type system refactor

python/builtins.py

@@ -161,6 +161,8 @@ def list::pop(self, i=-1):
     return res
 
 def list::__eq__(self, other):
+    if type(self) is not type(other):
+        return False
     if len(self) != len(other):
         return False
     for i in range(len(self)):
@@ -188,8 +190,24 @@ tuple.__contains__ = list.__contains__
 
 
 class property:
-    def __init__(self, fget):
+    def __init__(self, fget, fset=None):
         self.fget = fget
+        self.fset = fset
 
     def __get__(self, obj):
         return self.fget(obj)
+    
+    def __set__(self, obj, value):
+        if self.fset is None:
+            raise AttributeError("readonly property")
+        self.fset(obj, value)
+
+class staticmethod:
+    def __init__(self, f):
+        self.f = f
+
+    def __get__(self, obj):
+        return self.f
+    
+def type::__repr__(self):
+    return "<class '" + self.__name__ + "'>"

src/common.h

@@ -29,8 +29,8 @@
 #include <random>
 #include <chrono>
 
-#define PK_VERSION				"0.9.4"
+#define PK_VERSION				"0.9.5"
-#define PK_EXTRA_CHECK 			0
+#define PK_EXTRA_CHECK 			1
 
 #if (defined(__ANDROID__) && __ANDROID_API__ <= 22) || defined(__EMSCRIPTEN__)
 #define PK_ENABLE_FILEIO 		0

src/compiler.h

@@ -996,14 +996,18 @@ private:
         } else if(match(TK("pass"))){
             consume_end_stmt();
         } else {
+            int begin = co()->codes.size();
             EXPR_ANY();
+            int end = co()->codes.size();
             consume_end_stmt();
             // If last op is not an assignment, pop the result.
             uint8_t last_op = co()->codes.back().op;
             if( last_op!=OP_STORE_NAME && last_op!=OP_STORE_REF &&
             last_op!=OP_INPLACE_BINARY_OP && last_op!=OP_INPLACE_BITWISE_OP &&
             last_op!=OP_STORE_ALL_NAMES && last_op!=OP_STORE_CLASS_ATTR){
-                if(last_op == OP_BUILD_TUPLE_REF) co()->codes.back().op = OP_BUILD_TUPLE;
+                for(int i=begin; i<end; i++){
+                    if(co()->codes[i].op==OP_BUILD_TUPLE_REF) co()->codes[i].op = OP_BUILD_TUPLE;
+                }
                 if(mode()==REPL_MODE && name_scope() == NAME_GLOBAL) emit(OP_PRINT_EXPR, -1, true);
                 emit(OP_POP_TOP, -1, true);
             }

src/memory.h

@@ -69,7 +69,7 @@ public:
     T* get() const { return _t(); }
 
     int use_count() const { 
-        if(is_tagged()) return 1;
+        if(is_tagged()) return 0;
         return counter ? *counter : 0;
     }
 

src/obj.h

@@ -122,7 +122,7 @@ struct Py_ : PyObject {
 };
 
 #define OBJ_GET(T, obj) (((Py_<T>*)((obj).get()))->_value)
-#define OBJ_NAME(obj) OBJ_GET(Str, (obj)->attr(__name__))
+#define OBJ_NAME(obj) OBJ_GET(Str, vm->getattr(obj, __name__))
 
 const int kTpIntIndex = 2;
 const int kTpFloatIndex = 3;

src/pocketpy.h

@@ -65,7 +65,9 @@ void init_builtins(VM* _vm) {
     _vm->bind_builtin_func<0>("super", [](VM* vm, Args& args) {
         const PyVar* self = vm->top_frame()->f_locals().try_get(m_self);
         if(self == nullptr) vm->TypeError("super() can only be called in a class");
-        return vm->new_object(vm->tp_super, *self);
+        // base should be CURRENT_CLASS_BASE
+        Type base = vm->_all_types[(*self)->type.index].base;
+        return vm->new_object(vm->tp_super, Super(*self, base));
     });
 
     _vm->bind_builtin_func<1>("id", [](VM* vm, Args& args) {
@@ -164,8 +166,6 @@ void init_builtins(VM* _vm) {
     _vm->bind_method<1>("object", "__ne__", CPP_LAMBDA(VAR(args[0] != args[1])));
 
     _vm->bind_static_method<1>("type", "__new__", CPP_LAMBDA(vm->_t(args[0])));
-    _vm->bind_method<0>("type", "__repr__", CPP_LAMBDA(VAR("<class '" + OBJ_GET(Str, args[0]->attr(__name__)) + "'>")));
-
     _vm->bind_static_method<-1>("range", "__new__", [](VM* vm, Args& args) {
         Range r;
         switch (args.size()) {
@@ -490,7 +490,7 @@ void init_builtins(VM* _vm) {
     /************ PyTuple ************/
     _vm->bind_static_method<1>("tuple", "__new__", [](VM* vm, Args& args) {
         List list = CAST(List, vm->asList(args[0]));
-        return VAR(std::move(list));
+        return VAR(Tuple::from_list(std::move(list)));
     });
 
     _vm->bind_method<0>("tuple", "__iter__", [](VM* vm, Args& args) {
@@ -505,7 +505,7 @@ void init_builtins(VM* _vm) {
             s.normalize(self.size());
             List new_list;
             for(size_t i = s.start; i < s.stop; i++) new_list.push_back(self[i]);
-            return VAR(std::move(new_list));
+            return VAR(Tuple::from_list(std::move(new_list)));
         }
 
         int index = CAST(int, args[1]);
@@ -745,11 +745,10 @@ void VM::post_init(){
     add_module_io(this);
     add_module_os(this);
     add_module_c(this);
-    _lazy_modules["functools"] = kPythonLibs["functools"];
+
-    _lazy_modules["collections"] = kPythonLibs["collections"];
+    for(const char* name: {"this", "functools", "collections", "heapq", "bisect"}){
-    _lazy_modules["heapq"] = kPythonLibs["heapq"];
+        _lazy_modules[name] = kPythonLibs[name];
-    _lazy_modules["bisect"] = kPythonLibs["bisect"];
+    }
-    _lazy_modules["this"] = kPythonLibs["this"];
 
     CodeObject_ code = compile(kPythonLibs["builtins"], "<builtins>", EXEC_MODE);
     this->_exec(code, this->builtins);
@@ -757,6 +756,17 @@ void VM::post_init(){
     this->_exec(code, this->builtins);
     code = compile(kPythonLibs["set"], "<builtins>", EXEC_MODE);
     this->_exec(code, this->builtins);
+
+    // property is defined in builtins.py so we need to add it after builtins is loaded
+    _t(tp_object)->attr().set(__class__, property(CPP_LAMBDA(vm->_t(args[0]))));
+    _t(tp_type)->attr().set(__base__, property([](VM* vm, Args& args){
+        const PyTypeInfo& info = vm->_all_types[OBJ_GET(Type, args[0]).index];
+        return info.base.index == -1 ? vm->None : vm->_all_types[info.base.index].obj;
+    }));
+    _t(tp_type)->attr().set(__name__, property([](VM* vm, Args& args){
+        const PyTypeInfo& info = vm->_all_types[OBJ_GET(Type, args[0]).index];
+        return VAR(info.name);
+    }));
 }
 
 }   // namespace pkpy

src/vm.h

@@ -33,18 +33,23 @@ public:
     PyVar next();
 };
 
+struct PyTypeInfo{
+    PyVar obj;
+    Type base;
+    Str name;
+};
+
 class VM {
     VM* vm;     // self reference for simplify code
 public:
     std::stack< std::unique_ptr<Frame> > callstack;
     PyVar _py_op_call;
     PyVar _py_op_yield;
-    std::vector<PyVar> _all_types;
+    std::vector<PyTypeInfo> _all_types;
 
     PyVar run_frame(Frame* frame);
 
-    NameDict _types;
+    NameDict _modules;                          // loaded modules
-    NameDict _modules;                            // loaded modules
     std::map<StrName, Str> _lazy_modules;       // lazy loaded modules
     PyVar None, True, False, Ellipsis;
 
@@ -98,13 +103,22 @@ public:
         return call(_t(tp_list), one_arg(iterable));
     }
 
+    PyVar* find_name_in_mro(PyObject* cls, StrName name){
+        PyVar* val;
+        do{
+            val = cls->attr().try_get(name);
+            if(val != nullptr) return val;
+            Type cls_t = static_cast<Py_<Type>*>(cls)->_value;
+            Type base = _all_types[cls_t.index].base;
+            if(base.index == -1) break;
+            cls = _all_types[base.index].obj.get();
+        }while(true);
+        return nullptr;
+    }
+
     PyVar fast_call(StrName name, Args&& args){
-        PyObject* cls = _t(args[0]).get();
+        PyVar* val = find_name_in_mro(_t(args[0]).get(), name);
-        while(cls != None.get()) {
+        if(val != nullptr) return call(*val, std::move(args));
-            PyVar* val = cls->attr().try_get(name);
-            if(val != nullptr) return call(*val, std::move(args));
-            cls = cls->attr(__base__).get();
-        }
         AttributeError(args[0], name);
         return nullptr;
     }
@@ -160,11 +174,26 @@ public:
         return _exec();
     }
 
-    Type _new_type_object(StrName name, Type base=0) {
+    PyVar property(NativeFuncRaw fget){
+        PyVar p = builtins->attr("property");
+        PyVar method = new_object(tp_native_function, NativeFunc(fget, 1, false));
+        return call(p, one_arg(method));
+    }
+
+    PyVar new_type_object(PyVar mod, StrName name, Type base){
         PyVar obj = make_sp<PyObject, Py_<Type>>(tp_type, _all_types.size());
-        setattr(obj, __base__, _t(base));
+        PyTypeInfo info{
-        _types.set(name, obj);
+            .obj = obj,
-        _all_types.push_back(obj);
+            .base = base,
+            .name = (mod!=nullptr && mod!=builtins) ? Str(OBJ_NAME(mod)+"."+name.str()): name.str()
+        };
+        if(mod != nullptr) mod->attr().set(name, obj);
+        _all_types.push_back(info);
+        return obj;
+    }
+
+    Type _new_type_object(StrName name, Type base=0) {
+        PyVar obj = new_type_object(nullptr, name, base);
         return OBJ_GET(Type, obj);
     }
 
@@ -192,14 +221,23 @@ public:
         return make_sp<PyObject, Py_<std::decay_t<T>>>(type, std::move(_value));
     }
 
-    template<int ARGC>
+    PyVar _find_type(const Str& type){
-    void bind_func(Str typeName, Str funcName, NativeFuncRaw fn) {
+        PyVar* obj = builtins->attr().try_get(type);
-        bind_func<ARGC>(_types[typeName], funcName, fn);     
+        if(!obj){
+            for(auto& t: _all_types) if(t.name == type) return t.obj;
+            throw std::runtime_error("type not found: " + type);
+        }
+        return *obj;
     }
 
     template<int ARGC>
-    void bind_method(Str typeName, Str funcName, NativeFuncRaw fn) {
+    void bind_func(Str type, Str name, NativeFuncRaw fn) {
-        bind_method<ARGC>(_types[typeName], funcName, fn);
+        bind_func<ARGC>(_find_type(type), name, fn);
+    }
+
+    template<int ARGC>
+    void bind_method(Str type, Str name, NativeFuncRaw fn) {
+        bind_method<ARGC>(_find_type(type), name, fn);
     }
 
     template<int ARGC, typename... Args>
@@ -208,13 +246,13 @@ public:
     }
 
     template<int ARGC>
-    void _bind_methods(std::vector<Str> typeNames, Str funcName, NativeFuncRaw fn) {
+    void _bind_methods(std::vector<Str> types, Str name, NativeFuncRaw fn) {
-        for(auto& typeName : typeNames) bind_method<ARGC>(typeName, funcName, fn);
+        for(auto& type: types) bind_method<ARGC>(type, name, fn);
     }
 
     template<int ARGC>
-    void bind_builtin_func(Str funcName, NativeFuncRaw fn) {
+    void bind_builtin_func(Str name, NativeFuncRaw fn) {
-        bind_func<ARGC>(builtins, funcName, fn);
+        bind_func<ARGC>(builtins, name, fn);
     }
 
     int normalized_index(int index, int size){
@@ -276,13 +314,13 @@ public:
     }
 
     inline PyVar& _t(Type t){
-        return _all_types[t.index];
+        return _all_types[t.index].obj;
     }
 
     inline PyVar& _t(const PyVar& obj){
         if(is_int(obj)) return _t(tp_int);
         if(is_float(obj)) return _t(tp_float);
-        return _all_types[OBJ_GET(Type, _t(obj->type)).index];
+        return _all_types[OBJ_GET(Type, _t(obj->type)).index].obj;
     }
 
     ~VM() {
@@ -292,6 +330,14 @@ public:
         }
     }
 
+    inline PyVarOrNull getattr(const PyVar& obj, StrName name, bool throw_err=true, bool class_only=false){
+        return getattr(&obj, name, throw_err, class_only);
+    }
+    template<typename T>
+    inline void setattr(PyVar& obj, StrName name, T&& value){
+        setattr(&obj, name, std::forward<T>(value));
+    }
+
     CodeObject_ compile(Str source, Str filename, CompileMode mode);
     void post_init();
     PyVar num_negated(const PyVar& obj);
@@ -299,15 +345,14 @@ public:
     const PyVar& asBool(const PyVar& obj);
     i64 hash(const PyVar& obj);
     PyVar asRepr(const PyVar& obj);
-    PyVar new_type_object(PyVar mod, StrName name, PyVar base);
     PyVar new_module(StrName name);
     Str disassemble(CodeObject_ co);
     void init_builtin_types();
     PyVar call(const PyVar& _callable, Args args, const Args& kwargs, bool opCall);
     void unpack_args(Args& args);
-    PyVarOrNull getattr(const PyVar& obj, StrName name, bool throw_err=true, bool class_only=false);
+    PyVarOrNull getattr(const PyVar* obj, StrName name, bool throw_err=true, bool class_only=false);
     template<typename T>
-    void setattr(PyVar& obj, StrName name, T&& value);
+    void setattr(PyVar* obj, StrName name, T&& value);
     template<int ARGC>
     void bind_method(PyVar obj, Str funcName, NativeFuncRaw fn);
     template<int ARGC>
@@ -542,21 +587,9 @@ PyVar VM::asRepr(const PyVar& obj){
     return call(obj, __repr__);
 }
 
-PyVar VM::new_type_object(PyVar mod, StrName name, PyVar base){
-    if(!is_type(base, tp_type)) UNREACHABLE();
-    PyVar obj = make_sp<PyObject, Py_<Type>>(tp_type, _all_types.size());
-    setattr(obj, __base__, base);
-    Str fullName = name.str();
-    if(mod != builtins) fullName = OBJ_NAME(mod) + "." + name.str();
-    setattr(obj, __name__, VAR(fullName));
-    setattr(mod, name, obj);
-    _all_types.push_back(obj);
-    return obj;
-}
-
 PyVar VM::new_module(StrName name) {
     PyVar obj = new_object(tp_module, DummyModule());
-    setattr(obj, __name__, VAR(name.str()));
+    obj->attr().set(__name__, VAR(name.str()));
     _modules.set(name, obj);
     return obj;
 }
@@ -631,15 +664,13 @@ Str VM::disassemble(CodeObject_ co){
 }
 
 void VM::init_builtin_types(){
-    PyVar _tp_object = make_sp<PyObject, Py_<Type>>(1, 0);
+    // Py_(Type type, T&& val)
-    PyVar _tp_type = make_sp<PyObject, Py_<Type>>(1, 1);
+    PyVar _tp_object = make_sp<PyObject, Py_<Type>>(Type(1), Type(0));
-    _all_types.push_back(_tp_object);
+    PyVar _tp_type = make_sp<PyObject, Py_<Type>>(Type(1), Type(1));
-    _all_types.push_back(_tp_type);
+    _all_types.push_back({.obj = _tp_object, .base = -1, .name = "object"});
+    _all_types.push_back({.obj = _tp_type, .base = 0, .name = "type"});
     tp_object = 0; tp_type = 1;
 
-    _types.set("object", _tp_object);
-    _types.set("type", _tp_type);
-
     tp_int = _new_type_object("int");
     tp_float = _new_type_object("float");
     if(tp_int.index != kTpIntIndex || tp_float.index != kTpFloatIndex) UNREACHABLE();
@@ -665,25 +696,27 @@ void VM::init_builtin_types(){
     this->Ellipsis = new_object(_new_type_object("ellipsis"), DUMMY_VAL);
     this->True = new_object(tp_bool, true);
     this->False = new_object(tp_bool, false);
-    this->builtins = new_module("builtins");
-    this->_main = new_module("__main__");
     this->_py_op_call = new_object(_new_type_object("_py_op_call"), DUMMY_VAL);
     this->_py_op_yield = new_object(_new_type_object("_py_op_yield"), DUMMY_VAL);
+    this->builtins = new_module("builtins");
+    this->_main = new_module("__main__");
     
-    setattr(_t(tp_type), __base__, _t(tp_object));
+    // setup public types
-    setattr(_t(tp_object), __base__, None);
+    builtins->attr().set("type", _t(tp_type));
-    
+    builtins->attr().set("object", _t(tp_object));
-    for(auto [k, v]: _types.items()){
+    builtins->attr().set("bool", _t(tp_bool));
-        setattr(v, __name__, VAR(k.str()));
+    builtins->attr().set("int", _t(tp_int));
-    }
+    builtins->attr().set("float", _t(tp_float));
-
+    builtins->attr().set("str", _t(tp_str));
-    std::vector<Str> pb_types = {"type", "object", "bool", "int", "float", "str", "list", "tuple", "range"};
+    builtins->attr().set("list", _t(tp_list));
-    for (auto& name : pb_types) {
+    builtins->attr().set("tuple", _t(tp_tuple));
-        setattr(builtins, name, _types[name]);
+    builtins->attr().set("range", _t(tp_range));
-    }
 
     post_init();
-    for(auto [k, v]: _types.items()) v->attr()._try_perfect_rehash();
+    for(int i=0; i<_all_types.size(); i++){
+        auto& t = _all_types[i];
+        t.obj->attr()._try_perfect_rehash();
+    }
     for(auto [k, v]: _modules.items()) v->attr()._try_perfect_rehash();
 }
 
@@ -785,77 +818,75 @@ void VM::unpack_args(Args& args){
     args = Args::from_list(std::move(unpacked));
 }
 
-PyVarOrNull VM::getattr(const PyVar& obj, StrName name, bool throw_err, bool class_only) {
+using Super = std::pair<PyVar, Type>;
-    PyVar* val;
-    PyObject* cls;
 
-    if(is_type(obj, tp_super)){
+// https://docs.python.org/3/howto/descriptor.html#invocation-from-an-instance
-        const PyVar* root = &obj;
+PyVarOrNull VM::getattr(const PyVar* obj, StrName name, bool throw_err, bool class_only){
-        int depth = 1;
+    PyObject* objtype = _t(*obj).get();
-        while(true){
+    if(is_type(*obj, tp_super)){
-            root = &OBJ_GET(PyVar, *root);
+        const Super& super = OBJ_GET(Super, *obj);
-            if(!is_type(*root, tp_super)) break;
+        obj = &super.first;
-            depth++;
+        objtype = _t(super.second).get();
-        }
-        cls = _t(*root).get();
-        for(int i=0; i<depth; i++) cls = cls->attr(__base__).get();
-
-        if(!class_only){
-            val = (*root)->attr().try_get(name);
-            if(val != nullptr) return *val;
-        }
-    }else{
-        if(!class_only && !obj.is_tagged() && obj->is_attr_valid()){
-            val = obj->attr().try_get(name);
-            if(val != nullptr) return *val;
-        }
-        cls = _t(obj).get();
     }
-
+    PyVar* cls_var = find_name_in_mro(objtype, name);
-    while(cls != None.get()) {
+    if(cls_var != nullptr){
-        val = cls->attr().try_get(name);
+        // handle descriptor
-        if(val != nullptr){
+        PyVar* descr_get = _t(*cls_var)->attr().try_get(__get__);
-            PyVarOrNull descriptor = getattr(*val, __get__, false, true);
+        if(descr_get != nullptr) return call(*descr_get, two_args(*cls_var, *obj));
-            if(descriptor != nullptr) return call(descriptor, one_arg(obj));
-            if(is_type(*val, tp_function) || is_type(*val, tp_native_function)){
-                return VAR(BoundMethod(obj, *val));
-            }else{
-                return *val;
-            }
-        }else{
-            // this operation is expensive!!!
-            const Str& s = name.str();
-            if(s.empty() || s[0] != '_'){
-                PyVar* interceptor = cls->attr().try_get(__getattr__);
-                if(interceptor != nullptr){
-                    return call(*interceptor, two_args(obj, VAR(s)));
-                }
-            }
-        }
-        cls = cls->attr(__base__).get();
     }
-    if(throw_err) AttributeError(obj, name);
+    // handle instance __dict__
+    if(!class_only && !(*obj).is_tagged() && (*obj)->is_attr_valid()){
+        PyVar* val = (*obj)->attr().try_get(name);
+        if(val != nullptr) return *val;
+    }
+    if(cls_var != nullptr){
+        // bound method is non-data descriptor
+        if(is_type(*cls_var, tp_function) || is_type(*cls_var, tp_native_function)){
+            return VAR(BoundMethod(*obj, *cls_var));
+        }
+        return *cls_var;
+    }
+    if(throw_err) AttributeError(*obj, name);
     return nullptr;
 }
 
 template<typename T>
-void VM::setattr(PyVar& obj, StrName name, T&& value) {
+void VM::setattr(PyVar* obj, StrName name, T&& value){
-    if(obj.is_tagged()) TypeError("cannot set attribute");
+    static_assert(std::is_same_v<std::decay_t<T>, PyVar>);
-    PyObject* p = obj.get();
+    PyObject* objtype = _t(*obj).get();
-    while(p->type == tp_super) p = static_cast<PyVar*>(p->value())->get();
+    if(is_type(*obj, tp_super)){
-    if(!p->is_attr_valid()) TypeError("cannot set attribute");
+        Super& super = OBJ_GET(Super, *obj);
-    p->attr().set(name, std::forward<T>(value));
+        obj = &super.first;
+        objtype = _t(super.second).get();
+    }
+    PyVar* cls_var = find_name_in_mro(objtype, name);
+    if(cls_var != nullptr){
+        // handle descriptor
+        const PyVar& cls_var_t = _t(*cls_var);
+        if(cls_var_t->attr().contains(__get__)){
+            PyVar* descr_set = cls_var_t->attr().try_get(__set__);
+            if(descr_set != nullptr){
+                call(*descr_set, three_args(*cls_var, *obj, std::forward<T>(value)));
+            }else{
+                TypeError("readonly attribute: " + name.str().escape(true));
+            }
+            return;
+        }
+    }
+    // handle instance __dict__
+    if((*obj).is_tagged() || !(*obj)->is_attr_valid()) TypeError("cannot set attribute");
+    (*obj)->attr().set(name, std::forward<T>(value));
 }
 
 template<int ARGC>
-void VM::bind_method(PyVar obj, Str funcName, NativeFuncRaw fn) {
+void VM::bind_method(PyVar obj, Str name, NativeFuncRaw fn) {
     check_type(obj, tp_type);
-    setattr(obj, funcName, VAR(NativeFunc(fn, ARGC, true)));
+    obj->attr().set(name, VAR(NativeFunc(fn, ARGC, true)));
 }
 
 template<int ARGC>
-void VM::bind_func(PyVar obj, Str funcName, NativeFuncRaw fn) {
+void VM::bind_func(PyVar obj, Str name, NativeFuncRaw fn) {
-    setattr(obj, funcName, VAR(NativeFunc(fn, ARGC, false)));
+    obj->attr().set(name, VAR(NativeFunc(fn, ARGC, false)));
 }
 
 void VM::_error(Exception e){

tests/40_class.py

@@ -13,7 +13,7 @@ a = A(1, 2)
 assert a.add() == 3
 assert a.sub() == -1
 
-assert a.__base__ is object
+assert A.__base__ is object
 
 class B(A):
     def __init__(self, a, b, c):