Merge pull request #31 from blueloveTH/test

Test

amalgamate.py

@@ -2,7 +2,7 @@ with open("src/opcodes.h", "rt", encoding='utf-8') as f:
 	OPCODES_TEXT = f.read()
 
 pipeline = [
-	["common.h", "hash_table5.hpp", "memory.h", "str.h", "safestl.h", "builtins.h", "error.h"],
+	["common.h", "memory.h", "str.h", "tuplelist.h", "namedict.h", "builtins.h", "error.h"],
 	["obj.h", "parser.h", "ref.h", "codeobject.h", "frame.h"],
 	["vm.h", "ceval.h", "compiler.h", "repl.h"],
 	["iter.h", "pocketpy.h"]

src/builtins.h

@@ -210,7 +210,7 @@ list.__new__ = lambda obj: [i for i in obj]
 
 # https://github.com/python/cpython/blob/main/Objects/dictobject.c
 class dict:
-    def __init__(self, capacity=16):
+    def __init__(self, capacity=12):
         self._capacity = capacity
         self._a = [None] * self._capacity
         self._len = 0
@@ -243,7 +243,7 @@ class dict:
         else:
             self._a[i] = [key, value]
             self._len += 1
-            if self._len > self._capacity * 0.8:
+            if self._len > self._capacity * 0.67:
                 self._capacity *= 2
                 self.__rehash()
 

src/codeobject.h

@@ -65,6 +65,8 @@ struct CodeObject {
     std::vector<CodeBlock> blocks = { CodeBlock{NO_BLOCK, -1} };
     std::map<StrName, int> labels;
 
+    int ideal_locals_capacity = 4;
+
     void optimize(VM* vm);
 
     bool add_label(StrName label){

src/common.h

@@ -24,19 +24,17 @@
 #include <functional>
 #include <iostream>
 #include <map>
+#include <algorithm>
 // #include <filesystem>
 // namespace fs = std::filesystem;
 
-#define EMH_EXT 1
-#define EMH_FIND_HIT 1
-
 #ifdef POCKETPY_H
 #define UNREACHABLE() throw std::runtime_error( "L" + std::to_string(__LINE__) + " UNREACHABLE()!");
 #else
 #define UNREACHABLE() throw std::runtime_error( __FILE__ + std::string(":") + std::to_string(__LINE__) + " UNREACHABLE()!");
 #endif
 
-#define PK_VERSION "0.8.9"
+#define PK_VERSION "0.9.0"
 
 #if defined(__EMSCRIPTEN__) || defined(__arm__) || defined(__i386__)
 typedef int32_t i64;
@@ -46,7 +44,9 @@ typedef int64_t i64;
 typedef double f64;
 #endif
 
-struct Dummy { char _; };
+struct Dummy {  };
+struct DummyInstance {  };
+struct DummyModule { };
 #define DUMMY_VAL Dummy()
 
 struct Type {
@@ -64,4 +64,8 @@ struct Type {
 //#define THREAD_LOCAL thread_local
 #define THREAD_LOCAL
 
-#define RAW(T) std::remove_const_t<std::remove_reference_t<T>>
+#define RAW(T) std::remove_const_t<std::remove_reference_t<T>>
+
+const float kLocalsLoadFactor = 0.67;
+const float kInstAttrLoadFactor = 0.67;
+const float kTypeAttrLoadFactor = 0.34;

src/compiler.h

@@ -947,7 +947,7 @@ __LISTCOMP:
             // If last op is not an assignment, pop the result.
             uint8_t last_op = co()->codes.back().op;
             if( last_op!=OP_STORE_NAME && last_op!=OP_STORE_REF && last_op!=OP_INPLACE_BINARY_OP && last_op!=OP_INPLACE_BITWISE_OP){
-                if(mode()==REPL_MODE && parser->indents.top()==0) emit(OP_PRINT_EXPR, -1, true);
+                if(mode()==REPL_MODE && name_scope() == NAME_GLOBAL) emit(OP_PRINT_EXPR, -1, true);
                 emit(OP_POP_TOP, -1, true);
             }
         }

src/error.h

@@ -1,6 +1,7 @@
 #pragma once
 
-#include "safestl.h"
+#include "namedict.h"
+#include "tuplelist.h"
 
 struct NeedMoreLines {
     NeedMoreLines(bool is_compiling_class) : is_compiling_class(is_compiling_class) {}

src/frame.h

@@ -9,7 +9,7 @@ struct Frame {
     int _ip = -1;
     int _next_ip = 0;
 
-    const CodeObject_ co;
+    const CodeObject* co;
     PyVar _module;
     pkpy::shared_ptr<pkpy::NameDict> _locals;
     pkpy::shared_ptr<pkpy::NameDict> _closure;
@@ -24,9 +24,9 @@ struct Frame {
         return _closure->try_get(name);
     }
 
-    Frame(const CodeObject_ co, PyVar _module,
+    Frame(const CodeObject_& co, const PyVar& _module,
         pkpy::shared_ptr<pkpy::NameDict> _locals=nullptr, pkpy::shared_ptr<pkpy::NameDict> _closure=nullptr)
-        : co(co), _module(_module), _locals(_locals), _closure(_closure), id(kFrameGlobalId++) { }
+        : co(co.get()), _module(_module), _locals(_locals), _closure(_closure), id(kFrameGlobalId++) { }
 
     inline const Bytecode& next_bytecode() {
         _ip = _next_ip++;

src/memory.h

@@ -80,7 +80,7 @@ namespace pkpy{
             return reinterpret_cast<__VAL>(counter);
         }
 
-        inline bool is_tagged() const {
+        inline constexpr bool is_tagged() const {
             if constexpr(!std::is_same_v<T, PyObject>) return false;
             return (reinterpret_cast<i64>(counter) & 0b11) != 0b00;
         }
@@ -147,3 +147,8 @@ struct SmallArrayPool {
         }
     }
 };
+
+
+typedef pkpy::shared_ptr<PyObject> PyVar;
+typedef PyVar PyVarOrNull;
+typedef PyVar PyVarRef;

src/namedict.h

@@ -0,0 +1,166 @@
+#pragma once
+
+#include "common.h"
+#include "memory.h"
+#include "str.h"
+
+namespace pkpy{
+
+    struct NameDictNode{
+        StrName first;
+        PyVar second;
+        inline bool empty() const { return first.empty(); }
+    };
+
+    struct NameDict {
+        int _capacity;
+        int _size;
+        float _load_factor;
+        NameDictNode* _a;
+
+        NameDict(int capacity=4, float load_factor=0.67):
+            _capacity(capacity), _size(0), _load_factor(load_factor) {
+            _a = new NameDictNode[_capacity];
+        }
+
+        NameDict(const NameDict& other) {
+            this->_capacity = other._capacity;
+            this->_size = other._size;
+            this->_a = new NameDictNode[_capacity];
+            for(int i=0; i<_capacity; i++) _a[i] = other._a[i];
+        }
+        
+        NameDict& operator=(const NameDict&) = delete;
+        NameDict(NameDict&&) = delete;
+        NameDict& operator=(NameDict&&) = delete;
+
+        int size() const { return _size; }
+
+//https://github.com/python/cpython/blob/main/Objects/dictobject.c#L175
+#define HASH_PROBE(key, ok, i) \
+    int i = (key).index % _capacity; \
+    bool ok = false; \
+    while(!_a[i].empty()) { \
+        if(_a[i].first == (key)) { ok = true; break; } \
+        i = (5*i + 1) % _capacity; \
+    }
+
+#define HASH_PROBE_OVERRIDE(key, ok, i) \
+    i = (key).index % _capacity; \
+    ok = false; \
+    while(!_a[i].empty()) { \
+        if(_a[i].first == (key)) { ok = true; break; } \
+        i = (5*i + 1) % _capacity; \
+    }
+
+        const PyVar& operator[](StrName key) const {
+            HASH_PROBE(key, ok, i);
+            if(!ok) throw std::out_of_range("NameDict key not found");
+            return _a[i].second;
+        }
+
+        [[nodiscard]] PyVar& operator[](StrName key){
+            HASH_PROBE(key, ok, i);
+            if(!ok) {
+                _a[i].first = key;
+                _size++;
+                if(_size > _capacity * _load_factor){
+                    _rehash_2x();
+                    HASH_PROBE_OVERRIDE(key, ok, i);
+                }
+            }
+            return _a[i].second;
+        }
+
+        void _rehash_2x(){
+            NameDictNode* old_a = _a;
+            int old_capacity = _capacity;
+            _capacity *= 2;
+            _size = 0;
+            _a = new NameDictNode[_capacity];
+            for(int i=0; i<old_capacity; i++){
+                if(old_a[i].empty()) continue;
+                HASH_PROBE(old_a[i].first, ok, j);
+                if(ok) UNREACHABLE();
+                _a[j].first = old_a[i].first;
+                _a[j].second = std::move(old_a[i].second);
+                _size++;
+            }
+            delete[] old_a;
+        }
+
+        inline PyVar* try_get(StrName key){
+            HASH_PROBE(key, ok, i);
+            if(!ok) return nullptr;
+            return &_a[i].second;
+        }
+
+        inline bool try_set(StrName key, PyVar&& value){
+            HASH_PROBE(key, ok, i);
+            if(!ok) return false;
+            _a[i].second = std::move(value);
+            return true;
+        }
+
+        inline bool contains(StrName key) const {
+            HASH_PROBE(key, ok, i);
+            return ok;
+        }
+
+        ~NameDict(){ delete[] _a;}
+
+        struct iterator {
+            const NameDict* _dict;
+            int i;
+            iterator() = default;
+            iterator(const NameDict* dict, int i): _dict(dict), i(i) { _skip_empty(); }
+            inline void _skip_empty(){ while(i < _dict->_capacity && _dict->_a[i].empty()) i++;}
+            inline iterator& operator++(){ i++; _skip_empty(); return *this;}
+
+            inline bool operator!=(const iterator& other) const { return i != other.i; }
+            inline bool operator==(const iterator& other) const { return i == other.i; }
+
+            inline NameDictNode* operator->() const { return &_dict->_a[i]; }
+        };
+
+        template<typename T>
+        void emplace(StrName key, T&& value){
+            HASH_PROBE(key, ok, i);
+            if(!ok) {
+                _a[i].first = key;
+                _size++;
+                if(_size > _capacity * _load_factor){
+                    _rehash_2x();
+                    HASH_PROBE_OVERRIDE(key, ok, i);
+                }
+            }
+            _a[i].second = std::forward<T>(value);
+        }
+
+        void insert(iterator begin, iterator end){
+            for(auto it = begin; it != end; ++it){
+                emplace(it->first, it->second);
+            }
+        }
+
+        iterator find(StrName key) const{
+            HASH_PROBE(key, ok, i);
+            if(!ok) return end();
+            return iterator(this, i);
+        }
+
+        void erase(StrName key){
+            HASH_PROBE(key, ok, i);
+            if(!ok) throw std::out_of_range("NameDict key not found");
+            _a[i] = NameDictNode();
+            _size--;
+        }
+
+        inline iterator begin() const { return iterator(this, 0); }
+        inline iterator end() const { return iterator(this, _capacity); }
+
+    #undef HASH_PROBE
+    #undef HASH_PROBE_OVERRIDE
+    };
+
+} // namespace pkpy

src/obj.h

@@ -1,6 +1,7 @@
 #pragma once
 
-#include "safestl.h"
+#include "namedict.h"
+#include "tuplelist.h"
 
 struct CodeObject;
 struct Frame;
@@ -25,7 +26,7 @@ struct Function {
     CodeObject_ code;
     std::vector<StrName> args;
     StrName starred_arg;                // empty if no *arg
-    pkpy::NameDict kwargs;          // empty if no k=v
+    pkpy::NameDict kwargs;              // empty if no k=v
     std::vector<StrName> kwargs_order;
 
     // runtime settings
@@ -97,8 +98,10 @@ struct Py_ : PyObject {
     Py_(Type type, T&& val): PyObject(type), _value(std::move(val)) { _init(); }
 
     inline void _init() noexcept {
-        if constexpr (std::is_same_v<T, Dummy> || std::is_same_v<T, Type>) {
+        if constexpr (std::is_same_v<T, Type> || std::is_same_v<T, DummyModule>) {
-            _attr = new pkpy::NameDict();
+            _attr = new pkpy::NameDict(8, kTypeAttrLoadFactor);
+        }else if constexpr(std::is_same_v<T, DummyInstance>){
+            _attr = new pkpy::NameDict(4, kInstAttrLoadFactor);
         }else{
             _attr = nullptr;
         }

src/pocketpy.h

@@ -774,6 +774,22 @@ void add_module_random(VM* vm){
     vm->_exec(code, mod);
 }
 
+void VM::post_init(){
+    init_builtins(this);
+    add_module_sys(this);
+    add_module_time(this);
+    add_module_json(this);
+    add_module_math(this);
+    add_module_re(this);
+    add_module_dis(this);
+    add_module_random(this);
+    add_module_io(this);
+    add_module_os(this);
+
+    CodeObject_ code = compile(kBuiltinsCode, "<builtins>", EXEC_MODE);
+    this->_exec(code, this->builtins);
+}
+
 
 class _PkExported{
 public:
@@ -879,21 +895,7 @@ extern "C" {
     __EXPORT
     /// Create a virtual machine.
     VM* pkpy_new_vm(bool use_stdio){
-        VM* vm = PKPY_ALLOCATE(VM, use_stdio);
+        return PKPY_ALLOCATE(VM, use_stdio);
-        init_builtins(vm);
-        add_module_sys(vm);
-        add_module_time(vm);
-        add_module_json(vm);
-        add_module_math(vm);
-        add_module_re(vm);
-        add_module_dis(vm);
-        add_module_random(vm);
-        add_module_io(vm);
-        add_module_os(vm);
-
-        CodeObject_ code = vm->compile(kBuiltinsCode, "<builtins>", EXEC_MODE);
-        vm->_exec(code, vm->builtins);
-        return vm;
     }
 
     __EXPORT

src/ref.h

@@ -20,7 +20,7 @@ struct NameRef : BaseRef {
     const std::pair<StrName, NameScope> pair;
     inline StrName name() const { return pair.first; }
     inline NameScope scope() const { return pair.second; }
-    NameRef(std::pair<StrName, NameScope>& pair) : pair(pair) {}
+    NameRef(const std::pair<StrName, NameScope>& pair) : pair(pair) {}
 
     PyVar get(VM* vm, Frame* frame) const;
     void set(VM* vm, Frame* frame, PyVar val) const;

src/str.h

@@ -140,7 +140,6 @@ struct StrName {
     StrName(const Str& s): index(get(s).index) {}
     inline const Str& str() const { return _r_interned[index]; }
     inline bool empty() const { return index == -1; }
-    inline void reset() { index = -1; }
 
     inline bool operator==(const StrName& other) const noexcept {
         return this->index == other.index;

src/tuplelist.h

@@ -4,46 +4,29 @@
 #include "memory.h"
 #include "str.h"
 
-struct PyObject;
-typedef pkpy::shared_ptr<PyObject> PyVar;
-typedef PyVar PyVarOrNull;
-typedef PyVar PyVarRef;
-
-#include "hash_table5.hpp"
 namespace pkpy {
-	template<typename... Args>
+    class List: public std::vector<PyVar> {
-	using HashMap = emhash5::HashMap<Args...>;
+        PyVar& at(size_t) = delete;
-}
 
-namespace pkpy{
+        inline void _check_index(size_t i) const {
-class List: public std::vector<PyVar> {
+            if (i >= size()){
-    PyVar& at(size_t) = delete;
+                auto msg = "std::vector index out of range, " + std::to_string(i) + " not in [0, " + std::to_string(size()) + ")";
-
+                throw std::out_of_range(msg);
-    inline void _check_index(size_t i) const {
+            }
-        if (i >= size()){
+        }
-            auto msg = "std::vector index out of range, " + std::to_string(i) + " not in [0, " + std::to_string(size()) + ")";
+    public:
-            throw std::out_of_range(msg);
+        PyVar& operator[](size_t i) {
+            _check_index(i);
+            return std::vector<PyVar>::operator[](i);
         }
-    }
-public:
-    PyVar& operator[](size_t i) {
-        _check_index(i);
-        return std::vector<PyVar>::operator[](i);
-    }
 
-    const PyVar& operator[](size_t i) const {
+        const PyVar& operator[](size_t i) const {
-        _check_index(i);
+            _check_index(i);
-        return std::vector<PyVar>::operator[](i);
+            return std::vector<PyVar>::operator[](i);
-    }
+        }
 
-    using std::vector<PyVar>::vector;
+        using std::vector<PyVar>::vector;
-};
+    };
-
-
-}
-
-namespace pkpy {
-    typedef HashMap<StrName, PyVar> NameDict;
 
     class Args {
         static THREAD_LOCAL SmallArrayPool<PyVar, 10> _pool;
@@ -140,8 +123,5 @@ namespace pkpy {
     }
 
     typedef Args Tuple;
-
-    // declare static members
     THREAD_LOCAL SmallArrayPool<PyVar, 10> Args::_pool;
-    // THREAD_LOCAL SmallArrayPool<NameDictNode, 1> NameDict::_pool;
 }   // namespace pkpy

src/vm.h

@@ -25,7 +25,7 @@ public:
 
     pkpy::NameDict _types;
     pkpy::NameDict _modules;                            // loaded modules
-    pkpy::HashMap<StrName, Str> _lazy_modules;       // lazy loaded modules
+    std::map<StrName, Str> _lazy_modules;       // lazy loaded modules
     PyVar None, True, False, Ellipsis;
 
     bool use_stdio;
@@ -129,11 +129,11 @@ public:
             PyVar* new_f = _callable->attr().try_get(__new__);
             PyVar obj;
             if(new_f != nullptr){
-                obj = call(*new_f, args, kwargs, false);
+                obj = call(*new_f, std::move(args), kwargs, false);
             }else{
-                obj = new_object(_callable, DUMMY_VAL);
+                obj = new_object(_callable, DummyInstance());
                 PyVarOrNull init_f = getattr(obj, __init__, false);
-                if (init_f != nullptr) call(init_f, args, kwargs, false);
+                if (init_f != nullptr) call(init_f, std::move(args), kwargs, false);
             }
             return obj;
         }
@@ -151,28 +151,29 @@ public:
             return f(this, args);
         } else if(is_type(*callable, tp_function)){
             const pkpy::Function& fn = PyFunction_AS_C(*callable);
-            pkpy::shared_ptr<pkpy::NameDict> _locals = pkpy::make_shared<pkpy::NameDict>();
+            auto locals = pkpy::make_shared<pkpy::NameDict>(
-            pkpy::NameDict& locals = *_locals;
+                fn.code->ideal_locals_capacity, kLocalsLoadFactor
+            );
 
             int i = 0;
             for(StrName name : fn.args){
                 if(i < args.size()){
-                    locals.emplace(name, args[i++]);
+                    locals->emplace(name, std::move(args[i++]));
                     continue;
                 }
                 TypeError("missing positional argument " + name.str().escape(true));
             }
 
-            locals.insert(fn.kwargs.begin(), fn.kwargs.end());
+            locals->insert(fn.kwargs.begin(), fn.kwargs.end());
 
             if(!fn.starred_arg.empty()){
                 pkpy::List vargs;        // handle *args
-                while(i < args.size()) vargs.push_back(args[i++]);
+                while(i < args.size()) vargs.push_back(std::move(args[i++]));
-                locals.emplace(fn.starred_arg, PyTuple(std::move(vargs)));
+                locals->emplace(fn.starred_arg, PyTuple(std::move(vargs)));
             }else{
                 for(StrName key : fn.kwargs_order){
                     if(i < args.size()){
-                        locals[key] = args[i++];
+                        locals->emplace(key, std::move(args[i++]));
                     }else{
                         break;
                     }
@@ -185,11 +186,10 @@ public:
                 if(!fn.kwargs.contains(key)){
                     TypeError(key.escape(true) + " is an invalid keyword argument for " + fn.name + "()");
                 }
-                const PyVar& val = kwargs[i+1];
+                locals->emplace(key, kwargs[i+1]);
-                locals[key] = val;
             }
-            PyVar _module = fn._module != nullptr ? fn._module : top_frame()->_module;
+            const PyVar& _module = fn._module != nullptr ? fn._module : top_frame()->_module;
-            auto _frame = _new_frame(fn.code, _module, _locals, fn._closure);
+            auto _frame = _new_frame(fn.code, _module, locals, fn._closure);
             if(fn.code->is_generator){
                 return PyIter(pkpy::make_shared<BaseIter, Generator>(
                     this, std::move(_frame)));
@@ -212,10 +212,12 @@ public:
         }catch (const pkpy::Exception& e){
             *_stderr << e.summary() << '\n';
         }
+#ifdef _NDEBUG
         catch (const std::exception& e) {
             *_stderr << "An std::exception occurred! It could be a bug.\n";
             *_stderr << e.what() << '\n';
         }
+#endif
         callstack = {};
         return nullptr;
     }
@@ -322,7 +324,7 @@ public:
     }
 
     PyVar new_module(StrName name) {
-        PyVar obj = new_object(tp_module, DUMMY_VAL);
+        PyVar obj = new_object(tp_module, DummyModule());
         setattr(obj, __name__, PyStr(name.str()));
         _modules[name] = obj;
         return obj;
@@ -634,8 +636,12 @@ public:
         for (auto& name : pb_types) {
             setattr(builtins, name, _types[name]);
         }
+
+        post_init();
     }
 
+    void post_init();
+
     i64 hash(const PyVar& obj){
         if (is_type(obj, tp_str)) return PyStr_AS_C(obj).hash();
         if (is_int(obj)) return PyInt_AS_C(obj);
@@ -735,13 +741,13 @@ public:
 PyVar NameRef::get(VM* vm, Frame* frame) const{
     PyVar* val;
     val = frame->f_locals().try_get(name());
-    if(val) return *val;
+    if(val != nullptr) return *val;
     val = frame->f_closure_try_get(name());
-    if(val) return *val;
+    if(val != nullptr) return *val;
     val = frame->f_globals().try_get(name());
-    if(val) return *val;
+    if(val != nullptr) return *val;
     val = vm->builtins->attr().try_get(name());
-    if(val) return *val;
+    if(val != nullptr) return *val;
     vm->NameError(name());
     return nullptr;
 }
@@ -750,14 +756,9 @@ void NameRef::set(VM* vm, Frame* frame, PyVar val) const{
     switch(scope()) {
         case NAME_LOCAL: frame->f_locals()[name()] = std::move(val); break;
         case NAME_GLOBAL:
-        {
+            if(frame->f_locals().try_set(name(), std::move(val))) return;
-            PyVar* existing = frame->f_locals().try_get(name());
+            frame->f_globals()[name()] = std::move(val);
-            if(existing != nullptr){
+            break;
-                *existing = std::move(val);
-            }else{
-                frame->f_globals()[name()] = std::move(val);
-            }
-        } break;
         default: UNREACHABLE();
     }
 }
@@ -857,6 +858,12 @@ PyVar pkpy::NativeFunc::operator()(VM* vm, pkpy::Args& args) const{
 }
 
 void CodeObject::optimize(VM* vm){
+    int n = 0;
+    for(auto& p: names) if(p.second == NAME_LOCAL) n++;
+    int base_n = (int)(n / kLocalsLoadFactor + 1.5);
+    ideal_locals_capacity = 2;
+    while(ideal_locals_capacity < base_n) ideal_locals_capacity *= 2;
+
     for(int i=1; i<codes.size(); i++){
         if(codes[i].op == OP_UNARY_NEGATIVE && codes[i-1].op == OP_LOAD_CONST){
             codes[i].op = OP_NO_OP;