From 65440c2034466b8dda20d87be65533194724dea5 Mon Sep 17 00:00:00 2001 From: BLUELOVETH Date: Sat, 16 Mar 2024 18:37:09 +0800 Subject: [PATCH] Fix #197 (#227) * init * some optimize * Update frame.h * remove `LOAD_INTEGER` * Update vm.cpp * some optimize * some fix * Revert "remove `LOAD_INTEGER`" This reverts commit c0b965aee2f64fbfae0b20f41d714688649d20cf. * some fix * Update expr.cpp * some fix * Update retype.yml --- README.md | 6 +--- docs/retype.yml | 2 +- include/pocketpy/common.h | 2 +- include/pocketpy/expr.h | 1 + include/pocketpy/frame.h | 55 +++++++++++++++++++++++------ include/pocketpy/profiler.h | 6 ++-- include/pocketpy/vm.h | 8 ++--- src/ceval.cpp | 29 +++++----------- src/expr.cpp | 24 ++++++------- src/iter.cpp | 2 +- src/pocketpy.cpp | 6 ++-- src/profiler.cpp | 21 +++++++---- src/vm.cpp | 69 +++++++++++++++++++------------------ 13 files changed, 131 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index b37ce732..732ab9bf 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Some variables can be set to control the build process: + `PK_BUILD_SHARED_LIB` - Build the shared (for C-APIs only) + `PK_ENABLE_OS` - Enable OS related features (default mode is sandboxed) -It is safe to use `main` branch in production. +It is safe to use `main` branch in production if CI is green. ### Compile Flags @@ -197,10 +197,6 @@ Your sponsorship will help us develop pkpy continuously. An excellent learning material. It illustrates how Python's virtual machine works. -+ [box2d](https://box2d.org/) - - The world's best 2D physics engine, written by Erin Catto. `box2d` now becomes a built-in module in pkpy `v1.1.3` and later. - ## Star History diff --git a/docs/retype.yml b/docs/retype.yml index 53741ea6..215dff05 100644 --- a/docs/retype.yml +++ b/docs/retype.yml @@ -3,7 +3,7 @@ output: .retype url: https://pocketpy.dev branding: title: pocketpy - label: v1.4.1 + label: v1.4.3 logo: "./static/logo.png" favicon: "./static/logo.png" meta: diff --git a/include/pocketpy/common.h b/include/pocketpy/common.h index 014f8a66..9bcbb352 100644 --- a/include/pocketpy/common.h +++ b/include/pocketpy/common.h @@ -21,7 +21,7 @@ #include #include -#define PK_VERSION "1.4.2" +#define PK_VERSION "1.4.3" #include "config.h" #include "export.h" diff --git a/include/pocketpy/expr.h b/include/pocketpy/expr.h index 5cd06956..d030c2ad 100644 --- a/include/pocketpy/expr.h +++ b/include/pocketpy/expr.h @@ -105,6 +105,7 @@ struct CodeEmitContext{ void exit_block(); void emit_expr(); // clear the expression stack and generate bytecode int emit_(Opcode opcode, uint16_t arg, int line, bool is_virtual=false); + int emit_int(i64 value, int line); void patch_jump(int index); bool add_label(StrName name); int add_varname(StrName name); diff --git a/include/pocketpy/frame.h b/include/pocketpy/frame.h index b413bbec..7c4f3064 100644 --- a/include/pocketpy/frame.h +++ b/include/pocketpy/frame.h @@ -84,18 +84,21 @@ struct Frame { const CodeObject* co; PyObject* _module; - PyObject* _callable; // weak ref + PyObject* _callable; // a function object or nullptr (global scope) FastLocals _locals; NameDict& f_globals() noexcept { return _module->attr(); } PyObject* f_closure_try_get(StrName name); - Frame(PyObject** p0, const CodeObject* co, PyObject* _module, PyObject* _callable) - : _ip(-1), _next_ip(0), _sp_base(p0), co(co), _module(_module), _callable(_callable), _locals(co, p0) { } + // function scope + Frame(PyObject** p0, const CodeObject* co, PyObject* _module, PyObject* _callable, PyObject** _locals_base) + : _ip(-1), _next_ip(0), _sp_base(p0), co(co), _module(_module), _callable(_callable), _locals(co, _locals_base) { } + // exec/eval Frame(PyObject** p0, const CodeObject* co, PyObject* _module, PyObject* _callable, FastLocals _locals) : _ip(-1), _next_ip(0), _sp_base(p0), co(co), _module(_module), _callable(_callable), _locals(_locals) { } + // global scope Frame(PyObject** p0, const CodeObject_& co, PyObject* _module) : _ip(-1), _next_ip(0), _sp_base(p0), co(co.get()), _module(_module), _callable(nullptr), _locals(co.get(), p0) {} @@ -125,14 +128,46 @@ struct Frame { } }; -using CallstackContainer = small_vector_no_copy_and_move; +struct LinkedFrame{ + LinkedFrame* f_back; + Frame frame; + template + LinkedFrame(LinkedFrame* f_back, Args&&... args) : f_back(f_back), frame(std::forward(args)...) {} +}; -struct FrameId{ - CallstackContainer* data; - int index; - FrameId(CallstackContainer* data, int index) : data(data), index(index) {} - Frame* operator->() const { return &data->operator[](index); } - Frame* get() const { return &data->operator[](index); } +struct CallStack{ + static_assert(sizeof(LinkedFrame) <= 64 && std::is_trivially_destructible_v); + + LinkedFrame* _tail; + int _size; + CallStack(): _tail(nullptr), _size(0) {} + + int size() const { return _size; } + bool empty() const { return _size == 0; } + void clear(){ while(!empty()) pop(); } + + template + void emplace(Args&&... args){ + _tail = new(pool64_alloc()) LinkedFrame(_tail, std::forward(args)...); + ++_size; + } + + void pop(){ +#if PK_DEBUG_EXTRA_CHECK + if(empty()) PK_FATAL_ERROR(); +#endif + LinkedFrame* p = _tail; + _tail = p->f_back; + pool64_dealloc(p); + --_size; + } + + Frame& top() const { return _tail->frame; } + + template + void apply(Func&& f){ + for(LinkedFrame* p = _tail; p != nullptr; p = p->f_back) f(p->frame); + } }; }; // namespace pkpy \ No newline at end of file diff --git a/include/pocketpy/profiler.h b/include/pocketpy/profiler.h index e1f77c94..4ed36695 100644 --- a/include/pocketpy/profiler.h +++ b/include/pocketpy/profiler.h @@ -14,7 +14,7 @@ struct _LineRecord{ }; struct _FrameRecord{ - FrameId frame; + LinkedFrame* frame; clock_t prev_time; _LineRecord* prev_record; }; @@ -26,8 +26,8 @@ struct LineProfiler{ std::set functions; void begin(); - void _step(FrameId frame); - void _step_end(FrameId frame, int line); + void _step(LinkedFrame*); + void _step_end(LinkedFrame*, int); void end(); Str stats(); }; diff --git a/include/pocketpy/vm.h b/include/pocketpy/vm.h index 2c1e413a..6aa0ad1f 100644 --- a/include/pocketpy/vm.h +++ b/include/pocketpy/vm.h @@ -94,7 +94,7 @@ class VM { public: ManagedHeap heap; ValueStack s_data; - stack_no_copy callstack; + CallStack callstack; std::vector _all_types; NameDict _modules; // loaded modules @@ -151,7 +151,7 @@ public: VM(bool enable_os=true); - FrameId top_frame(); + Frame* top_frame(); void _pop_frame(); PyObject* py_str(PyObject* obj); @@ -364,9 +364,9 @@ public: } Type _tp(PyObject* obj){ + if(!is_tagged(obj)) return obj->type; if(is_int(obj)) return tp_int; - if(is_float(obj)) return tp_float; - return obj->type; + return tp_float; } PyObject* _t(PyObject* obj){ diff --git a/src/ceval.cpp b/src/ceval.cpp index d331ee91..f9a20639 100644 --- a/src/ceval.cpp +++ b/src/ceval.cpp @@ -44,19 +44,12 @@ bool VM::py_ge(PyObject* _0, PyObject* _1){ #undef BINARY_F_COMPARE -// static i64 _py_sint(PyObject* obj) noexcept { -// return (i64)(PK_BITS(obj) >> 2); -// } - PyObject* VM::_run_top_frame(){ - FrameId frame = top_frame(); - const int base_id = frame.index; + Frame* frame = top_frame(); + const Frame* base_frame = frame; bool need_raise = false; while(true){ -#if PK_DEBUG_EXTRA_CHECK - if(frame.index < base_id) PK_FATAL_ERROR(); -#endif try{ if(need_raise){ need_raise = false; _raise(); } /**********************************************************************/ @@ -67,14 +60,14 @@ PyObject* VM::_run_top_frame(){ { #define CEVAL_STEP_CALLBACK() \ - if(_ceval_on_step) _ceval_on_step(this, frame.get(), byte); \ - if(_profiler) _profiler->_step(frame); + if(_ceval_on_step) _ceval_on_step(this, frame, byte); \ + if(_profiler) _profiler->_step(callstack._tail); #define DISPATCH_OP_CALL() { frame = top_frame(); goto __NEXT_FRAME; } __NEXT_FRAME: // cache const CodeObject* co = frame->co; - const auto& co_consts = co->consts; + PyObject** co_consts = const_cast(co->consts.data()); const Bytecode* co_codes = co->codes.data(); Bytecode byte = co_codes[frame->next_bytecode()]; @@ -598,7 +591,7 @@ __NEXT_STEP:; TARGET(RETURN_VALUE){ PyObject* _0 = byte.arg == BC_NOARG ? POPX() : None; _pop_frame(); - if(frame.index == base_id){ // [ frameBase<- ] + if(frame == base_frame){ // [ frameBase<- ] return _0; }else{ frame = top_frame(); @@ -826,16 +819,12 @@ __NEXT_STEP:; }catch(UnhandledException){ PyObject* e_obj = POPX(); Exception& _e = PK_OBJ_GET(Exception, e_obj); + bool is_base_frame_to_be_popped = frame == base_frame; _pop_frame(); - if(callstack.empty()){ -#if PK_DEBUG_FULL_EXCEPTION - std::cerr << _e.summary() << std::endl; -#endif - throw _e; - } + if(callstack.empty()) throw _e; // propagate to the top level frame = top_frame(); PUSH(e_obj); - if(frame.index < base_id) throw ToBeRaisedException(); + if(is_base_frame_to_be_popped) throw ToBeRaisedException(); need_raise = true; }catch(ToBeRaisedException){ need_raise = true; diff --git a/src/expr.cpp b/src/expr.cpp index 791e3a8c..a359178e 100644 --- a/src/expr.cpp +++ b/src/expr.cpp @@ -64,6 +64,14 @@ namespace pkpy{ return i; } + int CodeEmitContext::emit_int(i64 _val, int line){ + if(is_imm_int(_val)){ + return emit_(OP_LOAD_INTEGER, (uint16_t)_val, line); + }else{ + return emit_(OP_LOAD_CONST, add_const(VAR(_val)), line); + } + } + void CodeEmitContext::patch_jump(int index) { int target = co->codes.size(); co->codes[index].arg = target; @@ -246,11 +254,7 @@ namespace pkpy{ VM* vm = ctx->vm; if(std::holds_alternative(value)){ i64 _val = std::get(value); - if(is_imm_int(_val)){ - ctx->emit_(OP_LOAD_INTEGER, (uint16_t)_val, line); - return; - } - ctx->emit_(OP_LOAD_CONST, ctx->add_const(VAR(_val)), line); + ctx->emit_int(_val, line); return; } if(std::holds_alternative(value)){ @@ -272,11 +276,7 @@ namespace pkpy{ LiteralExpr* lit = static_cast(child.get()); if(std::holds_alternative(lit->value)){ i64 _val = -std::get(lit->value); - if(is_imm_int(_val)){ - ctx->emit_(OP_LOAD_INTEGER, (uint16_t)_val, line); - }else{ - ctx->emit_(OP_LOAD_CONST, ctx->add_const(VAR(_val)), line); - } + ctx->emit_int(_val, line); return; } if(std::holds_alternative(lit->value)){ @@ -594,8 +594,8 @@ namespace pkpy{ // vectorcall protocol for(auto& item: args) item->emit_(ctx); for(auto& item: kwargs){ - uint16_t index = StrName(item.first.sv()).index; - ctx->emit_(OP_LOAD_INTEGER, index, line); + i64 _val = StrName(item.first.sv()).index; + ctx->emit_int(_val, line); item.second->emit_(ctx); } int KWARGC = kwargs.size(); diff --git a/src/iter.cpp b/src/iter.cpp index 0cd6908e..e62108f1 100644 --- a/src/iter.cpp +++ b/src/iter.cpp @@ -48,7 +48,7 @@ namespace pkpy{ // restore the context for(PyObject* obj: s_backup) vm->s_data.push(obj); s_backup.clear(); - vm->callstack.push(std::move(frame)); + vm->callstack.emplace(std::move(frame)); PyObject* ret; try{ diff --git a/src/pocketpy.cpp b/src/pocketpy.cpp index ce4df652..53e61949 100644 --- a/src/pocketpy.cpp +++ b/src/pocketpy.cpp @@ -77,7 +77,7 @@ void init_builtins(VM* _vm) { class_arg = args[0]; self_arg = args[1]; }else if(args.size() == 0){ - FrameId frame = vm->top_frame(); + Frame* frame = vm->top_frame(); if(frame->_callable != nullptr){ class_arg = PK_OBJ_GET(Function, frame->_callable)._class; if(frame->_locals.size() > 0) self_arg = frame->_locals[0]; @@ -184,7 +184,7 @@ void init_builtins(VM* _vm) { CodeObject_ code = vm->compile(CAST(Str&, args[0]), "", EVAL_MODE, true); PyObject* globals = args[1]; if(globals == vm->None){ - FrameId frame = vm->top_frame(); + Frame* frame = vm->top_frame(); return vm->_exec(code.get(), frame->_module, frame->_callable, frame->_locals); } vm->check_non_tagged_type(globals, vm->tp_mappingproxy); @@ -196,7 +196,7 @@ void init_builtins(VM* _vm) { CodeObject_ code = vm->compile(CAST(Str&, args[0]), "", EXEC_MODE, true); PyObject* globals = args[1]; if(globals == vm->None){ - FrameId frame = vm->top_frame(); + Frame* frame = vm->top_frame(); vm->_exec(code.get(), frame->_module, frame->_callable, frame->_locals); return vm->None; } diff --git a/src/profiler.cpp b/src/profiler.cpp index c5c8399f..35eb60bf 100644 --- a/src/profiler.cpp +++ b/src/profiler.cpp @@ -18,16 +18,17 @@ void LineProfiler::begin(){ frames.clear(); } -void LineProfiler::_step(FrameId frame){ +void LineProfiler::_step(LinkedFrame* linked_frame){ + Frame* frame = &linked_frame->frame; auto line_info = frame->co->lines[frame->_ip]; if(line_info.is_virtual) return; std::string_view filename = frame->co->src->filename.sv(); int line = line_info.lineno; if(frames.empty()){ - frames.push({frame, clock(), nullptr}); + frames.push({linked_frame, clock(), nullptr}); }else{ - _step_end(frame, line); + _step_end(linked_frame, line); } auto& file_records = records[filename]; @@ -43,13 +44,19 @@ void LineProfiler::_step(FrameId frame){ frames.top().prev_record = &file_records.at(line); } -void LineProfiler::_step_end(FrameId frame, int line){ +void LineProfiler::_step_end(LinkedFrame* linked_frame, int line){ clock_t now = clock(); _FrameRecord& top_frame_record = frames.top(); _LineRecord* prev_record = top_frame_record.prev_record; - int id_delta = frame.index - top_frame_record.frame.index; - PK_ASSERT(id_delta >= -1 && id_delta <= 1); + int id_delta; + if(linked_frame == top_frame_record.frame){ + id_delta = 0; + }else if(linked_frame->f_back == top_frame_record.frame){ + id_delta = 1; + }else{ + id_delta = -1; // unsafe + } // current line is about to change if(prev_record->line != line){ @@ -60,7 +67,7 @@ void LineProfiler::_step_end(FrameId frame, int line){ } if(id_delta == 1){ - frames.push({frame, now, nullptr}); + frames.push({linked_frame, now, nullptr}); }else{ if(id_delta == -1) frames.pop(); } diff --git a/src/vm.cpp b/src/vm.cpp index e4f38144..a9605ffb 100644 --- a/src/vm.cpp +++ b/src/vm.cpp @@ -119,16 +119,15 @@ namespace pkpy{ PK_UNREACHABLE(); } - FrameId VM::top_frame(){ + Frame* VM::top_frame(){ #if PK_DEBUG_EXTRA_CHECK if(callstack.empty()) PK_FATAL_ERROR(); #endif - return FrameId(&callstack.container(), callstack.size()-1); + return &callstack.top(); } void VM::_pop_frame(){ - Frame* frame = &callstack.top(); - s_data.reset(frame->_sp_base); + s_data.reset(callstack.top()._sp_base); callstack.pop(); } @@ -652,7 +651,7 @@ void VM::_log_s_data(const char* title) { if(f._sp_base == nullptr) PK_FATAL_ERROR(); sp_bases[f._sp_base] += 1; } - FrameId frame = top_frame(); + Frame* frame = top_frame(); int line = frame->co->lines[frame->_ip]; ss << frame->co->name << ":" << line << " ["; for(PyObject** p=s_data.begin(); p!=s_data.end(); p++){ @@ -838,7 +837,7 @@ void VM::_prepare_py_call(PyObject** buffer, ArgsView args, ArgsView kwargs, con } for(int j=0; jkw_to_index.try_get_likely_found(key); // if key is an explicit key, set as local variable if(index >= 0){ @@ -861,13 +860,16 @@ PyObject* VM::vectorcall(int ARGC, int KWARGC, bool op_call){ // [callable, , args..., kwargs...] // ^p0 ^p1 ^_sp PyObject* callable = p1[-(ARGC + 2)]; + Type callable_t = _tp(callable); + bool method_call = p1[-(ARGC + 1)] != PY_NULL; // handle boundmethod, do a patch - if(is_non_tagged_type(callable, tp_bound_method)){ + if(callable_t == tp_bound_method){ if(method_call) PK_FATAL_ERROR(); BoundMethod& bm = PK_OBJ_GET(BoundMethod, callable); callable = bm.func; // get unbound method + callable_t = _tp(callable); p1[-(ARGC + 2)] = bm.func; p1[-(ARGC + 1)] = bm.self; method_call = true; @@ -880,26 +882,7 @@ PyObject* VM::vectorcall(int ARGC, int KWARGC, bool op_call){ PyObject** _base = args.begin(); PyObject* buffer[PK_MAX_CO_VARNAMES]; - if(is_non_tagged_type(callable, tp_native_func)){ - const auto& f = PK_OBJ_GET(NativeFunc, callable); - PyObject* ret; - if(f.decl != nullptr){ - int co_nlocals = f.decl->code->varnames.size(); - _prepare_py_call(buffer, args, kwargs, f.decl); - // copy buffer back to stack - s_data.reset(_base + co_nlocals); - for(int j=0; jis_generator){ s_data.reset(p0); return _py_generator( - Frame(nullptr, co, fn._module, callable), + Frame(nullptr, co, fn._module, callable, nullptr), ArgsView(buffer, buffer + co_nlocals) ); } @@ -941,13 +924,32 @@ PyObject* VM::vectorcall(int ARGC, int KWARGC, bool op_call){ for(int j=0; jcode->varnames.size(); + _prepare_py_call(buffer, args, kwargs, f.decl); + // copy buffer back to stack + s_data.reset(_base + co_nlocals); + for(int j=0; j_ip; @@ -1259,7 +1262,7 @@ void VM::_raise(bool re_raise){ void ManagedHeap::mark() { for(PyObject* obj: _no_gc) PK_OBJ_MARK(obj); - for(auto& frame : vm->callstack.container()) frame._gc_mark(); + vm->callstack.apply([](Frame& frame){ frame._gc_mark(); }); for(PyObject* obj: vm->s_data) PK_OBJ_MARK(obj); for(auto [_, co]: vm->_cached_codes) co->_gc_mark(); if(vm->_last_exception) PK_OBJ_MARK(vm->_last_exception);