diff --git a/src/ceval.h b/src/ceval.h index 526850e4..fd164d02 100644 --- a/src/ceval.h +++ b/src/ceval.h @@ -18,6 +18,9 @@ __NEXT_STEP:; //heap._auto_collect(this); const Bytecode& byte = frame->next_bytecode(); + + // std::cout << frame->stack_info() << " " << OP_NAMES[byte.op] << std::endl; + switch (byte.op) { case OP_NO_OP: DISPATCH(); @@ -296,7 +299,7 @@ __NEXT_STEP:; } CodeObject_ code = compile(source, name.str(), EXEC_MODE); PyObject* new_mod = new_module(name); - _exec(code, new_mod); + _exec(code, new_mod, builtins); new_mod->attr()._try_perfect_rehash(); } frame->push(ext_mod); diff --git a/src/codeobject.h b/src/codeobject.h index dd2682e6..264f2db9 100644 --- a/src/codeobject.h +++ b/src/codeobject.h @@ -57,7 +57,7 @@ struct CodeObject { std::vector codes; List consts; std::vector names; - std::set global_names; + std::set global_names; std::vector blocks = { CodeBlock{NO_BLOCK, -1} }; std::map labels; std::vector func_decls; diff --git a/src/common.h b/src/common.h index 88875867..c337e398 100644 --- a/src/common.h +++ b/src/common.h @@ -28,16 +28,19 @@ #include #include -#define PK_VERSION "0.9.6" +#define PK_VERSION "0.9.7" // debug macros -#define DEBUG_NO_BUILTIN_MODULES 1 -#define DEBUG_MODE 1 +#define DEBUG_NO_BUILTIN_MODULES 0 +#define DEBUG_EXTRA_CHECK 1 +#define DEBUG_DIS_REPL 0 +#define DEBUG_DIS_REPL_MIN 1 +#define DEBUG_FULL_EXCEPTION 1 #if (defined(__ANDROID__) && __ANDROID_API__ <= 22) || defined(__EMSCRIPTEN__) #define PK_ENABLE_FILEIO 0 #else -#define PK_ENABLE_FILEIO 1 +#define PK_ENABLE_FILEIO (1-DEBUG_NO_BUILTIN_MODULES) #endif #if defined(__EMSCRIPTEN__) || defined(__arm__) || defined(__i386__) diff --git a/src/compiler.h b/src/compiler.h index 530e234e..725327ba 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -1,5 +1,7 @@ #pragma once +#include "codeobject.h" +#include "common.h" #include "expr.h" namespace pkpy{ @@ -152,6 +154,17 @@ class Compiler { parse_expression(PREC_TUPLE, push_stack); } + // special case for `for loop` and `comp` + Expr_ EXPR_VARS(){ + std::vector items; + do { + consume(TK("@id")); + items.push_back(make_expr(prev().str(), name_scope())); + } while(match(TK(","))); + if(items.size()==1) return std::move(items[0]); + return make_expr(std::move(items)); + } + template std::unique_ptr make_expr(Args&&... args) { std::unique_ptr expr = std::make_unique(std::forward(args)...); @@ -269,10 +282,9 @@ class Compiler { template void _consume_comp(Expr_ expr){ static_assert(std::is_base_of::value); - std::unique_ptr ce = std::make_unique(); + std::unique_ptr ce = make_expr(); ce->expr = std::move(expr); - EXPR_TUPLE(); // must be a lvalue - ce->vars = ctx()->s_expr.popx(); + ce->vars = EXPR_VARS(); consume(TK("in")); EXPR(); ce->iter = ctx()->s_expr.popx(); @@ -374,7 +386,12 @@ class Compiler { // PASS void exprName(){ - ctx()->s_expr.push(make_expr(prev().str(), name_scope())); + Str name = prev().str(); + NameScope scope = name_scope(); + if(ctx()->co->global_names.count(name)){ + scope = NAME_GLOBAL; + } + ctx()->s_expr.push(make_expr(name, scope)); } // PASS @@ -389,26 +406,65 @@ class Compiler { void exprSubscr() { auto e = make_expr(); e->a = ctx()->s_expr.popx(); - std::vector items; - do { - EXPR_TUPLE(); - items.push_back(ctx()->s_expr.popx()); - } while(match(TK(":"))); - consume(TK("]")); - switch(items.size()){ - case 1: - e->b = std::move(items[0]); - break; - case 2: case 3: { - auto slice = make_expr(); - slice->start = std::move(items[0]); - slice->stop = std::move(items[1]); - if(items.size()==3){ - slice->step = std::move(items[2]); - } - e->b = std::move(slice); - } break; - default: SyntaxError(); break; + auto slice = make_expr(); + bool is_slice = false; + // a[<0> : state<3> : state<5>] + int state = 0; + do{ + switch(state){ + case 0: + if(match(TK(":"))){ + is_slice=true; + state=2; + break; + } + if(match(TK("]"))) SyntaxError(); + EXPR_TUPLE(); + slice->start = ctx()->s_expr.popx(); + state=1; + break; + case 1: + if(match(TK(":"))){ + is_slice=true; + state=2; + break; + } + if(match(TK("]"))) goto __SUBSCR_END; + SyntaxError("expected ':' or ']'"); + break; + case 2: + if(match(TK(":"))){ + state=4; + break; + } + if(match(TK("]"))) goto __SUBSCR_END; + EXPR_TUPLE(); + slice->stop = ctx()->s_expr.popx(); + state=3; + break; + case 3: + if(match(TK(":"))){ + state=4; + break; + } + if(match(TK("]"))) goto __SUBSCR_END; + SyntaxError("expected ':' or ']'"); + break; + case 4: + if(match(TK("]"))) goto __SUBSCR_END; + EXPR_TUPLE(); + slice->step = ctx()->s_expr.popx(); + state=5; + break; + case 5: consume(TK("]")); goto __SUBSCR_END; + } + }while(true); +__SUBSCR_END: + if(is_slice){ + e->b = std::move(slice); + }else{ + if(state != 1) UNREACHABLE(); + e->b = std::move(slice->start); } ctx()->s_expr.push(std::move(e)); } @@ -535,8 +591,7 @@ class Compiler { // PASS void compile_for_loop() { - EXPR_TUPLE(); - Expr_ vars = ctx()->s_expr.popx(); + Expr_ vars = EXPR_VARS(); consume(TK("in")); EXPR(false); ctx()->emit(OP_GET_ITER, BC_NOARG, BC_KEEPLINE); @@ -550,31 +605,32 @@ class Compiler { } void compile_try_except() { - // ctx()->enter_block(TRY_EXCEPT); - // ctx()->emit(OP_TRY_BLOCK_ENTER, BC_NOARG, prev().line); - // compile_block_body(); - // ctx()->emit(OP_TRY_BLOCK_EXIT, BC_NOARG, BC_KEEPLINE); - // std::vector patches = { - // ctx()->emit(OP_JUMP_ABSOLUTE, BC_NOARG, BC_KEEPLINE) - // }; - // ctx()->exit_block(); - - // do { - // consume(TK("except")); - // if(match(TK("@id"))){ - // int name_idx = ctx()->add_name(prev().str(), NAME_SPECIAL); - // emit(OP_EXCEPTION_MATCH, name_idx); - // }else{ - // emit(OP_LOAD_TRUE); - // } - // int patch = emit(OP_POP_JUMP_IF_FALSE); - // emit(OP_POP_TOP); // pop the exception on match - // compile_block_body(); - // patches.push_back(emit(OP_JUMP_ABSOLUTE)); - // patch_jump(patch); - // }while(curr().type == TK("except")); - // emit(OP_RE_RAISE); // no match, re-raise - // for (int patch : patches) patch_jump(patch); + ctx()->enter_block(TRY_EXCEPT); + ctx()->emit(OP_TRY_BLOCK_ENTER, BC_NOARG, prev().line); + compile_block_body(); + ctx()->emit(OP_TRY_BLOCK_EXIT, BC_NOARG, BC_KEEPLINE); + std::vector patches = { + ctx()->emit(OP_JUMP_ABSOLUTE, BC_NOARG, BC_KEEPLINE) + }; + ctx()->exit_block(); + do { + consume(TK("except")); + if(match(TK("@id"))){ + int namei = ctx()->add_name(prev().str()); + ctx()->emit(OP_EXCEPTION_MATCH, namei, prev().line); + }else{ + ctx()->emit(OP_LOAD_TRUE, BC_NOARG, BC_KEEPLINE); + } + int patch = ctx()->emit(OP_POP_JUMP_IF_FALSE, BC_NOARG, BC_KEEPLINE); + // pop the exception on match + ctx()->emit(OP_POP_TOP, BC_NOARG, BC_KEEPLINE); + compile_block_body(); + patches.push_back(ctx()->emit(OP_JUMP_ABSOLUTE, BC_NOARG, BC_KEEPLINE)); + ctx()->patch_jump(patch); + }while(curr().type == TK("except")); + // no match, re-raise + ctx()->emit(OP_RE_RAISE, BC_NOARG, BC_KEEPLINE); + for (int patch : patches) ctx()->patch_jump(patch); } void compile_decorated(){ @@ -611,7 +667,7 @@ class Compiler { break; default: return false; } - std::cout << ctx()->_log_s_expr() << std::endl; + // std::cout << ctx()->_log_s_expr() << std::endl; Expr_ rhs = ctx()->s_expr.popx(); if(lhs_p->is_starred() || rhs->is_starred()){ @@ -785,7 +841,6 @@ class Compiler { if(enable_type_hints && match(TK(":"))) consume(TK("@id")); if(state == 0 && curr().type == TK("=")) state = 2; - switch (state) { case 0: decl->args.push_back(name); break; diff --git a/src/expr.h b/src/expr.h index da2e25ca..ec00dad3 100644 --- a/src/expr.h +++ b/src/expr.h @@ -357,8 +357,8 @@ struct DictItemExpr: Expr{ std::vector children() const override { return {key.get(), value.get()}; } void emit(CodeEmitContext* ctx) override { - key->emit(ctx); value->emit(ctx); + key->emit(ctx); // reverse order ctx->emit(OP_BUILD_TUPLE, 2, line); } }; @@ -462,9 +462,11 @@ struct CompExpr: Expr{ if(cond){ cond->emit(ctx); int patch = ctx->emit(OP_POP_JUMP_IF_FALSE, BC_NOARG, BC_KEEPLINE); + expr->emit(ctx); ctx->emit(op1(), BC_NOARG, BC_KEEPLINE); ctx->patch_jump(patch); }else{ + expr->emit(ctx); ctx->emit(op1(), BC_NOARG, BC_KEEPLINE); } ctx->emit(OP_LOOP_CONTINUE, BC_NOARG, BC_KEEPLINE); diff --git a/src/frame.h b/src/frame.h index 265255e6..23d85b60 100644 --- a/src/frame.h +++ b/src/frame.h @@ -14,6 +14,7 @@ struct Frame { const CodeObject* co; PyObject* _module; NameDict_ _locals; + NameDict_ _closure; const uint64_t id; std::vector>> s_try_block; const NameDict* names[5]; // name resolution array, zero terminated @@ -21,14 +22,16 @@ struct Frame { NameDict& f_locals() noexcept { return *_locals; } NameDict& f_globals() noexcept { return _module->attr(); } - Frame(const CodeObject_& co, PyObject* _module, NameDict_ _locals=nullptr, NameDict_ _closure=nullptr) - : co(co.get()), _module(_module), _locals(_locals), id(kFrameGlobalId++) { + Frame(const CodeObject_& co, PyObject* _module, PyObject* builtins, NameDict_ _locals=nullptr, NameDict_ _closure=nullptr) + : co(co.get()), _module(_module), _locals(_locals), _closure(_closure), id(kFrameGlobalId++) { memset(names, 0, sizeof(names)); int i = 0; if(_locals != nullptr) names[i++] = _locals.get(); if(_closure != nullptr) names[i++] = _closure.get(); - names[i++] = &_module->attr(); - // names[i++] = builtins + names[i++] = &_module->attr(); // borrowed reference + if(builtins != nullptr){ + names[i++] = &builtins->attr(); // borrowed reference + } } const Bytecode& next_bytecode() { @@ -41,26 +44,26 @@ struct Frame { return co->src->snapshot(line); } - // Str stack_info(){ - // StrStream ss; - // ss << "["; - // for(int i=0; i<_data.size(); i++){ - // ss << OBJ_TP_NAME(_data[i]); - // if(i != _data.size()-1) ss << ", "; - // } - // ss << "]"; - // return ss.str(); - // } + Str stack_info(){ + StrStream ss; + ss << "["; + for(int i=0; i<_data.size(); i++){ + ss << (i64)_data[i]; + if(i != _data.size()-1) ss << ", "; + } + ss << "]"; + return ss.str(); + } void pop(){ -#if DEBUG_MODE +#if DEBUG_EXTRA_CHECK if(_data.empty()) throw std::runtime_error("_data.empty() is true"); #endif _data.pop_back(); } PyObject* popx(){ -#if DEBUG_MODE +#if DEBUG_EXTRA_CHECK if(_data.empty()) throw std::runtime_error("_data.empty() is true"); #endif PyObject* ret = _data.back(); @@ -69,21 +72,21 @@ struct Frame { } PyObject*& top(){ -#if DEBUG_MODE +#if DEBUG_EXTRA_CHECK if(_data.empty()) throw std::runtime_error("_data.empty() is true"); #endif return _data.back(); } PyObject*& top_1(){ -#if DEBUG_MODE +#if DEBUG_EXTRA_CHECK if(_data.size() < 2) throw std::runtime_error("_data.size() < 2"); #endif return _data[_data.size()-2]; } PyObject*& top_2(){ -#if DEBUG_MODE +#if DEBUG_EXTRA_CHECK if(_data.size() < 3) throw std::runtime_error("_data.size() < 3"); #endif return _data[_data.size()-3]; @@ -152,12 +155,8 @@ struct Frame { void _mark() const { for(PyObject* obj : _data) OBJ_MARK(obj); OBJ_MARK(_module); - - int i = 0; // names[0] is ensured to be non-null - do{ - names[i++]->_mark(); - }while(names[i] != nullptr); - + _locals->_mark(); + _closure->_mark(); for(auto& p : s_try_block){ for(PyObject* obj : p.second) OBJ_MARK(obj); } diff --git a/src/pocketpy.h b/src/pocketpy.h index 58b75418..7ef361ed 100644 --- a/src/pocketpy.h +++ b/src/pocketpy.h @@ -15,7 +15,9 @@ inline CodeObject_ VM::compile(Str source, Str filename, CompileMode mode) { try{ return compiler.compile(); }catch(Exception& e){ - // std::cout << e.summary() << std::endl; +#if DEBUG_FULL_EXCEPTION + std::cerr << e.summary() << std::endl; +#endif _error(e); return nullptr; } @@ -93,12 +95,12 @@ inline void init_builtins(VM* _vm) { _vm->bind_builtin_func<1>("eval", [](VM* vm, Args& args) { CodeObject_ code = vm->compile(CAST(Str&, args[0]), "", EVAL_MODE); - return vm->_exec(code, vm->top_frame()->_module, vm->top_frame()->_locals); + return vm->_exec(code, vm->top_frame()->_module, vm->builtins, vm->top_frame()->_locals); }); _vm->bind_builtin_func<1>("exec", [](VM* vm, Args& args) { CodeObject_ code = vm->compile(CAST(Str&, args[0]), "", EXEC_MODE); - vm->_exec(code, vm->top_frame()->_module, vm->top_frame()->_locals); + vm->_exec(code, vm->top_frame()->_module, vm->builtins, vm->top_frame()->_locals); return vm->None; }); @@ -597,7 +599,7 @@ inline void add_module_json(VM* vm){ vm->bind_func<1>(mod, "loads", [](VM* vm, Args& args) { const Str& expr = CAST(Str&, args[0]); CodeObject_ code = vm->compile(expr, "", JSON_MODE); - return vm->_exec(code, vm->top_frame()->_module, vm->top_frame()->_locals); + return vm->_exec(code, vm->top_frame()->_module, vm->builtins, vm->top_frame()->_locals); }); vm->bind_func<1>(mod, "dumps", CPP_LAMBDA(vm->call(args[0], __json__, no_arg()))); @@ -750,7 +752,7 @@ inline void add_module_random(VM* vm){ PyObject* mod = vm->new_module("random"); Random::register_class(vm, mod); CodeObject_ code = vm->compile(kPythonLibs["random"], "random.py", EXEC_MODE); - vm->_exec(code, mod); + vm->_exec(code, mod, vm->builtins); } inline void add_module_gc(VM* vm){ @@ -778,11 +780,11 @@ inline void VM::post_init(){ } CodeObject_ code = compile(kPythonLibs["builtins"], "", EXEC_MODE); - this->_exec(code, this->builtins); + this->_exec(code, this->builtins, nullptr); code = compile(kPythonLibs["_dict"], "", EXEC_MODE); - this->_exec(code, this->builtins); + this->_exec(code, this->builtins, nullptr); code = compile(kPythonLibs["_set"], "", EXEC_MODE); - this->_exec(code, this->builtins); + this->_exec(code, this->builtins, nullptr); // property is defined in builtins.py so we need to add it after builtins is loaded _t(tp_object)->attr().set(__class__, property(CPP_LAMBDA(vm->_t(args[0])))); diff --git a/src/vm.h b/src/vm.h index 846b05f6..6197e70b 100644 --- a/src/vm.h +++ b/src/vm.h @@ -93,7 +93,7 @@ public: } Frame* top_frame() const { -#if DEBUG_MODE +#if DEBUG_EXTRA_CHECK if(callstack.empty()) UNREACHABLE(); #endif return callstack.top().get(); @@ -166,13 +166,15 @@ public: if(_module == nullptr) _module = _main; try { CodeObject_ code = compile(source, filename, mode); +#if DEBUG_DIS_REPL if(_module == _main) std::cout << disassemble(code) << '\n'; - return _exec(code, _module); +#endif + return _exec(code, _module, builtins); }catch (const Exception& e){ *_stderr << e.summary() << '\n'; } -#if !DEBUG_MODE +#if !DEBUG_FULL_EXCEPTION catch (const std::exception& e) { *_stderr << "An std::exception occurred! It could be a bug.\n"; *_stderr << e.what() << '\n'; @@ -607,11 +609,12 @@ inline Str VM::disassemble(CodeObject_ co){ argStr += " (" + BITWISE_SPECIAL_METHODS[byte.arg].str() + ")"; break; } - ss << argStr; - // ss << pad(argStr, 20); // may overflow - // ss << co->blocks[byte.block].to_string(); + ss << pad(argStr, 40); // may overflow + ss << co->blocks[byte.block].type; if(i != co->codes.size() - 1) ss << '\n'; } + +#if !DEBUG_DIS_REPL_MIN StrStream consts; consts << "co_consts: "; consts << CAST(Str, asRepr(VAR(co->consts))); @@ -624,7 +627,7 @@ inline Str VM::disassemble(CodeObject_ co){ } names << CAST(Str, asRepr(VAR(list))); ss << '\n' << consts.str() << '\n' << names.str(); - +#endif for(auto& decl: co->func_decls){ ss << "\n\n" << "Disassembly of " << decl->name.str() << ":\n"; ss << disassemble(decl->code); @@ -753,7 +756,7 @@ inline PyObject* VM::call(PyObject* callable, Args args, const Args& kwargs, boo locals->set(key, kwargs[i+1]); } PyObject* _module = fn._module != nullptr ? fn._module : top_frame()->_module; - auto _frame = _new_frame(fn.decl->code, _module, locals, fn._closure); + auto _frame = _new_frame(fn.decl->code, _module, builtins, locals, fn._closure); if(fn.decl->code->is_generator) return PyIter(Generator(this, std::move(_frame))); callstack.push(std::move(_frame)); if(opCall) return _py_op_call;