From c2be07b9cc155bbc3be1782e626399027bcfabe9 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sun, 9 Jun 2024 15:25:42 +0800 Subject: [PATCH] some fix --- build_g.sh | 2 +- include/pocketpy/common/config.h | 1 + include/pocketpy/common/vector.hpp | 15 +- include/pocketpy/compiler/compiler.hpp | 26 ++- include/pocketpy/compiler/expr.hpp | 57 +++++- include/pocketpy/compiler/lexer.hpp | 1 + include/pocketpy/objects/error.hpp | 6 +- scripts/run_tests.py | 17 +- src/common/str.cpp | 7 +- src/compiler/compiler.cpp | 235 +++++++++++++------------ src/compiler/expr.cpp | 13 +- src/compiler/lexer.cpp | 11 +- src/interpreter/vm.cpp | 19 +- tests/44_decorator.py | 41 ++++- tests/82_dataclasses.py | 12 -- tests/94_compile.py | 3 + 16 files changed, 287 insertions(+), 179 deletions(-) diff --git a/build_g.sh b/build_g.sh index c677933b..a634da62 100644 --- a/build_g.sh +++ b/build_g.sh @@ -2,6 +2,6 @@ python prebuild.py SRC=$(find src/ -name "*.cpp") -FLAGS="-std=c++17 -Og -stdlib=libc++ -Iinclude -frtti -Wfatal-errors -g -DDEBUG" +FLAGS="-std=c++17 -O0 -stdlib=libc++ -Iinclude -frtti -Wfatal-errors -g -DDEBUG" clang++ $FLAGS -o main src2/main.cpp $SRC diff --git a/include/pocketpy/common/config.h b/include/pocketpy/common/config.h index 55038ca3..547c4a50 100644 --- a/include/pocketpy/common/config.h +++ b/include/pocketpy/common/config.h @@ -37,6 +37,7 @@ #define PK_DEBUG_MEMORY_POOL 0 #define PK_DEBUG_NO_AUTO_GC 0 #define PK_DEBUG_GC_STATS 0 +#define PK_DEBUG_COMPILER 0 #ifndef PK_DEBUG_PRECOMPILED_EXEC #define PK_DEBUG_PRECOMPILED_EXEC 0 diff --git a/include/pocketpy/common/vector.hpp b/include/pocketpy/common/vector.hpp index 3d48dabc..ac7d4640 100644 --- a/include/pocketpy/common/vector.hpp +++ b/include/pocketpy/common/vector.hpp @@ -165,7 +165,10 @@ struct vector { int capacity() const { return _capacity; } - T& back() { return _data[_size - 1]; } + T& back() { + assert(_size > 0); + return _data[_size - 1]; + } T* begin() const { return _data; } @@ -173,9 +176,15 @@ struct vector { T* data() const { return _data; } - T& operator[] (int i) { return _data[i]; } + T& operator[] (int i) { + assert(i >= 0 && i < _size); + return _data[i]; + } - const T& operator[] (int i) const { return _data[i]; } + const T& operator[] (int i) const { + assert(i >= 0 && i < _size); + return _data[i]; + } void clear() { std::destroy(begin(), end()); diff --git a/include/pocketpy/compiler/compiler.hpp b/include/pocketpy/compiler/compiler.hpp index 6c2335d8..f5c7b1a3 100644 --- a/include/pocketpy/compiler/compiler.hpp +++ b/include/pocketpy/compiler/compiler.hpp @@ -24,23 +24,33 @@ struct Compiler { VM* vm; bool unknown_global_scope; // for eval/exec() call // for parsing token stream - int i = 0; + int __i = 0; const Token& tk(int i) const noexcept{ return lexer.nexts[i]; } - const Token& prev() const noexcept{ return tk(i - 1); } - const Token& curr() const noexcept{ return tk(i); } - const Token& next() const noexcept{ return tk(i + 1); } + const Token& prev() const noexcept{ return tk(__i - 1); } + const Token& curr() const noexcept{ return tk(__i); } + const Token& next() const noexcept{ return tk(__i + 1); } const Token& err() const noexcept{ - if(i >= lexer.nexts.size()) return prev(); + if(__i >= lexer.nexts.size()) return prev(); return curr(); } - void advance(int delta = 1) noexcept{ i += delta; } + void advance(int delta = 1) noexcept{ + __i += delta; +#if PK_DEBUG_COMPILER + if(__i>=0 && __ifilename.c_str(), + curr().line, + TK_STR(curr().type), + curr().str().escape().c_str() + ); + } +#endif + } CodeEmitContext* ctx() noexcept{ return &contexts.back(); } - vector& s_expr() noexcept{ return ctx()->s_expr; } - CompileMode mode() const noexcept{ return lexer.src->mode; } NameScope name_scope() const noexcept; diff --git a/include/pocketpy/compiler/expr.hpp b/include/pocketpy/compiler/expr.hpp index d63e4605..e5483ba8 100644 --- a/include/pocketpy/compiler/expr.hpp +++ b/include/pocketpy/compiler/expr.hpp @@ -60,7 +60,7 @@ struct CodeEmitContext{ VM* vm; FuncDecl_ func; // optional CodeObject_ co; // 1 CodeEmitContext <=> 1 CodeObject_ - vector s_expr; + vector _s_expr; int level; vector global_names; @@ -75,8 +75,6 @@ struct CodeEmitContext{ int get_loop() const noexcept; CodeBlock* enter_block(CodeBlockType type) noexcept; void exit_block() noexcept; - void emit_expr(bool emit = true) noexcept; // clear the expression stack and generate bytecode - void emit_decorators(int count) noexcept; int emit_(Opcode opcode, uint16_t arg, int line, bool is_virtual = false) noexcept; void revert_last_emit_() noexcept; int emit_int(i64 value, int line) noexcept; @@ -88,6 +86,59 @@ struct CodeEmitContext{ int add_func_decl(FuncDecl_ decl) noexcept; void emit_store_name(NameScope scope, StrName name, int line) noexcept; void try_merge_for_iter_store(int) noexcept; + // emit top -> pop -> delete + void s_emit_top() noexcept{ + s_debug_info("s_emit_top"); + Expr* e = _s_expr.popx_back(); + e->emit_(this); + delete_expr(e); + } + // push + void s_push(Expr* expr) noexcept{ + s_debug_info("s_push"); + _s_expr.push_back(expr); + } + // top + Expr* s_top() noexcept{ + return _s_expr.back(); + } + // size + int s_size() const noexcept{ + return _s_expr.size(); + } + // pop -> delete + void s_pop() noexcept{ + s_debug_info("s_pop"); + Expr* e = _s_expr.popx_back(); + delete_expr(e); + } + // pop move + Expr* s_popx() noexcept{ + s_debug_info("s_popx"); + return _s_expr.popx_back(); + } + // clean + void s_clean() noexcept{ + s_debug_info("s_clean"); + for(Expr* e: _s_expr) delete_expr(e); + _s_expr.clear(); + } + // emit decorators + void s_emit_decorators(int count) noexcept; + // debug stack +#if PK_DEBUG_COMPILER + void s_debug_info(const char* op) noexcept{ + SStream ss; + for(int i=0; i nexts; small_vector_2 indents; int brackets_level = 0; + bool used = false; char peekchar() const noexcept { return *curr_char; } diff --git a/include/pocketpy/objects/error.hpp b/include/pocketpy/objects/error.hpp index c24ddc87..a125329d 100644 --- a/include/pocketpy/objects/error.hpp +++ b/include/pocketpy/objects/error.hpp @@ -33,14 +33,14 @@ struct Exception { PyObject* _self; // weak reference struct Frame { - SourceData* src; // weak ref + std::shared_ptr src; // weak ref int lineno; const char* cursor; std::string name; Str snapshot() const { return src->snapshot(lineno, cursor, name); } - Frame(SourceData* src, int lineno, const char* cursor, std::string_view name) : + Frame(std::shared_ptr src, int lineno, const char* cursor, std::string_view name) : src(src), lineno(lineno), cursor(cursor), name(name) {} }; @@ -79,7 +79,7 @@ struct TopLevelException : std::exception { struct Error{ const char* type; - SourceData* src; + std::shared_ptr src; int lineno; const char* cursor; char msg[100]; diff --git a/scripts/run_tests.py b/scripts/run_tests.py index c125cc60..6f88edc0 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -39,13 +39,7 @@ def test_dir(path): print('CPython:', str(sys.version).replace('\n', '')) print('System:', '64-bit' if sys.maxsize > 2**32 else '32-bit') -if len(sys.argv) == 2: - assert 'benchmark' in sys.argv[1] - test_dir('benchmarks/') -else: - test_dir('tests/') - - # test interactive mode +def test_repl(): print("[REPL Test Enabled]") if sys.platform in ['linux', 'darwin']: cmd = './main' @@ -74,4 +68,13 @@ exit() assert 'ans_1: 3' in res.stdout, res.stdout assert 'ans_2: abc' in res.stdout, res.stdout + +if len(sys.argv) == 2: + assert 'benchmark' in sys.argv[1] + test_dir('benchmarks/') +else: + test_dir('tests/') + # test_repl() + + print("ALL TESTS PASSED") diff --git a/src/common/str.cpp b/src/common/str.cpp index b91f8e58..e7aa417a 100644 --- a/src/common/str.cpp +++ b/src/common/str.cpp @@ -391,9 +391,10 @@ StrName StrName::get(std::string_view s) { Str SStream::str() { // after this call, the buffer is no longer valid - buffer.reserve(buffer.size() + 1); // allocate one more byte for '\0' - buffer[buffer.size()] = '\0'; // set '\0' - return Str(buffer.detach()); + buffer.push_back('\0'); + auto detached = buffer.detach(); + detached.second--; // remove the last '\0' + return Str(detached); } SStream& SStream::operator<< (const Str& s) { diff --git a/src/compiler/compiler.cpp b/src/compiler/compiler.cpp index ed80b660..60ec72f0 100644 --- a/src/compiler/compiler.cpp +++ b/src/compiler/compiler.cpp @@ -6,7 +6,7 @@ namespace pkpy { #define consume(expected) if(!match(expected)) return SyntaxError("expected '%s', got '%s'", TK_STR(expected), TK_STR(curr().type)); #define consume_end_stmt() if(!match_end_stmt()) return SyntaxError("expected statement end") -#define check(B) err = B; if(err) return err +#define check(B) if((err = B)) return err PrattRule Compiler::rules[kTokenCount]; @@ -18,7 +18,7 @@ NameScope Compiler::name_scope() const noexcept{ CodeObject_ Compiler::push_global_context() noexcept{ CodeObject_ co = std::make_shared(lexer.src, lexer.src->filename); - co->start_line = i == 0 ? 1 : prev().line; + co->start_line = __i == 0 ? 1 : prev().line; contexts.push_back(CodeEmitContext(vm, co, contexts.size())); return co; } @@ -26,7 +26,7 @@ CodeObject_ Compiler::push_global_context() noexcept{ FuncDecl_ Compiler::push_f_context(Str name) noexcept{ FuncDecl_ decl = std::make_shared(); decl->code = std::make_shared(lexer.src, name); - decl->code->start_line = i == 0 ? 1 : prev().line; + decl->code->start_line = __i == 0 ? 1 : prev().line; decl->nested = name_scope() == NAME_LOCAL; contexts.push_back(CodeEmitContext(vm, decl->code, contexts.size())); contexts.back().func = decl; @@ -34,13 +34,13 @@ FuncDecl_ Compiler::push_f_context(Str name) noexcept{ } Error* Compiler::pop_context() noexcept{ - assert(s_expr().empty()); + assert(ctx()->s_size() == 0); // add a `return None` in the end as a guard // previously, we only do this if the last opcode is not a return // however, this is buggy...since there may be a jump to the end (out of bound) even if the last opcode is a return ctx()->emit_(OP_RETURN_VALUE, 1, BC_KEEPLINE, true); // find the last valid token - int j = i - 1; + int j = __i - 1; while(tk(j).type == TK("@eol") || tk(j).type == TK("@dedent") || tk(j).type == TK("@eof")) j--; ctx()->co->end_line = tk(j).line; @@ -205,8 +205,8 @@ Error* Compiler::EXPR_TUPLE(bool allow_slice) noexcept{ } while(match(TK(","))); TupleExpr* e = make_expr(count); for(int i=count-1; i>=0; i--) - e->items[i] = s_expr().popx_back(); - s_expr().push_back(e); + e->items[i] = ctx()->s_popx(); + ctx()->s_push(e); return NULL; } @@ -214,40 +214,40 @@ Error* Compiler::EXPR_VARS() noexcept{ int count = 0; do { consume(TK("@id")); - s_expr().push_back(make_expr(prev().str(), name_scope())); + ctx()->s_push(make_expr(prev().str(), name_scope())); count += 1; } while(match(TK(","))); if(count > 1){ TupleExpr* e = make_expr(count); for(int i=count-1; i>=0; i--) - e->items[i] = s_expr().popx_back(); - s_expr().push_back(e); + e->items[i] = ctx()->s_popx(); + ctx()->s_push(e); } return NULL; } Error* Compiler::exprLiteral() noexcept{ - s_expr().push_back(make_expr(prev().value)); + ctx()->s_push(make_expr(prev().value)); return NULL; } Error* Compiler::exprLong() noexcept{ - s_expr().push_back(make_expr(prev().str())); + ctx()->s_push(make_expr(prev().str())); return NULL; } Error* Compiler::exprImag() noexcept{ - s_expr().push_back(make_expr(std::get(prev().value))); + ctx()->s_push(make_expr(std::get(prev().value))); return NULL; } Error* Compiler::exprBytes() noexcept{ - s_expr().push_back(make_expr(std::get(prev().value))); + ctx()->s_push(make_expr(std::get(prev().value))); return NULL; } Error* Compiler::exprFString() noexcept{ - s_expr().push_back(make_expr(std::get(prev().value))); + ctx()->s_push(make_expr(std::get(prev().value))); return NULL; } @@ -261,38 +261,38 @@ Error* Compiler::exprLambda() noexcept{ } // https://github.com/pocketpy/pocketpy/issues/37 check(parse_expression(PREC_LAMBDA + 1)); - ctx()->emit_expr(); + ctx()->s_emit_top(); ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE); check(pop_context()); LambdaExpr* e = make_expr(decl); e->line = line; - s_expr().push_back(e); + ctx()->s_push(e); return NULL; } Error* Compiler::exprOr() noexcept{ auto e = make_expr(); - e->lhs = s_expr().popx_back(); + e->lhs = ctx()->s_popx(); Error* err = parse_expression(PREC_LOGICAL_OR + 1); if(err){ delete_expr(e); return err; } - e->rhs = s_expr().popx_back(); - s_expr().push_back(e); + e->rhs = ctx()->s_popx(); + ctx()->s_push(e); return NULL; } Error* Compiler::exprAnd() noexcept{ auto e = make_expr(); - e->lhs = s_expr().popx_back(); + e->lhs = ctx()->s_popx(); Error* err = parse_expression(PREC_LOGICAL_AND + 1); if(err){ delete_expr(e); return err; } - e->rhs = s_expr().popx_back(); - s_expr().push_back(e); + e->rhs = ctx()->s_popx(); + ctx()->s_push(e); return NULL; } @@ -305,31 +305,31 @@ Error* Compiler::exprTernary() noexcept{ check(parse_expression(PREC_TERNARY + 1)); // [true_expr, cond, false_expr] auto e = make_expr(); e->line = line; - e->false_expr = s_expr().popx_back(); - e->cond = s_expr().popx_back(); - e->true_expr = s_expr().popx_back(); - s_expr().push_back(e); + e->false_expr = ctx()->s_popx(); + e->cond = ctx()->s_popx(); + e->true_expr = ctx()->s_popx(); + ctx()->s_push(e); return NULL; } Error* Compiler::exprBinaryOp() noexcept{ auto e = make_expr(prev().type); - e->lhs = s_expr().popx_back(); + e->lhs = ctx()->s_popx(); Error* err = parse_expression(rules[e->op].precedence + 1); if(err){ delete_expr(e); return err; } - e->rhs = s_expr().popx_back(); - s_expr().push_back(std::move(e)); + e->rhs = ctx()->s_popx(); + ctx()->s_push(std::move(e)); return NULL; } Error* Compiler::exprNot() noexcept{ Error* err; check(parse_expression(PREC_LOGICAL_NOT + 1)); - NotExpr* e = make_expr(s_expr().popx_back()); - s_expr().push_back(e); + NotExpr* e = make_expr(ctx()->s_popx()); + ctx()->s_push(e); return NULL; } @@ -338,10 +338,10 @@ Error* Compiler::exprUnaryOp() noexcept{ TokenIndex op = prev().type; check(parse_expression(PREC_UNARY + 1)); switch(op) { - case TK("-"): s_expr().push_back(make_expr(s_expr().popx_back())); break; - case TK("~"): s_expr().push_back(make_expr(s_expr().popx_back())); break; - case TK("*"): s_expr().push_back(make_expr(s_expr().popx_back(), 1)); break; - case TK("**"): s_expr().push_back(make_expr(s_expr().popx_back(), 2)); break; + case TK("-"): ctx()->s_push(make_expr(ctx()->s_popx())); break; + case TK("~"): ctx()->s_push(make_expr(ctx()->s_popx())); break; + case TK("*"): ctx()->s_push(make_expr(ctx()->s_popx(), 1)); break; + case TK("**"): ctx()->s_push(make_expr(ctx()->s_popx(), 2)); break; default: assert(false); } return NULL; @@ -353,9 +353,9 @@ Error* Compiler::exprGroup() noexcept{ check(EXPR_TUPLE()); // () is just for change precedence match_newlines_repl(); consume(TK(")")); - if(s_expr().back()->is_tuple()) return NULL; - Expr* g = make_expr(s_expr().popx_back()); - s_expr().push_back(g); + if(ctx()->s_top()->is_tuple()) return NULL; + Expr* g = make_expr(ctx()->s_popx()); + ctx()->s_push(g); return NULL; } @@ -372,11 +372,11 @@ Error* Compiler::consume_comp(Opcode op0, Opcode op1) noexcept{ has_cond = true; } CompExpr* ce = make_expr(op0, op1); - if(has_cond) ce->cond = s_expr().popx_back(); - ce->iter = s_expr().popx_back(); - ce->vars = s_expr().popx_back(); - ce->expr = s_expr().popx_back(); - s_expr().push_back(ce); + if(has_cond) ce->cond = ctx()->s_popx(); + ce->iter = ctx()->s_popx(); + ce->vars = ctx()->s_popx(); + ce->expr = ctx()->s_popx(); + ctx()->s_push(ce); match_newlines_repl(); return NULL; } @@ -401,8 +401,8 @@ Error* Compiler::exprList() noexcept{ ListExpr* e = make_expr(count); e->line = line; // override line for(int i=count-1; i>=0; i--) - e->items[i] = s_expr().popx_back(); - s_expr().push_back(e); + e->items[i] = ctx()->s_popx(); + ctx()->s_push(e); return NULL; } @@ -414,30 +414,31 @@ Error* Compiler::exprMap() noexcept{ match_newlines_repl(); if(curr().type == TK("}")) break; check(EXPR()); // [key] - int star_level = s_expr().back()->star_level(); + int star_level = ctx()->s_top()->star_level(); if(star_level == 2 || curr().type == TK(":")) { parsing_dict = true; } if(parsing_dict) { if(star_level == 2) { DictItemExpr* dict_item = make_expr(); dict_item->key = NULL; - dict_item->value = s_expr().popx_back(); - s_expr().push_back(dict_item); + dict_item->value = ctx()->s_popx(); + ctx()->s_push(dict_item); } else { consume(TK(":")); check(EXPR()); DictItemExpr* dict_item = make_expr(); - dict_item->value = s_expr().popx_back(); - dict_item->key = s_expr().popx_back(); - s_expr().push_back(dict_item); + dict_item->value = ctx()->s_popx(); + dict_item->key = ctx()->s_popx(); + ctx()->s_push(dict_item); } } count += 1; match_newlines_repl(); if(count == 1 && match(TK("for"))) { - if(parsing_dict) + if(parsing_dict){ check(consume_comp(OP_BUILD_DICT, OP_DICT_ADD)); - else + }else{ check(consume_comp(OP_BUILD_SET, OP_SET_ADD)); + } consume(TK("}")); return NULL; } @@ -452,16 +453,16 @@ Error* Compiler::exprMap() noexcept{ se = make_expr(count); } for(int i=count-1; i>=0; i--) - se->items[i] = s_expr().popx_back(); - s_expr().push_back(se); + se->items[i] = ctx()->s_popx(); + ctx()->s_push(se); return NULL; } Error* Compiler::exprCall() noexcept{ Error* err; CallExpr* e = make_expr(); - e->callable = s_expr().popx_back(); - s_expr().push_back(e); // push onto the stack in advance + e->callable = ctx()->s_popx(); + ctx()->s_push(e); // push onto the stack in advance do { match_newlines_repl(); if(curr().type == TK(")")) break; @@ -470,16 +471,16 @@ Error* Compiler::exprCall() noexcept{ StrName key(prev().sv()); consume(TK("=")); check(EXPR()); - e->kwargs.push_back({key, s_expr().popx_back()}); + e->kwargs.push_back({key, ctx()->s_popx()}); } else { check(EXPR()); - if(s_expr().back()->star_level() == 2) { + if(ctx()->s_top()->star_level() == 2) { // **kwargs - e->kwargs.push_back({"**", s_expr().popx_back()}); + e->kwargs.push_back({"**", ctx()->s_popx()}); } else { // positional argument if(!e->kwargs.empty()) return SyntaxError("positional argument follows keyword argument"); - e->args.push_back(s_expr().popx_back()); + e->args.push_back(ctx()->s_popx()); } } match_newlines_repl(); @@ -492,32 +493,32 @@ Error* Compiler::exprName() noexcept{ StrName name(prev().sv()); NameScope scope = name_scope(); if(ctx()->global_names.contains(name)) { scope = NAME_GLOBAL; } - s_expr().push_back(make_expr(name, scope)); + ctx()->s_push(make_expr(name, scope)); return NULL; } Error* Compiler::exprAttrib() noexcept{ consume(TK("@id")); - s_expr().push_back(make_expr(s_expr().popx_back(), StrName::get(prev().sv()))); + ctx()->s_push(make_expr(ctx()->s_popx(), StrName::get(prev().sv()))); return NULL; } Error* Compiler::exprSlice0() noexcept{ Error* err; SliceExpr* slice = make_expr(); - s_expr().push_back(slice); // push onto the stack in advance + ctx()->s_push(slice); // push onto the stack in advance if(is_expression()) { // : check(EXPR()); - slice->stop = s_expr().popx_back(); + slice->stop = ctx()->s_popx(); // try optional step if(match(TK(":"))) { // :: check(EXPR()); - slice->step = s_expr().popx_back(); + slice->step = ctx()->s_popx(); } } else if(match(TK(":"))) { if(is_expression()) { // :: check(EXPR()); - slice->step = s_expr().popx_back(); + slice->step = ctx()->s_popx(); } // else :: } // else : return NULL; @@ -526,38 +527,40 @@ Error* Compiler::exprSlice0() noexcept{ Error* Compiler::exprSlice1() noexcept{ Error* err; SliceExpr* slice = make_expr(); - slice->start = s_expr().popx_back(); - s_expr().push_back(slice); // push onto the stack in advance + slice->start = ctx()->s_popx(); + ctx()->s_push(slice); // push onto the stack in advance if(is_expression()) { // : check(EXPR()); - slice->stop = s_expr().popx_back(); + slice->stop = ctx()->s_popx(); // try optional step if(match(TK(":"))) { // :: check(EXPR()); - slice->step = s_expr().popx_back(); + slice->step = ctx()->s_popx(); } } else if(match(TK(":"))) { // :: check(EXPR()); - slice->step = s_expr().popx_back(); + slice->step = ctx()->s_popx(); } // else : return NULL; } Error* Compiler::exprSubscr() noexcept{ Error* err; - SubscrExpr* e = make_expr(); - s_expr().push_back(e); + int line = prev().line; match_newlines_repl(); - e->lhs = s_expr().popx_back(); // a check(EXPR_TUPLE(true)); - e->rhs = s_expr().popx_back(); // a[] match_newlines_repl(); - consume(TK("]")); + consume(TK("]")); // [lhs, rhs] + SubscrExpr* e = make_expr(); + e->line = line; + e->rhs = ctx()->s_popx(); // [lhs] + e->lhs = ctx()->s_popx(); // [] + ctx()->s_push(e); return NULL; } Error* Compiler::exprLiteral0() noexcept{ - s_expr().push_back(make_expr(prev().type)); + ctx()->s_push(make_expr(prev().type)); return NULL; } @@ -697,7 +700,7 @@ Error* Compiler::parse_expression(int precedence, bool allow_slice) noexcept{ Error* Compiler::compile_if_stmt() noexcept{ Error* err; check(EXPR()); // condition - ctx()->emit_expr(); + ctx()->s_emit_top(); int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line); err = compile_block_body(); if(err) return err; @@ -721,7 +724,7 @@ Error* Compiler::compile_while_loop() noexcept{ Error* err; CodeBlock* block = ctx()->enter_block(CodeBlockType::WHILE_LOOP); check(EXPR()); // condition - ctx()->emit_expr(); + ctx()->s_emit_top(); int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line); check(compile_block_body()); ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE, true); @@ -740,11 +743,11 @@ Error* Compiler::compile_for_loop() noexcept{ check(EXPR_VARS()); // [vars] consume(TK("in")); check(EXPR_TUPLE()); // [vars, iter] - ctx()->emit_expr(); // [vars] + ctx()->s_emit_top(); // [vars] ctx()->emit_(OP_GET_ITER_NEW, BC_NOARG, BC_KEEPLINE); CodeBlock* block = ctx()->enter_block(CodeBlockType::FOR_LOOP); int for_codei = ctx()->emit_(OP_FOR_ITER, ctx()->curr_iblock, BC_KEEPLINE); - Expr* vars = s_expr().popx_back(); + Expr* vars = ctx()->s_popx(); bool ok = vars->emit_store(ctx()); delete_expr(vars); if(!ok) return SyntaxError(); // this error occurs in `vars` instead of this line, but...nevermind @@ -776,7 +779,7 @@ Error* Compiler::compile_try_except() noexcept{ consume(TK("except")); if(is_expression()) { check(EXPR()); // push assumed type on to the stack - ctx()->emit_expr(); + ctx()->s_emit_top(); ctx()->emit_(OP_EXCEPTION_MATCH, BC_NOARG, prev().line); if(match(TK("as"))) { consume(TK("@id")); @@ -836,10 +839,6 @@ Error* Compiler::compile_decorated() noexcept{ if(!match_newlines_repl()) return SyntaxError(); } while(match(TK("@"))); - array decorators(count); - for(int i = count - 1; i >= 0; i--) - decorators[i] = s_expr().popx_back(); - if(match(TK("class"))) { check(compile_class(count)); } else { @@ -863,7 +862,7 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{ case TK("&="): case TK("|="): case TK("^="): { - Expr* lhs_p = s_expr().back(); + Expr* lhs_p = ctx()->s_top(); if(lhs_p->is_starred()) return SyntaxError(); if(ctx()->is_compiling_class){ return SyntaxError("can't use inplace operator in class definition"); @@ -873,14 +872,15 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{ // a.x += 1; a should be evaluated only once // -1 to remove =; inplace=true auto e = make_expr(prev().type - 1, true); - e->lhs = s_expr().popx_back(); + e->lhs = ctx()->s_popx(); check(EXPR_TUPLE()); - e->rhs = s_expr().popx_back(); + e->rhs = ctx()->s_popx(); if(e->rhs->is_starred()) return SyntaxError(); e->emit_(ctx()); bool ok = lhs_p->emit_store_inplace(ctx()); if(!ok) return SyntaxError(); *is_assign = true; + return NULL; } case TK("="): { int n = 0; @@ -889,16 +889,17 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{ n += 1; } // stack size is n+1 - ctx()->emit_expr(); // emit and pop + ctx()->s_emit_top(); // emit and pop for(int j = 1; j < n; j++) ctx()->emit_(OP_DUP_TOP, BC_NOARG, BC_KEEPLINE); for(int j = 0; j < n; j++) { - auto e = s_expr().popx_back(); + auto e = ctx()->s_popx(); if(e->is_starred()) return SyntaxError(); bool ok = e->emit_store(ctx()); if(!ok) return SyntaxError(); } *is_assign = true; + return NULL; } default: *is_assign = false; } @@ -928,14 +929,14 @@ Error* Compiler::compile_stmt() noexcept{ case TK("yield"): if(contexts.size() <= 1) return SyntaxError("'yield' outside function"); check(EXPR_TUPLE()); - ctx()->emit_expr(); + ctx()->s_emit_top(); ctx()->emit_(OP_YIELD_VALUE, BC_NOARG, kw_line); consume_end_stmt(); break; case TK("yield from"): if(contexts.size() <= 1) return SyntaxError("'yield from' outside function"); check(EXPR_TUPLE()); - ctx()->emit_expr(); + ctx()->s_emit_top(); ctx()->emit_(OP_GET_ITER_NEW, BC_NOARG, kw_line); ctx()->enter_block(CodeBlockType::FOR_LOOP); @@ -950,7 +951,7 @@ Error* Compiler::compile_stmt() noexcept{ ctx()->emit_(OP_RETURN_VALUE, 1, kw_line); } else { check(EXPR_TUPLE()); - ctx()->emit_expr(); + ctx()->s_emit_top(); consume_end_stmt(); ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, kw_line); } @@ -968,12 +969,12 @@ Error* Compiler::compile_stmt() noexcept{ /*************************************************/ case TK("assert"): { check(EXPR()); // condition - ctx()->emit_expr(); + ctx()->s_emit_top(); int index = ctx()->emit_(OP_POP_JUMP_IF_TRUE, BC_NOARG, kw_line); int has_msg = 0; if(match(TK(","))) { check(EXPR()); // message - ctx()->emit_expr(); + ctx()->s_emit_top(); has_msg = 1; } ctx()->emit_(OP_RAISE_ASSERT, has_msg, kw_line); @@ -990,19 +991,19 @@ Error* Compiler::compile_stmt() noexcept{ break; case TK("raise"): { check(EXPR()); - ctx()->emit_expr(); + ctx()->s_emit_top(); ctx()->emit_(OP_RAISE, BC_NOARG, kw_line); consume_end_stmt(); } break; case TK("del"): { check(EXPR_TUPLE()); - Expr* e = s_expr().popx_back(); + Expr* e = ctx()->s_popx(); if(!e->emit_del(ctx())) return SyntaxError(); consume_end_stmt(); } break; case TK("with"): { check(EXPR()); // [ ] - ctx()->emit_expr(); + ctx()->s_emit_top(); ctx()->enter_block(CodeBlockType::CONTEXT_MANAGER); Expr* as_name = nullptr; if(match(TK("as"))) { @@ -1045,13 +1046,13 @@ Error* Compiler::compile_stmt() noexcept{ bool is_typed_name = false; // e.g. x: int // eat variable's type hint if it is a single name - if(s_expr().back()->is_name()) { + if(ctx()->s_top()->is_name()) { if(match(TK(":"))) { check(consume_type_hints()); is_typed_name = true; if(ctx()->is_compiling_class) { - NameExpr* ne = static_cast(s_expr().back()); + NameExpr* ne = static_cast(ctx()->s_top()); ctx()->emit_(OP_ADD_CLASS_ANNOTATION, ne->name.index, BC_KEEPLINE); } } @@ -1059,18 +1060,18 @@ Error* Compiler::compile_stmt() noexcept{ bool is_assign = false; check(try_compile_assignment(&is_assign)); if(!is_assign) { - if(!s_expr().empty() && s_expr().back()->is_starred()) { + if(ctx()->s_size() > 0 && ctx()->s_top()->is_starred()) { return SyntaxError(); } if(!is_typed_name) { - ctx()->emit_expr(); + ctx()->s_emit_top(); if((mode() == CELL_MODE || mode() == REPL_MODE) && name_scope() == NAME_GLOBAL) { ctx()->emit_(OP_PRINT_EXPR, BC_NOARG, BC_KEEPLINE); } else { ctx()->emit_(OP_POP_TOP, BC_NOARG, BC_KEEPLINE); } } else { - ctx()->emit_expr(false); + ctx()->s_pop(); } } consume_end_stmt(); @@ -1083,7 +1084,7 @@ Error* Compiler::compile_stmt() noexcept{ Error* Compiler::consume_type_hints() noexcept{ Error* err; check(EXPR()); - Expr* e = s_expr().popx_back(); + Expr* e = ctx()->s_popx(); delete_expr(e); return NULL; } @@ -1096,7 +1097,7 @@ Error* Compiler::compile_class(int decorators) noexcept{ if(match(TK("("))) { if(is_expression()) { check(EXPR()); - base = s_expr().popx_back(); + base = ctx()->s_popx(); } consume(TK(")")); } @@ -1114,10 +1115,9 @@ Error* Compiler::compile_class(int decorators) noexcept{ check(compile_block_body()); ctx()->is_compiling_class = false; - assert(s_expr().size() == decorators); if(decorators > 0) { ctx()->emit_(OP_BEGIN_CLASS_DECORATION, BC_NOARG, BC_KEEPLINE); - ctx()->emit_decorators(decorators); + ctx()->s_emit_decorators(decorators); ctx()->emit_(OP_END_CLASS_DECORATION, BC_NOARG, BC_KEEPLINE); } @@ -1209,8 +1209,7 @@ Error* Compiler::compile_function(int decorators) noexcept{ } ctx()->emit_(OP_LOAD_FUNCTION, ctx()->add_func_decl(decl), prev().line); - assert(s_expr().size() == decorators); - ctx()->emit_decorators(decorators); + ctx()->s_emit_decorators(decorators); if(!ctx()->is_compiling_class) { auto e = make_expr(decl_name, name_scope()); @@ -1273,11 +1272,15 @@ Compiler::Compiler(VM* vm, std::string_view source, const Str& filename, Compile } Error* Compiler::compile(CodeObject_* out) noexcept{ - assert(i == 0); // make sure it is the first time to compile + assert(__i == 0); // make sure it is the first time to compile Error* err; check(lexer.run()); + // for(int i=0; iemit_expr(); + ctx()->s_emit_top(); consume(TK("@eof")); ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE); check(pop_context()); @@ -1293,7 +1296,7 @@ Error* Compiler::compile(CodeObject_* out) noexcept{ return NULL; } else if(mode() == JSON_MODE) { check(EXPR()); - Expr* e = s_expr().popx_back(); + Expr* e = ctx()->s_popx(); if(!e->is_json_object()) return SyntaxError("expect a JSON object, literal or array"); consume(TK("@eof")); e->emit_(ctx()); @@ -1314,7 +1317,7 @@ Error* Compiler::compile(CodeObject_* out) noexcept{ Compiler::~Compiler(){ for(CodeEmitContext& ctx: contexts){ - for(Expr* e: ctx.s_expr) delete_expr(e); + ctx.s_clean(); } } diff --git a/src/compiler/expr.cpp b/src/compiler/expr.cpp index 7f5ce795..92a80753 100644 --- a/src/compiler/expr.cpp +++ b/src/compiler/expr.cpp @@ -40,18 +40,11 @@ void CodeEmitContext::exit_block() noexcept{ } } -// clear the expression stack and generate bytecode -void CodeEmitContext::emit_expr(bool emit) noexcept{ - // assert(s_expr.size() == 1); - Expr* e = s_expr.popx_back(); - if(emit) e->emit_(this); - delete_expr(e); -} - -void CodeEmitContext::emit_decorators(int count) noexcept{ +void CodeEmitContext::s_emit_decorators(int count) noexcept{ + assert(s_size() >= count); // [obj] for(int i=0; iemit_(this); // [obj, f] emit_(OP_ROT_TWO, BC_NOARG, deco->line); // [f, obj] emit_(OP_LOAD_NULL, BC_NOARG, BC_KEEPLINE); // [f, obj, NULL] diff --git a/src/compiler/lexer.cpp b/src/compiler/lexer.cpp index e9a6c259..eb7e051d 100644 --- a/src/compiler/lexer.cpp +++ b/src/compiler/lexer.cpp @@ -501,7 +501,7 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{ Error* Lexer::_error(bool lexer_err, const char* type, const char* msg, va_list args, i64 userdata) noexcept{ PK_THREAD_LOCAL Error err; err.type = type; - err.src = src.get(); + err.src = src; if(lexer_err){ err.lineno = current_line; err.cursor = curr_char; @@ -529,16 +529,19 @@ Error* Lexer::SyntaxError(const char* fmt, ...) noexcept{ Lexer::Lexer(VM* vm, std::shared_ptr src) noexcept : vm(vm), src(src){ this->token_start = src->source.c_str(); this->curr_char = src->source.c_str(); - this->nexts.push_back(Token{TK("@sof"), token_start, 0, current_line, brackets_level, {}}); - this->indents.push_back(0); } Error* Lexer::run() noexcept{ + assert(!this->used); + this->used = true; if(src->is_precompiled) { from_precompiled(); return NULL; } - assert(curr_char == src->source.c_str()); + // push initial tokens + this->nexts.push_back(Token{TK("@sof"), token_start, 0, current_line, brackets_level, {}}); + this->indents.push_back(0); + bool eof = false; while(!eof) { Error* err = lex_one_token(&eof); diff --git a/src/interpreter/vm.cpp b/src/interpreter/vm.cpp index 0a393361..74e9f73b 100644 --- a/src/interpreter/vm.cpp +++ b/src/interpreter/vm.cpp @@ -182,7 +182,9 @@ PyVar VM::exec(std::string_view source, Str filename, CompileMode mode, PyObject #endif CodeObject_ code = compile(source, filename, mode); return _exec(code, _module); - } catch(TopLevelException e) { stderr_write(e.summary() + "\n"); } catch(const std::exception& e) { + } catch(TopLevelException e) { + stderr_write(e.summary() + "\n"); + } catch(const std::exception& e) { Str msg = "An std::exception occurred! It could be a bug.\n"; msg = msg + e.what() + "\n"; stderr_write(msg); @@ -1374,12 +1376,13 @@ PyObject* VM::bind(PyObject* obj, const char* sig, NativeFuncC fn, any userdata, } PyObject* VM::bind(PyObject* obj, const char* sig, const char* docstring, NativeFuncC fn, any userdata, BindType bt) { - CodeObject_ co; - try { - // fn(a, b, *c, d=1) -> None - co = compile(_S("def ", sig, " : pass"), "", EXEC_MODE); - } catch(TopLevelException) { throw std::runtime_error("invalid signature: " + std::string(sig)); } - if(co->func_decls.size() != 1) { throw std::runtime_error("expected 1 function declaration"); } + char buffer[256]; + int length = snprintf(buffer, sizeof(buffer), "def %s : pass", sig); + std::string_view source(buffer, length); + // fn(a, b, *c, d=1) -> None + CodeObject_ co = compile(source, "", EXEC_MODE); + assert(co->func_decls.size() == 1); + FuncDecl_ decl = co->func_decls[0]; decl->docstring = docstring; PyObject* f_obj = heap.gcnew(tp_native_func, fn, decl, std::move(userdata)); @@ -1455,7 +1458,7 @@ void VM::__raise_exc(bool re_raise) { int current_line = frame->co->lines[actual_ip].lineno; // current line auto current_f_name = frame->co->name.sv(); // current function name if(frame->_callable == nullptr) current_f_name = ""; // not in a function - e.st_push(frame->co->src.get(), current_line, nullptr, current_f_name); + e.st_push(frame->co->src, current_line, nullptr, current_f_name); if(next_ip >= 0) { throw InternalException(InternalExceptionType::Handled, next_ip); diff --git a/tests/44_decorator.py b/tests/44_decorator.py index 925d4125..b90f8321 100644 --- a/tests/44_decorator.py +++ b/tests/44_decorator.py @@ -38,4 +38,43 @@ def fib(n): return n return fib(n-1) + fib(n-2) -assert fib(32) == 2178309 \ No newline at end of file +assert fib(32) == 2178309 + +def wrapped(cls): + return int + +@wrapped +@wrapped +@wrapped +@wrapped +class A: + def __init__(self) -> None: + pass + +assert A('123') == 123 + +# validate the decorator order +res = [] + +def w(x): + res.append('w') + return x + +def w1(x): + res.append('w1') + return x + +def w2(x): + res.append('w2') + return x + +@w +@w1 +@w2 +def f(): + pass + +f() +assert res == ['w2', 'w1', 'w'] + + diff --git a/tests/82_dataclasses.py b/tests/82_dataclasses.py index 81320231..7908efe0 100644 --- a/tests/82_dataclasses.py +++ b/tests/82_dataclasses.py @@ -15,15 +15,3 @@ assert asdict(A(1, '555')) == {'x': 1, 'y': '555'} assert A(1, 'N') == A(1, 'N') assert A(1, 'N') != A(1, 'M') -def wrapped(cls): - return int - -@wrapped -@wrapped -@wrapped -@wrapped -class A: - def __init__(self) -> None: - pass - -assert A('123') == 123 diff --git a/tests/94_compile.py b/tests/94_compile.py index fd90e88a..a0dd1847 100644 --- a/tests/94_compile.py +++ b/tests/94_compile.py @@ -1,2 +1,5 @@ +assert eval("1+2") == 3 + code = compile("1+2", "", "eval") +# print(code) assert eval(code) == 3