From b87cd6ed521dafebaf361656c1cad1f74037f32e Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Tue, 6 Jun 2023 13:52:58 +0800 Subject: [PATCH 1/4] ... --- docs/features/differences.md | 21 +++++----- src/ceval.h | 78 ++++++++++++++++++++++++++-------- src/common.h | 5 ++- src/compiler.h | 32 ++++++++++---- src/dict.h | 8 ++++ src/expr.h | 72 +++++++++++++++++++------------- src/memory.h | 1 + src/obj.h | 19 ++++++++- src/opcodes.h | 11 +++-- src/str.h | 22 +++++++++- src/vm.h | 81 ++++++++++++++++++++++++++++-------- tests/21_functions.py | 27 +++++++++++- 12 files changed, 285 insertions(+), 92 deletions(-) diff --git a/docs/features/differences.md b/docs/features/differences.md index c7baa8b7..8bfc40e8 100644 --- a/docs/features/differences.md +++ b/docs/features/differences.md @@ -20,17 +20,16 @@ The easiest way to test a feature is to [try it on your browser](https://pocketp ## Unimplemented features -1. `**kwargs` in function definition. -2. `__getattr__` and `__setattr__`. -3. Descriptor protocol `__get__` and `__set__`. However, `@property` is implemented. -4. `__slots__` in class definition. -5. One element tuple. `(1,)` is not supported. -6. Unpacking in `list` and `dict` literals, e.g. `[1, 2, *a]`. -7. Access the exception object in try..except. -8. `else` clause in try..except. -9. Inplace methods like `__iadd__` and `__imul__`. -10. `__del__` in class definition. -11. Multiple inheritance. +1. `__getattr__` and `__setattr__`. +2. Descriptor protocol `__get__` and `__set__`. However, `@property` is implemented. +3. `__slots__` in class definition. +4. One element tuple. `(1,)` is not supported. +5. Unpacking in `list` and `dict` literals, e.g. `[1, 2, *a]`. +6. Access the exception object in try..except. +7. `else` clause in try..except. +8. Inplace methods like `__iadd__` and `__imul__`. +9. `__del__` in class definition. +10. Multiple inheritance. ## Different behaviors diff --git a/src/ceval.h b/src/ceval.h index 72b8d84d..fa5564c9 100644 --- a/src/ceval.h +++ b/src/ceval.h @@ -91,7 +91,7 @@ __NEXT_STEP:; TARGET(LOAD_ELLIPSIS) PUSH(Ellipsis); DISPATCH(); TARGET(LOAD_FUNCTION) { FuncDecl_ decl = co->func_decls[byte.arg]; - bool is_simple = decl->starred_arg==-1 && decl->kwargs.size()==0 && !decl->code->is_generator; + bool is_simple = decl->starred_kwarg==-1 && decl->starred_arg==-1 && decl->kwargs.size()==0 && !decl->code->is_generator; int argc = decl->args.size(); PyObject* obj; if(decl->nested){ @@ -236,6 +236,11 @@ __NEXT_STEP:; } DISPATCH(); /*****************************************/ + TARGET(BUILD_TUPLE) + _0 = VAR(STACK_VIEW(byte.arg).to_tuple()); + STACK_SHRINK(byte.arg); + PUSH(_0); + DISPATCH(); TARGET(BUILD_LIST) _0 = VAR(STACK_VIEW(byte.arg).to_list()); STACK_SHRINK(byte.arg); @@ -263,11 +268,6 @@ __NEXT_STEP:; _0 = POPX(); // start PUSH(VAR(Slice(_0, _1, _2))); DISPATCH(); - TARGET(BUILD_TUPLE) - _0 = VAR(STACK_VIEW(byte.arg).to_tuple()); - STACK_SHRINK(byte.arg); - PUSH(_0); - DISPATCH(); TARGET(BUILD_STRING) { std::stringstream ss; ArgsView view = STACK_VIEW(byte.arg); @@ -276,6 +276,40 @@ __NEXT_STEP:; PUSH(VAR(ss.str())); } DISPATCH(); /*****************************************/ + TARGET(BUILD_TUPLE_UNPACK) { + auto _lock = heap.gc_scope_lock(); + List list; + _unpack_as_list(STACK_VIEW(byte.arg), list); + STACK_SHRINK(byte.arg); + _0 = VAR(Tuple(std::move(list))); + PUSH(_0); + } DISPATCH(); + TARGET(BUILD_LIST_UNPACK) { + auto _lock = heap.gc_scope_lock(); + List list; + _unpack_as_list(STACK_VIEW(byte.arg), list); + STACK_SHRINK(byte.arg); + _0 = VAR(std::move(list)); + PUSH(_0); + } DISPATCH(); + TARGET(BUILD_DICT_UNPACK) { + auto _lock = heap.gc_scope_lock(); + Dict dict(this); + _unpack_as_dict(STACK_VIEW(byte.arg), dict); + STACK_SHRINK(byte.arg); + _0 = VAR(std::move(dict)); + PUSH(_0); + } DISPATCH(); + TARGET(BUILD_SET_UNPACK) { + auto _lock = heap.gc_scope_lock(); + List list; + _unpack_as_list(STACK_VIEW(byte.arg), list); + STACK_SHRINK(byte.arg); + _0 = VAR(std::move(list)); + _0 = call(builtins->attr(set), _0); + PUSH(_0); + } DISPATCH(); + /*****************************************/ #define PREDICT_INT_OP(op) \ if(is_both_int(TOP(), SECOND())){ \ _1 = POPX(); \ @@ -426,9 +460,6 @@ __NEXT_STEP:; frame->jump_abs_break(index); } DISPATCH(); /*****************************************/ - TARGET(BEGIN_CALL) - PUSH(PY_BEGIN_CALL); - DISPATCH(); TARGET(CALL) _0 = vectorcall( byte.arg & 0xFFFF, // ARGC @@ -438,6 +469,23 @@ __NEXT_STEP:; if(_0 == PY_OP_CALL) DISPATCH_OP_CALL(); PUSH(_0); DISPATCH(); + TARGET(CALL_TP) + // [callable, , args: tuple, kwargs: dict] + _2 = POPX(); + _1 = POPX(); + for(PyObject* obj: _CAST(Tuple&, _1)) PUSH(obj); + _CAST(Dict&, _2).apply([this](PyObject* k, PyObject* v){ + PUSH(VAR(StrName(CAST(Str&, k)).index)); + PUSH(v); + }); + _0 = vectorcall( + _CAST(Tuple&, _1).size(), // ARGC + _CAST(Dict&, _2).size(), // KWARGC + true + ); + if(_0 == PY_OP_CALL) DISPATCH_OP_CALL(); + PUSH(_0); + DISPATCH(); TARGET(RETURN_VALUE) _0 = POPX(); _pop_frame(); @@ -471,6 +519,9 @@ __NEXT_STEP:; TARGET(UNARY_NOT) TOP() = VAR(!py_bool(TOP())); DISPATCH(); + TARGET(UNARY_STAR) + TOP() = VAR(StarWrapper(byte.arg, TOP())); + DISPATCH(); /*****************************************/ TARGET(GET_ITER) TOP() = py_iter(TOP()); @@ -528,15 +579,6 @@ __NEXT_STEP:; } PUSH(VAR(extras)); } DISPATCH(); - TARGET(UNPACK_UNLIMITED) { - auto _lock = heap.gc_scope_lock(); // lock the gc via RAII!! - _0 = py_iter(POPX()); - _1 = py_next(_0); - while(_1 != StopIteration){ - PUSH(_1); - _1 = py_next(_0); - } - } DISPATCH(); /*****************************************/ TARGET(BEGIN_CLASS) _name = StrName(byte.arg); diff --git a/src/common.h b/src/common.h index 20929837..59f0616b 100644 --- a/src/common.h +++ b/src/common.h @@ -29,7 +29,7 @@ #include #include -#define PK_VERSION "1.0.1" +#define PK_VERSION "1.0.2" // debug macros #define DEBUG_NO_BUILTIN_MODULES 0 @@ -153,6 +153,8 @@ struct Type { #define FATAL_ERROR() throw std::runtime_error( __FILE__ + std::string(":") + std::to_string(__LINE__) + " FATAL_ERROR()!"); #endif +#define PK_ASSERT(x) if(!(x)) FATAL_ERROR(); + inline const float kInstAttrLoadFactor = 0.67f; inline const float kTypeAttrLoadFactor = 0.5f; @@ -173,7 +175,6 @@ inline bool is_both_int(PyObject* a, PyObject* b) noexcept { // special singals, is_tagged() for them is true inline PyObject* const PY_NULL = (PyObject*)0b000011; // tagged null -inline PyObject* const PY_BEGIN_CALL = (PyObject*)0b010011; inline PyObject* const PY_OP_CALL = (PyObject*)0b100011; inline PyObject* const PY_OP_YIELD = (PyObject*)0b110011; diff --git a/src/compiler.h b/src/compiler.h index 214b1b39..890e9d70 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -90,7 +90,7 @@ class Compiler { rules[TK("*")] = { METHOD(exprUnaryOp), METHOD(exprBinaryOp), PREC_FACTOR }; rules[TK("/")] = { nullptr, METHOD(exprBinaryOp), PREC_FACTOR }; rules[TK("//")] = { nullptr, METHOD(exprBinaryOp), PREC_FACTOR }; - rules[TK("**")] = { nullptr, METHOD(exprBinaryOp), PREC_EXPONENT }; + rules[TK("**")] = { METHOD(exprUnaryOp), METHOD(exprBinaryOp), PREC_EXPONENT }; rules[TK(">")] = { nullptr, METHOD(exprBinaryOp), PREC_COMPARISION }; rules[TK("<")] = { nullptr, METHOD(exprBinaryOp), PREC_COMPARISION }; rules[TK("==")] = { nullptr, METHOD(exprBinaryOp), PREC_COMPARISION }; @@ -280,7 +280,10 @@ class Compiler { ctx()->s_expr.push(make_expr(ctx()->s_expr.popx())); break; case TK("*"): - ctx()->s_expr.push(make_expr(ctx()->s_expr.popx())); + ctx()->s_expr.push(make_expr(1, ctx()->s_expr.popx())); + break; + case TK("**"): + ctx()->s_expr.push(make_expr(2, ctx()->s_expr.popx())); break; default: FATAL_ERROR(); } @@ -387,9 +390,15 @@ class Compiler { EXPR(); e->kwargs.push_back({key, ctx()->s_expr.popx()}); } else{ - if(!e->kwargs.empty()) SyntaxError("positional argument follows keyword argument"); EXPR(); - e->args.push_back(ctx()->s_expr.popx()); + if(ctx()->s_expr.top()->star_level() == 2){ + // **kwargs + e->kwargs.push_back({"**", ctx()->s_expr.popx()}); + }else{ + // positional argument + if(!e->kwargs.empty()) SyntaxError("positional argument follows keyword argument"); + e->args.push_back(ctx()->s_expr.popx()); + } } match_newlines_repl(); } while (match(TK(","))); @@ -876,6 +885,7 @@ __SUBSCR_END: void _compile_f_args(FuncDecl_ decl, bool enable_type_hints){ int state = 0; // 0 for args, 1 for *args, 2 for k=v, 3 for **kwargs do { + if(state > 3) SyntaxError(); if(state == 3) SyntaxError("**kwargs should be the last argument"); match_newlines(); if(match(TK("*"))){ @@ -894,14 +904,17 @@ __SUBSCR_END: SyntaxError("duplicate argument name"); } } - if(decl->starred_arg!=-1 && decl->code->varnames[decl->starred_arg] == name){ - SyntaxError("duplicate argument name"); - } for(auto& kv: decl->kwargs){ if(decl->code->varnames[kv.key] == name){ SyntaxError("duplicate argument name"); } } + if(decl->starred_arg!=-1 && decl->code->varnames[decl->starred_arg] == name){ + SyntaxError("duplicate argument name"); + } + if(decl->starred_kwarg!=-1 && decl->code->varnames[decl->starred_kwarg] == name){ + SyntaxError("duplicate argument name"); + } // eat type hints if(enable_type_hints && match(TK(":"))) consume_type_hints(); @@ -924,7 +937,10 @@ __SUBSCR_END: } decl->kwargs.push_back(FuncDecl::KwArg{index, value}); } break; - case 3: SyntaxError("**kwargs is not supported yet"); break; + case 3: + decl->starred_kwarg = index; + state+=1; + break; } } while (match(TK(","))); } diff --git a/src/dict.h b/src/dict.h index 5fe69625..a83fbdf3 100644 --- a/src/dict.h +++ b/src/dict.h @@ -124,6 +124,14 @@ struct Dict{ return v; } + template + void apply(__Func f) const { + for(int i=0; i<_capacity; i++){ + if(_items[i].first == nullptr) continue; + f(_items[i].first, _items[i].second); + } + } + void clear(){ memset(_items, 0, _capacity * sizeof(Item)); _size = 0; diff --git a/src/expr.h b/src/expr.h index ceda17fc..80f0f34d 100644 --- a/src/expr.h +++ b/src/expr.h @@ -19,11 +19,12 @@ struct Expr{ virtual void emit(CodeEmitContext* ctx) = 0; virtual std::string str() const = 0; - virtual bool is_starred() const { return false; } virtual bool is_literal() const { return false; } virtual bool is_json_object() const { return false; } virtual bool is_attrib() const { return false; } virtual bool is_compare() const { return false; } + virtual int star_level() const { return 0; } + bool is_starred() const { return star_level() > 0; } // for OP_DELETE_XXX [[nodiscard]] virtual bool emit_del(CodeEmitContext* ctx) { return false; } @@ -183,24 +184,25 @@ struct NameExpr: Expr{ }; struct StarredExpr: Expr{ + int level; Expr_ child; - StarredExpr(Expr_&& child): child(std::move(child)) {} - std::string str() const override { return "Starred()"; } + StarredExpr(int level, Expr_&& child): level(level), child(std::move(child)) {} + std::string str() const override { return fmt("Starred(level=", level, ")"); } - bool is_starred() const override { return true; } + int star_level() const override { return level; } void emit(CodeEmitContext* ctx) override { child->emit(ctx); - ctx->emit(OP_UNPACK_UNLIMITED, BC_NOARG, line); + ctx->emit(OP_UNARY_STAR, level, line); } bool emit_store(CodeEmitContext* ctx) override { + if(level != 1) return false; // simply proxy to child return child->emit_store(ctx); } }; - struct NotExpr: Expr{ Expr_ child; NotExpr(Expr_&& child): child(std::move(child)) {} @@ -265,16 +267,13 @@ struct LiteralExpr: Expr{ if(std::holds_alternative(value)){ return std::to_string(std::get(value)); } - if(std::holds_alternative(value)){ return std::to_string(std::get(value)); } - if(std::holds_alternative(value)){ Str s = std::get(value).escape(); return s.str(); } - FATAL_ERROR(); } @@ -644,37 +643,54 @@ struct AttribExpr: Expr{ struct CallExpr: Expr{ Expr_ callable; std::vector args; + // **a will be interpreted as a special keyword argument: {"**": a} std::vector> kwargs; std::string str() const override { return "Call()"; } - bool need_unpack() const { - for(auto& item: args) if(item->is_starred()) return true; - return false; - } - void emit(CodeEmitContext* ctx) override { - VM* vm = ctx->vm; - if(need_unpack()) ctx->emit(OP_BEGIN_CALL, BC_NOARG, line); + bool vargs = false; + bool vkwargs = false; + for(auto& arg: args) if(arg->is_starred()) vargs = true; + for(auto& item: kwargs) if(item.second->is_starred()) vkwargs = true; + // if callable is a AttrExpr, we should try to use `fast_call` instead of use `boundmethod` proxy if(callable->is_attrib()){ auto p = static_cast(callable.get()); - p->emit_method(ctx); + p->emit_method(ctx); // OP_LOAD_METHOD }else{ callable->emit(ctx); ctx->emit(OP_LOAD_NULL, BC_NOARG, BC_KEEPLINE); } - // emit args - for(auto& item: args) item->emit(ctx); - // emit kwargs - for(auto& item: kwargs){ - int index = StrName::get(item.first.sv()).index; - ctx->emit(OP_LOAD_CONST, ctx->add_const(VAR(index)), line); - item.second->emit(ctx); + + if(vargs || vkwargs){ + for(auto& item: args) item->emit(ctx); + ctx->emit(OP_BUILD_TUPLE_UNPACK, (int)args.size(), line); + + for(auto& item: kwargs){ + item.second->emit(ctx); + if(item.second->is_starred()){ + if(item.second->star_level() != 2) FATAL_ERROR(); + }else{ + // k=v + int index = ctx->add_const(py_var(ctx->vm, item.first)); + ctx->emit(OP_LOAD_CONST, index, line); + ctx->emit(OP_BUILD_TUPLE, 2, line); + } + } + ctx->emit(OP_BUILD_DICT_UNPACK, (int)kwargs.size(), line); + ctx->emit(OP_CALL_TP, BC_NOARG, line); + }else{ + // vectorcall protocal + for(auto& item: args) item->emit(ctx); + for(auto& item: kwargs){ + int index = StrName(item.first.sv()).index; + ctx->emit(OP_LOAD_INTEGER, index, line); + item.second->emit(ctx); + } + int KWARGC = (int)kwargs.size(); + int ARGC = (int)args.size(); + ctx->emit(OP_CALL, (KWARGC<<16)|ARGC, line); } - int KWARGC = (int)kwargs.size(); - int ARGC = (int)args.size(); - if(need_unpack()) ARGC = 0xFFFF; - ctx->emit(OP_CALL, (KWARGC<<16)|ARGC, line); } }; diff --git a/src/memory.h b/src/memory.h index 9b087cd4..9a0387a7 100644 --- a/src/memory.h +++ b/src/memory.h @@ -238,6 +238,7 @@ struct MemoryPool{ } }; +// TODO: make them thread-safe inline MemoryPool<64> pool64; inline MemoryPool<128> pool128; diff --git a/src/obj.h b/src/obj.h index edf2426e..4a746cb8 100644 --- a/src/obj.h +++ b/src/obj.h @@ -60,8 +60,9 @@ struct FuncDecl { }; CodeObject_ code; // code object of this function pod_vector args; // indices in co->varnames - int starred_arg = -1; // index in co->varnames, -1 if no *arg pod_vector kwargs; // indices in co->varnames + int starred_arg = -1; // index in co->varnames, -1 if no *arg + int starred_kwarg = -1; // index in co->varnames, -1 if no **kwarg bool nested = false; // whether this function is nested void _gc_mark() const; }; @@ -101,6 +102,12 @@ struct Range { i64 step = 1; }; +struct StarWrapper{ + int level; // either 1 or 2 + PyObject* obj; + StarWrapper(int level, PyObject* obj) : level(level), obj(obj) {} +}; + struct Bytes{ std::vector _data; bool _ok; @@ -335,6 +342,16 @@ struct Py_ final: PyObject { } }; +template<> +struct Py_ final: PyObject { + StarWrapper _value; + void* value() override { return &_value; } + Py_(Type type, StarWrapper val): PyObject(type), _value(val) {} + void _obj_gc_mark() override { + OBJ_MARK(_value.obj); + } +}; + template<> struct Py_ final: PyObject { Property _value; diff --git a/src/opcodes.h b/src/opcodes.h index 4fdfba5d..8733c230 100644 --- a/src/opcodes.h +++ b/src/opcodes.h @@ -38,13 +38,18 @@ OPCODE(DELETE_GLOBAL) OPCODE(DELETE_ATTR) OPCODE(DELETE_SUBSCR) /**************************/ +OPCODE(BUILD_TUPLE) OPCODE(BUILD_LIST) OPCODE(BUILD_DICT) OPCODE(BUILD_SET) OPCODE(BUILD_SLICE) -OPCODE(BUILD_TUPLE) OPCODE(BUILD_STRING) /**************************/ +OPCODE(BUILD_TUPLE_UNPACK) +OPCODE(BUILD_LIST_UNPACK) +OPCODE(BUILD_DICT_UNPACK) +OPCODE(BUILD_SET_UNPACK) +/**************************/ OPCODE(BINARY_TRUEDIV) OPCODE(BINARY_POW) @@ -81,8 +86,8 @@ OPCODE(LOOP_CONTINUE) OPCODE(LOOP_BREAK) OPCODE(GOTO) /**************************/ -OPCODE(BEGIN_CALL) OPCODE(CALL) +OPCODE(CALL_TP) OPCODE(RETURN_VALUE) OPCODE(YIELD_VALUE) /**************************/ @@ -92,6 +97,7 @@ OPCODE(SET_ADD) /**************************/ OPCODE(UNARY_NEGATIVE) OPCODE(UNARY_NOT) +OPCODE(UNARY_STAR) /**************************/ OPCODE(GET_ITER) OPCODE(FOR_ITER) @@ -102,7 +108,6 @@ OPCODE(IMPORT_STAR) /**************************/ OPCODE(UNPACK_SEQUENCE) OPCODE(UNPACK_EX) -OPCODE(UNPACK_UNLIMITED) /**************************/ OPCODE(BEGIN_CLASS) OPCODE(END_CLASS) diff --git a/src/str.h b/src/str.h index 72839d56..98df4b1c 100644 --- a/src/str.h +++ b/src/str.h @@ -124,19 +124,37 @@ struct Str{ return memcmp(data, other.data, size) != 0; } + bool operator==(const std::string_view other) const { + if(size != (int)other.size()) return false; + return memcmp(data, other.data(), size) == 0; + } + + bool operator!=(const std::string_view other) const { + if(size != (int)other.size()) return true; + return memcmp(data, other.data(), size) != 0; + } + + bool operator==(const char* p) const { + return *this == std::string_view(p); + } + + bool operator!=(const char* p) const { + return *this != std::string_view(p); + } + bool operator<(const Str& other) const { int ret = strncmp(data, other.data, std::min(size, other.size)); if(ret != 0) return ret < 0; return size < other.size; } - bool operator<(const std::string_view& other) const { + bool operator<(const std::string_view other) const { int ret = strncmp(data, other.data(), std::min(size, (int)other.size())); if(ret != 0) return ret < 0; return size < (int)other.size(); } - friend bool operator<(const std::string_view& other, const Str& str){ + friend bool operator<(const std::string_view other, const Str& str){ return str > other; } diff --git a/src/vm.h b/src/vm.h index 604e7f30..dc11486b 100644 --- a/src/vm.h +++ b/src/vm.h @@ -134,7 +134,7 @@ public: Type tp_function, tp_native_func, tp_bound_method; Type tp_slice, tp_range, tp_module; Type tp_super, tp_exception, tp_bytes, tp_mappingproxy; - Type tp_dict, tp_property; + Type tp_dict, tp_property, tp_star_wrapper; const bool enable_os; @@ -637,6 +637,8 @@ public: #if DEBUG_CEVAL_STEP void _log_s_data(const char* title = nullptr); #endif + void _unpack_as_list(ArgsView args, List& list); + void _unpack_as_dict(ArgsView args, Dict& dict); PyObject* vectorcall(int ARGC, int KWARGC=0, bool op_call=false); CodeObject_ compile(Str source, Str filename, CompileMode mode, bool unknown_global_scope=false); PyObject* py_negate(PyObject* obj); @@ -686,6 +688,7 @@ DEF_NATIVE_2(Bytes, tp_bytes) DEF_NATIVE_2(MappingProxy, tp_mappingproxy) DEF_NATIVE_2(Dict, tp_dict) DEF_NATIVE_2(Property, tp_property) +DEF_NATIVE_2(StarWrapper, tp_star_wrapper) #undef DEF_NATIVE_2 @@ -1063,7 +1066,6 @@ inline void VM::_log_s_data(const char* title) { if(sp_bases[p] > 0) ss << " "; PyObject* obj = *p; if(obj == nullptr) ss << "(nil)"; - else if(obj == PY_BEGIN_CALL) ss << "BEGIN_CALL"; else if(obj == PY_NULL) ss << "NULL"; else if(is_int(obj)) ss << CAST(i64, obj); else if(is_float(obj)) ss << CAST(f64, obj); @@ -1121,6 +1123,7 @@ inline void VM::init_builtin_types(){ tp_mappingproxy = _new_type_object("mappingproxy"); tp_dict = _new_type_object("dict"); tp_property = _new_type_object("property"); + tp_star_wrapper = _new_type_object("_star_wrapper"); this->None = heap._new(_new_type_object("NoneType"), {}); this->Ellipsis = heap._new(_new_type_object("ellipsis"), {}); @@ -1154,21 +1157,47 @@ inline void VM::init_builtin_types(){ this->_main = new_module("__main__"); } -inline PyObject* VM::vectorcall(int ARGC, int KWARGC, bool op_call){ - bool is_varargs = ARGC == 0xFFFF; - PyObject** p0; - PyObject** p1 = s_data._sp - KWARGC*2; - if(is_varargs){ - p0 = p1 - 1; - while(*p0 != PY_BEGIN_CALL) p0--; - // [BEGIN_CALL, callable, , args..., kwargs...] - // ^p0 ^p1 ^_sp - ARGC = p1 - (p0 + 3); - }else{ - p0 = p1 - ARGC - 2 - (int)is_varargs; - // [callable, , args..., kwargs...] - // ^p0 ^p1 ^_sp +// `heap.gc_scope_lock();` needed before calling this function +inline void VM::_unpack_as_list(ArgsView args, List& list){ + for(PyObject* obj: args){ + if(is_non_tagged_type(obj, tp_star_wrapper)){ + const StarWrapper& w = _CAST(StarWrapper&, obj); + // maybe this check should be done in the compile time + if(w.level != 1) TypeError("expected level 1 star wrapper"); + PyObject* _0 = py_iter(w.obj); + PyObject* _1 = py_next(_0); + while(_1 != StopIteration){ + list.push_back(_1); + _1 = py_next(_0); + } + }else{ + list.push_back(obj); + } } +} + +// `heap.gc_scope_lock();` needed before calling this function +inline void VM::_unpack_as_dict(ArgsView args, Dict& dict){ + for(PyObject* obj: args){ + if(is_non_tagged_type(obj, tp_star_wrapper)){ + const StarWrapper& w = _CAST(StarWrapper&, obj); + // maybe this check should be done in the compile time + if(w.level != 2) TypeError("expected level 2 star wrapper"); + const Dict& other = CAST(Dict&, w.obj); + dict.update(other); + }else{ + const Tuple& t = CAST(Tuple&, obj); + if(t.size() != 2) TypeError("expected tuple of length 2"); + dict.set(t[0], t[1]); + } + } +} + +inline PyObject* VM::vectorcall(int ARGC, int KWARGC, bool op_call){ + PyObject** p1 = s_data._sp - KWARGC*2; + PyObject** p0 = p1 - ARGC - 2; + // [callable, , args..., kwargs...] + // ^p0 ^p1 ^_sp PyObject* callable = p1[-(ARGC + 2)]; bool method_call = p1[-(ARGC + 1)] != PY_NULL; @@ -1249,11 +1278,27 @@ inline PyObject* VM::vectorcall(int ARGC, int KWARGC, bool op_call){ if(i < args.size()) TypeError(fmt("too many arguments", " (", fn.decl->code->name, ')')); } + PyObject* vkwargs; + if(fn.decl->starred_kwarg != -1){ + vkwargs = VAR(Dict(this)); + buffer[fn.decl->starred_kwarg] = vkwargs; + }else{ + vkwargs = nullptr; + } + for(int i=0; ivarnames_inv.try_get(key); - if(index<0) TypeError(fmt(key.escape(), " is an invalid keyword argument for ", co->name, "()")); - buffer[index] = kwargs[i+1]; + if(index < 0){ + if(vkwargs == nullptr){ + TypeError(fmt(key.escape(), " is an invalid keyword argument for ", co->name, "()")); + }else{ + Dict& dict = _CAST(Dict&, vkwargs); + dict.set(VAR(key.sv()), kwargs[i+1]); + } + }else{ + buffer[index] = kwargs[i+1]; + } } if(co->is_generator){ diff --git a/tests/21_functions.py b/tests/21_functions.py index c2d90aae..1d01be0e 100644 --- a/tests/21_functions.py +++ b/tests/21_functions.py @@ -47,6 +47,9 @@ assert f(10, 1, 2, 3) == 18 def f(a, b, *c, d=2, e=5): return a + b + d + e + sum(c) +def g(*args, **kwargs): + return f(*args, **kwargs) + assert f(1, 2, 3, 4) == 17 assert f(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) == 62 assert f(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, d=1, e=2) == 58 @@ -56,6 +59,13 @@ assert f(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, e=1) == 58 assert f(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20) == 217 assert f(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, d=1, e=2) == 213 +assert g(1, 2, 3, 4) == 17 +assert g(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) == 62 +assert g(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, d=1, e=2) == 58 +assert g(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, e=1, d=2) == 58 +assert g(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, d=1) == 61 +assert g(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, e=1) == 58 + a = 1 b = 2 @@ -66,4 +76,19 @@ def f(): f() assert a == 3 -assert b == 4 \ No newline at end of file +assert b == 4 + +def g(a, b, *args, c=1, d=2, **kwargs): + S = a + b + c + d + sum(args) + return S, kwargs + +S, kwargs = g(1, 2, 3, 4, 5, c=4, e=5, f=6) +# a = 1 +# b = 2 +# c = 4 +# d = 2 +# sum(args) = 3 + 4 + 5 = 12 +# S = 1 + 2 + 4 + 2 + 12 = 21 + +assert S == 21 +assert kwargs == {'e': 5, 'f': 6} \ No newline at end of file From 636dbdd64aec876820cb2971ed6b0df4734a7dec Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Fri, 9 Jun 2023 22:16:53 +0800 Subject: [PATCH 2/4] ... --- src/compiler.h | 4 ---- src/expr.h | 33 ++++++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/compiler.h b/src/compiler.h index 890e9d70..e2568c85 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -322,7 +322,6 @@ class Compiler { if (curr().type == TK("]")) break; EXPR(); items.push_back(ctx()->s_expr.popx()); - if(items.back()->is_starred()) SyntaxError(); match_newlines_repl(); if(items.size()==1 && match(TK("for"))){ _consume_comp(std::move(items[0])); @@ -351,12 +350,9 @@ class Compiler { auto dict_item = make_expr(); dict_item->key = ctx()->s_expr.popx(); dict_item->value = ctx()->s_expr.popx(); - if(dict_item->key->is_starred()) SyntaxError(); - if(dict_item->value->is_starred()) SyntaxError(); items.push_back(std::move(dict_item)); }else{ items.push_back(ctx()->s_expr.popx()); - if(items.back()->is_starred()) SyntaxError(); } match_newlines_repl(); if(items.size()==1 && match(TK("for"))){ diff --git a/src/expr.h b/src/expr.h index 80f0f34d..f2f70669 100644 --- a/src/expr.h +++ b/src/expr.h @@ -370,10 +370,16 @@ struct DictItemExpr: Expr{ Expr_ value; std::string str() const override { return "DictItem()"; } + int star_level() const override { return value->star_level(); } + void emit(CodeEmitContext* ctx) override { - value->emit(ctx); - key->emit(ctx); // reverse order - ctx->emit(OP_BUILD_TUPLE, 2, line); + if(is_starred()){ + value->emit(ctx); + }else{ + value->emit(ctx); + key->emit(ctx); // reverse order + ctx->emit(OP_BUILD_TUPLE, 2, line); + } } }; @@ -391,7 +397,11 @@ struct SequenceExpr: Expr{ struct ListExpr: SequenceExpr{ using SequenceExpr::SequenceExpr; std::string str() const override { return "List()"; } - Opcode opcode() const override { return OP_BUILD_LIST; } + + Opcode opcode() const override { + for(auto& e: items) if(e->is_starred()) return OP_BUILD_LIST_UNPACK; + return OP_BUILD_LIST; + } bool is_json_object() const override { return true; } }; @@ -399,7 +409,10 @@ struct ListExpr: SequenceExpr{ struct DictExpr: SequenceExpr{ using SequenceExpr::SequenceExpr; std::string str() const override { return "Dict()"; } - Opcode opcode() const override { return OP_BUILD_DICT; } + Opcode opcode() const override { + for(auto& e: items) if(e->is_starred()) return OP_BUILD_DICT_UNPACK; + return OP_BUILD_DICT; + } bool is_json_object() const override { return true; } }; @@ -407,13 +420,19 @@ struct DictExpr: SequenceExpr{ struct SetExpr: SequenceExpr{ using SequenceExpr::SequenceExpr; std::string str() const override { return "Set()"; } - Opcode opcode() const override { return OP_BUILD_SET; } + Opcode opcode() const override { + for(auto& e: items) if(e->is_starred()) return OP_BUILD_SET_UNPACK; + return OP_BUILD_SET; + } }; struct TupleExpr: SequenceExpr{ using SequenceExpr::SequenceExpr; std::string str() const override { return "Tuple()"; } - Opcode opcode() const override { return OP_BUILD_TUPLE; } + Opcode opcode() const override { + for(auto& e: items) if(e->is_starred()) return OP_BUILD_TUPLE_UNPACK; + return OP_BUILD_TUPLE; + } bool emit_store(CodeEmitContext* ctx) override { // TOS is an iterable From 754808412b068e7cca3c716727487bc6869c5bf5 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Fri, 9 Jun 2023 22:28:36 +0800 Subject: [PATCH 3/4] add unpack builder --- src/compiler.h | 18 +++++++++++++----- src/expr.h | 3 ++- tests/05_list.py | 11 ++++++++++- tests/06_tuple.py | 11 ++++++++++- tests/07_dict.py | 16 +++++++++++++++- tests/08_set.py | 11 ++++++++++- 6 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/compiler.h b/src/compiler.h index e2568c85..b55d6c9f 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -343,13 +343,21 @@ class Compiler { match_newlines_repl(); if (curr().type == TK("}")) break; EXPR(); - if(curr().type == TK(":")) parsing_dict = true; + int star_level = ctx()->s_expr.top()->star_level(); + if(star_level==2 || curr().type == TK(":")){ + parsing_dict = true; + } if(parsing_dict){ - consume(TK(":")); - EXPR(); auto dict_item = make_expr(); - dict_item->key = ctx()->s_expr.popx(); - dict_item->value = ctx()->s_expr.popx(); + if(star_level == 2){ + dict_item->key = nullptr; + dict_item->value = ctx()->s_expr.popx(); + }else{ + consume(TK(":")); + EXPR(); + dict_item->key = ctx()->s_expr.popx(); + dict_item->value = ctx()->s_expr.popx(); + } items.push_back(std::move(dict_item)); }else{ items.push_back(ctx()->s_expr.popx()); diff --git a/src/expr.h b/src/expr.h index f2f70669..713a5592 100644 --- a/src/expr.h +++ b/src/expr.h @@ -366,7 +366,7 @@ struct SliceExpr: Expr{ }; struct DictItemExpr: Expr{ - Expr_ key; + Expr_ key; // maybe nullptr if it is **kwargs Expr_ value; std::string str() const override { return "DictItem()"; } @@ -374,6 +374,7 @@ struct DictItemExpr: Expr{ void emit(CodeEmitContext* ctx) override { if(is_starred()){ + PK_ASSERT(key == nullptr); value->emit(ctx); }else{ value->emit(ctx); diff --git a/tests/05_list.py b/tests/05_list.py index 2e5a740e..b7f93e5c 100644 --- a/tests/05_list.py +++ b/tests/05_list.py @@ -74,4 +74,13 @@ assert a == [8, 2, 4, 2, 9] b = [(1, 2), (3, 3), (5, 1)] b.sort(key=lambda x:x[1]) -assert b == [(5, 1), (1, 2), (3,3)] \ No newline at end of file +assert b == [(5, 1), (1, 2), (3,3)] + +# unpack expression +a = [1, 2, 3] +b = [*a, 4, 5] +assert b == [1, 2, 3, 4, 5] + +a = [] +b = [*a, 1, 2, 3, *a, *a] +assert b == [1, 2, 3] \ No newline at end of file diff --git a/tests/06_tuple.py b/tests/06_tuple.py index 7f88ddf1..7392ad32 100644 --- a/tests/06_tuple.py +++ b/tests/06_tuple.py @@ -5,4 +5,13 @@ assert b == 2 a,b = b,a assert a == 2 assert b == 1 -assert len(tup) == 6 \ No newline at end of file +assert len(tup) == 6 + +# unpack expression +a = 1, 2, 3 +b = *a, 4, 5 +assert b == (1, 2, 3, 4, 5) + +a = tuple([]) +b = *a, 1, 2, 3, *a, *a +assert b == (1, 2, 3) \ No newline at end of file diff --git a/tests/07_dict.py b/tests/07_dict.py index def4359c..114e9b8e 100644 --- a/tests/07_dict.py +++ b/tests/07_dict.py @@ -52,4 +52,18 @@ assert a == {1: 2, 3: 4} assert a.pop(1) == 2 assert a == {3: 4} assert a.pop(3) == 4 -assert a == {} \ No newline at end of file +assert a == {} + +# unpack expression +a = {1:2, 3:4} +b = {**a, 5:6, **a} +assert b == {1: 2, 3: 4, 5: 6} + +a = {} +b = {**a, 1:2, 3:4} +assert b == {1: 2, 3: 4} + +a = {1:2, 3:4, 7:8} +b = {**a, 1:5, 3:6} +c = {**a, **b} +assert c == {1: 5, 3: 6, 7: 8} \ No newline at end of file diff --git a/tests/08_set.py b/tests/08_set.py index 1f69a1c3..ffa146b4 100644 --- a/tests/08_set.py +++ b/tests/08_set.py @@ -76,4 +76,13 @@ assert type({}) is dict assert {1,2}.issubset({1,2,3}) assert {1,2,3}.issuperset({1,2}) assert {1,2,3}.isdisjoint({4,5,6}) -assert not {1,2,3}.isdisjoint({2,3,4}) \ No newline at end of file +assert not {1,2,3}.isdisjoint({2,3,4}) + +# unpack expression +a = {1, 2, 3} +b = {*a, 4, 5, *a, *a} +assert b == {1, 2, 3, 4, 5} + +a = set() +b = {*a, 1, 2, 3, *a, *a} +assert b == {1, 2, 3} \ No newline at end of file From fe00542124a3fce04e31c259b55e92417dd0e2d0 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Fri, 9 Jun 2023 22:39:16 +0800 Subject: [PATCH 4/4] ... --- src/expr.h | 3 ++- tests/46_star.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/expr.h b/src/expr.h index 713a5592..7651f5c6 100644 --- a/src/expr.h +++ b/src/expr.h @@ -687,13 +687,14 @@ struct CallExpr: Expr{ ctx->emit(OP_BUILD_TUPLE_UNPACK, (int)args.size(), line); for(auto& item: kwargs){ - item.second->emit(ctx); if(item.second->is_starred()){ if(item.second->star_level() != 2) FATAL_ERROR(); + item.second->emit(ctx); }else{ // k=v int index = ctx->add_const(py_var(ctx->vm, item.first)); ctx->emit(OP_LOAD_CONST, index, line); + item.second->emit(ctx); ctx->emit(OP_BUILD_TUPLE, 2, line); } } diff --git a/tests/46_star.py b/tests/46_star.py index f21e31b8..e1034feb 100644 --- a/tests/46_star.py +++ b/tests/46_star.py @@ -52,4 +52,15 @@ try: x = f1(*[1, 2, 3, 4]) exit(1) except TypeError: - pass \ No newline at end of file + pass + + +def g(*args, **kwargs): + return args, kwargs + +def f(a, b, *args, c=1, **kwargs): + return g(a, b, *args, c=c, **kwargs) + +args, kwargs = f(1, 2, 3, 4, c=5, d=6, e=-6.0) +assert args == (1, 2, 3, 4) +assert kwargs == {'c': 5, 'd': 6, 'e': -6.0} \ No newline at end of file