diff --git a/.github/workflows.zip b/.github/workflows.zip new file mode 100644 index 00000000..9daa7ca1 Binary files /dev/null and b/.github/workflows.zip differ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index c6f931bc..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,207 +0,0 @@ -name: build - -on: - push: - paths-ignore: - - 'docs/**' - - 'web/**' - - '**.md' - pull_request: - paths-ignore: - - 'docs/**' - - 'web/**' - - '**.md' -jobs: - build_win32_amalgamated: - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ilammy/msvc-dev-cmd@v1 - - name: Compile - shell: powershell - run: | - python amalgamate.py - cd amalgamated - cl.exe /std:c11 /utf-8 /Ox /I. pocketpy.c main.c /link /out:pkpy.exe - build_win32: - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ilammy/msvc-dev-cmd@v1 - - name: Compile - shell: bash - run: | - mkdir -p output/x86_64 - python cmake_build.py - cp main.exe output/x86_64 - cp pocketpy.dll output/x86_64 - - uses: actions/upload-artifact@v4 - with: - name: windows - path: output - - name: Unit Test - run: python scripts/run_tests.py - - name: Benchmark - run: python scripts/run_tests.py benchmark - build_linux: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Clang - uses: egor-tensin/setup-clang@v1 - with: - version: 15 - platform: x64 - - name: Install dependencies - run: sudo apt-get install -y libclang-rt-15-dev - - name: Unit Test with Coverage - run: bash run_tests.sh - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - directory: .coverage - if: github.ref == 'refs/heads/main' - - name: Compile and Test - run: | - mkdir -p output/x86_64 - python cmake_build.py - python scripts/run_tests.py - cp main output/x86_64 - cp libpocketpy.so output/x86_64 - env: - CC: clang - - uses: actions/upload-artifact@v4 - with: - name: linux - path: output - - name: Benchmark - run: python scripts/run_tests.py benchmark - build_linux_x86: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup Alpine Linux for aarch64 - uses: jirutka/setup-alpine@v1 - with: - arch: x86 - packages: gcc g++ make cmake libc-dev linux-headers python3 - - name: Build and Test - run: | - uname -m - python cmake_build.py - python scripts/run_tests.py - python scripts/run_tests.py benchmark - shell: alpine.sh --root {0} - build_darwin: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Compile and Test - run: | - python cmake_build.py - python scripts/run_tests.py - - name: Benchmark - run: python scripts/run_tests.py benchmark - - name: Test Amalgamated Build - run: python amalgamate.py - build_android: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: nttld/setup-ndk@v1 - id: setup-ndk - with: - ndk-version: r23 - local-cache: false - add-to-path: false - - name: Compile Shared Library - run: | - bash build_android.sh arm64-v8a - bash build_android.sh armeabi-v7a - bash build_android.sh x86_64 - - mkdir -p output/arm64-v8a - mkdir -p output/armeabi-v7a - mkdir -p output/x86_64 - - cp build/android/arm64-v8a/libpocketpy.so output/arm64-v8a - cp build/android/armeabi-v7a/libpocketpy.so output/armeabi-v7a - cp build/android/x86_64/libpocketpy.so output/x86_64 - env: - ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} - - uses: actions/upload-artifact@v4 - with: - name: android - path: output - build_ios: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Compile Frameworks - run: | - git clone https://github.com/leetal/ios-cmake --depth 1 ~/ios-cmake - bash build_ios.sh - mkdir -p output - cp -r build/pocketpy.xcframework output/pocketpy.xcframework - - uses: actions/upload-artifact@v4 - with: - name: ios - path: output - - merge: - runs-on: ubuntu-latest - needs: [ build_win32, build_linux, build_darwin, build_android, build_ios ] - steps: - - name: "Create output directory" - run: "mkdir $GITHUB_WORKSPACE/output" - - - name: "Merge win32" - uses: actions/download-artifact@v4.1.7 - with: - name: windows - path: $GITHUB_WORKSPACE/output/windows - - - name: "Merge linux" - uses: actions/download-artifact@v4.1.7 - with: - name: linux - path: $GITHUB_WORKSPACE/output/linux - - # - name: "Merge darwin" - # uses: actions/download-artifact@v4.1.7 - # with: - # name: macos - # path: $GITHUB_WORKSPACE/output/macos - - - name: "Merge android" - uses: actions/download-artifact@v4.1.7 - with: - name: android - path: $GITHUB_WORKSPACE/output/android - - - name: "Merge ios" - uses: actions/download-artifact@v4.1.7 - with: - name: ios - path: $GITHUB_WORKSPACE/output/ios - - - name: "Upload merged artifact" - uses: actions/upload-artifact@v4.3.3 - with: - name: all-in-one - path: $GITHUB_WORKSPACE/output diff --git a/.github/workflows/pybind11.yml b/.github/workflows/pybind11.yml deleted file mode 100644 index 7578b030..00000000 --- a/.github/workflows/pybind11.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: PKBIND Build and Test - -on: - push: - paths-ignore: - - "docs/**" - - "web/**" - - "**.md" - pull_request: - paths-ignore: - - "docs/**" - - "web/**" - - "**.md" - -jobs: - build_linux: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up GCC - run: | - sudo apt-get update - sudo apt-get install -y gcc g++ - - - name: Set up CMake - uses: jwlawson/actions-setup-cmake@v1.10 - - - name: Test - run: | - cd include/pybind11/tests - cmake -B build - cmake --build build --config Release --parallel - ./build/PKBIND_TEST - - build_win: - runs-on: windows-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up MSVC - uses: ilammy/msvc-dev-cmd@v1 - - - name: Set up CMake - uses: jwlawson/actions-setup-cmake@v1.10 - - - name: Test - run: | - cd include\pybind11\tests - cmake -B build - cmake --build build --config Release --parallel - build\Release\PKBIND_TEST.exe - - build_mac: - runs-on: macos-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Clang - run: | - brew install llvm - echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> ~/.zshrc - source ~/.zshrc - - - name: Set up CMake - uses: jwlawson/actions-setup-cmake@v1.10 - - - name: Test - run: | - cd include/pybind11/tests - cmake -B build -DENABLE_TEST=ON - cmake --build build --config Release --parallel - ./build/PKBIND_TEST diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml deleted file mode 100644 index a2add796..00000000 --- a/.github/workflows/website.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: website - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -permissions: - contents: write - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - ################################################### - - uses: actions/setup-node@v3.1.1 - - name: Retype build - run: | - python scripts/gen_docs.py - cd docs - npm install retypeapp -g - retype build - ################################################### - - name: Setup emsdk - uses: mymindstorm/setup-emsdk@v12 - with: - version: latest - actions-cache-folder: 'emsdk-cache' - - name: Compile - run: | - bash build_web.sh - mv web docs/.retype/static - ################################################### - - uses: crazy-max/ghaction-github-pages@v3 - with: - target_branch: gh-pages - build_dir: docs/.retype - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: github.ref == 'refs/heads/main' diff --git a/include/pocketpy/interpreter/frame.h b/include/pocketpy/interpreter/frame.h index 5a8b1709..04e5a6ab 100644 --- a/include/pocketpy/interpreter/frame.h +++ b/include/pocketpy/interpreter/frame.h @@ -31,28 +31,37 @@ void UnwindTarget__delete(UnwindTarget* self); typedef struct Frame { struct Frame* f_back; - const Bytecode* ip; const CodeObject* co; + py_StackRef p0; // unwinding base py_GlobalRef module; - py_StackRef p0; // unwinding base - py_StackRef locals; // locals base - bool has_function; // is p0 a function? - bool is_dynamic; // is dynamic frame? + py_Ref globals; // a module object or a dict object + py_Ref locals; // locals base or a proxy object (such as dict) + bool is_p0_function; + bool is_locals_proxy; + int ip; UnwindTarget* uw_list; } Frame; Frame* Frame__new(const CodeObject* co, - py_GlobalRef module, py_StackRef p0, - py_StackRef locals, - bool has_function); + py_GlobalRef module, + py_Ref globals, + py_Ref locals, + bool is_p0_function, + bool is_locals_proxy); void Frame__delete(Frame* self); -int Frame__ip(const Frame* self); int Frame__lineno(const Frame* self); int Frame__iblock(const Frame* self); -py_TValue* Frame__f_locals_try_get(Frame* self, py_Name name); -py_TValue* Frame__f_closure_try_get(Frame* self, py_Name name); + +int Frame__getglobal(Frame* self, py_Name name) PY_RAISE PY_RETURN; +bool Frame__setglobal(Frame* self, py_Name name, py_TValue* val) PY_RAISE; +int Frame__delglobal(Frame* self, py_Name name) PY_RAISE; + +py_Ref Frame__getclosure(Frame* self, py_Name name); + +py_StackRef Frame__getlocal_noproxy(Frame* self, py_Name name); + int Frame__prepare_jump_exception_handler(Frame* self, ValueStack*); diff --git a/include/pocketpy/objects/base.h b/include/pocketpy/objects/base.h index 3df4fc0e..18c522ba 100644 --- a/include/pocketpy/objects/base.h +++ b/include/pocketpy/objects/base.h @@ -19,5 +19,6 @@ typedef struct py_TValue { PyObject* _obj; c11_vec2 _vec2; c11_vec2i _vec2i; + void* _ptr; }; } py_TValue; diff --git a/include/pocketpy/objects/codeobject.h b/include/pocketpy/objects/codeobject.h index 9df8cc0f..1583ba3b 100644 --- a/include/pocketpy/objects/codeobject.h +++ b/include/pocketpy/objects/codeobject.h @@ -23,7 +23,6 @@ typedef enum FuncType { typedef enum NameScope { NAME_LOCAL, NAME_GLOBAL, - NAME_GLOBAL_UNKNOWN, } NameScope; typedef enum CodeBlockType { @@ -128,11 +127,12 @@ void FuncDecl__gc_mark(const FuncDecl* self); // runtime function typedef struct Function { FuncDecl_ decl; - py_TValue module; // weak ref - PyObject* clazz; // weak ref - NameDict* closure; // strong ref - py_CFunction cfunc; // wrapped C function + py_GlobalRef module; // maybe NULL, weak ref + py_Ref globals; // maybe NULL, strong ref + NameDict* closure; // maybe NULL, strong ref + PyObject* clazz; // weak ref; for super() + py_CFunction cfunc; // wrapped C function; for decl-based binding } Function; -void Function__ctor(Function* self, FuncDecl_ decl, py_TValue* module); +void Function__ctor(Function* self, FuncDecl_ decl, py_GlobalRef module, py_Ref globals); void Function__dtor(Function* self); diff --git a/include/pocketpy/pocketpy.h b/include/pocketpy/pocketpy.h index d8a70302..4997b6da 100644 --- a/include/pocketpy/pocketpy.h +++ b/include/pocketpy/pocketpy.h @@ -204,6 +204,8 @@ PK_API void py_newboundmethod(py_OutRef out, py_Ref self, py_Ref func); PK_API py_Name py_name(const char*); /// Convert a name to a null-terminated string. PK_API const char* py_name2str(py_Name); +/// Convert a name to a python `str` object with cache. +PK_API py_GlobalRef py_name2ref(py_Name); /// Convert a `c11_sv` to a name. PK_API py_Name py_namev(c11_sv); /// Convert a name to a `c11_sv`. diff --git a/pyrightconfig.json b/pyrightconfig.json index d08b4f98..b820ef59 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -3,5 +3,6 @@ "reportMissingModuleSource": "none", "reportArgumentType": "none", "reportWildcardImportFromLibrary": "none", + "reportRedeclaration": "none", "pythonVersion": "3.12" } diff --git a/src/common/strname.c b/src/common/strname.c index 8922a37b..6ba3805e 100644 --- a/src/common/strname.c +++ b/src/common/strname.c @@ -6,16 +6,19 @@ #include +typedef struct { + char* data; // null-terminated data + int size; // size of the data excluding the null-terminator + py_TValue* ref; // cached `str` object (lazy initialized) +} RInternedEntry; + // TODO: use a more efficient data structure static c11_smallmap_s2n _interned; -static c11_vector /*T=char* */ _r_interned; +static c11_vector /* T=RInternedEntry */ _r_interned; void py_Name__initialize() { c11_smallmap_s2n__ctor(&_interned); - for(int i = 0; i < _r_interned.length; i++) { - PK_FREE(c11__at(char*, &_r_interned, i)); - } - c11_vector__ctor(&_r_interned, sizeof(c11_sv)); + c11_vector__ctor(&_r_interned, sizeof(RInternedEntry)); #define MAGIC_METHOD(x) \ if(x != py_name(#x)) abort(); @@ -26,7 +29,7 @@ void py_Name__initialize() { void py_Name__finalize() { // free all char* for(int i = 0; i < _r_interned.length; i++) { - PK_FREE(c11__getitem(char*, &_r_interned, i)); + PK_FREE(c11__getitem(RInternedEntry, &_r_interned, i).data); } c11_smallmap_s2n__dtor(&_interned); c11_vector__dtor(&_r_interned); @@ -35,7 +38,6 @@ void py_Name__finalize() { py_Name py_name(const char* name) { return py_namev((c11_sv){name, strlen(name)}); } py_Name py_namev(c11_sv name) { - // TODO: PK_GLOBAL_SCOPE_LOCK() uint16_t index = c11_smallmap_s2n__get(&_interned, name, 0); if(index != 0) return index; // generate new index @@ -44,7 +46,11 @@ py_Name py_namev(c11_sv name) { char* p = PK_MALLOC(name.size + 1); memcpy(p, name.data, name.size); p[name.size] = '\0'; - c11_vector__push(char*, &_r_interned, p); + RInternedEntry entry; + entry.data = p; + entry.size = name.size; + entry.ref = NULL; + c11_vector__push(RInternedEntry, &_r_interned, entry); index = _r_interned.length; // 1-based // save to _interned c11_smallmap_s2n__set(&_interned, (c11_sv){p, name.size}, index); @@ -54,11 +60,24 @@ py_Name py_namev(c11_sv name) { const char* py_name2str(py_Name index) { assert(index > 0 && index <= _interned.length); - return c11__getitem(char*, &_r_interned, index - 1); + return c11__getitem(RInternedEntry, &_r_interned, index - 1).data; } c11_sv py_name2sv(py_Name index) { assert(index > 0 && index <= _interned.length); - const char* p = py_name2str(index); - return (c11_sv){p, strlen(p)}; + RInternedEntry entry = c11__getitem(RInternedEntry, &_r_interned, index - 1); + return (c11_sv){entry.data, entry.size}; +} + +py_GlobalRef py_name2ref(py_Name index) { + assert(index > 0 && index <= _interned.length); + RInternedEntry entry = c11__getitem(RInternedEntry, &_r_interned, index - 1); + if(entry.ref == NULL){ + entry.ref = PK_MALLOC(16); // ... + c11_sv sv; + sv.data = entry.data; + sv.size = entry.size; + py_newstrv(entry.ref, sv); + } + return entry.ref; } diff --git a/src/compiler/compiler.c b/src/compiler/compiler.c index eeefad2d..6adc55ce 100644 --- a/src/compiler/compiler.c +++ b/src/compiler/compiler.c @@ -102,17 +102,22 @@ void NameExpr__emit_(Expr* self_, Ctx* ctx) { NameExpr* self = (NameExpr*)self_; int index = c11_smallmap_n2i__get(&ctx->co->varnames_inv, self->name, -1); if(self->scope == NAME_LOCAL && index >= 0) { + // we know this is a local variable Ctx__emit_(ctx, OP_LOAD_FAST, index, self->line); } else { - Opcode op = ctx->level <= 1 ? OP_LOAD_GLOBAL : OP_LOAD_NONLOCAL; - if(ctx->is_compiling_class && self->scope == NAME_GLOBAL) { - // if we are compiling a class, we should use OP_LOAD_ATTR_GLOBAL instead of - // OP_LOAD_GLOBAL this supports @property.setter - op = OP_LOAD_CLASS_GLOBAL; - // exec()/eval() won't work with OP_LOAD_ATTR_GLOBAL in class body + Opcode op; + // otherwise, if we are running dynamically, force `OP_LOAD_NAME` + if(ctx->co->src->is_dynamic) { + op = OP_LOAD_NAME; + // `OP_LOAD_NAME` won't handle `OP_LOAD_CLASS_GLOBAL` + // so `exec()` will raise an error for @property.setter } else { - // we cannot determine the scope when calling exec()/eval() - if(self->scope == NAME_GLOBAL_UNKNOWN) op = OP_LOAD_NAME; + op = ctx->level <= 1 ? OP_LOAD_GLOBAL : OP_LOAD_NONLOCAL; + if(ctx->is_compiling_class && self->scope == NAME_GLOBAL) { + // if we are compiling a class, we should use `OP_LOAD_CLASS_GLOBAL` + // this is for @property.setter + op = OP_LOAD_CLASS_GLOBAL; + } } Ctx__emit_(ctx, op, self->name, self->line); } @@ -124,8 +129,11 @@ bool NameExpr__emit_del(Expr* self_, Ctx* ctx) { case NAME_LOCAL: Ctx__emit_(ctx, OP_DELETE_FAST, Ctx__add_varname(ctx, self->name), self->line); break; - case NAME_GLOBAL: Ctx__emit_(ctx, OP_DELETE_GLOBAL, self->name, self->line); break; - case NAME_GLOBAL_UNKNOWN: Ctx__emit_(ctx, OP_DELETE_NAME, self->name, self->line); break; + case NAME_GLOBAL: { + Opcode op = ctx->co->src->is_dynamic ? OP_DELETE_NAME : OP_DELETE_GLOBAL; + Ctx__emit_(ctx, op, self->name, self->line); + break; + } default: c11__unreachable(); } return true; @@ -1219,8 +1227,10 @@ static int Ctx__add_const(Ctx* self, py_Ref v) { static void Ctx__emit_store_name(Ctx* self, NameScope scope, py_Name name, int line) { switch(scope) { case NAME_LOCAL: Ctx__emit_(self, OP_STORE_FAST, Ctx__add_varname(self, name), line); break; - case NAME_GLOBAL: Ctx__emit_(self, OP_STORE_GLOBAL, name, line); break; - case NAME_GLOBAL_UNKNOWN: Ctx__emit_(self, OP_STORE_NAME, name, line); break; + case NAME_GLOBAL: { + Opcode op = self->co->src->is_dynamic ? OP_STORE_NAME : OP_STORE_GLOBAL; + Ctx__emit_(self, op, name, line); + } break; default: c11__unreachable(); } } @@ -1331,9 +1341,7 @@ static void Compiler__dtor(Compiler* self) { if((err = B)) return err static NameScope name_scope(Compiler* self) { - NameScope s = self->contexts.length > 1 ? NAME_LOCAL : NAME_GLOBAL; - if(self->src->is_dynamic && s == NAME_GLOBAL) s = NAME_GLOBAL_UNKNOWN; - return s; + return self->contexts.length > 1 ? NAME_LOCAL : NAME_GLOBAL; } Error* SyntaxError(Compiler* self, const char* fmt, ...) { @@ -1720,7 +1728,7 @@ static Error* exprName(Compiler* self) { NameScope scope = name_scope(self); // promote this name to global scope if needed if(c11_smallmap_n2i__contains(&ctx()->global_names, name)) { - if(scope == NAME_GLOBAL_UNKNOWN) return SyntaxError(self, "cannot use global keyword here"); + if(self->src->is_dynamic) return SyntaxError(self, "cannot use global keyword here"); scope = NAME_GLOBAL; } NameExpr* e = NameExpr__new(prev()->line, name, scope); diff --git a/src/interpreter/ceval.c b/src/interpreter/ceval.c index 1ff28e10..ad81c526 100644 --- a/src/interpreter/ceval.c +++ b/src/interpreter/ceval.c @@ -33,7 +33,7 @@ static bool stack_format_object(VM* self, c11_sv spec); } while(0) #define DISPATCH_JUMP_ABSOLUTE(__target) \ do { \ - frame->ip = c11__at(Bytecode, &frame->co->codes, __target); \ + frame->ip = __target; \ goto __NEXT_STEP; \ } while(0) @@ -86,15 +86,18 @@ static bool unpack_dict_to_buffer(py_Ref key, py_Ref val, void* ctx) { FrameResult VM__run_top_frame(VM* self) { Frame* frame = self->top_frame; + Bytecode* codes; + const Frame* base_frame = frame; while(true) { Bytecode byte; __NEXT_FRAME: + codes = frame->co->codes.data; frame->ip++; __NEXT_STEP: - byte = *frame->ip; + byte = codes[frame->ip]; #ifndef NDEBUG pk_print_stack(self, frame, byte); @@ -176,7 +179,7 @@ FrameResult VM__run_top_frame(VM* self) { CHECK_STACK_OVERFLOW(); FuncDecl_ decl = c11__getitem(FuncDecl_, &frame->co->func_decls, byte.arg); Function* ud = py_newobject(SP(), tp_function, 0, sizeof(Function)); - Function__ctor(ud, decl, frame->module); + Function__ctor(ud, decl, frame->module, frame->globals); if(decl->nested) { ud->closure = FastLocals__to_namedict(frame->locals, frame->co); py_Name name = py_name(decl->code.name->data); @@ -200,10 +203,10 @@ FrameResult VM__run_top_frame(VM* self) { DISPATCH(); } case OP_LOAD_NAME: { - assert(frame->is_dynamic); + // assert(frame->is_dynamic); py_Name name = byte.arg; py_TValue* tmp; - py_newstr(SP()++, py_name2str(name)); + py_assign(SP()++, py_name2ref(name)); // locals if(!py_isnone(&frame->p0[1])) { if(py_getitem(&frame->p0[1], TOP())) { @@ -217,6 +220,7 @@ FrameResult VM__run_top_frame(VM* self) { } } } + // `LOAD_ // globals if(py_getitem(&frame->p0[0], TOP())) { py_assign(TOP(), py_retval()); @@ -239,16 +243,18 @@ FrameResult VM__run_top_frame(VM* self) { } case OP_LOAD_NONLOCAL: { py_Name name = byte.arg; - py_Ref tmp = Frame__f_closure_try_get(frame, name); + py_Ref tmp = Frame__getclosure(frame, name); if(tmp != NULL) { PUSH(tmp); DISPATCH(); } - tmp = py_getdict(frame->module, name); - if(tmp != NULL) { - PUSH(tmp); + int res = Frame__getglobal(frame, name); + if(res == 1) { + PUSH(&self->last_retval); DISPATCH(); } + if(res == -1) goto __ERROR; + tmp = py_getdict(&self->builtins, name); if(tmp != NULL) { PUSH(tmp); @@ -259,12 +265,13 @@ FrameResult VM__run_top_frame(VM* self) { } case OP_LOAD_GLOBAL: { py_Name name = byte.arg; - py_Ref tmp = py_getdict(frame->module, name); - if(tmp != NULL) { - PUSH(tmp); + int res = Frame__getglobal(frame, name); + if(res == 1) { + PUSH(&self->last_retval); DISPATCH(); } - tmp = py_getdict(&self->builtins, name); + if(res == -1) goto __ERROR; + py_Ref tmp = py_getdict(&self->builtins, name); if(tmp != NULL) { PUSH(tmp); DISPATCH(); @@ -289,11 +296,12 @@ FrameResult VM__run_top_frame(VM* self) { DISPATCH(); } // load global if attribute not found - tmp = py_getdict(frame->module, name); - if(tmp) { - PUSH(tmp); + int res = Frame__getglobal(frame, name); + if(res == 1) { + PUSH(&self->last_retval); DISPATCH(); } + if(res == -1) goto __ERROR; tmp = py_getdict(&self->builtins, name); if(tmp) { PUSH(tmp); @@ -337,9 +345,9 @@ FrameResult VM__run_top_frame(VM* self) { } case OP_STORE_FAST: frame->locals[byte.arg] = POPX(); DISPATCH(); case OP_STORE_NAME: { - assert(frame->is_dynamic); + // assert(frame->is_dynamic); py_Name name = byte.arg; - py_newstr(SP()++, py_name2str(name)); + py_assign(SP()++, py_name2ref(name)); // [value, name] if(!py_isnone(&frame->p0[1])) { // locals @@ -369,7 +377,7 @@ FrameResult VM__run_top_frame(VM* self) { DISPATCH(); } case OP_STORE_GLOBAL: { - py_setdict(frame->module, byte.arg, TOP()); + if(!Frame__setglobal(frame, byte.arg, TOP())) goto __ERROR; POP(); DISPATCH(); } @@ -407,9 +415,9 @@ FrameResult VM__run_top_frame(VM* self) { DISPATCH(); } case OP_DELETE_NAME: { - assert(frame->is_dynamic); + // assert(frame->is_dynamic); py_Name name = byte.arg; - py_newstr(SP()++, py_name2str(name)); + py_assign(SP()++, py_name2ref(name)); if(!py_isnone(&frame->p0[1])) { // locals if(py_delitem(&frame->p0[1], TOP())) { @@ -439,12 +447,12 @@ FrameResult VM__run_top_frame(VM* self) { } case OP_DELETE_GLOBAL: { py_Name name = byte.arg; - bool ok = py_deldict(frame->module, name); - if(!ok) { - NameError(name); - goto __ERROR; - } - DISPATCH(); + int res = Frame__delglobal(frame, name); + if(res == 1) DISPATCH(); + if(res == -1) goto __ERROR; + // res == 0 + NameError(name); + goto __ERROR; } case OP_DELETE_ATTR: { @@ -860,7 +868,7 @@ FrameResult VM__run_top_frame(VM* self) { ImportError("cannot import name '%n'", name); goto __ERROR; } else { - py_setdict(frame->module, name, value); + if(!Frame__setglobal(frame, name, value)) goto __ERROR; } } } else { @@ -869,7 +877,7 @@ FrameResult VM__run_top_frame(VM* self) { if(!kv->key) continue; c11_sv name = py_name2sv(kv->key); if(name.size == 0 || name.data[0] == '_') continue; - py_setdict(frame->module, kv->key, &kv->value); + if(!Frame__setglobal(frame, kv->key, &kv->value)) goto __ERROR; } } POP(); @@ -998,8 +1006,7 @@ FrameResult VM__run_top_frame(VM* self) { case OP_END_CLASS: { // [cls or decorated] py_Name name = byte.arg; - // set into f_globals - py_setdict(frame->module, name, TOP()); + if(!Frame__setglobal(frame, name, TOP())) goto __ERROR; if(py_istype(TOP(), tp_type)) { // call on_end_subclass @@ -1166,7 +1173,7 @@ FrameResult VM__run_top_frame(VM* self) { py_BaseException__stpush(&self->curr_exception, frame->co->src, Frame__lineno(frame), - frame->has_function ? frame->co->name->data : NULL); + frame->is_p0_function ? frame->co->name->data : NULL); __ERROR_RE_RAISE: do { } while(0); @@ -1183,6 +1190,7 @@ FrameResult VM__run_top_frame(VM* self) { return RES_ERROR; } frame = self->top_frame; + codes = frame->co->codes.data; goto __ERROR; } } diff --git a/src/interpreter/frame.c b/src/interpreter/frame.c index 64c7e5ee..18747897 100644 --- a/src/interpreter/frame.c +++ b/src/interpreter/frame.c @@ -23,7 +23,7 @@ NameDict* FastLocals__to_namedict(py_TValue* locals, const CodeObject* co) { NameDict* dict = NameDict__new(); c11__foreach(c11_smallmap_n2i_KV, &co->varnames_inv, entry) { py_TValue value = locals[entry->value]; - if(!py_isnil(&value)) { NameDict__set(dict, entry->key, value); } + if(!py_isnil(&value)) NameDict__set(dict, entry->key, value); } return dict; } @@ -39,19 +39,23 @@ UnwindTarget* UnwindTarget__new(UnwindTarget* next, int iblock, int offset) { void UnwindTarget__delete(UnwindTarget* self) { PK_FREE(self); } Frame* Frame__new(const CodeObject* co, - py_GlobalRef module, py_StackRef p0, - py_StackRef locals, - bool has_function) { + py_GlobalRef module, + py_Ref globals, + py_Ref locals, + bool is_p0_function, + bool is_locals_proxy) { + assert(module->type == tp_module || module->type == tp_dict); Frame* self = FixedMemoryPool__alloc(&pk_current_vm->pool_frame); self->f_back = NULL; - self->ip = (Bytecode*)co->codes.data - 1; self->co = co; - self->module = module; self->p0 = p0; + self->module = module; + self->globals = globals; self->locals = locals; - self->has_function = has_function; - self->is_dynamic = co->src->is_dynamic; + self->is_p0_function = is_p0_function; + self->is_locals_proxy = is_locals_proxy; + self->ip = -1; self->uw_list = NULL; return self; } @@ -99,30 +103,60 @@ void Frame__set_unwind_target(Frame* self, py_TValue* sp) { } void Frame__gc_mark(Frame* self) { - pk__mark_value(self->module); + pk__mark_value(self->globals); + if(self->is_locals_proxy) pk__mark_value(self->locals); CodeObject__gc_mark(self->co); } -py_TValue* Frame__f_closure_try_get(Frame* self, py_Name name) { - if(!self->has_function) return NULL; - Function* ud = py_touserdata(self->p0); - if(ud->closure == NULL) return NULL; - return NameDict__try_get(ud->closure, name); -} - -int Frame__ip(const Frame* self) { return self->ip - (Bytecode*)self->co->codes.data; } - int Frame__lineno(const Frame* self) { - int ip = Frame__ip(self); + int ip = self->ip; return c11__getitem(BytecodeEx, &self->co->codes_ex, ip).lineno; } int Frame__iblock(const Frame* self) { - int ip = Frame__ip(self); + int ip = self->ip; return c11__getitem(BytecodeEx, &self->co->codes_ex, ip).iblock; } -py_TValue* Frame__f_locals_try_get(Frame* self, py_Name name) { - assert(!self->is_dynamic); +int Frame__getglobal(Frame* self, py_Name name) { + if(self->globals->type == tp_module) { + py_ItemRef item = py_getdict(self->globals, name); + if(item != NULL) { + py_assign(py_retval(), item); + return 1; + } + return 0; + } else { + return py_dict_getitem(self->globals, py_name2ref(name)); + } +} + +bool Frame__setglobal(Frame* self, py_Name name, py_TValue* val) { + if(self->globals->type == tp_module) { + py_setdict(self->globals, name, val); + return true; + } else { + return py_dict_setitem(self->globals, py_name2ref(name), val); + } +} + +int Frame__delglobal(Frame* self, py_Name name) { + if(self->globals->type == tp_module) { + bool found = py_deldict(self->globals, name); + return found ? 1 : 0; + } else { + return py_dict_delitem(self->globals, py_name2ref(name)); + } +} + +py_StackRef Frame__getlocal_noproxy(Frame* self, py_Name name) { + assert(!self->is_locals_proxy); return FastLocals__try_get_by_name(self->locals, self->co, name); +} + +py_Ref Frame__getclosure(Frame* self, py_Name name) { + if(!self->is_p0_function) return NULL; + Function* ud = py_touserdata(self->p0); + if(ud->closure == NULL) return NULL; + return NameDict__try_get(ud->closure, name); } \ No newline at end of file diff --git a/src/interpreter/vm.c b/src/interpreter/vm.c index 5ef35650..44f41737 100644 --- a/src/interpreter/vm.c +++ b/src/interpreter/vm.c @@ -431,8 +431,8 @@ static bool co->name->data); } else { // add to **kwargs - bool ok = py_dict_setitem_by_str(&buffer[decl->starred_kwarg], - py_name2str(key), + bool ok = py_dict_setitem(&buffer[decl->starred_kwarg], + py_name2ref(key), &p1[2 * j + 1]); if(!ok) return false; } @@ -480,7 +480,7 @@ FrameResult VM__vectorcall(VM* self, uint16_t argc, uint16_t kwargc, bool opcall // submit the call if(!fn->cfunc) { // python function - VM__push_frame(self, Frame__new(co, &fn->module, p0, argv, true)); + VM__push_frame(self, Frame__new(co, p0, fn->module, fn->globals, argv, true, false)); return opcall ? RES_CALL : VM__run_top_frame(self); } else { // decl-based binding @@ -509,7 +509,7 @@ FrameResult VM__vectorcall(VM* self, uint16_t argc, uint16_t kwargc, bool opcall // submit the call if(!fn->cfunc) { // python function - VM__push_frame(self, Frame__new(co, &fn->module, p0, argv, true)); + VM__push_frame(self, Frame__new(co, p0, fn->module, fn->globals, argv, true, false)); return opcall ? RES_CALL : VM__run_top_frame(self); } else { // decl-based binding @@ -525,7 +525,7 @@ FrameResult VM__vectorcall(VM* self, uint16_t argc, uint16_t kwargc, bool opcall // copy buffer back to stack self->stack.sp = argv + co->nlocals; memcpy(argv, self->__vectorcall_buffer, co->nlocals * sizeof(py_TValue)); - Frame* frame = Frame__new(co, &fn->module, p0, argv, true); + Frame* frame = Frame__new(co, p0, fn->module, fn->globals, argv, true, false); pk_newgenerator(py_retval(), frame, p0, self->stack.sp); self->stack.sp = p0; // reset the stack return RES_RETURN; diff --git a/src/modules/enum.c b/src/modules/enum.c index 11bb57f9..f63fa3a1 100644 --- a/src/modules/enum.c +++ b/src/modules/enum.c @@ -7,7 +7,7 @@ static bool Enum__wrapper_field(py_Name name, py_Ref value, void* ctx) { if(name_sv.size == 0 || name_sv.data[0] == '_') return true; py_push(ctx); py_pushnil(); - py_newstr(py_pushtmp(), py_name2str(name)); + py_assign(py_pushtmp(), py_name2ref(name)); py_push(value); bool ok = py_vectorcall(2, 0); if(!ok) return false; diff --git a/src/objects/codeobject.c b/src/objects/codeobject.c index ba90874b..6483bfe5 100644 --- a/src/objects/codeobject.c +++ b/src/objects/codeobject.c @@ -159,12 +159,13 @@ void CodeObject__dtor(CodeObject* self) { c11_vector__dtor(&self->func_decls); } -void Function__ctor(Function* self, FuncDecl_ decl, py_TValue* module) { +void Function__ctor(Function* self, FuncDecl_ decl, py_GlobalRef module, py_Ref globals) { PK_INCREF(decl); self->decl = decl; - self->module = module ? *module : *py_NIL(); - self->clazz = NULL; + self->module = module; + self->globals = globals; self->closure = NULL; + self->clazz = NULL; self->cfunc = NULL; } diff --git a/src/public/exec.c b/src/public/exec.c index de5a09fc..a4e07eeb 100644 --- a/src/public/exec.c +++ b/src/public/exec.c @@ -59,7 +59,9 @@ bool pk_exec(CodeObject* co, py_Ref module) { py_StackRef sp = vm->stack.sp; if(co->src->is_dynamic) sp -= 3; // [globals, locals, code] - Frame* frame = Frame__new(co, module, sp, sp, false); + const bool is_p0_function = false; + const bool is_locals_proxy = true; + Frame* frame = Frame__new(co, sp, module, module, sp, is_p0_function, is_locals_proxy); VM__push_frame(vm, frame); FrameResult res = VM__run_top_frame(vm); if(res == RES_ERROR) return false; diff --git a/src/public/modules.c b/src/public/modules.c index 74d00564..9d40d78c 100644 --- a/src/public/modules.c +++ b/src/public/modules.c @@ -498,23 +498,23 @@ void py_newglobals(py_Ref out) { pk_mappingproxy__namedict(out, &pk_current_vm->main); return; } - if(frame->is_dynamic) { - py_assign(out, &frame->p0[0]); + if(frame->globals->type == tp_module) { + pk_mappingproxy__namedict(out, frame->globals); } else { - pk_mappingproxy__namedict(out, frame->module); + *out = *frame->globals; // dict } } void py_newlocals(py_Ref out) { Frame* frame = pk_current_vm->top_frame; - if(frame->is_dynamic) { - py_assign(out, &frame->p0[1]); + if(!frame || !frame->is_p0_function) { + py_newglobals(out); return; } - if(frame->has_function) { + if(!frame->is_locals_proxy){ pk_mappingproxy__locals(out, frame); - } else { - py_newglobals(out); + }else{ + *out = *frame->locals; } } @@ -563,7 +563,7 @@ static bool _builtins_execdyn(const char* title, int argc, py_Ref argv, enum py_ CodeObject* co = py_touserdata(code); if(!co->src->is_dynamic) { if(argc != 1) - return ValueError("code object is not dynamic, so globals and locals must be None"); + return ValueError("code object is not dynamic, `globals` and `locals` must be None"); py_shrink(3); } Frame* frame = pk_current_vm->top_frame; @@ -736,6 +736,7 @@ py_TValue pk_builtins__register() { static void function__gc_mark(void* ud) { Function* func = ud; + if(func->globals) pk__mark_value(func->globals); if(func->closure) pk__mark_namedict(func->closure); FuncDecl__gc_mark(func->decl); } @@ -779,7 +780,7 @@ static bool super__new__(int argc, py_Ref argv) { py_Ref self_arg = NULL; if(argc == 1) { // super() - if(frame->has_function) { + if(frame->is_p0_function && !frame->is_locals_proxy) { py_TValue* callable = frame->p0; if(callable->type == tp_boundmethod) callable = py_getslot(frame->p0, 1); if(callable->type == tp_function) { diff --git a/src/public/py_mappingproxy.c b/src/public/py_mappingproxy.c index b8da64e0..343d8c4a 100644 --- a/src/public/py_mappingproxy.c +++ b/src/public/py_mappingproxy.c @@ -4,6 +4,7 @@ #include "pocketpy/objects/object.h" #include "pocketpy/interpreter/vm.h" #include "pocketpy/common/sstream.h" +#include void pk_mappingproxy__namedict(py_Ref out, py_Ref object) { py_newobject(out, tp_namedict, 1, 0); @@ -59,7 +60,7 @@ static bool namedict_items(int argc, py_Ref argv) { if(py_isnil(ti->magic_0 + j)) continue; py_Ref slot = py_list_emplace(py_retval()); py_newtuple(slot, 2); - py_newstr(py_tuple_getitem(slot, 0), py_name2str(j + PK_MAGIC_SLOTS_UNCOMMON_LENGTH)); + py_assign(py_tuple_getitem(slot, 0), py_name2ref(j + PK_MAGIC_SLOTS_UNCOMMON_LENGTH)); py_assign(py_tuple_getitem(slot, 1), ti->magic_0 + j); } if(ti->magic_1) { @@ -67,7 +68,7 @@ static bool namedict_items(int argc, py_Ref argv) { if(py_isnil(ti->magic_1 + j)) continue; py_Ref slot = py_list_emplace(py_retval()); py_newtuple(slot, 2); - py_newstr(py_tuple_getitem(slot, 0), py_name2str(j)); + py_assign(py_tuple_getitem(slot, 0), py_name2ref(j)); py_assign(py_tuple_getitem(slot, 1), ti->magic_1 + j); } } @@ -76,7 +77,7 @@ static bool namedict_items(int argc, py_Ref argv) { py_Ref slot = py_list_emplace(py_retval()); py_newtuple(slot, 2); NameDict_KV* kv = c11__at(NameDict_KV, dict, i); - py_newstr(py_tuple_getitem(slot, 0), py_name2str(kv->key)); + py_assign(py_tuple_getitem(slot, 0), py_name2ref(kv->key)); py_assign(py_tuple_getitem(slot, 1), &kv->value); } return true; @@ -107,17 +108,21 @@ py_Type pk_namedict__register() { ////////////////////// void pk_mappingproxy__locals(py_Ref out, Frame* frame) { - assert(frame->has_function && !frame->is_dynamic); - Frame** ud = py_newobject(out, tp_locals, 0, sizeof(Frame*)); - *ud = frame; + assert(frame->is_p0_function && !frame->is_locals_proxy); + out->type = tp_locals; + out->is_ptr = false; + out->extra = 0; + // this is a weak reference + // locals() will expire when the frame is destroyed + out->_ptr = frame; } static bool locals__getitem__(int argc, py_Ref argv) { PY_CHECK_ARGC(2); PY_CHECK_ARG_TYPE(1, tp_str); - Frame** ud = py_touserdata(argv); + Frame* frame = argv->_ptr; py_Name name = py_namev(py_tosv(py_arg(1))); - py_Ref slot = Frame__f_locals_try_get(*ud, name); + py_Ref slot = Frame__getlocal_noproxy(frame, name); if(!slot || py_isnil(slot)) return KeyError(py_arg(1)); py_assign(py_retval(), slot); return true; @@ -126,9 +131,9 @@ static bool locals__getitem__(int argc, py_Ref argv) { static bool locals__setitem__(int argc, py_Ref argv) { PY_CHECK_ARGC(3); PY_CHECK_ARG_TYPE(1, tp_str); - Frame** ud = py_touserdata(argv); + Frame* frame = argv->_ptr; py_Name name = py_namev(py_tosv(py_arg(1))); - py_Ref slot = Frame__f_locals_try_get(*ud, name); + py_Ref slot = Frame__getlocal_noproxy(frame, name); if(!slot) return KeyError(py_arg(1)); py_assign(slot, py_arg(2)); py_newnone(py_retval()); @@ -138,9 +143,9 @@ static bool locals__setitem__(int argc, py_Ref argv) { static bool locals__delitem__(int argc, py_Ref argv) { PY_CHECK_ARGC(2); PY_CHECK_ARG_TYPE(1, tp_str); - Frame** ud = py_touserdata(argv); + Frame* frame = argv->_ptr; py_Name name = py_namev(py_tosv(py_arg(1))); - py_Ref res = Frame__f_locals_try_get(*ud, name); + py_Ref res = Frame__getlocal_noproxy(frame, name); if(!res || py_isnil(res)) return KeyError(py_arg(1)); py_newnil(res); py_newnone(py_retval()); @@ -150,9 +155,9 @@ static bool locals__delitem__(int argc, py_Ref argv) { static bool locals__contains__(int argc, py_Ref argv) { PY_CHECK_ARGC(2); PY_CHECK_ARG_TYPE(1, tp_str); - Frame** ud = py_touserdata(argv); + Frame* frame = argv->_ptr; py_Name name = py_namev(py_tosv(py_arg(1))); - py_Ref slot = Frame__f_locals_try_get(*ud, name); + py_Ref slot = Frame__getlocal_noproxy(frame, name); py_newbool(py_retval(), slot && !py_isnil(slot)); return true; } diff --git a/src/public/py_object.c b/src/public/py_object.c index 7e9a3e04..827c0065 100644 --- a/src/public/py_object.c +++ b/src/public/py_object.c @@ -84,7 +84,7 @@ static bool type__base__(int argc, py_Ref argv) { static bool type__name__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); py_TypeInfo* ti = pk__type_info(py_totype(argv)); - py_newstr(py_retval(), py_name2str(ti->name)); + py_assign(py_retval(), py_name2ref(ti->name)); return true; } diff --git a/src/public/py_ops.c b/src/public/py_ops.c index 53c1fa43..9495ff4a 100644 --- a/src/public/py_ops.c +++ b/src/public/py_ops.c @@ -153,7 +153,7 @@ bool py_getattr(py_Ref self, py_Name name) { if(fallback) { py_push(fallback); py_push(self); - py_newstr(py_pushtmp(), py_name2str(name)); + py_assign(py_pushtmp(), py_name2ref(name)); return py_vectorcall(1, 0); } diff --git a/src/public/values.c b/src/public/values.c index 27d59e06..5db42560 100644 --- a/src/public/values.c +++ b/src/public/values.c @@ -104,7 +104,7 @@ py_Name decl->docstring = docstring; // construct the function Function* ud = py_newobject(out, tp_function, slots, sizeof(Function)); - Function__ctor(ud, decl, NULL); + Function__ctor(ud, decl, NULL, NULL); ud->cfunc = f; CodeObject__dtor(&code); PK_DECREF(source); diff --git a/tests/66_eval.py b/tests/66_eval.py index 3d1137b6..54692830 100644 --- a/tests/66_eval.py +++ b/tests/66_eval.py @@ -67,3 +67,8 @@ try: exit(1) except NameError: pass + +# https://github.com/pocketpy/pocketpy/issues/339 +code = '\nprint(x)\ndef f():\n print(x)\nf()\n' +x = 33 +exec(code, {'x': 42}) \ No newline at end of file diff --git a/tests/67_locals_vs_globals.py b/tests/67_locals_vs_globals.py new file mode 100644 index 00000000..010a2a12 --- /dev/null +++ b/tests/67_locals_vs_globals.py @@ -0,0 +1,45 @@ +# https://gist.github.com/dean0x7d/df5ce97e4a1a05be4d56d1378726ff92 + +a = 1 +my_locals = {"b": 2} + +# With user-defined locals: +exec(""" +import sys +assert locals() != globals() +assert "sys" in locals() +assert "sys" not in globals() +assert "a" not in locals() +assert "a" in globals() +# print(a) # checks `locals()` first, fails, but finds it in `globals()` +assert (a == 1), a +assert "b" in locals() +assert "b" not in globals() +# print(b) +assert (b == 2), b +def main(): + assert locals() != globals() + assert "sys" not in locals() # not the same `locals()` as the outer scope + assert "sys" not in globals() # and `sys` isn't in `globals()`, same as before + assert "b" not in locals() # again, not the same `locals()` as the outer scope +main() +""", globals(), my_locals) + +assert "sys" in my_locals # side effect +assert "sys" not in globals() + + +# With default locals: +exec(""" +import sys +assert locals() == globals() +assert "sys" in locals() +assert "sys" in globals() +def main(): + assert locals() != globals() + assert "sys" not in locals() # not the same locals as the outer scope + assert "sys" in globals() # but now be can access `sys` via `globals()` +main() +""", globals()) + +assert "sys" in globals() \ No newline at end of file