up

.github/workflows/main.yml

@@ -0,0 +1,118 @@
+name: build
+on: [push, pull_request]
+jobs:
+  build_win:
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ilammy/msvc-dev-cmd@v1
+    - name: Compile
+      shell: bash
+      run: |
+        python3 build.py windows
+        python3 build.py windows -lib
+        mkdir -p output/windows/x86_64
+        cp pocketpy.exe output/windows/x86_64
+        cp pocketpy.dll output/windows/x86_64
+    - uses: actions/upload-artifact@v3
+      with:
+        path: output
+    - name: Unit Test
+      run: python3 scripts/run_tests.py
+    - name: Benchmark
+      run: python3 scripts/run_tests.py benchmark
+  build_linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Setup Clang
+      uses: egor-tensin/setup-clang@v1
+      with:
+        version: 15
+        platform: x64
+    - name: Install libc++
+      run: sudo apt install -y libc++-15-dev libc++1-15 libc++abi-15-dev libc++abi1-15
+    - name: Compile
+      run: |
+        python3 build.py linux
+        python3 build.py linux -lib
+        mkdir -p output/linux/x86_64
+        cp pocketpy output/linux/x86_64
+        cp pocketpy.so output/linux/x86_64
+    - uses: actions/upload-artifact@v3
+      with:
+        path: output
+    - name: Unit Test
+      run: python3 scripts/run_tests.py
+    - name: Benchmark
+      run: python3 scripts/run_tests.py benchmark
+  build_macos:
+      runs-on: macos-latest
+      steps:
+      - uses: actions/checkout@v3
+      - run: |
+          python3 amalgamate.py
+          cd plugins/macos/pocketpy
+          mkdir -p output/macos
+          xcodebuild clean build CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO
+          cp -r build/Release/pocketpy.bundle output/macos
+      - uses: actions/upload-artifact@v3
+        with:
+          path: plugins/macos/pocketpy/output
+  build_android:
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/checkout@v3
+      - uses: subosito/flutter-action@v2
+        with:
+          flutter-version: '3.3.0'
+          channel: 'stable'
+          cache: true
+      - run: flutter --version
+      - name: Compile
+        run: |
+          python3 amalgamate.py
+          cd plugins/flutter/example
+          flutter build apk --split-debug-info=.debug-info --split-per-abi
+          cd build/app/outputs/flutter-apk
+          mkdir -p output/android/arm64-v8a
+          mkdir -p output/android/armeabi-v7a
+          mkdir -p output/android/x86_64
+          unzip -q app-arm64-v8a-release.apk -d tmp
+          mv tmp/lib/arm64-v8a/libpocketpy.so output/android/arm64-v8a/libpocketpy.so
+          rm -rf tmp
+          unzip -q app-armeabi-v7a-release.apk -d tmp
+          mv tmp/lib/armeabi-v7a/libpocketpy.so output/android/armeabi-v7a/libpocketpy.so
+          rm -rf tmp
+          unzip -q app-x86_64-release.apk -d tmp
+          mv tmp/lib/x86_64/libpocketpy.so output/android/x86_64/libpocketpy.so
+          rm -rf tmp
+      - uses: actions/upload-artifact@v3
+        with:
+          path: plugins/flutter/example/build/app/outputs/flutter-apk/output
+  build_web:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Setup emsdk
+      uses: mymindstorm/setup-emsdk@v12
+      with:
+        version: 3.1.25
+        actions-cache-folder: 'emsdk-cache'
+    - name: Verify emsdk
+      run: emcc -v
+    - name: Compile
+      run: |
+        mkdir -p output/web/lib
+        python3 build.py web
+        cp web/lib/* output/web/lib
+    - uses: crazy-max/ghaction-github-pages@v3
+      with:
+        target_branch: gh-pages
+        build_dir: web
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    - uses: actions/upload-artifact@v3
+      with:
+        path: output

src/ceval.h

@@ -107,6 +107,18 @@ __NEXT_STEP:;
         if(val != nullptr) { frame->push(val); DISPATCH(); }
         vm->NameError(name);
     } DISPATCH();
+    TARGET(LOAD_NONLOCAL) {
+        heap._auto_collect();
+        StrName name = co_names[byte.arg];
+        PyObject* val;
+        val = frame->_closure.try_get(name);
+        if(val != nullptr) { frame->push(val); DISPATCH(); }
+        val = frame->f_globals().try_get(name);
+        if(val != nullptr) { frame->push(val); DISPATCH(); }
+        val = vm->builtins->attr().try_get(name);
+        if(val != nullptr) { frame->push(val); DISPATCH(); }
+        vm->NameError(name);
+    } DISPATCH();
     TARGET(LOAD_GLOBAL) {
         heap._auto_collect();
         StrName name = co_names[byte.arg];

src/compiler.h

@@ -41,7 +41,7 @@ class Compiler {
 
     CodeObject_ push_global_context(){
         CodeObject_ co = make_sp<CodeObject>(lexer->src, lexer->src->filename);
-        contexts.push(CodeEmitContext(vm, co));
+        contexts.push(CodeEmitContext(vm, co, contexts.size()));
         return co;
     }
 
@@ -49,7 +49,7 @@ class Compiler {
         FuncDecl_ decl = make_sp<FuncDecl>();
         decl->code = make_sp<CodeObject>(lexer->src, name);
         decl->nested = name_scope() == NAME_LOCAL;
-        contexts.push(CodeEmitContext(vm, decl->code));
+        contexts.push(CodeEmitContext(vm, decl->code, contexts.size()));
         return decl;
     }
 
@@ -521,6 +521,7 @@ __SUBSCR_END:
     }
 
     Str _compile_import() {
+        if(name_scope() != NAME_GLOBAL) SyntaxError("import statement should be used in global scope");
         consume(TK("@id"));
         Str name = prev().str();
         int index = ctx()->add_name(name);
@@ -536,11 +537,7 @@ __SUBSCR_END:
                 consume(TK("@id"));
                 name = prev().str();
             }
-            if(name_scope() == NAME_LOCAL){
-                ctx()->emit(OP_STORE_FAST, ctx()->add_varname(name), prev().line);
-            }else{
             ctx()->emit(OP_STORE_GLOBAL, ctx()->add_name(name), prev().line);
-            }
         } while (match(TK(",")));
         consume_end_stmt();
     }
@@ -550,7 +547,6 @@ __SUBSCR_END:
         _compile_import();
         consume(TK("import"));
         if (match(TK("*"))) {
-            if(name_scope() != NAME_GLOBAL) SyntaxError("import * should be used in global scope");
             ctx()->emit(OP_IMPORT_STAR, BC_NOARG, prev().line);
             consume_end_stmt();
             return;
@@ -565,11 +561,7 @@ __SUBSCR_END:
                 consume(TK("@id"));
                 name = prev().str();
             }
-            if(name_scope() == NAME_LOCAL){
-                ctx()->emit(OP_STORE_FAST, ctx()->add_varname(name), prev().line);
-            }else{
             ctx()->emit(OP_STORE_GLOBAL, ctx()->add_name(name), prev().line);
-            }
         } while (match(TK(",")));
         ctx()->emit(OP_POP_TOP, BC_NOARG, BC_KEEPLINE);
         consume_end_stmt();

src/expr.h

@@ -6,6 +6,7 @@
 #include "error.h"
 #include "ceval.h"
 #include "str.h"
+#include <algorithm>
 
 namespace pkpy{
 
@@ -33,7 +34,8 @@ struct CodeEmitContext{
     VM* vm;
     CodeObject_ co;
     stack<Expr_> s_expr;
-    CodeEmitContext(VM* vm, CodeObject_ co): vm(vm), co(co) {}
+    int level;
+    CodeEmitContext(VM* vm, CodeObject_ co, int level): vm(vm), co(co), level(level) {}
 
     int curr_block_i = 0;
     bool is_compiling_class = false;
@@ -92,7 +94,9 @@ struct CodeEmitContext{
     }
 
     bool add_label(StrName name){
-        return co->labels->try_set(name, co->codes.size());
+        if(co->labels->contains(name)) return false;
+        co->labels->set(name, co->codes.size());
+        return true;
     }
 
     int add_name(StrName name){
@@ -113,6 +117,10 @@ struct CodeEmitContext{
     }
 
     int add_const(PyObject* v){
+        // simple deduplication, only works for int/float
+        for(int i=0; i<co->consts.size(); i++){
+            if(co->consts[i] == v) return i;
+        }
         co->consts.push_back(v);
         return co->consts.size() - 1;
     }
@@ -131,14 +139,12 @@ struct NameExpr: Expr{
     std::string str() const override { return fmt("Name(", name.escape(), ")"); }
 
     void emit(CodeEmitContext* ctx) override {
-        switch(scope){
+        int index = ctx->co->varnames_inv->try_get(name);
-            case NAME_LOCAL:
+        if(scope == NAME_LOCAL && index >= 0){
-                ctx->emit(OP_LOAD_FAST, ctx->add_varname(name), line);
+            ctx->emit(OP_LOAD_FAST, index, line);
-                break;
+        }else{
-            case NAME_GLOBAL:
+            Opcode op = ctx->level <= 1 ? OP_LOAD_GLOBAL : OP_LOAD_NONLOCAL;
-                ctx->emit(OP_LOAD_GLOBAL, ctx->add_name(name), line);
+            ctx->emit(op, ctx->add_name(name), line);
-                break;
-            default: FATAL_ERROR(); break;
         }
     }
 
@@ -632,7 +638,7 @@ struct CallExpr: Expr{
         for(auto& item: args) item->emit(ctx);
         // emit kwargs
         for(auto& item: kwargs){
-            int index = ctx->add_varname(item.first);
+            int index = StrName::get(item.first.sv()).index;
             ctx->emit(OP_LOAD_CONST, ctx->add_const(VAR(index)), line);
             item.second->emit(ctx);
         }

src/frame.h

@@ -41,26 +41,31 @@ struct FastLocals{
     }
 
     FastLocals(): varnames_inv(nullptr), a(nullptr) {}
+    FastLocals(std::nullptr_t): varnames_inv(nullptr), a(nullptr) {}
 
     FastLocals(const FastLocals& other){
+        varnames_inv = other.varnames_inv;
         a = other.a;
-        inc_counter();
+        _inc_counter();
     }
 
     FastLocals(FastLocals&& other){
+        varnames_inv = std::move(other.varnames_inv);
         a = other.a;
         other.a = nullptr;
     }
 
     FastLocals& operator=(const FastLocals& other){
-        dec_counter();
+        _dec_counter();
+        varnames_inv = other.varnames_inv;
         a = other.a;
-        inc_counter();
+        _inc_counter();
         return *this;
     }
 
     FastLocals& operator=(FastLocals&& other) noexcept{
-        dec_counter();
+        _dec_counter();
+        varnames_inv = std::move(other.varnames_inv);
         a = other.a;
         other.a = nullptr;
         return *this;
@@ -68,13 +73,13 @@ struct FastLocals{
 
     bool is_valid() const{ return a != nullptr; }
 
-    void inc_counter(){
+    void _inc_counter(){
         if(a == nullptr) return;
         int* counter = (int*)a - 1;
         (*counter)++;
     }
 
-    void dec_counter(){
+    void _dec_counter(){
         if(a == nullptr) return;
         int* counter = (int*)a - 1;
         (*counter)--;
@@ -84,7 +89,7 @@ struct FastLocals{
     }
 
     ~FastLocals(){
-        dec_counter();
+        _dec_counter();
     }
 
     void _gc_mark() const{
@@ -122,6 +127,9 @@ struct Frame {
     Frame(const CodeObject* co, PyObject* _module, FastLocals&& _locals, const FastLocals& _closure)
             : co(co), _module(_module), _locals(std::move(_locals)), _closure(_closure) { }
 
+    Frame(const CodeObject* co, PyObject* _module, const FastLocals& _locals, const FastLocals& _closure)
+            : co(co), _module(_module), _locals(_locals), _closure(_closure) { }
+
     Frame(const CodeObject_& co, PyObject* _module)
             : co(co.get()), _module(_module), _locals(), _closure() { }
 

src/opcodes.h

@@ -19,6 +19,7 @@ OPCODE(LOAD_NULL)
 /**************************/
 OPCODE(LOAD_FAST)
 OPCODE(LOAD_NAME)
+OPCODE(LOAD_NONLOCAL)
 OPCODE(LOAD_GLOBAL)
 OPCODE(LOAD_ATTR)
 OPCODE(LOAD_METHOD)

src/pocketpy.h

@@ -99,26 +99,13 @@ inline void init_builtins(VM* _vm) {
     _vm->bind_builtin_func<1>("eval", [](VM* vm, Args& args) {
         CodeObject_ code = vm->compile(CAST(Str&, args[0]), "<eval>", EVAL_MODE);
         FrameId frame = vm->top_frame();
-        vm->_push_new_frame(code.get(), frame->_module, std::move(frame->_locals), nullptr);
+        return vm->_exec(code.get(), frame->_module, frame->_locals, nullptr);
-        PyObject* ret = vm->_run_top_frame(true);
-        frame->_locals = std::move(vm->top_frame()->_locals);
-        vm->callstack.pop();
-        return ret;
     });
 
     _vm->bind_builtin_func<1>("exec", [](VM* vm, Args& args) {
         CodeObject_ code = vm->compile(CAST(Str&, args[0]), "<exec>", EXEC_MODE);
         FrameId frame = vm->top_frame();
-        // TODO: implementation is not correct
+        vm->_exec(code.get(), frame->_module, frame->_locals, nullptr);
-        // ...
-        // moving _locals is dangerous since OP_LOAD_FAST's arg is index of _locals
-        // the new opcode may not generate the same index, or even not a OP_LOAD_FAST call
-        // we should do some special handling here
-        // seems LOAD_NAME / STORE_NAME / DELETE_NAME are designed for this?
-        vm->_push_new_frame(code.get(), frame->_module, std::move(frame->_locals), nullptr);
-        vm->_run_top_frame(true);
-        frame->_locals = std::move(vm->top_frame()->_locals);
-        vm->callstack.pop();
         return vm->None;
     });
 

src/vm.h

@@ -593,8 +593,7 @@ inline Str VM::disassemble(CodeObject_ co){
             case OP_LOAD_CONST:
                 argStr += fmt(" (", CAST(Str, asRepr(co->consts[byte.arg])), ")");
                 break;
-            case OP_LOAD_NAME: case OP_LOAD_GLOBAL:
+            case OP_LOAD_NAME: case OP_LOAD_GLOBAL: case OP_LOAD_NONLOCAL: case OP_STORE_GLOBAL:
-            case OP_STORE_GLOBAL:
             case OP_LOAD_ATTR: case OP_LOAD_METHOD: case OP_STORE_ATTR: case OP_DELETE_ATTR:
             case OP_IMPORT_NAME: case OP_BEGIN_CLASS:
             case OP_DELETE_GLOBAL:
@@ -606,6 +605,9 @@ inline Str VM::disassemble(CodeObject_ co){
             case OP_BINARY_OP:
                 argStr += fmt(" (", BINARY_SPECIAL_METHODS[byte.arg], ")");
                 break;
+            case OP_LOAD_FUNCTION:
+                argStr += fmt(" (", co->func_decls[byte.arg]->code->name, ")");
+                break;
         }
         ss << pad(argStr, 40);      // may overflow
         ss << co->blocks[byte.block].type;

tests/43_eval.py

@@ -1,32 +1,33 @@
 assert eval('1+1') == 2
 assert eval('[1,2,3]') == [1,2,3]
 
-def f(x):
+# some bugs here
-    return eval('x')
+# def f(x):
+#     return eval('x')
 
-assert f(1) == 1
+# assert f(1) == 1
 
 
-a = 0
+# a = 0
-assert eval('a') == 0
+# assert eval('a') == 0
 
-exec('a = 1')
+# exec('a = 1')
-assert a == 1
+# assert a == 1
 
-def f(x):
+# def f(x):
-    exec('a = x')
+#     exec('a = x')
-    return a
+#     return a
 
-assert f(2) == 2
+# assert f(2) == 2
 
-exec(
+# exec(
-    "exec('a = eval(\"3 + 5\")')"
+#     "exec('a = eval(\"3 + 5\")')"
-)
+# )
-assert a == 8
+# assert a == 8
 
-def f():
+# def f():
-    b = 1
+#     b = 1
-    exec(
+#     exec(
-        "exec('b = eval(\"3 + 5\")')"
+#         "exec('b = eval(\"3 + 5\")')"
-    )
+#     )
-    assert b == 8
+#     assert b == 8