cjson cleanup

.github/workflows/main.yml

@@ -20,14 +20,9 @@ jobs:
       shell: bash
       run: |
         mkdir -p output/windows/x86_64
-        mkdir build
+        python3 cmake_build.py
-        cd build
+        cp main.exe output/windows/x86_64
-        cmake ..
+        cp pocketpy.dll output/windows/x86_64
-        cmake --build . --config Release
-        cp Release/main.exe ../output/windows/x86_64
-        cp Release/pocketpy.dll ../output/windows/x86_64
-        cp Release/main.exe ../
-        cp Release/pocketpy.dll ../
     - uses: actions/upload-artifact@v3
       with:
         path: output
@@ -59,14 +54,9 @@ jobs:
         export CXX=clang++
         export CC=clang
         mkdir -p output/linux/x86_64
-        mkdir build
+        python3 cmake_build.py
-        cd build
+        cp main output/linux/x86_64
-        cmake .. -DPK_USE_BOX2D=ON
+        cp libpocketpy.so output/linux/x86_64
-        cmake --build . --config Release
-        cp main ../output/linux/x86_64
-        cp libpocketpy.so ../output/linux/x86_64
-        cp main ../
-        cp libpocketpy.so ../
     - uses: actions/upload-artifact@v3
       with:
         path: output
@@ -87,13 +77,7 @@ jobs:
       - name: Build and Test
         run: |
           uname -m
-          mkdir build
+          python3 cmake_build.py
-          cd build
-          cmake ..
-          cmake --build . --config Release
-          cp main ../
-          cp libpocketpy.so ../
-          cd ..
           python3 scripts/run_tests.py
           python3 scripts/run_tests.py benchmark
         shell: alpine.sh {0}
@@ -103,13 +87,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Compile and Test
         run: |
-          mkdir build
+          python3 cmake_build.py
-          cd build
-          cmake ..
-          cmake --build . --config Release
-          cp main ../
-          cp libpocketpy.dylib ../
-          cd ..
           python3 scripts/run_tests.py
       - name: Benchmark
         run: python3 scripts/run_tests.py benchmark

.gitignore

@@ -36,7 +36,4 @@ pypi/
 libpocketpy.dylib
 
 .xmake/
-tests/res/WorldMap_Free_layout.ldtk
-tests/res/WorldMap_GridVania_layout.ldtk
 
-*.gcda

3rd/box2d/CMakeLists.txt

@@ -14,7 +14,7 @@ aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/src/rope BOX2D_SRC_3)
 
 aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/src BOX2D_BINDINGS_SRC)
 
-set(CMAKE_CXX_FLAGS_RELEASE "-O2")
+set(CMAKE_CXX_FLAGS "-O2")
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 add_library(

3rd/cjson/CMakeLists.txt

@@ -8,8 +8,8 @@ set(CMAKE_CXX_STANDARD 17)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/include)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/../pocketpy/include)
 
-set(CMAKE_C_FLAGS_RELEASE "-O2")
+set(CMAKE_C_FLAGS "-O2")
-set(CMAKE_CXX_FLAGS_RELEASE "-O2")
+set(CMAKE_CXX_FLAGS "-O2")
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 add_library(

CMakeLists.txt

@@ -34,9 +34,13 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}/include)
 aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/src POCKETPY_SRC)
 
 option(PK_USE_BOX2D "Use Box2D" OFF)
-option(PK_USE_CJSON "Use cJSON" ON)
+option(PK_USE_CJSON "Use cJSON" OFF)
 option(PK_USE_DYLIB "Use dylib" OFF)
 
+if(PK_USE_DYLIB)
+    add_definitions(-DPK_USE_DYLIB)
+endif()
+
 if(PK_USE_BOX2D)
     add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/3rd/box2d)
     include_directories(${CMAKE_CURRENT_LIST_DIR}/3rd/box2d/include)
@@ -61,8 +65,6 @@ else()
   option(PK_BUILD_STATIC_LIB "Build static library" ON)
 endif()
 
-option(PK_EXPORT_CXX_SYMBOLS "Export C++ symbols" OFF)
-
 if(PK_BUILD_SHARED_LIB)
     add_library(${PROJECT_NAME} SHARED ${POCKETPY_SRC})
 elseif(PK_BUILD_STATIC_LIB)
@@ -76,7 +78,6 @@ else()
 endif()
 
 if(PK_USE_DYLIB)
-    add_definitions(-DPK_USE_DYLIB)
     target_link_libraries(${PROJECT_NAME} ${CMAKE_DL_LIBS})
 endif()
 
@@ -84,16 +85,6 @@ if(PK_USE_BOX2D)
     target_link_libraries(${PROJECT_NAME} box2d)
 endif()
 
-target_link_libraries(${PROJECT_NAME} cjson)
+if(PK_USE_CJSON)
-
+    target_link_libraries(${PROJECT_NAME} cjson)
-if(PK_EXPORT_CXX_SYMBOLS AND MSVC)
-    set_target_properties(${PROJECT_NAME} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif()
-
-# enable link time optimization
-# include(CheckIPOSupported)
-# check_ipo_supported(RESULT result)
-# if(result)
-#     message(STATUS "LTO enabled")
-#     set_target_properties(${PROJECT_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE)
-# endif()

benchmarks/ldtk_cjson.py

@@ -0,0 +1,22 @@
+import sys
+is_pkpy = not hasattr(sys, 'getrefcount')
+
+if is_pkpy:
+    import cjson as json
+else:
+    import json
+
+_2489KB = 'WorldMap_GridVania_layout.ldtk'
+_1093KB = 'WorldMap_Free_layout.ldtk'
+_339KB = 'Typical_2D_platformer_example.ldtk'
+
+with open(f'res/{_2489KB}', 'r') as f:
+    json_content = f.read()
+
+data: dict = json.loads(json_content)
+assert isinstance(data, dict)
+
+# dumped: str = json.dumps(data)
+# loaded: dict = json.loads(dumped)
+# assert len(data) == len(loaded)
+# assert data == loaded

benchmarks/ldtk_json.py

@@ -1,22 +1,19 @@
-try:
+import json
-    import cjson as json
-    print('[INFO] cjson is used')
-except ImportError:
-    import json
 
 _2489KB = 'WorldMap_GridVania_layout.ldtk'
 _1093KB = 'WorldMap_Free_layout.ldtk'
 _339KB = 'Typical_2D_platformer_example.ldtk'
 
-with open(f'res/{_339KB}', 'r') as f:
+with open(f'res/{_2489KB}', 'r') as f:
     json_content = f.read()
 
 data: dict = json.loads(json_content)
 assert isinstance(data, dict)
 
-dumped: str = json.dumps(data)
+# dumped: str = json.dumps(data)
-loaded: dict = json.loads(dumped)
+# loaded: dict = json.loads(dumped)
-assert data == loaded
+# assert len(data) == len(loaded)
+# assert data == loaded
 
 # import pickle
 ##### very very slow!! DO NOT RUN IT

build.ps1

@@ -1,12 +0,0 @@
-if (Test-Path build) {
-    Remove-Item -Recurse -Force build
-}
-
-New-Item -ItemType Directory -Path build
-Push-Location build
-
-cmake ..
-cmake --build . --config Release
-
-Copy-Item "Release\main.exe" -Destination ".." # Note: NTFS uses backslash (\) instead of slashes (*nix, /) 
-Copy-Item "Release\pocketpy.dll" -Destination ".."

cmake_build.py

@@ -0,0 +1,23 @@
+import os
+import sys
+import shutil
+
+if not os.path.exists("build"):
+    os.mkdir("build")
+
+os.chdir("build")
+
+os.system(r"""
+cmake .. -DPK_USE_CJSON=ON -DPK_USE_BOX2D=ON
+cmake --build . --config Release
+""")
+
+if sys.platform == "win32":
+    shutil.copy("Release/main.exe", "../main.exe")
+    shutil.copy("Release/pocketpy.dll", "../pocketpy.dll")
+elif sys.platform == "darwin":
+    shutil.copy("main", "../main")
+    shutil.copy("libpocketpy.dylib", "../libpocketpy.dylib")
+else:
+    shutil.copy("main", "../main")
+    shutil.copy("libpocketpy.so", "../libpocketpy.so")

include/pocketpy/obj.h

@@ -144,6 +144,8 @@ struct PyObject{
     NameDict& attr() noexcept { return *_attr; }
     PyObject* attr(StrName name) const noexcept { return (*_attr)[name]; }
 
+    // PyObject* operator[](StrName name) const noexcept { return (*_attr)[name]; }
+
     virtual void _obj_gc_mark() = 0;
     virtual void* _value_ptr() = 0;
 

scripts/run_tests.py

@@ -2,9 +2,22 @@ import os
 import sys
 import time
 
+class WorkDir:
+    def __init__(self, next):
+        self.prev = os.getcwd()
+        self.next = next
+
+    def __enter__(self):
+        os.chdir(self.next)
+
+    def __exit__(self, *args, **kwargs):
+        os.chdir(self.prev)
+
 def test_file(filepath, cpython=False):
     if cpython:
-        return os.system("python3 " + filepath) == 0
+        x, y = os.path.split(filepath)
+        with WorkDir(x):
+            return os.system("python3 " + y) == 0
     if sys.platform == 'win32':
         return os.system("main.exe " + filepath) == 0
     else:

src/compiler.cpp

@@ -32,24 +32,27 @@ namespace pkpy{
         // however, this is buggy...since there may be a jump to the end (out of bound) even if the last opcode is a return
         ctx()->emit(OP_LOAD_NONE, BC_NOARG, BC_KEEPLINE);
         ctx()->emit(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE);
-        // ctx()->co->optimize(vm);
+        // some check here
+        std::vector<Bytecode>& codes = ctx()->co->codes;
         if(ctx()->co->varnames.size() > PK_MAX_CO_VARNAMES){
             SyntaxError("maximum number of local variables exceeded");
         }
-        FuncDecl_ func = contexts.top().func;
+        if(ctx()->co->consts.size() > 65535){
-        if(func){
+            // std::map<std::string, int> counts;
-            func->is_simple = true;
+            // for(PyObject* c: ctx()->co->consts){
-            if(func->code->is_generator) func->is_simple = false;
+            //     std::string key = obj_type_name(vm, vm->_tp(c)).str();
-            if(func->kwargs.size() > 0) func->is_simple = false;
+            //     counts[key] += 1;
-            if(func->starred_arg >= 0) func->is_simple = false;
+            // }
-            if(func->starred_kwarg >= 0) func->is_simple = false;
+            // for(auto pair: counts){
+            //     std::cout << pair.first << ": " << pair.second << std::endl;
+            // }
+            SyntaxError("maximum number of constants exceeded");
         }
-        // pre-compute LOOP_BREAK and LOOP_CONTINUE and FOR_ITER
-        std::vector<Bytecode>& codes = ctx()->co->codes;
         if(codes.size() > 65535 && ctx()->co->src->mode != JSON_MODE){
             // json mode does not contain jump instructions, so it is safe to ignore this check
-            SyntaxError("maximum number of opcodes exceeded, please split your code into smaller functions");
+            SyntaxError("maximum number of opcodes exceeded");
         }
+        // pre-compute LOOP_BREAK and LOOP_CONTINUE and FOR_ITER
         for(int i=0; i<codes.size(); i++){
             Bytecode& bc = codes[i];
             if(bc.op == OP_LOOP_CONTINUE){
@@ -60,6 +63,14 @@ namespace pkpy{
                 bc.arg = ctx()->co->_get_block_codei(i).end;
             }
         }
+        FuncDecl_ func = contexts.top().func;
+        if(func){
+            func->is_simple = true;
+            if(func->code->is_generator) func->is_simple = false;
+            if(func->kwargs.size() > 0) func->is_simple = false;
+            if(func->starred_arg >= 0) func->is_simple = false;
+            if(func->starred_kwarg >= 0) func->is_simple = false;
+        }
         contexts.pop();
     }
 

src/expr.cpp

@@ -1,5 +1,4 @@
 #include "pocketpy/expr.h"
-#include "pocketpy/codeobject.h"
 
 namespace pkpy{
 
@@ -88,6 +87,15 @@ namespace pkpy{
         for(int i=0; i<co->consts.size(); i++){
             if(co->consts[i] == v) return i;
         }
+        // string deduplication
+        if(is_non_tagged_type(v, vm->tp_str)){
+            const Str& v_str = PK_OBJ_GET(Str, v);
+            for(int i=0; i<co->consts.size(); i++){
+                if(is_non_tagged_type(co->consts[i], vm->tp_str)){
+                    if(PK_OBJ_GET(Str, co->consts[i]) == v_str) return i;
+                }
+            }
+        }
         co->consts.push_back(v);
         return co->consts.size() - 1;
     }

src/pocketpy.cpp

@@ -139,7 +139,7 @@ void init_builtins(VM* _vm) {
         vm->check_non_tagged_type(class_arg, vm->tp_type);
         Type type = PK_OBJ_GET(Type, class_arg);
         if(!vm->isinstance(self_arg, type)){
-            Str _0 = obj_type_name(vm, PK_OBJ_GET(Type, vm->_t(self_arg)));
+            Str _0 = obj_type_name(vm, vm->_tp(self_arg));
             Str _1 = obj_type_name(vm, type);
             vm->TypeError("super(): " + _0.escape() + " is not an instance of " + _1.escape());
         }
@@ -1331,7 +1331,11 @@ void init_builtins(VM* _vm) {
         self.apply([&](PyObject* k, PyObject* v){
             if(!first) ss << ", ";
             first = false;
-            Str key = CAST(Str&, k).escape(false);
+            if(!is_non_tagged_type(k, vm->tp_str)){
+                vm->TypeError(fmt("json keys must be string, got ", obj_type_name(vm, vm->_tp(k))));
+                UNREACHABLE();
+            }
+            Str key = _CAST(Str&, k).escape(false);
             Str value = CAST(Str&, vm->py_json(v));
             ss << key << ": " << value;
         });