add msgpack

Update CMakeLists.txt

Update CMakeLists.txt

Update CMakeLists.txt

.github/workflows/main.yml

@@ -36,7 +36,7 @@ jobs:
       shell: bash
       run: |
         mkdir -p output/x86_64
-        python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON
+        python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON -DPK_BUILD_MODULE_MSGPACK=ON
         cp main.exe output/x86_64
         cp pocketpy.dll output/x86_64
     - uses: actions/upload-artifact@v4
@@ -76,7 +76,7 @@ jobs:
       run: |
         python scripts/check_pragma_once.py include
         mkdir -p output/x86_64
-        python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON
+        python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON -DPK_BUILD_MODULE_MSGPACK=ON
         python scripts/run_tests.py
         cp main output/x86_64
         cp libpocketpy.so output/x86_64
@@ -96,7 +96,7 @@ jobs:
           submodules: recursive
       - name: Compile and Test
         run: |
-          python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON
+          python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON -DPK_BUILD_MODULE_MSGPACK=ON
           python scripts/run_tests.py
       - name: Benchmark
         run: python scripts/run_tests.py benchmark

3rd/msgpack/CMakeLists.txt

@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(msgpack)
+
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_C_STANDARD_REQUIRED ON)
+
+add_library(${PROJECT_NAME} STATIC
+    ${CMAKE_CURRENT_LIST_DIR}/src/mpack.c
+    ${CMAKE_CURRENT_LIST_DIR}/src/bindings.c
+)
+
+target_include_directories(${PROJECT_NAME} PRIVATE
+    ${CMAKE_CURRENT_LIST_DIR}/include
+)

3rd/msgpack/src/bindings.c

@@ -0,0 +1,114 @@
+#include "pocketpy.h"
+
+#include "mpack.h"
+#include <assert.h>
+
+static bool mpack_to_py(mpack_node_t node) {
+    py_StackRef tmp = py_pushtmp();
+
+    mpack_type_t type = mpack_node_type(node);
+
+    switch(type) {
+        case mpack_type_nil: py_newnone(tmp); break;
+
+        case mpack_type_bool: py_newbool(tmp, mpack_node_bool(node)); break;
+
+        case mpack_type_int: py_newint(tmp, mpack_node_i64(node)); break;
+
+        case mpack_type_uint: py_newint(tmp, (int64_t)mpack_node_u64(node)); break;
+
+        case mpack_type_float: py_newfloat(tmp, mpack_node_float(node)); break;
+
+        case mpack_type_double: py_newfloat(tmp, mpack_node_double(node)); break;
+
+        case mpack_type_str: {
+            const char* str = mpack_node_str(node);
+            size_t len = mpack_node_strlen(node);
+            c11_sv sv = {str, (int)len};
+            py_newstrv(tmp, sv);
+            break;
+        }
+
+        case mpack_type_bin: {
+            const char* data = mpack_node_bin_data(node);
+            size_t len = mpack_node_bin_size(node);
+            unsigned char* byte_data = py_newbytes(tmp, len);
+            memcpy(byte_data, data, len);
+            break;
+        }
+
+        case mpack_type_array: {
+            size_t count = mpack_node_array_length(node);
+            py_newlistn(tmp, count);
+            for(int i = 0; i < count; i++) {
+                mpack_node_t child = mpack_node_array_at(node, i);
+                bool ok = mpack_to_py(child);
+                if(!ok) return false;
+                py_list_setitem(tmp, i, py_peek(-1));
+                py_pop();
+            }
+            break;
+        }
+
+        case mpack_type_map: {
+            size_t count = mpack_node_map_count(node);
+            py_newdict(tmp);
+            for(size_t i = 0; i < count; i++) {
+                mpack_node_t key_node = mpack_node_map_key_at(node, i);
+                mpack_node_t val_node = mpack_node_map_value_at(node, i);
+                if(mpack_node_type(key_node) != mpack_type_str) {
+                    return TypeError("msgpack: key must be strings");
+                }
+                if(!mpack_to_py(key_node)) return false;
+                if(!mpack_to_py(val_node)) return false;
+                bool ok = py_dict_setitem(tmp, py_peek(-2), py_peek(-1));
+                if(!ok) return false;
+                py_shrink(2);
+            }
+            break;
+        }
+        default: return ValueError("msgpack: invalid node type");
+    }
+    return true;
+}
+
+static bool msgpack_loads(int argc, py_Ref argv) {
+    PY_CHECK_ARGC(1);
+    PY_CHECK_ARG_TYPE(0, tp_bytes);
+    int size;
+    unsigned char* data = py_tobytes(argv, &size);
+
+    mpack_tree_t tree;
+    mpack_tree_init_data(&tree, (const char*)data, size);
+    mpack_tree_parse(&tree);
+
+    if(mpack_tree_error(&tree) != mpack_ok) {
+        mpack_tree_destroy(&tree);
+        return ValueError("msgpack: parse error");
+    }
+
+    mpack_node_t node = mpack_tree_root(&tree);
+    bool ok = mpack_to_py(node);
+    mpack_tree_destroy(&tree);
+    if(!ok) return false;
+
+    py_assign(py_retval(), py_peek(-1));
+    py_pop();
+    return true;
+}
+
+// static mpack_node_t py_to_mpack(py_Ref object) {
+
+// }
+
+static bool msgpack_dumps(int argc, py_Ref argv) {
+    PY_CHECK_ARGC(1);
+    return ValueError("not yet");
+}
+
+void pk__add_module_msgpack() {
+    py_GlobalRef mod = py_newmodule("msgpack");
+
+    py_bindfunc(mod, "loads", msgpack_loads);
+    py_bindfunc(mod, "dumps", msgpack_dumps);
+}

CMakeLists.txt

@@ -115,6 +115,10 @@ if(PK_BUILD_MODULE_CUTE_PNG)
     add_definitions(-DPK_BUILD_MODULE_CUTE_PNG)
 endif()
 
+if(PK_BUILD_MODULE_MSGPACK)
+    add_subdirectory(3rd/msgpack)
+    add_definitions(-DPK_BUILD_MODULE_MSGPACK)
+endif()
 
 if(PK_BUILD_SHARED_LIB)
     message(">> Building shared library")
@@ -180,6 +184,10 @@ if(PK_BUILD_MODULE_CUTE_PNG)
     target_link_libraries(${PROJECT_NAME} cute_png)
 endif()
 
+if(PK_BUILD_MODULE_MSGPACK)
+    target_link_libraries(${PROJECT_NAME} msgpack)
+endif()
+
 
 if(PK_ENABLE_MIMALLOC)
     target_link_libraries(${PROJECT_NAME} mimalloc-static)

CMakeOptions.txt

@@ -16,6 +16,7 @@ option(PK_ENABLE_MIMALLOC "" OFF)
 # modules
 option(PK_BUILD_MODULE_LZ4 "" OFF)
 option(PK_BUILD_MODULE_CUTE_PNG "" OFF)
+option(PK_BUILD_MODULE_MSGPACK "" OFF)
 
 # PK_IS_MAIN determines whether the project is being used from root
 # or if it is added as a dependency (through add_subdirectory for example).

build_android_libs.sh

@@ -18,6 +18,7 @@ cmake \
     -DPK_ENABLE_OS=OFF \
     -DPK_ENABLE_DETERMINISM=ON \
     -DPK_BUILD_MODULE_LZ4=ON \
-    -DPK_BUILD_MODULE_CUTE_PNG=ON
+    -DPK_BUILD_MODULE_CUTE_PNG=ON \
+    -DPK_BUILD_MODULE_MSGPACK=ON
 
 cmake --build . --config Release

build_darwin_libs.sh

@@ -11,6 +11,8 @@ FLAGS="-DPK_BUILD_STATIC_LIB=ON \
     -DPK_ENABLE_DETERMINISM=ON \
     -DPK_BUILD_MODULE_LZ4=ON \
     -DPK_BUILD_MODULE_CUTE_PNG=ON \
+    -DPK_BUILD_MODULE_MSGPACK=ON \
+    -DCMAKE_BUILD_TYPE=Release \
     "
 
 cmake -G Xcode $FLAGS ..

build_ios_libs.sh

@@ -13,6 +13,8 @@ FLAGS="-DCMAKE_TOOLCHAIN_FILE=3rd/ios.toolchain.cmake \
     -DPK_ENABLE_DETERMINISM=ON \
     -DPK_BUILD_MODULE_LZ4=ON \
     -DPK_BUILD_MODULE_CUTE_PNG=ON \
+    -DPK_BUILD_MODULE_MSGPACK=ON \
+    -DCMAKE_BUILD_TYPE=Release \
     "
 
 cmake -B os64 -G Xcode $FLAGS -DPLATFORM=OS64 ..

compile_flags.txt

@@ -5,3 +5,4 @@
 -Iinclude/
 -I3rd/lz4/
 -I3rd/cute_headers/include/
+-I3rd/msgpack/include/

docs/modules/msgpack.md

@@ -0,0 +1,16 @@
+---
+icon: package
+label: msgpack
+---
+
+!!!
+This module is optional. Set option `PK_BUILD_MODULE_MSGPACK` to `ON` in your `CMakeLists.txt` to enable it.
+!!!
+
+### `msgpack.loads(data: bytes)`
+
+Decode a msgpack bytes into a python object.
+
+### `msgpack.dumps(obj) -> bytes`
+
+Encode a python object into a msgpack bytes.

include/pocketpy/interpreter/modules.h

@@ -32,3 +32,9 @@ void pk__add_module_cute_png();
 #else
 #define pk__add_module_cute_png()
 #endif
+
+#ifdef PK_BUILD_MODULE_MSGPACK
+void pk__add_module_msgpack();
+#else
+#define pk__add_module_msgpack()
+#endif

plugins/flutter/pocketpy/src/CMakeLists.txt

@@ -15,6 +15,7 @@ set(PK_ENABLE_OS OFF CACHE BOOL "" FORCE)
 set(PK_ENABLE_DETERMINISM ON CACHE BOOL "" FORCE)
 set(PK_BUILD_MODULE_LZ4 ON CACHE BOOL "" FORCE)
 set(PK_BUILD_MODULE_CUTE_PNG ON CACHE BOOL "" FORCE)
+set(PK_BUILD_MODULE_MSGPACK ON CACHE BOOL "" FORCE)
 set(PK_BUILD_SHARED_LIB ON CACHE BOOL "" FORCE)
 
 FetchContent_Declare(

src/interpreter/vm.c

@@ -263,6 +263,7 @@ void VM__ctor(VM* self) {
     pk__add_module_conio();
     pk__add_module_lz4();       // optional
     pk__add_module_cute_png();  // optional
+    pk__add_module_msgpack();   // optional
     pk__add_module_pkpy();
 
     // add python builtins