From 075b9c776c8ef1558045d8447e4287227bf3e479 Mon Sep 17 00:00:00 2001 From: Chukobyte Date: Sun, 26 Nov 2023 20:02:12 -0500 Subject: [PATCH 1/3] Added the ability to override the import handler in c. --- .gitignore | 3 +++ include/pocketpy/pocketpy_c.h | 2 ++ src/pocketpy_c.cpp | 10 ++++++++++ src2/pocketpy_c.c | 4 ++++ 4 files changed, 19 insertions(+) diff --git a/.gitignore b/.gitignore index 4f300593..103d0abc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__/ .ipynb_checkpoints .DS_Store .coverage +.idea src/main gmon.out @@ -35,5 +36,7 @@ main pypi/ libpocketpy.dylib +cmake-build-* + .xmake/ diff --git a/include/pocketpy/pocketpy_c.h b/include/pocketpy/pocketpy_c.h index 7ec9cfe9..a0318859 100644 --- a/include/pocketpy/pocketpy_c.h +++ b/include/pocketpy/pocketpy_c.h @@ -13,6 +13,7 @@ extern "C" { typedef struct pkpy_vm_handle pkpy_vm; typedef int (*pkpy_CFunction)(pkpy_vm*); typedef void (*pkpy_COutputHandler)(pkpy_vm*, const char*, int); +typedef const char* (*pkpy_CImportHandler)(const char*); typedef int pkpy_CName; typedef int pkpy_CType; @@ -95,6 +96,7 @@ PK_EXPORT pkpy_CName pkpy_name(const char* s); PK_EXPORT pkpy_CString pkpy_name_to_string(pkpy_CName name); PK_EXPORT void pkpy_compile_to_string(pkpy_vm*, const char* source, const char* filename, int mode, bool* ok, char** out); PK_EXPORT void pkpy_set_output_handler(pkpy_vm*, pkpy_COutputHandler handler); +PK_EXPORT void pkpy_set_import_handler(pkpy_vm*, pkpy_CImportHandler handler); /* REPL */ PK_EXPORT void* pkpy_new_repl(pkpy_vm*); diff --git a/src/pocketpy_c.cpp b/src/pocketpy_c.cpp index dfa5ac40..2fde22e1 100644 --- a/src/pocketpy_c.cpp +++ b/src/pocketpy_c.cpp @@ -584,6 +584,16 @@ void pkpy_set_output_handler(pkpy_vm* vm_handle, pkpy_COutputHandler handler){ vm->_stdout = reinterpret_cast(handler); } +void pkpy_set_import_handler(pkpy_vm* vm_handle, pkpy_CImportHandler handler){ + VM* vm = (VM*) vm_handle; + static pkpy_CImportHandler activeHandler; + activeHandler = handler; + vm->_import_handler = [](const Str& str) -> pkpy::Bytes { + const char* returnString = activeHandler(str.c_str()); + return { Str(returnString) }; + }; +} + void* pkpy_new_repl(pkpy_vm* vm_handle){ return new REPL((VM*)vm_handle); } diff --git a/src2/pocketpy_c.c b/src2/pocketpy_c.c index 5892fe42..0d61dcd6 100644 --- a/src2/pocketpy_c.c +++ b/src2/pocketpy_c.c @@ -246,6 +246,10 @@ void pkpy_set_output_handler(pkpy_vm* vm, pkpy_COutputHandler handler) { } +void pkpy_set_import_handler(pkpy_vm* vm, pkpy_CImportHandler handler) { + +} + void* pkpy_new_repl(pkpy_vm* vm) { void* returnValue; return returnValue; From e00ddb86928a2631eb91851e5eb8412787915d6c Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Tue, 28 Nov 2023 23:16:32 +0800 Subject: [PATCH 2/3] some refactor Update vm.cpp --- .gitignore | 2 -- docs/quick-start/modules.md | 18 ------------------ include/pocketpy/io.h | 2 +- include/pocketpy/obj.h | 7 ++++++- include/pocketpy/pocketpy_c.h | 2 +- include/pocketpy/vm.h | 2 +- src/io.cpp | 18 +++++++++--------- src/pocketpy_c.cpp | 7 +------ src/vm.cpp | 21 +++++++++++++-------- 9 files changed, 32 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 103d0abc..1e8cf64a 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,5 @@ main pypi/ libpocketpy.dylib -cmake-build-* - .xmake/ diff --git a/docs/quick-start/modules.md b/docs/quick-start/modules.md index f30f53b2..396c0a67 100644 --- a/docs/quick-start/modules.md +++ b/docs/quick-start/modules.md @@ -60,24 +60,6 @@ When you do `import` a module, the VM will try to find it in the following order ### Customized import handler You can use `vm->_import_handler` to provide a custom import handler for the 3rd step. -if both `enable_os` and `PK_ENABLE_OS` are `true`, the default `import_handler` is as follows: - -```cpp -inline Bytes _default_import_handler(const Str& name){ - std::filesystem::path path(name.sv()); - bool exists = std::filesystem::exists(path); - if(!exists) return Bytes(); - std::string cname = name.str(); - FILE* fp = fopen(cname.c_str(), "rb"); - if(!fp) return Bytes(); - fseek(fp, 0, SEEK_END); - std::vector buffer(ftell(fp)); - fseek(fp, 0, SEEK_SET); - fread(buffer.data(), 1, buffer.size(), fp); - fclose(fp); - return Bytes(std::move(buffer)); -}; -``` ### Import module via cpp diff --git a/include/pocketpy/io.h b/include/pocketpy/io.h index 76806cad..f4fd5b0c 100644 --- a/include/pocketpy/io.h +++ b/include/pocketpy/io.h @@ -3,7 +3,7 @@ #include "cffi.h" namespace pkpy{ - Bytes _default_import_handler(const Str& name); + unsigned char* _default_import_handler(const char*, int, int*); void add_module_os(VM* vm); void add_module_io(VM* vm); } diff --git a/include/pocketpy/obj.h b/include/pocketpy/obj.h index 442d26d6..e6460d2e 100644 --- a/include/pocketpy/obj.h +++ b/include/pocketpy/obj.h @@ -74,7 +74,12 @@ struct Bytes{ Bytes& operator=(const Bytes& rhs) = delete; std::pair detach() noexcept; - ~Bytes(){ delete[] _data;} + ~Bytes(){ + if(_data != nullptr){ + free(_data); + _data = nullptr; + } + } }; using Super = std::pair; diff --git a/include/pocketpy/pocketpy_c.h b/include/pocketpy/pocketpy_c.h index a0318859..ce61dc65 100644 --- a/include/pocketpy/pocketpy_c.h +++ b/include/pocketpy/pocketpy_c.h @@ -13,7 +13,7 @@ extern "C" { typedef struct pkpy_vm_handle pkpy_vm; typedef int (*pkpy_CFunction)(pkpy_vm*); typedef void (*pkpy_COutputHandler)(pkpy_vm*, const char*, int); -typedef const char* (*pkpy_CImportHandler)(const char*); +typedef unsigned char* (*pkpy_CImportHandler)(const char*, int, int*); typedef int pkpy_CName; typedef int pkpy_CType; diff --git a/include/pocketpy/vm.h b/include/pocketpy/vm.h index c98760e7..f453b441 100644 --- a/include/pocketpy/vm.h +++ b/include/pocketpy/vm.h @@ -142,7 +142,7 @@ public: PrintFunc _stdout; PrintFunc _stderr; - Bytes (*_import_handler)(const Str& name); + unsigned char* (*_import_handler)(const char*, int, int*); // for quick access Type tp_object, tp_type, tp_int, tp_float, tp_bool, tp_str; diff --git a/src/io.cpp b/src/io.cpp index f0630f6e..b9dd3146 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -23,14 +23,13 @@ static size_t io_fread(void* buffer, size_t size, size_t count, FILE* fp){ } -Bytes _default_import_handler(const Str& name){ +unsigned char* _default_import_handler(const char* name_p, int name_size, int* out_size){ #if PK_ENABLE_OS - std::filesystem::path path(name.sv()); - bool exists = std::filesystem::exists(path); - if(!exists) return Bytes(); - std::string cname = name.str(); - FILE* fp = io_fopen(cname.c_str(), "rb"); - if(!fp) return Bytes(); + std::string name(name_p, name_size); + bool exists = std::filesystem::exists(std::filesystem::path(name)); + if(!exists) return nullptr; + FILE* fp = io_fopen(name.c_str(), "rb"); + if(!fp) return nullptr; fseek(fp, 0, SEEK_END); int buffer_size = ftell(fp); unsigned char* buffer = new unsigned char[buffer_size]; @@ -38,9 +37,10 @@ Bytes _default_import_handler(const Str& name){ size_t sz = io_fread(buffer, 1, buffer_size, fp); PK_UNUSED(sz); fclose(fp); - return Bytes(buffer, buffer_size); + *out_size = buffer_size; + return buffer; #else - return Bytes(); + return nullptr; #endif }; diff --git a/src/pocketpy_c.cpp b/src/pocketpy_c.cpp index 2fde22e1..aaa57a8a 100644 --- a/src/pocketpy_c.cpp +++ b/src/pocketpy_c.cpp @@ -586,12 +586,7 @@ void pkpy_set_output_handler(pkpy_vm* vm_handle, pkpy_COutputHandler handler){ void pkpy_set_import_handler(pkpy_vm* vm_handle, pkpy_CImportHandler handler){ VM* vm = (VM*) vm_handle; - static pkpy_CImportHandler activeHandler; - activeHandler = handler; - vm->_import_handler = [](const Str& str) -> pkpy::Bytes { - const char* returnString = activeHandler(str.c_str()); - return { Str(returnString) }; - }; + vm->_import_handler = handler; } void* pkpy_new_repl(pkpy_vm* vm_handle){ diff --git a/src/vm.cpp b/src/vm.cpp index be92d13d..6ac9b874 100644 --- a/src/vm.cpp +++ b/src/vm.cpp @@ -82,9 +82,11 @@ namespace pkpy{ callstack.reserve(8); _main = nullptr; _last_exception = nullptr; - _import_handler = [](const Str& name) { - PK_UNUSED(name); - return Bytes(); + _import_handler = [](const char* name_p, int name_size, int* out_size) -> unsigned char*{ + PK_UNUSED(name_p); + PK_UNUSED(name_size); + PK_UNUSED(out_size); + return nullptr; }; init_builtin_types(); } @@ -335,17 +337,20 @@ namespace pkpy{ bool is_init = false; auto it = _lazy_modules.find(name); if(it == _lazy_modules.end()){ - Bytes b = _import_handler(filename); - if(!b){ + int out_size; + unsigned char* out = _import_handler(filename.data, filename.size, &out_size); + if(out == nullptr){ filename = path.replace('.', kPlatformSep).str() + kPlatformSep + "__init__.py"; is_init = true; - b = _import_handler(filename); + out = _import_handler(filename.data, filename.size, &out_size); } - if(!b){ + if(out == nullptr){ if(throw_err) ImportError(fmt("module ", path.escape(), " not found")); else return nullptr; } - source = Str(b.str()); + PK_ASSERT(out_size >= 0) + source = Str(std::string_view((char*)out, out_size)); + free(out); }else{ source = it->second; _lazy_modules.erase(it); From b8b42a84144c43e157c2629dd19167066bb14a6e Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Tue, 28 Nov 2023 23:48:42 +0800 Subject: [PATCH 3/3] some fix --- include/pocketpy/obj.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/include/pocketpy/obj.h b/include/pocketpy/obj.h index e6460d2e..442d26d6 100644 --- a/include/pocketpy/obj.h +++ b/include/pocketpy/obj.h @@ -74,12 +74,7 @@ struct Bytes{ Bytes& operator=(const Bytes& rhs) = delete; std::pair detach() noexcept; - ~Bytes(){ - if(_data != nullptr){ - free(_data); - _data = nullptr; - } - } + ~Bytes(){ delete[] _data;} }; using Super = std::pair;