diff --git a/.gitignore b/.gitignore index 56dfad56..a59fc57c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ docs/C-API/functions.md cmake-build-* tmp/ profiler_report.json + +tmp/ diff --git a/docs/retype.yml b/docs/retype.yml index 95083190..d2613367 100644 --- a/docs/retype.yml +++ b/docs/retype.yml @@ -3,7 +3,7 @@ output: .retype url: https://pocketpy.dev branding: title: pocketpy - label: v2.1.6 + label: v2.1.7 logo: "./static/logo.png" favicon: "./static/logo.png" meta: diff --git a/include/pocketpy/common/serialize.h b/include/pocketpy/common/serialize.h index 6888893a..7250724d 100644 --- a/include/pocketpy/common/serialize.h +++ b/include/pocketpy/common/serialize.h @@ -42,8 +42,6 @@ void* c11_deserializer__read_bytes(c11_deserializer* self, int size); return *p; \ } - -DEF_ATOMIC_INLINE_RW(size, int) DEF_ATOMIC_INLINE_RW(i8, int8_t) DEF_ATOMIC_INLINE_RW(i16, int16_t) DEF_ATOMIC_INLINE_RW(i32, int32_t) @@ -53,3 +51,4 @@ DEF_ATOMIC_INLINE_RW(f64, double) DEF_ATOMIC_INLINE_RW(type, py_Type) #undef DEF_ATOMIC_INLINE_RW + diff --git a/include/pocketpy/common/vector.h b/include/pocketpy/common/vector.h index 7f4e1808..0d556003 100644 --- a/include/pocketpy/common/vector.h +++ b/include/pocketpy/common/vector.h @@ -22,6 +22,7 @@ bool c11_vector__contains(const c11_vector* self, void* elem); void* c11_vector__submit(c11_vector* self, int* length); void c11_vector__swap(c11_vector* self, c11_vector* other); int c11_vector__nextcap(c11_vector* self); +void c11_vector__extend(c11_vector* self, const void* p, int size); #define c11__getitem(T, self, index) (((T*)(self)->data)[index]) #define c11__setitem(T, self, index, value) ((T*)(self)->data)[index] = value; @@ -40,17 +41,6 @@ int c11_vector__nextcap(c11_vector* self); #define c11_vector__back(T, self) (((T*)(self)->data)[(self)->length - 1]) -#define c11_vector__extend(T, self, p, size) \ - do { \ - int min_capacity = (self)->length + (size); \ - if((self)->capacity < min_capacity) { \ - int nextcap = c11_vector__nextcap(self); \ - c11_vector__reserve((self), c11__max(nextcap, min_capacity)); \ - } \ - memcpy((T*)(self)->data + (self)->length, (p), (size) * sizeof(T)); \ - (self)->length += (size); \ - } while(0) - #define c11_vector__insert(T, self, index, elem) \ do { \ if((self)->length == (self)->capacity) { \ diff --git a/include/pocketpy/config.h b/include/pocketpy/config.h index 2fc917e5..8f5d42ea 100644 --- a/include/pocketpy/config.h +++ b/include/pocketpy/config.h @@ -1,10 +1,10 @@ #pragma once // clang-format off -#define PK_VERSION "2.1.6" +#define PK_VERSION "2.1.7" #define PK_VERSION_MAJOR 2 #define PK_VERSION_MINOR 1 -#define PK_VERSION_PATCH 6 +#define PK_VERSION_PATCH 7 /*************** feature settings ***************/ #ifndef PK_ENABLE_OS // can be overridden by cmake diff --git a/include/pocketpy/interpreter/modules.h b/include/pocketpy/interpreter/modules.h index ed7f33d5..6054cce3 100644 --- a/include/pocketpy/interpreter/modules.h +++ b/include/pocketpy/interpreter/modules.h @@ -18,6 +18,7 @@ void pk__add_module_pickle(); void pk__add_module_base64(); void pk__add_module_importlib(); void pk__add_module_unicodedata(); +void pk__add_module_py_compile(); void pk__add_module_stdc(); void pk__add_module_vmath(); diff --git a/include/pocketpy/interpreter/vm.h b/include/pocketpy/interpreter/vm.h index 599b5307..7b3512de 100644 --- a/include/pocketpy/interpreter/vm.h +++ b/include/pocketpy/interpreter/vm.h @@ -62,7 +62,6 @@ typedef struct VM { void* ctx; // user-defined context CachedNames cached_names; - NameDict compile_time_funcs; py_StackRef curr_class; py_StackRef curr_decl_based_function; // this is for get current function without frame diff --git a/include/pocketpy/objects/codeobject.h b/include/pocketpy/objects/codeobject.h index 7a8452f1..8eda0c1d 100644 --- a/include/pocketpy/objects/codeobject.h +++ b/include/pocketpy/objects/codeobject.h @@ -45,26 +45,26 @@ typedef enum Opcode { } Opcode; typedef struct Bytecode { - uint8_t op; + uint16_t op; uint16_t arg; } Bytecode; void Bytecode__set_signed_arg(Bytecode* self, int arg); bool Bytecode__is_forward_jump(const Bytecode* self); -typedef struct CodeBlock { - CodeBlockType type; - int parent; // parent index in blocks - int start; // start index of this block in codes, inclusive - int end; // end index of this block in codes, exclusive - int end2; // ... -} CodeBlock; - typedef struct BytecodeEx { - int lineno; // line number for each bytecode - int iblock; // block index + int32_t lineno; // line number for each bytecode + int32_t iblock; // block index } BytecodeEx; +typedef struct CodeBlock { + int32_t type; + int32_t parent; // parent index in blocks + int32_t start; // start index of this block in codes, inclusive + int32_t end; // end index of this block in codes, exclusive + int32_t end2; // ... +} CodeBlock; + typedef struct CodeObject { SourceData_ src; c11_string* name; @@ -74,8 +74,8 @@ typedef struct CodeObject { c11_vector /*T=py_TValue*/ consts; // constants c11_vector /*T=py_Name*/ varnames; // local variables - c11_vector /*T=py_Name*/ names; - int nlocals; + c11_vector /*T=py_Name*/ names; // non-local names + int nlocals; // number of local variables c11_smallmap_n2d varnames_inv; c11_smallmap_n2d names_inv; @@ -95,7 +95,7 @@ void CodeObject__gc_mark(const CodeObject* self, c11_vector* p_stack); // Serialization void* CodeObject__dumps(const CodeObject* co, int* size); -const char* CodeObject__loads(CodeObject* co, const void* data, int size); +char* CodeObject__loads(const void* data, int size, CodeObject* out); typedef struct FuncDeclKwArg { int index; // index in co->varnames @@ -114,7 +114,7 @@ typedef struct FuncDecl { int starred_kwarg; // index in co->varnames, -1 if no **kwarg bool nested; // whether this function is nested - const char* docstring; // docstring of this function (weak ref) + char* docstring; FuncType type; c11_smallmap_n2d kw_to_index; @@ -129,6 +129,7 @@ void FuncDecl__add_kwarg(FuncDecl* self, py_Name name, const py_TValue* value); void FuncDecl__add_starred_arg(FuncDecl* self, py_Name name); void FuncDecl__add_starred_kwarg(FuncDecl* self, py_Name name); void FuncDecl__gc_mark(const FuncDecl* self, c11_vector* p_stack); +void FuncDecl__dtor(FuncDecl* self); // runtime function typedef struct Function { diff --git a/include/pocketpy/pocketpy.h b/include/pocketpy/pocketpy.h index 6eac3fcf..87051c0a 100644 --- a/include/pocketpy/pocketpy.h +++ b/include/pocketpy/pocketpy.h @@ -77,8 +77,8 @@ typedef void (*py_TraceFunc)(py_Frame* frame, enum py_TraceEvent); /// A struct contains the callbacks of the VM. typedef struct py_Callbacks { - /// Used by `__import__` to load a source module. - char* (*importfile)(const char*); + /// Used by `__import__` to load a source or compiled module. + char* (*importfile)(const char* path, int* data_size); /// Called before `importfile` to lazy-import a C module. PY_MAYBENULL py_GlobalRef (*lazyimport)(const char*); /// Used by `print` to output a string. @@ -298,10 +298,6 @@ PK_API void py_bindproperty(py_Type type, const char* name, py_CFunction getter, py_CFunction setter); /// Bind a magic method to type. PK_API void py_bindmagic(py_Type type, py_Name name, py_CFunction f); -/// Bind a compile-time function via "decl-based" style. -PK_API void py_macrobind(const char* sig, py_CFunction f); -/// Get a compile-time function by name. -PK_API py_ItemRef py_macroget(py_Name name); /************* Value Cast *************/ diff --git a/include/typings/py_compile.pyi b/include/typings/py_compile.pyi new file mode 100644 index 00000000..85d5abba --- /dev/null +++ b/include/typings/py_compile.pyi @@ -0,0 +1 @@ +def compile(file: str, cfile: str) -> None: ... diff --git a/plugins/flutter/pocketpy/ios/pocketpy.podspec b/plugins/flutter/pocketpy/ios/pocketpy.podspec index e35ca07a..ce2eee34 100644 --- a/plugins/flutter/pocketpy/ios/pocketpy.podspec +++ b/plugins/flutter/pocketpy/ios/pocketpy.podspec @@ -35,7 +35,7 @@ A new Flutter FFI plugin project. s.prepare_command = <<-CMD rm -rf pocketpy - git clone --branch v2.1.6 --depth 1 https://github.com/pocketpy/pocketpy.git + git clone --branch v2.1.7 --depth 1 https://github.com/pocketpy/pocketpy.git cd pocketpy git submodule update --init --recursive --depth 1 bash build_ios_libs.sh diff --git a/plugins/flutter/pocketpy/macos/pocketpy.podspec b/plugins/flutter/pocketpy/macos/pocketpy.podspec index 00defef9..57b9087e 100644 --- a/plugins/flutter/pocketpy/macos/pocketpy.podspec +++ b/plugins/flutter/pocketpy/macos/pocketpy.podspec @@ -32,7 +32,7 @@ A new Flutter FFI plugin project. s.prepare_command = <<-CMD rm -rf pocketpy - git clone --branch v2.1.6 --depth 1 https://github.com/pocketpy/pocketpy.git + git clone --branch v2.1.7 --depth 1 https://github.com/pocketpy/pocketpy.git cd pocketpy git submodule update --init --recursive --depth 1 bash build_darwin_libs.sh diff --git a/plugins/flutter/pocketpy/pubspec.yaml b/plugins/flutter/pocketpy/pubspec.yaml index 575994b3..72f6058d 100644 --- a/plugins/flutter/pocketpy/pubspec.yaml +++ b/plugins/flutter/pocketpy/pubspec.yaml @@ -1,6 +1,6 @@ name: pocketpy description: A lightweight Python interpreter for game engines. It supports Android/iOS/Windows/Linux/MacOS. -version: 2.1.6 +version: 2.1.7 homepage: https://pocketpy.dev repository: https://github.com/pocketpy/pocketpy diff --git a/plugins/flutter/pocketpy/src/CMakeLists.txt b/plugins/flutter/pocketpy/src/CMakeLists.txt index aa673595..31fb767f 100644 --- a/plugins/flutter/pocketpy/src/CMakeLists.txt +++ b/plugins/flutter/pocketpy/src/CMakeLists.txt @@ -21,7 +21,7 @@ set(PK_BUILD_SHARED_LIB ON CACHE BOOL "" FORCE) FetchContent_Declare( pocketpy GIT_REPOSITORY https://github.com/pocketpy/pocketpy.git - GIT_TAG v2.1.6 + GIT_TAG v2.1.7 ) FetchContent_MakeAvailable(pocketpy) diff --git a/src/common/chunkedvector.c b/src/common/chunkedvector.c index f965c32b..36aebfa4 100644 --- a/src/common/chunkedvector.c +++ b/src/common/chunkedvector.c @@ -10,7 +10,7 @@ PK_INLINE int c11__bit_length(unsigned long x) { #if(defined(__clang__) || defined(__GNUC__)) return x == 0 ? 0 : (int)sizeof(unsigned long) * 8 - __builtin_clzl(x); #elif defined(_MSC_VER) - static_assert(sizeof(unsigned long) <= 4, "unsigned long is greater than 4 bytes"); + _Static_assert(sizeof(unsigned long) <= 4, "unsigned long is greater than 4 bytes"); unsigned long msb; if(_BitScanReverse(&msb, x)) { return (int)msb + 1; } return 0; diff --git a/src/common/serialize.c b/src/common/serialize.c index 8a8b7521..e0830c3b 100644 --- a/src/common/serialize.c +++ b/src/common/serialize.c @@ -31,7 +31,7 @@ void c11_serializer__write_cstr(c11_serializer *self, const char* cstr) { } void c11_serializer__write_bytes(c11_serializer* self, const void* data, int size){ - c11_vector__extend(uint8_t, &self->data, data, size); + c11_vector__extend(&self->data, data, size); } void* c11_serializer__submit(c11_serializer* self, int* size){ diff --git a/src/common/sourcedata.c b/src/common/sourcedata.c index aff54ff0..85765156 100644 --- a/src/common/sourcedata.c +++ b/src/common/sourcedata.c @@ -8,10 +8,16 @@ static void SourceData__ctor(struct SourceData* self, const char* filename, enum py_CompileMode mode, bool is_dynamic) { - self->filename = c11_string__new(filename); self->mode = mode; + self->is_dynamic = is_dynamic; + self->filename = c11_string__new(filename); c11_vector__ctor(&self->line_starts, sizeof(const char*)); + if(!source) { + self->source = NULL; + return; + } + // Skip utf8 BOM if there is any. if(strncmp(source, "\xEF\xBB\xBF", 3) == 0) source += 3; // Drop all '\r' @@ -34,13 +40,12 @@ static void SourceData__ctor(struct SourceData* self, self->source->data[last_index + 1] = '\0'; } - self->is_dynamic = is_dynamic; c11_vector__push(const char*, &self->line_starts, self->source->data); } static void SourceData__dtor(struct SourceData* self) { c11_string__delete(self->filename); - c11_string__delete(self->source); + if(self->source) c11_string__delete(self->source); c11_vector__dtor(&self->line_starts); } @@ -59,7 +64,7 @@ bool SourceData__get_line(const struct SourceData* self, int lineno, const char** st, const char** ed) { - if(lineno < 0) return false; + if(lineno < 0 || !self->source) return false; lineno -= 1; if(lineno < 0) lineno = 0; const char* _start = c11__getitem(const char*, &self->line_starts, lineno); diff --git a/src/common/sstream.c b/src/common/sstream.c index b9cda467..36fc223c 100644 --- a/src/common/sstream.c +++ b/src/common/sstream.c @@ -79,7 +79,7 @@ void c11_sbuf__write_cstr(c11_sbuf* self, const char* str) { } void c11_sbuf__write_cstrn(c11_sbuf* self, const char* str, int n) { - c11_vector__extend(char, &self->data, str, n); + c11_vector__extend(&self->data, str, n); } void c11_sbuf__write_quoted(c11_sbuf* self, c11_sv sv, char quote) { diff --git a/src/common/vector.c b/src/common/vector.c index b1a152f6..cffea737 100644 --- a/src/common/vector.c +++ b/src/common/vector.c @@ -75,4 +75,15 @@ int c11_vector__nextcap(c11_vector* self) { // increase by 25% return self->capacity + (self->capacity >> 2); } -} \ No newline at end of file +} + +void c11_vector__extend(c11_vector* self, const void* p, int size) { + int min_capacity = self->length + size; + if(self->capacity < min_capacity) { + int nextcap = c11_vector__nextcap(self); + c11_vector__reserve((self), c11__max(nextcap, min_capacity)); + } + void* dst = (char*)self->data + self->length * self->elem_size; + memcpy(dst, p, size * self->elem_size); + self->length += size; +} diff --git a/src/compiler/compiler.c b/src/compiler/compiler.c index ce50303d..b374244c 100644 --- a/src/compiler/compiler.c +++ b/src/compiler/compiler.c @@ -76,7 +76,6 @@ static int Ctx__prepare_loop_divert(Ctx* self, int line, bool is_break); static int Ctx__enter_block(Ctx* self, CodeBlockType type); static void Ctx__exit_block(Ctx* self); static int Ctx__emit_(Ctx* self, Opcode opcode, uint16_t arg, int line); -// static void Ctx__revert_last_emit_(Ctx* self); static int Ctx__emit_int(Ctx* self, int64_t value, int line); static void Ctx__patch_jump(Ctx* self, int index); static void Ctx__emit_jump(Ctx* self, int target, int line); @@ -1177,7 +1176,7 @@ static void Ctx__s_emit_decorators(Ctx* self, int count) { } static int Ctx__emit_(Ctx* self, Opcode opcode, uint16_t arg, int line) { - Bytecode bc = {(uint8_t)opcode, arg}; + Bytecode bc = {(uint16_t)opcode, arg}; BytecodeEx bcx = {line, self->curr_iblock}; c11_vector__push(Bytecode, &self->co->codes, bc); c11_vector__push(BytecodeEx, &self->co->codes_ex, bcx); @@ -1187,11 +1186,6 @@ static int Ctx__emit_(Ctx* self, Opcode opcode, uint16_t arg, int line) { return i; } -// static void Ctx__revert_last_emit_(Ctx* self) { -// c11_vector__pop(&self->co->codes); -// c11_vector__pop(&self->co->codes_ex); -// } - static int Ctx__emit_int(Ctx* self, int64_t value, int line) { if(INT16_MIN <= value && value <= INT16_MAX) { return Ctx__emit_(self, OP_LOAD_SMALL_INT, (uint16_t)value, line); @@ -1881,65 +1875,11 @@ static Error* exprMap(Compiler* self) { static Error* read_literal(Compiler* self, py_Ref out); -static Error* exprCompileTimeCall(Compiler* self, py_ItemRef func, int line) { - Error* err; - py_push(func); - py_pushnil(); - - uint16_t argc = 0; - uint16_t kwargc = 0; - // copied from `exprCall` - do { - if(curr()->type == TK_RPAREN) break; - if(curr()->type == TK_ID && next()->type == TK_ASSIGN) { - consume(TK_ID); - py_Name key = py_namev(Token__sv(prev())); - consume(TK_ASSIGN); - // k=v - py_pushname(key); - check(read_literal(self, py_pushtmp())); - kwargc += 1; - } else { - if(kwargc > 0) { - return SyntaxError(self, "positional argument follows keyword argument"); - } - check(read_literal(self, py_pushtmp())); - argc += 1; - } - } while(match(TK_COMMA)); - consume(TK_RPAREN); - - py_StackRef p0 = py_peek(0); - bool ok = py_vectorcall(argc, kwargc); - if(!ok) { - char* msg = py_formatexc(); - py_clearexc(p0); - err = SyntaxError(self, "compile-time call error:\n%s", msg); - PK_FREE(msg); - return err; - } - - // TODO: optimize string dedup - int index = Ctx__add_const(ctx(), py_retval()); - Ctx__s_push(ctx(), (Expr*)LoadConstExpr__new(line, index)); - return NULL; -} - static Error* exprCall(Compiler* self) { Error* err; Expr* callable = Ctx__s_popx(ctx()); int line = prev()->line; - if(callable->vt->is_name) { - NameExpr* ne = (NameExpr*)callable; - py_ItemRef func = py_macroget(ne->name); - if(func != NULL) { - py_StackRef p0 = py_peek(0); - err = exprCompileTimeCall(self, func, line); - if(err != NULL) py_clearexc(p0); - return err; - } - } - + CallExpr* e = CallExpr__new(line, callable); Ctx__s_push(ctx(), (Expr*)e); // push onto the stack in advance do { @@ -2425,7 +2365,7 @@ static Error* compile_function(Compiler* self, int decorators) { py_TValue* consts = decl->code.consts.data; py_TValue* c = &consts[codes[0].arg]; if(py_isstr(c)) { - decl->docstring = py_tostr(c); + decl->docstring = c11_strdup(py_tostr(c)); codes[0].op = OP_NO_OP; codes[1].op = OP_NO_OP; } diff --git a/src/interpreter/vm.c b/src/interpreter/vm.c index 7117b680..e64e3edd 100644 --- a/src/interpreter/vm.c +++ b/src/interpreter/vm.c @@ -12,7 +12,7 @@ #include #include -static char* pk_default_importfile(const char* path) { +static char* pk_default_importfile(const char* path, int* data_size) { #if PK_ENABLE_OS FILE* f = fopen(path, "rb"); if(f == NULL) return NULL; @@ -23,6 +23,7 @@ static char* pk_default_importfile(const char* path) { size = fread(buffer, 1, size, f); buffer[size] = 0; fclose(f); + if(data_size) *data_size = (int)size; return buffer; #else return NULL; @@ -112,7 +113,6 @@ void VM__ctor(VM* self) { self->stack.end = self->stack.begin + PK_VM_STACK_SIZE; CachedNames__ctor(&self->cached_names); - NameDict__ctor(&self->compile_time_funcs, PK_TYPE_ATTR_LOAD_FACTOR); /* Init Builtin Types */ // 0: unused @@ -260,6 +260,7 @@ void VM__ctor(VM* self) { pk__add_module_base64(); pk__add_module_importlib(); pk__add_module_unicodedata(); + pk__add_module_py_compile(); pk__add_module_conio(); pk__add_module_lz4(); // optional @@ -306,7 +307,6 @@ void VM__dtor(VM* self) { BinTree__dtor(&self->modules); FixedMemoryPool__dtor(&self->pool_frame); CachedNames__dtor(&self->cached_names); - NameDict__dtor(&self->compile_time_funcs); c11_vector__dtor(&self->types); } @@ -671,12 +671,6 @@ void ManagedHeap__mark(ManagedHeap* self) { CachedNames_KV* kv = c11_chunkedvector__at(&vm->cached_names.entries, i); pk__mark_value(&kv->val); } - // mark compile time functions - for(int i = 0; i < vm->compile_time_funcs.capacity; i++) { - NameDict_KV* kv = &vm->compile_time_funcs.items[i]; - if(kv->key == NULL) continue; - pk__mark_value(&kv->value); - } // mark types int types_length = vm->types.length; // 0-th type is placeholder diff --git a/src/modules/pickle.c b/src/modules/pickle.c index 1c2618fc..51a7d2fb 100644 --- a/src/modules/pickle.c +++ b/src/modules/pickle.c @@ -59,7 +59,7 @@ static void PickleObject__dtor(PickleObject* self) { static bool PickleObject__py_submit(PickleObject* self, py_OutRef out); static void PickleObject__write_bytes(PickleObject* buf, const void* data, int size) { - c11_vector__extend(char, &buf->codes, data, size); + c11_vector__extend(&buf->codes, data, size); } static void c11_sbuf__write_type_path(c11_sbuf* path_buf, py_Type type) { diff --git a/src/modules/py_compile.c b/src/modules/py_compile.c new file mode 100644 index 00000000..56d97621 --- /dev/null +++ b/src/modules/py_compile.c @@ -0,0 +1,51 @@ +#include "pocketpy/pocketpy.h" +#include "pocketpy/interpreter/vm.h" +#include + +static bool py_compile_compile(int argc, py_Ref argv) { + PY_CHECK_ARGC(2); + PY_CHECK_ARG_TYPE(0, tp_str); + PY_CHECK_ARG_TYPE(1, tp_str); + const char* src_path = py_tostr(py_arg(0)); + const char* dst_path = py_tostr(py_arg(1)); + // read + FILE* fp = fopen(src_path, "rb"); + if(fp == NULL) { + const char* msg = strerror(errno); + return OSError("[Errno %d] %s: '%s'", errno, msg, src_path); + } + fseek(fp, 0, SEEK_END); + long size = ftell(fp); + fseek(fp, 0, SEEK_SET); + char* buffer = PK_MALLOC(size + 1); + size = fread(buffer, 1, size, fp); + buffer[size] = 0; + fclose(fp); + // compile + bool ok = py_compile(buffer, src_path, EXEC_MODE, false); + PK_FREE(buffer); + if(!ok) return false; + // dump + py_assign(py_pushtmp(), py_retval()); + int bc_size; + void* bc_data = CodeObject__dumps(py_touserdata(py_peek(-1)), &bc_size); + py_pop(); + // write + fp = fopen(dst_path, "wb"); + if(fp == NULL) { + PK_FREE(bc_data); + const char* msg = strerror(errno); + return OSError("[Errno %d] %s: '%s'", errno, msg, dst_path); + } + fwrite(bc_data, 1, bc_size, fp); + fclose(fp); + PK_FREE(bc_data); + py_newnone(py_retval()); + return true; +} + +void pk__add_module_py_compile() { + py_Ref mod = py_newmodule("py_compile"); + + py_bindfunc(mod, "compile", py_compile_compile); +} \ No newline at end of file diff --git a/src/objects/codeobject.c b/src/objects/codeobject.c index b0fe51d5..d75f419b 100644 --- a/src/objects/codeobject.c +++ b/src/objects/codeobject.c @@ -1,5 +1,6 @@ #include "pocketpy/objects/codeobject.h" #include "pocketpy/common/utils.h" +#include "pocketpy/config.h" #include "pocketpy/pocketpy.h" #include #include @@ -17,11 +18,12 @@ bool Bytecode__is_forward_jump(const Bytecode* self) { (op == OP_FOR_ITER || op == OP_FOR_ITER_YIELD_VALUE); } -static void FuncDecl__dtor(FuncDecl* self) { +void FuncDecl__dtor(FuncDecl* self) { CodeObject__dtor(&self->code); c11_vector__dtor(&self->args); c11_vector__dtor(&self->kwargs); c11_smallmap_n2d__dtor(&self->kw_to_index); + if(self->docstring) py_free(self->docstring); } FuncDecl_ FuncDecl__rcnew(SourceData_ src, c11_sv name) { @@ -30,7 +32,7 @@ FuncDecl_ FuncDecl__rcnew(SourceData_ src, c11_sv name) { self->rc.dtor = (void (*)(void*))FuncDecl__dtor; CodeObject__ctor(&self->code, src, name); - c11_vector__ctor(&self->args, sizeof(int)); + c11_vector__ctor(&self->args, sizeof(int32_t)); c11_vector__ctor(&self->kwargs, sizeof(FuncDeclKwArg)); self->starred_arg = -1; @@ -66,8 +68,8 @@ bool FuncDecl__is_duplicated_arg(const FuncDecl* decl, py_Name name) { } void FuncDecl__add_arg(FuncDecl* self, py_Name name) { - int index = CodeObject__add_varname(&self->code, name); - c11_vector__push(int, &self->args, index); + int32_t index = CodeObject__add_varname(&self->code, name); + c11_vector__push(int32_t, &self->args, index); } void FuncDecl__add_kwarg(FuncDecl* self, py_Name name, const py_TValue* value) { @@ -89,32 +91,6 @@ void FuncDecl__add_starred_kwarg(FuncDecl* self, py_Name name) { self->starred_kwarg = index; } -FuncDecl_ FuncDecl__build(c11_sv name, - c11_sv* args, - int argc, - c11_sv starred_arg, - c11_sv* kwargs, - int kwargc, - py_Ref kwdefaults, // a tuple contains default values - c11_sv starred_kwarg, - const char* docstring) { - SourceData_ source = SourceData__rcnew("pass", "", EXEC_MODE, false); - FuncDecl_ decl = FuncDecl__rcnew(source, name); - for(int i = 0; i < argc; i++) { - FuncDecl__add_arg(decl, py_namev(args[i])); - } - if(starred_arg.size) { FuncDecl__add_starred_arg(decl, py_namev(starred_arg)); } - assert(py_istype(kwdefaults, tp_tuple)); - assert(py_tuple_len(kwdefaults) == kwargc); - for(int i = 0; i < kwargc; i++) { - FuncDecl__add_kwarg(decl, py_namev(kwargs[i]), py_tuple_getitem(kwdefaults, i)); - } - if(starred_kwarg.size) FuncDecl__add_starred_kwarg(decl, py_namev(starred_kwarg)); - decl->docstring = docstring; - PK_DECREF(source); - return decl; -} - void CodeObject__ctor(CodeObject* self, SourceData_ src, c11_sv name) { self->src = src; PK_INCREF(src); diff --git a/src/objects/codeobject_ser.c b/src/objects/codeobject_ser.c index 76628ded..958ceff1 100644 --- a/src/objects/codeobject_ser.c +++ b/src/objects/codeobject_ser.c @@ -8,328 +8,333 @@ #define CODEOBJECT_VER_MINOR 0 // Forward declarations -static void FuncDecl__serialize(c11_serializer* s, const FuncDecl* decl); -static FuncDecl_ FuncDecl__deserialize(c11_deserializer* d); -static void CodeObject__serialize(c11_serializer* s, const CodeObject* co); -static void CodeObject__deserialize(c11_deserializer* d, CodeObject* co); +static void FuncDecl__serialize(c11_serializer* s, + const FuncDecl* decl, + const struct SourceData* parent_src); +static FuncDecl_ FuncDecl__deserialize(c11_deserializer* d, SourceData_ embedded_src); +static void CodeObject__serialize(c11_serializer* s, + const CodeObject* co, + const struct SourceData* parent_src); +static CodeObject CodeObject__deserialize(c11_deserializer* d, SourceData_ embedded_src); // Serialize a py_TValue constant // Supported types: None, int, float, bool, str, bytes, tuple, Ellipsis -static void TValue__serialize(c11_serializer* s, const py_TValue* val) { +static void TValue__serialize(c11_serializer* s, py_Ref val) { c11_serializer__write_type(s, val->type); + // 1. co_consts: int | float | str + // 2. function defaults: see `read_literal()` in compiler.c switch(val->type) { - case tp_NoneType: - case tp_ellipsis: - // No additional data needed - break; - case tp_int: - c11_serializer__write_i64(s, val->_i64); - break; - case tp_float: - c11_serializer__write_f64(s, val->_f64); - break; - case tp_bool: - c11_serializer__write_i8(s, val->_bool ? 1 : 0); - break; + case tp_int: c11_serializer__write_i64(s, val->_i64); break; + case tp_float: c11_serializer__write_f64(s, val->_f64); break; case tp_str: { c11_sv sv = py_tosv((py_Ref)val); - c11_serializer__write_size(s, sv.size); + c11_serializer__write_i32(s, sv.size); c11_serializer__write_bytes(s, sv.data, sv.size); break; } - case tp_bytes: { - int size; - unsigned char* data = py_tobytes((py_Ref)val, &size); - c11_serializer__write_size(s, size); - c11_serializer__write_bytes(s, data, size); + case tp_bool: { + bool value = py_tobool(val); + c11_serializer__write_i8(s, value ? 1 : 0); break; } + case tp_NoneType: break; + case tp_ellipsis: break; case tp_tuple: { - int len = py_tuple_len((py_Ref)val); - c11_serializer__write_size(s, len); - py_ObjectRef data = py_tuple_data((py_Ref)val); + int len = py_tuple_len(val); + c11_serializer__write_i32(s, len); for(int i = 0; i < len; i++) { - TValue__serialize(s, &data[i]); + py_Ref item = py_tuple_getitem(val, i); + TValue__serialize(s, item); } break; } - default: - c11__abort("TValue__serialize: unsupported type %d", val->type); + default: c11__abort("TValue__serialize: invalid type '%s'", py_tpname(val->type)); } } // Deserialize a py_TValue constant -static void TValue__deserialize(c11_deserializer* d, py_TValue* val) { +static void TValue__deserialize(c11_deserializer* d, py_OutRef out) { py_Type type = c11_deserializer__read_type(d); switch(type) { - case tp_NoneType: - py_newnone(val); - break; - case tp_ellipsis: - py_newellipsis(val); - break; case tp_int: { - int64_t v = c11_deserializer__read_i64(d); - py_newint(val, v); + py_i64 v = c11_deserializer__read_i64(d); + py_newint(out, v); break; } case tp_float: { - double v = c11_deserializer__read_f64(d); - py_newfloat(val, v); - break; - } - case tp_bool: { - int8_t v = c11_deserializer__read_i8(d); - py_newbool(val, v != 0); + py_f64 v = c11_deserializer__read_f64(d); + py_newfloat(out, v); break; } case tp_str: { - int size = c11_deserializer__read_size(d); - char* data = py_newstrn(val, size); - memcpy(data, c11_deserializer__read_bytes(d, size), size); + int size = c11_deserializer__read_i32(d); + char* dst = py_newstrn(out, size); + char* src = c11_deserializer__read_bytes(d, size); + memcpy(dst, src, size); break; } - case tp_bytes: { - int size = c11_deserializer__read_size(d); - unsigned char* data = py_newbytes(val, size); - memcpy(data, c11_deserializer__read_bytes(d, size), size); + case tp_bool: { + bool v = c11_deserializer__read_i8(d) != 0; + py_newbool(out, v); + break; + } + case tp_NoneType: { + py_newnone(out); + break; + } + case tp_ellipsis: { + py_newellipsis(out); break; } case tp_tuple: { - int len = c11_deserializer__read_size(d); - py_ObjectRef data = py_newtuple(val, len); + int len = c11_deserializer__read_i32(d); + py_newtuple(out, len); for(int i = 0; i < len; i++) { - TValue__deserialize(d, &data[i]); + py_ItemRef item = py_tuple_getitem(out, i); + TValue__deserialize(d, item); } break; } default: - c11__abort("TValue__deserialize: unsupported type %d", type); + c11__abort("TValue__deserialize: invalid type '%s'", py_tpname(type)); } } // Serialize CodeObject -static void CodeObject__serialize(c11_serializer* s, const CodeObject* co) { +static void CodeObject__serialize(c11_serializer* s, + const CodeObject* co, + const struct SourceData* parent_src) { // SourceData - c11_serializer__write_cstr(s, co->src->filename->data); - c11_serializer__write_cstr(s, co->src->source->data); - c11_serializer__write_i8(s, (int8_t)co->src->mode); - c11_serializer__write_i8(s, co->src->is_dynamic ? 1 : 0); + if(!parent_src) { + c11_serializer__write_i8(s, (int8_t)co->src->mode); + c11_serializer__write_i8(s, co->src->is_dynamic ? 1 : 0); + c11_serializer__write_cstr(s, co->src->filename->data); + } else { + c11__rtassert(co->src == parent_src); + } // name c11_serializer__write_cstr(s, co->name->data); // codes - c11_serializer__write_size(s, co->codes.length); + _Static_assert(sizeof(Bytecode) == sizeof(uint16_t) * 2, ""); + c11_serializer__write_i32(s, co->codes.length); c11_serializer__write_bytes(s, co->codes.data, co->codes.length * sizeof(Bytecode)); // codes_ex - c11_serializer__write_size(s, co->codes_ex.length); + _Static_assert(sizeof(BytecodeEx) == sizeof(int32_t) * 2, ""); + c11_serializer__write_i32(s, co->codes_ex.length); c11_serializer__write_bytes(s, co->codes_ex.data, co->codes_ex.length * sizeof(BytecodeEx)); // consts - c11_serializer__write_size(s, co->consts.length); + c11_serializer__write_i32(s, co->consts.length); for(int i = 0; i < co->consts.length; i++) { - py_TValue* val = c11__at(py_TValue, &co->consts, i); + py_Ref val = c11__at(py_TValue, &co->consts, i); TValue__serialize(s, val); } // varnames (as cstr via py_name2str) - c11_serializer__write_size(s, co->varnames.length); + c11_serializer__write_i32(s, co->varnames.length); for(int i = 0; i < co->varnames.length; i++) { py_Name name = c11__getitem(py_Name, &co->varnames, i); c11_serializer__write_cstr(s, py_name2str(name)); } // names (as cstr via py_name2str) - c11_serializer__write_size(s, co->names.length); + c11_serializer__write_i32(s, co->names.length); for(int i = 0; i < co->names.length; i++) { py_Name name = c11__getitem(py_Name, &co->names, i); c11_serializer__write_cstr(s, py_name2str(name)); } // nlocals - c11_serializer__write_size(s, co->nlocals); + c11_serializer__write_i32(s, co->nlocals); // blocks - c11_serializer__write_size(s, co->blocks.length); + _Static_assert(sizeof(CodeBlock) == sizeof(int32_t) * 5, ""); + c11_serializer__write_i32(s, co->blocks.length); c11_serializer__write_bytes(s, co->blocks.data, co->blocks.length * sizeof(CodeBlock)); // func_decls - c11_serializer__write_size(s, co->func_decls.length); + c11_serializer__write_i32(s, co->func_decls.length); for(int i = 0; i < co->func_decls.length; i++) { - FuncDecl_ decl = c11__getitem(FuncDecl_, &co->func_decls, i); - FuncDecl__serialize(s, decl); + const FuncDecl* decl = c11__getitem(FuncDecl_, &co->func_decls, i); + FuncDecl__serialize(s, decl, co->src); } // start_line, end_line - c11_serializer__write_size(s, co->start_line); - c11_serializer__write_size(s, co->end_line); + c11_serializer__write_i32(s, co->start_line); + c11_serializer__write_i32(s, co->end_line); } // Deserialize CodeObject (initialize co before calling) -static void CodeObject__deserialize(c11_deserializer* d, CodeObject* co) { +static CodeObject CodeObject__deserialize(c11_deserializer* d, SourceData_ embedded_src) { + CodeObject co; + // SourceData - const char* filename = c11_deserializer__read_cstr(d); - const char* source = c11_deserializer__read_cstr(d); - enum py_CompileMode mode = (enum py_CompileMode)c11_deserializer__read_i8(d); - bool is_dynamic = c11_deserializer__read_i8(d) != 0; - SourceData_ src = SourceData__rcnew(source, filename, mode, is_dynamic); + SourceData_ src; + if(embedded_src != NULL) { + src = embedded_src; + PK_INCREF(src); + } else { + enum py_CompileMode mode = (enum py_CompileMode)c11_deserializer__read_i8(d); + bool is_dynamic = c11_deserializer__read_i8(d) != 0; + const char* filename = c11_deserializer__read_cstr(d); + src = SourceData__rcnew(NULL, filename, mode, is_dynamic); + } // name const char* name = c11_deserializer__read_cstr(d); c11_sv name_sv = {name, strlen(name)}; // Initialize the CodeObject - CodeObject__ctor(co, src, name_sv); + CodeObject__ctor(&co, src, name_sv); PK_DECREF(src); // CodeObject__ctor increments ref count - // Clear the default root block that CodeObject__ctor adds - c11_vector__clear(&co->blocks); + c11_vector__clear(&co.blocks); // codes - int codes_len = c11_deserializer__read_size(d); - c11_vector__reserve(&co->codes, codes_len); - memcpy(co->codes.data, c11_deserializer__read_bytes(d, codes_len * sizeof(Bytecode)), codes_len * sizeof(Bytecode)); - co->codes.length = codes_len; - + int codes_len = c11_deserializer__read_i32(d); + c11_vector__extend(&co.codes, + c11_deserializer__read_bytes(d, codes_len * sizeof(Bytecode)), + codes_len); // codes_ex - int codes_ex_len = c11_deserializer__read_size(d); - c11_vector__reserve(&co->codes_ex, codes_ex_len); - memcpy(co->codes_ex.data, c11_deserializer__read_bytes(d, codes_ex_len * sizeof(BytecodeEx)), codes_ex_len * sizeof(BytecodeEx)); - co->codes_ex.length = codes_ex_len; + int codes_ex_len = c11_deserializer__read_i32(d); + c11_vector__extend(&co.codes_ex, + c11_deserializer__read_bytes(d, codes_ex_len * sizeof(BytecodeEx)), + codes_ex_len); // consts - int consts_len = c11_deserializer__read_size(d); + int consts_len = c11_deserializer__read_i32(d); for(int i = 0; i < consts_len; i++) { - py_TValue val; - TValue__deserialize(d, &val); - c11_vector__push(py_TValue, &co->consts, val); + py_Ref p_val = c11_vector__emplace(&co.consts); + TValue__deserialize(d, p_val); } // varnames - int varnames_len = c11_deserializer__read_size(d); + int varnames_len = c11_deserializer__read_i32(d); for(int i = 0; i < varnames_len; i++) { const char* s = c11_deserializer__read_cstr(d); py_Name n = py_name(s); - c11_vector__push(py_Name, &co->varnames, n); - c11_smallmap_n2d__set(&co->varnames_inv, n, i); + c11_vector__push(py_Name, &co.varnames, n); + c11_smallmap_n2d__set(&co.varnames_inv, n, i); } // names - int names_len = c11_deserializer__read_size(d); + int names_len = c11_deserializer__read_i32(d); for(int i = 0; i < names_len; i++) { const char* s = c11_deserializer__read_cstr(d); py_Name n = py_name(s); - c11_vector__push(py_Name, &co->names, n); - c11_smallmap_n2d__set(&co->names_inv, n, i); + c11_vector__push(py_Name, &co.names, n); + c11_smallmap_n2d__set(&co.names_inv, n, i); } // nlocals - co->nlocals = c11_deserializer__read_size(d); + co.nlocals = c11_deserializer__read_i32(d); // blocks - int blocks_len = c11_deserializer__read_size(d); - c11_vector__reserve(&co->blocks, blocks_len); - memcpy(co->blocks.data, c11_deserializer__read_bytes(d, blocks_len * sizeof(CodeBlock)), blocks_len * sizeof(CodeBlock)); - co->blocks.length = blocks_len; - + int blocks_len = c11_deserializer__read_i32(d); + c11_vector__extend(&co.blocks, + c11_deserializer__read_bytes(d, blocks_len * sizeof(CodeBlock)), + blocks_len); // func_decls - int func_decls_len = c11_deserializer__read_size(d); + int func_decls_len = c11_deserializer__read_i32(d); for(int i = 0; i < func_decls_len; i++) { - FuncDecl_ decl = FuncDecl__deserialize(d); - c11_vector__push(FuncDecl_, &co->func_decls, decl); + FuncDecl_ decl = FuncDecl__deserialize(d, src); + c11_vector__push(FuncDecl_, &co.func_decls, decl); } // start_line, end_line - co->start_line = c11_deserializer__read_size(d); - co->end_line = c11_deserializer__read_size(d); + co.start_line = c11_deserializer__read_i32(d); + co.end_line = c11_deserializer__read_i32(d); + + return co; } // Serialize FuncDecl -static void FuncDecl__serialize(c11_serializer* s, const FuncDecl* decl) { +static void FuncDecl__serialize(c11_serializer* s, + const FuncDecl* decl, + const struct SourceData* parent_src) { // CodeObject (embedded) - CodeObject__serialize(s, &decl->code); + CodeObject__serialize(s, &decl->code, parent_src); // args - c11_serializer__write_size(s, decl->args.length); - c11_serializer__write_bytes(s, decl->args.data, decl->args.length * sizeof(int)); + c11_serializer__write_i32(s, decl->args.length); + c11_serializer__write_bytes(s, decl->args.data, decl->args.length * sizeof(int32_t)); // kwargs - c11_serializer__write_size(s, decl->kwargs.length); + c11_serializer__write_i32(s, decl->kwargs.length); for(int i = 0; i < decl->kwargs.length; i++) { FuncDeclKwArg* kw = c11__at(FuncDeclKwArg, &decl->kwargs, i); - c11_serializer__write_size(s, kw->index); + c11_serializer__write_i32(s, kw->index); c11_serializer__write_cstr(s, py_name2str(kw->key)); TValue__serialize(s, &kw->value); } // starred_arg, starred_kwarg - c11_serializer__write_size(s, decl->starred_arg); - c11_serializer__write_size(s, decl->starred_kwarg); + c11_serializer__write_i32(s, decl->starred_arg); + c11_serializer__write_i32(s, decl->starred_kwarg); // nested c11_serializer__write_i8(s, decl->nested ? 1 : 0); // docstring - c11_serializer__write_cstr(s, decl->docstring ? decl->docstring : ""); + int has_docstring = decl->docstring != NULL ? 1 : 0; + c11_serializer__write_i8(s, has_docstring); + if(has_docstring) c11_serializer__write_cstr(s, decl->docstring); // type c11_serializer__write_i8(s, (int8_t)decl->type); } // Deserialize FuncDecl -static FuncDecl_ FuncDecl__deserialize(c11_deserializer* d) { - // We need to deserialize the CodeObject first to get src and name - // But FuncDecl__rcnew calls CodeObject__ctor, so we deserialize differently - - // Allocate FuncDecl manually +static FuncDecl_ FuncDecl__deserialize(c11_deserializer* d, SourceData_ embedded_src) { FuncDecl* self = PK_MALLOC(sizeof(FuncDecl)); self->rc.count = 1; - - // Deserialize CodeObject directly into self->code - CodeObject__deserialize(d, &self->code); - - // Initialize other fields - c11_vector__ctor(&self->args, sizeof(int)); + self->rc.dtor = (void (*)(void*))FuncDecl__dtor; + + c11_vector__ctor(&self->args, sizeof(int32_t)); c11_vector__ctor(&self->kwargs, sizeof(FuncDeclKwArg)); c11_smallmap_n2d__ctor(&self->kw_to_index); + // CodeObject (embedded) + self->code = CodeObject__deserialize(d, embedded_src); + // args - int args_len = c11_deserializer__read_size(d); - c11_vector__reserve(&self->args, args_len); - memcpy(self->args.data, c11_deserializer__read_bytes(d, args_len * sizeof(int)), args_len * sizeof(int)); - self->args.length = args_len; + int args_len = c11_deserializer__read_i32(d); + c11_vector__extend(&self->args, + c11_deserializer__read_bytes(d, args_len * sizeof(int32_t)), + args_len); // kwargs - int kwargs_len = c11_deserializer__read_size(d); + int kwargs_len = c11_deserializer__read_i32(d); for(int i = 0; i < kwargs_len; i++) { - FuncDeclKwArg kw; - kw.index = c11_deserializer__read_size(d); + FuncDeclKwArg* kw = c11_vector__emplace(&self->kwargs); + kw->index = c11_deserializer__read_i32(d); const char* key_str = c11_deserializer__read_cstr(d); - kw.key = py_name(key_str); - TValue__deserialize(d, &kw.value); - c11_vector__push(FuncDeclKwArg, &self->kwargs, kw); - c11_smallmap_n2d__set(&self->kw_to_index, kw.key, kw.index); + kw->key = py_name(key_str); + TValue__deserialize(d, &kw->value); + c11_smallmap_n2d__set(&self->kw_to_index, kw->key, kw->index); } - - // starred_arg, starred_kwarg - self->starred_arg = c11_deserializer__read_size(d); - self->starred_kwarg = c11_deserializer__read_size(d); + // starred_arg + self->starred_arg = c11_deserializer__read_i32(d); + // starred_kwarg + self->starred_kwarg = c11_deserializer__read_i32(d); // nested self->nested = c11_deserializer__read_i8(d) != 0; // docstring - const char* docstring = c11_deserializer__read_cstr(d); - self->docstring = docstring[0] ? c11_strdup(docstring) : NULL; + int has_docstring = c11_deserializer__read_i8(d); + if(has_docstring) { + const char* docstring = c11_deserializer__read_cstr(d); + self->docstring = c11_strdup(docstring); + } else { + self->docstring = NULL; + } // type self->type = (FuncType)c11_deserializer__read_i8(d); - - // Set destructor - self->rc.dtor = NULL; // Will be set properly when used - return self; } @@ -337,26 +342,26 @@ static FuncDecl_ FuncDecl__deserialize(c11_deserializer* d) { void* CodeObject__dumps(const CodeObject* co, int* size) { c11_serializer s; c11_serializer__ctor(&s, CODEOBJECT_MAGIC, CODEOBJECT_VER_MAJOR, CODEOBJECT_VER_MINOR); - CodeObject__serialize(&s, co); + CodeObject__serialize(&s, co, NULL); return c11_serializer__submit(&s, size); } -// Static error message buffer for CodeObject__loads -static _Thread_local char s_codeobject_error_msg[64]; - // Public API: Deserialize CodeObject from bytes // Returns error message or NULL on success -const char* CodeObject__loads(CodeObject* co, const void* data, int size) { +char* CodeObject__loads(const void* data, int size, CodeObject* out) { c11_deserializer d; c11_deserializer__ctor(&d, data, size); - - if(!c11_deserializer__check_header(&d, CODEOBJECT_MAGIC, CODEOBJECT_VER_MAJOR, CODEOBJECT_VER_MINOR)) { - memcpy(s_codeobject_error_msg, d.error_msg, sizeof(s_codeobject_error_msg)); + + if(!c11_deserializer__check_header(&d, + CODEOBJECT_MAGIC, + CODEOBJECT_VER_MAJOR, + CODEOBJECT_VER_MINOR)) { + char* error_msg = c11_strdup(d.error_msg); c11_deserializer__dtor(&d); - return s_codeobject_error_msg; + return error_msg; } - - CodeObject__deserialize(&d, co); + + *out = CodeObject__deserialize(&d, NULL); c11_deserializer__dtor(&d); return NULL; } diff --git a/src/public/Bindings.c b/src/public/Bindings.c index 28ca8950..47f8de11 100644 --- a/src/public/Bindings.c +++ b/src/public/Bindings.c @@ -46,16 +46,3 @@ void py_bindmagic(py_Type type, py_Name name, py_CFunction f) { py_Ref tmp = py_emplacedict(py_tpobject(type), name); py_newnativefunc(tmp, f); } - -void py_macrobind(const char* sig, py_CFunction f) { - py_Ref tmp = py_pushtmp(); - py_Name name = py_newfunction(tmp, sig, f, NULL, 0); - NameDict__set(&pk_current_vm->compile_time_funcs, name, tmp); - py_pop(); -} - -py_ItemRef py_macroget(py_Name name) { - NameDict* d = &pk_current_vm->compile_time_funcs; - if(d->length == 0) return NULL; - return NameDict__try_get(d, name); -} diff --git a/src/public/GlobalSetup.c b/src/public/GlobalSetup.c index 704fa354..a6913348 100644 --- a/src/public/GlobalSetup.c +++ b/src/public/GlobalSetup.c @@ -32,6 +32,10 @@ void py_initialize() { _Static_assert(sizeof(py_TValue) == 24, "sizeof(py_TValue) != 24"); _Static_assert(offsetof(py_TValue, extra) == 4, "offsetof(py_TValue, extra) != 4"); + // check sizes + _Static_assert(sizeof(float) == 4, ""); + _Static_assert(sizeof(double) == 8, ""); + pk_current_vm = pk_all_vm[0] = &pk_default_vm; // initialize some convenient references diff --git a/src/public/ModuleSystem.c b/src/public/ModuleSystem.c index 3e1b25a7..f59b41ae 100644 --- a/src/public/ModuleSystem.c +++ b/src/public/ModuleSystem.c @@ -140,20 +140,39 @@ int py_import(const char* path_cstr) { c11_string* filename = c11_string__new3("%s.py", slashed_path->data); bool need_free = true; + bool is_pyc = false; const char* data = load_kPythonLib(path_cstr); + int data_size = -1; + if(data != NULL) { need_free = false; goto __SUCCESS; } - data = vm->callbacks.importfile(filename->data); + data = vm->callbacks.importfile(filename->data, &data_size); if(data != NULL) goto __SUCCESS; + c11_string__delete(filename); + filename = c11_string__new3("%s.pyc", slashed_path->data); + data = vm->callbacks.importfile(filename->data, &data_size); + if(data != NULL) { + is_pyc = true; + goto __SUCCESS; + } + c11_string__delete(filename); filename = c11_string__new3("%s%c__init__.py", slashed_path->data, PK_PLATFORM_SEP); - data = vm->callbacks.importfile(filename->data); + data = vm->callbacks.importfile(filename->data, &data_size); if(data != NULL) goto __SUCCESS; + c11_string__delete(filename); + filename = c11_string__new3("%s%c__init__.pyc", slashed_path->data, PK_PLATFORM_SEP); + data = vm->callbacks.importfile(filename->data, &data_size); + if(data != NULL) { + is_pyc = true; + goto __SUCCESS; + } + c11_string__delete(filename); c11_string__delete(slashed_path); // not found @@ -162,8 +181,24 @@ int py_import(const char* path_cstr) { __SUCCESS: do { } while(0); + py_GlobalRef mod = py_newmodule(path_cstr); - bool ok = py_exec((const char*)data, filename->data, EXEC_MODE, mod); + + bool ok; + if(is_pyc) { + CodeObject co; + char* err = CodeObject__loads(data, data_size, &co); + if(err == NULL) { + c11__rtassert(co.src->mode == EXEC_MODE); + ok = pk_exec(&co, mod); + } else { + RuntimeError("failed to load %s: %s", filename->data, err); + ok = false; + } + } else { + ok = py_exec(data, filename->data, EXEC_MODE, mod); + } + py_assign(py_retval(), mod); c11_string__delete(filename); @@ -180,11 +215,13 @@ bool py_importlib_reload(py_Ref module) { c11_sv path = c11_string__sv(mi->path); c11_string* slashed_path = c11_sv__replace(path, '.', PK_PLATFORM_SEP); c11_string* filename = c11_string__new3("%s.py", slashed_path->data); - char* data = vm->callbacks.importfile(filename->data); + // Here we only consider source modules. + // Because compiled modules have no source file (it cannot be reloaded) + char* data = vm->callbacks.importfile(filename->data, NULL); if(data == NULL) { c11_string__delete(filename); filename = c11_string__new3("%s%c__init__.py", slashed_path->data, PK_PLATFORM_SEP); - data = vm->callbacks.importfile(filename->data); + data = vm->callbacks.importfile(filename->data, NULL); } c11_string__delete(slashed_path); if(data == NULL) return ImportError("module '%v' not found", path); diff --git a/src/public/PyList.c b/src/public/PyList.c index 761c0a73..67e15d71 100644 --- a/src/public/PyList.c +++ b/src/public/PyList.c @@ -203,8 +203,8 @@ static bool list__add__(int argc, py_Ref argv) { List* list_1 = py_touserdata(_1); py_newlist(py_retval()); List* list = py_touserdata(py_retval()); - c11_vector__extend(py_TValue, list, list_0->data, list_0->length); - c11_vector__extend(py_TValue, list, list_1->data, list_1->length); + c11_vector__extend(list, list_0->data, list_0->length); + c11_vector__extend(list, list_1->data, list_1->length); } else { py_newnotimplemented(py_retval()); } @@ -221,7 +221,7 @@ static bool list__mul__(int argc, py_Ref argv) { List* list = py_touserdata(py_retval()); List* list_0 = py_touserdata(_0); for(int i = 0; i < n; i++) { - c11_vector__extend(py_TValue, list, list_0->data, list_0->length); + c11_vector__extend(list, list_0->data, list_0->length); } } else { py_newnotimplemented(py_retval()); @@ -264,7 +264,7 @@ static bool list_extend(int argc, py_Ref argv) { py_TValue* p; int length = pk_arrayview(py_arg(1), &p); if(length == -1) return TypeError("extend() argument must be a list or tuple"); - c11_vector__extend(py_TValue, self, p, length); + c11_vector__extend(self, p, length); py_newnone(py_retval()); return true; } @@ -293,7 +293,7 @@ static bool list_copy(int argc, py_Ref argv) { py_newlist(py_retval()); List* self = py_touserdata(py_arg(0)); List* list = py_touserdata(py_retval()); - c11_vector__extend(py_TValue, list, self->data, self->length); + c11_vector__extend(list, self->data, self->length); return true; } diff --git a/src/public/ValueCreation.c b/src/public/ValueCreation.c index 426cc8e7..3c496f89 100644 --- a/src/public/ValueCreation.c +++ b/src/public/ValueCreation.c @@ -121,7 +121,7 @@ py_Name py_newfunction(py_OutRef out, c11__abort("py_newfunction(): invalid signature '%s'", sig); } FuncDecl_ decl = c11__getitem(FuncDecl_, &code.func_decls, 0); - decl->docstring = docstring; + if(docstring) decl->docstring = c11_strdup(docstring); // construct the function Function* ud = py_newobject(out, tp_function, slots, sizeof(Function)); Function__ctor(ud, decl, NULL, NULL); diff --git a/tests/922_py_compile.py b/tests/922_py_compile.py new file mode 100644 index 00000000..be414a5f --- /dev/null +++ b/tests/922_py_compile.py @@ -0,0 +1,22 @@ +from py_compile import compile +import os + +if not os.path.exists('tmp'): + os.mkdir('tmp') + +compile('python/heapq.py', 'tmp/heapq1.pyc') +assert os.path.exists('tmp/heapq1.pyc') + +os.chdir('tmp') +import heapq1 +import heapq + +a = [1, 2, -3, 2, 1, 5, 11, 123] * 10 +b = a.copy() + +heapq.heapify(a) +heapq1.heapify(b) + +assert a == b + +os.remove('heapq1.pyc')