diff --git a/.gitignore b/.gitignore index 56dfad56..a59fc57c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ docs/C-API/functions.md cmake-build-* tmp/ profiler_report.json + +tmp/ diff --git a/compileall.py b/compileall.py new file mode 100644 index 00000000..b2e17001 --- /dev/null +++ b/compileall.py @@ -0,0 +1,28 @@ +import sys +import os + +if len(sys.argv) != 4: + print('Usage: python compileall.py ') + exit(1) + +pkpy_exe = sys.argv[1] +source_dir = sys.argv[2] +output_dir = sys.argv[3] + +def do_compile(src_path, dst_path): + cmd = f'{pkpy_exe} --compile "{src_path}" "{dst_path}"' + print(src_path) + assert os.system(cmd) == 0 + +for root, _, files in os.walk(source_dir): + for file in files: + if not file.endswith('.py'): + continue + src_path = os.path.join(root, file) + dst_path = os.path.join( + output_dir, + os.path.relpath(root, source_dir), + file + 'c' + ) + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + do_compile(src_path, dst_path) diff --git a/docs/retype.yml b/docs/retype.yml index 95083190..d2613367 100644 --- a/docs/retype.yml +++ b/docs/retype.yml @@ -3,7 +3,7 @@ output: .retype url: https://pocketpy.dev branding: title: pocketpy - label: v2.1.6 + label: v2.1.7 logo: "./static/logo.png" favicon: "./static/logo.png" meta: diff --git a/include/pocketpy/common/serialize.h b/include/pocketpy/common/serialize.h new file mode 100644 index 00000000..1d5b1d06 --- /dev/null +++ b/include/pocketpy/common/serialize.h @@ -0,0 +1,56 @@ +#pragma once + +#include + +#include "pocketpy/common/vector.h" +#include "pocketpy/common/str.h" + + +typedef struct c11_serializer { + c11_vector data; +} c11_serializer; + +void c11_serializer__ctor(c11_serializer* self, int16_t magic, int8_t major_ver, int8_t minor_ver); +void c11_serializer__dtor(c11_serializer* self); +void c11_serializer__write_cstr(c11_serializer* self, const char*); +void c11_serializer__write_bytes(c11_serializer* self, const void* data, int size); +void* c11_serializer__submit(c11_serializer* self, int* size); + +typedef struct c11_deserializer { + char error_msg[64]; + const uint8_t* data; + int size; + int index; + int8_t major_ver; + int8_t minor_ver; +} c11_deserializer; + +void c11_deserializer__ctor(c11_deserializer* self, const void* data, int size); +void c11_deserializer__dtor(c11_deserializer* self); +bool c11_deserializer__check_header(c11_deserializer* self, int16_t magic, int8_t major_ver, int8_t minor_ver_min); +const char* c11_deserializer__read_cstr(c11_deserializer* self); +void* c11_deserializer__read_bytes(c11_deserializer* self, int size); + + +#define DEF_ATOMIC_INLINE_RW(name, T) \ + static inline void c11_serializer__write_##name(c11_serializer* self, T value){ \ + c11_serializer__write_bytes(self, &value, sizeof(T)); \ + } \ + static inline T c11_deserializer__read_##name(c11_deserializer* self){ \ + const void* p = self->data + self->index; \ + self->index += sizeof(T); \ + T retval;\ + memcpy(&retval, p, sizeof(T)); \ + return retval; \ + } + +DEF_ATOMIC_INLINE_RW(i8, int8_t) +DEF_ATOMIC_INLINE_RW(i16, int16_t) +DEF_ATOMIC_INLINE_RW(i32, int32_t) +DEF_ATOMIC_INLINE_RW(i64, int64_t) +DEF_ATOMIC_INLINE_RW(f32, float) +DEF_ATOMIC_INLINE_RW(f64, double) +DEF_ATOMIC_INLINE_RW(type, py_Type) + +#undef DEF_ATOMIC_INLINE_RW + diff --git a/include/pocketpy/common/vector.h b/include/pocketpy/common/vector.h index 7f4e1808..0d556003 100644 --- a/include/pocketpy/common/vector.h +++ b/include/pocketpy/common/vector.h @@ -22,6 +22,7 @@ bool c11_vector__contains(const c11_vector* self, void* elem); void* c11_vector__submit(c11_vector* self, int* length); void c11_vector__swap(c11_vector* self, c11_vector* other); int c11_vector__nextcap(c11_vector* self); +void c11_vector__extend(c11_vector* self, const void* p, int size); #define c11__getitem(T, self, index) (((T*)(self)->data)[index]) #define c11__setitem(T, self, index, value) ((T*)(self)->data)[index] = value; @@ -40,17 +41,6 @@ int c11_vector__nextcap(c11_vector* self); #define c11_vector__back(T, self) (((T*)(self)->data)[(self)->length - 1]) -#define c11_vector__extend(T, self, p, size) \ - do { \ - int min_capacity = (self)->length + (size); \ - if((self)->capacity < min_capacity) { \ - int nextcap = c11_vector__nextcap(self); \ - c11_vector__reserve((self), c11__max(nextcap, min_capacity)); \ - } \ - memcpy((T*)(self)->data + (self)->length, (p), (size) * sizeof(T)); \ - (self)->length += (size); \ - } while(0) - #define c11_vector__insert(T, self, index, elem) \ do { \ if((self)->length == (self)->capacity) { \ diff --git a/include/pocketpy/config.h b/include/pocketpy/config.h index 2fc917e5..8f5d42ea 100644 --- a/include/pocketpy/config.h +++ b/include/pocketpy/config.h @@ -1,10 +1,10 @@ #pragma once // clang-format off -#define PK_VERSION "2.1.6" +#define PK_VERSION "2.1.7" #define PK_VERSION_MAJOR 2 #define PK_VERSION_MINOR 1 -#define PK_VERSION_PATCH 6 +#define PK_VERSION_PATCH 7 /*************** feature settings ***************/ #ifndef PK_ENABLE_OS // can be overridden by cmake diff --git a/include/pocketpy/interpreter/vm.h b/include/pocketpy/interpreter/vm.h index 599b5307..7b3512de 100644 --- a/include/pocketpy/interpreter/vm.h +++ b/include/pocketpy/interpreter/vm.h @@ -62,7 +62,6 @@ typedef struct VM { void* ctx; // user-defined context CachedNames cached_names; - NameDict compile_time_funcs; py_StackRef curr_class; py_StackRef curr_decl_based_function; // this is for get current function without frame diff --git a/include/pocketpy/objects/codeobject.h b/include/pocketpy/objects/codeobject.h index 4d61e98f..a6fd24c3 100644 --- a/include/pocketpy/objects/codeobject.h +++ b/include/pocketpy/objects/codeobject.h @@ -45,26 +45,26 @@ typedef enum Opcode { } Opcode; typedef struct Bytecode { - uint8_t op; + uint16_t op; uint16_t arg; } Bytecode; void Bytecode__set_signed_arg(Bytecode* self, int arg); bool Bytecode__is_forward_jump(const Bytecode* self); -typedef struct CodeBlock { - CodeBlockType type; - int parent; // parent index in blocks - int start; // start index of this block in codes, inclusive - int end; // end index of this block in codes, exclusive - int end2; // ... -} CodeBlock; - typedef struct BytecodeEx { - int lineno; // line number for each bytecode - int iblock; // block index + int32_t lineno; // line number for each bytecode + int32_t iblock; // block index } BytecodeEx; +typedef struct CodeBlock { + int32_t type; + int32_t parent; // parent index in blocks + int32_t start; // start index of this block in codes, inclusive + int32_t end; // end index of this block in codes, exclusive + int32_t end2; // ... +} CodeBlock; + typedef struct CodeObject { SourceData_ src; c11_string* name; @@ -74,8 +74,8 @@ typedef struct CodeObject { c11_vector /*T=py_TValue*/ consts; // constants c11_vector /*T=py_Name*/ varnames; // local variables - c11_vector /*T=py_Name*/ names; - int nlocals; + c11_vector /*T=py_Name*/ names; // non-local names + int nlocals; // number of local variables c11_smallmap_n2d varnames_inv; c11_smallmap_n2d names_inv; @@ -93,6 +93,10 @@ int CodeObject__add_varname(CodeObject* self, py_Name name); int CodeObject__add_name(CodeObject* self, py_Name name); void CodeObject__gc_mark(const CodeObject* self, c11_vector* p_stack); +// Serialization +void* CodeObject__dumps(const CodeObject* co, int* size); +char* CodeObject__loads(const void* data, int size, const char* filename, CodeObject* out); + typedef struct FuncDeclKwArg { int index; // index in co->varnames py_Name key; // name of this argument @@ -110,7 +114,7 @@ typedef struct FuncDecl { int starred_kwarg; // index in co->varnames, -1 if no **kwarg bool nested; // whether this function is nested - const char* docstring; // docstring of this function (weak ref) + char* docstring; FuncType type; c11_smallmap_n2d kw_to_index; @@ -125,6 +129,7 @@ void FuncDecl__add_kwarg(FuncDecl* self, py_Name name, const py_TValue* value); void FuncDecl__add_starred_arg(FuncDecl* self, py_Name name); void FuncDecl__add_starred_kwarg(FuncDecl* self, py_Name name); void FuncDecl__gc_mark(const FuncDecl* self, c11_vector* p_stack); +void FuncDecl__dtor(FuncDecl* self); // runtime function typedef struct Function { diff --git a/include/pocketpy/pocketpy.h b/include/pocketpy/pocketpy.h index 8fa0296f..22432871 100644 --- a/include/pocketpy/pocketpy.h +++ b/include/pocketpy/pocketpy.h @@ -79,8 +79,8 @@ typedef void (*py_TraceFunc)(py_Frame* frame, enum py_TraceEvent); /// A struct contains the callbacks of the VM. typedef struct py_Callbacks { - /// Used by `__import__` to load a source module. - char* (*importfile)(const char*); + /// Used by `__import__` to load a source or compiled module. + char* (*importfile)(const char* path, int* data_size); /// Called before `importfile` to lazy-import a C module. PY_MAYBENULL py_GlobalRef (*lazyimport)(const char*); /// Used by `print` to output a string. @@ -182,6 +182,9 @@ PK_API bool py_compile(const char* source, const char* filename, enum py_CompileMode mode, bool is_dynamic) PY_RAISE PY_RETURN; +/// Compile a `.py` file into a `.pyc` file. +PK_API bool py_compilefile(const char* src_path, + const char* dst_path) PY_RAISE; /// Run a source string. /// @param source source string. /// @param filename filename (for error messages). @@ -300,10 +303,6 @@ PK_API void py_bindproperty(py_Type type, const char* name, py_CFunction getter, py_CFunction setter); /// Bind a magic method to type. PK_API void py_bindmagic(py_Type type, py_Name name, py_CFunction f); -/// Bind a compile-time function via "decl-based" style. -PK_API void py_macrobind(const char* sig, py_CFunction f); -/// Get a compile-time function by name. -PK_API py_ItemRef py_macroget(py_Name name); /************* Value Cast *************/ diff --git a/include/pybind11/tests/module.cpp b/include/pybind11/tests/module.cpp index feaf8aa1..d12cba0d 100644 --- a/include/pybind11/tests/module.cpp +++ b/include/pybind11/tests/module.cpp @@ -94,8 +94,8 @@ struct import_callback { _importfile = nullptr; }; - static char* importfile(const char* path) { - if(value.empty()) return _importfile(path); + static char* importfile(const char* path, int* data_size) { + if(value.empty()) return _importfile(path, data_size); // +1 for the null terminator char* cstr = new char[value.size() + 1]; diff --git a/plugins/flutter/pocketpy/ios/pocketpy.podspec b/plugins/flutter/pocketpy/ios/pocketpy.podspec index e35ca07a..ce2eee34 100644 --- a/plugins/flutter/pocketpy/ios/pocketpy.podspec +++ b/plugins/flutter/pocketpy/ios/pocketpy.podspec @@ -35,7 +35,7 @@ A new Flutter FFI plugin project. s.prepare_command = <<-CMD rm -rf pocketpy - git clone --branch v2.1.6 --depth 1 https://github.com/pocketpy/pocketpy.git + git clone --branch v2.1.7 --depth 1 https://github.com/pocketpy/pocketpy.git cd pocketpy git submodule update --init --recursive --depth 1 bash build_ios_libs.sh diff --git a/plugins/flutter/pocketpy/macos/pocketpy.podspec b/plugins/flutter/pocketpy/macos/pocketpy.podspec index 00defef9..57b9087e 100644 --- a/plugins/flutter/pocketpy/macos/pocketpy.podspec +++ b/plugins/flutter/pocketpy/macos/pocketpy.podspec @@ -32,7 +32,7 @@ A new Flutter FFI plugin project. s.prepare_command = <<-CMD rm -rf pocketpy - git clone --branch v2.1.6 --depth 1 https://github.com/pocketpy/pocketpy.git + git clone --branch v2.1.7 --depth 1 https://github.com/pocketpy/pocketpy.git cd pocketpy git submodule update --init --recursive --depth 1 bash build_darwin_libs.sh diff --git a/plugins/flutter/pocketpy/pubspec.yaml b/plugins/flutter/pocketpy/pubspec.yaml index 575994b3..72f6058d 100644 --- a/plugins/flutter/pocketpy/pubspec.yaml +++ b/plugins/flutter/pocketpy/pubspec.yaml @@ -1,6 +1,6 @@ name: pocketpy description: A lightweight Python interpreter for game engines. It supports Android/iOS/Windows/Linux/MacOS. -version: 2.1.6 +version: 2.1.7 homepage: https://pocketpy.dev repository: https://github.com/pocketpy/pocketpy diff --git a/plugins/flutter/pocketpy/src/CMakeLists.txt b/plugins/flutter/pocketpy/src/CMakeLists.txt index aa673595..31fb767f 100644 --- a/plugins/flutter/pocketpy/src/CMakeLists.txt +++ b/plugins/flutter/pocketpy/src/CMakeLists.txt @@ -21,7 +21,7 @@ set(PK_BUILD_SHARED_LIB ON CACHE BOOL "" FORCE) FetchContent_Declare( pocketpy GIT_REPOSITORY https://github.com/pocketpy/pocketpy.git - GIT_TAG v2.1.6 + GIT_TAG v2.1.7 ) FetchContent_MakeAvailable(pocketpy) diff --git a/src/common/chunkedvector.c b/src/common/chunkedvector.c index f965c32b..36aebfa4 100644 --- a/src/common/chunkedvector.c +++ b/src/common/chunkedvector.c @@ -10,7 +10,7 @@ PK_INLINE int c11__bit_length(unsigned long x) { #if(defined(__clang__) || defined(__GNUC__)) return x == 0 ? 0 : (int)sizeof(unsigned long) * 8 - __builtin_clzl(x); #elif defined(_MSC_VER) - static_assert(sizeof(unsigned long) <= 4, "unsigned long is greater than 4 bytes"); + _Static_assert(sizeof(unsigned long) <= 4, "unsigned long is greater than 4 bytes"); unsigned long msb; if(_BitScanReverse(&msb, x)) { return (int)msb + 1; } return 0; diff --git a/src/common/serialize.c b/src/common/serialize.c new file mode 100644 index 00000000..f3920641 --- /dev/null +++ b/src/common/serialize.c @@ -0,0 +1,102 @@ +#include "pocketpy/common/serialize.h" + +// >>> ord('🥕') +// 129365 +// >>> ord('🍋') +// 127819 + +static uint32_t c11__checksum_32bit(const void* data, int size){ + const uint8_t* p = (const uint8_t*)data; + uint32_t res = 0; + for(int i = 0; i < size; i++){ + res = res * 31 + p[i]; + } + return res; +} + +void c11_serializer__ctor(c11_serializer* self, int16_t magic, int8_t major_ver, int8_t minor_ver){ + c11_vector__ctor(&self->data, 1); + c11_serializer__write_i16(self, magic); + c11_serializer__write_i8(self, major_ver); + c11_serializer__write_i8(self, minor_ver); +} + +void c11_serializer__dtor(c11_serializer* self){ + c11_vector__dtor(&self->data); +} + +void c11_serializer__write_cstr(c11_serializer *self, const char* cstr) { + int len = (int)strlen(cstr); + c11_serializer__write_bytes(self, cstr, len + 1); +} + +void c11_serializer__write_bytes(c11_serializer* self, const void* data, int size){ + c11_vector__extend(&self->data, data, size); +} + +void* c11_serializer__submit(c11_serializer* self, int* size){ + uint32_t checksum = c11__checksum_32bit(self->data.data, self->data.length); + c11_serializer__write_bytes(self, &checksum, sizeof(uint32_t)); + return c11_vector__submit(&self->data, size); +} + +void c11_deserializer__ctor(c11_deserializer* self, const void* data, int size){ + self->data = (const uint8_t*)data; + self->size = size; + self->index = 0; + self->error_msg[0] = '\0'; +} + +void c11_deserializer__dtor(c11_deserializer* self){ + // nothing to do +} + +bool c11_deserializer__error(c11_deserializer* self, const char* msg){ + snprintf(self->error_msg, sizeof(self->error_msg), "%s", msg); + return false; +} + +bool c11_deserializer__check_header(c11_deserializer* self, int16_t magic, int8_t major_ver, int8_t minor_ver_min){ + if(self->size < 8){ + return c11_deserializer__error(self, "bad header: size < 8"); + } + // read 16bit magic + int16_t file_magic = c11_deserializer__read_i16(self); + if(file_magic != magic){ + return c11_deserializer__error(self, "bad header: magic mismatch"); + } + // read 16bit version + self->major_ver = c11_deserializer__read_i8(self); + self->minor_ver = c11_deserializer__read_i8(self); + + // check checksum + uint32_t checksum; + memcpy(&checksum, self->data + self->size - 4, sizeof(uint32_t)); + uint32_t actual_checksum = c11__checksum_32bit(self->data, self->size - 4); + if(checksum != actual_checksum){ + return c11_deserializer__error(self, "bad header: checksum mismatch"); + } + // check version + if(self->major_ver != major_ver){ + return c11_deserializer__error(self, "bad header: major version mismatch"); + } + if(self->minor_ver < minor_ver_min){ + // file_ver: 1.1, require_ver: 1.0 => ok + // file_ver: 1.1, require_ver: 1.1 => ok + // file_ver: 1.1, require_ver: 1.2 => error + return c11_deserializer__error(self, "bad header: minor version mismatch"); + } + return true; +} + +const char* c11_deserializer__read_cstr(c11_deserializer* self){ + const char* p = (const char*)(self->data + self->index); + self->index += (strlen(p) + 1); + return p; +} + +void* c11_deserializer__read_bytes(c11_deserializer* self, int size){ + void* p = (void*)(self->data + self->index); + self->index += size; + return p; +} \ No newline at end of file diff --git a/src/common/sourcedata.c b/src/common/sourcedata.c index aff54ff0..85765156 100644 --- a/src/common/sourcedata.c +++ b/src/common/sourcedata.c @@ -8,10 +8,16 @@ static void SourceData__ctor(struct SourceData* self, const char* filename, enum py_CompileMode mode, bool is_dynamic) { - self->filename = c11_string__new(filename); self->mode = mode; + self->is_dynamic = is_dynamic; + self->filename = c11_string__new(filename); c11_vector__ctor(&self->line_starts, sizeof(const char*)); + if(!source) { + self->source = NULL; + return; + } + // Skip utf8 BOM if there is any. if(strncmp(source, "\xEF\xBB\xBF", 3) == 0) source += 3; // Drop all '\r' @@ -34,13 +40,12 @@ static void SourceData__ctor(struct SourceData* self, self->source->data[last_index + 1] = '\0'; } - self->is_dynamic = is_dynamic; c11_vector__push(const char*, &self->line_starts, self->source->data); } static void SourceData__dtor(struct SourceData* self) { c11_string__delete(self->filename); - c11_string__delete(self->source); + if(self->source) c11_string__delete(self->source); c11_vector__dtor(&self->line_starts); } @@ -59,7 +64,7 @@ bool SourceData__get_line(const struct SourceData* self, int lineno, const char** st, const char** ed) { - if(lineno < 0) return false; + if(lineno < 0 || !self->source) return false; lineno -= 1; if(lineno < 0) lineno = 0; const char* _start = c11__getitem(const char*, &self->line_starts, lineno); diff --git a/src/common/sstream.c b/src/common/sstream.c index b9cda467..36fc223c 100644 --- a/src/common/sstream.c +++ b/src/common/sstream.c @@ -79,7 +79,7 @@ void c11_sbuf__write_cstr(c11_sbuf* self, const char* str) { } void c11_sbuf__write_cstrn(c11_sbuf* self, const char* str, int n) { - c11_vector__extend(char, &self->data, str, n); + c11_vector__extend(&self->data, str, n); } void c11_sbuf__write_quoted(c11_sbuf* self, c11_sv sv, char quote) { diff --git a/src/common/vector.c b/src/common/vector.c index b1a152f6..cffea737 100644 --- a/src/common/vector.c +++ b/src/common/vector.c @@ -75,4 +75,15 @@ int c11_vector__nextcap(c11_vector* self) { // increase by 25% return self->capacity + (self->capacity >> 2); } -} \ No newline at end of file +} + +void c11_vector__extend(c11_vector* self, const void* p, int size) { + int min_capacity = self->length + size; + if(self->capacity < min_capacity) { + int nextcap = c11_vector__nextcap(self); + c11_vector__reserve((self), c11__max(nextcap, min_capacity)); + } + void* dst = (char*)self->data + self->length * self->elem_size; + memcpy(dst, p, size * self->elem_size); + self->length += size; +} diff --git a/src/compiler/compiler.c b/src/compiler/compiler.c index ce50303d..b374244c 100644 --- a/src/compiler/compiler.c +++ b/src/compiler/compiler.c @@ -76,7 +76,6 @@ static int Ctx__prepare_loop_divert(Ctx* self, int line, bool is_break); static int Ctx__enter_block(Ctx* self, CodeBlockType type); static void Ctx__exit_block(Ctx* self); static int Ctx__emit_(Ctx* self, Opcode opcode, uint16_t arg, int line); -// static void Ctx__revert_last_emit_(Ctx* self); static int Ctx__emit_int(Ctx* self, int64_t value, int line); static void Ctx__patch_jump(Ctx* self, int index); static void Ctx__emit_jump(Ctx* self, int target, int line); @@ -1177,7 +1176,7 @@ static void Ctx__s_emit_decorators(Ctx* self, int count) { } static int Ctx__emit_(Ctx* self, Opcode opcode, uint16_t arg, int line) { - Bytecode bc = {(uint8_t)opcode, arg}; + Bytecode bc = {(uint16_t)opcode, arg}; BytecodeEx bcx = {line, self->curr_iblock}; c11_vector__push(Bytecode, &self->co->codes, bc); c11_vector__push(BytecodeEx, &self->co->codes_ex, bcx); @@ -1187,11 +1186,6 @@ static int Ctx__emit_(Ctx* self, Opcode opcode, uint16_t arg, int line) { return i; } -// static void Ctx__revert_last_emit_(Ctx* self) { -// c11_vector__pop(&self->co->codes); -// c11_vector__pop(&self->co->codes_ex); -// } - static int Ctx__emit_int(Ctx* self, int64_t value, int line) { if(INT16_MIN <= value && value <= INT16_MAX) { return Ctx__emit_(self, OP_LOAD_SMALL_INT, (uint16_t)value, line); @@ -1881,65 +1875,11 @@ static Error* exprMap(Compiler* self) { static Error* read_literal(Compiler* self, py_Ref out); -static Error* exprCompileTimeCall(Compiler* self, py_ItemRef func, int line) { - Error* err; - py_push(func); - py_pushnil(); - - uint16_t argc = 0; - uint16_t kwargc = 0; - // copied from `exprCall` - do { - if(curr()->type == TK_RPAREN) break; - if(curr()->type == TK_ID && next()->type == TK_ASSIGN) { - consume(TK_ID); - py_Name key = py_namev(Token__sv(prev())); - consume(TK_ASSIGN); - // k=v - py_pushname(key); - check(read_literal(self, py_pushtmp())); - kwargc += 1; - } else { - if(kwargc > 0) { - return SyntaxError(self, "positional argument follows keyword argument"); - } - check(read_literal(self, py_pushtmp())); - argc += 1; - } - } while(match(TK_COMMA)); - consume(TK_RPAREN); - - py_StackRef p0 = py_peek(0); - bool ok = py_vectorcall(argc, kwargc); - if(!ok) { - char* msg = py_formatexc(); - py_clearexc(p0); - err = SyntaxError(self, "compile-time call error:\n%s", msg); - PK_FREE(msg); - return err; - } - - // TODO: optimize string dedup - int index = Ctx__add_const(ctx(), py_retval()); - Ctx__s_push(ctx(), (Expr*)LoadConstExpr__new(line, index)); - return NULL; -} - static Error* exprCall(Compiler* self) { Error* err; Expr* callable = Ctx__s_popx(ctx()); int line = prev()->line; - if(callable->vt->is_name) { - NameExpr* ne = (NameExpr*)callable; - py_ItemRef func = py_macroget(ne->name); - if(func != NULL) { - py_StackRef p0 = py_peek(0); - err = exprCompileTimeCall(self, func, line); - if(err != NULL) py_clearexc(p0); - return err; - } - } - + CallExpr* e = CallExpr__new(line, callable); Ctx__s_push(ctx(), (Expr*)e); // push onto the stack in advance do { @@ -2425,7 +2365,7 @@ static Error* compile_function(Compiler* self, int decorators) { py_TValue* consts = decl->code.consts.data; py_TValue* c = &consts[codes[0].arg]; if(py_isstr(c)) { - decl->docstring = py_tostr(c); + decl->docstring = c11_strdup(py_tostr(c)); codes[0].op = OP_NO_OP; codes[1].op = OP_NO_OP; } diff --git a/src/interpreter/py_compile.c b/src/interpreter/py_compile.c new file mode 100644 index 00000000..bfecedb9 --- /dev/null +++ b/src/interpreter/py_compile.c @@ -0,0 +1,39 @@ +#include "pocketpy/pocketpy.h" +#include "pocketpy/interpreter/vm.h" +#include + +bool py_compilefile(const char* src_path, const char* dst_path) { + // read + FILE* fp = fopen(src_path, "rb"); + if(fp == NULL) { + const char* msg = strerror(errno); + return OSError("[Errno %d] %s: '%s'", errno, msg, src_path); + } + fseek(fp, 0, SEEK_END); + long size = ftell(fp); + fseek(fp, 0, SEEK_SET); + char* buffer = PK_MALLOC(size + 1); + size = fread(buffer, 1, size, fp); + buffer[size] = 0; + fclose(fp); + // compile + bool ok = py_compile(buffer, src_path, EXEC_MODE, false); + PK_FREE(buffer); + if(!ok) return false; + // dump + py_assign(py_pushtmp(), py_retval()); + int bc_size; + void* bc_data = CodeObject__dumps(py_touserdata(py_peek(-1)), &bc_size); + py_pop(); + // write + fp = fopen(dst_path, "wb"); + if(fp == NULL) { + PK_FREE(bc_data); + const char* msg = strerror(errno); + return OSError("[Errno %d] %s: '%s'", errno, msg, dst_path); + } + fwrite(bc_data, 1, bc_size, fp); + fclose(fp); + PK_FREE(bc_data); + return true; +} diff --git a/src/interpreter/vm.c b/src/interpreter/vm.c index 7117b680..89e38ea8 100644 --- a/src/interpreter/vm.c +++ b/src/interpreter/vm.c @@ -12,7 +12,7 @@ #include #include -static char* pk_default_importfile(const char* path) { +static char* pk_default_importfile(const char* path, int* data_size) { #if PK_ENABLE_OS FILE* f = fopen(path, "rb"); if(f == NULL) return NULL; @@ -23,6 +23,7 @@ static char* pk_default_importfile(const char* path) { size = fread(buffer, 1, size, f); buffer[size] = 0; fclose(f); + if(data_size) *data_size = (int)size; return buffer; #else return NULL; @@ -112,7 +113,6 @@ void VM__ctor(VM* self) { self->stack.end = self->stack.begin + PK_VM_STACK_SIZE; CachedNames__ctor(&self->cached_names); - NameDict__ctor(&self->compile_time_funcs, PK_TYPE_ATTR_LOAD_FACTOR); /* Init Builtin Types */ // 0: unused @@ -306,7 +306,6 @@ void VM__dtor(VM* self) { BinTree__dtor(&self->modules); FixedMemoryPool__dtor(&self->pool_frame); CachedNames__dtor(&self->cached_names); - NameDict__dtor(&self->compile_time_funcs); c11_vector__dtor(&self->types); } @@ -671,12 +670,6 @@ void ManagedHeap__mark(ManagedHeap* self) { CachedNames_KV* kv = c11_chunkedvector__at(&vm->cached_names.entries, i); pk__mark_value(&kv->val); } - // mark compile time functions - for(int i = 0; i < vm->compile_time_funcs.capacity; i++) { - NameDict_KV* kv = &vm->compile_time_funcs.items[i]; - if(kv->key == NULL) continue; - pk__mark_value(&kv->value); - } // mark types int types_length = vm->types.length; // 0-th type is placeholder diff --git a/src/modules/pickle.c b/src/modules/pickle.c index 1c2618fc..51a7d2fb 100644 --- a/src/modules/pickle.c +++ b/src/modules/pickle.c @@ -59,7 +59,7 @@ static void PickleObject__dtor(PickleObject* self) { static bool PickleObject__py_submit(PickleObject* self, py_OutRef out); static void PickleObject__write_bytes(PickleObject* buf, const void* data, int size) { - c11_vector__extend(char, &buf->codes, data, size); + c11_vector__extend(&buf->codes, data, size); } static void c11_sbuf__write_type_path(c11_sbuf* path_buf, py_Type type) { diff --git a/src/objects/codeobject.c b/src/objects/codeobject.c index b0fe51d5..68524a05 100644 --- a/src/objects/codeobject.c +++ b/src/objects/codeobject.c @@ -17,11 +17,12 @@ bool Bytecode__is_forward_jump(const Bytecode* self) { (op == OP_FOR_ITER || op == OP_FOR_ITER_YIELD_VALUE); } -static void FuncDecl__dtor(FuncDecl* self) { +void FuncDecl__dtor(FuncDecl* self) { CodeObject__dtor(&self->code); c11_vector__dtor(&self->args); c11_vector__dtor(&self->kwargs); c11_smallmap_n2d__dtor(&self->kw_to_index); + if(self->docstring) py_free(self->docstring); } FuncDecl_ FuncDecl__rcnew(SourceData_ src, c11_sv name) { @@ -30,7 +31,7 @@ FuncDecl_ FuncDecl__rcnew(SourceData_ src, c11_sv name) { self->rc.dtor = (void (*)(void*))FuncDecl__dtor; CodeObject__ctor(&self->code, src, name); - c11_vector__ctor(&self->args, sizeof(int)); + c11_vector__ctor(&self->args, sizeof(int32_t)); c11_vector__ctor(&self->kwargs, sizeof(FuncDeclKwArg)); self->starred_arg = -1; @@ -66,8 +67,8 @@ bool FuncDecl__is_duplicated_arg(const FuncDecl* decl, py_Name name) { } void FuncDecl__add_arg(FuncDecl* self, py_Name name) { - int index = CodeObject__add_varname(&self->code, name); - c11_vector__push(int, &self->args, index); + int32_t index = CodeObject__add_varname(&self->code, name); + c11_vector__push(int32_t, &self->args, index); } void FuncDecl__add_kwarg(FuncDecl* self, py_Name name, const py_TValue* value) { @@ -89,32 +90,6 @@ void FuncDecl__add_starred_kwarg(FuncDecl* self, py_Name name) { self->starred_kwarg = index; } -FuncDecl_ FuncDecl__build(c11_sv name, - c11_sv* args, - int argc, - c11_sv starred_arg, - c11_sv* kwargs, - int kwargc, - py_Ref kwdefaults, // a tuple contains default values - c11_sv starred_kwarg, - const char* docstring) { - SourceData_ source = SourceData__rcnew("pass", "", EXEC_MODE, false); - FuncDecl_ decl = FuncDecl__rcnew(source, name); - for(int i = 0; i < argc; i++) { - FuncDecl__add_arg(decl, py_namev(args[i])); - } - if(starred_arg.size) { FuncDecl__add_starred_arg(decl, py_namev(starred_arg)); } - assert(py_istype(kwdefaults, tp_tuple)); - assert(py_tuple_len(kwdefaults) == kwargc); - for(int i = 0; i < kwargc; i++) { - FuncDecl__add_kwarg(decl, py_namev(kwargs[i]), py_tuple_getitem(kwdefaults, i)); - } - if(starred_kwarg.size) FuncDecl__add_starred_kwarg(decl, py_namev(starred_kwarg)); - decl->docstring = docstring; - PK_DECREF(source); - return decl; -} - void CodeObject__ctor(CodeObject* self, SourceData_ src, c11_sv name) { self->src = src; PK_INCREF(src); diff --git a/src/objects/codeobject_ser.c b/src/objects/codeobject_ser.c new file mode 100644 index 00000000..4dcb29e4 --- /dev/null +++ b/src/objects/codeobject_ser.c @@ -0,0 +1,368 @@ +#include "pocketpy/objects/codeobject.h" +#include "pocketpy/common/serialize.h" +#include "pocketpy/common/utils.h" + +// Magic number for CodeObject serialization: "CO" = 0x434F +#define CODEOBJECT_MAGIC 0x434F +#define CODEOBJECT_VER_MAJOR 1 +#define CODEOBJECT_VER_MINOR 0 +#define CODEOBJECT_VER_MINOR_MIN 0 + +// Forward declarations +static void FuncDecl__serialize(c11_serializer* s, + const FuncDecl* decl, + const struct SourceData* parent_src); +static FuncDecl_ FuncDecl__deserialize(c11_deserializer* d, SourceData_ embedded_src); +static void CodeObject__serialize(c11_serializer* s, + const CodeObject* co, + const struct SourceData* parent_src); +static CodeObject CodeObject__deserialize(c11_deserializer* d, const char* filename, SourceData_ embedded_src); + +// Serialize a py_TValue constant +// Supported types: None, int, float, bool, str, bytes, tuple, Ellipsis +static void TValue__serialize(c11_serializer* s, py_Ref val) { + c11_serializer__write_type(s, val->type); + // 1. co_consts: int | float | str + // 2. function defaults: see `read_literal()` in compiler.c + switch(val->type) { + case tp_int: c11_serializer__write_i64(s, val->_i64); break; + case tp_float: c11_serializer__write_f64(s, val->_f64); break; + case tp_str: { + c11_sv sv = py_tosv((py_Ref)val); + c11_serializer__write_i32(s, sv.size); + c11_serializer__write_bytes(s, sv.data, sv.size); + break; + } + case tp_bool: { + bool value = py_tobool(val); + c11_serializer__write_i8(s, value ? 1 : 0); + break; + } + case tp_NoneType: break; + case tp_ellipsis: break; + case tp_tuple: { + int len = py_tuple_len(val); + c11_serializer__write_i32(s, len); + for(int i = 0; i < len; i++) { + py_Ref item = py_tuple_getitem(val, i); + TValue__serialize(s, item); + } + break; + } + default: c11__abort("TValue__serialize: invalid type '%s'", py_tpname(val->type)); + } +} + +// Deserialize a py_TValue constant +static void TValue__deserialize(c11_deserializer* d, py_OutRef out) { + py_Type type = c11_deserializer__read_type(d); + switch(type) { + case tp_int: { + py_i64 v = c11_deserializer__read_i64(d); + py_newint(out, v); + break; + } + case tp_float: { + py_f64 v = c11_deserializer__read_f64(d); + py_newfloat(out, v); + break; + } + case tp_str: { + int size = c11_deserializer__read_i32(d); + char* dst = py_newstrn(out, size); + char* src = c11_deserializer__read_bytes(d, size); + memcpy(dst, src, size); + break; + } + case tp_bool: { + bool v = c11_deserializer__read_i8(d) != 0; + py_newbool(out, v); + break; + } + case tp_NoneType: { + py_newnone(out); + break; + } + case tp_ellipsis: { + py_newellipsis(out); + break; + } + case tp_tuple: { + int len = c11_deserializer__read_i32(d); + py_newtuple(out, len); + for(int i = 0; i < len; i++) { + py_ItemRef item = py_tuple_getitem(out, i); + TValue__deserialize(d, item); + } + break; + } + default: + c11__abort("TValue__deserialize: invalid type '%s'", py_tpname(type)); + } +} + +// Serialize CodeObject +static void CodeObject__serialize(c11_serializer* s, + const CodeObject* co, + const struct SourceData* parent_src) { + // SourceData + if(parent_src) { + c11__rtassert(co->src == parent_src); + } + + // name + c11_serializer__write_cstr(s, co->name->data); + + // codes + _Static_assert(sizeof(Bytecode) == sizeof(uint16_t) * 2, ""); + c11_serializer__write_i32(s, co->codes.length); + c11_serializer__write_bytes(s, co->codes.data, co->codes.length * sizeof(Bytecode)); + + // codes_ex + _Static_assert(sizeof(BytecodeEx) == sizeof(int32_t) * 2, ""); + c11_serializer__write_i32(s, co->codes_ex.length); + c11_serializer__write_bytes(s, co->codes_ex.data, co->codes_ex.length * sizeof(BytecodeEx)); + + // consts + c11_serializer__write_i32(s, co->consts.length); + for(int i = 0; i < co->consts.length; i++) { + py_Ref val = c11__at(py_TValue, &co->consts, i); + TValue__serialize(s, val); + } + + // varnames (as cstr via py_name2str) + c11_serializer__write_i32(s, co->varnames.length); + for(int i = 0; i < co->varnames.length; i++) { + py_Name name = c11__getitem(py_Name, &co->varnames, i); + c11_serializer__write_cstr(s, py_name2str(name)); + } + + // names (as cstr via py_name2str) + c11_serializer__write_i32(s, co->names.length); + for(int i = 0; i < co->names.length; i++) { + py_Name name = c11__getitem(py_Name, &co->names, i); + c11_serializer__write_cstr(s, py_name2str(name)); + } + + // nlocals + c11_serializer__write_i32(s, co->nlocals); + + // blocks + _Static_assert(sizeof(CodeBlock) == sizeof(int32_t) * 5, ""); + c11_serializer__write_i32(s, co->blocks.length); + c11_serializer__write_bytes(s, co->blocks.data, co->blocks.length * sizeof(CodeBlock)); + + // func_decls + c11_serializer__write_i32(s, co->func_decls.length); + for(int i = 0; i < co->func_decls.length; i++) { + const FuncDecl* decl = c11__getitem(FuncDecl_, &co->func_decls, i); + FuncDecl__serialize(s, decl, co->src); + } + + // start_line, end_line + c11_serializer__write_i32(s, co->start_line); + c11_serializer__write_i32(s, co->end_line); +} + +// Deserialize CodeObject (initialize co before calling) +static CodeObject CodeObject__deserialize(c11_deserializer* d, const char* filename, SourceData_ embedded_src) { + CodeObject co; + + // SourceData + SourceData_ src; + if(embedded_src != NULL) { + c11__rtassert(filename == NULL); + src = embedded_src; + PK_INCREF(src); + } else { + c11__rtassert(filename != NULL); + src = SourceData__rcnew(NULL, filename, EXEC_MODE, false); + } + + // name + const char* name = c11_deserializer__read_cstr(d); + c11_sv name_sv = {name, strlen(name)}; + + // Initialize the CodeObject + CodeObject__ctor(&co, src, name_sv); + PK_DECREF(src); // CodeObject__ctor increments ref count + // Clear the default root block that CodeObject__ctor adds + c11_vector__clear(&co.blocks); + + // codes + int codes_len = c11_deserializer__read_i32(d); + c11_vector__extend(&co.codes, + c11_deserializer__read_bytes(d, codes_len * sizeof(Bytecode)), + codes_len); + // codes_ex + int codes_ex_len = c11_deserializer__read_i32(d); + c11_vector__extend(&co.codes_ex, + c11_deserializer__read_bytes(d, codes_ex_len * sizeof(BytecodeEx)), + codes_ex_len); + + // consts + int consts_len = c11_deserializer__read_i32(d); + for(int i = 0; i < consts_len; i++) { + py_Ref p_val = c11_vector__emplace(&co.consts); + TValue__deserialize(d, p_val); + } + + // varnames + int varnames_len = c11_deserializer__read_i32(d); + for(int i = 0; i < varnames_len; i++) { + const char* s = c11_deserializer__read_cstr(d); + py_Name n = py_name(s); + c11_vector__push(py_Name, &co.varnames, n); + c11_smallmap_n2d__set(&co.varnames_inv, n, i); + } + + // names + int names_len = c11_deserializer__read_i32(d); + for(int i = 0; i < names_len; i++) { + const char* s = c11_deserializer__read_cstr(d); + py_Name n = py_name(s); + c11_vector__push(py_Name, &co.names, n); + c11_smallmap_n2d__set(&co.names_inv, n, i); + } + + // nlocals + co.nlocals = c11_deserializer__read_i32(d); + + // blocks + int blocks_len = c11_deserializer__read_i32(d); + c11_vector__extend(&co.blocks, + c11_deserializer__read_bytes(d, blocks_len * sizeof(CodeBlock)), + blocks_len); + // func_decls + int func_decls_len = c11_deserializer__read_i32(d); + for(int i = 0; i < func_decls_len; i++) { + FuncDecl_ decl = FuncDecl__deserialize(d, src); + c11_vector__push(FuncDecl_, &co.func_decls, decl); + } + + // start_line, end_line + co.start_line = c11_deserializer__read_i32(d); + co.end_line = c11_deserializer__read_i32(d); + + return co; +} + +// Serialize FuncDecl +static void FuncDecl__serialize(c11_serializer* s, + const FuncDecl* decl, + const struct SourceData* parent_src) { + // CodeObject (embedded) + CodeObject__serialize(s, &decl->code, parent_src); + + // args + c11_serializer__write_i32(s, decl->args.length); + c11_serializer__write_bytes(s, decl->args.data, decl->args.length * sizeof(int32_t)); + + // kwargs + c11_serializer__write_i32(s, decl->kwargs.length); + for(int i = 0; i < decl->kwargs.length; i++) { + FuncDeclKwArg* kw = c11__at(FuncDeclKwArg, &decl->kwargs, i); + c11_serializer__write_i32(s, kw->index); + c11_serializer__write_cstr(s, py_name2str(kw->key)); + TValue__serialize(s, &kw->value); + } + + // starred_arg, starred_kwarg + c11_serializer__write_i32(s, decl->starred_arg); + c11_serializer__write_i32(s, decl->starred_kwarg); + + // nested + c11_serializer__write_i8(s, decl->nested ? 1 : 0); + + // docstring + int has_docstring = decl->docstring != NULL ? 1 : 0; + c11_serializer__write_i8(s, has_docstring); + if(has_docstring) c11_serializer__write_cstr(s, decl->docstring); + + // type + c11_serializer__write_i8(s, (int8_t)decl->type); +} + +// Deserialize FuncDecl +static FuncDecl_ FuncDecl__deserialize(c11_deserializer* d, SourceData_ embedded_src) { + FuncDecl* self = PK_MALLOC(sizeof(FuncDecl)); + self->rc.count = 1; + self->rc.dtor = (void (*)(void*))FuncDecl__dtor; + + c11_vector__ctor(&self->args, sizeof(int32_t)); + c11_vector__ctor(&self->kwargs, sizeof(FuncDeclKwArg)); + c11_smallmap_n2d__ctor(&self->kw_to_index); + + // CodeObject (embedded) + self->code = CodeObject__deserialize(d, NULL, embedded_src); + + // args + int args_len = c11_deserializer__read_i32(d); + c11_vector__extend(&self->args, + c11_deserializer__read_bytes(d, args_len * sizeof(int32_t)), + args_len); + + // kwargs + int kwargs_len = c11_deserializer__read_i32(d); + for(int i = 0; i < kwargs_len; i++) { + FuncDeclKwArg* kw = c11_vector__emplace(&self->kwargs); + kw->index = c11_deserializer__read_i32(d); + const char* key_str = c11_deserializer__read_cstr(d); + kw->key = py_name(key_str); + TValue__deserialize(d, &kw->value); + c11_smallmap_n2d__set(&self->kw_to_index, kw->key, kw->index); + } + // starred_arg + self->starred_arg = c11_deserializer__read_i32(d); + // starred_kwarg + self->starred_kwarg = c11_deserializer__read_i32(d); + + // nested + self->nested = c11_deserializer__read_i8(d) != 0; + + // docstring + int has_docstring = c11_deserializer__read_i8(d); + if(has_docstring) { + const char* docstring = c11_deserializer__read_cstr(d); + self->docstring = c11_strdup(docstring); + } else { + self->docstring = NULL; + } + + // type + self->type = (FuncType)c11_deserializer__read_i8(d); + return self; +} + +// Public API: Serialize CodeObject to bytes +void* CodeObject__dumps(const CodeObject* co, int* size) { + c11_serializer s; + c11_serializer__ctor(&s, CODEOBJECT_MAGIC, CODEOBJECT_VER_MAJOR, CODEOBJECT_VER_MINOR); + CodeObject__serialize(&s, co, NULL); + return c11_serializer__submit(&s, size); +} + +// Public API: Deserialize CodeObject from bytes +// Returns error message or NULL on success +char* CodeObject__loads(const void* data, int size, const char* filename, CodeObject* out) { + c11_deserializer d; + c11_deserializer__ctor(&d, data, size); + + if(!c11_deserializer__check_header(&d, + CODEOBJECT_MAGIC, + CODEOBJECT_VER_MAJOR, + CODEOBJECT_VER_MINOR_MIN)) { + char* error_msg = c11_strdup(d.error_msg); + c11_deserializer__dtor(&d); + return error_msg; + } + + *out = CodeObject__deserialize(&d, filename, NULL); + c11_deserializer__dtor(&d); + return NULL; +} + +#undef CODEOBJECT_MAGIC +#undef CODEOBJECT_VER_MAJOR +#undef CODEOBJECT_VER_MINOR +#undef CODEOBJECT_VER_MINOR_MIN diff --git a/src/public/Bindings.c b/src/public/Bindings.c index 28ca8950..47f8de11 100644 --- a/src/public/Bindings.c +++ b/src/public/Bindings.c @@ -46,16 +46,3 @@ void py_bindmagic(py_Type type, py_Name name, py_CFunction f) { py_Ref tmp = py_emplacedict(py_tpobject(type), name); py_newnativefunc(tmp, f); } - -void py_macrobind(const char* sig, py_CFunction f) { - py_Ref tmp = py_pushtmp(); - py_Name name = py_newfunction(tmp, sig, f, NULL, 0); - NameDict__set(&pk_current_vm->compile_time_funcs, name, tmp); - py_pop(); -} - -py_ItemRef py_macroget(py_Name name) { - NameDict* d = &pk_current_vm->compile_time_funcs; - if(d->length == 0) return NULL; - return NameDict__try_get(d, name); -} diff --git a/src/public/GlobalSetup.c b/src/public/GlobalSetup.c index 704fa354..d4d044fc 100644 --- a/src/public/GlobalSetup.c +++ b/src/public/GlobalSetup.c @@ -32,6 +32,10 @@ void py_initialize() { _Static_assert(sizeof(py_TValue) == 24, "sizeof(py_TValue) != 24"); _Static_assert(offsetof(py_TValue, extra) == 4, "offsetof(py_TValue, extra) != 4"); + // check sizes + _Static_assert(sizeof(float) == 4, ""); + _Static_assert(sizeof(double) == 8, ""); + pk_current_vm = pk_all_vm[0] = &pk_default_vm; // initialize some convenient references @@ -117,7 +121,7 @@ void py_sys_setargv(int argc, char** argv) { py_GlobalRef sys = py_getmodule("sys"); py_Ref argv_list = py_getdict(sys, py_name("argv")); py_list_clear(argv_list); - for(int i = 0; i < argc; i++) { + for(int i = 1; i < argc; i++) { py_newstr(py_list_emplace(argv_list), argv[i]); } } diff --git a/src/public/ModuleSystem.c b/src/public/ModuleSystem.c index 3e1b25a7..c641e68c 100644 --- a/src/public/ModuleSystem.c +++ b/src/public/ModuleSystem.c @@ -140,20 +140,39 @@ int py_import(const char* path_cstr) { c11_string* filename = c11_string__new3("%s.py", slashed_path->data); bool need_free = true; + bool is_pyc = false; const char* data = load_kPythonLib(path_cstr); + int data_size = -1; + if(data != NULL) { need_free = false; goto __SUCCESS; } - data = vm->callbacks.importfile(filename->data); + data = vm->callbacks.importfile(filename->data, &data_size); if(data != NULL) goto __SUCCESS; + c11_string__delete(filename); + filename = c11_string__new3("%s.pyc", slashed_path->data); + data = vm->callbacks.importfile(filename->data, &data_size); + if(data != NULL) { + is_pyc = true; + goto __SUCCESS; + } + c11_string__delete(filename); filename = c11_string__new3("%s%c__init__.py", slashed_path->data, PK_PLATFORM_SEP); - data = vm->callbacks.importfile(filename->data); + data = vm->callbacks.importfile(filename->data, &data_size); if(data != NULL) goto __SUCCESS; + c11_string__delete(filename); + filename = c11_string__new3("%s%c__init__.pyc", slashed_path->data, PK_PLATFORM_SEP); + data = vm->callbacks.importfile(filename->data, &data_size); + if(data != NULL) { + is_pyc = true; + goto __SUCCESS; + } + c11_string__delete(filename); c11_string__delete(slashed_path); // not found @@ -162,8 +181,25 @@ int py_import(const char* path_cstr) { __SUCCESS: do { } while(0); + py_GlobalRef mod = py_newmodule(path_cstr); - bool ok = py_exec((const char*)data, filename->data, EXEC_MODE, mod); + + bool ok; + if(is_pyc) { + CodeObject co; + char* err = CodeObject__loads(data, data_size, filename->data, &co); + if(err == NULL) { + c11__rtassert(co.src->mode == EXEC_MODE); + c11__rtassert(co.src->is_dynamic == false); + ok = pk_exec(&co, mod); + } else { + RuntimeError("failed to load %s: %s", filename->data, err); + ok = false; + } + } else { + ok = py_exec(data, filename->data, EXEC_MODE, mod); + } + py_assign(py_retval(), mod); c11_string__delete(filename); @@ -180,11 +216,13 @@ bool py_importlib_reload(py_Ref module) { c11_sv path = c11_string__sv(mi->path); c11_string* slashed_path = c11_sv__replace(path, '.', PK_PLATFORM_SEP); c11_string* filename = c11_string__new3("%s.py", slashed_path->data); - char* data = vm->callbacks.importfile(filename->data); + // Here we only consider source modules. + // Because compiled modules have no source file (it cannot be reloaded) + char* data = vm->callbacks.importfile(filename->data, NULL); if(data == NULL) { c11_string__delete(filename); filename = c11_string__new3("%s%c__init__.py", slashed_path->data, PK_PLATFORM_SEP); - data = vm->callbacks.importfile(filename->data); + data = vm->callbacks.importfile(filename->data, NULL); } c11_string__delete(slashed_path); if(data == NULL) return ImportError("module '%v' not found", path); diff --git a/src/public/PyList.c b/src/public/PyList.c index 761c0a73..67e15d71 100644 --- a/src/public/PyList.c +++ b/src/public/PyList.c @@ -203,8 +203,8 @@ static bool list__add__(int argc, py_Ref argv) { List* list_1 = py_touserdata(_1); py_newlist(py_retval()); List* list = py_touserdata(py_retval()); - c11_vector__extend(py_TValue, list, list_0->data, list_0->length); - c11_vector__extend(py_TValue, list, list_1->data, list_1->length); + c11_vector__extend(list, list_0->data, list_0->length); + c11_vector__extend(list, list_1->data, list_1->length); } else { py_newnotimplemented(py_retval()); } @@ -221,7 +221,7 @@ static bool list__mul__(int argc, py_Ref argv) { List* list = py_touserdata(py_retval()); List* list_0 = py_touserdata(_0); for(int i = 0; i < n; i++) { - c11_vector__extend(py_TValue, list, list_0->data, list_0->length); + c11_vector__extend(list, list_0->data, list_0->length); } } else { py_newnotimplemented(py_retval()); @@ -264,7 +264,7 @@ static bool list_extend(int argc, py_Ref argv) { py_TValue* p; int length = pk_arrayview(py_arg(1), &p); if(length == -1) return TypeError("extend() argument must be a list or tuple"); - c11_vector__extend(py_TValue, self, p, length); + c11_vector__extend(self, p, length); py_newnone(py_retval()); return true; } @@ -293,7 +293,7 @@ static bool list_copy(int argc, py_Ref argv) { py_newlist(py_retval()); List* self = py_touserdata(py_arg(0)); List* list = py_touserdata(py_retval()); - c11_vector__extend(py_TValue, list, self->data, self->length); + c11_vector__extend(list, self->data, self->length); return true; } diff --git a/src/public/ValueCreation.c b/src/public/ValueCreation.c index 426cc8e7..3c496f89 100644 --- a/src/public/ValueCreation.c +++ b/src/public/ValueCreation.c @@ -121,7 +121,7 @@ py_Name py_newfunction(py_OutRef out, c11__abort("py_newfunction(): invalid signature '%s'", sig); } FuncDecl_ decl = c11__getitem(FuncDecl_, &code.func_decls, 0); - decl->docstring = docstring; + if(docstring) decl->docstring = c11_strdup(docstring); // construct the function Function* ud = py_newobject(out, tp_function, slots, sizeof(Function)); Function__ctor(ud, decl, NULL, NULL); diff --git a/src2/main.c b/src2/main.c index d78cdcb5..683b3bf4 100644 --- a/src2/main.c +++ b/src2/main.c @@ -34,7 +34,9 @@ int main(int argc, char** argv) { bool profile = false; bool debug = false; - const char* filename = NULL; + bool compile = false; + const char* arg1 = NULL; + const char* arg2 = NULL; for(int i = 1; i < argc; i++) { if(strcmp(argv[i], "--profile") == 0) { @@ -45,11 +47,19 @@ int main(int argc, char** argv) { debug = true; continue; } - if(filename == NULL) { - filename = argv[i]; + if(strcmp(argv[i], "--compile") == 0) { + compile = true; continue; } - printf("Usage: pocketpy [--profile] [--debug] filename\n"); + if(arg1 == NULL) { + arg1 = argv[i]; + continue; + } + if(arg2 == NULL) { + arg2 = argv[i]; + continue; + } + printf("Usage: pocketpy [--profile] [--debug] [--compile] filename\n"); } if(debug && profile) { @@ -57,9 +67,22 @@ int main(int argc, char** argv) { return 1; } + if(compile && (debug || profile)) { + printf("Error: --compile cannot be used with --debug or --profile.\n"); + return 1; + } + py_initialize(); py_sys_setargv(argc, argv); + if(compile) { + bool ok = py_compilefile(arg1, arg2); + if(!ok) py_printexc(); + py_finalize(); + return ok ? 0 : 1; + } + + const char* filename = arg1; if(filename == NULL) { if(profile) printf("Warning: --profile is ignored in REPL mode.\n"); if(debug) printf("Warning: --debug is ignored in REPL mode.\n"); diff --git a/tests/801_sys.py b/tests/801_sys.py index 2a91949a..9b0912b1 100644 --- a/tests/801_sys.py +++ b/tests/801_sys.py @@ -1,4 +1,5 @@ import sys -assert len(sys.argv) == 2 -assert (sys.argv[1] == 'tests/801_sys.py'), sys.argv +filename = 'tests/801_sys.py' +assert (sys.argv == [filename]), sys.argv + diff --git a/tests/922_py_compile.py b/tests/922_py_compile.py new file mode 100644 index 00000000..522d5cf8 --- /dev/null +++ b/tests/922_py_compile.py @@ -0,0 +1,26 @@ +try: + import os +except ImportError: + print('os is not enabled, skipping test...') + exit(0) + +import sys +if sys.platform == 'win32': + exe_name = 'main.exe' +else: + exe_name = './main' +assert os.system(f'{exe_name} --compile python/heapq.py heapq1.pyc') == 0 +assert os.path.exists('heapq1.pyc') + +import heapq1 +import heapq + +a = [1, 2, -3, 2, 1, 5, 11, 123] * 10 +b = a.copy() + +heapq.heapify(a) +heapq1.heapify(b) + +assert a == b + +os.remove('heapq1.pyc')