diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ccccfdaf..ea10acf6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,7 +36,7 @@ jobs: shell: bash run: | mkdir -p output/x86_64 - python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON + python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON cp main.exe output/x86_64 cp pocketpy.dll output/x86_64 - uses: actions/upload-artifact@v4 @@ -76,7 +76,7 @@ jobs: run: | python scripts/check_pragma_once.py include mkdir -p output/x86_64 - python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON + python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON python scripts/run_tests.py cp main output/x86_64 cp libpocketpy.so output/x86_64 @@ -96,7 +96,7 @@ jobs: submodules: recursive - name: Compile and Test run: | - python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON + python cmake_build.py Release -DPK_BUILD_MODULE_LZ4=ON -DPK_BUILD_MODULE_CUTE_PNG=ON python scripts/run_tests.py - name: Benchmark run: python scripts/run_tests.py benchmark @@ -162,7 +162,7 @@ jobs: uses: jirutka/setup-alpine@v1 with: arch: ${{ matrix.arch }} - packages: gcc g++ make cmake libc-dev linux-headers python3 + packages: gcc g++ make cmake libc-dev linux-headers python3 git - name: Build and Test run: | echo "Building for architecture: ${{ matrix.arch }}" diff --git a/.gitignore b/.gitignore index f4b0ceb0..5d28157c 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ docs/C-API/functions.md cmake-build-* +tmp/ \ No newline at end of file diff --git a/3rd/cute_png/CMakeLists.txt b/3rd/cute_png/CMakeLists.txt new file mode 100644 index 00000000..515be561 --- /dev/null +++ b/3rd/cute_png/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.10) + +project(cute_png) + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) + +add_library(${PROJECT_NAME} STATIC src/cute_png.c) + +target_include_directories(${PROJECT_NAME} PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/include +) diff --git a/3rd/cute_png/include/cute_png.h b/3rd/cute_png/include/cute_png.h new file mode 100644 index 00000000..ab24b254 --- /dev/null +++ b/3rd/cute_png/include/cute_png.h @@ -0,0 +1,1852 @@ +/* + ------------------------------------------------------------------------------ + Licensing information can be found at the end of the file. + ------------------------------------------------------------------------------ + + cute_png.h - v1.05 + + To create implementation (the function definitions) + #define CUTE_PNG_IMPLEMENTATION + in *one* C/CPP file (translation unit) that includes this file + + + SUMMARY: + + This header wraps some very nice functions by Richard Mitton from his + tigr (Tiny Graphics) library, with some additional features and small + bug-fixes. + + + Revision history: + 1.00 (12/23/2016) initial release + 1.01 (03/08/2017) tRNS chunk support for paletted images + 1.02 (10/23/2017) support for explicitly loading paletted png images + 1.03 (11/12/2017) construct atlas in memory + 1.04 (08/23/2018) various bug fixes for filter and word decoder + added `cp_load_blank` + 1.05 (11/10/2022) added `cp_save_png_to_memory` + + + EXAMPLES: + + Loading a PNG from disk, then freeing it + cp_image_t img = cp_load_png("images/pic.png"); + ... + free(img.pix); + CUTE_PNG_MEMSET(&img, 0, sizeof(img)); + + Loading a PNG from memory, then freeing it + cp_image_t img = cp_load_png_mem(memory, sizeof(memory)); + ... + free(img.pix); + CUTE_PNG_MEMSET(&img, 0, sizeof(img)); + + Saving a PNG to disk + cp_save_png("images/example.png", &img); + // img is just a raw RGBA buffer, and can come from anywhere, + // not only from cp_load*** functions + + Creating a texture atlas in memory + int w = 1024; + int h = 1024; + cp_atlas_image_t* imgs_out = (cp_atlas_image_t*)malloc(sizeof(cp_atlas_image_t) * my_png_count); + cp_image_t atlas_img = cp_make_atlas(w, int h, my_png_array, my_png_count, imgs_out); + // just pass an array of pointers to images along with the image count. Make sure to also + // provide an array of `cp_atlas_image_t` for `cp_make_atlas` to output important UV info for the + // images that fit into the atlas. + + Using the default atlas saver + int errors = cp_default_save_atlas("atlas.png", "atlas.txt", atlas_img, atlas_imgs, img_count, names_of_all_images ? names_of_all_images : 0); + if (errors) { ... } + // Atlas info (like uv coordinates) are in "atlas.txt", and the image was writen to "atlas.png". + // atlas_imgs was an array of `cp_atlas_image_t` from the `cp_make_atlas` function. + + Inflating a DEFLATE block (decompressing memory stored in DEFLATE format) + cp_inflate(in, in_bytes, out, out_bytes); + // this function requires knowledge of the un-compressed size + // does *not* do any internal realloc! Will return errors if an + // attempt to overwrite the out buffer is made + + CUSTOMIZATION + + There are various macros in this header you can customize by defining them before + including cute_png.h. Simply define one to override the default behavior. + + CUTE_PNG_ALLOCA + CUTE_PNG_ALLOC + CUTE_PNG_FREE + CUTE_PNG_CALLOC + CUTE_PNG_REALLOC + CUTE_PNG_MEMCPY + CUTE_PNG_MEMCMP + CUTE_PNG_MEMSET + CUTE_PNG_ASSERT + CUTE_PNG_FPRINTF + CUTE_PNG_SEEK_SET + CUTE_PNG_SEEK_END + CUTE_PNG_FILE + CUTE_PNG_FOPEN + CUTE_PNG_FSEEK + CUTE_PNG_FREAD + CUTE_PNG_FTELL + CUTE_PNG_FWRITE + CUTE_PNG_FCLOSE + CUTE_PNG_FERROR + CUTE_PNG_ATLAS_MUST_FIT + CUTE_PNG_ATLAS_FLIP_Y_AXIS_FOR_UV + CUTE_PNG_ATLAS_EMPTY_COLOR +*/ + +/* + Contributors: + Zachary Carter 1.01 - bug catch for tRNS chunk in paletted images + Dennis Korpel 1.03 - fix some pointer/memory related bugs + Dennis Korpel 1.04 - fix for filter on first row of pixels +*/ + +#if !defined(CUTE_PNG_H) + +#ifdef _WIN32 + #if !defined(_CRT_SECURE_NO_WARNINGS) + #define _CRT_SECURE_NO_WARNINGS + #endif +#endif + +#ifndef CUTE_PNG_ATLAS_MUST_FIT + #define CUTE_PNG_ATLAS_MUST_FIT 1 // returns error from cp_make_atlas if *any* input image does not fit +#endif // CUTE_PNG_ATLAS_MUST_FIT + +#ifndef CUTE_PNG_ATLAS_FLIP_Y_AXIS_FOR_UV + #define CUTE_PNG_ATLAS_FLIP_Y_AXIS_FOR_UV 1 // flips output uv coordinate's y. Can be useful to "flip image on load" +#endif // CUTE_PNG_ATLAS_FLIP_Y_AXIS_FOR_UV + +#ifndef CUTE_PNG_ATLAS_EMPTY_COLOR + #define CUTE_PNG_ATLAS_EMPTY_COLOR 0x000000FF // the fill color for empty areas in a texture atlas (RGBA) +#endif // CUTE_PNG_ATLAS_EMPTY_COLOR + +#include +#include + +typedef struct cp_pixel_t cp_pixel_t; +typedef struct cp_image_t cp_image_t; +typedef struct cp_indexed_image_t cp_indexed_image_t; +typedef struct cp_atlas_image_t cp_atlas_image_t; + +// Read this in the event of errors from any function +extern const char* cp_error_reason; + +// return 1 for success, 0 for failures +int cp_inflate(void* in, int in_bytes, void* out, int out_bytes); +int cp_save_png(const char* file_name, const cp_image_t* img); + +typedef struct cp_saved_png_t +{ + int size; // Size of the `data` buffer. + void* data; // Pointer to the saved png in memory. + // NULL if something went wrong. + // Call CUTE_PNG_FREE on `data` when done. +} cp_saved_png_t; + +// Saves a png file to memory. +// Call CUTE_PNG_FREE on .data when done. +cp_saved_png_t cp_save_png_to_memory(const cp_image_t* img); + +// Constructs an atlas image in-memory. The atlas pixels are stored in the returned image. free the pixels +// when done with them. The user must provide an array of cp_atlas_image_t for the `imgs` param. `imgs` holds +// information about uv coordinates for an associated image in the `pngs` array. Output image has NULL +// pixels buffer in the event of errors. +cp_image_t cp_make_atlas(int atlasWidth, int atlasHeight, const cp_image_t* pngs, int png_count, cp_atlas_image_t* imgs_out); + +// A decent "default" function, ready to use out-of-the-box. Saves out an easy to parse text formatted info file +// along with an atlas image. `names` param can be optionally NULL. +int cp_default_save_atlas(const char* out_path_image, const char* out_path_atlas_txt, const cp_image_t* atlas, const cp_atlas_image_t* imgs, int img_count, const char** names); + +// these two functions return cp_image_t::pix as 0 in event of errors +// call free on cp_image_t::pix when done, or call cp_free_png +cp_image_t cp_load_png(const char *file_name); +cp_image_t cp_load_png_mem(const void *png_data, int png_length); +cp_image_t cp_load_blank(int w, int h); // Alloc's pixels, but `pix` memory is uninitialized. +void cp_free_png(cp_image_t* img); +void cp_flip_image_horizontal(cp_image_t* img); + +// Reads the w/h of the png without doing any other decompression or parsing. +void cp_load_png_wh(const void* png_data, int png_length, int* w, int* h); + +// loads indexed (paletted) pngs, but does not depalette the image into RGBA pixels +// these two functions return cp_indexed_image_t::pix as 0 in event of errors +// call free on cp_indexed_image_t::pix when done, or call cp_free_indexed_png +cp_indexed_image_t cp_load_indexed_png(const char* file_name); +cp_indexed_image_t cp_load_indexed_png_mem(const void *png_data, int png_length); +void cp_free_indexed_png(cp_indexed_image_t* img); + +// converts paletted image into a standard RGBA image +// call free on cp_image_t::pix when done +cp_image_t cp_depallete_indexed_image(cp_indexed_image_t* img); + +// Pre-process the pixels to transform the image data to a premultiplied alpha format. +// Resource: http://www.essentialmath.com/GDC2015/VanVerth_Jim_DoingMathwRGB.pdf +void cp_premultiply(cp_image_t* img); + +struct cp_pixel_t +{ + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t a; +}; + +struct cp_image_t +{ + int w; + int h; + cp_pixel_t* pix; +}; + +struct cp_indexed_image_t +{ + int w; + int h; + uint8_t* pix; + uint8_t palette_len; + cp_pixel_t palette[256]; +}; + +struct cp_atlas_image_t +{ + int img_index; // index into the `imgs` array + int w, h; // pixel w/h of original image + float minx, miny; // u coordinate + float maxx, maxy; // v coordinate + int fit; // non-zero if image fit and was placed into the atlas +}; + +#define CUTE_PNG_H +#endif + +#ifdef CUTE_PNG_IMPLEMENTATION +#ifndef CUTE_PNG_IMPLEMENTATION_ONCE +#define CUTE_PNG_IMPLEMENTATION_ONCE + +#if !defined(CUTE_PNG_ALLOCA) + #define CUTE_PNG_ALLOCA alloca + + #ifdef _WIN32 + #include + #elif defined(__linux__) + #include + #endif +#endif + +#if !defined(CUTE_PNG_ALLOC) + #include + #define CUTE_PNG_ALLOC malloc +#endif + +#if !defined(CUTE_PNG_FREE) + #include + #define CUTE_PNG_FREE free +#endif + +#if !defined(CUTE_PNG_CALLOC) + #include + #define CUTE_PNG_CALLOC calloc +#endif + +#if !defined(CUTE_PNG_REALLOC) + #include + #define CUTE_PNG_REALLOC realloc +#endif + +#if !defined(CUTE_PNG_MEMCPY) + #include + #define CUTE_PNG_MEMCPY memcpy +#endif + +#if !defined(CUTE_PNG_MEMCMP) + #include + #define CUTE_PNG_MEMCMP memcmp +#endif + +#if !defined(CUTE_PNG_MEMSET) + #include + #define CUTE_PNG_MEMSET memset +#endif + +#if !defined(CUTE_PNG_ASSERT) + #include + #define CUTE_PNG_ASSERT assert +#endif + +#if !defined(CUTE_PNG_FPRINTF) + #include + #define CUTE_PNG_FPRINTF fprintf +#endif + +#if !defined(CUTE_PNG_SEEK_SET) + #include + #define CUTE_PNG_SEEK_SET SEEK_SET +#endif + +#if !defined(CUTE_PNG_SEEK_END) + #include + #define CUTE_PNG_SEEK_END SEEK_END +#endif + +#if !defined(CUTE_PNG_FILE) + #include + #define CUTE_PNG_FILE FILE +#endif + +#if !defined(CUTE_PNG_FOPEN) + #include + #define CUTE_PNG_FOPEN fopen +#endif + +#if !defined(CUTE_PNG_FSEEK) + #include + #define CUTE_PNG_FSEEK fseek +#endif + +#if !defined(CUTE_PNG_FREAD) + #include + #define CUTE_PNG_FREAD fread +#endif + +#if !defined(CUTE_PNG_FTELL) + #include + #define CUTE_PNG_FTELL ftell +#endif + +#if !defined(CUTE_PNG_FWRITE) + #include + #define CUTE_PNG_FWRITE fwrite +#endif + +#if !defined(CUTE_PNG_FCLOSE) + #include + #define CUTE_PNG_FCLOSE fclose +#endif + +#if !defined(CUTE_PNG_FERROR) + #include + #define CUTE_PNG_FERROR ferror +#endif + +static cp_pixel_t cp_make_pixel_a(uint8_t r, uint8_t g, uint8_t b, uint8_t a) +{ + cp_pixel_t p; + p.r = r; p.g = g; p.b = b; p.a = a; + return p; +} + +static cp_pixel_t cp_make_pixel(uint8_t r, uint8_t g, uint8_t b) +{ + cp_pixel_t p; + p.r = r; p.g = g; p.b = b; p.a = 0xFF; + return p; +} + +const char* cp_error_reason; +#define CUTE_PNG_FAIL() do { goto cp_err; } while (0) +#define CUTE_PNG_CHECK(X, Y) do { if (!(X)) { cp_error_reason = Y; CUTE_PNG_FAIL(); } } while (0) +#define CUTE_PNG_CALL(X) do { if (!(X)) goto cp_err; } while (0) +#define CUTE_PNG_LOOKUP_BITS 9 +#define CUTE_PNG_LOOKUP_COUNT (1 << CUTE_PNG_LOOKUP_BITS) +#define CUTE_PNG_LOOKUP_MASK (CUTE_PNG_LOOKUP_COUNT - 1) +#define CUTE_PNG_DEFLATE_MAX_BITLEN 15 + +// DEFLATE tables from RFC 1951 +uint8_t cp_fixed_table[288 + 32] = { + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, +}; // 3.2.6 +uint8_t cp_permutation_order[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; // 3.2.7 +uint8_t cp_len_extra_bits[29 + 2] = { 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0, 0,0 }; // 3.2.5 +uint32_t cp_len_base[29 + 2] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 0,0 }; // 3.2.5 +uint8_t cp_dist_extra_bits[30 + 2] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13, 0,0 }; // 3.2.5 +uint32_t cp_dist_base[30 + 2] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 0,0 }; // 3.2.5 + +typedef struct cp_state_t +{ + uint64_t bits; + int count; + uint32_t* words; + int word_count; + int word_index; + int bits_left; + + int final_word_available; + uint32_t final_word; + + char* out; + char* out_end; + char* begin; + + uint16_t lookup[CUTE_PNG_LOOKUP_COUNT]; + uint32_t lit[288]; + uint32_t dst[32]; + uint32_t len[19]; + uint32_t nlit; + uint32_t ndst; + uint32_t nlen; +} cp_state_t; + +static int cp_would_overflow(cp_state_t* s, int num_bits) +{ + return (s->bits_left + s->count) - num_bits < 0; +} + +static char* cp_ptr(cp_state_t* s) +{ + CUTE_PNG_ASSERT(!(s->bits_left & 7)); + return (char*)(s->words + s->word_index) - (s->count / 8); +} + +static uint64_t cp_peak_bits(cp_state_t* s, int num_bits_to_read) +{ + if (s->count < num_bits_to_read) + { + if (s->word_index < s->word_count) + { + uint32_t word = s->words[s->word_index++]; + s->bits |= (uint64_t)word << s->count; + s->count += 32; + CUTE_PNG_ASSERT(s->word_index <= s->word_count); + } + + else if (s->final_word_available) + { + uint32_t word = s->final_word; + s->bits |= (uint64_t)word << s->count; + s->count += s->bits_left; + s->final_word_available = 0; + } + } + + return s->bits; +} + +static uint32_t cp_consume_bits(cp_state_t* s, int num_bits_to_read) +{ + CUTE_PNG_ASSERT(s->count >= num_bits_to_read); + uint32_t bits = s->bits & (((uint64_t)1 << num_bits_to_read) - 1); + s->bits >>= num_bits_to_read; + s->count -= num_bits_to_read; + s->bits_left -= num_bits_to_read; + return bits; +} + +static uint32_t cp_read_bits(cp_state_t* s, int num_bits_to_read) +{ + CUTE_PNG_ASSERT(num_bits_to_read <= 32); + CUTE_PNG_ASSERT(num_bits_to_read >= 0); + CUTE_PNG_ASSERT(s->bits_left > 0); + CUTE_PNG_ASSERT(s->count <= 64); + CUTE_PNG_ASSERT(!cp_would_overflow(s, num_bits_to_read)); + cp_peak_bits(s, num_bits_to_read); + uint32_t bits = cp_consume_bits(s, num_bits_to_read); + return bits; +} + +static char* cp_read_file_to_memory(const char* path, int* size) +{ + char* data = 0; + CUTE_PNG_FILE* fp = CUTE_PNG_FOPEN(path, "rb"); + int sizeNum = 0; + + if (fp) + { + CUTE_PNG_FSEEK(fp, 0, CUTE_PNG_SEEK_END); + sizeNum = CUTE_PNG_FTELL(fp); + CUTE_PNG_FSEEK(fp, 0, CUTE_PNG_SEEK_SET); + data = (char*)CUTE_PNG_ALLOC(sizeNum + 1); + CUTE_PNG_FREAD(data, sizeNum, 1, fp); + data[sizeNum] = 0; + CUTE_PNG_FCLOSE(fp); + } + + if (size) *size = sizeNum; + return data; +} + +static uint32_t cp_rev16(uint32_t a) +{ + a = ((a & 0xAAAA) >> 1) | ((a & 0x5555) << 1); + a = ((a & 0xCCCC) >> 2) | ((a & 0x3333) << 2); + a = ((a & 0xF0F0) >> 4) | ((a & 0x0F0F) << 4); + a = ((a & 0xFF00) >> 8) | ((a & 0x00FF) << 8); + return a; +} + +// RFC 1951 section 3.2.2 +static int cp_build(cp_state_t* s, uint32_t* tree, uint8_t* lens, int sym_count) +{ + int n, codes[16], first[16], counts[16] = { 0 }; + + // Frequency count + for (n = 0; n < sym_count; n++) counts[lens[n]]++; + + // Distribute codes + counts[0] = codes[0] = first[0] = 0; + for (n = 1; n <= 15; ++n) + { + codes[n] = (codes[n - 1] + counts[n - 1]) << 1; + first[n] = first[n - 1] + counts[n - 1]; + } + + if (s) CUTE_PNG_MEMSET(s->lookup, 0, sizeof(s->lookup)); + for (int i = 0; i < sym_count; ++i) + { + int len = lens[i]; + + if (len != 0) + { + CUTE_PNG_ASSERT(len < 16); + uint32_t code = codes[len]++; + uint32_t slot = first[len]++; + tree[slot] = (code << (32 - len)) | (i << 4) | len; + + if (s && len <= CUTE_PNG_LOOKUP_BITS) + { + int j = cp_rev16(code) >> (16 - len); + while (j < (1 << CUTE_PNG_LOOKUP_BITS)) + { + s->lookup[j] = (uint16_t)((len << CUTE_PNG_LOOKUP_BITS) | i); + j += (1 << len); + } + } + } + } + + int max_index = first[15]; + return max_index; +} + +static int cp_stored(cp_state_t* s) +{ + char* p; + + // 3.2.3 + // skip any remaining bits in current partially processed byte + cp_read_bits(s, s->count & 7); + + // 3.2.4 + // read LEN and NLEN, should complement each other + uint16_t LEN = (uint16_t)cp_read_bits(s, 16); + uint16_t NLEN = (uint16_t)cp_read_bits(s, 16); + CUTE_PNG_CHECK(LEN == (uint16_t)(~NLEN), "Failed to find LEN and NLEN as complements within stored (uncompressed) stream."); + CUTE_PNG_CHECK(s->bits_left / 8 <= (int)LEN, "Stored block extends beyond end of input stream."); + p = cp_ptr(s); + CUTE_PNG_MEMCPY(s->out, p, LEN); + s->out += LEN; + return 1; + +cp_err: + return 0; +} + +// 3.2.6 +static int cp_fixed(cp_state_t* s) +{ + s->nlit = cp_build(s, s->lit, cp_fixed_table, 288); + s->ndst = cp_build(0, s->dst, cp_fixed_table + 288, 32); + return 1; +} + +static int cp_decode(cp_state_t* s, uint32_t* tree, int hi) +{ + uint64_t bits = cp_peak_bits(s, 16); + uint32_t search = (cp_rev16((uint32_t)bits) << 16) | 0xFFFF; + int lo = 0; + while (lo < hi) + { + int guess = (lo + hi) >> 1; + if (search < tree[guess]) hi = guess; + else lo = guess + 1; + } + + uint32_t key = tree[lo - 1]; + uint32_t len = (32 - (key & 0xF)); + CUTE_PNG_ASSERT((search >> len) == (key >> len)); + + int code = cp_consume_bits(s, key & 0xF); + (void)code; + return (key >> 4) & 0xFFF; +} + +// 3.2.7 +static int cp_dynamic(cp_state_t* s) +{ + uint8_t lenlens[19] = { 0 }; + + int nlit = 257 + cp_read_bits(s, 5); + int ndst = 1 + cp_read_bits(s, 5); + int nlen = 4 + cp_read_bits(s, 4); + + for (int i = 0 ; i < nlen; ++i) + lenlens[cp_permutation_order[i]] = (uint8_t)cp_read_bits(s, 3); + + // Build the tree for decoding code lengths + s->nlen = cp_build(0, s->len, lenlens, 19); + uint8_t lens[288 + 32]; + + for (int n = 0; n < nlit + ndst;) + { + int sym = cp_decode(s, s->len, s->nlen); + switch (sym) + { + case 16: for (int i = 3 + cp_read_bits(s, 2); i; --i, ++n) lens[n] = lens[n - 1]; break; + case 17: for (int i = 3 + cp_read_bits(s, 3); i; --i, ++n) lens[n] = 0; break; + case 18: for (int i = 11 + cp_read_bits(s, 7); i; --i, ++n) lens[n] = 0; break; + default: lens[n++] = (uint8_t)sym; break; + } + } + + s->nlit = cp_build(s, s->lit, lens, nlit); + s->ndst = cp_build(0, s->dst, lens + nlit, ndst); + return 1; +} + +// 3.2.3 +static int cp_block(cp_state_t* s) +{ + while (1) + { + int symbol = cp_decode(s, s->lit, s->nlit); + + if (symbol < 256) + { + CUTE_PNG_CHECK(s->out + 1 <= s->out_end, "Attempted to overwrite out buffer while outputting a symbol."); + *s->out = (char)symbol; + s->out += 1; + } + + else if (symbol > 256) + { + symbol -= 257; + int length = cp_read_bits(s, cp_len_extra_bits[symbol]) + cp_len_base[symbol]; + int distance_symbol = cp_decode(s, s->dst, s->ndst); + int backwards_distance = cp_read_bits(s, cp_dist_extra_bits[distance_symbol]) + cp_dist_base[distance_symbol]; + CUTE_PNG_CHECK(s->out - backwards_distance >= s->begin, "Attempted to write before out buffer (invalid backwards distance)."); + CUTE_PNG_CHECK(s->out + length <= s->out_end, "Attempted to overwrite out buffer while outputting a string."); + char* src = s->out - backwards_distance; + char* dst = s->out; + s->out += length; + + switch (backwards_distance) + { + case 1: // very common in images + CUTE_PNG_MEMSET(dst, *src, length); + break; + default: while (length--) *dst++ = *src++; + } + } + + else break; + } + + return 1; + +cp_err: + return 0; +} + +// 3.2.3 +int cp_inflate(void* in, int in_bytes, void* out, int out_bytes) +{ + cp_state_t* s = (cp_state_t*)CUTE_PNG_CALLOC(1, sizeof(cp_state_t)); + s->bits = 0; + s->count = 0; + s->word_index = 0; + s->bits_left = in_bytes * 8; + + // s->words is the in-pointer rounded up to a multiple of 4 + int first_bytes = (int) ((( (size_t) in + 3) & ~3) - (size_t) in); + s->words = (uint32_t*)((char*)in + first_bytes); + s->word_count = (in_bytes - first_bytes) / 4; + int last_bytes = ((in_bytes - first_bytes) & 3); + + for (int i = 0; i < first_bytes; ++i) + s->bits |= (uint64_t)(((uint8_t*)in)[i]) << (i * 8); + + s->final_word_available = last_bytes ? 1 : 0; + s->final_word = 0; + for(int i = 0; i < last_bytes; i++) + s->final_word |= ((uint8_t*)in)[in_bytes - last_bytes+i] << (i * 8); + + s->count = first_bytes * 8; + + s->out = (char*)out; + s->out_end = s->out + out_bytes; + s->begin = (char*)out; + + int count = 0; + int bfinal; + do + { + bfinal = cp_read_bits(s, 1); + int btype = cp_read_bits(s, 2); + + switch (btype) + { + case 0: CUTE_PNG_CALL(cp_stored(s)); break; + case 1: cp_fixed(s); CUTE_PNG_CALL(cp_block(s)); break; + case 2: cp_dynamic(s); CUTE_PNG_CALL(cp_block(s)); break; + case 3: CUTE_PNG_CHECK(0, "Detected unknown block type within input stream."); + } + + ++count; + } + while (!bfinal); + + CUTE_PNG_FREE(s); + return 1; + +cp_err: + CUTE_PNG_FREE(s); + return 0; +} + +static uint8_t cp_paeth(uint8_t a, uint8_t b, uint8_t c) +{ + int p = a + b - c; + int pa = abs(p - a); + int pb = abs(p - b); + int pc = abs(p - c); + return (pa <= pb && pa <= pc) ? a : (pb <= pc) ? b : c; +} + +typedef struct cp_save_png_data_t +{ + uint32_t crc; + uint32_t adler; + uint32_t bits; + uint32_t prev; + uint32_t runlen; + int buflen; + int bufcap; + char* buffer; +} cp_save_png_data_t; + +uint32_t CP_CRC_TABLE[] = { + 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, + 0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c +}; + +static void cp_put8(cp_save_png_data_t* s, uint32_t a) +{ + if (s->buflen >= s->bufcap) + { + s->bufcap *= 2; + s->buffer = (char*)CUTE_PNG_REALLOC(s->buffer, s->bufcap); + } + s->buffer[s->buflen++] = a; + s->crc = (s->crc >> 4) ^ CP_CRC_TABLE[(s->crc & 15) ^ (a & 15)]; + s->crc = (s->crc >> 4) ^ CP_CRC_TABLE[(s->crc & 15) ^ (a >> 4)]; +} + +static void cp_update_adler(cp_save_png_data_t* s, uint32_t v) +{ + uint32_t s1 = s->adler & 0xFFFF; + uint32_t s2 = (s->adler >> 16) & 0xFFFF; + s1 = (s1 + v) % 65521; + s2 = (s2 + s1) % 65521; + s->adler = (s2 << 16) + s1; +} + +static void cp_put32(cp_save_png_data_t* s, uint32_t v) +{ + cp_put8(s, (v >> 24) & 0xFF); + cp_put8(s, (v >> 16) & 0xFF); + cp_put8(s, (v >> 8) & 0xFF); + cp_put8(s, v & 0xFF); +} + +static void cp_put_bits(cp_save_png_data_t* s, uint32_t data, uint32_t bitcount) +{ + while (bitcount--) + { + uint32_t prev = s->bits; + s->bits = (s->bits >> 1) | ((data & 1) << 7); + data >>= 1; + + if (prev & 1) + { + cp_put8(s, s->bits); + s->bits = 0x80; + } + } +} + +static void cp_put_bitsr(cp_save_png_data_t* s, uint32_t data, uint32_t bitcount) +{ + while (bitcount--) + cp_put_bits(s, data >> bitcount, 1); +} + +static void cp_begin_chunk(cp_save_png_data_t* s, const char* id, uint32_t len) +{ + cp_put32(s, len); + s->crc = 0xFFFFFFFF; + cp_put8(s, (unsigned char)id[0]); + cp_put8(s, (unsigned char)id[1]); + cp_put8(s, (unsigned char)id[2]); + cp_put8(s, (unsigned char)id[3]); +} + +static void cp_encode_literal(cp_save_png_data_t* s, uint32_t v) +{ + // Encode a literal/length using the built-in tables. + // Could do better with a custom table but whatever. + if (v < 144) cp_put_bitsr(s, 0x030 + v - 0, 8); + else if (v < 256) cp_put_bitsr(s, 0x190 + v - 144, 9); + else if (v < 280) cp_put_bitsr(s, 0x000 + v - 256, 7); + else cp_put_bitsr(s, 0x0c0 + v - 280, 8); +} + +static void cp_encode_len(cp_save_png_data_t* s, uint32_t code, uint32_t bits, uint32_t len) +{ + cp_encode_literal(s, code + (len >> bits)); + cp_put_bits(s, len, bits); + cp_put_bits(s, 0, 5); +} + +static void cp_end_run(cp_save_png_data_t* s) +{ + s->runlen--; + cp_encode_literal(s, s->prev); + + if (s->runlen >= 67) cp_encode_len(s, 277, 4, s->runlen - 67); + else if (s->runlen >= 35) cp_encode_len(s, 273, 3, s->runlen - 35); + else if (s->runlen >= 19) cp_encode_len(s, 269, 2, s->runlen - 19); + else if (s->runlen >= 11) cp_encode_len(s, 265, 1, s->runlen - 11); + else if (s->runlen >= 3) cp_encode_len(s, 257, 0, s->runlen - 3); + else while (s->runlen--) cp_encode_literal(s, s->prev); +} + +static void cp_encode_byte(cp_save_png_data_t *s, uint8_t v) +{ + cp_update_adler(s, v); + + // Simple RLE compression. We could do better by doing a search + // to find matches, but this works pretty well TBH. + if (s->prev == v && s->runlen < 115) s->runlen++; + + else + { + if (s->runlen) cp_end_run(s); + + s->prev = v; + s->runlen = 1; + } +} + +static void cp_save_header(cp_save_png_data_t* s, cp_image_t* img) +{ + const unsigned char* hdr = (const unsigned char*)"\211PNG\r\n\032\n"; + for (int i = 0; i < 8; ++i) { + cp_put8(s, *hdr++); + } + cp_begin_chunk(s, "IHDR", 13); + cp_put32(s, img->w); + cp_put32(s, img->h); + cp_put8(s, 8); // bit depth + cp_put8(s, 6); // RGBA + cp_put8(s, 0); // compression (deflate) + cp_put8(s, 0); // filter (standard) + cp_put8(s, 0); // interlace off + cp_put32(s, ~s->crc); +} + +static void cp_save_data(cp_save_png_data_t* s, cp_image_t* img, long dataPos, long* dataSize) +{ + cp_begin_chunk(s, "IDAT", 0); + cp_put8(s, 0x08); // zlib compression method + cp_put8(s, 0x1D); // zlib compression flags + cp_put_bits(s, 3, 3); // zlib last block + fixed dictionary + + for (int y = 0; y < img->h; ++y) + { + cp_pixel_t *row = &img->pix[y * img->w]; + cp_pixel_t prev = cp_make_pixel_a(0, 0, 0, 0); + + cp_encode_byte(s, 1); // sub filter + for (int x = 0; x < img->w; ++x) + { + cp_encode_byte(s, row[x].r - prev.r); + cp_encode_byte(s, row[x].g - prev.g); + cp_encode_byte(s, row[x].b - prev.b); + cp_encode_byte(s, row[x].a - prev.a); + prev = row[x]; + } + } + + cp_end_run(s); + cp_encode_literal(s, 256); // terminator + while (s->bits != 0x80) cp_put_bits(s, 0, 1); + cp_put32(s, s->adler); + *dataSize = (s->buflen - dataPos) - 8; + cp_put32(s, ~s->crc); +} + +cp_saved_png_t cp_save_png_to_memory(const cp_image_t* img) +{ + cp_saved_png_t result = { 0 }; + cp_save_png_data_t s = { 0 }; + long dataPos, dataSize, fileSize; + if (!img) return result; + + s.adler = 1; + s.bits = 0x80; + s.prev = 0xFFFF; + s.bufcap = 1024; + s.buffer = (char*)CUTE_PNG_ALLOC(1024); + + cp_save_header(&s, (cp_image_t*)img); + dataPos = s.buflen; + cp_save_data(&s, (cp_image_t*)img, dataPos, &dataSize); + + // End chunk. + cp_begin_chunk(&s, "IEND", 0); + cp_put32(&s, ~s.crc); + + // Write back payload size. + fileSize = s.buflen; + s.buflen = dataPos; + cp_put32(&s, dataSize); + + result.size = fileSize; + result.data = s.buffer; + return result; +} + +int cp_save_png(const char* file_name, const cp_image_t* img) +{ + cp_saved_png_t s; + long err; + CUTE_PNG_FILE* fp = CUTE_PNG_FOPEN(file_name, "wb"); + if (!fp) return 1; + s = cp_save_png_to_memory(img); + CUTE_PNG_FWRITE(s.data, s.size, 1, fp); + err = CUTE_PNG_FERROR(fp); + CUTE_PNG_FCLOSE(fp); + CUTE_PNG_FREE(s.data); + return !err; +} + +typedef struct cp_raw_png_t +{ + const uint8_t* p; + const uint8_t* end; +} cp_raw_png_t; + +static uint32_t cp_make32(const uint8_t* s) +{ + return (s[0] << 24) | (s[1] << 16) | (s[2] << 8) | s[3]; +} + +static const uint8_t* cp_chunk(cp_raw_png_t* png, const char* chunk, uint32_t minlen) +{ + uint32_t len = cp_make32(png->p); + const uint8_t* start = png->p; + + if (!CUTE_PNG_MEMCMP(start + 4, chunk, 4) && len >= minlen) + { + int offset = len + 12; + + if (png->p + offset <= png->end) + { + png->p += offset; + return start + 8; + } + } + + return 0; +} + +static const uint8_t* cp_find(cp_raw_png_t* png, const char* chunk, uint32_t minlen) +{ + const uint8_t *start; + while (png->p < png->end) + { + uint32_t len = cp_make32(png->p); + start = png->p; + png->p += len + 12; + + if (!CUTE_PNG_MEMCMP(start+4, chunk, 4) && len >= minlen && png->p <= png->end) + return start + 8; + } + + return 0; +} + +static int cp_unfilter(int w, int h, int bpp, uint8_t* raw) +{ + int len = w * bpp; + uint8_t *prev; + int x; + + if (h > 0) + { +#define FILTER_LOOP_FIRST(A) for (x = bpp; x < len; x++) raw[x] += A; break + switch (*raw++) + { + case 0: break; + case 1: FILTER_LOOP_FIRST(raw[x - bpp]); + case 2: break; + case 3: FILTER_LOOP_FIRST(raw[x - bpp] / 2); + case 4: FILTER_LOOP_FIRST(cp_paeth(raw[x - bpp], 0, 0)); + default: return 0; + } +#undef FILTER_LOOP_FIRST + } + + prev = raw; + raw += len; + + for (int y = 1; y < h; y++, prev = raw, raw += len) + { +#define FILTER_LOOP(A, B) for (x = 0 ; x < bpp; x++) raw[x] += A; for (; x < len; x++) raw[x] += B; break + switch (*raw++) + { + case 0: break; + case 1: FILTER_LOOP(0 , raw[x - bpp] ); + case 2: FILTER_LOOP(prev[x] , prev[x]); + case 3: FILTER_LOOP(prev[x] / 2, (raw[x - bpp] + prev[x]) / 2); + case 4: FILTER_LOOP(prev[x] , cp_paeth(raw[x - bpp], prev[x], prev[x -bpp])); + default: return 0; + } +#undef FILTER_LOOP + } + + return 1; +} + +static void cp_convert(int bpp, int w, int h, uint8_t* src, cp_pixel_t* dst) +{ + for (int y = 0; y < h; y++) + { + // skip filter byte + src++; + + for (int x = 0; x < w; x++, src += bpp) + { + switch (bpp) + { + case 1: *dst++ = cp_make_pixel(src[0], src[0], src[0]); break; + case 2: *dst++ = cp_make_pixel_a(src[0], src[0], src[0], src[1]); break; + case 3: *dst++ = cp_make_pixel(src[0], src[1], src[2]); break; + case 4: *dst++ = cp_make_pixel_a(src[0], src[1], src[2], src[3]); break; + } + } + } +} + +// http://www.libpng.org/pub/png/spec/1.2/PNG-Chunks.html#C.tRNS +static uint8_t cp_get_alpha_for_indexed_image(int index, const uint8_t* trns, uint32_t trns_len) +{ + if (!trns) return 255; + else if ((uint32_t)index >= trns_len) return 255; + else return trns[index]; +} + +static void cp_depalette(int w, int h, uint8_t* src, cp_pixel_t* dst, const uint8_t* plte, const uint8_t* trns, uint32_t trns_len) +{ + for (int y = 0; y < h; ++y) + { + // skip filter byte + ++src; + + for (int x = 0; x < w; ++x, ++src) + { + int c = *src; + uint8_t r = plte[c * 3]; + uint8_t g = plte[c * 3 + 1]; + uint8_t b = plte[c * 3 + 2]; + uint8_t a = cp_get_alpha_for_indexed_image(c, trns, trns_len); + *dst++ = cp_make_pixel_a(r, g, b, a); + } + } +} + +static uint32_t cp_get_chunk_byte_length(const uint8_t* chunk) +{ + return cp_make32(chunk - 8); +} + +static int cp_out_size(cp_image_t* img, int bpp) +{ + return (img->w + 1) * img->h * bpp; +} + +cp_image_t cp_load_png_mem(const void* png_data, int png_length) +{ + const char* sig = "\211PNG\r\n\032\n"; + const uint8_t* ihdr, *first, *plte, *trns; + int bit_depth, color_type, bpp, w, h, pix_bytes; + int compression, filter, interlace; + int datalen, offset; + uint8_t* out; + cp_image_t img = { 0 }; + uint8_t* data = 0; + cp_raw_png_t png; + png.p = (uint8_t*)png_data; + png.end = (uint8_t*)png_data + png_length; + + CUTE_PNG_CHECK(!CUTE_PNG_MEMCMP(png.p, sig, 8), "incorrect file signature (is this a png file?)"); + png.p += 8; + + ihdr = cp_chunk(&png, "IHDR", 13); + CUTE_PNG_CHECK(ihdr, "unable to find IHDR chunk"); + bit_depth = ihdr[8]; + color_type = ihdr[9]; + CUTE_PNG_CHECK(bit_depth == 8, "only bit-depth of 8 is supported"); + + switch (color_type) + { + case 0: bpp = 1; break; // greyscale + case 2: bpp = 3; break; // RGB + case 3: bpp = 1; break; // paletted + case 4: bpp = 2; break; // grey+alpha + case 6: bpp = 4; break; // RGBA + default: CUTE_PNG_CHECK(0, "unknown color type"); + } + + // +1 for filter byte (which is dumb! just stick this at file header...) + w = cp_make32(ihdr) + 1; + h = cp_make32(ihdr + 4); + CUTE_PNG_CHECK(w >= 1, "invalid IHDR chunk found, image width was less than 1"); + CUTE_PNG_CHECK(h >= 1, "invalid IHDR chunk found, image height was less than 1"); + CUTE_PNG_CHECK((int64_t) w * h * sizeof(cp_pixel_t) < INT_MAX, "image too large"); + pix_bytes = w * h * sizeof(cp_pixel_t); + img.w = w - 1; + img.h = h; + img.pix = (cp_pixel_t*)CUTE_PNG_ALLOC(pix_bytes); + CUTE_PNG_CHECK(img.pix, "unable to allocate raw image space"); + + compression = ihdr[10]; + filter = ihdr[11]; + interlace = ihdr[12]; + CUTE_PNG_CHECK(!compression, "only standard compression DEFLATE is supported"); + CUTE_PNG_CHECK(!filter, "only standard adaptive filtering is supported"); + CUTE_PNG_CHECK(!interlace, "interlacing is not supported"); + + // PLTE must come before any IDAT chunk + first = png.p; + plte = cp_find(&png, "PLTE", 0); + if (!plte) png.p = first; + else first = png.p; + + // tRNS can come after PLTE + trns = cp_find(&png, "tRNS", 0); + if (!trns) png.p = first; + else first = png.p; + + // Compute length of the DEFLATE stream through IDAT chunk data sizes + datalen = 0; + for (const uint8_t* idat = cp_find(&png, "IDAT", 0); idat; idat = cp_chunk(&png, "IDAT", 0)) + { + uint32_t len = cp_get_chunk_byte_length(idat); + datalen += len; + } + + // Copy in IDAT chunk data sections to form the compressed DEFLATE stream + png.p = first; + data = (uint8_t*)CUTE_PNG_ALLOC(datalen); + offset = 0; + for (const uint8_t* idat = cp_find(&png, "IDAT", 0); idat; idat = cp_chunk(&png, "IDAT", 0)) + { + uint32_t len = cp_get_chunk_byte_length(idat); + CUTE_PNG_MEMCPY(data + offset, idat, len); + offset += len; + } + + // check for proper zlib structure in DEFLATE stream + CUTE_PNG_CHECK(data && datalen >= 6, "corrupt zlib structure in DEFLATE stream"); + CUTE_PNG_CHECK((data[0] & 0x0f) == 0x08, "only zlib compression method (RFC 1950) is supported"); + CUTE_PNG_CHECK((data[0] & 0xf0) <= 0x70, "innapropriate window size detected"); + CUTE_PNG_CHECK(!(data[1] & 0x20), "preset dictionary is present and not supported"); + + // check for integer overflow + CUTE_PNG_CHECK(cp_out_size(&img, 4) >= 1, "invalid image size found"); + CUTE_PNG_CHECK(cp_out_size(&img, bpp) >= 1, "invalid image size found"); + + out = (uint8_t*)img.pix + cp_out_size(&img, 4) - cp_out_size(&img, bpp); + CUTE_PNG_CHECK(cp_inflate(data + 2, datalen - 6, out, pix_bytes), "DEFLATE algorithm failed"); + CUTE_PNG_CHECK(cp_unfilter(img.w, img.h, bpp, out), "invalid filter byte found"); + + if (color_type == 3) + { + CUTE_PNG_CHECK(plte, "color type of indexed requires a PLTE chunk"); + uint32_t trns_len = trns ? cp_get_chunk_byte_length(trns) : 0; + cp_depalette(img.w, img.h, out, img.pix, plte, trns, trns_len); + } + else cp_convert(bpp, img.w, img.h, out, img.pix); + + CUTE_PNG_FREE(data); + return img; + +cp_err: + CUTE_PNG_FREE(data); + CUTE_PNG_FREE(img.pix); + img.pix = 0; + + return img; +} + +cp_image_t cp_load_blank(int w, int h) +{ + cp_image_t img; + img.w = w; + img.h = h; + img.pix = (cp_pixel_t*)CUTE_PNG_ALLOC(w * h * sizeof(cp_pixel_t)); + return img; +} + +cp_image_t cp_load_png(const char *file_name) +{ + cp_image_t img = { 0 }; + int len; + void* data = cp_read_file_to_memory(file_name, &len); + if (!data) return img; + img = cp_load_png_mem(data, len); + CUTE_PNG_FREE(data); + return img; +} + +void cp_free_png(cp_image_t* img) +{ + CUTE_PNG_FREE(img->pix); + img->pix = 0; + img->w = img->h = 0; +} + +void cp_flip_image_horizontal(cp_image_t* img) +{ + cp_pixel_t* pix = img->pix; + int w = img->w; + int h = img->h; + int flips = h / 2; + for (int i = 0; i < flips; ++i) + { + cp_pixel_t* a = pix + w * i; + cp_pixel_t* b = pix + w * (h - i - 1); + for (int j = 0; j < w; ++j) + { + cp_pixel_t t = *a; + *a = *b; + *b = t; + ++a; + ++b; + } + } +} + +void cp_load_png_wh(const void* png_data, int png_length, int* w_out, int* h_out) +{ + const char* sig = "\211PNG\r\n\032\n"; + const uint8_t* ihdr; + cp_raw_png_t png; + int w, h; + png.p = (uint8_t*)png_data; + png.end = (uint8_t*)png_data + png_length; + + if (w_out) *w_out = 0; + if (h_out) *h_out = 0; + + CUTE_PNG_CHECK(!CUTE_PNG_MEMCMP(png.p, sig, 8), "incorrect file signature (is this a png file?)"); + png.p += 8; + + ihdr = cp_chunk(&png, "IHDR", 13); + CUTE_PNG_CHECK(ihdr, "unable to find IHDR chunk"); + + // +1 for filter byte (which is dumb! just stick this at file header...) + w = cp_make32(ihdr) + 1; + h = cp_make32(ihdr + 4); + if (w_out) *w_out = w - 1; + if (h_out) *h_out = h; + + cp_err:; +} + +cp_indexed_image_t cp_load_indexed_png(const char* file_name) +{ + cp_indexed_image_t img = { 0 }; + int len; + void* data = cp_read_file_to_memory(file_name, &len); + if (!data) return img; + img = cp_load_indexed_png_mem(data, len); + CUTE_PNG_FREE(data); + return img; +} + +static void cp_unpack_indexed_rows(int w, int h, uint8_t* src, uint8_t* dst) +{ + for (int y = 0; y < h; ++y) + { + // skip filter byte + ++src; + + for (int x = 0; x < w; ++x, ++src) + { + *dst++ = *src; + } + } +} + +void cp_unpack_palette(cp_pixel_t* dst, const uint8_t* plte, int plte_len, const uint8_t* trns, int trns_len) +{ + for (int i = 0; i < plte_len * 3; i += 3) + { + unsigned char r = plte[i]; + unsigned char g = plte[i + 1]; + unsigned char b = plte[i + 2]; + unsigned char a = cp_get_alpha_for_indexed_image(i / 3, trns, trns_len); + cp_pixel_t p = cp_make_pixel_a(r, g, b, a); + *dst++ = p; + } +} + +cp_indexed_image_t cp_load_indexed_png_mem(const void *png_data, int png_length) +{ + const char* sig = "\211PNG\r\n\032\n"; + const uint8_t* ihdr, *first, *plte, *trns; + int bit_depth, color_type, bpp, w, h, pix_bytes; + int compression, filter, interlace; + int datalen, offset; + int plte_len; + uint8_t* out; + cp_indexed_image_t img = { 0 }; + uint8_t* data = 0; + cp_raw_png_t png; + png.p = (uint8_t*)png_data; + png.end = (uint8_t*)png_data + png_length; + + CUTE_PNG_CHECK(!CUTE_PNG_MEMCMP(png.p, sig, 8), "incorrect file signature (is this a png file?)"); + png.p += 8; + + ihdr = cp_chunk(&png, "IHDR", 13); + CUTE_PNG_CHECK(ihdr, "unable to find IHDR chunk"); + bit_depth = ihdr[8]; + color_type = ihdr[9]; + bpp = 1; // bytes per pixel + CUTE_PNG_CHECK(bit_depth == 8, "only bit-depth of 8 is supported"); + CUTE_PNG_CHECK(color_type == 3, "only indexed png images (images with a palette) are valid for cp_load_indexed_png_mem"); + + // +1 for filter byte (which is dumb! just stick this at file header...) + w = cp_make32(ihdr) + 1; + h = cp_make32(ihdr + 4); + CUTE_PNG_CHECK((int64_t) w * h * sizeof(uint8_t) < INT_MAX, "image too large"); + pix_bytes = w * h * sizeof(uint8_t); + img.w = w - 1; + img.h = h; + img.pix = (uint8_t*)CUTE_PNG_ALLOC(pix_bytes); + CUTE_PNG_CHECK(img.pix, "unable to allocate raw image space"); + + compression = ihdr[10]; + filter = ihdr[11]; + interlace = ihdr[12]; + CUTE_PNG_CHECK(!compression, "only standard compression DEFLATE is supported"); + CUTE_PNG_CHECK(!filter, "only standard adaptive filtering is supported"); + CUTE_PNG_CHECK(!interlace, "interlacing is not supported"); + + // PLTE must come before any IDAT chunk + first = png.p; + plte = cp_find(&png, "PLTE", 0); + if (!plte) png.p = first; + else first = png.p; + + // tRNS can come after PLTE + trns = cp_find(&png, "tRNS", 0); + if (!trns) png.p = first; + else first = png.p; + + // Compute length of the DEFLATE stream through IDAT chunk data sizes + datalen = 0; + for (const uint8_t* idat = cp_find(&png, "IDAT", 0); idat; idat = cp_chunk(&png, "IDAT", 0)) + { + uint32_t len = cp_get_chunk_byte_length(idat); + datalen += len; + } + + // Copy in IDAT chunk data sections to form the compressed DEFLATE stream + png.p = first; + data = (uint8_t*)CUTE_PNG_ALLOC(datalen); + offset = 0; + for (const uint8_t* idat = cp_find(&png, "IDAT", 0); idat; idat = cp_chunk(&png, "IDAT", 0)) + { + uint32_t len = cp_get_chunk_byte_length(idat); + CUTE_PNG_MEMCPY(data + offset, idat, len); + offset += len; + } + + // check for proper zlib structure in DEFLATE stream + CUTE_PNG_CHECK(data && datalen >= 6, "corrupt zlib structure in DEFLATE stream"); + CUTE_PNG_CHECK((data[0] & 0x0f) == 0x08, "only zlib compression method (RFC 1950) is supported"); + CUTE_PNG_CHECK((data[0] & 0xf0) <= 0x70, "innapropriate window size detected"); + CUTE_PNG_CHECK(!(data[1] & 0x20), "preset dictionary is present and not supported"); + + out = img.pix; + CUTE_PNG_CHECK(cp_inflate(data + 2, datalen - 6, out, pix_bytes), "DEFLATE algorithm failed"); + CUTE_PNG_CHECK(cp_unfilter(img.w, img.h, bpp, out), "invalid filter byte found"); + cp_unpack_indexed_rows(img.w, img.h, out, img.pix); + + plte_len = cp_get_chunk_byte_length(plte) / 3; + cp_unpack_palette(img.palette, plte, plte_len, trns, cp_get_chunk_byte_length(trns)); + img.palette_len = (uint8_t)plte_len; + + CUTE_PNG_FREE(data); + return img; + +cp_err: + CUTE_PNG_FREE(data); + CUTE_PNG_FREE(img.pix); + img.pix = 0; + + return img; +} + +void cp_free_indexed_png(cp_indexed_image_t* img) +{ + CUTE_PNG_FREE(img->pix); + img->pix = 0; + img->w = img->h = 0; +} + +cp_image_t cp_depallete_indexed_image(cp_indexed_image_t* img) +{ + cp_image_t out = { 0 }; + out.w = img->w; + out.h = img->h; + out.pix = (cp_pixel_t*)CUTE_PNG_ALLOC(sizeof(cp_pixel_t) * out.w * out.h); + + cp_pixel_t* dst = out.pix; + uint8_t* src = img->pix; + + for (int y = 0; y < out.h; ++y) + { + for (int x = 0; x < out.w; ++x) + { + int index = *src++; + cp_pixel_t p = img->palette[index]; + *dst++ = p; + } + } + + return out; +} + +typedef struct cp_v2i_t +{ + int x; + int y; +} cp_v2i_t; + +typedef struct cp_integer_image_t +{ + int img_index; + cp_v2i_t size; + cp_v2i_t min; + cp_v2i_t max; + int fit; +} cp_integer_image_t; + +static cp_v2i_t cp_v2i(int x, int y) +{ + cp_v2i_t v; + v.x = x; + v.y = y; + return v; +} + +static cp_v2i_t cp_sub(cp_v2i_t a, cp_v2i_t b) +{ + cp_v2i_t v; + v.x = a.x - b.x; + v.y = a.y - b.y; + return v; +} + +static cp_v2i_t cp_add(cp_v2i_t a, cp_v2i_t b) +{ + cp_v2i_t v; + v.x = a.x + b.x; + v.y = a.y + b.y; + return v; +} + +typedef struct cp_atlas_node_t +{ + cp_v2i_t size; + cp_v2i_t min; + cp_v2i_t max; +} cp_atlas_node_t; + +static cp_atlas_node_t* cp_best_fit(int sp, const cp_image_t* png, cp_atlas_node_t* nodes) +{ + int bestVolume = INT_MAX; + cp_atlas_node_t *best_node = 0; + int width = png->w; + int height = png->h; + int png_volume = width * height; + + for (int i = 0; i < sp; ++i) + { + cp_atlas_node_t *node = nodes + i; + int can_contain = node->size.x >= width && node->size.y >= height; + if (can_contain) + { + int node_volume = node->size.x * node->size.y; + if (node_volume == png_volume) return node; + if (node_volume < bestVolume) + { + bestVolume = node_volume; + best_node = node; + } + } + } + + return best_node; +} + +static int cp_perimeter_pred(cp_integer_image_t* a, cp_integer_image_t* b) +{ + int perimeterA = 2 * (a->size.x + a->size.y); + int perimeterB = 2 * (b->size.x + b->size.y); + return perimeterB < perimeterA; +} + +void cp_premultiply(cp_image_t* img) +{ + int w = img->w; + int h = img->h; + int stride = w * sizeof(cp_pixel_t); + uint8_t* data = (uint8_t*)img->pix; + + for(int i = 0; i < (int)stride * h; i += sizeof(cp_pixel_t)) + { + float a = (float)data[i + 3] / 255.0f; + float r = (float)data[i + 0] / 255.0f; + float g = (float)data[i + 1] / 255.0f; + float b = (float)data[i + 2] / 255.0f; + r *= a; + g *= a; + b *= a; + data[i + 0] = (uint8_t)(r * 255.0f); + data[i + 1] = (uint8_t)(g * 255.0f); + data[i + 2] = (uint8_t)(b * 255.0f); + } +} + +static void cp_qsort(cp_integer_image_t* items, int count) +{ + if (count <= 1) return; + + cp_integer_image_t pivot = items[count - 1]; + int low = 0; + for (int i = 0; i < count - 1; ++i) + { + if (cp_perimeter_pred(items + i, &pivot)) + { + cp_integer_image_t tmp = items[i]; + items[i] = items[low]; + items[low] = tmp; + low++; + } + } + + items[count - 1] = items[low]; + items[low] = pivot; + cp_qsort(items, low); + cp_qsort(items + low + 1, count - 1 - low); +} + +static void cp_write_pixel(char* mem, long color) { + mem[0] = (color >> 24) & 0xFF; + mem[1] = (color >> 16) & 0xFF; + mem[2] = (color >> 8) & 0xFF; + mem[3] = (color >> 0) & 0xFF; +} + +cp_image_t cp_make_atlas(int atlas_width, int atlas_height, const cp_image_t* pngs, int png_count, cp_atlas_image_t* imgs_out) +{ + float w0, h0, div, wTol, hTol; + int atlas_image_size, atlas_stride, sp; + void* atlas_pixels = 0; + int atlas_node_capacity = png_count * 2; + cp_image_t atlas_image; + cp_integer_image_t* images = 0; + cp_atlas_node_t* nodes = 0; + + atlas_image.w = atlas_width; + atlas_image.h = atlas_height; + atlas_image.pix = 0; + + CUTE_PNG_CHECK(pngs, "pngs array was NULL"); + CUTE_PNG_CHECK(imgs_out, "imgs_out array was NULL"); + + images = (cp_integer_image_t*)CUTE_PNG_ALLOCA(sizeof(cp_integer_image_t) * png_count); + nodes = (cp_atlas_node_t*)CUTE_PNG_ALLOC(sizeof(cp_atlas_node_t) * atlas_node_capacity); + CUTE_PNG_CHECK(images, "out of mem"); + CUTE_PNG_CHECK(nodes, "out of mem"); + + for (int i = 0; i < png_count; ++i) + { + const cp_image_t* png = pngs + i; + cp_integer_image_t* image = images + i; + image->fit = 0; + image->size = cp_v2i(png->w, png->h); + image->img_index = i; + } + + // Sort PNGs from largest to smallest + cp_qsort(images, png_count); + + // stack pointer, the stack is the nodes array which we will + // allocate nodes from as necessary. + sp = 1; + + nodes[0].min = cp_v2i(0, 0); + nodes[0].max = cp_v2i(atlas_width, atlas_height); + nodes[0].size = cp_v2i(atlas_width, atlas_height); + + // Nodes represent empty space in the atlas. Placing a texture into the + // atlas involves splitting a node into two smaller pieces (or, if a + // perfect fit is found, deleting the node). + for (int i = 0; i < png_count; ++i) + { + cp_integer_image_t* image = images + i; + const cp_image_t* png = pngs + image->img_index; + int width = png->w; + int height = png->h; + cp_atlas_node_t *best_fit = cp_best_fit(sp, png, nodes); + if (CUTE_PNG_ATLAS_MUST_FIT) CUTE_PNG_CHECK(best_fit, "Not enough room to place image in atlas."); + else if (!best_fit) + { + image->fit = 0; + continue; + } + + image->min = best_fit->min; + image->max = cp_add(image->min, image->size); + + if (best_fit->size.x == width && best_fit->size.y == height) + { + cp_atlas_node_t* last_node = nodes + --sp; + *best_fit = *last_node; + image->fit = 1; + + continue; + } + + image->fit = 1; + + if (sp == atlas_node_capacity) + { + int new_capacity = atlas_node_capacity * 2; + cp_atlas_node_t* new_nodes = (cp_atlas_node_t*)CUTE_PNG_ALLOC(sizeof(cp_atlas_node_t) * new_capacity); + CUTE_PNG_CHECK(new_nodes, "out of mem"); + CUTE_PNG_MEMCPY(new_nodes, nodes, sizeof(cp_atlas_node_t) * sp); + + // best_fit became a dangling pointer, so relocate it + best_fit = new_nodes + (best_fit - nodes); + CUTE_PNG_FREE(nodes); + + nodes = new_nodes; + atlas_node_capacity = new_capacity; + } + + cp_atlas_node_t* new_node = nodes + sp++; + new_node->min = best_fit->min; + + // Split bestFit along x or y, whichever minimizes + // fragmentation of empty space + cp_v2i_t d = cp_sub(best_fit->size, cp_v2i(width, height)); + if (d.x < d.y) + { + new_node->size.x = d.x; + new_node->size.y = height; + new_node->min.x += width; + + best_fit->size.y = d.y; + best_fit->min.y += height; + } + + else + { + new_node->size.x = width; + new_node->size.y = d.y; + new_node->min.y += height; + + best_fit->size.x = d.x; + best_fit->min.x += width; + } + + new_node->max = cp_add(new_node->min, new_node->size); + } + + // Write the final atlas image, use CUTE_PNG_ATLAS_EMPTY_COLOR as base color + atlas_stride = atlas_width * sizeof(cp_pixel_t); + atlas_image_size = atlas_width * atlas_height * sizeof(cp_pixel_t); + atlas_pixels = CUTE_PNG_ALLOC(atlas_image_size); + CUTE_PNG_CHECK(atlas_pixels, "out of mem"); + + for(int i = 0; i < atlas_image_size; i += sizeof(cp_pixel_t)) { + cp_write_pixel((char*)atlas_pixels + i, CUTE_PNG_ATLAS_EMPTY_COLOR); + } + + for (int i = 0; i < png_count; ++i) + { + cp_integer_image_t* image = images + i; + + if (image->fit) + { + const cp_image_t* png = pngs + image->img_index; + char* pixels = (char*)png->pix; + cp_v2i_t min = image->min; + cp_v2i_t max = image->max; + int atlas_offset = min.x * sizeof(cp_pixel_t); + int tex_stride = png->w * sizeof(cp_pixel_t); + + for (int row = min.y, y = 0; row < max.y; ++row, ++y) + { + void* row_ptr = (char*)atlas_pixels + (row * atlas_stride + atlas_offset); + CUTE_PNG_MEMCPY(row_ptr, pixels + y * tex_stride, tex_stride); + } + } + } + + atlas_image.pix = (cp_pixel_t*)atlas_pixels; + + // squeeze UVs inward by 128th of a pixel + // this prevents atlas bleeding. tune as necessary for good results. + w0 = 1.0f / (float)(atlas_width); + h0 = 1.0f / (float)(atlas_height); + div = 1.0f / 128.0f; + wTol = w0 * div; + hTol = h0 * div; + + for (int i = 0; i < png_count; ++i) + { + cp_integer_image_t* image = images + i; + cp_atlas_image_t* img_out = imgs_out + i; + + img_out->img_index = image->img_index; + img_out->w = image->size.x; + img_out->h = image->size.y; + img_out->fit = image->fit; + + if (image->fit) + { + cp_v2i_t min = image->min; + cp_v2i_t max = image->max; + + float min_x = (float)min.x * w0 + wTol; + float min_y = (float)min.y * h0 + hTol; + float max_x = (float)max.x * w0 - wTol; + float max_y = (float)max.y * h0 - hTol; + + // flip image on y axis + if (CUTE_PNG_ATLAS_FLIP_Y_AXIS_FOR_UV) + { + float tmp = min_y; + min_y = max_y; + max_y = tmp; + } + + img_out->minx = min_x; + img_out->miny = min_y; + img_out->maxx = max_x; + img_out->maxy = max_y; + } + } + + CUTE_PNG_FREE(nodes); + return atlas_image; + +cp_err: + CUTE_PNG_FREE(atlas_pixels); + CUTE_PNG_FREE(nodes); + atlas_image.pix = 0; + return atlas_image; +} + +int cp_default_save_atlas(const char* out_path_image, const char* out_path_atlas_txt, const cp_image_t* atlas, const cp_atlas_image_t* imgs, int img_count, const char** names) +{ + CUTE_PNG_FILE* fp = CUTE_PNG_FOPEN(out_path_atlas_txt, "wt"); + CUTE_PNG_CHECK(fp, "unable to open out_path_atlas_txt in cp_default_save_atlas"); + + CUTE_PNG_FPRINTF(fp, "%s\n%d\n\n", out_path_image, img_count); + + for (int i = 0; i < img_count; ++i) + { + const cp_atlas_image_t* image = imgs + i; + const char* name = names ? names[image->img_index] : 0; + + if (image->fit) + { + int width = image->w; + int height = image->h; + float min_x = image->minx; + float min_y = image->miny; + float max_x = image->maxx; + float max_y = image->maxy; + + if (name) CUTE_PNG_FPRINTF(fp, "{ \"%s\", w = %d, h = %d, u = { %.10f, %.10f }, v = { %.10f, %.10f } }\n", name, width, height, min_x, min_y, max_x, max_y); + else CUTE_PNG_FPRINTF(fp, "{ w = %d, h = %d, u = { %.10f, %.10f }, v = { %.10f, %.10f } }\n", width, height, min_x, min_y, max_x, max_y); + } + } + + // Save atlas image PNG to disk + CUTE_PNG_CHECK(cp_save_png(out_path_image, atlas), "failed to save atlas image to disk"); + +cp_err: + CUTE_PNG_FCLOSE(fp); + return 0; +} + +#endif // CUTE_PNG_IMPLEMENTATION_ONCE +#endif // CUTE_PNG_IMPLEMENTATION + +/* + ------------------------------------------------------------------------------ + This software is available under 2 licenses - you may choose the one you like. + ------------------------------------------------------------------------------ + ALTERNATIVE A - zlib license + Copyright (c) 2019 Randy Gaul http://www.randygaul.net + This software is provided 'as-is', without any express or implied warranty. + In no event will the authors be held liable for any damages arising from + the use of this software. + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not + be misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + ------------------------------------------------------------------------------ + ALTERNATIVE B - Public Domain (www.unlicense.org) + This is free and unencumbered software released into the public domain. + Anyone is free to copy, modify, publish, use, compile, sell, or distribute this + software, either in source code form or as a compiled binary, for any purpose, + commercial or non-commercial, and by any means. + In jurisdictions that recognize copyright laws, the author or authors of this + software dedicate any and all copyright interest in the software to the public + domain. We make this dedication for the benefit of the public at large and to + the detriment of our heirs and successors. We intend this dedication to be an + overt act of relinquishment in perpetuity of all present and future rights to + this software under copyright law. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ------------------------------------------------------------------------------ +*/ \ No newline at end of file diff --git a/3rd/cute_png/src/cute_png.c b/3rd/cute_png/src/cute_png.c new file mode 100644 index 00000000..57547b46 --- /dev/null +++ b/3rd/cute_png/src/cute_png.c @@ -0,0 +1,56 @@ +#include "pocketpy.h" + +#define CUTE_PNG_IMPLEMENTATION +#include "cute_png.h" + +static bool cute_png_loads(int argc, py_Ref argv) { + PY_CHECK_ARGC(1); + PY_CHECK_ARG_TYPE(0, tp_bytes); + int size; + unsigned char* data = py_tobytes(argv, &size); + cp_image_t image = cp_load_png_mem(data, size); + if(image.pix == NULL) return ValueError("cute_png: %s", cp_error_reason); + py_newarray2d(py_retval(), image.w, image.h); + for(int y = 0; y < image.h; y++) { + for(int x = 0; x < image.w; x++) { + cp_pixel_t pixel = image.pix[y * image.w + x]; + py_ObjectRef slot = py_array2d_getitem(py_retval(), x, y); + c11_color32 color; + color.r = pixel.r; + color.g = pixel.g; + color.b = pixel.b; + color.a = pixel.a; + py_newcolor32(slot, color); + } + } + return true; +} + +static bool cute_png_dumps(int argc, py_Ref argv) { + PY_CHECK_ARGC(1); + PY_CHECK_ARG_TYPE(0, tp_array2d); + int width = py_array2d_getwidth(argv); + int height = py_array2d_getheight(argv); + cp_image_t image = cp_load_blank(width, height); + for(int y = 0; y < height; y++) { + for(int x = 0; x < width; x++) { + py_ObjectRef slot = py_array2d_getitem(argv, x, y); + if(!py_checktype(slot, tp_color32)) return false; + c11_color32 color = py_tocolor32(slot); + cp_pixel_t pixel = cp_make_pixel_a(color.r, color.g, color.b, color.a); + image.pix[y * width + x] = pixel; + } + } + cp_saved_png_t saved_image = cp_save_png_to_memory(&image); + assert(saved_image.data != NULL); + unsigned char* data = py_newbytes(py_retval(), saved_image.size); + memcpy(data, saved_image.data, saved_image.size); + return true; +} + +void pk__add_module_cute_png() { + py_GlobalRef mod = py_newmodule("cute_png"); + + py_bindfunc(mod, "loads", cute_png_loads); + py_bindfunc(mod, "dumps", cute_png_dumps); +} \ No newline at end of file diff --git a/3rd/libhv/CMakeLists.txt b/3rd/libhv/CMakeLists.txt index 786adf9d..f17a066c 100644 --- a/3rd/libhv/CMakeLists.txt +++ b/3rd/libhv/CMakeLists.txt @@ -2,8 +2,6 @@ cmake_minimum_required(VERSION 3.10) project(libhv_bindings) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 14) diff --git a/3rd/lz4/CMakeLists.txt b/3rd/lz4/CMakeLists.txt index 25040e5e..17d125fc 100644 --- a/3rd/lz4/CMakeLists.txt +++ b/3rd/lz4/CMakeLists.txt @@ -5,12 +5,12 @@ project(lz4) set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) -add_library(lz4 STATIC lz4/lib/lz4.c) +add_library(${PROJECT_NAME} STATIC lz4/lib/lz4.c) -target_include_directories(lz4 PRIVATE +target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/lz4/lib ) -target_include_directories(lz4 INTERFACE +target_include_directories(${PROJECT_NAME} INTERFACE ${CMAKE_CURRENT_LIST_DIR} ) diff --git a/CMakeLists.txt b/CMakeLists.txt index f9ec20fb..2fd4aca2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,10 @@ cmake_minimum_required(VERSION 3.10) project(pocketpy) +if(PK_BUILD_SHARED_LIB OR NOT PK_BUILD_STATIC_MAIN) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) +endif() + set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) @@ -13,8 +17,7 @@ endif() if(MSVC) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8 /jumptablerdata") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /experimental:c11atomics") - add_compile_options(/wd4267 /wd4244 /wd4146) + add_compile_options(/wd4267 /wd4244 /wd4146 /experimental:c11atomics) if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Ox") @@ -47,19 +50,57 @@ file(GLOB_RECURSE POCKETPY_SRC ${CMAKE_CURRENT_LIST_DIR}/src/*.c) if(PK_ENABLE_OS) add_definitions(-DPK_ENABLE_OS=1) +else() + add_definitions(-DPK_ENABLE_OS=0) +endif() + +if(PK_ENABLE_THREADS) + add_definitions(-DPK_ENABLE_THREADS=1) +else() + add_definitions(-DPK_ENABLE_THREADS=0) endif() if(PK_ENABLE_DETERMINISM) add_subdirectory(3rd/dmath/dmath) add_definitions(-DPK_ENABLE_DETERMINISM=1) +else() + add_definitions(-DPK_ENABLE_DETERMINISM=0) endif() if(PK_ENABLE_WATCHDOG) add_definitions(-DPK_ENABLE_WATCHDOG=1) +else() + add_definitions(-DPK_ENABLE_WATCHDOG=0) endif() if(PK_ENABLE_CUSTOM_SNAME) add_definitions(-DPK_ENABLE_CUSTOM_SNAME=1) +else() + add_definitions(-DPK_ENABLE_CUSTOM_SNAME=0) +endif() + +if(PK_ENABLE_MIMALLOC) + message(">> Fetching mimalloc") + include(FetchContent) + + FetchContent_Declare( + mimalloc + GIT_REPOSITORY https://github.com/microsoft/mimalloc.git + GIT_TAG v3.1.5 + ) + + set(MI_OVERRIDE OFF CACHE BOOL "" FORCE) + set(MI_NO_USE_CXX ON CACHE BOOL "" FORCE) + set(MI_BUILD_SHARED OFF CACHE BOOL "" FORCE) + set(MI_BUILD_OBJECT OFF CACHE BOOL "" FORCE) + set(MI_BUILD_STATIC ON CACHE BOOL "" FORCE) + set(MI_BUILD_TESTS OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(mimalloc) + include_directories(${mimalloc_SOURCE_DIR}/include) + + add_definitions(-DPK_ENABLE_MIMALLOC=1) +else() + add_definitions(-DPK_ENABLE_MIMALLOC=0) endif() @@ -73,6 +114,12 @@ if(PK_BUILD_MODULE_LIBHV) add_definitions(-DPK_BUILD_MODULE_LIBHV) endif() +if(PK_BUILD_MODULE_CUTE_PNG) + add_subdirectory(3rd/cute_png) + add_definitions(-DPK_BUILD_MODULE_CUTE_PNG) +endif() + + if(PK_BUILD_SHARED_LIB) message(">> Building shared library") add_library(${PROJECT_NAME} SHARED ${POCKETPY_SRC}) @@ -104,8 +151,10 @@ if(PK_ENABLE_DETERMINISM) endif() endif() -find_package(Threads REQUIRED) -target_link_libraries(${PROJECT_NAME} Threads::Threads) +if(PK_ENABLE_THREADS) + find_package(Threads REQUIRED) + target_link_libraries(${PROJECT_NAME} Threads::Threads) +endif() if(UNIX AND NOT APPLE) if(NOT PK_ENABLE_DETERMINISM) @@ -133,4 +182,13 @@ endif() if(PK_BUILD_MODULE_LIBHV) target_link_libraries(${PROJECT_NAME} libhv_bindings) +endif() + +if(PK_BUILD_MODULE_CUTE_PNG) + target_link_libraries(${PROJECT_NAME} cute_png) +endif() + + +if(PK_ENABLE_MIMALLOC) + target_link_libraries(${PROJECT_NAME} mimalloc-static) endif() \ No newline at end of file diff --git a/CMakeOptions.txt b/CMakeOptions.txt index 10f4fa20..3dc0813a 100644 --- a/CMakeOptions.txt +++ b/CMakeOptions.txt @@ -6,15 +6,17 @@ else() endif() # system features -option(PK_ENABLE_OS "" OFF) +option(PK_ENABLE_OS "" ON) +option(PK_ENABLE_THREADS "" ON) option(PK_ENABLE_DETERMINISM "" OFF) option(PK_ENABLE_WATCHDOG "" OFF) option(PK_ENABLE_CUSTOM_SNAME "" OFF) +option(PK_ENABLE_MIMALLOC "" OFF) # modules option(PK_BUILD_MODULE_LZ4 "" OFF) option(PK_BUILD_MODULE_LIBHV "" OFF) - +option(PK_BUILD_MODULE_CUTE_PNG "" OFF) # PK_IS_MAIN determines whether the project is being used from root # or if it is added as a dependency (through add_subdirectory for example). diff --git a/README.md b/README.md index 85602abe..baad208d 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Developers are able to write Python bindings via C-API or pybind11 compatible in Please see https://pocketpy.dev for details and try the following resources. + [Live Python Demo](https://pocketpy.dev/static/web/): Run Python code in your browser + [Live C Examples](https://pocketpy.github.io/examples/): Explore C-APIs in your browser ++ [Godot Extension](https://github.com/pocketpy/godot-pocketpy): Use pocketpy in Godot Engine ## Supported Platforms @@ -186,6 +187,7 @@ And these are the results of the primes benchmark on Intel i5-12400F, WSL (Ubunt | | Description | |-----------------------------------------------------------------|--------------------------------------------------------------------------| +| [godot-pocketpy](https://github.com/pocketpy/godot-pocketpy) | Godot extension for using pocketpy in Godot Engine. | | [TIC-80](https://github.com/nesbox/TIC-80) | TIC-80 is a fantasy computer for making, playing and sharing tiny games. | | [py-js](https://github.com/shakfu/py-js) | Python3 externals for Max / MSP. | | [crescent](https://github.com/chukobyte/crescent) | Crescent is a cross-platform 2D fighting and beat-em-up game engine. | diff --git a/build_android.sh b/build_android.sh index 479626c9..db14c0b6 100644 --- a/build_android.sh +++ b/build_android.sh @@ -16,6 +16,7 @@ cmake \ -DPK_BUILD_SHARED_LIB=ON \ -DCMAKE_BUILD_TYPE=Release \ -DPK_BUILD_MODULE_LZ4=ON \ - -DPK_BUILD_MODULE_LIBHV=ON + -DPK_BUILD_MODULE_LIBHV=ON \ + -DPK_BUILD_CUTE_PNG=ON cmake --build . --config Release diff --git a/cmake_build.py b/cmake_build.py index 0e47e990..954f9650 100644 --- a/cmake_build.py +++ b/cmake_build.py @@ -20,7 +20,7 @@ assert config in ['Debug', 'Release', 'RelWithDebInfo'] os.chdir("build") -code = os.system(f"cmake .. -DPK_ENABLE_OS=ON -DPK_ENABLE_DETERMINISM=ON -DCMAKE_BUILD_TYPE={config} {extra_flags}") +code = os.system(f"cmake .. -DPK_ENABLE_MIMALLOC=ON -DPK_ENABLE_DETERMINISM=ON -DCMAKE_BUILD_TYPE={config} {extra_flags}") assert code == 0 code = os.system(f"cmake --build . --config {config} -j 4") assert code == 0 diff --git a/compile_flags.txt b/compile_flags.txt index d0397504..e98f6100 100644 --- a/compile_flags.txt +++ b/compile_flags.txt @@ -4,4 +4,5 @@ -std=c11 -Iinclude/ -I3rd/lz4/ --I3rd/libhv/include/ \ No newline at end of file +-I3rd/libhv/include/ +-I3rd/cute_headers/include/ \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index bf31d29b..e112d063 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,8 @@ Developers are able to write Python bindings via C-API or pybind11 compatible in + [Live Python Demo](https://pocketpy.dev/static/web/): Run Python code in your browser + [Live C Examples](https://pocketpy.github.io/examples/): Explore C-APIs in your browser ++ [Godot Extension](https://github.com/pocketpy/godot-pocketpy): Use pocketpy in Godot Engine ++ [Flutter Plugin](https://pub.dev/packages/pocketpy): Use pocketpy in Flutter apps ## What it looks like diff --git a/docs/modules/cute_png.md b/docs/modules/cute_png.md new file mode 100644 index 00000000..1af88aaa --- /dev/null +++ b/docs/modules/cute_png.md @@ -0,0 +1,14 @@ +--- +icon: package +label: cute_png +--- + +!!! +This module is optional. Set option `PK_BUILD_MODULE_CUTE_PNG` to `ON` in your `CMakeLists.txt` to enable it. +!!! + +Wraps [cute_png.h](https://github.com/RandyGaul/cute_headers/blob/master/cute_png.h) for PNG image loading and saving. + +#### Source code + +:::code source="../../include/typings/cute_png.pyi" ::: diff --git a/include/pocketpy/common/chunkedvector.h b/include/pocketpy/common/chunkedvector.h index 682210b7..6468a930 100644 --- a/include/pocketpy/common/chunkedvector.h +++ b/include/pocketpy/common/chunkedvector.h @@ -1,6 +1,7 @@ #pragma once #include "pocketpy/common/vector.h" + typedef struct c11_chunkedvector_chunk { int length; int capacity; diff --git a/include/pocketpy/common/socket.h b/include/pocketpy/common/socket.h index 87e21974..ff98c2cc 100644 --- a/include/pocketpy/common/socket.h +++ b/include/pocketpy/common/socket.h @@ -1,7 +1,10 @@ #pragma once +#include "pocketpy/config.h" #include +#if PK_ENABLE_OS + typedef void* c11_socket_handler; enum c11_address_family { C11_AF_INET = 2 }; @@ -23,3 +26,5 @@ int c11_socket_close(c11_socket_handler socket); int c11_socket_set_block(c11_socket_handler socket, int flag); c11_socket_handler c11_socket_invalid_socket_handler(); int c11_socket_get_last_error(); + +#endif // PK_ENABLE_OS \ No newline at end of file diff --git a/include/pocketpy/common/str.h b/include/pocketpy/common/str.h index 83fcf20f..5ccaf5f7 100644 --- a/include/pocketpy/common/str.h +++ b/include/pocketpy/common/str.h @@ -67,6 +67,7 @@ c11_string* c11_sv__replace2(c11_sv self, c11_sv old, c11_sv new_); c11_vector /* T=c11_sv */ c11_sv__split(c11_sv self, char sep); c11_vector /* T=c11_sv */ c11_sv__split2(c11_sv self, c11_sv sep); +c11_vector /* T=c11_sv */ c11_sv__splitwhitespace(c11_sv self); // misc int c11__unicode_index_to_byte(const char* data, int i); diff --git a/include/pocketpy/common/threads.h b/include/pocketpy/common/threads.h index 0791433a..d1d1fa51 100644 --- a/include/pocketpy/common/threads.h +++ b/include/pocketpy/common/threads.h @@ -1,5 +1,9 @@ #pragma once +#include "pocketpy/config.h" + +#if PK_ENABLE_THREADS + #include #include @@ -17,3 +21,5 @@ typedef int c11_thrd_retval_t; bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg); void c11_thrd_yield(); + +#endif \ No newline at end of file diff --git a/include/pocketpy/common/utils.h b/include/pocketpy/common/utils.h index 5a219ac9..ef6f1134 100644 --- a/include/pocketpy/common/utils.h +++ b/include/pocketpy/common/utils.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #define PK_REGION(name) 1 diff --git a/include/pocketpy/common/vector.h b/include/pocketpy/common/vector.h index 93f784f4..7f4e1808 100644 --- a/include/pocketpy/common/vector.h +++ b/include/pocketpy/common/vector.h @@ -1,7 +1,5 @@ #pragma once -#include -#include #include #include @@ -23,6 +21,7 @@ void* c11_vector__emplace(c11_vector* self); bool c11_vector__contains(const c11_vector* self, void* elem); void* c11_vector__submit(c11_vector* self, int* length); void c11_vector__swap(c11_vector* self, c11_vector* other); +int c11_vector__nextcap(c11_vector* self); #define c11__getitem(T, self, index) (((T*)(self)->data)[index]) #define c11__setitem(T, self, index, value) ((T*)(self)->data)[index] = value; @@ -30,7 +29,9 @@ void c11_vector__swap(c11_vector* self, c11_vector* other); #define c11_vector__push(T, self, elem) \ do { \ - if((self)->length == (self)->capacity) c11_vector__reserve((self), (self)->capacity * 2); \ + if((self)->length == (self)->capacity) { \ + c11_vector__reserve((self), c11_vector__nextcap((self))); \ + } \ ((T*)(self)->data)[(self)->length] = (elem); \ (self)->length++; \ } while(0) @@ -42,15 +43,19 @@ void c11_vector__swap(c11_vector* self, c11_vector* other); #define c11_vector__extend(T, self, p, size) \ do { \ int min_capacity = (self)->length + (size); \ - if((self)->capacity < min_capacity) \ - c11_vector__reserve((self), c11__max((self)->capacity * 2, min_capacity)); \ + if((self)->capacity < min_capacity) { \ + int nextcap = c11_vector__nextcap(self); \ + c11_vector__reserve((self), c11__max(nextcap, min_capacity)); \ + } \ memcpy((T*)(self)->data + (self)->length, (p), (size) * sizeof(T)); \ (self)->length += (size); \ } while(0) #define c11_vector__insert(T, self, index, elem) \ do { \ - if((self)->length == (self)->capacity) c11_vector__reserve((self), (self)->capacity * 2); \ + if((self)->length == (self)->capacity) { \ + c11_vector__reserve((self), c11_vector__nextcap(self)); \ + } \ T* p = (T*)(self)->data + (index); \ memmove(p + 1, p, ((self)->length - (index)) * sizeof(T)); \ *p = (elem); \ diff --git a/include/pocketpy/config.h b/include/pocketpy/config.h index 91ad1d4e..53109114 100644 --- a/include/pocketpy/config.h +++ b/include/pocketpy/config.h @@ -1,17 +1,20 @@ #pragma once // clang-format off -#define PK_VERSION "2.1.0" +#define PK_VERSION "2.1.1" #define PK_VERSION_MAJOR 2 #define PK_VERSION_MINOR 1 -#define PK_VERSION_PATCH 0 +#define PK_VERSION_PATCH 1 /*************** feature settings ***************/ -// Whether to compile os-related modules or not #ifndef PK_ENABLE_OS // can be overridden by cmake #define PK_ENABLE_OS 1 #endif +#ifndef PK_ENABLE_THREADS // can be overridden by cmake +#define PK_ENABLE_THREADS 1 +#endif + #ifndef PK_ENABLE_DETERMINISM // must be enabled from cmake #define PK_ENABLE_DETERMINISM 0 #endif @@ -24,24 +27,15 @@ #define PK_ENABLE_CUSTOM_SNAME 0 #endif +#ifndef PK_ENABLE_MIMALLOC // can be overridden by cmake +#define PK_ENABLE_MIMALLOC 0 +#endif + // GC min threshold #ifndef PK_GC_MIN_THRESHOLD // can be overridden by cmake #define PK_GC_MIN_THRESHOLD 32768 #endif -// Memory allocation functions -#ifndef PK_MALLOC - #ifndef __cplusplus - #define PK_MALLOC(size) malloc(size) - #define PK_REALLOC(ptr, size) realloc(ptr, size) - #define PK_FREE(ptr) free(ptr) - #else - #define PK_MALLOC(size) std::malloc(size) - #define PK_REALLOC(ptr, size) std::realloc(ptr, size) - #define PK_FREE(ptr) std::free(ptr) - #endif -#endif - // This is the maximum size of the value stack in py_TValue units // The actual size in bytes equals `sizeof(py_TValue) * PK_VM_STACK_SIZE` #ifndef PK_VM_STACK_SIZE // can be overridden by cmake @@ -81,3 +75,31 @@ #define restrict #endif #endif + +#if PK_ENABLE_THREADS + #define PK_THREAD_LOCAL _Thread_local +#else + #define PK_THREAD_LOCAL +#endif + +// Memory allocation functions +#ifndef PK_MALLOC + #if PK_ENABLE_MIMALLOC + #include "mimalloc.h" + #define PK_MALLOC(size) mi_malloc(size) + #define PK_REALLOC(ptr, size) mi_realloc(ptr, size) + #define PK_FREE(ptr) mi_free(ptr) + #else + #ifndef __cplusplus + #include + #define PK_MALLOC(size) malloc(size) + #define PK_REALLOC(ptr, size) realloc(ptr, size) + #define PK_FREE(ptr) free(ptr) + #else + #include + #define PK_MALLOC(size) std::malloc(size) + #define PK_REALLOC(ptr, size) std::realloc(ptr, size) + #define PK_FREE(ptr) std::free(ptr) + #endif + #endif +#endif diff --git a/include/pocketpy/export.h b/include/pocketpy/export.h index d115a8e0..44104bae 100644 --- a/include/pocketpy/export.h +++ b/include/pocketpy/export.h @@ -60,4 +60,16 @@ #define PK_DEPRECATED __attribute__((deprecated)) #else #define PK_DEPRECATED -#endif \ No newline at end of file +#endif + +#ifdef NDEBUG + #if defined(__GNUC__) + #define PK_INLINE __attribute__((always_inline)) inline + #elif defined(_MSC_VER) + #define PK_INLINE __forceinline + #else + #define PK_INLINE inline + #endif +#else + #define PK_INLINE +#endif diff --git a/include/pocketpy/interpreter/array2d.h b/include/pocketpy/interpreter/array2d.h index 6bf052a2..1357b04d 100644 --- a/include/pocketpy/interpreter/array2d.h +++ b/include/pocketpy/interpreter/array2d.h @@ -31,7 +31,7 @@ typedef struct c11_array2d_view { c11_vec2i origin; } c11_array2d_view; -c11_array2d* py_newarray2d(py_OutRef out, int n_cols, int n_rows); +c11_array2d* c11_newarray2d(py_OutRef out, int n_cols, int n_rows); /* chunked_array2d */ #define SMALLMAP_T__HEADER diff --git a/include/pocketpy/interpreter/bindings.h b/include/pocketpy/interpreter/bindings.h new file mode 100644 index 00000000..5bc25594 --- /dev/null +++ b/include/pocketpy/interpreter/bindings.h @@ -0,0 +1,11 @@ +#pragma once + +#include "pocketpy/pocketpy.h" + +bool generator__next__(int argc, py_Ref argv); +bool array2d_like_iterator__next__(int argc, py_Ref argv); +bool list_iterator__next__(int argc, py_Ref argv); +bool tuple_iterator__next__(int argc, py_Ref argv); +bool dict_items__next__(int argc, py_Ref argv); +bool range_iterator__next__(int argc, py_Ref argv); +bool str_iterator__next__(int argc, py_Ref argv); \ No newline at end of file diff --git a/include/pocketpy/interpreter/modules.h b/include/pocketpy/interpreter/modules.h index 76d1f993..7f3385f2 100644 --- a/include/pocketpy/interpreter/modules.h +++ b/include/pocketpy/interpreter/modules.h @@ -31,4 +31,10 @@ void pk__add_module_pkpy(); void pk__add_module_libhv(); #else #define pk__add_module_libhv() -#endif \ No newline at end of file +#endif + +#ifdef PK_BUILD_MODULE_CUTE_PNG +void pk__add_module_cute_png(); +#else +#define pk__add_module_cute_png() +#endif diff --git a/include/pocketpy/interpreter/objectpool.h b/include/pocketpy/interpreter/objectpool.h index c9700583..85911161 100644 --- a/include/pocketpy/interpreter/objectpool.h +++ b/include/pocketpy/interpreter/objectpool.h @@ -5,14 +5,19 @@ #define kPoolArenaSize (120 * 1024) #define kMultiPoolCount 5 -#define kPoolMaxBlockSize (32*kMultiPoolCount) +#define kPoolMaxBlockSize (32 * kMultiPoolCount) typedef struct PoolArena { int block_size; int block_count; int unused_length; - int* unused; - char data[kPoolArenaSize]; + + union { + char data[kPoolArenaSize]; + int64_t _align64; + }; + + int unused[]; } PoolArena; typedef struct Pool { diff --git a/include/pocketpy/interpreter/typeinfo.h b/include/pocketpy/interpreter/typeinfo.h index 5dbdcb97..80790894 100644 --- a/include/pocketpy/interpreter/typeinfo.h +++ b/include/pocketpy/interpreter/typeinfo.h @@ -14,7 +14,12 @@ typedef struct py_TypeInfo { py_GlobalRef module; bool is_python; // is it a python class? (not derived from c object) - bool is_sealed; // can it be subclassed? + bool is_final; // can it be subclassed? + + bool (*getattribute)(py_Ref self, py_Name name) PY_RAISE PY_RETURN; + bool (*setattribute)(py_Ref self, py_Name name, py_Ref val) PY_RAISE PY_RETURN; + bool (*delattribute)(py_Ref self, py_Name name) PY_RAISE; + bool (*getunboundmethod)(py_Ref self, py_Name name) PY_RETURN; py_TValue annotations; py_Dtor dtor; // destructor for this type, NULL if no dtor @@ -30,7 +35,7 @@ py_Type pk_newtype(const char* name, const py_GlobalRef module, void (*dtor)(void*), bool is_python, - bool is_sealed); + bool is_final); py_Type pk_newtypewithmode(py_Name name, @@ -38,4 +43,4 @@ py_Type pk_newtypewithmode(py_Name name, const py_GlobalRef module, void (*dtor)(void*), bool is_python, - bool is_sealed, enum py_CompileMode mode); \ No newline at end of file + bool is_final, enum py_CompileMode mode); \ No newline at end of file diff --git a/include/pocketpy/interpreter/types.h b/include/pocketpy/interpreter/types.h index 5c4dd460..56535e7a 100644 --- a/include/pocketpy/interpreter/types.h +++ b/include/pocketpy/interpreter/types.h @@ -3,26 +3,22 @@ #include "pocketpy/common/vector.h" #include "pocketpy/objects/base.h" -#define PK_DICT_MAX_COLLISION 4 - typedef struct { uint64_t hash; py_TValue key; py_TValue val; } DictEntry; -typedef struct { - int _[PK_DICT_MAX_COLLISION]; -} DictIndex; - typedef struct { int length; uint32_t capacity; - DictIndex* indices; + uint32_t null_index_value; + bool index_is_short; + void* indices; c11_vector /*T=DictEntry*/ entries; } Dict; typedef c11_vector List; void c11_chunked_array2d__mark(void* ud, c11_vector* p_stack); -void function__gc_mark(void* ud, c11_vector* p_stack); \ No newline at end of file +void function__gc_mark(void* ud, c11_vector* p_stack); diff --git a/include/pocketpy/interpreter/vm.h b/include/pocketpy/interpreter/vm.h index 7436ffea..d216e612 100644 --- a/include/pocketpy/interpreter/vm.h +++ b/include/pocketpy/interpreter/vm.h @@ -67,7 +67,7 @@ typedef struct VM { NameDict compile_time_funcs; py_StackRef curr_class; - py_StackRef curr_decl_based_function; + py_StackRef curr_decl_based_function; // this is for get current function without frame TraceInfo trace_info; WatchdogInfo watchdog_info; LineProfiler line_profiler; @@ -92,7 +92,6 @@ bool pk__parse_int_slice(py_Ref slice, bool pk__normalize_index(int* index, int length); bool pk__object_new(int argc, py_Ref argv); -py_TypeInfo* pk_typeinfo(py_Type type); bool pk_wrapper__self(int argc, py_Ref argv); @@ -113,7 +112,6 @@ const char* pk_opname(Opcode op); int pk_arrayview(py_Ref self, py_TValue** p); bool pk_wrapper__arrayequal(py_Type type, int argc, py_Ref argv); -bool pk_arrayiter(py_Ref val); bool pk_arraycontains(py_Ref self, py_Ref val); bool pk_loadmethod(py_StackRef self, py_Name name); @@ -139,7 +137,8 @@ py_Type pk_dict__register(); py_Type pk_dict_items__register(); py_Type pk_list__register(); py_Type pk_tuple__register(); -py_Type pk_array_iterator__register(); +py_Type pk_list_iterator__register(); +py_Type pk_tuple_iterator__register(); py_Type pk_slice__register(); py_Type pk_function__register(); py_Type pk_nativefunc__register(); diff --git a/include/pocketpy/objects/base.h b/include/pocketpy/objects/base.h index bc656f5e..f041b178 100644 --- a/include/pocketpy/objects/base.h +++ b/include/pocketpy/objects/base.h @@ -19,8 +19,10 @@ typedef struct py_TValue { PyObject* _obj; c11_vec2 _vec2; c11_vec2i _vec2i; + c11_vec3 _vec3; + c11_vec3i _vec3i; c11_color32 _color32; void* _ptr; - char _chars[8]; + char _chars[16]; }; } py_TValue; diff --git a/include/pocketpy/objects/iterator.h b/include/pocketpy/objects/iterator.h new file mode 100644 index 00000000..c80be597 --- /dev/null +++ b/include/pocketpy/objects/iterator.h @@ -0,0 +1,15 @@ +#pragma once + +#include "pocketpy/objects/base.h" +#include "pocketpy/common/vector.h" + +typedef struct tuple_iterator { + py_TValue* p; + int length; + int index; +} tuple_iterator; + +typedef struct list_iterator { + c11_vector* vec; + int index; +} list_iterator; diff --git a/include/pocketpy/pocketpy.h b/include/pocketpy/pocketpy.h index 33e5f5db..cb1c7858 100644 --- a/include/pocketpy/pocketpy.h +++ b/include/pocketpy/pocketpy.h @@ -30,7 +30,14 @@ typedef void (*py_Dtor)(void*); #ifdef PK_IS_PUBLIC_INCLUDE typedef struct py_TValue { - int64_t _[2]; + py_Type type; + bool is_ptr; + int extra; + + union { + int64_t _i64; + char _chars[16]; + }; } py_TValue; #endif @@ -70,8 +77,10 @@ typedef void (*py_TraceFunc)(py_Frame* frame, enum py_TraceEvent); /// A struct contains the callbacks of the VM. typedef struct py_Callbacks { - /// Used by `__import__` to load source code of a module. + /// Used by `__import__` to load a source module. char* (*importfile)(const char*); + /// Called before `importfile` to lazy-import a C module. + py_GlobalRef (*lazyimport)(const char*); /// Used by `print` to output a string. void (*print)(const char*); /// Flush the output buffer of `print`. @@ -124,6 +133,13 @@ PK_API int py_gc_collect(); /// Setup the callbacks for the current VM. PK_API py_Callbacks* py_callbacks(); +/// Wrapper for `PK_MALLOC(size)`. +PK_API void* py_malloc(size_t size); +/// Wrapper for `PK_REALLOC(ptr, size)`. +PK_API void* py_realloc(void* ptr, size_t size); +/// Wrapper for `PK_FREE(ptr)`. +PK_API void py_free(void* ptr); + /// Begin the watchdog with `timeout` in milliseconds. /// `PK_ENABLE_WATCHDOG` must be defined to `1` to use this feature. /// You need to call `py_watchdog_end()` later. @@ -201,6 +217,8 @@ PK_API py_GlobalRef py_NIL(); /// Create an `int` object. PK_API void py_newint(py_OutRef, py_i64); +/// Create a trivial value object. +PK_API void py_newtrivial(py_OutRef out, py_Type type, void* data, int size); /// Create a `float` object. PK_API void py_newfloat(py_OutRef, py_f64); /// Create a `bool` object. @@ -282,6 +300,8 @@ PK_API void* py_newobject(py_OutRef out, py_Type type, int slots, int udsize); /// Convert an `int` object in python to `int64_t`. PK_API py_i64 py_toint(py_Ref); +/// Get the address of the trivial value object. +PK_API void* py_totrivial(py_Ref); /// Convert a `float` object in python to `double`. PK_API py_f64 py_tofloat(py_Ref); /// Cast a `int` or `float` object in python to `double`. @@ -350,6 +370,15 @@ PK_API py_GlobalRef py_tpobject(py_Type type); PK_API const char* py_tpname(py_Type type); /// Call a type to create a new instance. PK_API bool py_tpcall(py_Type type, int argc, py_Ref argv) PY_RAISE PY_RETURN; +/// Disable the type for subclassing. +PK_API void py_tpsetfinal(py_Type type); +/// Set attribute hooks for the given type. +PK_API void py_tphookattributes(py_Type type, + bool (*getattribute)(py_Ref self, py_Name name) PY_RAISE PY_RETURN, + bool (*setattribute)(py_Ref self, py_Name name, py_Ref val) + PY_RAISE PY_RETURN, + bool (*delattribute)(py_Ref self, py_Name name) PY_RAISE, + bool (*getunboundmethod)(py_Ref self, py_Name name) PY_RETURN); /// Check if the object is an instance of the given type exactly. /// Raise `TypeError` if the check fails. @@ -389,6 +418,7 @@ PK_API void py_setglobal(py_Name name, py_Ref val); PK_API py_ItemRef py_getbuiltin(py_Name name); /// Get the last return value. +/// Please note that `py_retval()` cannot be used as input argument. PK_API py_GlobalRef py_retval(); /// Get an item from the object's `__dict__`. @@ -715,19 +745,33 @@ PK_API bool /// noexcept PK_API int py_dict_len(py_Ref self); +/************* random module *************/ +PK_API void py_newRandom(py_OutRef out); +PK_API void py_Random_seed(py_Ref self, py_i64 seed); +PK_API py_f64 py_Random_random(py_Ref self); +PK_API py_f64 py_Random_uniform(py_Ref self, py_f64 a, py_f64 b); +PK_API py_i64 py_Random_randint(py_Ref self, py_i64 a, py_i64 b); + +/************* array2d module *************/ +PK_API void py_newarray2d(py_OutRef out, int width, int height); +PK_API int py_array2d_getwidth(py_Ref self); +PK_API int py_array2d_getheight(py_Ref self); +PK_API py_ObjectRef py_array2d_getitem(py_Ref self, int x, int y); +PK_API void py_array2d_setitem(py_Ref self, int x, int y, py_Ref val); + /************* vmath module *************/ -void py_newvec2(py_OutRef out, c11_vec2); -void py_newvec3(py_OutRef out, c11_vec3); -void py_newvec2i(py_OutRef out, c11_vec2i); -void py_newvec3i(py_OutRef out, c11_vec3i); -void py_newcolor32(py_OutRef out, c11_color32); -c11_mat3x3* py_newmat3x3(py_OutRef out); -c11_vec2 py_tovec2(py_Ref self); -c11_vec3 py_tovec3(py_Ref self); -c11_vec2i py_tovec2i(py_Ref self); -c11_vec3i py_tovec3i(py_Ref self); -c11_mat3x3* py_tomat3x3(py_Ref self); -c11_color32 py_tocolor32(py_Ref self); +PK_API void py_newvec2(py_OutRef out, c11_vec2); +PK_API void py_newvec3(py_OutRef out, c11_vec3); +PK_API void py_newvec2i(py_OutRef out, c11_vec2i); +PK_API void py_newvec3i(py_OutRef out, c11_vec3i); +PK_API void py_newcolor32(py_OutRef out, c11_color32); +PK_API c11_mat3x3* py_newmat3x3(py_OutRef out); +PK_API c11_vec2 py_tovec2(py_Ref self); +PK_API c11_vec3 py_tovec3(py_Ref self); +PK_API c11_vec2i py_tovec2i(py_Ref self); +PK_API c11_vec3i py_tovec3i(py_Ref self); +PK_API c11_mat3x3* py_tomat3x3(py_Ref self); +PK_API c11_color32 py_tocolor32(py_Ref self); /************* Others *************/ @@ -755,10 +799,11 @@ enum py_PredefinedType { tp_bool, tp_str, tp_str_iterator, - tp_list, // c11_vector - tp_tuple, // N slots - tp_array_iterator, - tp_slice, // 3 slots (start, stop, step) + tp_list, // c11_vector + tp_tuple, // N slots + tp_list_iterator, // 1 slot + tp_tuple_iterator, // 1 slot + tp_slice, // 3 slots (start, stop, step) tp_range, tp_range_iterator, tp_module, diff --git a/include/pocketpy/xmacros/fixedhash.h b/include/pocketpy/xmacros/fixedhash.h index 7713e495..866d0a97 100644 --- a/include/pocketpy/xmacros/fixedhash.h +++ b/include/pocketpy/xmacros/fixedhash.h @@ -124,3 +124,4 @@ bool METHOD(contains)(NAME* self, K key) { #undef less #undef partial_less #undef equal +#undef hash diff --git a/include/pybind11/internal/error.h b/include/pybind11/internal/error.h index 3aa9ad66..086fda0e 100644 --- a/include/pybind11/internal/error.h +++ b/include/pybind11/internal/error.h @@ -14,7 +14,7 @@ public: // get the python exception object object& exception() { return m_exception; } - ~python_error() { std::free(m_what); } + ~python_error() { py_free(m_what); } bool match(py_Type type) const { return py_isinstance(m_exception.ptr(), type); } diff --git a/include/typings/cute_png.pyi b/include/typings/cute_png.pyi new file mode 100644 index 00000000..1ed40734 --- /dev/null +++ b/include/typings/cute_png.pyi @@ -0,0 +1,5 @@ +from array2d import array2d +from vmath import color32 + +def loads(data: bytes) -> array2d[color32]: ... +def dumps(image: array2d[color32]) -> bytes: ... diff --git a/include/typings/vmath.pyi b/include/typings/vmath.pyi index 73df8992..f2c21fc6 100644 --- a/include/typings/vmath.pyi +++ b/include/typings/vmath.pyi @@ -186,6 +186,7 @@ class color32: def __eq__(self, other: object) -> bool: ... def __ne__(self, other: object) -> bool: ... def __repr__(self) -> str: ... + def __hash__(self) -> int: ... @property def r(self) -> int: ... diff --git a/src/common/algorithm.c b/src/common/algorithm.c index b02e9920..1b165868 100644 --- a/src/common/algorithm.c +++ b/src/common/algorithm.c @@ -1,7 +1,6 @@ #include "pocketpy/common/algorithm.h" #include "pocketpy/config.h" #include -#include static bool _stable_sort_merge(char* a, char* a_end, diff --git a/src/common/chunkedvector.c b/src/common/chunkedvector.c index e6eedc0e..3f06ba7d 100644 --- a/src/common/chunkedvector.c +++ b/src/common/chunkedvector.c @@ -1,15 +1,12 @@ #include "pocketpy/common/chunkedvector.h" - -#include -#include -#include "pocketpy/common/utils.h" -#include "pocketpy/config.h" +#include "pocketpy/pocketpy.h" +#include #if defined(_MSC_VER) #include #endif -inline static int c11_bit_length(unsigned long x) { +PK_INLINE static int c11__bit_length(unsigned long x) { #if(defined(__clang__) || defined(__GNUC__)) return x == 0 ? 0 : (int)sizeof(unsigned long) * 8 - __builtin_clzl(x); #elif defined(_MSC_VER) @@ -39,7 +36,6 @@ void c11_chunkedvector__ctor(c11_chunkedvector* self, int elem_size, int initial self->initial_chunks = initial_chunks; void* chunks_data = PK_MALLOC(elem_size * ((1U << (unsigned int)initial_chunks) - 1)); for(int i = 0; i < initial_chunks; i++) { - // TODO: optimize? c11_chunkedvector_chunk chunk = {.length = 0, .capacity = 1U << i, .data = (char*)chunks_data + elem_size * ((1U << i) - 1U)}; @@ -59,10 +55,11 @@ void c11_chunkedvector__dtor(c11_chunkedvector* self) { void* c11_chunkedvector__emplace(c11_chunkedvector* self) { if(self->length == self->capacity) { - #ifndef NDEBUG - c11_chunkedvector_chunk last_chunk = c11_vector__back(c11_chunkedvector_chunk, &self->chunks); - assert(last_chunk.capacity == last_chunk.length); - #endif +#ifndef NDEBUG + c11_chunkedvector_chunk last_chunk = + c11_vector__back(c11_chunkedvector_chunk, &self->chunks); + assert(last_chunk.capacity == last_chunk.length); +#endif c11_chunkedvector_chunk chunk = { .length = 0, .capacity = 1U << (unsigned int)self->chunks.length, @@ -70,9 +67,14 @@ void* c11_chunkedvector__emplace(c11_chunkedvector* self) { self->capacity += chunk.capacity; c11_vector__push(c11_chunkedvector_chunk, &self->chunks, chunk); } - int last_chunk_index = c11_bit_length(self->length + 1) - 1; +#if 1 + int last_chunk_index = c11__bit_length(self->length + 1) - 1; c11_chunkedvector_chunk* last_chunk = c11__at(c11_chunkedvector_chunk, &self->chunks, last_chunk_index); +#else + // This is not correct, because there is some pre-allocated chunks + c11_chunkedvector_chunk* last_chunk = &c11_vector__back(c11_chunkedvector_chunk, &self->chunks); +#endif void* p = (char*)last_chunk->data + self->elem_size * last_chunk->length; last_chunk->length++; self->length++; @@ -80,7 +82,7 @@ void* c11_chunkedvector__emplace(c11_chunkedvector* self) { } void* c11_chunkedvector__at(c11_chunkedvector* self, int index) { - int chunk_index = c11_bit_length(index + 1) - 1; + int chunk_index = c11__bit_length(index + 1) - 1; c11_chunkedvector_chunk* chunk = c11__at(c11_chunkedvector_chunk, &self->chunks, chunk_index); return (char*)chunk->data + (index + 1 - (1U << (unsigned int)chunk_index)) * self->elem_size; } diff --git a/src/common/memorypool.c b/src/common/memorypool.c index b0dd9bed..70b6960d 100644 --- a/src/common/memorypool.c +++ b/src/common/memorypool.c @@ -1,7 +1,6 @@ #include "pocketpy/common/memorypool.h" -#include "pocketpy/pocketpy.h" +#include "pocketpy/config.h" -#include #include void FixedMemoryPool__ctor(FixedMemoryPool* self, int BlockSize, int BlockCount) { diff --git a/src/common/name.c b/src/common/name.c index d10f1597..46d0a863 100644 --- a/src/common/name.c +++ b/src/common/name.c @@ -2,9 +2,9 @@ #include "pocketpy/common/name.h" #include "pocketpy/common/str.h" -#include "pocketpy/common/threads.h" #include "pocketpy/pocketpy.h" -#include +#include "pocketpy/common/threads.h" +#include typedef struct NameBucket NameBucket; @@ -17,7 +17,9 @@ typedef struct NameBucket { static struct { NameBucket* table[0x10000]; +#if PK_ENABLE_THREADS atomic_flag lock; +#endif } pk_string_table; #define MAGIC_METHOD(x) py_Name x; @@ -43,9 +45,11 @@ void pk_names_finalize() { } py_Name py_namev(c11_sv name) { +#if PK_ENABLE_THREADS while(atomic_flag_test_and_set(&pk_string_table.lock)) { c11_thrd_yield(); } +#endif uint64_t hash = c11_sv__hash(name); int index = hash & 0xFFFF; NameBucket* p = pk_string_table.table[index]; @@ -61,7 +65,9 @@ py_Name py_namev(c11_sv name) { p = p->next; } if(found) { +#if PK_ENABLE_THREADS atomic_flag_clear(&pk_string_table.lock); +#endif return (py_Name)p; } @@ -78,7 +84,9 @@ py_Name py_namev(c11_sv name) { assert(prev->next == NULL); prev->next = bucket; } +#if PK_ENABLE_THREADS atomic_flag_clear(&pk_string_table.lock); +#endif return (py_Name)bucket; } diff --git a/src/common/socket.c b/src/common/socket.c index 17ebcbdc..8f1ac18d 100644 --- a/src/common/socket.c +++ b/src/common/socket.c @@ -1,5 +1,7 @@ #include "pocketpy/common/socket.h" +#if PK_ENABLE_OS + #include #if defined (_WIN32) || defined (_WIN64) @@ -120,3 +122,5 @@ int c11_socket_get_last_error(){ #undef SOCKET_HANDLERTOFD #undef SOCKET_FDTOHANDLER + +#endif // PK_ENABLE_OS \ No newline at end of file diff --git a/src/common/sourcedata.c b/src/common/sourcedata.c index fee47d74..c3a6c1eb 100644 --- a/src/common/sourcedata.c +++ b/src/common/sourcedata.c @@ -1,7 +1,6 @@ #include "pocketpy/objects/sourcedata.h" #include "pocketpy/common/sstream.h" #include -#include #include static void SourceData__ctor(struct SourceData* self, @@ -25,6 +24,16 @@ static void SourceData__ctor(struct SourceData* self, source++; } self->source = c11_sbuf__submit(&ss); + // remove trailing newline + int last_index = self->source->size - 1; + while(last_index >= 0 && isspace(self->source->data[last_index])) { + last_index--; + } + if(last_index >= 0) { + self->source->size = last_index + 1; + self->source->data[last_index + 1] = '\0'; + } + self->is_dynamic = is_dynamic; c11_vector__push(const char*, &self->line_starts, self->source->data); } diff --git a/src/common/sstream.c b/src/common/sstream.c index 69b30876..b9cda467 100644 --- a/src/common/sstream.c +++ b/src/common/sstream.c @@ -4,8 +4,8 @@ #include "pocketpy/pocketpy.h" #include -#include #include +#include #include #include diff --git a/src/common/str.c b/src/common/str.c index 0982c455..642883fc 100644 --- a/src/common/str.c +++ b/src/common/str.c @@ -2,11 +2,10 @@ #include "pocketpy/common/sstream.h" #include "pocketpy/common/utils.h" -#include -#include -#include #include +#include #include +#include c11_string* c11_string__new(const char* data) { return c11_string__new2(data, strlen(data)); } @@ -190,6 +189,26 @@ uint64_t c11_sv__hash(c11_sv self) { return hash; } +c11_vector /* T=c11_sv */ c11_sv__splitwhitespace(c11_sv self) { + c11_vector retval; + c11_vector__ctor(&retval, sizeof(c11_sv)); + const char* data = self.data; + int i = 0; + for(int j = 0; j < self.size; j++) { + if(isspace(data[j])) { + assert(j >= i); + c11_sv tmp = {data + i, j - i}; + c11_vector__push(c11_sv, &retval, tmp); + i = j + 1; + } + } + if(i <= self.size) { + c11_sv tmp = {data + i, self.size - i}; + c11_vector__push(c11_sv, &retval, tmp); + } + return retval; +} + c11_vector /* T=c11_sv */ c11_sv__split(c11_sv self, char sep) { c11_vector retval; c11_vector__ctor(&retval, sizeof(c11_sv)); diff --git a/src/common/threads.c b/src/common/threads.c index a9414a97..a2bb62cc 100644 --- a/src/common/threads.c +++ b/src/common/threads.c @@ -1,6 +1,7 @@ -#include "pocketpy/export.h" #include "pocketpy/common/threads.h" +#if PK_ENABLE_THREADS + #if PK_USE_PTHREADS bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg) { @@ -19,4 +20,6 @@ bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* a void c11_thrd_yield() { thrd_yield(); } -#endif \ No newline at end of file +#endif + +#endif // PK_ENABLE_THREADS \ No newline at end of file diff --git a/src/common/vector.c b/src/common/vector.c index 32b97fe4..b1a152f6 100644 --- a/src/common/vector.c +++ b/src/common/vector.c @@ -1,6 +1,5 @@ #include "pocketpy/common/vector.h" -#include #include #include "pocketpy/common/utils.h" #include "pocketpy/config.h" @@ -40,7 +39,7 @@ void c11_vector__reserve(c11_vector* self, int capacity) { void c11_vector__clear(c11_vector* self) { self->length = 0; } void* c11_vector__emplace(c11_vector* self) { - if(self->length == self->capacity) c11_vector__reserve(self, self->capacity * 2); + if(self->length == self->capacity) { c11_vector__reserve(self, c11_vector__nextcap(self)); } void* p = (char*)self->data + (size_t)self->elem_size * (size_t)self->length; self->length++; return p; @@ -63,8 +62,17 @@ void* c11_vector__submit(c11_vector* self, int* length) { return retval; } -void c11_vector__swap(c11_vector *self, c11_vector *other){ +void c11_vector__swap(c11_vector* self, c11_vector* other) { c11_vector tmp = *self; *self = *other; *other = tmp; } + +int c11_vector__nextcap(c11_vector* self) { + if(self->capacity < 1024) { + return self->capacity * 2; + } else { + // increase by 25% + return self->capacity + (self->capacity >> 2); + } +} \ No newline at end of file diff --git a/src/compiler/compiler.c b/src/compiler/compiler.c index b3935b80..c9fc95ea 100644 --- a/src/compiler/compiler.c +++ b/src/compiler/compiler.c @@ -28,6 +28,7 @@ typedef struct ExprVt { bool is_subscr; // SubscrExpr bool is_starred; // StarredExpr bool is_binary; // BinaryExpr + bool is_ternary; // TernaryExpr void (*dtor)(Expr*); } ExprVt; @@ -873,7 +874,11 @@ void TernaryExpr__emit_(Expr* self_, Ctx* ctx) { } TernaryExpr* TernaryExpr__new(int line) { - const static ExprVt Vt = {.dtor = TernaryExpr__dtor, .emit_ = TernaryExpr__emit_}; + const static ExprVt Vt = { + .dtor = TernaryExpr__dtor, + .emit_ = TernaryExpr__emit_, + .is_ternary = true, + }; TernaryExpr* self = PK_MALLOC(sizeof(TernaryExpr)); self->vt = &Vt; self->line = line; @@ -1681,6 +1686,10 @@ static Error* exprTernary(Compiler* self) { e->cond = Ctx__s_popx(ctx()); e->true_expr = Ctx__s_popx(ctx()); Ctx__s_push(ctx(), (Expr*)e); + + if(e->cond->vt->is_ternary || e->false_expr->vt->is_ternary || e->true_expr->vt->is_ternary) { + return SyntaxError(self, "nested ternary expressions without `()` are ambiguous"); + } return NULL; } @@ -2856,13 +2865,14 @@ static Error* compile_stmt(Compiler* self) { bool is_typed_name = false; // e.g. x: int // eat variable's type hint if it is a single name - if(Ctx__s_top(ctx())->vt->is_name) { + const ExprVt* top_vt = Ctx__s_top(ctx())->vt; + if(top_vt->is_name || top_vt->is_attrib) { if(match(TK_COLON)) { c11_sv type_hint; check(consume_type_hints_sv(self, &type_hint)); is_typed_name = true; - if(ctx()->is_compiling_class) { + if(ctx()->is_compiling_class && top_vt->is_name) { NameExpr* ne = (NameExpr*)Ctx__s_top(ctx()); int index = Ctx__add_const_string(ctx(), type_hint); Ctx__emit_(ctx(), OP_LOAD_CONST, index, BC_KEEPLINE); diff --git a/src/interpreter/ceval.c b/src/interpreter/ceval.c index 7c6ece74..4427de9a 100644 --- a/src/interpreter/ceval.c +++ b/src/interpreter/ceval.c @@ -7,6 +7,7 @@ #include "pocketpy/pocketpy.h" #include "pocketpy/objects/error.h" #include +#include #include static bool stack_format_object(VM* self, c11_sv spec); @@ -30,6 +31,12 @@ static bool stack_format_object(VM* self, c11_sv spec); goto __NEXT_STEP; \ } while(0) +#define RESET_CO_CACHE() \ + do { \ + co_codes = frame->co->codes.data; \ + co_names = frame->co->names.data; \ + } while(0) + /* Stack manipulation macros */ // https://github.com/python/cpython/blob/3.9/Python/ceval.c#L1123 #define TOP() (self->stack.sp - 1) @@ -82,526 +89,523 @@ FrameResult VM__run_top_frame(VM* self) { py_Frame* frame = self->top_frame; Bytecode* co_codes; py_Name* co_names; + Bytecode byte; const py_Frame* base_frame = frame; - while(true) { - Bytecode byte; - __NEXT_FRAME: - if(self->recursion_depth >= self->max_recursion_depth) { - py_exception(tp_RecursionError, "maximum recursion depth exceeded"); - goto __ERROR; - } - // NOTE: remember to change another occurrence after __ERROR_RE_RAISE: - co_codes = frame->co->codes.data; - co_names = frame->co->names.data; - frame->ip++; +__NEXT_FRAME: + if(self->recursion_depth >= self->max_recursion_depth) { + py_exception(tp_RecursionError, "maximum recursion depth exceeded"); + goto __ERROR; + } + RESET_CO_CACHE(); + frame->ip++; - __NEXT_STEP: - byte = co_codes[frame->ip]; +__NEXT_STEP: + byte = co_codes[frame->ip]; - if(self->trace_info.func) { - SourceLocation loc = Frame__source_location(frame); - SourceLocation prev_loc = self->trace_info.prev_loc; - if(loc.lineno != prev_loc.lineno || loc.src != prev_loc.src) { - if(prev_loc.src) PK_DECREF(prev_loc.src); - PK_INCREF(loc.src); - self->trace_info.prev_loc = loc; - self->trace_info.func(frame, TRACE_EVENT_LINE); - } + if(self->trace_info.func) { + SourceLocation loc = Frame__source_location(frame); + SourceLocation prev_loc = self->trace_info.prev_loc; + if(loc.lineno != prev_loc.lineno || loc.src != prev_loc.src) { + if(prev_loc.src) PK_DECREF(prev_loc.src); + PK_INCREF(loc.src); + self->trace_info.prev_loc = loc; + self->trace_info.func(frame, TRACE_EVENT_LINE); } + } #if PK_ENABLE_WATCHDOG - if(self->watchdog_info.max_reset_time > 0) { - clock_t now = clock(); - if(now > self->watchdog_info.max_reset_time) { - self->watchdog_info.max_reset_time = 0; - TimeoutError("watchdog timeout"); - goto __ERROR; - } + if(self->watchdog_info.max_reset_time > 0) { + clock_t now = clock(); + if(now > self->watchdog_info.max_reset_time) { + self->watchdog_info.max_reset_time = 0; + TimeoutError("watchdog timeout"); + goto __ERROR; } + } #endif #ifndef NDEBUG - pk_print_stack(self, frame, byte); + pk_print_stack(self, frame, byte); #endif - switch((Opcode)byte.op) { - case OP_NO_OP: DISPATCH(); - /*****************************************/ - case OP_POP_TOP: POP(); DISPATCH(); - case OP_DUP_TOP: PUSH(TOP()); DISPATCH(); - case OP_DUP_TOP_TWO: - // [a, b] - PUSH(SECOND()); // [a, b, a] - PUSH(SECOND()); // [a, b, a, b] - DISPATCH(); - case OP_ROT_TWO: { - py_TValue tmp = *TOP(); - *TOP() = *SECOND(); - *SECOND() = tmp; - DISPATCH(); + switch((Opcode)byte.op) { + case OP_NO_OP: DISPATCH(); + /*****************************************/ + case OP_POP_TOP: POP(); DISPATCH(); + case OP_DUP_TOP: PUSH(TOP()); DISPATCH(); + case OP_DUP_TOP_TWO: + // [a, b] + PUSH(SECOND()); // [a, b, a] + PUSH(SECOND()); // [a, b, a, b] + DISPATCH(); + case OP_ROT_TWO: { + py_TValue tmp = *TOP(); + *TOP() = *SECOND(); + *SECOND() = tmp; + DISPATCH(); + } + case OP_ROT_THREE: { + // [a, b, c] -> [c, a, b] + py_TValue tmp = *TOP(); + *TOP() = *SECOND(); + *SECOND() = *THIRD(); + *THIRD() = tmp; + DISPATCH(); + } + case OP_PRINT_EXPR: + if(TOP()->type != tp_NoneType) { + bool ok = py_repr(TOP()); + if(!ok) goto __ERROR; + self->callbacks.print(py_tostr(&self->last_retval)); + self->callbacks.print("\n"); } - case OP_ROT_THREE: { - // [a, b, c] -> [c, a, b] - py_TValue tmp = *TOP(); - *TOP() = *SECOND(); - *SECOND() = *THIRD(); - *THIRD() = tmp; - DISPATCH(); - } - case OP_PRINT_EXPR: - if(TOP()->type != tp_NoneType) { - bool ok = py_repr(TOP()); - if(!ok) goto __ERROR; - self->callbacks.print(py_tostr(&self->last_retval)); - self->callbacks.print("\n"); + POP(); + DISPATCH(); + /*****************************************/ + case OP_LOAD_CONST: { + PUSH(c11__at(py_TValue, &frame->co->consts, byte.arg)); + DISPATCH(); + } + case OP_LOAD_NONE: { + py_newnone(SP()++); + DISPATCH(); + } + case OP_LOAD_TRUE: { + py_newbool(SP()++, true); + DISPATCH(); + } + case OP_LOAD_FALSE: { + py_newbool(SP()++, false); + DISPATCH(); + } + /*****************************************/ + case OP_LOAD_SMALL_INT: { + py_newint(SP()++, (int16_t)byte.arg); + DISPATCH(); + } + /*****************************************/ + case OP_LOAD_ELLIPSIS: { + py_newellipsis(SP()++); + DISPATCH(); + } + case OP_LOAD_FUNCTION: { + FuncDecl_ decl = c11__getitem(FuncDecl_, &frame->co->func_decls, byte.arg); + Function* ud = py_newobject(SP(), tp_function, 0, sizeof(Function)); + Function__ctor(ud, decl, frame->module, frame->globals); + if(decl->nested) { + if(frame->is_locals_special) { + RuntimeError("cannot create closure from special locals"); + goto __ERROR; } - POP(); - DISPATCH(); + ud->closure = FastLocals__to_namedict(frame->locals, frame->co); + py_Name name = py_name(decl->code.name->data); + // capture itself to allow recursion + NameDict__set(ud->closure, name, SP()); + } + SP()++; + DISPATCH(); + } + case OP_LOAD_NULL: + py_newnil(SP()++); + DISPATCH(); /*****************************************/ - case OP_LOAD_CONST: { - PUSH(c11__at(py_TValue, &frame->co->consts, byte.arg)); + case OP_LOAD_FAST: { + assert(!frame->is_locals_special); + py_Ref val = &frame->locals[byte.arg]; + if(!py_isnil(val)) { + PUSH(val); DISPATCH(); } - case OP_LOAD_NONE: { - py_newnone(SP()++); - DISPATCH(); - } - case OP_LOAD_TRUE: { - py_newbool(SP()++, true); - DISPATCH(); - } - case OP_LOAD_FALSE: { - py_newbool(SP()++, false); - DISPATCH(); - } - /*****************************************/ - case OP_LOAD_SMALL_INT: { - py_newint(SP()++, (int16_t)byte.arg); - DISPATCH(); - } - /*****************************************/ - case OP_LOAD_ELLIPSIS: { - py_newellipsis(SP()++); - DISPATCH(); - } - case OP_LOAD_FUNCTION: { - FuncDecl_ decl = c11__getitem(FuncDecl_, &frame->co->func_decls, byte.arg); - Function* ud = py_newobject(SP(), tp_function, 0, sizeof(Function)); - Function__ctor(ud, decl, frame->module, frame->globals); - if(decl->nested) { - if(frame->is_locals_special) { - RuntimeError("cannot create closure from special locals"); + py_Name name = c11__getitem(py_Name, &frame->co->varnames, byte.arg); + UnboundLocalError(name); + goto __ERROR; + } + case OP_LOAD_NAME: { + assert(frame->is_locals_special); + py_Name name = co_names[byte.arg]; + // locals + switch(frame->locals->type) { + case tp_locals: { + py_Frame* noproxy = frame->locals->_ptr; + py_Ref slot = Frame__getlocal_noproxy(noproxy, name); + if(slot == NULL) break; + if(py_isnil(slot)) { + UnboundLocalError(name); goto __ERROR; } - ud->closure = FastLocals__to_namedict(frame->locals, frame->co); - py_Name name = py_name(decl->code.name->data); - // capture itself to allow recursion - NameDict__set(ud->closure, name, SP()); - } - SP()++; - DISPATCH(); - } - case OP_LOAD_NULL: - py_newnil(SP()++); - DISPATCH(); - /*****************************************/ - case OP_LOAD_FAST: { - assert(!frame->is_locals_special); - py_Ref val = &frame->locals[byte.arg]; - if(!py_isnil(val)) { - PUSH(val); + PUSH(slot); DISPATCH(); } + case tp_dict: { + int res = py_dict_getitem(frame->locals, py_name2ref(name)); + if(res == 1) { + PUSH(&self->last_retval); + DISPATCH(); + } + if(res == 0) break; + assert(res == -1); + goto __ERROR; + } + case tp_nil: break; + default: c11__unreachable(); + } + // globals + int res = Frame__getglobal(frame, name); + if(res == 1) { + PUSH(&self->last_retval); + DISPATCH(); + } + if(res == -1) goto __ERROR; + // builtins + py_Ref tmp = py_getdict(self->builtins, name); + if(tmp != NULL) { + PUSH(tmp); + DISPATCH(); + } + NameError(name); + goto __ERROR; + } + case OP_LOAD_NONLOCAL: { + py_Name name = co_names[byte.arg]; + py_Ref tmp = Frame__getclosure(frame, name); + if(tmp != NULL) { + PUSH(tmp); + DISPATCH(); + } + int res = Frame__getglobal(frame, name); + if(res == 1) { + PUSH(&self->last_retval); + DISPATCH(); + } + if(res == -1) goto __ERROR; + + tmp = py_getdict(self->builtins, name); + if(tmp != NULL) { + PUSH(tmp); + DISPATCH(); + } + NameError(name); + goto __ERROR; + } + case OP_LOAD_GLOBAL: { + py_Name name = co_names[byte.arg]; + int res = Frame__getglobal(frame, name); + if(res == 1) { + PUSH(&self->last_retval); + DISPATCH(); + } + if(res == -1) goto __ERROR; + py_Ref tmp = py_getdict(self->builtins, name); + if(tmp != NULL) { + PUSH(tmp); + DISPATCH(); + } + NameError(name); + goto __ERROR; + } + case OP_LOAD_ATTR: { + py_Name name = co_names[byte.arg]; + if(py_getattr(TOP(), name)) { + py_assign(TOP(), py_retval()); + } else { + goto __ERROR; + } + DISPATCH(); + } + case OP_LOAD_CLASS_GLOBAL: { + assert(self->curr_class); + py_Name name = co_names[byte.arg]; + py_Ref tmp = py_getdict(self->curr_class, name); + if(tmp) { + PUSH(tmp); + DISPATCH(); + } + // load global if attribute not found + int res = Frame__getglobal(frame, name); + if(res == 1) { + PUSH(&self->last_retval); + DISPATCH(); + } + if(res == -1) goto __ERROR; + tmp = py_getdict(self->builtins, name); + if(tmp) { + PUSH(tmp); + DISPATCH(); + } + NameError(name); + goto __ERROR; + } + case OP_LOAD_METHOD: { + // [self] -> [unbound, self] + py_Name name = co_names[byte.arg]; + bool ok = py_pushmethod(name); + if(!ok) { + // fallback to getattr + if(py_getattr(TOP(), name)) { + py_assign(TOP(), py_retval()); + py_newnil(SP()++); + } else { + goto __ERROR; + } + } + DISPATCH(); + } + case OP_LOAD_SUBSCR: { + // [a, b] -> a[b] + py_Ref magic = py_tpfindmagic(SECOND()->type, __getitem__); + if(magic) { + if(magic->type == tp_nativefunc) { + if(!py_callcfunc(magic->_cfunc, 2, SECOND())) goto __ERROR; + POP(); + py_assign(TOP(), py_retval()); + } else { + INSERT_THIRD(); // [?, a, b] + *THIRD() = *magic; // [__getitem__, a, b] + vectorcall_opcall(1, 0); + } + DISPATCH(); + } + TypeError("'%t' object is not subscriptable", SECOND()->type); + goto __ERROR; + } + case OP_STORE_FAST: { + assert(!frame->is_locals_special); + frame->locals[byte.arg] = POPX(); + DISPATCH(); + } + case OP_STORE_NAME: { + assert(frame->is_locals_special); + py_Name name = co_names[byte.arg]; + switch(frame->locals->type) { + case tp_locals: { + py_Frame* noproxy = frame->locals->_ptr; + py_Ref slot = Frame__getlocal_noproxy(noproxy, name); + if(slot == NULL) { + UnboundLocalError(name); + goto __ERROR; + } + *slot = POPX(); + DISPATCH(); + } + case tp_dict: { + if(!py_dict_setitem(frame->locals, py_name2ref(name), TOP())) goto __ERROR; + POP(); + DISPATCH(); + } + case tp_nil: { + // globals + if(!Frame__setglobal(frame, name, TOP())) goto __ERROR; + POP(); + DISPATCH(); + } + default: c11__unreachable(); + } + } + case OP_STORE_GLOBAL: { + py_Name name = co_names[byte.arg]; + if(!Frame__setglobal(frame, name, TOP())) goto __ERROR; + POP(); + DISPATCH(); + } + case OP_STORE_ATTR: { + // [val, a] -> a.b = val + py_Name name = co_names[byte.arg]; + if(!py_setattr(TOP(), name, SECOND())) goto __ERROR; + STACK_SHRINK(2); + DISPATCH(); + } + case OP_STORE_SUBSCR: { + // [val, a, b] -> a[b] = val + py_Ref magic = py_tpfindmagic(SECOND()->type, __setitem__); + if(magic) { + PUSH(THIRD()); // [val, a, b, val] + if(magic->type == tp_nativefunc) { + if(!py_callcfunc(magic->_cfunc, 3, THIRD())) goto __ERROR; + STACK_SHRINK(4); + } else { + *FOURTH() = *magic; // [__setitem__, a, b, val] + if(!py_vectorcall(2, 0)) goto __ERROR; + } + DISPATCH(); + } + TypeError("'%t' object does not support item assignment", SECOND()->type); + goto __ERROR; + } + case OP_DELETE_FAST: { + assert(!frame->is_locals_special); + py_Ref tmp = &frame->locals[byte.arg]; + if(py_isnil(tmp)) { py_Name name = c11__getitem(py_Name, &frame->co->varnames, byte.arg); UnboundLocalError(name); goto __ERROR; } - case OP_LOAD_NAME: { - assert(frame->is_locals_special); - py_Name name = co_names[byte.arg]; - // locals - switch(frame->locals->type) { - case tp_locals: { - py_Frame* noproxy = frame->locals->_ptr; - py_Ref slot = Frame__getlocal_noproxy(noproxy, name); - if(slot == NULL) break; - if(py_isnil(slot)) { - UnboundLocalError(name); - goto __ERROR; - } - PUSH(slot); - DISPATCH(); - } - case tp_dict: { - int res = py_dict_getitem(frame->locals, py_name2ref(name)); - if(res == 1) { - PUSH(&self->last_retval); - DISPATCH(); - } - if(res == 0) break; - assert(res == -1); + py_newnil(tmp); + DISPATCH(); + } + case OP_DELETE_NAME: { + assert(frame->is_locals_special); + py_Name name = co_names[byte.arg]; + switch(frame->locals->type) { + case tp_locals: { + py_Frame* noproxy = frame->locals->_ptr; + py_Ref slot = Frame__getlocal_noproxy(noproxy, name); + if(slot == NULL || py_isnil(slot)) { + UnboundLocalError(name); goto __ERROR; } - case tp_nil: break; - default: c11__unreachable(); - } - // globals - int res = Frame__getglobal(frame, name); - if(res == 1) { - PUSH(&self->last_retval); + py_newnil(slot); DISPATCH(); } - if(res == -1) goto __ERROR; - // builtins - py_Ref tmp = py_getdict(self->builtins, name); - if(tmp != NULL) { - PUSH(tmp); - DISPATCH(); + case tp_dict: { + int res = py_dict_delitem(frame->locals, py_name2ref(name)); + if(res == 1) DISPATCH(); + if(res == 0) UnboundLocalError(name); + goto __ERROR; } - NameError(name); - goto __ERROR; + case tp_nil: { + // globals + int res = Frame__delglobal(frame, name); + if(res == 1) DISPATCH(); + if(res == 0) NameError(name); + goto __ERROR; + } + default: c11__unreachable(); } - case OP_LOAD_NONLOCAL: { - py_Name name = co_names[byte.arg]; - py_Ref tmp = Frame__getclosure(frame, name); - if(tmp != NULL) { - PUSH(tmp); - DISPATCH(); - } - int res = Frame__getglobal(frame, name); - if(res == 1) { - PUSH(&self->last_retval); - DISPATCH(); - } - if(res == -1) goto __ERROR; + } + case OP_DELETE_GLOBAL: { + py_Name name = co_names[byte.arg]; + int res = Frame__delglobal(frame, name); + if(res == 1) DISPATCH(); + if(res == -1) goto __ERROR; + NameError(name); + goto __ERROR; + } - tmp = py_getdict(self->builtins, name); - if(tmp != NULL) { - PUSH(tmp); - DISPATCH(); - } - NameError(name); - goto __ERROR; - } - case OP_LOAD_GLOBAL: { - py_Name name = co_names[byte.arg]; - int res = Frame__getglobal(frame, name); - if(res == 1) { - PUSH(&self->last_retval); - DISPATCH(); - } - if(res == -1) goto __ERROR; - py_Ref tmp = py_getdict(self->builtins, name); - if(tmp != NULL) { - PUSH(tmp); - DISPATCH(); - } - NameError(name); - goto __ERROR; - } - case OP_LOAD_ATTR: { - py_Name name = co_names[byte.arg]; - if(py_getattr(TOP(), name)) { - py_assign(TOP(), py_retval()); + case OP_DELETE_ATTR: { + py_Name name = co_names[byte.arg]; + if(!py_delattr(TOP(), name)) goto __ERROR; + DISPATCH(); + } + + case OP_DELETE_SUBSCR: { + // [a, b] -> del a[b] + py_Ref magic = py_tpfindmagic(SECOND()->type, __delitem__); + if(magic) { + if(magic->type == tp_nativefunc) { + if(!py_callcfunc(magic->_cfunc, 2, SECOND())) goto __ERROR; + STACK_SHRINK(2); } else { - goto __ERROR; - } - DISPATCH(); - } - case OP_LOAD_CLASS_GLOBAL: { - assert(self->curr_class); - py_Name name = co_names[byte.arg]; - py_Ref tmp = py_getdict(self->curr_class, name); - if(tmp) { - PUSH(tmp); - DISPATCH(); - } - // load global if attribute not found - int res = Frame__getglobal(frame, name); - if(res == 1) { - PUSH(&self->last_retval); - DISPATCH(); - } - if(res == -1) goto __ERROR; - tmp = py_getdict(self->builtins, name); - if(tmp) { - PUSH(tmp); - DISPATCH(); - } - NameError(name); - goto __ERROR; - } - case OP_LOAD_METHOD: { - // [self] -> [unbound, self] - py_Name name = co_names[byte.arg]; - bool ok = py_pushmethod(name); - if(!ok) { - // fallback to getattr - if(py_getattr(TOP(), name)) { - py_assign(TOP(), py_retval()); - py_newnil(SP()++); - } else { - goto __ERROR; - } - } - DISPATCH(); - } - case OP_LOAD_SUBSCR: { - // [a, b] -> a[b] - py_Ref magic = py_tpfindmagic(SECOND()->type, __getitem__); - if(magic) { - if(magic->type == tp_nativefunc) { - if(!py_callcfunc(magic->_cfunc, 2, SECOND())) goto __ERROR; - POP(); - py_assign(TOP(), py_retval()); - } else { - INSERT_THIRD(); // [?, a, b] - *THIRD() = *magic; // [__getitem__, a, b] - vectorcall_opcall(1, 0); - } - DISPATCH(); - } - TypeError("'%t' object is not subscriptable", SECOND()->type); - goto __ERROR; - } - case OP_STORE_FAST: { - assert(!frame->is_locals_special); - frame->locals[byte.arg] = POPX(); - DISPATCH(); - } - case OP_STORE_NAME: { - assert(frame->is_locals_special); - py_Name name = co_names[byte.arg]; - switch(frame->locals->type) { - case tp_locals: { - py_Frame* noproxy = frame->locals->_ptr; - py_Ref slot = Frame__getlocal_noproxy(noproxy, name); - if(slot == NULL) { - UnboundLocalError(name); - goto __ERROR; - } - *slot = POPX(); - DISPATCH(); - } - case tp_dict: { - if(!py_dict_setitem(frame->locals, py_name2ref(name), TOP())) goto __ERROR; - POP(); - DISPATCH(); - } - case tp_nil: { - // globals - if(!Frame__setglobal(frame, name, TOP())) goto __ERROR; - POP(); - DISPATCH(); - } - default: c11__unreachable(); - } - } - case OP_STORE_GLOBAL: { - py_Name name = co_names[byte.arg]; - if(!Frame__setglobal(frame, name, TOP())) goto __ERROR; - POP(); - DISPATCH(); - } - case OP_STORE_ATTR: { - // [val, a] -> a.b = val - py_Name name = co_names[byte.arg]; - if(!py_setattr(TOP(), name, SECOND())) goto __ERROR; - STACK_SHRINK(2); - DISPATCH(); - } - case OP_STORE_SUBSCR: { - // [val, a, b] -> a[b] = val - py_Ref magic = py_tpfindmagic(SECOND()->type, __setitem__); - if(magic) { - PUSH(THIRD()); // [val, a, b, val] - if(magic->type == tp_nativefunc) { - if(!py_callcfunc(magic->_cfunc, 3, THIRD())) goto __ERROR; - STACK_SHRINK(4); - } else { - *FOURTH() = *magic; // [__setitem__, a, b, val] - if(!py_vectorcall(2, 0)) goto __ERROR; - } - DISPATCH(); - } - TypeError("'%t' object does not support item assignment", SECOND()->type); - goto __ERROR; - } - case OP_DELETE_FAST: { - assert(!frame->is_locals_special); - py_Ref tmp = &frame->locals[byte.arg]; - if(py_isnil(tmp)) { - py_Name name = c11__getitem(py_Name, &frame->co->varnames, byte.arg); - UnboundLocalError(name); - goto __ERROR; - } - py_newnil(tmp); - DISPATCH(); - } - case OP_DELETE_NAME: { - assert(frame->is_locals_special); - py_Name name = co_names[byte.arg]; - switch(frame->locals->type) { - case tp_locals: { - py_Frame* noproxy = frame->locals->_ptr; - py_Ref slot = Frame__getlocal_noproxy(noproxy, name); - if(slot == NULL || py_isnil(slot)) { - UnboundLocalError(name); - goto __ERROR; - } - py_newnil(slot); - DISPATCH(); - } - case tp_dict: { - int res = py_dict_delitem(frame->locals, py_name2ref(name)); - if(res == 1) DISPATCH(); - if(res == 0) UnboundLocalError(name); - goto __ERROR; - } - case tp_nil: { - // globals - int res = Frame__delglobal(frame, name); - if(res == 1) DISPATCH(); - if(res == 0) NameError(name); - goto __ERROR; - } - default: c11__unreachable(); - } - } - case OP_DELETE_GLOBAL: { - py_Name name = co_names[byte.arg]; - int res = Frame__delglobal(frame, name); - if(res == 1) DISPATCH(); - if(res == -1) goto __ERROR; - NameError(name); - goto __ERROR; - } - - case OP_DELETE_ATTR: { - py_Name name = co_names[byte.arg]; - if(!py_delattr(TOP(), name)) goto __ERROR; - DISPATCH(); - } - - case OP_DELETE_SUBSCR: { - // [a, b] -> del a[b] - py_Ref magic = py_tpfindmagic(SECOND()->type, __delitem__); - if(magic) { - if(magic->type == tp_nativefunc) { - if(!py_callcfunc(magic->_cfunc, 2, SECOND())) goto __ERROR; - STACK_SHRINK(2); - } else { - INSERT_THIRD(); // [?, a, b] - *THIRD() = *magic; // [__delitem__, a, b] - if(!py_vectorcall(1, 0)) goto __ERROR; - } - DISPATCH(); - } - TypeError("'%t' object does not support item deletion", SECOND()->type); - goto __ERROR; - } - /*****************************************/ - case OP_BUILD_IMAG: { - // [x] - py_Ref f = py_getdict(self->builtins, py_name("complex")); - assert(f != NULL); - py_TValue tmp = *TOP(); - *TOP() = *f; // [complex] - py_newnil(SP()++); // [complex, NULL] - py_newint(SP()++, 0); // [complex, NULL, 0] - *SP()++ = tmp; // [complex, NULL, 0, x] - vectorcall_opcall(2, 0); - DISPATCH(); - } - case OP_BUILD_BYTES: { - int size; - py_Ref string = c11__at(py_TValue, &frame->co->consts, byte.arg); - const char* data = py_tostrn(string, &size); - unsigned char* p = py_newbytes(SP()++, size); - memcpy(p, data, size); - DISPATCH(); - } - case OP_BUILD_TUPLE: { - py_TValue tmp; - py_Ref p = py_newtuple(&tmp, byte.arg); - py_TValue* begin = SP() - byte.arg; - for(int i = 0; i < byte.arg; i++) - p[i] = begin[i]; - SP() = begin; - PUSH(&tmp); - DISPATCH(); - } - case OP_BUILD_LIST: { - py_TValue tmp; - py_newlistn(&tmp, byte.arg); - py_TValue* begin = SP() - byte.arg; - for(int i = 0; i < byte.arg; i++) { - py_list_setitem(&tmp, i, begin + i); - } - SP() = begin; - PUSH(&tmp); - DISPATCH(); - } - case OP_BUILD_DICT: { - py_TValue* begin = SP() - byte.arg * 2; - py_Ref tmp = py_pushtmp(); - py_newdict(tmp); - for(int i = 0; i < byte.arg * 2; i += 2) { - bool ok = py_dict_setitem(tmp, begin + i, begin + i + 1); - if(!ok) goto __ERROR; - } - SP() = begin; - PUSH(tmp); - DISPATCH(); - } - case OP_BUILD_SET: { - py_TValue* begin = SP() - byte.arg; - py_Ref typeobject_set = py_getdict(self->builtins, py_name("set")); - assert(typeobject_set != NULL); - py_push(typeobject_set); - py_pushnil(); - if(!py_vectorcall(0, 0)) goto __ERROR; - py_push(py_retval()); // empty set - py_Name id_add = py_name("add"); - for(int i = 0; i < byte.arg; i++) { - py_push(TOP()); - if(!py_pushmethod(id_add)) { - c11__abort("OP_BUILD_SET: failed to load method 'add'"); - } - py_push(begin + i); + INSERT_THIRD(); // [?, a, b] + *THIRD() = *magic; // [__delitem__, a, b] if(!py_vectorcall(1, 0)) goto __ERROR; } - py_TValue tmp = *TOP(); - SP() = begin; - PUSH(&tmp); DISPATCH(); } - case OP_BUILD_SLICE: { - // [start, stop, step] - py_TValue tmp; - py_newslice(&tmp); - py_setslot(&tmp, 0, THIRD()); - py_setslot(&tmp, 1, SECOND()); - py_setslot(&tmp, 2, TOP()); - STACK_SHRINK(3); - PUSH(&tmp); - DISPATCH(); + TypeError("'%t' object does not support item deletion", SECOND()->type); + goto __ERROR; + } + /*****************************************/ + case OP_BUILD_IMAG: { + // [x] + py_Ref f = py_getdict(self->builtins, py_name("complex")); + assert(f != NULL); + py_TValue tmp = *TOP(); + *TOP() = *f; // [complex] + py_newnil(SP()++); // [complex, NULL] + py_newint(SP()++, 0); // [complex, NULL, 0] + *SP()++ = tmp; // [complex, NULL, 0, x] + vectorcall_opcall(2, 0); + DISPATCH(); + } + case OP_BUILD_BYTES: { + int size; + py_Ref string = c11__at(py_TValue, &frame->co->consts, byte.arg); + const char* data = py_tostrn(string, &size); + unsigned char* p = py_newbytes(SP()++, size); + memcpy(p, data, size); + DISPATCH(); + } + case OP_BUILD_TUPLE: { + py_TValue tmp; + py_Ref p = py_newtuple(&tmp, byte.arg); + py_TValue* begin = SP() - byte.arg; + for(int i = 0; i < byte.arg; i++) + p[i] = begin[i]; + SP() = begin; + PUSH(&tmp); + DISPATCH(); + } + case OP_BUILD_LIST: { + py_TValue tmp; + py_newlistn(&tmp, byte.arg); + py_TValue* begin = SP() - byte.arg; + for(int i = 0; i < byte.arg; i++) { + py_list_setitem(&tmp, i, begin + i); } - case OP_BUILD_STRING: { - py_TValue* begin = SP() - byte.arg; - c11_sbuf ss; - c11_sbuf__ctor(&ss); - for(int i = 0; i < byte.arg; i++) { - if(!py_str(begin + i)) goto __ERROR; - c11_sbuf__write_sv(&ss, py_tosv(&self->last_retval)); + SP() = begin; + PUSH(&tmp); + DISPATCH(); + } + case OP_BUILD_DICT: { + py_TValue* begin = SP() - byte.arg * 2; + py_Ref tmp = py_pushtmp(); + py_newdict(tmp); + for(int i = 0; i < byte.arg * 2; i += 2) { + bool ok = py_dict_setitem(tmp, begin + i, begin + i + 1); + if(!ok) goto __ERROR; + } + SP() = begin; + PUSH(tmp); + DISPATCH(); + } + case OP_BUILD_SET: { + py_TValue* begin = SP() - byte.arg; + py_Ref typeobject_set = py_getdict(self->builtins, py_name("set")); + assert(typeobject_set != NULL); + py_push(typeobject_set); + py_pushnil(); + if(!py_vectorcall(0, 0)) goto __ERROR; + py_push(py_retval()); // empty set + py_Name id_add = py_name("add"); + for(int i = 0; i < byte.arg; i++) { + py_push(TOP()); + if(!py_pushmethod(id_add)) { + c11__abort("OP_BUILD_SET: failed to load method 'add'"); } - SP() = begin; - c11_sbuf__py_submit(&ss, SP()++); - DISPATCH(); + py_push(begin + i); + if(!py_vectorcall(1, 0)) goto __ERROR; } - /*****************************/ + py_TValue tmp = *TOP(); + SP() = begin; + PUSH(&tmp); + DISPATCH(); + } + case OP_BUILD_SLICE: { + // [start, stop, step] + py_TValue tmp; + py_newslice(&tmp); + py_setslot(&tmp, 0, THIRD()); + py_setslot(&tmp, 1, SECOND()); + py_setslot(&tmp, 2, TOP()); + STACK_SHRINK(3); + PUSH(&tmp); + DISPATCH(); + } + case OP_BUILD_STRING: { + py_TValue* begin = SP() - byte.arg; + c11_sbuf ss; + c11_sbuf__ctor(&ss); + for(int i = 0; i < byte.arg; i++) { + if(!py_str(begin + i)) goto __ERROR; + c11_sbuf__write_sv(&ss, py_tosv(&self->last_retval)); + } + SP() = begin; + c11_sbuf__py_submit(&ss, SP()++); + DISPATCH(); + } + /*****************************/ #define CASE_BINARY_OP(label, op, rop) \ case label: { \ if(!pk_stack_binaryop(self, op, rop)) goto __ERROR; \ @@ -609,643 +613,638 @@ FrameResult VM__run_top_frame(VM* self) { *TOP() = self->last_retval; \ DISPATCH(); \ } - CASE_BINARY_OP(OP_BINARY_ADD, __add__, __radd__) - CASE_BINARY_OP(OP_BINARY_SUB, __sub__, __rsub__) - CASE_BINARY_OP(OP_BINARY_MUL, __mul__, __rmul__) - CASE_BINARY_OP(OP_BINARY_TRUEDIV, __truediv__, __rtruediv__) - CASE_BINARY_OP(OP_BINARY_FLOORDIV, __floordiv__, __rfloordiv__) - CASE_BINARY_OP(OP_BINARY_MOD, __mod__, __rmod__) - CASE_BINARY_OP(OP_BINARY_POW, __pow__, __rpow__) - CASE_BINARY_OP(OP_BINARY_LSHIFT, __lshift__, 0) - CASE_BINARY_OP(OP_BINARY_RSHIFT, __rshift__, 0) - CASE_BINARY_OP(OP_BINARY_AND, __and__, 0) - CASE_BINARY_OP(OP_BINARY_OR, __or__, 0) - CASE_BINARY_OP(OP_BINARY_XOR, __xor__, 0) - CASE_BINARY_OP(OP_BINARY_MATMUL, __matmul__, 0) - CASE_BINARY_OP(OP_COMPARE_LT, __lt__, __gt__) - CASE_BINARY_OP(OP_COMPARE_LE, __le__, __ge__) - CASE_BINARY_OP(OP_COMPARE_EQ, __eq__, __eq__) - CASE_BINARY_OP(OP_COMPARE_NE, __ne__, __ne__) - CASE_BINARY_OP(OP_COMPARE_GT, __gt__, __lt__) - CASE_BINARY_OP(OP_COMPARE_GE, __ge__, __le__) + CASE_BINARY_OP(OP_BINARY_ADD, __add__, __radd__) + CASE_BINARY_OP(OP_BINARY_SUB, __sub__, __rsub__) + CASE_BINARY_OP(OP_BINARY_MUL, __mul__, __rmul__) + CASE_BINARY_OP(OP_BINARY_TRUEDIV, __truediv__, __rtruediv__) + CASE_BINARY_OP(OP_BINARY_FLOORDIV, __floordiv__, __rfloordiv__) + CASE_BINARY_OP(OP_BINARY_MOD, __mod__, __rmod__) + CASE_BINARY_OP(OP_BINARY_POW, __pow__, __rpow__) + CASE_BINARY_OP(OP_BINARY_LSHIFT, __lshift__, 0) + CASE_BINARY_OP(OP_BINARY_RSHIFT, __rshift__, 0) + CASE_BINARY_OP(OP_BINARY_AND, __and__, 0) + CASE_BINARY_OP(OP_BINARY_OR, __or__, 0) + CASE_BINARY_OP(OP_BINARY_XOR, __xor__, 0) + CASE_BINARY_OP(OP_BINARY_MATMUL, __matmul__, 0) + CASE_BINARY_OP(OP_COMPARE_LT, __lt__, __gt__) + CASE_BINARY_OP(OP_COMPARE_LE, __le__, __ge__) + CASE_BINARY_OP(OP_COMPARE_EQ, __eq__, __eq__) + CASE_BINARY_OP(OP_COMPARE_NE, __ne__, __ne__) + CASE_BINARY_OP(OP_COMPARE_GT, __gt__, __lt__) + CASE_BINARY_OP(OP_COMPARE_GE, __ge__, __le__) #undef CASE_BINARY_OP - case OP_IS_OP: { - bool res = py_isidentical(SECOND(), TOP()); - POP(); + case OP_IS_OP: { + bool res = py_isidentical(SECOND(), TOP()); + POP(); + if(byte.arg) res = !res; + py_newbool(TOP(), res); + DISPATCH(); + } + case OP_CONTAINS_OP: { + // [b, a] -> b __contains__ a (a in b) -> [retval] + py_Ref magic = py_tpfindmagic(SECOND()->type, __contains__); + if(magic) { + if(magic->type == tp_nativefunc) { + if(!py_callcfunc(magic->_cfunc, 2, SECOND())) goto __ERROR; + STACK_SHRINK(2); + } else { + INSERT_THIRD(); // [?, b, a] + *THIRD() = *magic; // [__contains__, a, b] + if(!py_vectorcall(1, 0)) goto __ERROR; + } + bool res = py_tobool(py_retval()); if(byte.arg) res = !res; - py_newbool(TOP(), res); + py_newbool(SP()++, res); DISPATCH(); } - case OP_CONTAINS_OP: { - // [b, a] -> b __contains__ a (a in b) -> [retval] - py_Ref magic = py_tpfindmagic(SECOND()->type, __contains__); - if(magic) { - if(magic->type == tp_nativefunc) { - if(!py_callcfunc(magic->_cfunc, 2, SECOND())) goto __ERROR; - STACK_SHRINK(2); - } else { - INSERT_THIRD(); // [?, b, a] - *THIRD() = *magic; // [__contains__, a, b] - if(!py_vectorcall(1, 0)) goto __ERROR; - } - bool res = py_tobool(py_retval()); - if(byte.arg) res = !res; - py_newbool(SP()++, res); - DISPATCH(); - } - TypeError("'%t' type does not support '__contains__'", SECOND()->type); - goto __ERROR; - } - /*****************************************/ - case OP_JUMP_FORWARD: DISPATCH_JUMP((int16_t)byte.arg); - case OP_POP_JUMP_IF_NOT_MATCH: { - int res = py_equal(SECOND(), TOP()); - if(res < 0) goto __ERROR; - STACK_SHRINK(2); - if(!res) DISPATCH_JUMP((int16_t)byte.arg); - DISPATCH(); - } - case OP_POP_JUMP_IF_FALSE: { - int res = py_bool(TOP()); - if(res < 0) goto __ERROR; - POP(); - if(!res) DISPATCH_JUMP((int16_t)byte.arg); - DISPATCH(); - } - case OP_POP_JUMP_IF_TRUE: { - int res = py_bool(TOP()); - if(res < 0) goto __ERROR; - POP(); - if(res) DISPATCH_JUMP((int16_t)byte.arg); - DISPATCH(); - } - case OP_JUMP_IF_TRUE_OR_POP: { - int res = py_bool(TOP()); - if(res < 0) goto __ERROR; - if(res) { - DISPATCH_JUMP((int16_t)byte.arg); - } else { - POP(); - DISPATCH(); - } - } - case OP_JUMP_IF_FALSE_OR_POP: { - int res = py_bool(TOP()); - if(res < 0) goto __ERROR; - if(!res) { - DISPATCH_JUMP((int16_t)byte.arg); - } else { - POP(); - DISPATCH(); - } - } - case OP_SHORTCUT_IF_FALSE_OR_POP: { - int res = py_bool(TOP()); - if(res < 0) goto __ERROR; - if(!res) { // [b, False] - STACK_SHRINK(2); // [] - py_newbool(SP()++, false); // [False] - DISPATCH_JUMP((int16_t)byte.arg); - } else { - POP(); // [b] - DISPATCH(); - } - } - case OP_LOOP_CONTINUE: { - DISPATCH_JUMP((int16_t)byte.arg); - } - case OP_LOOP_BREAK: { - DISPATCH_JUMP((int16_t)byte.arg); - } + TypeError("'%t' type does not support '__contains__'", SECOND()->type); + goto __ERROR; + } /*****************************************/ - case OP_CALL: { - ManagedHeap__collect_if_needed(&self->heap); - vectorcall_opcall(byte.arg & 0xFF, byte.arg >> 8); + case OP_JUMP_FORWARD: DISPATCH_JUMP((int16_t)byte.arg); + case OP_POP_JUMP_IF_NOT_MATCH: { + int res = py_equal(SECOND(), TOP()); + if(res < 0) goto __ERROR; + STACK_SHRINK(2); + if(!res) DISPATCH_JUMP((int16_t)byte.arg); + DISPATCH(); + } + case OP_POP_JUMP_IF_FALSE: { + int res = py_bool(TOP()); + if(res < 0) goto __ERROR; + POP(); + if(!res) DISPATCH_JUMP((int16_t)byte.arg); + DISPATCH(); + } + case OP_POP_JUMP_IF_TRUE: { + int res = py_bool(TOP()); + if(res < 0) goto __ERROR; + POP(); + if(res) DISPATCH_JUMP((int16_t)byte.arg); + DISPATCH(); + } + case OP_JUMP_IF_TRUE_OR_POP: { + int res = py_bool(TOP()); + if(res < 0) goto __ERROR; + if(res) { + DISPATCH_JUMP((int16_t)byte.arg); + } else { + POP(); DISPATCH(); } - case OP_CALL_VARGS: { - // [_0, _1, _2 | k1, v1, k2, v2] - uint16_t argc = byte.arg & 0xFF; - uint16_t kwargc = byte.arg >> 8; + } + case OP_JUMP_IF_FALSE_OR_POP: { + int res = py_bool(TOP()); + if(res < 0) goto __ERROR; + if(!res) { + DISPATCH_JUMP((int16_t)byte.arg); + } else { + POP(); + DISPATCH(); + } + } + case OP_SHORTCUT_IF_FALSE_OR_POP: { + int res = py_bool(TOP()); + if(res < 0) goto __ERROR; + if(!res) { // [b, False] + STACK_SHRINK(2); // [] + py_newbool(SP()++, false); // [False] + DISPATCH_JUMP((int16_t)byte.arg); + } else { + POP(); // [b] + DISPATCH(); + } + } + case OP_LOOP_CONTINUE: { + DISPATCH_JUMP((int16_t)byte.arg); + } + case OP_LOOP_BREAK: { + DISPATCH_JUMP((int16_t)byte.arg); + } + /*****************************************/ + case OP_CALL: { + ManagedHeap__collect_if_needed(&self->heap); + vectorcall_opcall(byte.arg & 0xFF, byte.arg >> 8); + DISPATCH(); + } + case OP_CALL_VARGS: { + // [_0, _1, _2 | k1, v1, k2, v2] + uint16_t argc = byte.arg & 0xFF; + uint16_t kwargc = byte.arg >> 8; - int n = 0; - py_TValue* sp = SP(); - py_TValue* p1 = sp - kwargc * 2; - py_TValue* base = p1 - argc; - py_TValue* buf = self->vectorcall_buffer; + int n = 0; + py_TValue* sp = SP(); + py_TValue* p1 = sp - kwargc * 2; + py_TValue* base = p1 - argc; + py_TValue* buf = self->vectorcall_buffer; - for(py_TValue* curr = base; curr != p1; curr++) { - if(curr->type != tp_star_wrapper) { - buf[n++] = *curr; - } else { - py_TValue* args = py_getslot(curr, 0); - py_TValue* p; - int length = pk_arrayview(args, &p); - if(length != -1) { - for(int j = 0; j < length; j++) { - buf[n++] = p[j]; - } - argc += length - 1; - } else { - TypeError("*args must be a list or tuple, got '%t'", args->type); - goto __ERROR; + for(py_TValue* curr = base; curr != p1; curr++) { + if(curr->type != tp_star_wrapper) { + buf[n++] = *curr; + } else { + py_TValue* args = py_getslot(curr, 0); + py_TValue* p; + int length = pk_arrayview(args, &p); + if(length != -1) { + for(int j = 0; j < length; j++) { + buf[n++] = p[j]; } + argc += length - 1; + } else { + TypeError("*args must be a list or tuple, got '%t'", args->type); + goto __ERROR; } } + } - for(py_TValue* curr = p1; curr != sp; curr += 2) { - if(curr[1].type != tp_star_wrapper) { - buf[n++] = curr[0]; - buf[n++] = curr[1]; + for(py_TValue* curr = p1; curr != sp; curr += 2) { + if(curr[1].type != tp_star_wrapper) { + buf[n++] = curr[0]; + buf[n++] = curr[1]; + } else { + assert(py_toint(&curr[0]) == 0); + py_TValue* kwargs = py_getslot(&curr[1], 0); + if(kwargs->type == tp_dict) { + py_TValue* p = buf + n; + if(!py_dict_apply(kwargs, unpack_dict_to_buffer, &p)) goto __ERROR; + n = p - buf; + kwargc += py_dict_len(kwargs) - 1; } else { - assert(py_toint(&curr[0]) == 0); - py_TValue* kwargs = py_getslot(&curr[1], 0); - if(kwargs->type == tp_dict) { - py_TValue* p = buf + n; - if(!py_dict_apply(kwargs, unpack_dict_to_buffer, &p)) goto __ERROR; - n = p - buf; - kwargc += py_dict_len(kwargs) - 1; - } else { - TypeError("**kwargs must be a dict, got '%t'", kwargs->type); - goto __ERROR; - } + TypeError("**kwargs must be a dict, got '%t'", kwargs->type); + goto __ERROR; } } - - memcpy(base, buf, n * sizeof(py_TValue)); - SP() = base + n; - - vectorcall_opcall(argc, kwargc); - DISPATCH(); } - case OP_RETURN_VALUE: { - CHECK_RETURN_FROM_EXCEPT_OR_FINALLY(); - if(byte.arg == BC_NOARG) { - self->last_retval = POPX(); - } else { - py_newnone(&self->last_retval); - } - VM__pop_frame(self); - if(frame == base_frame) { // [ frameBase<- ] - return RES_RETURN; - } else { - frame = self->top_frame; - PUSH(&self->last_retval); - goto __NEXT_FRAME; - } - DISPATCH(); + + memcpy(base, buf, n * sizeof(py_TValue)); + SP() = base + n; + + vectorcall_opcall(argc, kwargc); + DISPATCH(); + } + case OP_RETURN_VALUE: { + CHECK_RETURN_FROM_EXCEPT_OR_FINALLY(); + if(byte.arg == BC_NOARG) { + self->last_retval = POPX(); + } else { + py_newnone(&self->last_retval); } - case OP_YIELD_VALUE: { - CHECK_RETURN_FROM_EXCEPT_OR_FINALLY(); - if(byte.arg == 1) { - py_newnone(py_retval()); - } else { - py_assign(py_retval(), TOP()); - POP(); - } + VM__pop_frame(self); + if(frame == base_frame) { // [ frameBase<- ] + return RES_RETURN; + } else { + frame = self->top_frame; + PUSH(&self->last_retval); + goto __NEXT_FRAME; + } + DISPATCH(); + } + case OP_YIELD_VALUE: { + CHECK_RETURN_FROM_EXCEPT_OR_FINALLY(); + if(byte.arg == 1) { + py_newnone(py_retval()); + } else { + py_assign(py_retval(), TOP()); + POP(); + } + return RES_YIELD; + } + case OP_FOR_ITER_YIELD_VALUE: { + CHECK_RETURN_FROM_EXCEPT_OR_FINALLY(); + int res = py_next(TOP()); + if(res == -1) goto __ERROR; + if(res) { return RES_YIELD; + } else { + assert(self->last_retval.type == tp_StopIteration); + py_ObjectRef value = py_getslot(&self->last_retval, 0); + if(py_isnil(value)) value = py_None(); + *TOP() = *value; // [iter] -> [retval] + DISPATCH_JUMP((int16_t)byte.arg); } - case OP_FOR_ITER_YIELD_VALUE: { - CHECK_RETURN_FROM_EXCEPT_OR_FINALLY(); - int res = py_next(TOP()); - if(res == -1) goto __ERROR; - if(res) { - return RES_YIELD; - } else { - assert(self->last_retval.type == tp_StopIteration); - py_ObjectRef value = py_getslot(&self->last_retval, 0); - if(py_isnil(value)) value = py_None(); - *TOP() = *value; // [iter] -> [retval] - DISPATCH_JUMP((int16_t)byte.arg); - } - } - ///////// - case OP_LIST_APPEND: { - // [list, iter, value] - py_list_append(THIRD(), TOP()); - POP(); - DISPATCH(); - } - case OP_DICT_ADD: { - // [dict, iter, key, value] - bool ok = py_dict_setitem(FOURTH(), SECOND(), TOP()); - if(!ok) goto __ERROR; - STACK_SHRINK(2); - DISPATCH(); - } - case OP_SET_ADD: { - // [set, iter, value] - py_push(THIRD()); // [| set] - if(!py_pushmethod(py_name("add"))) { - c11__abort("OP_SET_ADD: failed to load method 'add'"); - } // [|add() set] - py_push(THIRD()); - if(!py_vectorcall(1, 0)) goto __ERROR; - POP(); - DISPATCH(); - } - ///////// - case OP_UNARY_NEGATIVE: { - if(!pk_callmagic(__neg__, 1, TOP())) goto __ERROR; - *TOP() = self->last_retval; - DISPATCH(); - } - case OP_UNARY_NOT: { - int res = py_bool(TOP()); - if(res < 0) goto __ERROR; - py_newbool(TOP(), !res); - DISPATCH(); - } - case OP_UNARY_STAR: { - py_TValue value = POPX(); - int* level = py_newobject(SP()++, tp_star_wrapper, 1, sizeof(int)); - *level = byte.arg; - py_setslot(TOP(), 0, &value); - DISPATCH(); - } - case OP_UNARY_INVERT: { - if(!pk_callmagic(__invert__, 1, TOP())) goto __ERROR; - *TOP() = self->last_retval; - DISPATCH(); - } - //////////////// - case OP_GET_ITER: { - if(!py_iter(TOP())) goto __ERROR; - *TOP() = *py_retval(); - DISPATCH(); - } - case OP_FOR_ITER: { - int res = py_next(TOP()); - if(res == -1) goto __ERROR; - if(res) { - PUSH(py_retval()); - DISPATCH(); - } else { - assert(self->last_retval.type == tp_StopIteration); - POP(); // [iter] -> [] - DISPATCH_JUMP((int16_t)byte.arg); - } - } - //////// - case OP_IMPORT_PATH: { - py_Ref path_object = c11__at(py_TValue, &frame->co->consts, byte.arg); - const char* path = py_tostr(path_object); - int res = py_import(path); - if(res == -1) goto __ERROR; - if(res == 0) { - ImportError("No module named '%s'", path); - goto __ERROR; - } + } + ///////// + case OP_LIST_APPEND: { + // [list, iter, value] + py_list_append(THIRD(), TOP()); + POP(); + DISPATCH(); + } + case OP_DICT_ADD: { + // [dict, iter, key, value] + bool ok = py_dict_setitem(FOURTH(), SECOND(), TOP()); + if(!ok) goto __ERROR; + STACK_SHRINK(2); + DISPATCH(); + } + case OP_SET_ADD: { + // [set, iter, value] + py_push(THIRD()); // [| set] + if(!py_pushmethod(py_name("add"))) { + c11__abort("OP_SET_ADD: failed to load method 'add'"); + } // [|add() set] + py_push(THIRD()); + if(!py_vectorcall(1, 0)) goto __ERROR; + POP(); + DISPATCH(); + } + ///////// + case OP_UNARY_NEGATIVE: { + if(!pk_callmagic(__neg__, 1, TOP())) goto __ERROR; + *TOP() = self->last_retval; + DISPATCH(); + } + case OP_UNARY_NOT: { + int res = py_bool(TOP()); + if(res < 0) goto __ERROR; + py_newbool(TOP(), !res); + DISPATCH(); + } + case OP_UNARY_STAR: { + py_TValue value = POPX(); + int* level = py_newobject(SP()++, tp_star_wrapper, 1, sizeof(int)); + *level = byte.arg; + py_setslot(TOP(), 0, &value); + DISPATCH(); + } + case OP_UNARY_INVERT: { + if(!pk_callmagic(__invert__, 1, TOP())) goto __ERROR; + *TOP() = self->last_retval; + DISPATCH(); + } + //////////////// + case OP_GET_ITER: { + if(!py_iter(TOP())) goto __ERROR; + *TOP() = *py_retval(); + DISPATCH(); + } + case OP_FOR_ITER: { + int res = py_next(TOP()); + if(res == -1) goto __ERROR; + if(res) { PUSH(py_retval()); DISPATCH(); + } else { + assert(self->last_retval.type == tp_StopIteration); + POP(); // [iter] -> [] + DISPATCH_JUMP((int16_t)byte.arg); } - case OP_POP_IMPORT_STAR: { - // [module] - NameDict* dict = PyObject__dict(TOP()->_obj); - py_ItemRef all = NameDict__try_get(dict, __all__); - if(all) { - py_TValue* p; - int length = pk_arrayview(all, &p); - if(length == -1) { - TypeError("'__all__' must be a list or tuple, got '%t'", all->type); - goto __ERROR; - } - for(int i = 0; i < length; i++) { - py_Name name = py_namev(py_tosv(p + i)); - py_ItemRef value = NameDict__try_get(dict, name); - if(value == NULL) { - ImportError("cannot import name '%n'", name); - goto __ERROR; - } else { - if(!Frame__setglobal(frame, name, value)) goto __ERROR; - } - } - } else { - for(int i = 0; i < dict->capacity; i++) { - NameDict_KV* kv = &dict->items[i]; - if(kv->key == NULL) continue; - c11_sv name = py_name2sv(kv->key); - if(name.size == 0 || name.data[0] == '_') continue; - if(!Frame__setglobal(frame, kv->key, &kv->value)) goto __ERROR; - } - } - POP(); - DISPATCH(); + } + //////// + case OP_IMPORT_PATH: { + py_Ref path_object = c11__at(py_TValue, &frame->co->consts, byte.arg); + const char* path = py_tostr(path_object); + int res = py_import(path); + if(res == -1) goto __ERROR; + if(res == 0) { + ImportError("No module named '%s'", path); + goto __ERROR; } - //////// - case OP_UNPACK_SEQUENCE: { + PUSH(py_retval()); + DISPATCH(); + } + case OP_POP_IMPORT_STAR: { + // [module] + NameDict* dict = PyObject__dict(TOP()->_obj); + py_ItemRef all = NameDict__try_get(dict, __all__); + if(all) { py_TValue* p; - int length; - - switch(TOP()->type) { - case tp_tuple: { - length = py_tuple_len(TOP()); - p = py_tuple_data(TOP()); - break; - } - case tp_list: { - length = py_list_len(TOP()); - p = py_list_data(TOP()); - break; - } - case tp_vec2i: { - length = 2; - if(byte.arg != length) break; - c11_vec2i val = py_tovec2i(TOP()); - POP(); - py_newint(SP()++, val.x); - py_newint(SP()++, val.y); - DISPATCH(); - } - case tp_vec2: { - length = 2; - if(byte.arg != length) break; - c11_vec2 val = py_tovec2(TOP()); - POP(); - py_newfloat(SP()++, val.x); - py_newfloat(SP()++, val.y); - DISPATCH(); - } - case tp_vec3i: { - length = 3; - if(byte.arg != length) break; - c11_vec3i val = py_tovec3i(TOP()); - POP(); - py_newint(SP()++, val.x); - py_newint(SP()++, val.y); - py_newint(SP()++, val.z); - DISPATCH(); - } - case tp_vec3: { - length = 3; - if(byte.arg != length) break; - c11_vec3 val = py_tovec3(TOP()); - POP(); - py_newfloat(SP()++, val.x); - py_newfloat(SP()++, val.y); - py_newfloat(SP()++, val.z); - DISPATCH(); - } - default: { - TypeError("expected list or tuple to unpack, got %t", TOP()->type); - goto __ERROR; - } - } - if(length != byte.arg) { - ValueError("expected %d values to unpack, got %d", byte.arg, length); + int length = pk_arrayview(all, &p); + if(length == -1) { + TypeError("'__all__' must be a list or tuple, got '%t'", all->type); goto __ERROR; } - POP(); for(int i = 0; i < length; i++) { - PUSH(p + i); + py_Name name = py_namev(py_tosv(p + i)); + py_ItemRef value = NameDict__try_get(dict, name); + if(value == NULL) { + ImportError("cannot import name '%n'", name); + goto __ERROR; + } else { + if(!Frame__setglobal(frame, name, value)) goto __ERROR; + } + } + } else { + for(int i = 0; i < dict->capacity; i++) { + NameDict_KV* kv = &dict->items[i]; + if(kv->key == NULL) continue; + c11_sv name = py_name2sv(kv->key); + if(name.size == 0 || name.data[0] == '_') continue; + if(!Frame__setglobal(frame, kv->key, &kv->value)) goto __ERROR; } - DISPATCH(); } - case OP_UNPACK_EX: { - py_TValue* p; - int length = pk_arrayview(TOP(), &p); - if(length == -1) { + POP(); + DISPATCH(); + } + //////// + case OP_UNPACK_SEQUENCE: { + py_TValue* p; + int length; + + switch(TOP()->type) { + case tp_tuple: { + length = py_tuple_len(TOP()); + p = py_tuple_data(TOP()); + break; + } + case tp_list: { + length = py_list_len(TOP()); + p = py_list_data(TOP()); + break; + } + case tp_vec2i: { + length = 2; + if(byte.arg != length) break; + c11_vec2i val = py_tovec2i(TOP()); + POP(); + py_newint(SP()++, val.x); + py_newint(SP()++, val.y); + DISPATCH(); + } + case tp_vec2: { + length = 2; + if(byte.arg != length) break; + c11_vec2 val = py_tovec2(TOP()); + POP(); + py_newfloat(SP()++, val.x); + py_newfloat(SP()++, val.y); + DISPATCH(); + } + case tp_vec3i: { + length = 3; + if(byte.arg != length) break; + c11_vec3i val = py_tovec3i(TOP()); + POP(); + py_newint(SP()++, val.x); + py_newint(SP()++, val.y); + py_newint(SP()++, val.z); + DISPATCH(); + } + case tp_vec3: { + length = 3; + if(byte.arg != length) break; + c11_vec3 val = py_tovec3(TOP()); + POP(); + py_newfloat(SP()++, val.x); + py_newfloat(SP()++, val.y); + py_newfloat(SP()++, val.z); + DISPATCH(); + } + default: { TypeError("expected list or tuple to unpack, got %t", TOP()->type); goto __ERROR; } - int exceed = length - byte.arg; - if(exceed < 0) { - ValueError("not enough values to unpack"); - goto __ERROR; - } - POP(); - for(int i = 0; i < byte.arg; i++) { - PUSH(p + i); - } - py_newlistn(SP()++, exceed); - for(int i = 0; i < exceed; i++) { - py_list_setitem(TOP(), i, p + byte.arg + i); - } - DISPATCH(); } - /////////// - case OP_BEGIN_CLASS: { - // [base] - py_Name name = co_names[byte.arg]; - py_Type base; - if(py_isnone(TOP())) { - base = tp_object; - } else { - if(!py_checktype(TOP(), tp_type)) goto __ERROR; - base = py_totype(TOP()); - } - POP(); - - py_TypeInfo* base_ti = pk_typeinfo(base); - if(base_ti->is_sealed) { - TypeError("type '%t' is not an acceptable base type", base); - goto __ERROR; - } - - py_Type type = pk_newtypewithmode(name, - base, - frame->module, - NULL, - base_ti->is_python, - false, - frame->co->src->mode); - PUSH(py_tpobject(type)); - self->curr_class = TOP(); - DISPATCH(); - } - case OP_END_CLASS: { - // [cls or decorated] - py_Name name = co_names[byte.arg]; - if(!Frame__setglobal(frame, name, TOP())) goto __ERROR; - - if(py_istype(TOP(), tp_type)) { - // call on_end_subclass - py_TypeInfo* ti = py_touserdata(TOP()); - if(ti->base != tp_object) { - py_TypeInfo* base_ti = ti->base_ti; - if(base_ti->on_end_subclass) base_ti->on_end_subclass(ti); - } - py_TValue* slot_eq = py_getdict(&ti->self, __eq__); - py_TValue* slot_ne = py_getdict(&ti->self, __ne__); - if(slot_eq && !slot_ne) { - TypeError("'%n' implements '__eq__' but not '__ne__'", ti->name); - goto __ERROR; - } - } - // class with decorator is unsafe currently - // it skips the above check - POP(); - self->curr_class = NULL; - DISPATCH(); - } - case OP_STORE_CLASS_ATTR: { - assert(self->curr_class); - py_Name name = co_names[byte.arg]; - // TOP() can be a function, classmethod or custom decorator - py_Ref actual_func = TOP(); - if(actual_func->type == tp_classmethod) { - actual_func = py_getslot(actual_func, 0); - } - if(actual_func->type == tp_function) { - Function* ud = py_touserdata(actual_func); - ud->clazz = self->curr_class->_obj; - } - py_setdict(self->curr_class, name, TOP()); - POP(); - DISPATCH(); - } - case OP_ADD_CLASS_ANNOTATION: { - assert(self->curr_class); - // [type_hint string] - py_TypeInfo* ti = py_touserdata(self->curr_class); - if(py_isnil(&ti->annotations)) py_newdict(&ti->annotations); - py_Name name = co_names[byte.arg]; - bool ok = py_dict_setitem_by_str(&ti->annotations, py_name2str(name), TOP()); - if(!ok) goto __ERROR; - POP(); - DISPATCH(); - } - /////////// - case OP_WITH_ENTER: { - // [expr] - py_push(TOP()); - if(!py_pushmethod(__enter__)) { - TypeError("'%t' object does not support the context manager protocol", - TOP()->type); - goto __ERROR; - } - vectorcall_opcall(0, 0); - DISPATCH(); - } - case OP_WITH_EXIT: { - // [expr] - py_push(TOP()); - if(!py_pushmethod(__exit__)) { - TypeError("'%t' object does not support the context manager protocol", - TOP()->type); - goto __ERROR; - } - if(!py_vectorcall(0, 0)) goto __ERROR; - POP(); - DISPATCH(); - } - /////////// - case OP_TRY_ENTER: { - Frame__set_unwind_target(frame, SP()); - DISPATCH(); - } - case OP_EXCEPTION_MATCH: { - if(!py_checktype(TOP(), tp_type)) goto __ERROR; - bool ok = py_isinstance(&self->curr_exception, py_totype(TOP())); - py_newbool(TOP(), ok); - DISPATCH(); - } - case OP_RAISE: { - // [exception] - if(py_istype(TOP(), tp_type)) { - if(!py_tpcall(py_totype(TOP()), 0, NULL)) goto __ERROR; - py_assign(TOP(), py_retval()); - } - if(!py_isinstance(TOP(), tp_BaseException)) { - TypeError("exceptions must derive from BaseException"); - goto __ERROR; - } - py_raise(TOP()); + if(length != byte.arg) { + ValueError("expected %d values to unpack, got %d", byte.arg, length); goto __ERROR; } - case OP_RAISE_ASSERT: { - if(byte.arg) { - if(!py_str(TOP())) goto __ERROR; - POP(); - py_exception(tp_AssertionError, "%s", py_tostr(py_retval())); - } else { - py_exception(tp_AssertionError, ""); - } - goto __ERROR; + POP(); + for(int i = 0; i < length; i++) { + PUSH(p + i); } - case OP_RE_RAISE: { - if(self->curr_exception.type) { - assert(!self->is_curr_exc_handled); - goto __ERROR_RE_RAISE; - } - DISPATCH(); - } - case OP_PUSH_EXCEPTION: { - assert(self->curr_exception.type); - PUSH(&self->curr_exception); - DISPATCH(); - } - case OP_BEGIN_EXC_HANDLING: { - assert(self->curr_exception.type); - self->is_curr_exc_handled = true; - DISPATCH(); - } - case OP_END_EXC_HANDLING: { - assert(self->curr_exception.type); - py_clearexc(NULL); - DISPATCH(); - } - case OP_BEGIN_FINALLY: { - if(self->curr_exception.type) { - assert(!self->is_curr_exc_handled); - // temporarily handle the exception if any - self->is_curr_exc_handled = true; - } - DISPATCH(); - } - case OP_END_FINALLY: { - if(byte.arg == BC_NOARG) { - if(self->curr_exception.type) { - assert(self->is_curr_exc_handled); - // revert the exception handling if needed - self->is_curr_exc_handled = false; - } - } else { - // break or continue inside finally block - py_clearexc(NULL); - } - DISPATCH(); - } - ////////////////// - case OP_FORMAT_STRING: { - py_Ref spec = c11__at(py_TValue, &frame->co->consts, byte.arg); - bool ok = stack_format_object(self, py_tosv(spec)); - if(!ok) goto __ERROR; - DISPATCH(); - } - default: c11__unreachable(); + DISPATCH(); } - - c11__unreachable(); - - __ERROR: - py_BaseException__stpush(&self->curr_exception, - frame->co->src, - Frame__lineno(frame), - !frame->is_locals_special ? frame->co->name->data : NULL); - __ERROR_RE_RAISE: - do { - } while(0); - int target = Frame__prepare_jump_exception_handler(frame, &self->stack); - if(target >= 0) { - // 1. Exception can be handled inside the current frame - DISPATCH_JUMP_ABSOLUTE(target); - } else { - // 2. Exception need to be propagated to the upper frame - bool is_base_frame_to_be_popped = frame == base_frame; - VM__pop_frame(self); - if(self->top_frame == NULL || is_base_frame_to_be_popped) { - // propagate to the top level - return RES_ERROR; + case OP_UNPACK_EX: { + py_TValue* p; + int length = pk_arrayview(TOP(), &p); + if(length == -1) { + TypeError("expected list or tuple to unpack, got %t", TOP()->type); + goto __ERROR; } - frame = self->top_frame; - co_codes = frame->co->codes.data; - co_names = frame->co->names.data; + int exceed = length - byte.arg; + if(exceed < 0) { + ValueError("not enough values to unpack"); + goto __ERROR; + } + POP(); + for(int i = 0; i < byte.arg; i++) { + PUSH(p + i); + } + py_newlistn(SP()++, exceed); + for(int i = 0; i < exceed; i++) { + py_list_setitem(TOP(), i, p + byte.arg + i); + } + DISPATCH(); + } + /////////// + case OP_BEGIN_CLASS: { + // [base] + py_Name name = co_names[byte.arg]; + py_Type base; + if(py_isnone(TOP())) { + base = tp_object; + } else { + if(!py_checktype(TOP(), tp_type)) goto __ERROR; + base = py_totype(TOP()); + } + POP(); + + py_TypeInfo* base_ti = pk_typeinfo(base); + if(base_ti->is_final) { + TypeError("type '%t' is not an acceptable base type", base); + goto __ERROR; + } + + py_Type type = pk_newtypewithmode(name, + base, + frame->module, + NULL, + base_ti->is_python, + false, + frame->co->src->mode); + PUSH(py_tpobject(type)); + self->curr_class = TOP(); + DISPATCH(); + } + case OP_END_CLASS: { + // [cls or decorated] + py_Name name = co_names[byte.arg]; + if(!Frame__setglobal(frame, name, TOP())) goto __ERROR; + + if(py_istype(TOP(), tp_type)) { + // call on_end_subclass + py_TypeInfo* ti = py_touserdata(TOP()); + if(ti->base != tp_object) { + py_TypeInfo* base_ti = ti->base_ti; + if(base_ti->on_end_subclass) base_ti->on_end_subclass(ti); + } + py_TValue* slot_eq = py_getdict(&ti->self, __eq__); + py_TValue* slot_ne = py_getdict(&ti->self, __ne__); + if(slot_eq && !slot_ne) { + TypeError("'%n' implements '__eq__' but not '__ne__'", ti->name); + goto __ERROR; + } + } + // class with decorator is unsafe currently + // it skips the above check + POP(); + self->curr_class = NULL; + DISPATCH(); + } + case OP_STORE_CLASS_ATTR: { + assert(self->curr_class); + py_Name name = co_names[byte.arg]; + // TOP() can be a function, classmethod or custom decorator + py_Ref actual_func = TOP(); + if(actual_func->type == tp_classmethod) { actual_func = py_getslot(actual_func, 0); } + if(actual_func->type == tp_function) { + Function* ud = py_touserdata(actual_func); + ud->clazz = self->curr_class->_obj; + } + py_setdict(self->curr_class, name, TOP()); + POP(); + DISPATCH(); + } + case OP_ADD_CLASS_ANNOTATION: { + assert(self->curr_class); + // [type_hint string] + py_TypeInfo* ti = py_touserdata(self->curr_class); + if(py_isnil(&ti->annotations)) py_newdict(&ti->annotations); + py_Name name = co_names[byte.arg]; + bool ok = py_dict_setitem_by_str(&ti->annotations, py_name2str(name), TOP()); + if(!ok) goto __ERROR; + POP(); + DISPATCH(); + } + /////////// + case OP_WITH_ENTER: { + // [expr] + py_push(TOP()); + if(!py_pushmethod(__enter__)) { + TypeError("'%t' object does not support the context manager protocol", TOP()->type); + goto __ERROR; + } + vectorcall_opcall(0, 0); + DISPATCH(); + } + case OP_WITH_EXIT: { + // [expr] + py_push(TOP()); + if(!py_pushmethod(__exit__)) { + TypeError("'%t' object does not support the context manager protocol", TOP()->type); + goto __ERROR; + } + if(!py_vectorcall(0, 0)) goto __ERROR; + POP(); + DISPATCH(); + } + /////////// + case OP_TRY_ENTER: { + Frame__set_unwind_target(frame, SP()); + DISPATCH(); + } + case OP_EXCEPTION_MATCH: { + if(!py_checktype(TOP(), tp_type)) goto __ERROR; + bool ok = py_isinstance(&self->curr_exception, py_totype(TOP())); + py_newbool(TOP(), ok); + DISPATCH(); + } + case OP_RAISE: { + // [exception] + if(py_istype(TOP(), tp_type)) { + if(!py_tpcall(py_totype(TOP()), 0, NULL)) goto __ERROR; + py_assign(TOP(), py_retval()); + } + if(!py_isinstance(TOP(), tp_BaseException)) { + TypeError("exceptions must derive from BaseException"); + goto __ERROR; + } + py_raise(TOP()); goto __ERROR; } + case OP_RAISE_ASSERT: { + if(byte.arg) { + if(!py_str(TOP())) goto __ERROR; + POP(); + py_exception(tp_AssertionError, "%s", py_tostr(py_retval())); + } else { + py_exception(tp_AssertionError, ""); + } + goto __ERROR; + } + case OP_RE_RAISE: { + if(self->curr_exception.type) { + assert(!self->is_curr_exc_handled); + goto __ERROR_RE_RAISE; + } + DISPATCH(); + } + case OP_PUSH_EXCEPTION: { + assert(self->curr_exception.type); + PUSH(&self->curr_exception); + DISPATCH(); + } + case OP_BEGIN_EXC_HANDLING: { + assert(self->curr_exception.type); + self->is_curr_exc_handled = true; + DISPATCH(); + } + case OP_END_EXC_HANDLING: { + assert(self->curr_exception.type); + py_clearexc(NULL); + DISPATCH(); + } + case OP_BEGIN_FINALLY: { + if(self->curr_exception.type) { + assert(!self->is_curr_exc_handled); + // temporarily handle the exception if any + self->is_curr_exc_handled = true; + } + DISPATCH(); + } + case OP_END_FINALLY: { + if(byte.arg == BC_NOARG) { + if(self->curr_exception.type) { + assert(self->is_curr_exc_handled); + // revert the exception handling if needed + self->is_curr_exc_handled = false; + } + } else { + // break or continue inside finally block + py_clearexc(NULL); + } + DISPATCH(); + } + ////////////////// + case OP_FORMAT_STRING: { + py_Ref spec = c11__at(py_TValue, &frame->co->consts, byte.arg); + bool ok = stack_format_object(self, py_tosv(spec)); + if(!ok) goto __ERROR; + DISPATCH(); + } + default: c11__unreachable(); } - return RES_RETURN; + c11__unreachable(); + +__ERROR: + py_BaseException__stpush(&self->curr_exception, + frame->co->src, + Frame__lineno(frame), + !frame->is_locals_special ? frame->co->name->data : NULL); +__ERROR_RE_RAISE: + do { + } while(0); + + int target = Frame__prepare_jump_exception_handler(frame, &self->stack); + if(target >= 0) { + // 1. Exception can be handled inside the current frame + DISPATCH_JUMP_ABSOLUTE(target); + } else { + // 2. Exception need to be propagated to the upper frame + bool is_base_frame_to_be_popped = frame == base_frame; + VM__pop_frame(self); + if(self->top_frame == NULL || is_base_frame_to_be_popped) { + // propagate to the top level + return RES_ERROR; + } + frame = self->top_frame; + RESET_CO_CACHE(); + goto __ERROR; + } + + c11__unreachable(); } const char* pk_op2str(py_Name op) { @@ -1304,7 +1303,10 @@ bool pk_stack_binaryop(VM* self, py_Name op, py_Name rop) { py_newbool(py_retval(), !res); return true; } - return TypeError("unsupported operand type(s) for '%s'", pk_op2str(op)); + + py_Type lhs_t = rop ? TOP()->type : SECOND()->type; + py_Type rhs_t = rop ? SECOND()->type : TOP()->type; + return TypeError("unsupported operand type(s) for '%s': '%t' and '%t'", pk_op2str(op), lhs_t, rhs_t); } bool py_binaryop(py_Ref lhs, py_Ref rhs, py_Name op, py_Name rop) { @@ -1522,6 +1524,7 @@ static bool stack_format_object(VM* self, c11_sv spec) { #undef SP #undef INSERT_THIRD #undef vectorcall_opcall +#undef RESET_CO_CACHE void py_sys_settrace(py_TraceFunc func, bool reset) { TraceInfo* info = &pk_current_vm->trace_info; diff --git a/src/interpreter/frame.c b/src/interpreter/frame.c index 2d5ee253..9ff16bf5 100644 --- a/src/interpreter/frame.c +++ b/src/interpreter/frame.c @@ -5,6 +5,7 @@ #include "pocketpy/objects/codeobject.h" #include "pocketpy/pocketpy.h" #include +#include void ValueStack__ctor(ValueStack* self) { self->sp = self->begin; diff --git a/src/interpreter/generator.c b/src/interpreter/generator.c index e5542736..1f7d4cda 100644 --- a/src/interpreter/generator.c +++ b/src/interpreter/generator.c @@ -4,6 +4,7 @@ #include "pocketpy/objects/base.h" #include "pocketpy/pocketpy.h" #include +#include void pk_newgenerator(py_Ref out, py_Frame* frame, py_TValue* begin, py_TValue* end) { Generator* ud = py_newobject(out, tp_generator, 1, sizeof(Generator)); @@ -20,7 +21,7 @@ void Generator__dtor(Generator* ud) { if(ud->frame) Frame__delete(ud->frame); } -static bool generator__next__(int argc, py_Ref argv) { +bool generator__next__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); Generator* ud = py_touserdata(argv); py_StackRef p0 = py_peek(0); diff --git a/src/interpreter/heap.c b/src/interpreter/heap.c index e50aaceb..2d061478 100644 --- a/src/interpreter/heap.c +++ b/src/interpreter/heap.c @@ -3,6 +3,7 @@ #include "pocketpy/interpreter/objectpool.h" #include "pocketpy/objects/base.h" #include "pocketpy/pocketpy.h" +#include void ManagedHeap__ctor(ManagedHeap* self) { MultiPool__ctor(&self->small_objects); @@ -33,25 +34,26 @@ void ManagedHeap__dtor(ManagedHeap* self) { void ManagedHeap__collect_if_needed(ManagedHeap* self) { if(!self->gc_enabled) return; if(self->gc_counter < self->gc_threshold) return; - self->gc_counter = 0; int freed = ManagedHeap__collect(self); // adjust `gc_threshold` based on `freed_ma` self->freed_ma[0] = self->freed_ma[1]; self->freed_ma[1] = self->freed_ma[2]; self->freed_ma[2] = freed; int avg_freed = (self->freed_ma[0] + self->freed_ma[1] + self->freed_ma[2]) / 3; - const int upper = PK_GC_MIN_THRESHOLD * 8; + const int upper = PK_GC_MIN_THRESHOLD * 16; const int lower = PK_GC_MIN_THRESHOLD / 2; float free_ratio = (float)avg_freed / self->gc_threshold; - int new_threshold = self->gc_threshold * (1 / free_ratio); + int new_threshold = self->gc_threshold * (1.5f / free_ratio); // printf("gc_threshold=%d, avg_freed=%d, new_threshold=%d\n", self->gc_threshold, avg_freed, // new_threshold); self->gc_threshold = c11__min(c11__max(new_threshold, lower), upper); } int ManagedHeap__collect(ManagedHeap* self) { + self->gc_counter = 0; ManagedHeap__mark(self); int freed = ManagedHeap__sweep(self); + // printf("GC: collected %d objects\n", freed); return freed; } diff --git a/src/interpreter/line_profier.c b/src/interpreter/line_profier.c index b9251356..a1ea48e9 100644 --- a/src/interpreter/line_profier.c +++ b/src/interpreter/line_profier.c @@ -1,4 +1,5 @@ #include "pocketpy/interpreter/line_profiler.h" +#include void LineProfiler__ctor(LineProfiler* self) { c11_smallmap_p2i__ctor(&self->records); diff --git a/src/interpreter/objectpool.c b/src/interpreter/objectpool.c index fb9685ca..08a18fae 100644 --- a/src/interpreter/objectpool.c +++ b/src/interpreter/objectpool.c @@ -1,12 +1,10 @@ #include "pocketpy/interpreter/objectpool.h" -#include "pocketpy/config.h" #include "pocketpy/objects/object.h" #include "pocketpy/common/sstream.h" #include #include -#include #include static PoolArena* PoolArena__new(int block_size) { @@ -16,7 +14,6 @@ static PoolArena* PoolArena__new(int block_size) { self->block_size = block_size; self->block_count = block_count; self->unused_length = block_count; - self->unused = PK_MALLOC(sizeof(int) * block_count); for(int i = 0; i < block_count; i++) { self->unused[i] = i; } @@ -29,7 +26,6 @@ static void PoolArena__delete(PoolArena* self) { PyObject* obj = (PyObject*)(self->data + i * self->block_size); if(obj->type != 0) PyObject__dtor(obj); } - PK_FREE(self->unused); PK_FREE(self); } diff --git a/src/interpreter/typeinfo.c b/src/interpreter/typeinfo.c index d8a07756..93fba34d 100644 --- a/src/interpreter/typeinfo.c +++ b/src/interpreter/typeinfo.c @@ -1,4 +1,5 @@ #include "pocketpy/interpreter/vm.h" +#include py_ItemRef pk_tpfindname(py_TypeInfo* ti, py_Name name) { assert(ti != NULL); @@ -10,17 +11,17 @@ py_ItemRef pk_tpfindname(py_TypeInfo* ti, py_Name name) { return NULL; } -py_ItemRef py_tpfindname(py_Type type, py_Name name) { +PK_INLINE py_ItemRef py_tpfindname(py_Type type, py_Name name) { py_TypeInfo* ti = pk_typeinfo(type); return pk_tpfindname(ti, name); } -py_Ref py_tpfindmagic(py_Type t, py_Name name) { +PK_INLINE py_Ref py_tpfindmagic(py_Type t, py_Name name) { // assert(py_ismagicname(name)); return py_tpfindname(t, name); } -py_Type py_tpbase(py_Type t) { +PK_INLINE py_Type py_tpbase(py_Type t) { assert(t); py_TypeInfo* ti = pk_typeinfo(t); return ti->base; @@ -44,7 +45,7 @@ const char* py_tpname(py_Type type) { return py_name2str(name); } -py_TypeInfo* pk_typeinfo(py_Type type) { +PK_INLINE py_TypeInfo* pk_typeinfo(py_Type type) { #ifndef NDEBUG int length = pk_current_vm->types.length; if(type < 0 || type >= length) { @@ -60,11 +61,11 @@ static void py_TypeInfo__common_init(py_Name name, const py_GlobalRef module, void (*dtor)(void*), bool is_python, - bool is_sealed, + bool is_final, py_TypeInfo* self, py_TValue* typeobject) { py_TypeInfo* base_ti = base ? pk_typeinfo(base) : NULL; - if(base_ti && base_ti->is_sealed) { + if(base_ti && base_ti->is_final) { c11__abort("type '%s' is not an acceptable base type", py_name2str(base_ti->name)); } @@ -78,7 +79,12 @@ static void py_TypeInfo__common_init(py_Name name, if(!dtor && base) dtor = base_ti->dtor; self->is_python = is_python; - self->is_sealed = is_sealed; + self->is_final = is_final; + + self->getattribute = NULL; + self->setattribute = NULL; + self->delattribute = NULL; + self->getunboundmethod = NULL; self->annotations = *py_NIL(); self->dtor = dtor; @@ -90,7 +96,7 @@ py_Type pk_newtype(const char* name, const py_GlobalRef module, void (*dtor)(void*), bool is_python, - bool is_sealed) { + bool is_final) { py_Type index = pk_current_vm->types.length; py_TypeInfo* self = py_newobject(py_retval(), tp_type, -1, sizeof(py_TypeInfo)); py_TypeInfo__common_init(py_name(name), @@ -99,7 +105,7 @@ py_Type pk_newtype(const char* name, module, dtor, is_python, - is_sealed, + is_final, self, py_retval()); TypePointer* pointer = c11_vector__emplace(&pk_current_vm->types); @@ -113,7 +119,7 @@ py_Type pk_newtypewithmode(py_Name name, const py_GlobalRef module, void (*dtor)(void*), bool is_python, - bool is_sealed, + bool is_final, enum py_CompileMode mode) { if(mode == RELOAD_MODE && module != NULL) { py_ItemRef old_class = py_getdict(module, name); @@ -131,7 +137,7 @@ py_Type pk_newtypewithmode(py_Name name, module, dtor, is_python, - is_sealed, + is_final, self, &self->self); TypePointer* pointer = c11__at(TypePointer, &pk_current_vm->types, index); @@ -141,7 +147,7 @@ py_Type pk_newtypewithmode(py_Name name, } } - return pk_newtype(py_name2str(name), base, module, dtor, is_python, is_sealed); + return pk_newtype(py_name2str(name), base, module, dtor, is_python, is_final); } py_Type py_newtype(const char* name, py_Type base, const py_GlobalRef module, void (*dtor)(void*)) { @@ -149,4 +155,23 @@ py_Type py_newtype(const char* name, py_Type base, const py_GlobalRef module, vo py_Type type = pk_newtype(name, base, module, dtor, false, false); if(module) py_setdict(module, py_name(name), py_tpobject(type)); return type; -} \ No newline at end of file +} + +void py_tpsetfinal(py_Type type) { + assert(type); + py_TypeInfo* ti = pk_typeinfo(type); + ti->is_final = true; +} + +void py_tphookattributes(py_Type type, + bool (*getattribute)(py_Ref self, py_Name name), + bool (*setattribute)(py_Ref self, py_Name name, py_Ref val), + bool (*delattribute)(py_Ref self, py_Name name), + bool (*getunboundmethod)(py_Ref self, py_Name name)) { + assert(type); + py_TypeInfo* ti = pk_typeinfo(type); + ti->getattribute = getattribute; + ti->setattribute = setattribute; + ti->delattribute = delattribute; + ti->getunboundmethod = getunboundmethod; +} diff --git a/src/interpreter/vm.c b/src/interpreter/vm.c index c402c1ee..f81383df 100644 --- a/src/interpreter/vm.c +++ b/src/interpreter/vm.c @@ -1,6 +1,5 @@ #include "pocketpy/interpreter/vm.h" #include "pocketpy/common/memorypool.h" -#include "pocketpy/common/sstream.h" #include "pocketpy/common/utils.h" #include "pocketpy/interpreter/generator.h" #include "pocketpy/interpreter/modules.h" @@ -10,6 +9,7 @@ #include "pocketpy/common/_generated.h" #include "pocketpy/pocketpy.h" #include +#include static char* pk_default_importfile(const char* path) { #if PK_ENABLE_OS @@ -59,6 +59,7 @@ void VM__ctor(VM* self) { self->main = NULL; self->callbacks.importfile = pk_default_importfile; + self->callbacks.lazyimport = NULL; self->callbacks.print = pk_default_print; self->callbacks.flush = pk_default_flush; self->callbacks.getchr = pk_default_getchr; @@ -109,7 +110,8 @@ void VM__ctor(VM* self) { validate(tp_list, pk_list__register()); validate(tp_tuple, pk_tuple__register()); - validate(tp_array_iterator, pk_array_iterator__register()); + validate(tp_list_iterator, pk_list_iterator__register()); + validate(tp_tuple_iterator, pk_tuple_iterator__register()); validate(tp_slice, pk_slice__register()); validate(tp_range, pk_range__register()); @@ -234,6 +236,7 @@ void VM__ctor(VM* self) { pk__add_module_conio(); pk__add_module_lz4(); // optional pk__add_module_libhv(); // optional + pk__add_module_cute_png(); // optional pk__add_module_pkpy(); // add python builtins diff --git a/src/interpreter/vmx.c b/src/interpreter/vmx.c index 4334981b..abb00a2b 100644 --- a/src/interpreter/vmx.c +++ b/src/interpreter/vmx.c @@ -1,4 +1,5 @@ #include "pocketpy/interpreter/vm.h" +#include void pk_print_stack(VM* self, py_Frame* frame, Bytecode byte) { return; @@ -123,5 +124,8 @@ py_Ref py_name2ref(py_Name name) { void PyObject__dtor(PyObject* self) { py_Dtor dtor = c11__getitem(TypePointer, &pk_current_vm->types, self->type).dtor; if(dtor) dtor(PyObject__userdata(self)); - if(self->slots == -1) NameDict__dtor(PyObject__dict(self)); + if(self->slots == -1) { + NameDict* dict = PyObject__dict(self); + NameDict__dtor(dict); + } } \ No newline at end of file diff --git a/src/modules/array2d.c b/src/modules/array2d.c index b1c01110..c599f20d 100644 --- a/src/modules/array2d.c +++ b/src/modules/array2d.c @@ -16,7 +16,7 @@ static bool c11_array2d__set(c11_array2d* self, int col, int row, py_Ref value) return true; } -c11_array2d* py_newarray2d(py_OutRef out, int n_cols, int n_rows) { +c11_array2d* c11_newarray2d(py_OutRef out, int n_cols, int n_rows) { int numel = n_cols * n_rows; c11_array2d* ud = py_newobject(out, tp_array2d, numel, sizeof(c11_array2d)); ud->header.n_cols = n_cols; @@ -112,7 +112,10 @@ static bool array2d_like_index(int argc, py_Ref argv) { int code = py_equal(item, value); if(code == -1) return false; if(code == 1) { - py_newvec2i(py_retval(), (c11_vec2i){{i, j}}); + py_newvec2i(py_retval(), + (c11_vec2i){ + {i, j} + }); return true; } } @@ -176,7 +179,7 @@ static bool array2d_like_map(int argc, py_Ref argv) { PY_CHECK_ARGC(2); c11_array2d_like* self = py_touserdata(argv); py_Ref f = py_arg(1); - c11_array2d* res = py_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); + c11_array2d* res = c11_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); for(int j = 0; j < self->n_rows; j++) { for(int i = 0; i < self->n_cols; i++) { py_Ref item = self->f_get(self, i, j); @@ -228,7 +231,7 @@ static bool _array2d_like_broadcasted_zip_with(int argc, py_Ref argv, py_Name op } else { other = NULL; } - c11_array2d* res = py_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); + c11_array2d* res = c11_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); for(int j = 0; j < self->n_rows; j++) { for(int i = 0; i < self->n_cols; i++) { py_Ref lhs = self->f_get(self, i, j); @@ -254,7 +257,7 @@ static bool array2d_like_zip_with(int argc, py_Ref argv) { c11_array2d_like* other = py_touserdata(py_arg(1)); py_Ref f = py_arg(2); if(!_array2d_like_check_same_shape(self, other)) return false; - c11_array2d* res = py_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); + c11_array2d* res = c11_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); for(int j = 0; j < self->n_rows; j++) { for(int i = 0; i < self->n_cols; i++) { py_push(f); @@ -299,7 +302,7 @@ DEF_ARRAY2D_LIKE__MAGIC_ZIP_WITH(__xor__, __xor__, 0) static bool array2d_like__invert__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); c11_array2d_like* self = py_touserdata(argv); - c11_array2d* res = py_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); + c11_array2d* res = c11_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); for(int j = 0; j < self->n_rows; j++) { for(int i = 0; i < self->n_cols; i++) { py_Ref item = self->f_get(self, i, j); @@ -316,7 +319,7 @@ static bool array2d_like_copy(int argc, py_Ref argv) { // def copy(self) -> 'array2d': ... PY_CHECK_ARGC(1); c11_array2d_like* self = py_touserdata(argv); - c11_array2d* res = py_newarray2d(py_retval(), self->n_cols, self->n_rows); + c11_array2d* res = c11_newarray2d(py_retval(), self->n_cols, self->n_rows); for(int j = 0; j < self->n_rows; j++) { for(int i = 0; i < self->n_cols; i++) { py_Ref item = self->f_get(self, i, j); @@ -637,7 +640,7 @@ static bool array2d_like_get_bounding_rect(int argc, py_Ref argv) { static bool array2d_like_count_neighbors(int argc, py_Ref argv) { PY_CHECK_ARGC(3); c11_array2d_like* self = py_touserdata(argv); - c11_array2d* res = py_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); + c11_array2d* res = c11_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); py_Ref value = py_arg(1); const char* neighborhood = py_tostr(py_arg(2)); @@ -703,7 +706,7 @@ static bool array2d_like_convolve(int argc, py_Ref argv) { int ksize = kernel->n_cols; if(ksize % 2 == 0) return ValueError("kernel size must be odd"); int ksize_half = ksize / 2; - c11_array2d* res = py_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); + c11_array2d* res = c11_newarray2d(py_pushtmp(), self->n_cols, self->n_rows); for(int j = 0; j < self->n_rows; j++) { for(int i = 0; i < self->n_cols; i++) { py_i64 sum = 0; @@ -800,7 +803,7 @@ static void register_array2d_like(py_Ref mod) { } } -static bool array2d_like_iterator__next__(int argc, py_Ref argv) { +bool array2d_like_iterator__next__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); c11_array2d_like_iterator* self = py_touserdata(argv); if(self->j >= self->array->n_rows) return StopIteration(); @@ -834,7 +837,7 @@ static bool array2d__new__(int argc, py_Ref argv) { int n_cols = argv[1]._i64; int n_rows = argv[2]._i64; if(n_cols <= 0 || n_rows <= 0) return ValueError("array2d() expected positive dimensions"); - c11_array2d* ud = py_newarray2d(py_pushtmp(), n_cols, n_rows); + c11_array2d* ud = c11_newarray2d(py_pushtmp(), n_cols, n_rows); // setup initial values if(py_callable(default_)) { for(int j = 0; j < n_rows; j++) { @@ -876,7 +879,7 @@ static bool array2d_fromlist_STATIC(int argc, py_Ref argv) { return ValueError("fromlist() expected a list of lists with the same length"); } } - c11_array2d* res = py_newarray2d(py_retval(), n_cols, n_rows); + c11_array2d* res = c11_newarray2d(py_retval(), n_cols, n_rows); for(int j = 0; j < n_rows; j++) { py_Ref row_j = py_list_getitem(argv, j); for(int i = 0; i < n_cols; i++) { @@ -1359,4 +1362,30 @@ void pk__add_module_array2d() { register_array2d(mod); register_array2d_view(mod); register_chunked_array2d(mod); +} + +void py_newarray2d(py_OutRef out, int width, int height) { c11_newarray2d(out, width, height); } + +int py_array2d_getwidth(py_Ref self) { + assert(self->type == tp_array2d); + c11_array2d* ud = py_touserdata(self); + return ud->header.n_cols; +} + +int py_array2d_getheight(py_Ref self) { + assert(self->type == tp_array2d); + c11_array2d* ud = py_touserdata(self); + return ud->header.n_rows; +} + +py_ObjectRef py_array2d_getitem(py_Ref self, int x, int y) { + assert(self->type == tp_array2d); + c11_array2d* ud = py_touserdata(self); + return c11_array2d__get(ud, x, y); +} + +void py_array2d_setitem(py_Ref self, int x, int y, py_Ref value) { + assert(self->type == tp_array2d); + c11_array2d* ud = py_touserdata(self); + c11_array2d__set(ud, x, y, value); } \ No newline at end of file diff --git a/src/modules/conio.c b/src/modules/conio.c index 12ca2af4..78f543c0 100644 --- a/src/modules/conio.c +++ b/src/modules/conio.c @@ -1,5 +1,4 @@ #include "pocketpy/pocketpy.h" -#include #if PY_SYS_PLATFORM == 0 diff --git a/src/modules/enum.c b/src/modules/enum.c index 9c7a5659..f0601f48 100644 --- a/src/modules/enum.c +++ b/src/modules/enum.c @@ -16,7 +16,7 @@ static bool Enum__wrapper_field(py_Name name, py_Ref value, void* ctx) { } static void Enum__on_end_subclass(py_TypeInfo* derived_ti) { - derived_ti->is_sealed = true; + derived_ti->is_final = true; py_applydict(&derived_ti->self, Enum__wrapper_field, &derived_ti->self); } diff --git a/src/modules/lz4.c b/src/modules/lz4.c index 514fa5f8..2e01ac59 100644 --- a/src/modules/lz4.c +++ b/src/modules/lz4.c @@ -1,6 +1,5 @@ #ifdef PK_BUILD_MODULE_LZ4 -#include #include #include #include "pocketpy/pocketpy.h" diff --git a/src/modules/math.c b/src/modules/math.c index 294350e7..4ef3f81e 100644 --- a/src/modules/math.c +++ b/src/modules/math.c @@ -1,8 +1,4 @@ #include "pocketpy/pocketpy.h" - -#include "pocketpy/common/utils.h" -#include "pocketpy/objects/object.h" -#include "pocketpy/common/sstream.h" #include "pocketpy/interpreter/vm.h" #include diff --git a/src/modules/os.c b/src/modules/os.c index abeb1e06..cceb1718 100644 --- a/src/modules/os.c +++ b/src/modules/os.c @@ -2,7 +2,7 @@ #include "pocketpy/pocketpy.h" #include "pocketpy/interpreter/vm.h" -#if PK_ENABLE_OS == 1 +#if PK_ENABLE_OS #include diff --git a/src/modules/pickle.c b/src/modules/pickle.c index 497e0fb1..8a0ce1cd 100644 --- a/src/modules/pickle.c +++ b/src/modules/pickle.c @@ -656,7 +656,7 @@ bool py_pickle_loads_body(const unsigned char* p, int memo_length, c11_smallmap_ case PKL_ARRAY2D: { int n_cols = pkl__read_int(&p); int n_rows = pkl__read_int(&p); - c11_array2d* arr = py_newarray2d(py_pushtmp(), n_cols, n_rows); + c11_array2d* arr = c11_newarray2d(py_pushtmp(), n_cols, n_rows); int total_size = arr->header.numel * sizeof(py_TValue); memcpy(arr->data, p, total_size); for(int i = 0; i < arr->header.numel; i++) { diff --git a/src/modules/pkpy.c b/src/modules/pkpy.c index 58bbc8a6..564551a7 100644 --- a/src/modules/pkpy.c +++ b/src/modules/pkpy.c @@ -106,6 +106,8 @@ static bool pkpy_watchdog_end(int argc, py_Ref argv) { } #endif +#if PK_ENABLE_THREADS + typedef struct c11_ComputeThread c11_ComputeThread; typedef struct { @@ -456,6 +458,8 @@ static void pk_ComputeThread__register(py_Ref mod) { py_bindmethod(type, "eval", ComputeThread_eval); } +#endif // PK_ENABLE_THREADS + static void pkpy_configmacros_add(py_Ref dict, const char* key, int val) { assert(dict->type == tp_dict); py_TValue tmp; @@ -508,11 +512,14 @@ void pk__add_module_pkpy() { py_bindfunc(mod, "watchdog_end", pkpy_watchdog_end); #endif +#if PK_ENABLE_THREADS pk_ComputeThread__register(mod); +#endif py_Ref configmacros = py_emplacedict(mod, py_name("configmacros")); py_newdict(configmacros); pkpy_configmacros_add(configmacros, "PK_ENABLE_OS", PK_ENABLE_OS); + pkpy_configmacros_add(configmacros, "PK_ENABLE_THREADS", PK_ENABLE_THREADS); pkpy_configmacros_add(configmacros, "PK_ENABLE_DETERMINISM", PK_ENABLE_DETERMINISM); pkpy_configmacros_add(configmacros, "PK_ENABLE_WATCHDOG", PK_ENABLE_WATCHDOG); pkpy_configmacros_add(configmacros, "PK_GC_MIN_THRESHOLD", PK_GC_MIN_THRESHOLD); diff --git a/src/modules/random.c b/src/modules/random.c index babd3aed..ef8bdd9c 100644 --- a/src/modules/random.c +++ b/src/modules/random.c @@ -145,7 +145,7 @@ static bool Random__init__(int argc, py_Ref argv) { // do nothing } else if(argc == 2) { mt19937* ud = py_touserdata(py_arg(0)); - if(!py_isnone(&argv[1])){ + if(!py_isnone(&argv[1])) { PY_CHECK_ARG_TYPE(1, tp_int); py_i64 seed = py_toint(py_arg(1)); mt19937__seed(ud, (uint32_t)seed); @@ -328,4 +328,32 @@ __ERROR: #undef MATRIX_A #undef UPPER_MASK #undef LOWER_MASK -#undef ADD_INST_BOUNDMETHOD \ No newline at end of file +#undef ADD_INST_BOUNDMETHOD + +void py_newRandom(py_OutRef out) { + py_Type type = py_gettype("random", py_name("Random")); + assert(type != 0); + mt19937* ud = py_newobject(out, type, 0, sizeof(mt19937)); + mt19937__ctor(ud); +} + +void py_Random_seed(py_Ref self, py_i64 seed) { + mt19937* ud = py_touserdata(self); + mt19937__seed(ud, (uint32_t)seed); +} + +py_f64 py_Random_random(py_Ref self) { + mt19937* ud = py_touserdata(self); + return mt19937__random(ud); +} + +py_f64 py_Random_uniform(py_Ref self, py_f64 a, py_f64 b) { + mt19937* ud = py_touserdata(self); + return mt19937__uniform(ud, a, b); +} + +py_i64 py_Random_randint(py_Ref self, py_i64 a, py_i64 b) { + mt19937* ud = py_touserdata(self); + if(a > b) { c11__abort("randint(a, b): a must be less than or equal to b"); } + return mt19937__randint(ud, a, b); +} \ No newline at end of file diff --git a/src/modules/traceback.c b/src/modules/traceback.c index 42d97dcb..5fec7afb 100644 --- a/src/modules/traceback.c +++ b/src/modules/traceback.c @@ -1,6 +1,4 @@ -#include "pocketpy/objects/base.h" #include "pocketpy/pocketpy.h" -#include static bool traceback_format_exc(int argc, py_Ref argv) { PY_CHECK_ARGC(0); diff --git a/src/modules/vmath.c b/src/modules/vmath.c index e0badc05..428ea2e0 100644 --- a/src/modules/vmath.c +++ b/src/modules/vmath.c @@ -58,25 +58,23 @@ c11_vec2i py_tovec2i(py_Ref self) { void py_newvec3(py_OutRef out, c11_vec3 v) { out->type = tp_vec3; out->is_ptr = false; - c11_vec3* data = (c11_vec3*)(&out->extra); - *data = v; + out->_vec3 = v; } c11_vec3 py_tovec3(py_Ref self) { assert(self->type == tp_vec3); - return *(c11_vec3*)(&self->extra); + return self->_vec3; } void py_newvec3i(py_OutRef out, c11_vec3i v) { out->type = tp_vec3i; out->is_ptr = false; - c11_vec3i* data = (c11_vec3i*)(&out->extra); - *data = v; + out->_vec3i = v; } c11_vec3i py_tovec3i(py_Ref self) { assert(self->type == tp_vec3i); - return *(c11_vec3i*)(&self->extra); + return self->_vec3i; } c11_mat3x3* py_newmat3x3(py_OutRef out) { @@ -1013,6 +1011,14 @@ static bool color32__repr__(int argc, py_Ref argv) { return true; } +static bool color32__hash__(int argc, py_Ref argv) { + PY_CHECK_ARGC(1); + c11_color32 color = py_tocolor32(argv); + uint32_t* color_int = (uint32_t*)&color; + py_newint(py_retval(), *color_int); + return true; +} + static bool color32_ansi_fg(int argc, py_Ref argv) { PY_CHECK_ARGC(2); c11_color32 color = py_tocolor32(argv); @@ -1258,6 +1264,7 @@ void pk__add_module_vmath() { py_bindmagic(color32, __repr__, color32__repr__); py_bindmagic(color32, __eq__, color32__eq__); py_bindmagic(color32, __ne__, color32__ne__); + py_bindmagic(color32, __hash__, color32__hash__); py_bindproperty(color32, "r", color32__r, NULL); py_bindproperty(color32, "g", color32__g, NULL); py_bindproperty(color32, "b", color32__b, NULL); diff --git a/src/objects/codeobject.c b/src/objects/codeobject.c index 9e441527..b0fe51d5 100644 --- a/src/objects/codeobject.c +++ b/src/objects/codeobject.c @@ -2,6 +2,7 @@ #include "pocketpy/common/utils.h" #include "pocketpy/pocketpy.h" #include +#include void Bytecode__set_signed_arg(Bytecode* self, int arg) { self->arg = (int16_t)arg; diff --git a/src/objects/namedict.c b/src/objects/namedict.c index b95c4448..86f006f6 100644 --- a/src/objects/namedict.c +++ b/src/objects/namedict.c @@ -4,21 +4,30 @@ #include #include #include -#include + +#define HASH_PROBE_0(__k, ok, i) \ + ok = false; \ + i = (uintptr_t)(__k)&self->mask; \ + do { \ + if(self->items[i].key == (__k)) { \ + ok = true; \ + break; \ + } \ + if(self->items[i].key == NULL) break; \ + i = (5 * i + 1) & self->mask; \ + } while(true); #define HASH_PROBE_1(__k, ok, i) \ ok = false; \ - i = (uintptr_t)(__k) & self->mask; \ + i = (uintptr_t)(__k)&self->mask; \ while(self->items[i].key != NULL) { \ if(self->items[i].key == (__k)) { \ ok = true; \ break; \ } \ - i = (i + 1) & self->mask; \ + i = (5 * i + 1) & self->mask; \ } -#define HASH_PROBE_0 HASH_PROBE_1 - static void NameDict__set_capacity_and_alloc_items(NameDict* self, int val) { self->capacity = val; self->critical_size = val * self->load_factor; @@ -101,18 +110,24 @@ bool NameDict__del(NameDict* self, py_Name key) { self->items[i].key = NULL; self->items[i].value = *py_NIL(); self->length--; - // tidy - uintptr_t pre_z = i; - uintptr_t z = (i + 1) & self->mask; - while(self->items[z].key != NULL) { - uintptr_t h = (uintptr_t)self->items[z].key & self->mask; - if(h != i) break; - // std::swap(_items[pre_z], _items[z]); - NameDict_KV tmp = self->items[pre_z]; - self->items[pre_z] = self->items[z]; - self->items[z] = tmp; - pre_z = z; - z = (z + 1) & self->mask; + /* tidy */ + uint32_t posToRemove = i; + uint32_t posToShift = posToRemove; + while(true) { + posToShift = (5 * posToShift + 1) & self->mask; + if(self->items[posToShift].key == NULL) break; + uintptr_t hash_z = (uintptr_t)self->items[posToShift].key; + uintptr_t insertPos = hash_z & self->mask; + bool cond1 = insertPos <= posToRemove; + bool cond2 = posToRemove <= posToShift; + if((cond1 && cond2) || + // chain wrapped around capacity + (posToShift < insertPos && (cond1 || cond2))) { + NameDict_KV tmp = self->items[posToRemove]; + self->items[posToRemove] = self->items[posToShift]; + self->items[posToShift] = tmp; + posToRemove = posToShift; + } } return true; } diff --git a/src/objects/object.c b/src/objects/object.c index 1e104aed..1a3f7ef4 100644 --- a/src/objects/object.c +++ b/src/objects/object.c @@ -2,14 +2,16 @@ #include "pocketpy/pocketpy.h" #include -void* PyObject__userdata(PyObject* self) { return self->flex + PK_OBJ_SLOTS_SIZE(self->slots); } +PK_INLINE void* PyObject__userdata(PyObject* self) { + return self->flex + PK_OBJ_SLOTS_SIZE(self->slots); +} -NameDict* PyObject__dict(PyObject* self) { +PK_INLINE NameDict* PyObject__dict(PyObject* self) { assert(self->slots == -1); return (NameDict*)(self->flex); } -py_TValue* PyObject__slots(PyObject* self) { +PK_INLINE py_TValue* PyObject__slots(PyObject* self) { assert(self->slots >= 0); return (py_TValue*)(self->flex); } \ No newline at end of file diff --git a/src/public/cast.c b/src/public/cast.c index 4deabef1..39d8fb66 100644 --- a/src/public/cast.c +++ b/src/public/cast.c @@ -9,6 +9,8 @@ int64_t py_toint(py_Ref self) { return self->_i64; } +void* py_totrivial(py_Ref self) { return &self->_chars; } + double py_tofloat(py_Ref self) { assert(self->type == tp_float); return self->_f64; diff --git a/src/public/internal.c b/src/public/internal.c index c7be8853..82a5a2b2 100644 --- a/src/public/internal.c +++ b/src/public/internal.c @@ -30,9 +30,7 @@ void py_initialize() { bool is_little_endian = *(char*)&x == 1; if(!is_little_endian) c11__abort("is_little_endian != true"); - // check py_TValue; 16 bytes to make py_arg() macro work - static_assert(sizeof(py_CFunction) <= 8, "sizeof(py_CFunction) > 8"); - static_assert(sizeof(py_TValue) == 16, "sizeof(py_TValue) != 16"); + static_assert(sizeof(py_TValue) == 24, "sizeof(py_TValue) != 24"); static_assert(offsetof(py_TValue, extra) == 4, "offsetof(py_TValue, extra) != 4"); pk_current_vm = pk_all_vm[0] = &pk_default_vm; @@ -47,6 +45,12 @@ void py_initialize() { pk_initialized = true; } +void* py_malloc(size_t size) { return PK_MALLOC(size); } + +void* py_realloc(void* ptr, size_t size) { return PK_REALLOC(ptr, size); } + +void py_free(void* ptr) { PK_FREE(ptr); } + py_GlobalRef py_True() { return &_True; } py_GlobalRef py_False() { return &_False; } @@ -178,7 +182,7 @@ bool py_vectorcall(uint16_t argc, uint16_t kwargc) { return VM__vectorcall(pk_current_vm, argc, kwargc, false) != RES_ERROR; } -py_Ref py_retval() { return &pk_current_vm->last_retval; } +PK_INLINE py_Ref py_retval() { return &pk_current_vm->last_retval; } bool py_pushmethod(py_Name name) { bool ok = pk_loadmethod(py_peek(-1), name); @@ -223,7 +227,21 @@ bool pk_loadmethod(py_StackRef self, py_Name name) { self_bak = *self; } - py_Ref cls_var = py_tpfindname(type, name); + py_TypeInfo* ti = pk_typeinfo(type); + + if(ti->getunboundmethod) { + bool ok = ti->getunboundmethod(self, name); + if(ok) { + assert(py_retval()->type == tp_nativefunc || py_retval()->type == tp_function); + self[0] = *py_retval(); + self[1] = self_bak; + return true; + } else { + return false; + } + } + + py_Ref cls_var = pk_tpfindname(ti, name); if(cls_var != NULL) { switch(cls_var->type) { case tp_function: @@ -238,7 +256,7 @@ bool pk_loadmethod(py_StackRef self, py_Name name) { break; case tp_classmethod: self[0] = *py_getslot(cls_var, 0); - self[1] = pk_typeinfo(type)->self; + self[1] = ti->self; break; default: c11__unreachable(); } diff --git a/src/public/modules.c b/src/public/modules.c index 7eb54c46..ccf740c9 100644 --- a/src/public/modules.c +++ b/src/public/modules.c @@ -28,32 +28,8 @@ static void py_ModuleInfo__dtor(py_ModuleInfo* mi) { c11_string__delete(mi->path); } -static bool module__name__(int argc, py_Ref argv) { - PY_CHECK_ARGC(1); - py_ModuleInfo* mi = py_touserdata(argv); - py_newstrv(py_retval(), c11_string__sv(mi->name)); - return true; -} - -static bool module__package__(int argc, py_Ref argv) { - PY_CHECK_ARGC(1); - py_ModuleInfo* mi = py_touserdata(argv); - py_newstrv(py_retval(), c11_string__sv(mi->package)); - return true; -} - -static bool module__path__(int argc, py_Ref argv) { - PY_CHECK_ARGC(1); - py_ModuleInfo* mi = py_touserdata(argv); - py_newstrv(py_retval(), c11_string__sv(mi->path)); - return true; -} - py_Type pk_module__register() { py_Type type = pk_newtype("module", tp_object, NULL, (py_Dtor)py_ModuleInfo__dtor, false, true); - py_bindproperty(type, "__name__", module__name__, NULL); - py_bindproperty(type, "__package__", module__package__, NULL); - py_bindproperty(type, "__path__", module__path__, NULL); return type; } @@ -85,6 +61,13 @@ py_Ref py_newmodule(const char* path) { BinTree__set(&pk_current_vm->modules, (void*)path, py_retval()); py_GlobalRef retval = py_getmodule(path); mi->self = retval; + + // setup __name__ + py_newstrv(py_emplacedict(retval, __name__), c11_string__sv(mi->name)); + // setup __package__ + py_newstrv(py_emplacedict(retval, __package__), c11_string__sv(mi->package)); + // setup __path__ + py_newstrv(py_emplacedict(retval, __path__), c11_string__sv(mi->path)); return retval; } @@ -146,6 +129,15 @@ int py_import(const char* path_cstr) { return true; } + if(vm->callbacks.lazyimport) { + py_GlobalRef lazymod = vm->callbacks.lazyimport(path_cstr); + if(lazymod) { + c11__rtassert(py_istype(lazymod, tp_module)); + py_assign(py_retval(), lazymod); + return 1; + } + } + // try import c11_string* slashed_path = c11_sv__replace(path, '.', PK_PLATFORM_SEP); c11_string* filename = c11_string__new3("%s.py", slashed_path->data); diff --git a/src/public/py_array.c b/src/public/py_array.c index 478333e7..bf55dcc5 100644 --- a/src/public/py_array.c +++ b/src/public/py_array.c @@ -1,13 +1,8 @@ #include "pocketpy/pocketpy.h" #include "pocketpy/objects/object.h" +#include "pocketpy/objects/iterator.h" #include "pocketpy/interpreter/vm.h" -typedef struct array_iterator { - py_TValue* p; - int length; - int index; -} array_iterator; - int pk_arrayview(py_Ref self, py_TValue** p) { if(self->type == tp_list) { *p = py_list_data(self); @@ -46,18 +41,6 @@ bool pk_wrapper__arrayequal(py_Type type, int argc, py_Ref argv) { return true; } -bool pk_arrayiter(py_Ref val) { - py_TValue* p; - int length = pk_arrayview(val, &p); - if(length == -1) return TypeError("expected list or tuple, got %t", val->type); - array_iterator* ud = py_newobject(py_retval(), tp_array_iterator, 1, sizeof(array_iterator)); - ud->p = p; - ud->length = length; - ud->index = 0; - py_setslot(py_retval(), 0, val); // keep a reference to the object - return true; -} - bool pk_arraycontains(py_Ref self, py_Ref val) { py_TValue* p; int length = pk_arrayview(self, &p); @@ -74,25 +57,39 @@ bool pk_arraycontains(py_Ref self, py_Ref val) { return true; } -static bool array_iterator__iter__(int argc, py_Ref argv) { +bool list_iterator__next__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); - *py_retval() = *argv; - return true; -} - -static bool array_iterator__next__(int argc, py_Ref argv) { - PY_CHECK_ARGC(1); - array_iterator* ud = py_touserdata(argv); - if(ud->index < ud->length) { - *py_retval() = ud->p[ud->index++]; + list_iterator* ud = py_touserdata(argv); + if(ud->index < ud->vec->length) { + py_TValue* res = c11__at(py_TValue, ud->vec, ud->index); + py_assign(py_retval(), res); + ud->index++; return true; } return StopIteration(); } -py_Type pk_array_iterator__register() { - py_Type type = pk_newtype("array_iterator", tp_object, NULL, NULL, false, true); - py_bindmagic(type, __iter__, array_iterator__iter__); - py_bindmagic(type, __next__, array_iterator__next__); +bool tuple_iterator__next__(int argc, py_Ref argv) { + PY_CHECK_ARGC(1); + tuple_iterator* ud = py_touserdata(argv); + if(ud->index < ud->length) { + py_assign(py_retval(), ud->p + ud->index); + ud->index++; + return true; + } + return StopIteration(); +} + +py_Type pk_list_iterator__register() { + py_Type type = pk_newtype("list_iterator", tp_object, NULL, NULL, false, true); + py_bindmagic(type, __iter__, pk_wrapper__self); + py_bindmagic(type, __next__, list_iterator__next__); + return type; +} + +py_Type pk_tuple_iterator__register() { + py_Type type = pk_newtype("tuple_iterator", tp_object, NULL, NULL, false, true); + py_bindmagic(type, __iter__, pk_wrapper__self); + py_bindmagic(type, __next__, tuple_iterator__next__); return type; } diff --git a/src/public/py_dict.c b/src/public/py_dict.c index 347b1f26..a3870cca 100644 --- a/src/public/py_dict.c +++ b/src/public/py_dict.c @@ -5,6 +5,16 @@ #include "pocketpy/interpreter/types.h" #include "pocketpy/interpreter/vm.h" +typedef struct { + Dict* dict; // weakref for slot 0 + Dict dict_backup; + DictEntry* curr; + DictEntry* end; + int mode; // 0: keys, 1: values, 2: items +} DictIterator; + +#define Dict__step(x) ((x) < mask ? (x) + 1 : 0) + static uint32_t Dict__next_cap(uint32_t cap) { switch(cap) { case 7: return 17; @@ -51,19 +61,36 @@ static uint32_t Dict__next_cap(uint32_t cap) { } } - - -typedef struct { - DictEntry* curr; - DictEntry* end; - int mode; // 0: keys, 1: values, 2: items -} DictIterator; +static uint64_t Dict__hash_2nd(uint64_t key) { + // https://gist.github.com/badboy/6267743 + key = (~key) + (key << 21); // key = (key << 21) - key - 1 + key = key ^ (key >> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >> 28); + key = key + (key << 31); + return key; +} static void Dict__ctor(Dict* self, uint32_t capacity, int entries_capacity) { self->length = 0; self->capacity = capacity; - self->indices = PK_MALLOC(self->capacity * sizeof(DictIndex)); - memset(self->indices, -1, self->capacity * sizeof(DictIndex)); + + size_t indices_size; + if(self->capacity < UINT16_MAX) { + self->index_is_short = true; + indices_size = self->capacity * sizeof(uint16_t); + self->null_index_value = UINT16_MAX; + } else { + self->index_is_short = false; + indices_size = self->capacity * sizeof(uint32_t); + self->null_index_value = UINT32_MAX; + } + + self->indices = PK_MALLOC(indices_size); + memset(self->indices, -1, indices_size); + c11_vector__ctor(&self->entries, sizeof(DictEntry)); c11_vector__reserve(&self->entries, entries_capacity); } @@ -75,70 +102,118 @@ static void Dict__dtor(Dict* self) { c11_vector__dtor(&self->entries); } -static bool Dict__try_get(Dict* self, py_TValue* key, DictEntry** out) { - py_i64 hash; - if(!py_hash(key, &hash)) return false; - int idx = (uint64_t)hash % self->capacity; - for(int i = 0; i < PK_DICT_MAX_COLLISION; i++) { - int idx2 = self->indices[idx]._[i]; - if(idx2 == -1) continue; +static uint32_t Dict__get_index(Dict* self, uint32_t index) { + if(self->index_is_short) { + uint16_t* indices = self->indices; + return indices[index]; + } else { + uint32_t* indices = self->indices; + return indices[index]; + } +} + +static void Dict__swap_null_index(Dict* self, uint32_t pre_z, uint32_t z) { + if(self->index_is_short) { + uint16_t* indices = self->indices; + assert(indices[pre_z] == UINT16_MAX); + indices[pre_z] = indices[z]; + indices[z] = UINT16_MAX; + } else { + uint32_t* indices = self->indices; + assert(indices[pre_z] == UINT32_MAX); + indices[pre_z] = indices[z]; + indices[z] = UINT32_MAX; + } +} + +static void Dict__set_index(Dict* self, uint32_t index, uint32_t value) { + if(self->index_is_short) { + uint16_t* indices = self->indices; + indices[index] = (uint16_t)value; + } else { + uint32_t* indices = self->indices; + indices[index] = value; + } +} + +static bool Dict__probe(Dict* self, + py_TValue* key, + uint64_t* p_hash, + uint32_t* p_idx, + DictEntry** p_entry) { + py_i64 h_user; + if(!py_hash(key, &h_user)) return false; + if(py_isstr(key)) { + *p_hash = (uint64_t)h_user; + } else { + *p_hash = Dict__hash_2nd((uint64_t)h_user); + } + uint32_t mask = self->capacity - 1; + uint32_t idx = (*p_hash) % self->capacity; + while(true) { + uint32_t idx2 = Dict__get_index(self, idx); + if(idx2 == self->null_index_value) break; DictEntry* entry = c11__at(DictEntry, &self->entries, idx2); - if(entry->hash == (uint64_t)hash) { + if(entry->hash == (*p_hash)) { int res = py_equal(&entry->key, key); if(res == 1) { - *out = entry; + *p_idx = idx; + *p_entry = entry; return true; } if(res == -1) return false; // error } + // try next index + idx = Dict__step(idx); } - *out = NULL; + // not found + *p_idx = idx; + *p_entry = NULL; return true; } +static bool Dict__try_get(Dict* self, py_TValue* key, DictEntry** out) { + uint64_t hash; + uint32_t idx; + return Dict__probe(self, key, &hash, &idx, out); +} + static void Dict__clear(Dict* self) { - memset(self->indices, -1, self->capacity * sizeof(DictIndex)); + size_t indices_size = self->index_is_short ? self->capacity * sizeof(uint16_t) + : self->capacity * sizeof(uint32_t); + memset(self->indices, -1, indices_size); c11_vector__clear(&self->entries); self->length = 0; } static void Dict__rehash_2x(Dict* self) { Dict old_dict = *self; - uint32_t new_capacity = self->capacity; - -__RETRY: - // use next capacity - new_capacity = Dict__next_cap(new_capacity); + uint32_t new_capacity = Dict__next_cap(old_dict.capacity); + uint32_t mask = new_capacity - 1; // create a new dict with new capacity Dict__ctor(self, new_capacity, old_dict.entries.capacity); // move entries from old dict to new dict for(int i = 0; i < old_dict.entries.length; i++) { DictEntry* old_entry = c11__at(DictEntry, &old_dict.entries, i); - if(py_isnil(&old_entry->key)) continue; - int idx = old_entry->hash % new_capacity; - bool success = false; - for(int i = 0; i < PK_DICT_MAX_COLLISION; i++) { - int idx2 = self->indices[idx]._[i]; - if(idx2 == -1) { - // insert new entry (empty slot) + if(py_isnil(&old_entry->key)) continue; // skip deleted + uint32_t idx = old_entry->hash % new_capacity; + while(true) { + uint32_t idx2 = Dict__get_index(self, idx); + if(idx2 == self->null_index_value) { c11_vector__push(DictEntry, &self->entries, *old_entry); - self->indices[idx]._[i] = self->entries.length - 1; + Dict__set_index(self, idx, self->entries.length - 1); self->length++; - success = true; break; } - } - if(!success) { - Dict__dtor(self); - goto __RETRY; + // try next index + idx = Dict__step(idx); } } - // done Dict__dtor(&old_dict); } static void Dict__compact_entries(Dict* self) { - int* mappings = PK_MALLOC(self->entries.length * sizeof(int)); + uint32_t* mappings = PK_MALLOC(self->entries.length * sizeof(uint32_t)); int n = 0; for(int i = 0; i < self->entries.length; i++) { @@ -153,96 +228,97 @@ static void Dict__compact_entries(Dict* self) { } self->entries.length = n; // update indices - for(uint32_t i = 0; i < self->capacity; i++) { - for(int j = 0; j < PK_DICT_MAX_COLLISION; j++) { - int idx = self->indices[i]._[j]; - if(idx == -1) continue; - self->indices[i]._[j] = mappings[idx]; - } + for(uint32_t idx = 0; idx < self->capacity; idx++) { + uint32_t idx2 = Dict__get_index(self, idx); + if(idx2 == self->null_index_value) continue; + Dict__set_index(self, idx, mappings[idx2]); } PK_FREE(mappings); } static bool Dict__set(Dict* self, py_TValue* key, py_TValue* val) { - py_i64 hash; - if(!py_hash(key, &hash)) return false; - int idx = (uint64_t)hash % self->capacity; - int bad_hash_count = 0; - for(int i = 0; i < PK_DICT_MAX_COLLISION; i++) { - int idx2 = self->indices[idx]._[i]; - if(idx2 == -1) { - // insert new entry - DictEntry* new_entry = c11_vector__emplace(&self->entries); - new_entry->hash = (uint64_t)hash; - new_entry->key = *key; - new_entry->val = *val; - self->indices[idx]._[i] = self->entries.length - 1; - self->length++; - return true; - } + uint64_t hash; + uint32_t idx; + DictEntry* entry; + if(!Dict__probe(self, key, &hash, &idx, &entry)) return false; + if(entry) { // update existing entry - DictEntry* entry = c11__at(DictEntry, &self->entries, idx2); - // check if they have the same hash - if(entry->hash == (uint64_t)hash) { - // check if they are equal - int res = py_equal(&entry->key, key); - if(res == 1) { - entry->val = *val; - return true; - } - if(res == -1) return false; // error - // res == 0 - bad_hash_count++; - } + entry->val = *val; + return true; } - // no empty slot found - if(bad_hash_count == PK_DICT_MAX_COLLISION) { - // all `PK_DICT_MAX_COLLISION` slots have the same hash but different keys - // we are unable to solve this collision via rehashing - return RuntimeError("dict: %d/%d/%d: maximum collision reached (hash=%i)", - self->entries.length, - self->entries.capacity, - self->capacity, - hash); - } - - if(self->capacity >= (uint32_t)self->entries.length * 10) { - return RuntimeError("dict: %d/%d/%d: minimum load factor reached", - self->entries.length, - self->entries.capacity, - self->capacity); - } - Dict__rehash_2x(self); - return Dict__set(self, key, val); + // insert new entry + DictEntry* new_entry = c11_vector__emplace(&self->entries); + new_entry->hash = hash; + new_entry->key = *key; + new_entry->val = *val; + Dict__set_index(self, idx, self->entries.length - 1); + self->length++; + // check if we need to rehash + float load_factor = (float)self->length / self->capacity; + if(load_factor > (self->index_is_short ? 0.3f : 0.4f)) Dict__rehash_2x(self); + return true; } /// Delete an entry from the dict. /// -1: error, 0: not found, 1: found and deleted static int Dict__pop(Dict* self, py_Ref key) { - py_i64 hash; - if(!py_hash(key, &hash)) return -1; - int idx = (uint64_t)hash % self->capacity; - for(int i = 0; i < PK_DICT_MAX_COLLISION; i++) { - int idx2 = self->indices[idx]._[i]; - if(idx2 == -1) continue; - DictEntry* entry = c11__at(DictEntry, &self->entries, idx2); - if(entry->hash == (uint64_t)hash) { - int res = py_equal(&entry->key, key); - if(res == 1) { - *py_retval() = entry->val; - py_newnil(&entry->key); - self->indices[idx]._[i] = -1; - self->length--; - if(self->length < self->entries.length / 2) Dict__compact_entries(self); - return 1; - } - if(res == -1) return -1; // error + // Dict__log_index(self, "before pop"); + uint64_t hash; + uint32_t idx; + DictEntry* entry; + if(!Dict__probe(self, key, &hash, &idx, &entry)) return -1; + if(!entry) return 0; // not found + + // found the entry, delete and return it + py_assign(py_retval(), &entry->val); + Dict__set_index(self, idx, self->null_index_value); + py_newnil(&entry->key); + py_newnil(&entry->val); + self->length--; + + /* tidy */ + // https://github.com/OpenHFT/Chronicle-Map/blob/820573a68471509ffc1b0584454f4a67c0be1b84/src/main/java/net/openhft/chronicle/hash/impl/CompactOffHeapLinearHashTable.java#L156 + uint32_t mask = self->capacity - 1; + uint32_t posToRemove = idx; + uint32_t posToShift = posToRemove; + // int probe_count = 0; + // int swap_count = 0; + while(true) { + posToShift = Dict__step(posToShift); + uint32_t idx_z = Dict__get_index(self, posToShift); + if(idx_z == self->null_index_value) break; + uint64_t hash_z = c11__at(DictEntry, &self->entries, idx_z)->hash; + uint32_t insertPos = (uint64_t)hash_z % self->capacity; + // the following condition essentially means circular permutations + // of three (r = posToRemove, s = posToShift, i = insertPos) + // positions are accepted: + // [...i..r...s.] or + // [...r..s...i.] or + // [...s..i...r.] + bool cond1 = insertPos <= posToRemove; + bool cond2 = posToRemove <= posToShift; + if((cond1 && cond2) || + // chain wrapped around capacity + (posToShift < insertPos && (cond1 || cond2))) { + Dict__swap_null_index(self, posToRemove, posToShift); + posToRemove = posToShift; + // swap_count++; } + // probe_count++; } - return 0; + // printf("Dict__pop: probe_count=%d, swap_count=%d\n", probe_count, swap_count); + // compact entries if necessary + if(self->entries.length > 16 && (self->length < self->entries.length >> 1)) { + Dict__compact_entries(self); // compact entries + } + // Dict__log_index(self, "after pop"); + return 1; } static void DictIterator__ctor(DictIterator* self, Dict* dict, int mode) { + assert(mode >= 0 && mode <= 2); + self->dict = dict; + self->dict_backup = *dict; // backup the dict self->curr = dict->entries.data; self->end = self->curr + dict->entries.length; self->mode = mode; @@ -257,18 +333,22 @@ static DictEntry* DictIterator__next(DictIterator* self) { return retval; } +static bool DictIterator__modified(DictIterator* self) { + return memcmp(self->dict, &self->dict_backup, sizeof(Dict)) != 0; +} + /////////////////////////////// static bool dict__new__(int argc, py_Ref argv) { py_Type cls = py_totype(argv); int slots = cls == tp_dict ? 0 : -1; Dict* ud = py_newobject(py_retval(), cls, slots, sizeof(Dict)); - Dict__ctor(ud, 7, 8); + Dict__ctor(ud, 17, 4); return true; } void py_newdict(py_OutRef out) { Dict* ud = py_newobject(out, tp_dict, 0, sizeof(Dict)); - Dict__ctor(ud, 7, 8); + Dict__ctor(ud, 17, 4); } static bool dict__init__(int argc, py_Ref argv) { @@ -426,12 +506,17 @@ static bool dict_copy(int argc, py_Ref argv) { PY_CHECK_ARGC(1); Dict* self = py_touserdata(argv); Dict* new_dict = py_newobject(py_retval(), tp_dict, 0, sizeof(Dict)); - new_dict->capacity = self->capacity; new_dict->length = self->length; + new_dict->capacity = self->capacity; + new_dict->null_index_value = self->null_index_value; + new_dict->index_is_short = self->index_is_short; + // copy entries new_dict->entries = c11_vector__copy(&self->entries); // copy indices - new_dict->indices = PK_MALLOC(new_dict->capacity * sizeof(DictIndex)); - memcpy(new_dict->indices, self->indices, new_dict->capacity * sizeof(DictIndex)); + size_t indices_size = self->index_is_short ? self->capacity * sizeof(uint16_t) + : self->capacity * sizeof(uint32_t); + new_dict->indices = PK_MALLOC(indices_size); + memcpy(new_dict->indices, self->indices, indices_size); return true; } @@ -525,9 +610,10 @@ py_Type pk_dict__register() { } ////////////////////////// -static bool dict_items__next__(int argc, py_Ref argv) { +bool dict_items__next__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); DictIterator* iter = py_touserdata(py_arg(0)); + if(DictIterator__modified(iter)) return RuntimeError("dictionary modified during iteration"); DictEntry* entry = (DictIterator__next(iter)); if(entry) { switch(iter->mode) { @@ -641,4 +727,4 @@ bool py_dict_apply(py_Ref self, bool (*f)(py_Ref, py_Ref, void*), void* ctx) { return true; } -#undef PK_DICT_MAX_COLLISION \ No newline at end of file +#undef Dict__step \ No newline at end of file diff --git a/src/public/py_list.c b/src/public/py_list.c index 35585a6e..1e9142fb 100644 --- a/src/public/py_list.c +++ b/src/public/py_list.c @@ -3,6 +3,7 @@ #include "pocketpy/common/utils.h" #include "pocketpy/interpreter/vm.h" #include "pocketpy/interpreter/types.h" +#include "pocketpy/objects/iterator.h" #include "pocketpy/common/sstream.h" void py_newlist(py_OutRef out) { @@ -394,7 +395,11 @@ static bool list_sort(int argc, py_Ref argv) { static bool list__iter__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); - return pk_arrayiter(argv); + list_iterator* ud = py_newobject(py_retval(), tp_list_iterator, 1, sizeof(list_iterator)); + ud->vec = py_touserdata(argv); + ud->index = 0; + py_setslot(py_retval(), 0, argv); // keep a reference to the object + return true; } static bool list__contains__(int argc, py_Ref argv) { diff --git a/src/public/py_number.c b/src/public/py_number.c index 554e783e..7e0bd5d6 100644 --- a/src/public/py_number.c +++ b/src/public/py_number.c @@ -175,6 +175,32 @@ static bool int__mod__(int argc, py_Ref argv) { return true; } +static bool float__mod__(int argc, py_Ref argv) { + PY_CHECK_ARGC(2); + py_f64 lhs = py_tofloat(&argv[0]); + py_f64 rhs; + if(try_castfloat(&argv[1], &rhs)) { + if(rhs == 0.0) return ZeroDivisionError("float modulo by zero"); + py_newfloat(py_retval(), fmod(lhs, rhs)); + return true; + } + py_newnotimplemented(py_retval()); + return true; +} + +static bool float__rmod__(int argc, py_Ref argv) { + PY_CHECK_ARGC(2); + py_f64 rhs = py_tofloat(&argv[0]); + py_f64 lhs; + if(try_castfloat(&argv[1], &lhs)) { + if(rhs == 0.0) return ZeroDivisionError("float modulo by zero"); + py_newfloat(py_retval(), fmod(lhs, rhs)); + return true; + } + py_newnotimplemented(py_retval()); + return true; +} + static bool int__divmod__(int argc, py_Ref argv) { PY_CHECK_ARGC(2); PY_CHECK_ARG_TYPE(1, tp_int); @@ -245,24 +271,6 @@ static bool float__repr__(int argc, py_Ref argv) { return true; } -union c11_8bytes { - py_i64 _i64; - py_f64 _f64; - - union { - uint32_t upper; - uint32_t lower; - } bits; -}; - -static py_i64 c11_8bytes__hash(union c11_8bytes u) { - // https://stackoverflow.com/questions/664014/what-integer-hash-function-are-good-that-accepts-an-integer-hash-key - const uint32_t C = 2654435761; - u.bits.upper *= C; - u.bits.lower *= C; - return u._i64; -} - static bool int__hash__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); py_assign(py_retval(), argv); @@ -272,8 +280,9 @@ static bool int__hash__(int argc, py_Ref argv) { static bool float__hash__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); py_f64 val = py_tofloat(&argv[0]); - union c11_8bytes u = {._f64 = val}; - py_newint(py_retval(), c11_8bytes__hash(u)); + py_i64 h_user; + memcpy(&h_user, &val, sizeof(py_f64)); + py_newint(py_retval(), h_user); return true; } @@ -525,6 +534,10 @@ void pk_number__register() { py_bindmagic(tp_int, __mod__, int__mod__); py_bindmagic(tp_int, __divmod__, int__divmod__); + // fmod + py_bindmagic(tp_float, __mod__, float__mod__); + py_bindmagic(tp_float, __rmod__, float__rmod__); + // int.__invert__ & int. py_bindmagic(tp_int, __invert__, int__invert__); diff --git a/src/public/py_ops.c b/src/public/py_ops.c index d384092c..8a3fcaf5 100644 --- a/src/public/py_ops.c +++ b/src/public/py_ops.c @@ -1,4 +1,5 @@ #include "pocketpy/interpreter/typeinfo.h" +#include "pocketpy/interpreter/bindings.h" #include "pocketpy/interpreter/vm.h" #include "pocketpy/objects/base.h" #include "pocketpy/pocketpy.h" @@ -77,12 +78,39 @@ bool py_iter(py_Ref val) { int py_next(py_Ref val) { VM* vm = pk_current_vm; - py_Ref tmp = py_tpfindmagic(val->type, __next__); - if(!tmp) { - TypeError("'%t' object is not an iterator", val->type); - return -1; + + switch(val->type) { + case tp_generator: + if(generator__next__(1, val)) return 1; + break; + case tp_array2d_like_iterator: + if(array2d_like_iterator__next__(1, val)) return 1; + break; + case tp_list_iterator: + if(list_iterator__next__(1, val)) return 1; + break; + case tp_tuple_iterator: + if(tuple_iterator__next__(1, val)) return 1; + break; + case tp_dict_iterator: + if(dict_items__next__(1, val)) return 1; + break; + case tp_range_iterator: + if(range_iterator__next__(1, val)) return 1; + break; + case tp_str_iterator: + if(str_iterator__next__(1, val)) return 1; + break; + default: { + py_Ref tmp = py_tpfindmagic(val->type, __next__); + if(!tmp) { + TypeError("'%t' object is not an iterator", val->type); + return -1; + } + if(py_call(tmp, 1, val)) return 1; + break; + } } - if(py_call(tmp, 1, val)) return 1; if(vm->curr_exception.type == tp_StopIteration) { vm->last_retval = vm->curr_exception; py_clearexc(NULL); @@ -94,6 +122,8 @@ int py_next(py_Ref val) { bool py_getattr(py_Ref self, py_Name name) { // https://docs.python.org/3/howto/descriptor.html#invocation-from-an-instance py_TypeInfo* ti = pk_typeinfo(self->type); + if(ti->getattribute) return ti->getattribute(self, name); + py_Ref cls_var = pk_tpfindname(ti, name); if(cls_var) { // handle descriptor @@ -184,8 +214,10 @@ bool py_getattr(py_Ref self, py_Name name) { } bool py_setattr(py_Ref self, py_Name name, py_Ref val) { - py_Type type = self->type; - py_Ref cls_var = py_tpfindname(type, name); + py_TypeInfo* ti = pk_typeinfo(self->type); + if(ti->setattribute) return ti->setattribute(self, name, val); + + py_Ref cls_var = pk_tpfindname(ti, name); if(cls_var) { // handle descriptor if(py_istype(cls_var, tp_property)) { @@ -211,6 +243,9 @@ bool py_setattr(py_Ref self, py_Name name, py_Ref val) { } bool py_delattr(py_Ref self, py_Name name) { + py_TypeInfo* ti = pk_typeinfo(self->type); + if(ti->delattribute) return ti->delattribute(self, name); + if(self->is_ptr && self->_obj->slots == -1) { if(py_deldict(self, name)) return true; return AttributeError(self, name); diff --git a/src/public/py_range.c b/src/public/py_range.c index c78f93ec..b580ab57 100644 --- a/src/public/py_range.c +++ b/src/public/py_range.c @@ -68,9 +68,9 @@ static bool range_iterator__new__(int argc, py_Ref argv) { return true; } -static bool range_iterator__next__(int argc, py_Ref argv) { +bool range_iterator__next__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); - RangeIterator* ud = py_touserdata(py_arg(0)); + RangeIterator* ud = py_touserdata(argv); if(ud->range.step > 0) { if(ud->current >= ud->range.stop) return StopIteration(); } else { diff --git a/src/public/py_str.c b/src/public/py_str.c index e7145ca1..c01f2c18 100644 --- a/src/public/py_str.c +++ b/src/public/py_str.c @@ -9,7 +9,7 @@ void py_newstr(py_OutRef out, const char* data) { py_newstrv(out, (c11_sv){data, strlen(data)}); } char* py_newstrn(py_OutRef out, int size) { - if(size < 8) { + if(size < 16) { out->type = tp_str; out->is_ptr = false; c11_string* ud = (c11_string*)(&out->extra); @@ -327,7 +327,7 @@ static bool str_split(int argc, py_Ref argv) { if(argc > 2) return TypeError("split() takes at most 2 arguments"); if(argc == 1) { // sep = None - res = c11_sv__split(self, ' '); + res = c11_sv__splitwhitespace(self); discard_empty = true; } if(argc == 2) { @@ -513,7 +513,7 @@ py_Type pk_str__register() { return type; } -static bool str_iterator__next__(int argc, py_Ref argv) { +bool str_iterator__next__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); int* ud = py_touserdata(&argv[0]); int size; diff --git a/src/public/py_tuple.c b/src/public/py_tuple.c index 0471bd84..6d2e8112 100644 --- a/src/public/py_tuple.c +++ b/src/public/py_tuple.c @@ -3,6 +3,7 @@ #include "pocketpy/common/utils.h" #include "pocketpy/common/sstream.h" #include "pocketpy/objects/object.h" +#include "pocketpy/objects/iterator.h" #include "pocketpy/interpreter/vm.h" py_ObjectRef py_newtuple(py_OutRef out, int n) { @@ -144,7 +145,12 @@ static bool tuple__lt__(int argc, py_Ref argv) { static bool tuple__iter__(int argc, py_Ref argv) { PY_CHECK_ARGC(1); - return pk_arrayiter(argv); + tuple_iterator* ud = py_newobject(py_retval(), tp_tuple_iterator, 1, sizeof(tuple_iterator)); + ud->p = py_tuple_data(argv); + ud->length = py_tuple_len(argv); + ud->index = 0; + py_setslot(py_retval(), 0, argv); // keep a reference to the object + return true; } static bool tuple__contains__(int argc, py_Ref argv) { diff --git a/src/public/stack_ops.c b/src/public/stack_ops.c index 1c65000b..6a72e05f 100644 --- a/src/public/stack_ops.c +++ b/src/public/stack_ops.c @@ -8,12 +8,12 @@ py_Ref py_getreg(int i) { return pk_current_vm->reg + i; } void py_setreg(int i, py_Ref val) { pk_current_vm->reg[i] = *val; } -py_Ref py_getdict(py_Ref self, py_Name name) { +PK_INLINE py_Ref py_getdict(py_Ref self, py_Name name) { assert(self && self->is_ptr); return NameDict__try_get(PyObject__dict(self->_obj), name); } -void py_setdict(py_Ref self, py_Name name, py_Ref val) { +PK_INLINE void py_setdict(py_Ref self, py_Name name, py_Ref val) { assert(self && self->is_ptr); NameDict__set(PyObject__dict(self->_obj), name, val); } diff --git a/src/public/typecast.c b/src/public/typecast.c index a00fd491..69dbf690 100644 --- a/src/public/typecast.c +++ b/src/public/typecast.c @@ -4,7 +4,7 @@ #include "pocketpy/objects/object.h" #include "pocketpy/interpreter/vm.h" -bool py_istype(py_Ref self, py_Type type) { return self->type == type; } +PK_INLINE bool py_istype(py_Ref self, py_Type type) { return self->type == type; } bool py_checktype(py_Ref self, py_Type type) { if(self->type == type) return true; diff --git a/src/public/values.c b/src/public/values.c index 590a29ac..f696f85d 100644 --- a/src/public/values.c +++ b/src/public/values.c @@ -13,6 +13,13 @@ void py_newint(py_OutRef out, py_i64 val) { out->_i64 = val; } +void py_newtrivial(py_OutRef out, py_Type type, void* data, int size) { + out->type = type; + out->is_ptr = false; + assert(size <= 16); + memcpy(&out->_chars, data, size); +} + void py_newfloat(py_OutRef out, py_f64 val) { out->type = tp_float; out->is_ptr = false; diff --git a/src2/main.c b/src2/main.c index 10f5674d..7b28b923 100644 --- a/src2/main.c +++ b/src2/main.c @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -20,7 +19,7 @@ static char* read_file(const char* path) { fseek(file, 0, SEEK_END); long size = ftell(file); fseek(file, 0, SEEK_SET); - char* buffer = malloc(size + 1); + char* buffer = PK_MALLOC(size + 1); size = fread(buffer, 1, size, file); buffer[size] = 0; return buffer; @@ -86,7 +85,7 @@ int main(int argc, char** argv) { char* source = read_file(filename); if(source) { if(!py_exec(source, filename, EXEC_MODE, NULL)) py_printexc(); - free(source); + PK_FREE(source); } } diff --git a/tests/02_float.py b/tests/02_float.py index 06aadc6a..e2b4821c 100644 --- a/tests/02_float.py +++ b/tests/02_float.py @@ -109,3 +109,8 @@ assert abs(0.0) == 0.0 # exit(1) # except ValueError: # pass + +assert eq(10 % 4, 2) +assert eq(10.5 % 4, 2.5) +assert eq(10 % 4.5, 1.0) +assert eq(10.5 % 4.5, 1.5) \ No newline at end of file diff --git a/tests/04_str.py b/tests/04_str.py index dfe8a984..4a72746b 100644 --- a/tests/04_str.py +++ b/tests/04_str.py @@ -57,6 +57,9 @@ assert 'aa bb cccc'.split('cc') == ['aa bb ', '', ''] assert '.a.b.'.split('.') == ['', 'a', 'b', ''] assert '.a...b.'.split('.') == ['', 'a', '', '', 'b', ''] +# https://github.com/pocketpy/pocketpy/issues/378 +assert "a b \n c\td".split() == ['a', 'b', 'c', 'd'] + try: 'a'.split('') exit(1) @@ -228,3 +231,7 @@ assert f"{(1, 2, 3)}" == "(1, 2, 3)" # stack=[1,2,3,4] # assert f"{stack[2:]}" == '[3, 4]' + + +assert id('1' * 16) is not None +assert id('1' * 15) is None \ No newline at end of file diff --git a/tests/08_dict.py b/tests/08_dict.py index accd3e34..992603f7 100644 --- a/tests/08_dict.py +++ b/tests/08_dict.py @@ -115,30 +115,7 @@ assert a.pop(1) == 2 assert a.pop(1, None) is None -n = 2 ** 17 -a = {} -for i in range(n): - a[str(i)] = i - -for i in range(n): - y = a[str(i)] - -for i in range(n): - del a[str(i)] - -# namedict delete test -# class A: pass -# a = A() -# b = ['0', '1'] - -# for i in range(len(data)): -# z = data[i] -# setattr(a, str(z), i) -# b.append(z) -# if i % 3 == 0: -# y = b.pop() -# delattr(a, y) - +# test getitem d = {} for i in range(-1000, 1000): d[i] = i @@ -155,3 +132,37 @@ assert list(d) == ['1', 222, '333'] assert list(d.keys()) == ['1', 222, '333'] assert list(d.values()) == [1, 2, 3] assert list(d.items()) == [('1', 1), (222, 2), ('333', 3)] + +# test del +n = 2 ** 17 +a = {} +for i in range(n): + a[str(i)] = i +for i in range(n): + del a[str(i)] +assert len(a) == 0 + +# test del with int keys +if 0: + n = 2 ** 17 + a = {} + for i in range(n): + a[i] = i + for i in range(n): + del a[i] + assert len(a) == 0 + +####################### + +# namedict delete test +class A: pass +a = A() +b = ['0', '1'] + +for i in range(len(data)): + z = data[i] + setattr(a, str(z), i) + b.append(z) + if i % 3 == 0: + y = b.pop() + delattr(a, y) \ No newline at end of file diff --git a/tests/40_class.py b/tests/40_class.py index 845b488b..f18a4261 100644 --- a/tests/40_class.py +++ b/tests/40_class.py @@ -126,7 +126,13 @@ class MyClass: b, c = 1, 2 d = b + c + def __init__(self, m, n) -> None: + self.m: int = m + self.n: float = n + assert MyClass.a == (1, 2, 3) assert MyClass.b == 1 assert MyClass.c == 2 assert MyClass.d == 3 + +assert MyClass(1, 2).m == 1 \ No newline at end of file diff --git a/tests/66_eval.py b/tests/66_eval.py index 2a51c22a..93c3e7a5 100644 --- a/tests/66_eval.py +++ b/tests/66_eval.py @@ -74,3 +74,5 @@ exec(code, {'x': 42, 'res': res}) assert res == [42, 42] assert x == 33 +# test removing trailing newlines +assert eval('[1, 2, 3]\n \n') == [1, 2, 3] \ No newline at end of file diff --git a/tests/95_bugs.py b/tests/95_bugs.py index ea341717..fe9071bd 100644 --- a/tests/95_bugs.py +++ b/tests/95_bugs.py @@ -157,4 +157,16 @@ assert a == 5 a, b, c = (1, 2, 3) if True else (4, 5, 6) assert a == 1 assert b == 2 -assert c == 3 \ No newline at end of file +assert c == 3 + +# https://github.com/pocketpy/pocketpy/issues/376 +xs = [0] +res = [] +for x in xs: + res.append(x) + if x == 100: + break + xs.append(x+1) + +assert res == list(range(101)) +assert xs == res \ No newline at end of file diff --git a/tests/98_thread.py b/tests/98_thread.py index bc428315..6f4c5d2e 100644 --- a/tests/98_thread.py +++ b/tests/98_thread.py @@ -1,4 +1,11 @@ -from pkpy import ComputeThread +from pkpy import configmacros + +if configmacros['PK_ENABLE_THREADS'] == 1: + from pkpy import ComputeThread +else: + print('threads is not enabled, skipping test...') + exit() + import time thread_1 = ComputeThread(1) diff --git a/tests/99_extras.py b/tests/99_extras.py index b6b8d9ea..732fa283 100644 --- a/tests/99_extras.py +++ b/tests/99_extras.py @@ -103,9 +103,4 @@ class A: bad_dict = {A(): 1, A(): 2, A(): 3, A(): 4} assert len(bad_dict) == 4 -try: - bad_dict[A()] = 5 # error - exit(1) -except RuntimeError as e: - assert 'maximum collision reached' in str(e)