From 2d0db3dc717267a54624ad732ba78dd4f08e7ddc Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sat, 15 Jun 2024 13:34:47 +0800 Subject: [PATCH] add lower_bound --- include/pocketpy/common/algorithm.h | 11 +--- include/pocketpy/common/sstream.h | 9 +-- include/pocketpy/common/str.h | 14 +++-- include/pocketpy/common/vector.h | 27 ++++++--- include/pocketpy/common/vector.hpp | 4 +- include/pocketpy/objects/sourcedata.h | 6 +- include/pocketpy/objects/sourcedata.hpp | 2 +- src/common/algorithm.c | 25 +++++++- src/common/sourcedata.c | 28 ++++----- src/common/sstream.c | 28 +++++++-- src/common/str.c | 76 ++++++++++++------------- src/compiler/lexer.cpp | 19 +------ src/interpreter/iter.cpp | 2 +- src/modules/random.cpp | 2 +- 14 files changed, 138 insertions(+), 115 deletions(-) diff --git a/include/pocketpy/common/algorithm.h b/include/pocketpy/common/algorithm.h index f5a56509..4438327a 100644 --- a/include/pocketpy/common/algorithm.h +++ b/include/pocketpy/common/algorithm.h @@ -9,15 +9,10 @@ extern "C" { void *c11__lower_bound(const void *key, const void *ptr, int count, int size, bool (*less)(const void *, const void *)); +int *c11__lower_bound_int(int key, const int *ptr, int count); +double *c11__lower_bound_double(double key, const double *ptr, int count); + #ifdef __cplusplus } -namespace pkpy{ -template - T* lower_bound(T* begin, T* end, const T& value){ - return (T*)c11__lower_bound(&value, begin, end - begin, sizeof(T), [](const void* a, const void* b){ - return *(T*)a < *(T*)b; - }); - } -} // namespace pkpy #endif \ No newline at end of file diff --git a/include/pocketpy/common/sstream.h b/include/pocketpy/common/sstream.h index 6189e8ef..7525246b 100644 --- a/include/pocketpy/common/sstream.h +++ b/include/pocketpy/common/sstream.h @@ -1,15 +1,15 @@ #pragma once -#ifdef __cplusplus -extern "C" { -#endif - #include "pocketpy/common/vector.h" #include "pocketpy/common/str.h" #include "pocketpy/common/utils.h" #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct pkpy_SStream { c11_vector data; } pkpy_SStream; @@ -40,6 +40,7 @@ PK_INLINE pkpy_AnyStr pkpy_AnyStr__cstr(const char* x) { pkpy_AnyStr s; s.type = PK_INLINE pkpy_AnyStr pkpy_AnyStr__ptr(void* x) { pkpy_AnyStr s; s.type = 9; s._ptr = x; return s; } void pkpy_SStream__ctor(pkpy_SStream* self); +void pkpy_SStream__ctor2(pkpy_SStream* self, int capacity); void pkpy_SStream__dtor(pkpy_SStream* self); void pkpy_SStream__write_int(pkpy_SStream* self, int); diff --git a/include/pocketpy/common/str.h b/include/pocketpy/common/str.h index c09be739..28efd200 100644 --- a/include/pocketpy/common/str.h +++ b/include/pocketpy/common/str.h @@ -1,14 +1,13 @@ #pragma once +#include +#include "pocketpy/common/vector.h" +#include "pocketpy/common/utils.h" + #ifdef __cplusplus extern "C" { #endif -#include - -#include "pocketpy/common/vector.h" -#include "pocketpy/common/utils.h" - /* string_view */ typedef struct c11_string{ const char* data; @@ -29,7 +28,6 @@ PK_INLINE const char* pkpy_Str__data(const pkpy_Str* self){ return self->is_sso ? self->_inlined : self->_ptr; } -int pkpy_utils__u8_header(unsigned char c, bool suppress); void pkpy_Str__ctor(pkpy_Str* self, const char* data); void pkpy_Str__ctor2(pkpy_Str* self, const char* data, int size); void pkpy_Str__dtor(pkpy_Str* self); @@ -57,6 +55,10 @@ int pkpy_Str__count(const pkpy_Str* self, const pkpy_Str* sub); c11_vector/* T=c11_string */ pkpy_Str__split(const pkpy_Str* self, char sep); c11_vector/* T=c11_string */ pkpy_Str__split2(const pkpy_Str* self, const pkpy_Str* sep); +bool c11__isascii(const char* p, int size); +bool c11__is_unicode_Lo_char(int c); +int c11__u8_header(unsigned char c, bool suppress); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/include/pocketpy/common/vector.h b/include/pocketpy/common/vector.h index a3c4abf3..fb7f72f1 100644 --- a/include/pocketpy/common/vector.h +++ b/include/pocketpy/common/vector.h @@ -1,14 +1,13 @@ #pragma once +#include +#include +#include "pocketpy/common/algorithm.h" + #ifdef __cplusplus extern "C" { #endif -#include -#include - -#include "pocketpy/common/algorithm.h" - typedef struct c11_array{ void* data; int count; @@ -60,17 +59,29 @@ void c11_vector__clear(c11_vector* self); #define c11_vector__insert(T, self, index, elem) \ do{ \ if((self)->count == (self)->capacity) c11_vector__reserve((self), (self)->capacity*2); \ - memmove((T*)(self)->data + (index) + 1, (T*)(self)->data + (index), ((self)->count - (index)) * sizeof(T)); \ - ((T*)(self)->data)[index] = (elem); \ + T* p = (T*)(self)->data + (index); \ + memmove(p + 1, p, ((self)->count - (index)) * sizeof(T)); \ + *p = (elem); \ (self)->count++; \ }while(0) #define c11_vector__erase(T, self, index) \ do{ \ - memmove((T*)(self)->data + (index), (T*)(self)->data + (index) + 1, ((self)->count - (index) - 1) * sizeof(T)); \ + T* p = (T*)(self)->data + (index); \ + memmove(p, p + 1, ((self)->count - (index) - 1) * sizeof(T)); \ (self)->count--; \ }while(0) +#define c11_vector__reverse(T, self, start, end) \ + do{ \ + T* p = (T*)(self)->data + (start); \ + T* q = (T*)(self)->data + (end); \ + while(p < q){ \ + T tmp = *p; *p = *q; *q = tmp; \ + p++; q--; \ + } \ + }while(0) + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/include/pocketpy/common/vector.hpp b/include/pocketpy/common/vector.hpp index 4d05c4cf..66e035a0 100644 --- a/include/pocketpy/common/vector.hpp +++ b/include/pocketpy/common/vector.hpp @@ -428,13 +428,13 @@ struct small_map { Item* data() const { return _data.data(); } void insert(const K& key, const V& value) { - Item* it = lower_bound(_data.begin(), _data.end(), key); + Item* it = std::lower_bound(_data.begin(), _data.end(), key); assert(it == _data.end() || it->first != key); _data.insert(it, {key, value}); } V* try_get(const K& key) const { - auto it = lower_bound(_data.begin(), _data.end(), key); + auto it = std::lower_bound(_data.begin(), _data.end(), key); if(it == _data.end() || it->first != key) return nullptr; return &it->second; } diff --git a/include/pocketpy/objects/sourcedata.h b/include/pocketpy/objects/sourcedata.h index 7f8d7cc0..3b40175a 100644 --- a/include/pocketpy/objects/sourcedata.h +++ b/include/pocketpy/objects/sourcedata.h @@ -17,11 +17,11 @@ struct pkpy_SourceData { pkpy_Str filename; pkpy_Str source; - c11_vector line_starts; // contains "const char *" - c11_vector _precompiled_tokens; // contains "pkpy_Str" + c11_vector/*T=const char* */ line_starts; + c11_vector/*T=pkpy_Str*/ _precompiled_tokens; }; -void pkpy_SourceData__ctor(struct pkpy_SourceData *self, const char *source, int source_size, const pkpy_Str *filename, enum CompileMode mode); +void pkpy_SourceData__ctor(struct pkpy_SourceData *self, c11_string source, const pkpy_Str *filename, enum CompileMode mode); void pkpy_SourceData__dtor(struct pkpy_SourceData* self); bool pkpy_SourceData__get_line(const struct pkpy_SourceData *self, int lineno, const char **st, const char **ed); diff --git a/include/pocketpy/objects/sourcedata.hpp b/include/pocketpy/objects/sourcedata.hpp index a9b6ac19..f4ce2908 100644 --- a/include/pocketpy/objects/sourcedata.hpp +++ b/include/pocketpy/objects/sourcedata.hpp @@ -8,7 +8,7 @@ namespace pkpy { struct SourceData : public pkpy_SourceData { SourceData(std::string_view source, const Str& filename, CompileMode mode) { - pkpy_SourceData__ctor(this, source.data(), source.size(), &filename, mode); + pkpy_SourceData__ctor(this, {source.data(), (int)source.size()}, &filename, mode); } ~SourceData() { diff --git a/src/common/algorithm.c b/src/common/algorithm.c index 0db7d07c..5917e337 100644 --- a/src/common/algorithm.c +++ b/src/common/algorithm.c @@ -6,15 +6,34 @@ void *c11__lower_bound(const void *key, const void *ptr, int count, int size, int __len = count; while(__len != 0){ - int __l2 = (int)((unsigned int)__len >> 1); + int __l2 = (int)((unsigned int)__len / 2); char* __m = __first + __l2 * size; if(less(__m, key)){ - __first = __m; __m += size; + __first = __m; __len -= __l2 + 1; }else{ __len = __l2; } } return __first; -} \ No newline at end of file +} + +static bool c11__less_int(const void* a, const void* b){ + return *(int*)a < *(int*)b; +} + +static bool c11__less_double(const void* a, const void* b){ + return *(double*)a < *(double*)b; +} + +int *c11__lower_bound_int(int key, const int *ptr, int count) { + void* res = c11__lower_bound(&key, ptr, count, sizeof(int), c11__less_int); + return (int*)res; +} + +double *c11__lower_bound_double(double key, const double *ptr, int count) { + void* res = c11__lower_bound(&key, ptr, count, sizeof(double), c11__less_double); + return (double*)res; +} + diff --git a/src/common/sourcedata.c b/src/common/sourcedata.c index 496ea12c..b37a1810 100644 --- a/src/common/sourcedata.c +++ b/src/common/sourcedata.c @@ -4,31 +4,27 @@ #include #include -void pkpy_Str__take_buf(pkpy_Str *self, char *data, int size); - void pkpy_SourceData__ctor(struct pkpy_SourceData* self, - const char* source, - int source_size, + c11_string source, // may not be null-terminated const pkpy_Str* filename, enum CompileMode mode) { self->filename = pkpy_Str__copy(filename); // OPTIMIZEME? self->mode = mode; - c11_vector__ctor(&self->line_starts, sizeof(const char*)); c11_vector__ctor(&self->_precompiled_tokens, sizeof(pkpy_Str)); - int index = (strncmp(source, "\xEF\xBB\xBF", 3) == 0) ? 3 : 0; - int len = source_size - index; - for(int i = 0; i < source_size; ++i) - len -= (source[i] == '\r'); - - char *buf = (char*)malloc(len + 1), *p = buf; - buf[len] = '\0'; - for(; index < source_size; ++index) { - if(source[index] != '\r') *(p++) = source[index]; + int index = 0; + // Skip utf8 BOM if there is any. + if (source.size >= 3 && strncmp(source.data, "\xEF\xBB\xBF", 3) == 0) index += 3; + // Drop all '\r' + pkpy_SStream ss; + pkpy_SStream__ctor2(&ss, source.size + 1); + while(index < source.size){ + char c = source.data[index]; + if(c != '\r') pkpy_SStream__write_char(&ss, c); + index++; } - pkpy_Str__take_buf(&self->source, buf, len); - + self->source = pkpy_SStream__submit(&ss); self->is_precompiled = (strncmp(pkpy_Str__data(&self->source), "pkpy:", 5) == 0); c11_vector__push(const char*, &self->line_starts, pkpy_Str__data(&self->source)); } diff --git a/src/common/sstream.c b/src/common/sstream.c index ad99a93a..5b9b513f 100644 --- a/src/common/sstream.c +++ b/src/common/sstream.c @@ -10,6 +10,11 @@ void pkpy_SStream__ctor(pkpy_SStream* self) { c11_vector__ctor(&self->data, sizeof(char)); } +void pkpy_SStream__ctor2(pkpy_SStream* self, int capacity) { + c11_vector__ctor(&self->data, sizeof(char)); + c11_vector__reserve(&self->data, capacity); +} + void pkpy_SStream__dtor(pkpy_SStream* self) { c11_vector__dtor(&self->data); } @@ -24,10 +29,25 @@ void pkpy_SStream__write_int(pkpy_SStream* self, int i) { pkpy_SStream__write_cstr(self, buf); } -void pkpy_SStream__write_i64(pkpy_SStream* self, int64_t i) { - char buf[23]; // sign + 21 digits + null terminator - snprintf(buf, sizeof(buf), "%lld", i); - pkpy_SStream__write_cstr(self, buf); +void pkpy_SStream__write_i64(pkpy_SStream* self, int64_t val) { + // sign + 21 digits + null terminator + // str(-2**64).__len__() == 21 + c11_vector__reserve(&self->data, self->data.count + 23); + if(val == 0){ + pkpy_SStream__write_char(self, '0'); + return; + } + if(val < 0){ + pkpy_SStream__write_char(self, '-'); + val = -val; + } + int start = self->data.count; + while(val){ + c11_vector__push(char, &self->data, '0' + val % 10); + val /= 10; + } + int end = self->data.count - 1; + c11_vector__reverse(char, &self->data, start, end); } void pkpy_SStream__write_float(pkpy_SStream* self, float val, int precision){ diff --git a/src/common/str.c b/src/common/str.c index 9c3fb5fb..cfd48094 100644 --- a/src/common/str.c +++ b/src/common/str.c @@ -7,50 +7,13 @@ #include #include -int pkpy_utils__u8_header(unsigned char c, bool suppress) { - if((c & 0b10000000) == 0) return 1; - if((c & 0b11100000) == 0b11000000) return 2; - if((c & 0b11110000) == 0b11100000) return 3; - if((c & 0b11111000) == 0b11110000) return 4; - if((c & 0b11111100) == 0b11111000) return 5; - if((c & 0b11111110) == 0b11111100) return 6; - if(!suppress) PK_FATAL_ERROR("invalid utf8 char\n") - return 0; -} - void pkpy_Str__ctor(pkpy_Str *self, const char *data){ pkpy_Str__ctor2(self, data, strlen(data)); } -static void pkpy_Str__check_ascii(pkpy_Str *self, char *p) { - for(int i = 0; i < self->size; i++){ - if(!isascii(p[i])){ - self->is_ascii = false; - break; - } - } -} - -void pkpy_Str__take_buf(pkpy_Str *self, char *data, int size) { - self->size = size; - self->is_ascii = true; - self->is_sso = size < sizeof(self->_inlined); - char* p; - if(self->is_sso){ - p = self->_inlined; - memcpy(p, data, size); - p[size] = '\0'; - free(data); - }else{ - self->_ptr = data; - p = self->_ptr; - } - pkpy_Str__check_ascii(self, p); -} - void pkpy_Str__ctor2(pkpy_Str *self, const char *data, int size){ self->size = size; - self->is_ascii = true; + self->is_ascii = c11__isascii(data, size); self->is_sso = size < sizeof(self->_inlined); char* p; if(self->is_sso){ @@ -61,7 +24,6 @@ void pkpy_Str__ctor2(pkpy_Str *self, const char *data, int size){ } memcpy(p, data, size); p[size] = '\0'; - pkpy_Str__check_ascii(self, p); } void pkpy_Str__dtor(pkpy_Str *self){ @@ -288,7 +250,7 @@ int pkpy_Str__cmp2(const pkpy_Str *self, const char *other, int size){ pkpy_Str pkpy_Str__u8_getitem(const pkpy_Str *self, int i){ i = pkpy_Str__unicode_index_to_byte(self, i); - int size = pkpy_utils__u8_header(pkpy_Str__data(self)[i], false); + int size = c11__u8_header(pkpy_Str__data(self)[i], false); return pkpy_Str__slice2(self, i, i + size); } @@ -330,7 +292,7 @@ int pkpy_Str__unicode_index_to_byte(const pkpy_Str* self, int i) { const char* p = pkpy_Str__data(self); int j = 0; while(i > 0) { - j += pkpy_utils__u8_header(p[j], false); + j += c11__u8_header(p[j], false); i--; } return j; @@ -409,3 +371,35 @@ c11_vector/* T=c11_string */ pkpy_Str__split2(const pkpy_Str *self, const pkpy_S if(tmp.size != 0) c11_vector__push(c11_string, &retval, tmp); return retval; } + +bool c11__isascii(const char* p, int size){ + for(int i = 0; i < size; i++) + if((unsigned char)p[i] > 127) + return false; + return true; +} + +// clang-format off +static const int kLoRangeA[] = {170,186,443,448,660,1488,1519,1568,1601,1646,1649,1749,1774,1786,1791,1808,1810,1869,1969,1994,2048,2112,2144,2208,2230,2308,2365,2384,2392,2418,2437,2447,2451,2474,2482,2486,2493,2510,2524,2527,2544,2556,2565,2575,2579,2602,2610,2613,2616,2649,2654,2674,2693,2703,2707,2730,2738,2741,2749,2768,2784,2809,2821,2831,2835,2858,2866,2869,2877,2908,2911,2929,2947,2949,2958,2962,2969,2972,2974,2979,2984,2990,3024,3077,3086,3090,3114,3133,3160,3168,3200,3205,3214,3218,3242,3253,3261,3294,3296,3313,3333,3342,3346,3389,3406,3412,3423,3450,3461,3482,3507,3517,3520,3585,3634,3648,3713,3716,3718,3724,3749,3751,3762,3773,3776,3804,3840,3904,3913,3976,4096,4159,4176,4186,4193,4197,4206,4213,4238,4352,4682,4688,4696,4698,4704,4746,4752,4786,4792,4800,4802,4808,4824,4882,4888,4992,5121,5743,5761,5792,5873,5888,5902,5920,5952,5984,5998,6016,6108,6176,6212,6272,6279,6314,6320,6400,6480,6512,6528,6576,6656,6688,6917,6981,7043,7086,7098,7168,7245,7258,7401,7406,7413,7418,8501,11568,11648,11680,11688,11696,11704,11712,11720,11728,11736,12294,12348,12353,12447,12449,12543,12549,12593,12704,12784,13312,19968,40960,40982,42192,42240,42512,42538,42606,42656,42895,42999,43003,43011,43015,43020,43072,43138,43250,43259,43261,43274,43312,43360,43396,43488,43495,43514,43520,43584,43588,43616,43633,43642,43646,43697,43701,43705,43712,43714,43739,43744,43762,43777,43785,43793,43808,43816,43968,44032,55216,55243,63744,64112,64285,64287,64298,64312,64318,64320,64323,64326,64467,64848,64914,65008,65136,65142,65382,65393,65440,65474,65482,65490,65498,65536,65549,65576,65596,65599,65616,65664,66176,66208,66304,66349,66370,66384,66432,66464,66504,66640,66816,66864,67072,67392,67424,67584,67592,67594,67639,67644,67647,67680,67712,67808,67828,67840,67872,67968,68030,68096,68112,68117,68121,68192,68224,68288,68297,68352,68416,68448,68480,68608,68864,69376,69415,69424,69600,69635,69763,69840,69891,69956,69968,70006,70019,70081,70106,70108,70144,70163,70272,70280,70282,70287,70303,70320,70405,70415,70419,70442,70450,70453,70461,70480,70493,70656,70727,70751,70784,70852,70855,71040,71128,71168,71236,71296,71352,71424,71680,71935,72096,72106,72161,72163,72192,72203,72250,72272,72284,72349,72384,72704,72714,72768,72818,72960,72968,72971,73030,73056,73063,73066,73112,73440,73728,74880,77824,82944,92160,92736,92880,92928,93027,93053,93952,94032,94208,100352,110592,110928,110948,110960,113664,113776,113792,113808,123136,123214,123584,124928,126464,126469,126497,126500,126503,126505,126516,126521,126523,126530,126535,126537,126539,126541,126545,126548,126551,126553,126555,126557,126559,126561,126564,126567,126572,126580,126585,126590,126592,126603,126625,126629,126635,131072,173824,177984,178208,183984,194560}; +static const int kLoRangeB[] = {170,186,443,451,660,1514,1522,1599,1610,1647,1747,1749,1775,1788,1791,1808,1839,1957,1969,2026,2069,2136,2154,2228,2237,2361,2365,2384,2401,2432,2444,2448,2472,2480,2482,2489,2493,2510,2525,2529,2545,2556,2570,2576,2600,2608,2611,2614,2617,2652,2654,2676,2701,2705,2728,2736,2739,2745,2749,2768,2785,2809,2828,2832,2856,2864,2867,2873,2877,2909,2913,2929,2947,2954,2960,2965,2970,2972,2975,2980,2986,3001,3024,3084,3088,3112,3129,3133,3162,3169,3200,3212,3216,3240,3251,3257,3261,3294,3297,3314,3340,3344,3386,3389,3406,3414,3425,3455,3478,3505,3515,3517,3526,3632,3635,3653,3714,3716,3722,3747,3749,3760,3763,3773,3780,3807,3840,3911,3948,3980,4138,4159,4181,4189,4193,4198,4208,4225,4238,4680,4685,4694,4696,4701,4744,4749,4784,4789,4798,4800,4805,4822,4880,4885,4954,5007,5740,5759,5786,5866,5880,5900,5905,5937,5969,5996,6000,6067,6108,6210,6264,6276,6312,6314,6389,6430,6509,6516,6571,6601,6678,6740,6963,6987,7072,7087,7141,7203,7247,7287,7404,7411,7414,7418,8504,11623,11670,11686,11694,11702,11710,11718,11726,11734,11742,12294,12348,12438,12447,12538,12543,12591,12686,12730,12799,19893,40943,40980,42124,42231,42507,42527,42539,42606,42725,42895,42999,43009,43013,43018,43042,43123,43187,43255,43259,43262,43301,43334,43388,43442,43492,43503,43518,43560,43586,43595,43631,43638,43642,43695,43697,43702,43709,43712,43714,43740,43754,43762,43782,43790,43798,43814,43822,44002,55203,55238,55291,64109,64217,64285,64296,64310,64316,64318,64321,64324,64433,64829,64911,64967,65019,65140,65276,65391,65437,65470,65479,65487,65495,65500,65547,65574,65594,65597,65613,65629,65786,66204,66256,66335,66368,66377,66421,66461,66499,66511,66717,66855,66915,67382,67413,67431,67589,67592,67637,67640,67644,67669,67702,67742,67826,67829,67861,67897,68023,68031,68096,68115,68119,68149,68220,68252,68295,68324,68405,68437,68466,68497,68680,68899,69404,69415,69445,69622,69687,69807,69864,69926,69956,70002,70006,70066,70084,70106,70108,70161,70187,70278,70280,70285,70301,70312,70366,70412,70416,70440,70448,70451,70457,70461,70480,70497,70708,70730,70751,70831,70853,70855,71086,71131,71215,71236,71338,71352,71450,71723,71935,72103,72144,72161,72163,72192,72242,72250,72272,72329,72349,72440,72712,72750,72768,72847,72966,72969,73008,73030,73061,73064,73097,73112,73458,74649,75075,78894,83526,92728,92766,92909,92975,93047,93071,94026,94032,100343,101106,110878,110930,110951,111355,113770,113788,113800,113817,123180,123214,123627,125124,126467,126495,126498,126500,126503,126514,126519,126521,126523,126530,126535,126537,126539,126543,126546,126548,126551,126553,126555,126557,126559,126562,126564,126570,126578,126583,126588,126590,126601,126619,126627,126633,126651,173782,177972,178205,183969,191456,195101}; +// clang-format on + +bool c11__is_unicode_Lo_char(int c){ + if(c == 0x1f955) return true; + int index = c11__lower_bound_int(c, kLoRangeA, 476) - kLoRangeA; + if(c == kLoRangeA[index]) return true; + index -= 1; + if(index < 0) return false; + return c >= kLoRangeA[index] && c <= kLoRangeB[index]; +} + +int c11__u8_header(unsigned char c, bool suppress) { + if((c & 0b10000000) == 0) return 1; + if((c & 0b11100000) == 0b11000000) return 2; + if((c & 0b11110000) == 0b11100000) return 3; + if((c & 0b11111000) == 0b11110000) return 4; + if((c & 0b11111100) == 0b11111000) return 5; + if((c & 0b11111110) == 0b11111100) return 6; + if(!suppress) PK_FATAL_ERROR("invalid utf8 char\n") + return 0; +} \ No newline at end of file diff --git a/src/compiler/lexer.cpp b/src/compiler/lexer.cpp index e87ce605..0b3f1a35 100644 --- a/src/compiler/lexer.cpp +++ b/src/compiler/lexer.cpp @@ -7,11 +7,6 @@ namespace pkpy { -// clang-format off -static const uint32_t kLoRangeA[] = {170,186,443,448,660,1488,1519,1568,1601,1646,1649,1749,1774,1786,1791,1808,1810,1869,1969,1994,2048,2112,2144,2208,2230,2308,2365,2384,2392,2418,2437,2447,2451,2474,2482,2486,2493,2510,2524,2527,2544,2556,2565,2575,2579,2602,2610,2613,2616,2649,2654,2674,2693,2703,2707,2730,2738,2741,2749,2768,2784,2809,2821,2831,2835,2858,2866,2869,2877,2908,2911,2929,2947,2949,2958,2962,2969,2972,2974,2979,2984,2990,3024,3077,3086,3090,3114,3133,3160,3168,3200,3205,3214,3218,3242,3253,3261,3294,3296,3313,3333,3342,3346,3389,3406,3412,3423,3450,3461,3482,3507,3517,3520,3585,3634,3648,3713,3716,3718,3724,3749,3751,3762,3773,3776,3804,3840,3904,3913,3976,4096,4159,4176,4186,4193,4197,4206,4213,4238,4352,4682,4688,4696,4698,4704,4746,4752,4786,4792,4800,4802,4808,4824,4882,4888,4992,5121,5743,5761,5792,5873,5888,5902,5920,5952,5984,5998,6016,6108,6176,6212,6272,6279,6314,6320,6400,6480,6512,6528,6576,6656,6688,6917,6981,7043,7086,7098,7168,7245,7258,7401,7406,7413,7418,8501,11568,11648,11680,11688,11696,11704,11712,11720,11728,11736,12294,12348,12353,12447,12449,12543,12549,12593,12704,12784,13312,19968,40960,40982,42192,42240,42512,42538,42606,42656,42895,42999,43003,43011,43015,43020,43072,43138,43250,43259,43261,43274,43312,43360,43396,43488,43495,43514,43520,43584,43588,43616,43633,43642,43646,43697,43701,43705,43712,43714,43739,43744,43762,43777,43785,43793,43808,43816,43968,44032,55216,55243,63744,64112,64285,64287,64298,64312,64318,64320,64323,64326,64467,64848,64914,65008,65136,65142,65382,65393,65440,65474,65482,65490,65498,65536,65549,65576,65596,65599,65616,65664,66176,66208,66304,66349,66370,66384,66432,66464,66504,66640,66816,66864,67072,67392,67424,67584,67592,67594,67639,67644,67647,67680,67712,67808,67828,67840,67872,67968,68030,68096,68112,68117,68121,68192,68224,68288,68297,68352,68416,68448,68480,68608,68864,69376,69415,69424,69600,69635,69763,69840,69891,69956,69968,70006,70019,70081,70106,70108,70144,70163,70272,70280,70282,70287,70303,70320,70405,70415,70419,70442,70450,70453,70461,70480,70493,70656,70727,70751,70784,70852,70855,71040,71128,71168,71236,71296,71352,71424,71680,71935,72096,72106,72161,72163,72192,72203,72250,72272,72284,72349,72384,72704,72714,72768,72818,72960,72968,72971,73030,73056,73063,73066,73112,73440,73728,74880,77824,82944,92160,92736,92880,92928,93027,93053,93952,94032,94208,100352,110592,110928,110948,110960,113664,113776,113792,113808,123136,123214,123584,124928,126464,126469,126497,126500,126503,126505,126516,126521,126523,126530,126535,126537,126539,126541,126545,126548,126551,126553,126555,126557,126559,126561,126564,126567,126572,126580,126585,126590,126592,126603,126625,126629,126635,131072,173824,177984,178208,183984,194560}; -static const uint32_t kLoRangeB[] = {170,186,443,451,660,1514,1522,1599,1610,1647,1747,1749,1775,1788,1791,1808,1839,1957,1969,2026,2069,2136,2154,2228,2237,2361,2365,2384,2401,2432,2444,2448,2472,2480,2482,2489,2493,2510,2525,2529,2545,2556,2570,2576,2600,2608,2611,2614,2617,2652,2654,2676,2701,2705,2728,2736,2739,2745,2749,2768,2785,2809,2828,2832,2856,2864,2867,2873,2877,2909,2913,2929,2947,2954,2960,2965,2970,2972,2975,2980,2986,3001,3024,3084,3088,3112,3129,3133,3162,3169,3200,3212,3216,3240,3251,3257,3261,3294,3297,3314,3340,3344,3386,3389,3406,3414,3425,3455,3478,3505,3515,3517,3526,3632,3635,3653,3714,3716,3722,3747,3749,3760,3763,3773,3780,3807,3840,3911,3948,3980,4138,4159,4181,4189,4193,4198,4208,4225,4238,4680,4685,4694,4696,4701,4744,4749,4784,4789,4798,4800,4805,4822,4880,4885,4954,5007,5740,5759,5786,5866,5880,5900,5905,5937,5969,5996,6000,6067,6108,6210,6264,6276,6312,6314,6389,6430,6509,6516,6571,6601,6678,6740,6963,6987,7072,7087,7141,7203,7247,7287,7404,7411,7414,7418,8504,11623,11670,11686,11694,11702,11710,11718,11726,11734,11742,12294,12348,12438,12447,12538,12543,12591,12686,12730,12799,19893,40943,40980,42124,42231,42507,42527,42539,42606,42725,42895,42999,43009,43013,43018,43042,43123,43187,43255,43259,43262,43301,43334,43388,43442,43492,43503,43518,43560,43586,43595,43631,43638,43642,43695,43697,43702,43709,43712,43714,43740,43754,43762,43782,43790,43798,43814,43822,44002,55203,55238,55291,64109,64217,64285,64296,64310,64316,64318,64321,64324,64433,64829,64911,64967,65019,65140,65276,65391,65437,65470,65479,65487,65495,65500,65547,65574,65594,65597,65613,65629,65786,66204,66256,66335,66368,66377,66421,66461,66499,66511,66717,66855,66915,67382,67413,67431,67589,67592,67637,67640,67644,67669,67702,67742,67826,67829,67861,67897,68023,68031,68096,68115,68119,68149,68220,68252,68295,68324,68405,68437,68466,68497,68680,68899,69404,69415,69445,69622,69687,69807,69864,69926,69956,70002,70006,70066,70084,70106,70108,70161,70187,70278,70280,70285,70301,70312,70366,70412,70416,70440,70448,70451,70457,70461,70480,70497,70708,70730,70751,70831,70853,70855,71086,71131,71215,71236,71338,71352,71450,71723,71935,72103,72144,72161,72163,72192,72242,72250,72272,72329,72349,72440,72712,72750,72768,72847,72966,72969,73008,73030,73061,73064,73097,73112,73458,74649,75075,78894,83526,92728,92766,92909,92975,93047,93071,94026,94032,100343,101106,110878,110930,110951,111355,113770,113788,113800,113817,123180,123214,123627,125124,126467,126495,126498,126500,126503,126514,126519,126521,126523,126530,126535,126537,126539,126543,126546,126548,126551,126553,126555,126557,126559,126562,126564,126570,126578,126583,126588,126590,126601,126619,126627,126633,126651,173782,177972,178205,183969,191456,195101}; -// clang-format on - static bool is_possible_number_char(char c) noexcept{ switch(c) { // clang-format off @@ -25,16 +20,6 @@ static bool is_possible_number_char(char c) noexcept{ } } -static bool is_unicode_Lo_char(uint32_t c) noexcept{ - // open a hole for carrot - if(c == U'🥕') return true; - auto index = lower_bound(kLoRangeA, kLoRangeA + 476, c) - kLoRangeA; - if(c == kLoRangeA[index]) return true; - index -= 1; - if(index < 0) return false; - return c >= kLoRangeA[index] && c <= kLoRangeB[index]; -} - bool Lexer::match_n_chars(int n, char c0) noexcept{ const char* c = curr_char; for(int i = 0; i < n; i++) { @@ -108,7 +93,7 @@ Error* Lexer::eat_name() noexcept{ curr_char--; while(true) { unsigned char c = peekchar(); - int u8bytes = pkpy_utils__u8_header(c, true); + int u8bytes = c11__u8_header(c, true); if(u8bytes == 0) return SyntaxError("invalid char: %c", c); if(u8bytes == 1) { if(isalpha(c) || c == '_' || isdigit(c)) { @@ -135,7 +120,7 @@ Error* Lexer::eat_name() noexcept{ value |= (b & 0b00111111) << (6 * (u8bytes - k - 1)); } } - if(is_unicode_Lo_char(value)) + if(c11__is_unicode_Lo_char(value)) curr_char += u8bytes; else break; diff --git a/src/interpreter/iter.cpp b/src/interpreter/iter.cpp index d0fae96a..882c7f82 100644 --- a/src/interpreter/iter.cpp +++ b/src/interpreter/iter.cpp @@ -49,7 +49,7 @@ void StringIter::_register(VM* vm, PyObject* mod, PyObject* type) { Str& s = PK_OBJ_GET(Str, self.ref); if(self.i == s.size) return 0; int start = self.i; - int len = pkpy_utils__u8_header(s[self.i], false); + int len = c11__u8_header(s[self.i], false); self.i += len; vm->s_data.push(VAR(s.slice(start, self.i))); return 1; diff --git a/src/modules/random.cpp b/src/modules/random.cpp index 131aa298..03b59196 100644 --- a/src/modules/random.cpp +++ b/src/modules/random.cpp @@ -199,7 +199,7 @@ struct Random { List result(k); for(int i = 0; i < k; i++) { f64 r = self.gen.uniform(0.0, cum_weights[size - 1]); - int idx = lower_bound(cum_weights.begin(), cum_weights.end(), r) - cum_weights.begin(); + int idx = c11__lower_bound_double(r, cum_weights.begin(), cum_weights.size()) - cum_weights.begin(); result[i] = data[idx]; } return VAR(std::move(result));