Merge cf47fd21e4c1b285452c2d50c6d86e92ab0b9825 into 5845d0e9385a6f2d3791819fa0c18ce7ae5032ed

python/typing.py

@@ -57,3 +57,5 @@ assert_never = lambda x: x
 
 TypedDict = dict
 NotRequired = _PLACEHOLDER
+
+cast = lambda _, val: val

src/common/_generated.c

@@ -11,7 +11,7 @@ const char kPythonLibs_functools[] = "class cache:\n    def __init__(self, f):\n
 const char kPythonLibs_heapq[] = "# Heap queue algorithm (a.k.a. priority queue)\ndef heappush(heap, item):\n    \"\"\"Push item onto heap, maintaining the heap invariant.\"\"\"\n    heap.append(item)\n    _siftdown(heap, 0, len(heap)-1)\n\ndef heappop(heap):\n    \"\"\"Pop the smallest item off the heap, maintaining the heap invariant.\"\"\"\n    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty\n    if heap:\n        returnitem = heap[0]\n        heap[0] = lastelt\n        _siftup(heap, 0)\n        return returnitem\n    return lastelt\n\ndef heapreplace(heap, item):\n    \"\"\"Pop and return the current smallest value, and add the new item.\n\n    This is more efficient than heappop() followed by heappush(), and can be\n    more appropriate when using a fixed-size heap.  Note that the value\n    returned may be larger than item!  That constrains reasonable uses of\n    this routine unless written as part of a conditional replacement:\n\n        if item > heap[0]:\n            item = heapreplace(heap, item)\n    \"\"\"\n    returnitem = heap[0]    # raises appropriate IndexError if heap is empty\n    heap[0] = item\n    _siftup(heap, 0)\n    return returnitem\n\ndef heappushpop(heap, item):\n    \"\"\"Fast version of a heappush followed by a heappop.\"\"\"\n    if heap and heap[0] < item:\n        item, heap[0] = heap[0], item\n        _siftup(heap, 0)\n    return item\n\ndef heapify(x):\n    \"\"\"Transform list into a heap, in-place, in O(len(x)) time.\"\"\"\n    n = len(x)\n    # Transform bottom-up.  The largest index there's any point to looking at\n    # is the largest with a child index in-range, so must have 2*i + 1 < n,\n    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so\n    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is\n    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.\n    for i in reversed(range(n//2)):\n        _siftup(x, i)\n\n# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos\n# is the index of a leaf with a possibly out-of-order value.  Restore the\n# heap invariant.\ndef _siftdown(heap, startpos, pos):\n    newitem = heap[pos]\n    # Follow the path to the root, moving parents down until finding a place\n    # newitem fits.\n    while pos > startpos:\n        parentpos = (pos - 1) >> 1\n        parent = heap[parentpos]\n        if newitem < parent:\n            heap[pos] = parent\n            pos = parentpos\n            continue\n        break\n    heap[pos] = newitem\n\ndef _siftup(heap, pos):\n    endpos = len(heap)\n    startpos = pos\n    newitem = heap[pos]\n    # Bubble up the smaller child until hitting a leaf.\n    childpos = 2*pos + 1    # leftmost child position\n    while childpos < endpos:\n        # Set childpos to index of smaller child.\n        rightpos = childpos + 1\n        if rightpos < endpos and not heap[childpos] < heap[rightpos]:\n            childpos = rightpos\n        # Move the smaller child up.\n        heap[pos] = heap[childpos]\n        pos = childpos\n        childpos = 2*pos + 1\n    # The leaf at pos is empty now.  Put newitem there, and bubble it up\n    # to its final resting place (by sifting its parents down).\n    heap[pos] = newitem\n    _siftdown(heap, startpos, pos)";
 const char kPythonLibs_linalg[] = "from vmath import *";
 const char kPythonLibs_operator[] = "# https://docs.python.org/3/library/operator.html#mapping-operators-to-functions\n\ndef le(a, b): return a <= b\ndef lt(a, b): return a < b\ndef ge(a, b): return a >= b\ndef gt(a, b): return a > b\ndef eq(a, b): return a == b\ndef ne(a, b): return a != b\n\ndef and_(a, b): return a & b\ndef or_(a, b): return a | b\ndef xor(a, b): return a ^ b\ndef invert(a): return ~a\ndef lshift(a, b): return a << b\ndef rshift(a, b): return a >> b\n\ndef is_(a, b): return a is b\ndef is_not(a, b): return a is not b\ndef not_(a): return not a\ndef truth(a): return bool(a)\ndef contains(a, b): return b in a\n\ndef add(a, b): return a + b\ndef sub(a, b): return a - b\ndef mul(a, b): return a * b\ndef truediv(a, b): return a / b\ndef floordiv(a, b): return a // b\ndef mod(a, b): return a % b\ndef pow(a, b): return a ** b\ndef neg(a): return -a\ndef matmul(a, b): return a @ b\n\ndef getitem(a, b): return a[b]\ndef setitem(a, b, c): a[b] = c\ndef delitem(a, b): del a[b]\n\ndef iadd(a, b): a += b; return a\ndef isub(a, b): a -= b; return a\ndef imul(a, b): a *= b; return a\ndef itruediv(a, b): a /= b; return a\ndef ifloordiv(a, b): a //= b; return a\ndef imod(a, b): a %= b; return a\n# def ipow(a, b): a **= b; return a\n# def imatmul(a, b): a @= b; return a\ndef iand(a, b): a &= b; return a\ndef ior(a, b): a |= b; return a\ndef ixor(a, b): a ^= b; return a\ndef ilshift(a, b): a <<= b; return a\ndef irshift(a, b): a >>= b; return a\n\nclass attrgetter:\n    def __init__(self, attr):\n        self.attr = attr\n    def __call__(self, obj):\n        return getattr(obj, self.attr)\n";
-const char kPythonLibs_typing[] = "class _Placeholder:\n    def __init__(self, *args, **kwargs):\n        pass\n    def __getitem__(self, *args):\n        return self\n    def __call__(self, *args, **kwargs):\n        return self\n    def __and__(self, other):\n        return self\n    def __or__(self, other):\n        return self\n    def __xor__(self, other):\n        return self\n\n\n_PLACEHOLDER = _Placeholder()\n\nSequence = _PLACEHOLDER\nList = _PLACEHOLDER\nDict = _PLACEHOLDER\nTuple = _PLACEHOLDER\nSet = _PLACEHOLDER\nAny = _PLACEHOLDER\nUnion = _PLACEHOLDER\nOptional = _PLACEHOLDER\nCallable = _PLACEHOLDER\nType = _PLACEHOLDER\nTypeAlias = _PLACEHOLDER\nNewType = _PLACEHOLDER\n\nClassVar = _PLACEHOLDER\n\nLiteral = _PLACEHOLDER\nLiteralString = _PLACEHOLDER\n\nIterable = _PLACEHOLDER\nGenerator = _PLACEHOLDER\nIterator = _PLACEHOLDER\n\nHashable = _PLACEHOLDER\n\nTypeVar = _PLACEHOLDER\nSelf = _PLACEHOLDER\n\nProtocol = object\nGeneric = object\nNever = object\n\nTYPE_CHECKING = False\n\n# decorators\noverload = lambda x: x\nfinal = lambda x: x\n\n# exhaustiveness checking\nassert_never = lambda x: x\n\nTypedDict = dict\nNotRequired = _PLACEHOLDER\n";
+const char kPythonLibs_typing[] = "class _Placeholder:\n    def __init__(self, *args, **kwargs):\n        pass\n    def __getitem__(self, *args):\n        return self\n    def __call__(self, *args, **kwargs):\n        return self\n    def __and__(self, other):\n        return self\n    def __or__(self, other):\n        return self\n    def __xor__(self, other):\n        return self\n\n\n_PLACEHOLDER = _Placeholder()\n\nSequence = _PLACEHOLDER\nList = _PLACEHOLDER\nDict = _PLACEHOLDER\nTuple = _PLACEHOLDER\nSet = _PLACEHOLDER\nAny = _PLACEHOLDER\nUnion = _PLACEHOLDER\nOptional = _PLACEHOLDER\nCallable = _PLACEHOLDER\nType = _PLACEHOLDER\nTypeAlias = _PLACEHOLDER\nNewType = _PLACEHOLDER\n\nClassVar = _PLACEHOLDER\n\nLiteral = _PLACEHOLDER\nLiteralString = _PLACEHOLDER\n\nIterable = _PLACEHOLDER\nGenerator = _PLACEHOLDER\nIterator = _PLACEHOLDER\n\nHashable = _PLACEHOLDER\n\nTypeVar = _PLACEHOLDER\nSelf = _PLACEHOLDER\n\nProtocol = object\nGeneric = object\nNever = object\n\nTYPE_CHECKING = False\n\n# decorators\noverload = lambda x: x\nfinal = lambda x: x\n\n# exhaustiveness checking\nassert_never = lambda x: x\n\nTypedDict = dict\nNotRequired = _PLACEHOLDER\n\ncast = lambda _, val: val\n";
 
 const char* load_kPythonLib(const char* name) {
     if (strchr(name, '.') != NULL) return NULL;

src/modules/json.c

@@ -5,6 +5,221 @@
 #include "pocketpy/common/sstream.h"
 #include "pocketpy/interpreter/vm.h"
 #include <math.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+typedef struct {
+    const char* cur;
+    const char* end;
+} json_Parser;
+
+static bool json_Parser__match(json_Parser* self, char c) {
+    if(self->cur == self->end) return false;
+    if(*self->cur == c) {
+        self->cur++;
+        return true;
+    }
+    return false;
+}
+
+static void json_Parser__skip_whitespace(json_Parser* self) {
+    while(self->cur < self->end) {
+        char c = *self->cur;
+        if(c == ' ' || c == '\n' || c == '\r' || c == '\t') {
+            self->cur++;
+        } else {
+            break;
+        }
+    }
+}
+
+static bool json_Parser__parse_value(json_Parser* self, py_Ref out);
+
+static bool json_Parser__parse_object(json_Parser* self, py_Ref out) {
+    if(!json_Parser__match(self, '{')) return false;
+    py_newdict(out);
+    json_Parser__skip_whitespace(self);
+    if(json_Parser__match(self, '}')) return true;
+
+    while(true) {
+        json_Parser__skip_whitespace(self);
+        py_Ref key = py_pushtmp();
+        if(!json_Parser__parse_value(self, key)) return false;
+        if(!py_isstr(key)) return ValueError("json: expecting string as key");
+
+        json_Parser__skip_whitespace(self);
+        if(!json_Parser__match(self, ':')) return ValueError("json: expecting ':'");
+
+        json_Parser__skip_whitespace(self);
+        py_Ref value = py_pushtmp();
+        if(!json_Parser__parse_value(self, value)) return false;
+
+        py_dict_setitem(out, key, value);
+        py_pop();  // value
+        py_pop();  // key
+
+        json_Parser__skip_whitespace(self);
+        if(json_Parser__match(self, '}')) return true;
+        if(!json_Parser__match(self, ',')) return ValueError("json: expecting ',' or '}'");
+    }
+}
+
+static bool json_Parser__parse_array(json_Parser* self, py_Ref out) {
+    if(!json_Parser__match(self, '[')) return false;
+    py_newlist(out);
+    json_Parser__skip_whitespace(self);
+    if(json_Parser__match(self, ']')) return true;
+
+    while(true) {
+        json_Parser__skip_whitespace(self);
+        py_Ref value = py_pushtmp();
+        if(!json_Parser__parse_value(self, value)) return false;
+        py_list_append(out, value);
+        py_pop();  // value
+
+        json_Parser__skip_whitespace(self);
+        if(json_Parser__match(self, ']')) return true;
+        if(!json_Parser__match(self, ',')) return ValueError("json: expecting ',' or ']'");
+    }
+}
+
+static bool json_Parser__parse_string(json_Parser* self, py_Ref out) {
+    if(!json_Parser__match(self, '"')) return false;
+    c11_sbuf buf;
+    c11_sbuf__ctor(&buf);
+
+    while(self->cur < self->end) {
+        char c = *self->cur++;
+        if(c == '"') {
+            c11_sbuf__py_submit(&buf, out);
+            return true;
+        }
+        if(c == '\\') {
+            if(self->cur == self->end) break;
+            c = *self->cur++;
+            switch(c) {
+                case '"': c11_sbuf__write_char(&buf, '"'); break;
+                case '\\': c11_sbuf__write_char(&buf, '\\'); break;
+                case '/': c11_sbuf__write_char(&buf, '/'); break;
+                case 'b': c11_sbuf__write_char(&buf, '\b'); break;
+                case 'f': c11_sbuf__write_char(&buf, '\f'); break;
+                case 'n': c11_sbuf__write_char(&buf, '\n'); break;
+                case 'r': c11_sbuf__write_char(&buf, '\r'); break;
+                case 't': c11_sbuf__write_char(&buf, '\t'); break;
+                case 'u': {
+                    // TODO: support unicode escape \uXXXX
+                    // For now we just write \u and continue to avoid crash
+                    c11_sbuf__write_char(&buf, '\\');
+                    c11_sbuf__write_char(&buf, 'u');
+                    break;
+                }
+                default: c11_sbuf__write_char(&buf, c); break;
+            }
+        } else {
+            c11_sbuf__write_char(&buf, c);
+        }
+    }
+    c11_sbuf__dtor(&buf);
+    return ValueError("json: expecting '\"'");
+}
+
+static void json_Parser__ctor_true(py_Ref out) { py_newbool(out, true); }
+
+static void json_Parser__ctor_false(py_Ref out) { py_newbool(out, false); }
+
+static bool json_Parser__parse_keyword(json_Parser* self,
+                                       const char* keyword,
+                                       py_Ref out,
+                                       void (*ctor)(py_Ref)) {
+    int len = strlen(keyword);
+    if(self->end - self->cur >= len && memcmp(self->cur, keyword, len) == 0) {
+        self->cur += len;
+        ctor(out);
+        return true;
+    }
+    return false;
+}
+
+static bool json_Parser__parse_number(json_Parser* self, py_Ref out) {
+    const char* start = self->cur;
+    if(self->cur < self->end && *self->cur == '-') self->cur++;
+
+    if(self->cur == self->end) return false;
+
+    if(*self->cur == '0') {
+        self->cur++;
+        // do not allow leading zeros like 0123
+    } else if(isdigit(*self->cur)) {
+        self->cur++;
+        while(self->cur < self->end && isdigit(*self->cur))
+            self->cur++;
+    } else {
+        return false;
+    }
+
+    bool is_float = false;
+
+    // fraction
+    if(self->cur < self->end && *self->cur == '.') {
+        is_float = true;
+        self->cur++;
+        if(self->cur == self->end || !isdigit(*self->cur)) return false;
+        while(self->cur < self->end && isdigit(*self->cur))
+            self->cur++;
+    }
+
+    // exponent
+    if(self->cur < self->end && (*self->cur == 'e' || *self->cur == 'E')) {
+        is_float = true;
+        self->cur++;
+        if(self->cur < self->end && (*self->cur == '+' || *self->cur == '-')) self->cur++;
+        if(self->cur == self->end || !isdigit(*self->cur)) return false;
+        while(self->cur < self->end && isdigit(*self->cur))
+            self->cur++;
+    }
+
+    // parse
+    char* endptr;
+    if(is_float) {
+        double val = strtod(start, &endptr);
+        if(endptr != self->cur) return false;
+        py_newfloat(out, val);
+    } else {
+        long long val = strtoll(start, &endptr, 10);
+        if(endptr != self->cur) return false;
+        py_newint(out, val);
+    }
+    return true;
+}
+
+static bool json_Parser__parse_value(json_Parser* self, py_Ref out) {
+    json_Parser__skip_whitespace(self);
+    if(self->cur == self->end) return ValueError("json: unexpected end of input");
+
+    char c = *self->cur;
+    if(c == '{') return json_Parser__parse_object(self, out);
+    if(c == '[') return json_Parser__parse_array(self, out);
+    if(c == '"') return json_Parser__parse_string(self, out);
+    if(c == 't') return json_Parser__parse_keyword(self, "true", out, json_Parser__ctor_true);
+    if(c == 'f') return json_Parser__parse_keyword(self, "false", out, json_Parser__ctor_false);
+    if(c == 'n') return json_Parser__parse_keyword(self, "null", out, py_newnone);
+    if(c == '-' || isdigit(c)) return json_Parser__parse_number(self, out);
+
+    return ValueError("json: unexpected character '%c'", c);
+}
+
+bool py_json_loads(const char* source) {
+    json_Parser parser;
+    parser.cur = source;
+    parser.end = source + strlen(source);
+
+    if(!json_Parser__parse_value(&parser, py_retval())) return false;
+
+    json_Parser__skip_whitespace(&parser);
+    if(parser.cur != parser.end) return ValueError("json: extra data");
+    return true;
+}
 
 static bool json_loads(int argc, py_Ref argv) {
     PY_CHECK_ARGC(1);
@@ -183,9 +398,3 @@ bool py_json_dumps(py_Ref val, int indent) {
     c11_sbuf__py_submit(&buf, py_retval());
     return true;
 }
-
-bool py_json_loads(const char* source) {
-    py_GlobalRef mod = py_getmodule("json");
-    return py_exec(source, "<json>", EVAL_MODE, mod);
-}
-