From a07078ed4f198585e27d2e4207c363d97e35b450 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Thu, 16 Mar 2023 23:51:20 +0800 Subject: [PATCH] fix a bug and add `heapq` --- python/heapq.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++ src/compiler.h | 8 ++- src/parser.h | 4 +- src/pocketpy.h | 1 + tests/70_heapq.py | 9 ++++ tests/99_bugs.py | 5 +- 6 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 python/heapq.py create mode 100644 tests/70_heapq.py diff --git a/python/heapq.py b/python/heapq.py new file mode 100644 index 00000000..48432a8e --- /dev/null +++ b/python/heapq.py @@ -0,0 +1,129 @@ +# Heap queue algorithm (a.k.a. priority queue) + +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', + 'nlargest', 'nsmallest', 'heappushpop'] + +def heappush(heap, item): + """Push item onto heap, maintaining the heap invariant.""" + heap.append(item) + _siftdown(heap, 0, len(heap)-1) + +def heappop(heap): + """Pop the smallest item off the heap, maintaining the heap invariant.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup(heap, 0) + return returnitem + return lastelt + +def heapreplace(heap, item): + """Pop and return the current smallest value, and add the new item. + + This is more efficient than heappop() followed by heappush(), and can be + more appropriate when using a fixed-size heap. Note that the value + returned may be larger than item! That constrains reasonable uses of + this routine unless written as part of a conditional replacement: + + if item > heap[0]: + item = heapreplace(heap, item) + """ + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup(heap, 0) + return returnitem + +def heappushpop(heap, item): + """Fast version of a heappush followed by a heappop.""" + if heap and heap[0] < item: + item, heap[0] = heap[0], item + _siftup(heap, 0) + return item + +def heapify(x): + """Transform list into a heap, in-place, in O(len(x)) time.""" + n = len(x) + # Transform bottom-up. The largest index there's any point to looking at + # is the largest with a child index in-range, so must have 2*i + 1 < n, + # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so + # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is + # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. + for i in reversed(range(n//2)): + _siftup(x, i) + +# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos +# is the index of a leaf with a possibly out-of-order value. Restore the +# heap invariant. +def _siftdown(heap, startpos, pos): + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if newitem < parent: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem + +# The child indices of heap index pos are already heaps, and we want to make +# a heap at index pos too. We do this by bubbling the smaller child of +# pos up (and so on with that child's children, etc) until hitting a leaf, +# then using _siftdown to move the oddball originally at index pos into place. +# +# We *could* break out of the loop as soon as we find a pos where newitem <= +# both its children, but turns out that's not a good idea, and despite that +# many books write the algorithm that way. During a heap pop, the last array +# element is sifted in, and that tends to be large, so that comparing it +# against values starting from the root usually doesn't pay (= usually doesn't +# get us out of the loop early). See Knuth, Volume 3, where this is +# explained and quantified in an exercise. +# +# Cutting the # of comparisons is important, since these routines have no +# way to extract "the priority" from an array element, so that intelligence +# is likely to be hiding in custom comparison methods, or in array elements +# storing (priority, record) tuples. Comparisons are thus potentially +# expensive. +# +# On random arrays of length 1000, making this change cut the number of +# comparisons made by heapify() a little, and those made by exhaustive +# heappop() a lot, in accord with theory. Here are typical results from 3 +# runs (3 just to demonstrate how small the variance is): +# +# Compares needed by heapify Compares needed by 1000 heappops +# -------------------------- -------------------------------- +# 1837 cut to 1663 14996 cut to 8680 +# 1855 cut to 1659 14966 cut to 8678 +# 1847 cut to 1660 15024 cut to 8703 +# +# Building the heap by using heappush() 1000 times instead required +# 2198, 2148, and 2219 compares: heapify() is more efficient, when +# you can use it. +# +# The total compares needed by list.sort() on the same lists were 8627, +# 8627, and 8632 (this should be compared to the sum of heapify() and +# heappop() compares): list.sort() is (unsurprisingly!) more efficient +# for sorting. + +def _siftup(heap, pos): + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[childpos] < heap[rightpos]: + childpos = rightpos + # Move the smaller child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown(heap, startpos, pos) \ No newline at end of file diff --git a/src/compiler.h b/src/compiler.h index 8781d97d..1ecaec92 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -64,7 +64,7 @@ public: rules[TK("is not")] = { nullptr, METHOD(exprBinaryOp), PREC_TEST }; rules[TK("and") ] = { nullptr, METHOD(exprAnd), PREC_LOGICAL_AND }; rules[TK("or")] = { nullptr, METHOD(exprOr), PREC_LOGICAL_OR }; - rules[TK("not")] = { METHOD(exprUnaryOp), nullptr, PREC_UNARY }; + rules[TK("not")] = { METHOD(exprNot), nullptr, PREC_LOGICAL_NOT }; rules[TK("True")] = { METHOD(exprValue), NO_INFIX }; rules[TK("False")] = { METHOD(exprValue), NO_INFIX }; rules[TK("lambda")] = { METHOD(exprLambda), NO_INFIX }; @@ -506,12 +506,16 @@ private: } } + void exprNot() { + parse_expression((Precedence)(PREC_LOGICAL_NOT + 1)); + emit(OP_UNARY_NOT); + } + void exprUnaryOp() { TokenIndex op = parser->prev.type; parse_expression((Precedence)(PREC_UNARY + 1)); switch (op) { case TK("-"): emit(OP_UNARY_NEGATIVE); break; - case TK("not"): emit(OP_UNARY_NOT); break; case TK("*"): emit(OP_UNARY_STAR, co()->_rvalue); break; default: UNREACHABLE(); } diff --git a/src/parser.h b/src/parser.h index 9f9197e9..c867ea4a 100644 --- a/src/parser.h +++ b/src/parser.h @@ -67,6 +67,7 @@ struct Token{ } }; +// https://docs.python.org/3/reference/expressions.html enum Precedence { PREC_NONE, PREC_ASSIGNMENT, // = @@ -74,8 +75,9 @@ enum Precedence { PREC_TERNARY, // ?: PREC_LOGICAL_OR, // or PREC_LOGICAL_AND, // and + PREC_LOGICAL_NOT, // not PREC_EQUALITY, // == != - PREC_TEST, // in is + PREC_TEST, // in / is / is not / not in PREC_COMPARISION, // < > <= >= PREC_BITWISE_OR, // | PREC_BITWISE_XOR, // ^ diff --git a/src/pocketpy.h b/src/pocketpy.h index 3f5cab94..90f8d1ec 100644 --- a/src/pocketpy.h +++ b/src/pocketpy.h @@ -731,6 +731,7 @@ void VM::post_init(){ add_module_c(this); _lazy_modules["functools"] = kPythonLibs["functools"]; _lazy_modules["collections"] = kPythonLibs["collections"]; + _lazy_modules["heapq"] = kPythonLibs["heapq"]; CodeObject_ code = compile(kPythonLibs["builtins"], "", EXEC_MODE); this->_exec(code, this->builtins); diff --git a/tests/70_heapq.py b/tests/70_heapq.py new file mode 100644 index 00000000..636bfaa0 --- /dev/null +++ b/tests/70_heapq.py @@ -0,0 +1,9 @@ +from heapq import heapify, heappop, heappush +from random import randint + +a = [randint(0, 100) for i in range(1000)] +b = sorted(a) + +heapify(a) +for x in b: + assert heappop(a) == x \ No newline at end of file diff --git a/tests/99_bugs.py b/tests/99_bugs.py index d6b53bb9..9332374c 100644 --- a/tests/99_bugs.py +++ b/tests/99_bugs.py @@ -1,4 +1,7 @@ # https://github.com/blueloveTH/pocketpy/issues/37 mp = map(lambda x: x**2, [1, 2, 3, 4, 5] ) -assert list(mp) == [1, 4, 9, 16, 25] \ No newline at end of file +assert list(mp) == [1, 4, 9, 16, 25] + + +assert not 3>4 \ No newline at end of file