diff --git a/amalgamate.py b/amalgamate.py index cd658268..3aa7992b 100644 --- a/amalgamate.py +++ b/amalgamate.py @@ -9,7 +9,7 @@ pipeline = [ ["config.h", "export.h", "common.h", "memory.h", "vector.h", "str.h", "tuplelist.h", "namedict.h", "error.h", "lexer.h"], ["obj.h", "dict.h", "codeobject.h", "frame.h"], ["gc.h", "vm.h", "ceval.h", "expr.h", "compiler.h", "repl.h"], - ["_generated.h", "cffi.h", "bindings.h", "iter.h", "base64.h", "random.h", "re.h", "linalg.h", "easing.h", "io.h"], + ["_generated.h", "cffi.h", "bindings.h", "iter.h", "base64.h", "collections.h", "random.h", "re.h", "linalg.h", "easing.h", "io.h"], ["pocketpy.h", "pocketpy_c.h"] ] diff --git a/include/pocketpy/collections.h b/include/pocketpy/collections.h new file mode 100644 index 00000000..446f9da5 --- /dev/null +++ b/include/pocketpy/collections.h @@ -0,0 +1,13 @@ +#pragma once + +#include "obj.h" +#include "common.h" +#include "memory.h" +#include "str.h" +#include "iter.h" +#include "cffi.h" + +namespace pkpy +{ + void add_module_collections(VM *vm); +} // namespace pkpy \ No newline at end of file diff --git a/include/pocketpy/common.h b/include/pocketpy/common.h index c82265dc..555aabba 100644 --- a/include/pocketpy/common.h +++ b/include/pocketpy/common.h @@ -21,6 +21,7 @@ #include #include #include +#include #define PK_VERSION "1.2.7" diff --git a/include/pocketpy/pocketpy.h b/include/pocketpy/pocketpy.h index 81498d21..1748b0c3 100644 --- a/include/pocketpy/pocketpy.h +++ b/include/pocketpy/pocketpy.h @@ -13,6 +13,7 @@ #include "re.h" #include "random.h" #include "bindings.h" +#include "collections.h" namespace pkpy { diff --git a/include/pocketpy/vm.h b/include/pocketpy/vm.h index 90e64c5d..8493e51f 100644 --- a/include/pocketpy/vm.h +++ b/include/pocketpy/vm.h @@ -360,6 +360,7 @@ public: void TypeError(const Str& msg){ _error("TypeError", msg); } void IndexError(const Str& msg){ _error("IndexError", msg); } void ValueError(const Str& msg){ _error("ValueError", msg); } + void RuntimeError(const Str& msg){ _error("RuntimeError", msg); } void ZeroDivisionError(const Str& msg){ _error("ZeroDivisionError", msg); } void ZeroDivisionError(){ _error("ZeroDivisionError", "division by zero"); } void NameError(StrName name){ _error("NameError", fmt("name ", name.escape() + " is not defined")); } diff --git a/python/collections.py b/python/collections.py index 75d99567..9a6571df 100644 --- a/python/collections.py +++ b/python/collections.py @@ -1,110 +1,110 @@ -class _LinkedListNode: - def __init__(self, prev, next, value) -> None: - self.prev = prev - self.next = next - self.value = value +# class _LinkedListNode: +# def __init__(self, prev, next, value) -> None: +# self.prev = prev +# self.next = next +# self.value = value -class deque: - def __init__(self, iterable=None) -> None: - self.head = _LinkedListNode(None, None, None) - self.tail = _LinkedListNode(None, None, None) - self.head.next = self.tail - self.tail.prev = self.head - self.size = 0 - if iterable is not None: - for value in iterable: - self.append(value) +# class deque: +# def __init__(self, iterable=None) -> None: +# self.head = _LinkedListNode(None, None, None) +# self.tail = _LinkedListNode(None, None, None) +# self.head.next = self.tail +# self.tail.prev = self.head +# self.size = 0 +# if iterable is not None: +# for value in iterable: +# self.append(value) - def __getitem__(self, index): - assert 0 <= index < len(self) - node = self.head.next - for _ in range(index): - node = node.next - return node.value +# def __getitem__(self, index): +# assert 0 <= index < len(self) +# node = self.head.next +# for _ in range(index): +# node = node.next +# return node.value - def __setitem__(self, index, value): - assert 0 <= index < len(self) - node = self.head.next - for _ in range(index): - node = node.next - node.value = value +# def __setitem__(self, index, value): +# assert 0 <= index < len(self) +# node = self.head.next +# for _ in range(index): +# node = node.next +# node.value = value - def __delitem__(self, index): - assert 0 <= index < len(self) - node = self.head.next - for _ in range(index): - node = node.next - node.prev.next = node.next - node.next.prev = node.prev - self.size -= 1 +# def __delitem__(self, index): +# assert 0 <= index < len(self) +# node = self.head.next +# for _ in range(index): +# node = node.next +# node.prev.next = node.next +# node.next.prev = node.prev +# self.size -= 1 - def clear(self): - self.head.next = self.tail - self.tail.prev = self.head - self.size = 0 +# def clear(self): +# self.head.next = self.tail +# self.tail.prev = self.head +# self.size = 0 - def extend(self, iterable): - for value in iterable: - self.append(value) +# def extend(self, iterable): +# for value in iterable: +# self.append(value) - def append(self, value): - node = _LinkedListNode(self.tail.prev, self.tail, value) - self.tail.prev.next = node - self.tail.prev = node - self.size += 1 +# def append(self, value): +# node = _LinkedListNode(self.tail.prev, self.tail, value) +# self.tail.prev.next = node +# self.tail.prev = node +# self.size += 1 - def appendleft(self, value): - node = _LinkedListNode(self.head, self.head.next, value) - self.head.next.prev = node - self.head.next = node - self.size += 1 +# def appendleft(self, value): +# node = _LinkedListNode(self.head, self.head.next, value) +# self.head.next.prev = node +# self.head.next = node +# self.size += 1 - def pop(self): - assert self.size > 0 - node = self.tail.prev - node.prev.next = self.tail - self.tail.prev = node.prev - self.size -= 1 - return node.value +# def pop(self): +# assert self.size > 0 +# node = self.tail.prev +# node.prev.next = self.tail +# self.tail.prev = node.prev +# self.size -= 1 +# return node.value - def popleft(self): - assert self.size > 0 - node = self.head.next - node.next.prev = self.head - self.head.next = node.next - self.size -= 1 - return node.value +# def popleft(self): +# assert self.size > 0 +# node = self.head.next +# node.next.prev = self.head +# self.head.next = node.next +# self.size -= 1 +# return node.value - def copy(self): - new_list = deque() - for value in self: - new_list.append(value) - return new_list +# def copy(self): +# new_list = deque() +# for value in self: +# new_list.append(value) +# return new_list - def __len__(self): - return self.size +# def __len__(self): +# return self.size - def __iter__(self): - node = self.head.next - while node is not self.tail: - yield node.value - node = node.next +# def __iter__(self): +# node = self.head.next +# while node is not self.tail: +# yield node.value +# node = node.next - def __repr__(self) -> str: - a = list(self) - return f"deque({a})" +# def __repr__(self) -> str: +# a = list(self) +# return f"deque({a})" - def __eq__(self, __o: object) -> bool: - if not isinstance(__o, deque): - return False - if len(self) != len(__o): - return False - t1, t2 = self.head.next, __o.head.next - while t1 is not self.tail: - if t1.value != t2.value: - return False - t1, t2 = t1.next, t2.next - return True +# def __eq__(self, __o: object) -> bool: +# if not isinstance(__o, deque): +# return False +# if len(self) != len(__o): +# return False +# t1, t2 = self.head.next, __o.head.next +# while t1 is not self.tail: +# if t1.value != t2.value: +# return False +# t1, t2 = t1.next, t2.next +# return True def Counter(iterable): a = {} diff --git a/src/collections.cpp b/src/collections.cpp new file mode 100644 index 00000000..d5d1f17c --- /dev/null +++ b/src/collections.cpp @@ -0,0 +1,586 @@ +#include "pocketpy/collections.h" +namespace pkpy +{ + struct PyDequeIter // Iterator for the deque type + { + PY_CLASS(PyDequeIter, builtins, "_deque_iterator") + PyObject *ref; + bool is_reversed; + std::deque::iterator begin, end, current; + std::deque::reverse_iterator rbegin, rend, rcurrent; + PyDequeIter(PyObject *ref, std::deque::iterator begin, std::deque::iterator end) + : ref(ref), begin(begin), end(end), current(begin) + { + this->is_reversed = false; + } + PyDequeIter(PyObject *ref, std::deque::reverse_iterator rbegin, std::deque::reverse_iterator rend) + : ref(ref), rbegin(rbegin), rend(rend), rcurrent(rbegin) + { + this->is_reversed = true; + } + void _gc_mark() const { PK_OBJ_MARK(ref); } + static void _register(VM *vm, PyObject *mod, PyObject *type); + }; + void PyDequeIter::_register(VM *vm, PyObject *mod, PyObject *type) + { + // Iterator for the deque type + vm->_all_types[PK_OBJ_GET(Type, type)].subclass_enabled = false; + vm->bind_notimplemented_constructor(type); + + vm->bind__iter__(PK_OBJ_GET(Type, type), [](VM *vm, PyObject *obj) + { return obj; }); + vm->bind__next__(PK_OBJ_GET(Type, type), [](VM *vm, PyObject *obj) + { + PyDequeIter& self = _CAST(PyDequeIter&, obj); + if(self.is_reversed){ + if(self.rcurrent == self.rend) return vm->StopIteration; + PyObject* ret = *self.rcurrent; + ++self.rcurrent; + return ret; + } + else{ + if(self.current == self.end) return vm->StopIteration; + PyObject* ret = *self.current; + ++self.current; + return ret; + } }); + } + struct PyDeque + { + PY_CLASS(PyDeque, collections, deque); + PyDeque(VM *vm, PyObject *iterable, PyObject *maxlen); // constructor + // PyDeque members + std::deque dequeItems; + int maxlen = -1; // -1 means unbounded + bool bounded = false; // if true, maxlen is not -1 + void insertObj(bool front, bool back, int index, PyObject *item); // insert at index, used purely for internal purposes: append, appendleft, insert methods + PyObject *popObj(bool front, bool back, PyObject *item, VM *vm); // pop at index, used purely for internal purposes: pop, popleft, remove methods + int findIndex(VM *vm, PyObject *obj, int start, int stop); // find the index of the given object in the deque + std::stringstream getRepr(VM *vm, PyObject* thisObj); // get the string representation of the deque + // Special methods + static void _register(VM *vm, PyObject *mod, PyObject *type); // register the type + void _gc_mark() const; // needed for container types, mark all objects in the deque for gc + }; + void PyDeque::_register(VM *vm, PyObject *mod, PyObject *type) + { + vm->bind(type, "__new__(cls, iterable=None, maxlen=None)", + [](VM *vm, ArgsView args) + { + Type cls_t = PK_OBJ_GET(Type, args[0]); + PyObject *iterable = args[1]; + PyObject *maxlen = args[2]; + return vm->heap.gcnew(cls_t, vm, iterable, maxlen); + }); + // gets the item at the given index, if index is negative, it will be treated as index + len(deque) + // if the index is out of range, IndexError will be thrown --> required for [] operator + vm->bind(type, "__getitem__(self, index) -> PyObject", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + int index = CAST(int, args[1]); + index = vm->normalized_index(index, self.dequeItems.size()); // error is handled by the vm->normalized_index + return self.dequeItems.at(index); + }); + // sets the item at the given index, if index is negative, it will be treated as index + len(deque) + // if the index is out of range, IndexError will be thrown --> required for [] operator + vm->bind(type, "__setitem__(self, index, newValue) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + int index = CAST(int, args[1]); + PyObject *newValue = args[2]; + index = vm->normalized_index(index, self.dequeItems.size()); // error is handled by the vm->normalized_index + self.dequeItems.at(index) = newValue; + return vm->None; + }); + // erases the item at the given index, if index is negative, it will be treated as index + len(deque) + // if the index is out of range, IndexError will be thrown --> required for [] operator + vm->bind(type, "__delitem__(self, index) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + int index = CAST(int, args[1]); + index = vm->normalized_index(index, self.dequeItems.size()); // error is handled by the vm->normalized_index + self.dequeItems.erase(self.dequeItems.begin() + index); + return vm->None; + }); + // returns the length of the deque + vm->bind(type, "__len__(self) -> int", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + return VAR(self.dequeItems.size()); + }); + // returns an iterator for the deque + vm->bind(type, "__iter__(self) -> deque_iterator", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + return vm->heap.gcnew( + PyDequeIter::_type(vm), args[0], + self.dequeItems.begin(), self.dequeItems.end()); + }); + // returns a string representation of the deque + vm->bind(type, "__repr__(self) -> str", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + std::stringstream ss = self.getRepr(vm, args[0]); + return VAR(ss.str()); + }); + // returns a string representation of the deque + vm->bind(type, "__str__(self) -> str", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + std::stringstream ss = self.getRepr(vm, args[0]); + return VAR(ss.str()); + }); + // enables comparison between two deques, == and != are supported + vm->bind(type, "__eq__(self, other) -> bool", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + PyDeque &other = _CAST(PyDeque &, args[1]); + if (self.dequeItems.size() != other.dequeItems.size()) // trivial case + return VAR(false); + for (int i = 0; i < self.dequeItems.size(); i++) + if (!vm->py_equals(self.dequeItems[i], other.dequeItems[i])) + return VAR(false); + return VAR(true); + }); + // clear the deque + vm->bind(type, "clear(self) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + self.dequeItems.clear(); + return vm->None; + }); + // extend the deque with the given iterable + vm->bind(type, "extend(self, iterable) -> None", + [](VM *vm, ArgsView args) + { + auto _lock = vm->heap.gc_scope_lock(); // locking the heap + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *it = vm->py_iter(args[1]); // strong ref + PyObject *obj = vm->py_next(it); + while (obj != vm->StopIteration) + { + self.insertObj(false, true, -1, obj); + obj = vm->py_next(it); + } + return vm->None; + }); + // append at the end of the deque + vm->bind(type, "append(self, item) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *item = args[1]; + self.insertObj(false, true, -1, item); + return vm->None; + }); + // append at the beginning of the deque + vm->bind(type, "appendleft(self, item) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *item = args[1]; + self.insertObj(true, false, -1, item); + return vm->None; + }); + // pop from the end of the deque + vm->bind(type, "pop(self) -> PyObject", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + if (self.dequeItems.empty()) + { + vm->IndexError("pop from an empty deque"); + return vm->None; + } + return self.popObj(false, true, nullptr, vm); + }); + // pop from the beginning of the deque + vm->bind(type, "popleft(self) -> PyObject", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + if (self.dequeItems.empty()) + { + vm->IndexError("pop from an empty deque"); + return vm->None; + } + return self.popObj(true, false, nullptr, vm); + }); + // shallow copy of the deque + vm->bind(type, "copy(self) -> deque", + [](VM *vm, ArgsView args) + { + auto _lock = vm->heap.gc_scope_lock(); // locking the heap + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *newDequeObj = vm->heap.gcnew(PyDeque::_type(vm), vm, vm->None, vm->None); // create the empty deque + PyDeque &newDeque = _CAST(PyDeque &, newDequeObj); // cast it to PyDeque so we can use its methods + for (auto it = self.dequeItems.begin(); it != self.dequeItems.end(); ++it) + newDeque.insertObj(false, true, -1, *it); + return newDequeObj; + }); + // NEW: counts the number of occurences of the given object in the deque + vm->bind(type, "count(self, obj) -> int", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *obj = args[1]; + int cnt = 0, sz = self.dequeItems.size(); + for (auto it = self.dequeItems.begin(); it != self.dequeItems.end(); ++it) + { + if (vm->py_equals((*it), obj)) + cnt++; + if (sz != self.dequeItems.size())// mutating the deque during iteration is not allowed + vm->RuntimeError("deque mutated during iteration"); + } + return VAR(cnt); + }); + // NEW: extends the deque from the left + vm->bind(type, "extendleft(self, iterable) -> None", + [](VM *vm, ArgsView args) + { + auto _lock = vm->heap.gc_scope_lock(); + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *it = vm->py_iter(args[1]); // strong ref + PyObject *obj = vm->py_next(it); + while (obj != vm->StopIteration) + { + self.insertObj(true, false, -1, obj); + obj = vm->py_next(it); + } + return vm->None; + }); + // NEW: returns the index of the given object in the deque + vm->bind(type, "index(self, obj, start=None, stop=None) -> int", + [](VM *vm, ArgsView args) + { + // Return the position of x in the deque (at or after index start and before index stop). Returns the first match or raises ValueError if not found. + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *obj = args[1]; + int start = 0, stop = self.dequeItems.size(); // default values + if (!vm->py_equals(args[2], vm->None)) + start = CAST(int, args[2]); + if (!vm->py_equals(args[3], vm->None)) + stop = CAST(int, args[3]); + int index = self.findIndex(vm, obj, start, stop); + if (index != -1) + return VAR(index); + else + vm->ValueError(_CAST(Str &, vm->py_repr(obj)) + " is not in deque"); + return vm->None; + }); + // NEW: returns the index of the given object in the deque + vm->bind(type, "__contains__(self, obj) -> bool", + [](VM *vm, ArgsView args) + { + // Return the position of x in the deque (at or after index start and before index stop). Returns the first match or raises ValueError if not found. + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *obj = args[1]; + int start = 0, stop = self.dequeItems.size(); // default values + int index = self.findIndex(vm, obj, start, stop); + if (index != -1) + return VAR(true); + return VAR(false); + }); + // NEW: inserts the given object at the given index + vm->bind(type, "insert(self, index, obj) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + int index = CAST(int, args[1]); + PyObject *obj = args[2]; + if (self.bounded && self.dequeItems.size() == self.maxlen) + vm->IndexError("deque already at its maximum size"); + else + self.insertObj(false, false, index, obj); // this index shouldn't be fixed using vm->normalized_index, pass as is + return vm->None; + }); + // NEW: removes the first occurence of the given object from the deque + vm->bind(type, "remove(self, obj) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + PyObject *obj = args[1]; + PyObject *removed = self.popObj(false, false, obj, vm); + if (removed == nullptr) + vm->ValueError(_CAST(Str &, vm->py_repr(obj)) + " is not in list"); + return vm->None; + }); + // NEW: reverses the deque + vm->bind(type, "reverse(self) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + if (self.dequeItems.empty() || self.dequeItems.size() == 1) + return vm->None; // handle trivial cases + int sz = self.dequeItems.size(); + for (int i = 0; i < sz / 2; i++) + { + PyObject *tmp = self.dequeItems[i]; + self.dequeItems[i] = self.dequeItems[sz - i - 1]; // swapping + self.dequeItems[sz - i - 1] = tmp; + } + return vm->None; + }); + // NEW: rotates the deque by n steps + vm->bind(type, "rotate(self, n=1) -> None", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + int n = CAST(int, args[1]); + + if (n != 0 && !self.dequeItems.empty()) // trivial case + { + PyObject *tmp; // holds the object to be rotated + int direction = n > 0 ? 1 : -1; + n = abs(n); + n = n % self.dequeItems.size(); // make sure n is in range + while (n--) + { + if (direction == 1) + { + tmp = self.dequeItems.back(); + self.dequeItems.pop_back(); + self.dequeItems.push_front(tmp); + } + else + { + tmp = self.dequeItems.front(); + self.dequeItems.pop_front(); + self.dequeItems.push_back(tmp); + } + } + } + return vm->None; + }); + // NEW: getter and setter of property `maxlen` + vm->bind_property( + type, "maxlen: int", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + if (self.bounded) + return VAR(self.maxlen); + return vm->None; + }, + [](VM *vm, ArgsView args) + { + vm->AttributeError("attribute 'maxlen' of 'collections.deque' objects is not writable"); + return vm->None; + }); + // NEW: support pickle + vm->bind(type, "__getnewargs__(self) -> tuple[list, int]", + [](VM *vm, ArgsView args) + { + PyDeque &self = _CAST(PyDeque &, args[0]); + Tuple ret(2); + List list; + for (PyObject *obj : self.dequeItems) + { + list.push_back(obj); + } + ret[0] = VAR(std::move(list)); + if (self.bounded) + ret[1] = VAR(self.maxlen); + else + ret[1] = vm->None; + return VAR(ret); + }); + } + /// @brief initializes a new PyDeque object, actual initialization is done in __init__ + PyDeque::PyDeque(VM *vm, PyObject *iterable, PyObject *maxlen) + { + + if (!vm->py_equals(maxlen, vm->None)) // fix the maxlen first + { + int tmp = CAST(int, maxlen); + if (tmp < 0) + vm->ValueError("maxlen must be non-negative"); + else + { + this->maxlen = tmp; + this->bounded = true; + } + } + else + { + this->bounded = false; + this->maxlen = -1; + } + if (!vm->py_equals(iterable, vm->None)) + { + this->dequeItems.clear(); // clear the deque + auto _lock = vm->heap.gc_scope_lock(); // locking the heap + PyObject *it = vm->py_iter(iterable); // strong ref + PyObject *obj = vm->py_next(it); + while (obj != vm->StopIteration) + { + this->insertObj(false, true, -1, obj); + obj = vm->py_next(it); + } + } + } + std::stringstream PyDeque::getRepr(VM *vm, PyObject *thisObj) + { + std::stringstream ss; + ss << "deque(["; + for (auto it = this->dequeItems.begin(); it != this->dequeItems.end(); ++it) + { + if (*it == thisObj) + ss << "[...]"; + else + ss << CAST(Str &, vm->py_repr(*it)); + if (it != this->dequeItems.end() - 1) + ss << ", "; + } + this->bounded ? ss << "], maxlen=" << this->maxlen << ")" : ss << "])"; + return ss; + } + int PyDeque::findIndex(VM *vm, PyObject *obj, int start, int stop) + { + // the following code is special purpose normalization for this method, taken from CPython: _collectionsmodule.c file + if (start < 0) + { + start = this->dequeItems.size() + start; // try to fix for negative indices + if (start < 0) + start = 0; + } + if (stop < 0) + { + stop = this->dequeItems.size() + stop; // try to fix for negative indices + if (stop < 0) + stop = 0; + } + if (stop > this->dequeItems.size()) + stop = this->dequeItems.size(); + if (start > stop) + start = stop; // end of normalization + PK_ASSERT(start >= 0 && start <= this->dequeItems.size() && stop >= 0 && stop <= this->dequeItems.size() && start <= stop); // sanity check + int loopSize = std::min((int)(this->dequeItems.size()), stop); + int sz = this->dequeItems.size(); + for (int i = start; i < loopSize; i++) + { + if (vm->py_equals(this->dequeItems[i], obj)) + return i; + if (sz != this->dequeItems.size())// mutating the deque during iteration is not allowed + vm->RuntimeError("deque mutated during iteration"); + } + return -1; + } + + /// @brief pops or removes an item from the deque + /// @param front if true, pop from the front of the deque + /// @param back if true, pop from the back of the deque + /// @param item if front and back is not set, remove the first occurence of item from the deque + /// @param vm is needed for the py_equals + /// @return PyObject* if front or back is set, this is a pop operation and we return a PyObject*, if front and back are not set, this is a remove operation and we return the removed item or nullptr + PyObject *PyDeque::popObj(bool front, bool back, PyObject *item, VM *vm) + { + // error handling + if (front && back) + throw std::runtime_error("both front and back are set"); // this should never happen + if (front || back) + { + // front or back is set, we don't care about item, this is a pop operation and we return a PyObject* + if (this->dequeItems.empty()) + throw std::runtime_error("pop from an empty deque"); // shouldn't happen + PyObject *obj; + if (front) + { + obj = this->dequeItems.front(); + this->dequeItems.pop_front(); + } + else + { + obj = this->dequeItems.back(); + this->dequeItems.pop_back(); + } + return obj; + } + else + { + // front and back are not set, we care about item, this is a remove operation and we return the removed item or nullptr + int sz = this->dequeItems.size(); + for (auto it = this->dequeItems.begin(); it != this->dequeItems.end(); ++it) + { + bool found = vm->py_equals((*it), item); + if (sz != this->dequeItems.size()) // mutating the deque during iteration is not allowed + vm->IndexError("deque mutated during iteration"); + if (found) + { + PyObject *obj = *it; // keep a reference to the object for returning + this->dequeItems.erase(it); + return obj; + } + } + return nullptr; // not found + } + } + /// @brief inserts an item into the deque + /// @param front if true, insert at the front of the deque + /// @param back if true, insert at the back of the deque + /// @param index if front and back are not set, insert at the given index + /// @param item the item to insert + /// @return true if the item was inserted successfully, false if the deque is bounded and is already at its maximum size + void PyDeque::insertObj(bool front, bool back, int index, PyObject *item) // assume index is not fixed using the vm->normalized_index + { + // error handling + if (front && back) + throw std::runtime_error("both front and back are set"); // this should never happen + if (front || back) + { + // front or back is set, we don't care about index + if (this->bounded) + { + if (this->maxlen == 0) + return; // bounded and maxlen is 0, so we can't append + else if (this->dequeItems.size() == this->maxlen) + { + if (front) + this->dequeItems.pop_back(); // remove the last item + else if (back) + this->dequeItems.pop_front(); // remove the first item + } + } + if (front) + this->dequeItems.emplace_front(item); + else if (back) + this->dequeItems.emplace_back(item); + } + else + { + // front and back are not set, we care about index + if (index < 0) + index = this->dequeItems.size() + index; // try fixing for negative indices + if (index < 0) // still negative means insert at the beginning + this->dequeItems.push_front(item); + else if (index >= this->dequeItems.size()) // still out of range means insert at the end + this->dequeItems.push_back(item); + else + this->dequeItems.insert((this->dequeItems.begin() + index), item); // insert at the given index + } + } + /// @brief marks the deque items for garbage collection + void PyDeque::_gc_mark() const + { + for (PyObject *obj : this->dequeItems) + PK_OBJ_MARK(obj); + } + /// @brief registers the PyDeque class + /// @param vm is needed for the new_module and register_class + void add_module_collections(VM *vm) + { + PyObject *mod = vm->new_module("collections"); + PyDeque::register_class(vm, mod); + PyDequeIter::register_class(vm, vm->builtins); + CodeObject_ code = vm->compile(kPythonLibs["collections"], "collections.py", EXEC_MODE); + vm->_exec(code, mod); + } +} // namespace pkpypkpy diff --git a/src/pocketpy.cpp b/src/pocketpy.cpp index 5fc50fd2..de5d6ecd 100644 --- a/src/pocketpy.cpp +++ b/src/pocketpy.cpp @@ -1649,7 +1649,7 @@ void VM::post_init(){ add_module_base64(this); add_module_timeit(this); - for(const char* name: {"this", "functools", "collections", "heapq", "bisect", "pickle", "_long", "colorsys", "typing", "datetime"}){ + for(const char* name: {"this", "functools", "heapq", "bisect", "pickle", "_long", "colorsys", "typing", "datetime"}){ _lazy_modules[name] = kPythonLibs[name]; } @@ -1672,6 +1672,7 @@ void VM::post_init(){ add_module_linalg(this); add_module_easing(this); + add_module_collections(this); #ifdef PK_USE_BOX2D add_module_box2d(this); diff --git a/tests/70_collections.py b/tests/70_collections.py index 477b16d3..7d967d01 100644 --- a/tests/70_collections.py +++ b/tests/70_collections.py @@ -1,16 +1,831 @@ from collections import Counter, deque +import random +import pickle +import gc +import builtins q = deque() q.append(1) q.append(2) q.appendleft(3) q.append(4) - assert len(q) == 4 - assert q == deque([3, 1, 2, 4]) assert q.popleft() == 3 assert q.pop() == 4 - assert len(q) == 2 -assert q == deque([1, 2]) \ No newline at end of file +assert q == deque([1, 2]) + +# ADDING TESTS FROM CPYTHON's test_deque.py file + +############ TEST basics############### + + +def assertEqual(a, b): + assert a == b +def assertNotEqual(a, b): + assert a != b +def printFailed(function_name, *args, **kwargs): + print("X Failed Tests for {} for args: {} {}".format(str(function_name), str(args), str(kwargs))) + + +BIG = 100000 + + +def fail(): + raise SyntaxError + yield 1 + + +d = deque(range(-5125, -5000)) +# d.__init__(range(200)) # not supported +d = deque(range(200)) +for i in range(200, 400): + d.append(i) +for i in reversed(range(-200, 0)): + d.appendleft(i) + +assertEqual(list(d), list(range(-200, 400))) +assertEqual(len(d), 600) + +left = [d.popleft() for i in range(250)] +assertEqual(left, list(range(-200, 50))) +assertEqual(list(d), list(range(50, 400))) + +right = [d.pop() for i in range(250)] +right.reverse() +assertEqual(right, list(range(150, 400))) +assertEqual(list(d), list(range(50, 150))) + +####### TEST maxlen############### +try: + dq = deque() + dq.maxlen = -1 + printFailed("deque.maxlen", -1) + exit(1) +except AttributeError: + pass + +try: + dq = deque() + dq.maxlen = -2 + printFailed("deque.maxlen", -2) + exit(1) +except AttributeError: + pass + +it = iter(range(10)) +d = deque(it, maxlen=3) +assertEqual(list(it), []) +assertEqual(repr(d), 'deque([7, 8, 9], maxlen=3)') +assertEqual(list(d), [7, 8, 9]) +assertEqual(d, deque(range(10), 3)) +d.append(10) +assertEqual(list(d), [8, 9, 10]) +d.appendleft(7) +assertEqual(list(d), [7, 8, 9]) +d.extend([10, 11]) +assertEqual(list(d), [9, 10, 11]) +d.extendleft([8, 7]) +assertEqual(list(d), [7, 8, 9]) +d = deque(range(200), maxlen=10) +d.append(d) +assertEqual(repr(d)[-30:], ', 198, 199, [...]], maxlen=10)') +d = deque(range(10), maxlen=None) +assertEqual(repr(d), 'deque([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])') + +####### TEST maxlen = 0############### +it = iter(range(100)) +deque(it, maxlen=0) +assertEqual(list(it), []) + +it = iter(range(100)) +d = deque(maxlen=0) +d.extend(it) +assertEqual(list(it), []) + +it = iter(range(100)) +d = deque(maxlen=0) +d.extendleft(it) +assertEqual(list(it), []) + + +####### TEST maxlen attribute ############# + +assertEqual(deque().maxlen, None) +assertEqual(deque('abc').maxlen, None) +assertEqual(deque('abc', maxlen=4).maxlen, 4) +assertEqual(deque('abc', maxlen=2).maxlen, 2) +assertEqual(deque('abc', maxlen=0).maxlen, 0) +try: + d = deque('abc') + d.maxlen = 10 + printFailed("deque.maxlen", 10) + exit(1) +except AttributeError: + pass + +######### TEST count()################# +for s in ('', 'abracadabra', 'simsalabim'*500+'abc'): + s = list(s) + d = deque(s) + for letter in 'abcdefghijklmnopqrstuvwxyz': + assertEqual(s.count(letter), d.count(letter)) +try: + d.count() + printFailed("deque.count") + exit(1) +except TypeError: + pass + +try: + d.count(1, 2) + printFailed("deque.count", 1, 2) + exit(1) +except TypeError: + pass + + +class BadCompare: + def __eq__(self, other): + raise ArithmeticError + + +d = deque([1, 2, BadCompare(), 3]) + +try: + d.count(2) + printFailed("deque.count", 2) + exit(1) +except ArithmeticError: + pass + +d = deque([1, 2, 3]) +try: + d.count(BadCompare()) + printFailed("deque.count", "BadCompare()") + exit(1) +except ArithmeticError: + pass + + +class MutatingCompare: + def __eq__(self, other): + d.pop() + return True + + +m = MutatingCompare() +d = deque([1, 2, 3, m, 4, 5]) +m.d = d + +try: + d.count(3) + printFailed("deque.count", "MutatingCompare()") + exit(1) +except RuntimeError: + pass + +d = deque([None]*16) +for i in range(len(d)): + d.rotate(-1) +d.rotate(1) +assertEqual(d.count(1), 0) +assertEqual(d.count(None), 16) + +#### TEST comparisons == ##### + +d = deque('xabc') +d.popleft() +for e in [d, deque('abc'), deque('ab'), deque(), list(d)]: + assertEqual(d == e, type(d) == type(e) and list(d) == list(e)) + assertEqual(d != e, not (type(d) == type(e) and list(d) == list(e))) + +args = map(deque, ('', 'a', 'b', 'ab', 'ba', 'abc', 'xba', 'xabc', 'cba')) +for x in args: + for y in args: + assertEqual(x == y, list(x) == list(y)) + assertEqual(x != y, list(x) != list(y)) + # assertEqual(x < y, list(x) < list(y)) # not currently supported + # assertEqual(x <= y, list(x) <= list(y)) # not currently supported + # assertEqual(x > y, list(x) > list(y)) # not currently supported + # assertEqual(x >= y, list(x) >= list(y)) # not currently supported + + +############### TEST contains()################# + +n = 200 + +d = deque(range(n)) +for i in range(n): + assertEqual(i in d, True) +assertEqual((n+1) not in d, True) + + +class MutateCmp: + def __init__(self, deque, result): + self.deque = deque + self.result = result + + def __eq__(self, other): + self.deque.clear() + return self.result + + +# # Test detection of mutation during iteration +d = deque(range(n)) +d[n//2] = MutateCmp(d, False) +try: + n in d + printFailed("deque.__contains__", n) + exit(1) +except RuntimeError: + pass + + +class BadCmp: + def __eq__(self, other): + raise RuntimeError + + +# # Test detection of comparison exceptions +d = deque(range(n)) +d[n//2] = BadCmp() +try: + n in d + printFailed("deque.__contains__", n) + exit(1) +except RuntimeError: + pass + + +##### test_contains_count_stop_crashes##### + +class A: + def __eq__(self, other): + d.clear() + return NotImplemented + + +d = deque([A(), A()]) + +try: + _ = 3 in d + printFailed("deque.__contains__", 3) + exit(1) +except RuntimeError: + pass + +d = deque([A(), A()]) +try: + _ = d.count(3) + printFailed("deque.count", 3) + exit(1) +except RuntimeError: + pass + + +######## TEST extend()################ + + +d = deque('a') +try: + d.extend(1) + printFailed("deque.extend", 1) + exit(1) +except TypeError: + pass +d.extend('bcd') +assertEqual(list(d), list('abcd')) +d.extend(d) +assertEqual(list(d), list('abcdabcd')) + +###### TEST extend_left() ################ + +d = deque('a') +try: + d.extendleft(1) + printFailed("deque.extendleft", 1) + exit(1) +except TypeError: + pass +d.extendleft('bcd') +assertEqual(list(d), list(reversed('abcd'))) +d.extendleft(d) +assertEqual(list(d), list('abcddcba')) +d = deque() +d.extendleft(range(1000)) +assertEqual(list(d), list(reversed(range(1000)))) +try: + d.extendleft(fail()) + printFailed("deque.extendleft", fail()) + exit(1) +except SyntaxError: + pass + +##### TEST get_item ################ + +n = 200 +d = deque(range(n)) +l = list(range(n)) +for i in range(n): + d.popleft() + l.pop(0) + if random.random() < 0.5: + d.append(i) + l.append(i) + for j in range(1-len(l), len(l)): + assert d[j] == l[j] + +d = deque('superman') +assertEqual(d[0], 's') +assertEqual(d[-1], 'n') +d = deque() +try: + d.__getitem__(0) + printFailed("deque.__getitem__", 0) + exit(1) +except IndexError: + pass +try: + d.__getitem__(-1) + printFailed("deque.__getitem__", -1) + exit(1) +except IndexError: + pass + + +######### TEST index()############### +for n in 1, 2, 30, 40, 200: + + d = deque(range(n)) + for i in range(n): + assertEqual(d.index(i), i) + + try: + d.index(n+1) + printFailed("deque.index", n+1) + exit(1) + except ValueError: + pass + + # Test detection of mutation during iteration + d = deque(range(n)) + d[n//2] = MutateCmp(d, False) + + try: + d.index(n) + printFailed("deque.index", n) + exit(1) + except RuntimeError: + pass + + # Test detection of comparison exceptions + d = deque(range(n)) + d[n//2] = BadCmp() + + try: + d.index(n) + printFailed("deque.index", n) + exit(1) + except RuntimeError: + pass + + +# Test start and stop arguments behavior matches list.index() +# COMMENT: Current List behavior doesn't support start and stop arguments, so this test is not supported +# elements = 'ABCDEFGHI' +# nonelement = 'Z' +# d = deque(elements * 2) +# s = list(elements * 2) +# for start in range(-5 - len(s)*2, 5 + len(s) * 2): +# for stop in range(-5 - len(s)*2, 5 + len(s) * 2): +# for element in elements + 'Z': +# try: +# print(element, start, stop) +# target = s.index(element, start, stop) +# except ValueError: +# try: +# d.index(element, start, stop) +# print("X Failed Tests!") +# exit(1) +# except ValueError: +# continue +# # with assertRaises(ValueError): +# # d.index(element, start, stop) +# assertEqual(d.index(element, start, stop), target) + + +# Test large start argument +d = deque(range(0, 10000, 10)) +for step in range(100): + i = d.index(8500, 700) + assertEqual(d[i], 8500) + # Repeat test with a different internal offset + d.rotate() + +########### test_index_bug_24913############# +d = deque('A' * 3) +try: + d.index('A', 1, 0) + printFailed("deque.index", 'A', 1, 0) + exit(1) +except ValueError: + pass + +########### test_insert############# + # Test to make sure insert behaves like lists +elements = 'ABCDEFGHI' +for i in range(-5 - len(elements)*2, 5 + len(elements) * 2): + d = deque('ABCDEFGHI') + s = list('ABCDEFGHI') + d.insert(i, 'Z') + s.insert(i, 'Z') + assertEqual(list(d), s) + + +########### test_insert_bug_26194############# +data = 'ABC' +d = deque(data, maxlen=len(data)) +try: + d.insert(0, 'Z') + printFailed("deque.insert", 0, 'Z') + exit(1) +except IndexError: + pass + +elements = 'ABCDEFGHI' +for i in range(-len(elements), len(elements)): + d = deque(elements, maxlen=len(elements)+1) + d.insert(i, 'Z') + if i >= 0: + assertEqual(d[i], 'Z') + else: + assertEqual(d[i-1], 'Z') + + +######### test set_item ############# +n = 200 +d = deque(range(n)) +for i in range(n): + d[i] = 10 * i +assertEqual(list(d), [10*i for i in range(n)]) +l = list(d) +for i in range(1-n, 0, -1): + d[i] = 7*i + l[i] = 7*i +assertEqual(list(d), l) + + +########## test del_item ############# +n = 500 # O(n**2) test, don't make this too big +d = deque(range(n)) +try: + d.__delitem__(-n-1) + printFailed("deque.__delitem__", -n-1) + exit(1) +except IndexError: + pass + +try: + d.__delitem__(n) + printFailed("deque.__delitem__", n) + exit(1) +except IndexError: + pass +for i in range(n): + assertEqual(len(d), n-i) + j = random.randint(0, len(d)-1) + val = d[j] + assertEqual(val in d, True) + del d[j] + assertEqual(val in d, False) +assertEqual(len(d), 0) + + +######### test reverse()############### + +n = 500 # O(n**2) test, don't make this too big +data = [random.random() for i in range(n)] +for i in range(n): + d = deque(data[:i]) + r = d.reverse() + assertEqual(list(d), list(reversed(data[:i]))) + assertEqual(r, None) + d.reverse() + assertEqual(list(d), data[:i]) +try: + d.reverse(1) + printFailed("deque.reverse", 1) + exit(1) +except TypeError: + pass + +############ test rotate############# +s = tuple('abcde') +n = len(s) + +d = deque(s) +d.rotate(1) # verify rot(1) +assertEqual(''.join(d), 'eabcd') + +d = deque(s) +d.rotate(-1) # verify rot(-1) +assertEqual(''.join(d), 'bcdea') +d.rotate() # check default to 1 +assertEqual(tuple(d), s) + +for i in range(n*3): + d = deque(s) + e = deque(d) + d.rotate(i) # check vs. rot(1) n times + for j in range(i): + e.rotate(1) + assertEqual(tuple(d), tuple(e)) + d.rotate(-i) # check that it works in reverse + assertEqual(tuple(d), s) + e.rotate(n-i) # check that it wraps forward + assertEqual(tuple(e), s) + +for i in range(n*3): + d = deque(s) + e = deque(d) + d.rotate(-i) + for j in range(i): + e.rotate(-1) # check vs. rot(-1) n times + assertEqual(tuple(d), tuple(e)) + d.rotate(i) # check that it works in reverse + assertEqual(tuple(d), s) + e.rotate(i-n) # check that it wraps backaround + assertEqual(tuple(e), s) + +d = deque(s) +e = deque(s) +e.rotate(BIG+17) # verify on long series of rotates +dr = d.rotate +for i in range(BIG+17): + dr() +assertEqual(tuple(d), tuple(e)) +try: + d.rotate(1, 2) + printFailed("deque.rotate", 1, 2) + exit(1) +except TypeError: + pass + +try: + d.rotate(1, 10) + printFailed("deque.rotate", 1, 10) + exit(1) +except TypeError: + pass +d = deque() +d.rotate() # rotate an empty deque +assertEqual(d, deque()) + + +########## test len############# + +d = deque('ab') +assertEqual(len(d), 2) +d.popleft() +assertEqual(len(d), 1) +d.pop() +assertEqual(len(d), 0) +try: + d.pop() + printFailed("deque.pop") + exit(1) +except IndexError: + pass +assertEqual(len(d), 0) +d.append('c') +assertEqual(len(d), 1) +d.appendleft('d') +assertEqual(len(d), 2) +d.clear() +assertEqual(len(d), 0) + + +############## test underflow############# +d = deque() +try: + d.pop() + printFailed("deque.pop") + exit(1) +except IndexError: + pass +try: + d.popleft() + printFailed("deque.popleft") + exit(1) +except IndexError: + pass + +############## test clear############# +d = deque(range(100)) +assertEqual(len(d), 100) +d.clear() +assertEqual(len(d), 0) +assertEqual(list(d), []) +d.clear() # clear an empty deque +assertEqual(list(d), []) + + +############# test remove############# +d = deque('abcdefghcij') +d.remove('c') +assertEqual(d, deque('abdefghcij')) +d.remove('c') +assertEqual(d, deque('abdefghij')) +try: + d.remove('c') + printFailed("deque.remove", "c") + exit(1) +except ValueError: + pass +assertEqual(d, deque('abdefghij')) + +# Handle comparison errors +d = deque(['a', 'b', BadCmp(), 'c']) +e = deque(d) + +try: + d.remove('c') + printFailed("deque.remove", "c") + exit(1) +except RuntimeError: + pass +for x, y in zip(d, e): + # verify that original order and values are retained. + assertEqual(x is y, True) + +# Handle evil mutator +for match in (True, False): + d = deque(['ab']) + d.extend([MutateCmp(d, match), 'c']) + try: + d.remove('c') + printFailed("deque.remove", "c") + exit(1) + except IndexError: + pass + assertEqual(d, deque()) + + +########### test repr############# +d = deque(range(200)) +e = eval(repr(d)) +assertEqual(list(d), list(e)) +d.append(d) +assertEqual(repr(d)[-20:], '7, 198, 199, [...]])') + + +######### test init ############# + +try: + deque('abc', 2, 3) + printFailed("deque", 'abc', 2, 3) + exit(1) +except TypeError: + pass +try: + deque(1) + printFailed("deque", 1) + exit(1) +except TypeError: + pass + + +######### test hash ############# +try: + hash(deque('abcd')) +except TypeError: + pass + + +###### test long steady state queue pop left ######## +for size in (0, 1, 2, 100, 1000): + d = deque(range(size)) + append, pop = d.append, d.popleft + for i in range(size, BIG): + append(i) + x = pop() + if x != i - size: + assertEqual(x, i-size) + assertEqual(list(d), list(range(BIG-size, BIG))) + + +######## test long steady state queue pop right ######## +for size in (0, 1, 2, 100, 1000): + d = deque(reversed(range(size))) + append, pop = d.appendleft, d.pop + for i in range(size, BIG): + append(i) + x = pop() + if x != i - size: + assertEqual(x, i-size) + assertEqual(list(reversed(list(d))), + list(range(BIG-size, BIG))) + +###### test big queue popleft ######## +d = deque() +append, pop = d.append, d.popleft +for i in range(BIG): + append(i) +for i in range(BIG): + x = pop() + if x != i: + assertEqual(x, i) + +###### test big queue pop right ######## +d = deque() +append, pop = d.appendleft, d.pop +for i in range(BIG): + append(i) +for i in range(BIG): + x = pop() + if x != i: + assertEqual(x, i) + + +####### test big stack right######## +d = deque() +append, pop = d.append, d.pop +for i in range(BIG): + append(i) +for i in reversed(range(BIG)): + x = pop() + if x != i: + assertEqual(x, i) +assertEqual(len(d), 0) + + +##### test big stack left ######## +d = deque() +append, pop = d.appendleft, d.popleft +for i in range(BIG): + append(i) +for i in reversed(range(BIG)): + x = pop() + if x != i: + assertEqual(x, i) +assertEqual(len(d), 0) + + +##### test roundtrip iter init ######## +d = deque(range(200)) +e = deque(d) +assertNotEqual(id(d), id(e)) +assertEqual(list(d), list(e)) + + +########## test pickle ############# + +for d in deque(range(200)), deque(range(200), 100): + for i in range(5 + 1): + s = pickle.dumps(d) + e = pickle.loads(s) + assertNotEqual(id(e), id(d)) + assertEqual(list(e), list(d)) + assertEqual(e.maxlen, d.maxlen) + +######## test pickle recursive ######## +# the following doesn't work because the pickle module doesn't +# for d in deque('abc'), deque('abc', 3): +# d.append(d) +# for i in range(5 + 1): +# e = pickle.loads(pickle.dumps(d)) +# assertNotEqual(id(e), id(d)) +# assertEqual(id(e[-1]), id(e)) +# assertEqual(e.maxlen, d.maxlen) + + +### test copy ######## + +mut = [10] +d = deque([mut]) +e = d.copy() +assertEqual(list(d), list(e)) +mut[0] = 11 +assertNotEqual(id(d), id(e)) +assertEqual(list(d), list(e)) + +### test reversed#$#### + +for s in ('abcd', range(2000)): + assertEqual(list(reversed(deque(s))), list(reversed(s))) + + +# probably not supported +# klass = type(reversed(deque())) +# for s in ('abcd', range(2000)): +# assertEqual(list(klass(deque(s))), list(reversed(s))) + +d = deque() +for i in range(100): + d.append(1) + gc.collect() + + +print('✓', "ALL TEST PASSED!!")