Zxc200611/pocketpy
1
0
Fork 0
mirror of https://github.com/pocketpy/pocketpy synced 2025-10-21 12:00:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

update gc

Browse Source
This commit is contained in:
blueloveTH 2023-03-28 21:49:50 +08:00
parent 01121c339a
commit 85cfaa4e14
14 changed files with 248 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/ceval.h
View File

@ -5,8 +5,6 @@
namespace pkpy{
Str _read_file_cwd(const Str& name, bool* ok);
inline PyObject* VM::run_frame(Frame* frame){
while(frame->has_next_bytecode()){
const Bytecode& byte = frame->next_bytecode();
@ -183,11 +181,11 @@ inline PyObject* VM::run_frame(Frame* frame){
} continue;
case OP_RE_RAISE: _raise(); continue;
case OP_BUILD_LIST:
frame->push(VAR(frame->pop_n_values_reversed(this, byte.arg).move_to_list()));
frame->push(VAR(frame->pop_n_values_reversed(this, byte.arg).to_list()));
continue;
case OP_BUILD_MAP: {
Args items = frame->pop_n_values_reversed(this, byte.arg*2);
PyObject* obj = call(builtins->attr("dict"));
PyObject* obj = call(builtins->attr("dict"), no_arg());
for(int i=0; i<items.size(); i+=2){
call(obj, __setitem__, Args{items[i], items[i+1]});
}
@ -195,7 +193,7 @@ inline PyObject* VM::run_frame(Frame* frame){
} continue;
case OP_BUILD_SET: {
PyObject* list = VAR(
frame->pop_n_values_reversed(this, byte.arg).move_to_list()
frame->pop_n_values_reversed(this, byte.arg).to_list()
);
PyObject* obj = call(builtins->attr("set"), Args{list});
frame->push(obj);
@ -295,7 +293,7 @@ inline PyObject* VM::run_frame(Frame* frame){
} continue;
case OP_IMPORT_NAME: {
StrName name = frame->co->names[byte.arg].first;
PyObject** ext_mod = _modules.try_get(name);
PyObject* ext_mod = _modules.try_get(name);
if(ext_mod == nullptr){
Str source;
auto it2 = _lazy_modules.find(name);
@ -313,7 +311,7 @@ inline PyObject* VM::run_frame(Frame* frame){
frame->push(new_mod);
new_mod->attr()._try_perfect_rehash();
}else{
frame->push(*ext_mod);
frame->push(ext_mod);
}
} continue;
case OP_STORE_ALL_NAMES: {
@ -326,8 +324,8 @@ inline PyObject* VM::run_frame(Frame* frame){
}; continue;
case OP_YIELD_VALUE: return _py_op_yield;
// TODO: using "goto" inside with block may cause __exit__ not called
case OP_WITH_ENTER: call(frame->pop_value(this), __enter__); continue;
case OP_WITH_EXIT: call(frame->pop_value(this), __exit__); continue;
case OP_WITH_ENTER: call(frame->pop_value(this), __enter__, no_arg()); continue;
case OP_WITH_EXIT: call(frame->pop_value(this), __exit__, no_arg()); continue;
case OP_TRY_BLOCK_ENTER: frame->on_try_block_enter(); continue;
case OP_TRY_BLOCK_EXIT: frame->on_try_block_exit(); continue;
default: throw std::runtime_error(Str("opcode ") + OP_NAMES[byte.op] + " is not implemented");

16
src/codeobject.h
View File

@ -26,16 +26,11 @@ inline const char* OP_NAMES[] = {
struct Bytecode{
uint8_t op;
uint16_t block;
int arg;
int line;
uint16_t block;
};
inline Str pad(const Str& s, const int n){
if(s.size() >= n) return s.substr(0, n);
return s + std::string(n - s.size(), ' ');
}
enum CodeBlockType {
NO_BLOCK,
FOR_LOOP,
@ -49,19 +44,14 @@ struct CodeBlock {
int parent; // parent index in blocks
int start; // start index of this block in codes, inclusive
int end; // end index of this block in codes, exclusive
std::string to_string() const {
if(parent == -1) return "";
return "[B:" + std::to_string(type) + "]";
}
};
struct CodeObject {
shared_ptr<SourceData> src;
std::shared_ptr<SourceData> src;
Str name;
bool is_generator = false;
CodeObject(shared_ptr<SourceData> src, Str name) {
CodeObject(std::shared_ptr<SourceData> src, Str name) {
this->src = src;
this->name = name;
}

15
src/common.h
View File

@ -27,7 +27,7 @@
#include <set>
#include <algorithm>
#include <random>
#include <chrono>
#include <initializer_list>
#define PK_VERSION "0.9.5"
#define PK_EXTRA_CHECK 0
@ -54,20 +54,17 @@ namespace pkpy{
namespace std = ::std;
struct Dummy { };
struct DummyInstance { };
struct Dummy { };
struct DummyInstance { };
struct DummyModule { };
struct Type {
int index;
Type(): index(-1) {}
Type(int index): index(index) {}
bool operator==(Type other) const noexcept {
return this->index == other.index;
}
bool operator!=(Type other) const noexcept {
return this->index != other.index;
}
bool operator==(Type other) const noexcept { return this->index == other.index; }
bool operator!=(Type other) const noexcept { return this->index != other.index; }
operator int() const noexcept { return this->index; }
};
//#define THREAD_LOCAL thread_local

10
src/compiler.h
View File

@ -35,7 +35,7 @@ public:
Compiler(VM* vm, const char* source, Str filename, CompileMode mode){
this->vm = vm;
this->parser = std::make_unique<Parser>(
make_sp<SourceData>(source, filename, mode)
std::make_shared<SourceData>(source, filename, mode)
);
// http://journal.stuffwithstuff.com/2011/03/19/pratt-parsers-expression-parsing-made-easy/
@ -394,7 +394,7 @@ private:
_compile_f_args(func, false);
consume(TK(":"));
}
func.code = make_sp<CodeObject>(parser->src, func.name.str());
func.code = std::make_shared<CodeObject>(parser->src, func.name.str());
this->codes.push(func.code);
co()->_rvalue += 1; EXPR(); co()->_rvalue -= 1;
emit(OP_RETURN_VALUE);
@ -711,7 +711,7 @@ private:
int emit(Opcode opcode, int arg=-1, bool keepline=false) {
int line = parser->prev.line;
co()->codes.push_back(
Bytecode{(uint8_t)opcode, arg, line, (uint16_t)co()->_curr_block_i}
Bytecode{(uint8_t)opcode, (uint16_t)co()->_curr_block_i, arg, line}
);
int i = co()->codes.size() - 1;
if(keepline && i>=1) co()->codes[i].line = co()->codes[i-1].line;
@ -1090,7 +1090,7 @@ private:
if(match(TK("->"))){
if(!match(TK("None"))) consume(TK("@id"));
}
func.code = make_sp<CodeObject>(parser->src, func.name.str());
func.code = std::make_shared<CodeObject>(parser->src, func.name.str());
this->codes.push(func.code);
compile_block_body();
func.code->optimize(vm);
@ -1154,7 +1154,7 @@ public:
if(used) UNREACHABLE();
used = true;
CodeObject_ code = make_sp<CodeObject>(parser->src, Str("<module>"));
CodeObject_ code = std::make_shared<CodeObject>(parser->src, Str("<module>"));
codes.push(code);
lex_token(); lex_token();

2
src/frame.h
View File

@ -21,7 +21,7 @@ struct Frame {
NameDict& f_locals() noexcept { return _locals != nullptr ? *_locals : _module->attr(); }
NameDict& f_globals() noexcept { return _module->attr(); }
PyObject** f_closure_try_get(StrName name) noexcept {
PyObject* f_closure_try_get(StrName name) noexcept {
if(_closure == nullptr) return nullptr;
return _closure->try_get(name);
}

6
src/io.h
View File

@ -157,10 +157,10 @@ inline void add_module_os(VM* vm){
#else
namespace pkpy{
void add_module_io(VM* vm){}
void add_module_os(VM* vm){}
inline void add_module_io(VM* vm){}
inline void add_module_os(VM* vm){}
Str _read_file_cwd(const Str& name, bool* ok){
inline Str _read_file_cwd(const Str& name, bool* ok){
*ok = false;
return Str();
}

82
src/memory.h
View File

@ -4,79 +4,12 @@
namespace pkpy{
template <typename T>
struct shared_ptr {
int* counter;
#define _t() (T*)(counter + 1)
#define _inc_counter() if(counter) ++(*counter)
#define _dec_counter() if(counter && --(*counter) == 0) {((T*)(counter + 1))->~T(); free(counter);}
public:
shared_ptr() : counter(nullptr) {}
shared_ptr(int* counter) : counter(counter) {}
shared_ptr(const shared_ptr& other) : counter(other.counter) {
_inc_counter();
}
shared_ptr(shared_ptr&& other) noexcept : counter(other.counter) {
other.counter = nullptr;
}
~shared_ptr() { _dec_counter(); }
bool operator==(const shared_ptr& other) const { return counter == other.counter; }
bool operator!=(const shared_ptr& other) const { return counter != other.counter; }
bool operator<(const shared_ptr& other) const { return counter < other.counter; }
bool operator>(const shared_ptr& other) const { return counter > other.counter; }
bool operator<=(const shared_ptr& other) const { return counter <= other.counter; }
bool operator>=(const shared_ptr& other) const { return counter >= other.counter; }
bool operator==(std::nullptr_t) const { return counter == nullptr; }
bool operator!=(std::nullptr_t) const { return counter != nullptr; }
shared_ptr& operator=(const shared_ptr& other) {
_dec_counter();
counter = other.counter;
_inc_counter();
return *this;
}
shared_ptr& operator=(shared_ptr&& other) noexcept {
_dec_counter();
counter = other.counter;
other.counter = nullptr;
return *this;
}
T& operator*() const { return *_t(); }
T* operator->() const { return _t(); }
T* get() const { return _t(); }
int use_count() const {
return counter ? *counter : 0;
}
void reset(){
_dec_counter();
counter = nullptr;
}
};
#undef _t
#undef _inc_counter
#undef _dec_counter
template <typename T, typename... Args>
shared_ptr<T> make_sp(Args&&... args) {
int* p = (int*)malloc(sizeof(int) + sizeof(T));
*p = 1;
new(p+1) T(std::forward<Args>(args)...);
return shared_ptr<T>(p);
}
template<typename T, int __Bucket, int __BucketSize=32>
struct SmallArrayPool {
template<typename T, int __Bucket, int __BucketSize=32, bool __ZeroCheck=true>
struct FreeListA {
std::vector<T*> buckets[__Bucket+1];
T* alloc(int n){
if(n == 0) return nullptr;
if constexpr(__ZeroCheck) if(n == 0) return nullptr;
if(n > __Bucket || buckets[n].empty()){
return new T[n];
}else{
@ -87,7 +20,7 @@ struct SmallArrayPool {
}
void dealloc(T* p, int n){
if(n == 0) return;
if constexpr(__ZeroCheck) if(n == 0) return;
if(n > __Bucket || buckets[n].size() >= __BucketSize){
delete[] p;
}else{
@ -95,10 +28,11 @@ struct SmallArrayPool {
}
}
~SmallArrayPool(){
for(int i=1; i<=__Bucket; i++){
for(auto p: buckets[i]) delete[] p;
~FreeListA(){
for(int i=0; i<=__Bucket; i++){
for(T* p : buckets[i]) delete[] p;
}
}
};
}; // namespace pkpy

132
src/namedict.h
View File

@ -6,40 +6,7 @@
namespace pkpy{
const int kNameDictNodeSize = sizeof(StrName) + sizeof(void*);
template<int __Bucket, int __BucketSize=32>
struct DictArrayPool {
std::vector<StrName*> buckets[__Bucket+1];
StrName* alloc(uint16_t n){
StrName* _keys;
if(n > __Bucket || buckets[n].empty()){
_keys = (StrName*)malloc(kNameDictNodeSize * n);
memset((void*)_keys, 0, kNameDictNodeSize * n);
}else{
_keys = buckets[n].back();
memset((void*)_keys, 0, sizeof(StrName) * n);
buckets[n].pop_back();
}
return _keys;
}
void dealloc(StrName* head, uint16_t n){
if(n > __Bucket || buckets[n].size() >= __BucketSize){
free(head);
}else{
buckets[n].push_back(head);
}
}
~DictArrayPool(){
// let it leak, since this object is static
}
};
const std::vector<uint16_t> kHashSeeds = {9629, 43049, 13267, 59509, 39251, 1249, 35803, 54469, 27689, 9719, 34897, 18973, 30661, 19913, 27919, 32143, 3467, 28019, 1051, 39419, 1361, 28547, 48197, 2609, 24317, 22861, 41467, 17623, 52837, 59053, 33589, 32117};
static DictArrayPool<32> _dict_pool;
inline static uint16_t find_next_capacity(uint16_t n){
uint16_t x = 2;
@ -66,75 +33,61 @@ inline static uint16_t find_perfect_hash_seed(uint16_t capacity, const std::vect
}
struct NameDict {
using Item = std::pair<StrName, PyObject*>;
inline static FreeListA<Item, 32, 32, false> _pool;
uint16_t _capacity;
uint16_t _size;
float _load_factor;
uint16_t _hash_seed;
uint16_t _mask;
StrName* _keys;
PyObject*& value(uint16_t i){
return reinterpret_cast<PyObject**>(_keys + _capacity)[i];
}
PyObject* value(uint16_t i) const {
return reinterpret_cast<PyObject**>(_keys + _capacity)[i];
}
Item* _items;
NameDict(uint16_t capacity=2, float load_factor=0.67, uint16_t hash_seed=kHashSeeds[0]):
_capacity(capacity), _size(0), _load_factor(load_factor),
_hash_seed(hash_seed), _mask(capacity-1) {
_keys = _dict_pool.alloc(capacity);
}
_items = _pool.alloc(_capacity);
}
NameDict(const NameDict& other) {
memcpy(this, &other, sizeof(NameDict));
_keys = _dict_pool.alloc(_capacity);
_items = _pool.alloc(_capacity);
for(int i=0; i<_capacity; i++){
_keys[i] = other._keys[i];
value(i) = other.value(i);
_items[i] = other._items[i];
}
}
NameDict& operator=(const NameDict& other) {
_dict_pool.dealloc(_keys, _capacity);
_pool.dealloc(_items, _capacity);
memcpy(this, &other, sizeof(NameDict));
_keys = _dict_pool.alloc(_capacity);
_items = _pool.alloc(_capacity);
for(int i=0; i<_capacity; i++){
_keys[i] = other._keys[i];
value(i) = other.value(i);
_items[i] = other._items[i];
}
return *this;
}
~NameDict(){ _dict_pool.dealloc(_keys, _capacity); }
~NameDict(){ _pool.dealloc(_items, _capacity); }
NameDict(NameDict&&) = delete;
NameDict& operator=(NameDict&&) = delete;
uint16_t size() const { return _size; }
#define HASH_PROBE(key, ok, i) \
ok = false; \
i = _hash(key, _mask, _hash_seed); \
while(!_keys[i].empty()) { \
if(_keys[i] == (key)) { ok = true; break; } \
i = (i + 1) & _mask; \
#define HASH_PROBE(key, ok, i) \
ok = false; \
i = _hash(key, _mask, _hash_seed); \
while(!_items[i].first.empty()) { \
if(_items[i].first == (key)) { ok = true; break; } \
i = (i + 1) & _mask; \
}
PyObject* operator[](StrName key) const {
bool ok; uint16_t i;
HASH_PROBE(key, ok, i);
if(!ok) throw std::out_of_range("NameDict key not found: " + key.str());
return value(i);
return _items[i].second;
}
// PyObject*& get(StrName key){
// bool ok; uint16_t i;
// HASH_PROBE(key, ok, i);
// if(!ok) throw std::out_of_range("NameDict key not found: " + key.str());
// return value(i);
// }
template<typename T>
void set(StrName key, T&& val){
bool ok; uint16_t i;
@ -145,29 +98,27 @@ while(!_keys[i].empty()) { \
_rehash(true);
HASH_PROBE(key, ok, i);
}
_keys[i] = key;
_items[i].first = key;
}
value(i) = std::forward<T>(val);
_items[i].second = std::forward<T>(val);
}
void _rehash(bool resize){
StrName* old_keys = _keys;
PyObject** old_values = &value(0);
Item* old_items = _items;
uint16_t old_capacity = _capacity;
if(resize){
_capacity = find_next_capacity(_capacity * 2);
_mask = _capacity - 1;
}
_keys = _dict_pool.alloc(_capacity);
_items = _pool.alloc(_capacity);
for(uint16_t i=0; i<old_capacity; i++){
if(old_keys[i].empty()) continue;
if(old_items[i].first.empty()) continue;
bool ok; uint16_t j;
HASH_PROBE(old_keys[i], ok, j);
HASH_PROBE(old_items[i].first, ok, j);
if(ok) UNREACHABLE();
_keys[j] = old_keys[i];
value(j) = old_values[i]; // std::move makes a segfault
_items[j] = old_items[i];
}
_dict_pool.dealloc(old_keys, old_capacity);
_pool.dealloc(old_items, old_capacity);
}
void _try_perfect_rehash(){
@ -175,18 +126,18 @@ while(!_keys[i].empty()) { \
_rehash(false); // do not resize
}
PyObject** try_get(StrName key){
PyObject* try_get(StrName key){
bool ok; uint16_t i;
HASH_PROBE(key, ok, i);
if(!ok) return nullptr;
return &value(i);
return _items[i].second;
}
bool try_set(StrName key, PyObject* val){
bool ok; uint16_t i;
HASH_PROBE(key, ok, i);
if(!ok) return false;
value(i) = val;
_items[i].second = val;
return true;
}
@ -198,8 +149,8 @@ while(!_keys[i].empty()) { \
void update(const NameDict& other){
for(uint16_t i=0; i<other._capacity; i++){
if(other._keys[i].empty()) continue;
set(other._keys[i], other.value(i));
auto& item = other._items[i];
if(!item.first.empty()) set(item.first, item.second);
}
}
@ -207,15 +158,16 @@ while(!_keys[i].empty()) { \
bool ok; uint16_t i;
HASH_PROBE(key, ok, i);
if(!ok) throw std::out_of_range("NameDict key not found: " + key.str());
_keys[i] = StrName(); value(i) = nullptr;
_items[i].first = StrName();
_items[i].second = nullptr;
_size--;
}
std::vector<std::pair<StrName, PyObject*>> items() const {
std::vector<std::pair<StrName, PyObject*>> v;
std::vector<Item> items() const {
std::vector<Item> v;
for(uint16_t i=0; i<_capacity; i++){
if(_keys[i].empty()) continue;
v.push_back(std::make_pair(_keys[i], value(i)));
if(_items[i].first.empty()) continue;
v.push_back(_items[i]);
}
return v;
}
@ -223,16 +175,16 @@ while(!_keys[i].empty()) { \
std::vector<StrName> keys() const {
std::vector<StrName> v;
for(uint16_t i=0; i<_capacity; i++){
if(_keys[i].empty()) continue;
v.push_back(_keys[i]);
if(_items[i].first.empty()) continue;
v.push_back(_items[i].first);
}
return v;
}
void apply_v(void(*f)(PyObject*)) {
for(uint16_t i=0; i<_capacity; i++){
if(_keys[i].empty()) continue;
f(value(i));
if(_items[i].first.empty()) continue;
f(_items[i].second);
}
}
#undef HASH_PROBE

4
src/obj.h
View File

@ -13,8 +13,8 @@ struct BaseRef;
class VM;
typedef std::function<PyObject*(VM*, Args&)> NativeFuncRaw;
typedef shared_ptr<CodeObject> CodeObject_;
typedef shared_ptr<NameDict> NameDict_;
typedef std::shared_ptr<CodeObject> CodeObject_;
typedef std::shared_ptr<NameDict> NameDict_;
struct NativeFunc {
NativeFuncRaw f;

4
src/parser.h
View File

@ -95,7 +95,7 @@ enum Precedence {
// The context of the parsing phase for the compiler.
struct Parser {
shared_ptr<SourceData> src;
std::shared_ptr<SourceData> src;
const char* token_start;
const char* curr_char;
@ -290,7 +290,7 @@ struct Parser {
else set_next_token(one);
}
Parser(shared_ptr<SourceData> src) {
Parser(std::shared_ptr<SourceData> src) {
this->src = src;
this->token_start = src->source;
this->curr_char = src->source;

12
src/pocketpy.h
View File

@ -514,7 +514,7 @@ inline void init_builtins(VM* _vm) {
/************ PyTuple ************/
_vm->bind_static_method<1>("tuple", "__new__", [](VM* vm, Args& args) {
List list = CAST(List, vm->asList(args[0]));
return VAR(Tuple::from_list(std::move(list)));
return VAR(Tuple(std::move(list)));
});
_vm->bind_method<0>("tuple", "__iter__", [](VM* vm, Args& args) {
@ -529,7 +529,7 @@ inline void init_builtins(VM* _vm) {
s.normalize(self.size());
List new_list;
for(size_t i = s.start; i < s.stop; i++) new_list.push_back(self[i]);
return VAR(Tuple::from_list(std::move(new_list)));
return VAR(Tuple(std::move(new_list)));
}
int index = CAST(int, args[1]);
@ -601,7 +601,7 @@ inline void add_module_json(VM* vm){
return vm->_exec(code, vm->top_frame()->_module, vm->top_frame()->_locals);
});
vm->bind_func<1>(mod, "dumps", CPP_LAMBDA(vm->call(args[0], __json__)));
vm->bind_func<1>(mod, "dumps", CPP_LAMBDA(vm->call(args[0], __json__, no_arg())));
}
inline void add_module_math(VM* vm){
@ -850,10 +850,10 @@ extern "C" {
/// Return `__repr__` of the result.
/// If the variable is not found, return `nullptr`.
char* pkpy_vm_get_global(pkpy::VM* vm, const char* name){
pkpy::PyObject** val = vm->_main->attr().try_get(name);
pkpy::PyObject* val = vm->_main->attr().try_get(name);
if(val == nullptr) return nullptr;
try{
pkpy::Str repr = pkpy::CAST(pkpy::Str, vm->asRepr(*val));
pkpy::Str repr = pkpy::CAST(pkpy::Str, vm->asRepr(val));
return strdup(repr.c_str());
}catch(...){
return nullptr;
@ -955,7 +955,7 @@ extern "C" {
ss << f_header;
for(int i=0; i<args.size(); i++){
ss << ' ';
pkpy::PyObject* x = vm->call(args[i], pkpy::__json__);
pkpy::PyObject* x = vm->call(args[i], pkpy::__json__, pkpy::no_arg());
ss << pkpy::CAST(pkpy::Str&, x);
}
char* packet = strdup(ss.str().c_str());

10
src/ref.h
View File

@ -19,15 +19,15 @@ struct NameRef : BaseRef {
NameRef(const std::pair<StrName, NameScope>& pair) : pair(pair) {}
PyObject* get(VM* vm, Frame* frame) const{
PyObject** val;
PyObject* val;
val = frame->f_locals().try_get(name());
if(val != nullptr) return *val;
if(val != nullptr) return val;
val = frame->f_closure_try_get(name());
if(val != nullptr) return *val;
if(val != nullptr) return val;
val = frame->f_globals().try_get(name());
if(val != nullptr) return *val;
if(val != nullptr) return val;
val = vm->builtins->attr().try_get(name());
if(val != nullptr) return *val;
if(val != nullptr) return val;
vm->NameError(name());
return nullptr;
}

151
src/tuplelist.h
View File

@ -3,91 +3,84 @@
#include "common.h"
#include "memory.h"
#include "str.h"
#include <initializer_list>
namespace pkpy {
using List = std::vector<PyObject*>;
class Args {
static THREAD_LOCAL SmallArrayPool<PyObject*, 10> _pool;
using List = std::vector<PyObject*>;
PyObject** _args;
int _size;
class Args {
inline static THREAD_LOCAL FreeListA<PyObject*, 10> _pool;
void _alloc(int n){
this->_args = _pool.alloc(n);
this->_size = n;
}
PyObject** _args;
int _size;
public:
Args(int n){ _alloc(n); }
Args(const Args& other){
_alloc(other._size);
for(int i=0; i<_size; i++) _args[i] = other._args[i];
}
Args(Args&& other) noexcept {
this->_args = other._args;
this->_size = other._size;
other._args = nullptr;
other._size = 0;
}
Args(std::initializer_list<PyObject*> list) : Args(list.size()){
int i=0;
for(auto& p : list) _args[i++] = p;
}
static pkpy::Args from_list(List&& other) noexcept {
Args ret(other.size());
memcpy((void*)ret._args, (void*)other.data(), sizeof(void*)*ret.size());
memset((void*)other.data(), 0, sizeof(void*)*ret.size());
other.clear();
return ret;
}
PyObject*& operator[](int i){ return _args[i]; }
PyObject* operator[](int i) const { return _args[i]; }
Args& operator=(Args&& other) noexcept {
_pool.dealloc(_args, _size);
this->_args = other._args;
this->_size = other._size;
other._args = nullptr;
other._size = 0;
return *this;
}
int size() const { return _size; }
List move_to_list() noexcept {
List ret(_size);
memcpy((void*)ret.data(), (void*)_args, sizeof(void*)*_size);
memset((void*)_args, 0, sizeof(void*)*_size);
return ret;
}
void extend_self(PyObject* self){
PyObject** old_args = _args;
int old_size = _size;
_alloc(old_size+1);
_args[0] = self;
if(old_size == 0) return;
memcpy((void*)(_args+1), (void*)old_args, sizeof(void*)*old_size);
memset((void*)old_args, 0, sizeof(void*)*old_size);
_pool.dealloc(old_args, old_size);
}
~Args(){ _pool.dealloc(_args, _size); }
};
inline const Args& no_arg() {
static const Args _zero(0);
return _zero;
void _alloc(int n){
this->_args = _pool.alloc(n);
this->_size = n;
}
typedef Args Tuple;
inline THREAD_LOCAL SmallArrayPool<PyObject*, 10> Args::_pool;
public:
Args(int n){ _alloc(n); }
Args(const Args& other){
_alloc(other._size);
for(int i=0; i<_size; i++) _args[i] = other._args[i];
}
Args(Args&& other) noexcept {
this->_args = other._args;
this->_size = other._size;
other._args = nullptr;
other._size = 0;
}
Args(std::initializer_list<PyObject*> list) : Args(list.size()){
int i = 0;
for(PyObject* p : list) _args[i++] = p;
}
Args(List&& other) noexcept : Args(other.size()){
for(int i=0; i<_size; i++) _args[i] = other[i];
other.clear();
}
PyObject*& operator[](int i){ return _args[i]; }
PyObject* operator[](int i) const { return _args[i]; }
Args& operator=(Args&& other) noexcept {
_pool.dealloc(_args, _size);
this->_args = other._args;
this->_size = other._size;
other._args = nullptr;
other._size = 0;
return *this;
}
int size() const { return _size; }
List to_list() noexcept {
List ret(_size);
for(int i=0; i<_size; i++) ret[i] = _args[i];
return ret;
}
void extend_self(PyObject* self){
PyObject** old_args = _args;
int old_size = _size;
_alloc(old_size+1);
_args[0] = self;
for(int i=0; i<old_size; i++) _args[i+1] = old_args[i];
_pool.dealloc(old_args, old_size);
}
~Args(){ _pool.dealloc(_args, _size); }
};
inline const Args& no_arg() {
static const Args _zero(0);
return _zero;
}
typedef Args Tuple;
} // namespace pkpy

181
src/vm.h
View File

@ -7,24 +7,27 @@
namespace pkpy{
Str _read_file_cwd(const Str& name, bool* ok);
#define DEF_NATIVE_2(ctype, ptype) \
template<> inline ctype py_cast<ctype>(VM* vm, PyObject* obj) { \
template<> inline ctype py_cast<ctype>(VM* vm, PyObject* obj) { \
vm->check_type(obj, vm->ptype); \
return OBJ_GET(ctype, obj); \
} \
template<> inline ctype _py_cast<ctype>(VM* vm, PyObject* obj) { \
template<> inline ctype _py_cast<ctype>(VM* vm, PyObject* obj) { \
return OBJ_GET(ctype, obj); \
} \
template<> inline ctype& py_cast<ctype&>(VM* vm, PyObject* obj) { \
template<> inline ctype& py_cast<ctype&>(VM* vm, PyObject* obj) { \
vm->check_type(obj, vm->ptype); \
return OBJ_GET(ctype, obj); \
} \
template<> inline ctype& _py_cast<ctype&>(VM* vm, PyObject* obj) { \
template<> inline ctype& _py_cast<ctype&>(VM* vm, PyObject* obj) { \
return OBJ_GET(ctype, obj); \
} \
inline PyObject* py_var(VM* vm, const ctype& value) { return vm->gcnew(vm->ptype, value);} \
inline PyObject* py_var(VM* vm, ctype&& value) { return vm->gcnew(vm->ptype, std::move(value));}
class Generator: public BaseIter {
std::unique_ptr<Frame> frame;
int state; // 0,1,2
@ -61,6 +64,7 @@ public:
PyObject* False;
PyObject* Ellipsis;
// managed by _modules, need_gc=false
PyObject* builtins; // builtins module
PyObject* _main; // __main__ module
@ -83,12 +87,6 @@ public:
init_builtin_types();
}
PyObject* asStr(PyObject* obj){
PyObject* f = getattr(obj, __str__, false, true);
if(f != nullptr) return call(f);
return asRepr(obj);
}
Frame* top_frame() const {
#if PK_EXTRA_CHECK
if(callstack.empty()) UNREACHABLE();
@ -96,10 +94,16 @@ public:
return callstack.top().get();
}
PyObject* asStr(PyObject* obj){
PyObject* f = getattr(obj, __str__, false, true);
if(f != nullptr) return call(f, no_arg());
return asRepr(obj);
}
PyObject* asIter(PyObject* obj){
if(is_type(obj, tp_iterator)) return obj;
PyObject* iter_f = getattr(obj, __iter__, false, true);
if(iter_f != nullptr) return call(iter_f);
if(iter_f != nullptr) return call(iter_f, no_arg());
TypeError(OBJ_NAME(_t(obj)).escape(true) + " object is not iterable");
return nullptr;
}
@ -109,15 +113,15 @@ public:
return call(_t(tp_list), Args{iterable});
}
PyObject** find_name_in_mro(PyObject* cls, StrName name){
PyObject** val;
PyObject* find_name_in_mro(PyObject* cls, StrName name){
PyObject* val;
do{
val = cls->attr().try_get(name);
if(val != nullptr) return val;
Type cls_t = static_cast<Py_<Type>*>(cls)->_value;
Type base = _all_types[cls_t.index].base;
Type base = _all_types[cls_t].base;
if(base.index == -1) break;
cls = _all_types[base.index].obj;
cls = _all_types[base].obj;
}while(true);
return nullptr;
}
@ -126,7 +130,7 @@ public:
Type obj_t = OBJ_GET(Type, _t(obj));
do{
if(obj_t == cls_t) return true;
Type base = _all_types[obj_t.index].base;
Type base = _all_types[obj_t].base;
if(base.index == -1) break;
obj_t = base;
}while(true);
@ -134,8 +138,8 @@ public:
}
PyObject* fast_call(StrName name, Args&& args){
PyObject** val = find_name_in_mro(_t(args[0]), name);
if(val != nullptr) return call(*val, std::move(args));
PyObject* val = find_name_in_mro(_t(args[0]), name);
if(val != nullptr) return call(val, std::move(args));
AttributeError(args[0], name);
return nullptr;
}
@ -147,28 +151,19 @@ public:
return obj;
}
PyObject* call(PyObject* callable){
return call(callable, no_arg(), no_arg(), false);
}
template<typename ArgT>
std::enable_if_t<std::is_same_v<std::decay_t<ArgT>, Args>, PyObject*>
call(PyObject* _callable, ArgT&& args){
return call(_callable, std::forward<ArgT>(args), no_arg(), false);
call(PyObject* callable, ArgT&& args){
return call(callable, std::forward<ArgT>(args), no_arg(), false);
}
template<typename ArgT>
std::enable_if_t<std::is_same_v<std::decay_t<ArgT>, Args>, PyObject*>
call(PyObject* obj, const StrName name, ArgT&& args){
return call(getattr(obj, name, true, true), std::forward<ArgT>(args), no_arg(), false);
PyObject* callable = getattr(obj, name, true, true);
return call(callable, std::forward<ArgT>(args), no_arg(), false);
}
PyObject* call(PyObject* obj, StrName name){
return call(getattr(obj, name, true, true), no_arg(), no_arg(), false);
}
// repl mode is only for setting `frame->id` to 0
PyObject* exec(Str source, Str filename, CompileMode mode, PyObject* _module=nullptr){
if(_module == nullptr) _module = _main;
try {
@ -205,7 +200,6 @@ public:
}
PyObject* new_type_object(PyObject* mod, StrName name, Type base){
// use gcnew
PyObject* obj = new Py_<Type>(tp_type, _all_types.size());
PyTypeInfo info{
.obj = obj,
@ -223,12 +217,12 @@ public:
}
PyObject* _find_type(const Str& type){
PyObject** obj = builtins->attr().try_get(type);
if(!obj){
PyObject* obj = builtins->attr().try_get(type);
if(obj == nullptr){
for(auto& t: _all_types) if(t.name == type) return t.obj;
throw std::runtime_error("type not found: " + type);
}
return *obj;
return obj;
}
template<int ARGC>
@ -294,7 +288,6 @@ public:
else throw UnhandledException();
}
public:
void IOError(const Str& msg) { _error("IOError", msg); }
void NotImplementedError(){ _error("NotImplementedError", ""); }
void TypeError(const Str& msg){ _error("TypeError", msg); }
@ -332,7 +325,6 @@ public:
}
CodeObject_ compile(Str source, Str filename, CompileMode mode);
void post_init();
PyObject* num_negated(PyObject* obj);
f64 num_to_float(PyObject* obj);
PyObject* asBool(PyObject* obj);
@ -341,21 +333,20 @@ public:
PyObject* new_module(StrName name);
Str disassemble(CodeObject_ co);
void init_builtin_types();
PyObject* call(PyObject* _callable, Args args, const Args& kwargs, bool opCall);
PyObject* call(PyObject* callable, Args args, const Args& kwargs, bool opCall);
void unpack_args(Args& args);
PyObject* getattr(PyObject* obj, StrName name, bool throw_err=true, bool class_only=false);
template<typename T>
void setattr(PyObject* obj, StrName name, T&& value);
template<int ARGC>
void bind_method(PyObject* obj, Str funcName, NativeFuncRaw fn);
void bind_method(PyObject*, Str, NativeFuncRaw);
template<int ARGC>
void bind_func(PyObject* obj, Str funcName, NativeFuncRaw fn);
void _error(Exception e);
void bind_func(PyObject*, Str, NativeFuncRaw);
void _error(Exception);
PyObject* _exec();
template<typename P>
PyObject* PyRef(P&& value);
template<typename P> PyObject* PyRef(P&&);
const BaseRef* PyRef_AS_C(PyObject* obj);
void post_init();
};
inline PyObject* NativeFunc::operator()(VM* vm, Args& args) const{
@ -418,13 +409,13 @@ DEF_NATIVE_2(Slice, tp_slice)
DEF_NATIVE_2(Exception, tp_exception)
DEF_NATIVE_2(StarWrapper, tp_star_wrapper)
#define PY_CAST_INT(T) \
template<> inline T py_cast<T>(VM* vm, PyObject* obj){ \
vm->check_type(obj, vm->tp_int); \
return (T)(BITS(obj) >> 2); \
} \
template<> inline T _py_cast<T>(VM* vm, PyObject* obj){ \
return (T)(BITS(obj) >> 2); \
#define PY_CAST_INT(T) \
template<> inline T py_cast<T>(VM* vm, PyObject* obj){ \
vm->check_type(obj, vm->tp_int); \
return (T)(BITS(obj) >> 2); \
} \
template<> inline T _py_cast<T>(VM* vm, PyObject* obj){ \
return (T)(BITS(obj) >> 2); \
}
PY_CAST_INT(char)
@ -463,10 +454,10 @@ template<> inline double _py_cast<double>(VM* vm, PyObject* obj){
}
#define PY_VAR_INT(T) \
inline PyObject* py_var(VM* vm, T _val){ \
i64 val = static_cast<i64>(_val); \
if(((val << 2) >> 2) != val){ \
#define PY_VAR_INT(T) \
inline PyObject* py_var(VM* vm, T _val){ \
i64 val = static_cast<i64>(_val); \
if(((val << 2) >> 2) != val){ \
vm->_error("OverflowError", std::to_string(val) + " is out of range"); \
} \
val = (val << 2) | 0b01; \
@ -485,9 +476,9 @@ PY_VAR_INT(unsigned long)
PY_VAR_INT(unsigned long long)
#define PY_VAR_FLOAT(T) \
inline PyObject* py_var(VM* vm, T _val){ \
inline PyObject* py_var(VM* vm, T _val){ \
f64 val = static_cast<f64>(_val); \
i64 bits = BitsCvt(val)._int; \
i64 bits = BitsCvt(val)._int; \
bits = (bits >> 2) << 2; \
bits |= 0b10; \
return reinterpret_cast<PyObject*>(bits); \
@ -546,9 +537,9 @@ inline PyObject* VM::asBool(PyObject* obj){
if(obj == None) return False;
if(is_type(obj, tp_int)) return VAR(CAST(i64, obj) != 0);
if(is_type(obj, tp_float)) return VAR(CAST(f64, obj) != 0.0);
PyObject* len_fn = getattr(obj, __len__, false, true);
if(len_fn != nullptr){
PyObject* ret = call(len_fn);
PyObject* len_f = getattr(obj, __len__, false, true);
if(len_f != nullptr){
PyObject* ret = call(len_f, no_arg());
return VAR(CAST(i64, ret) > 0);
}
return True;
@ -578,7 +569,7 @@ inline i64 VM::hash(PyObject* obj){
}
inline PyObject* VM::asRepr(PyObject* obj){
return call(obj, __repr__);
return call(obj, __repr__, no_arg());
}
inline PyObject* VM::new_module(StrName name) {
@ -592,6 +583,11 @@ inline PyObject* VM::new_module(StrName name) {
}
inline Str VM::disassemble(CodeObject_ co){
auto pad = [](const Str& s, const int n){
if(s.size() >= n) return s.substr(0, n);
return s + std::string(n - s.size(), ' ');
};
std::vector<int> jumpTargets;
for(auto byte : co->codes){
if(byte.op == OP_JUMP_ABSOLUTE || byte.op == OP_SAFE_JUMP_ABSOLUTE || byte.op == OP_POP_JUMP_IF_FALSE){
@ -633,8 +629,9 @@ inline Str VM::disassemble(CodeObject_ co){
auto& x = co->names[(byte.arg >> 16) & 0xFFFF];
argStr += " (" + a.first.str() + '[' + x.first.str() + "])";
}
ss << pad(argStr, 20); // may overflow
ss << co->blocks[byte.block].to_string();
ss << argStr;
// ss << pad(argStr, 20); // may overflow
// ss << co->blocks[byte.block].to_string();
if(i != co->codes.size() - 1) ss << '\n';
}
StrStream consts;
@ -661,13 +658,11 @@ inline Str VM::disassemble(CodeObject_ co){
}
inline void VM::init_builtin_types(){
PyObject* _tp_object = new Py_<Type>(Type(1), Type(0));
PyObject* _tp_type = new Py_<Type>(Type(1), Type(1));
// PyTypeObject is managed by _all_types
// PyModuleObject is managed by _modules
// They are not managed by GC, so we use a simple "new"
_all_types.push_back({.obj = _tp_object, .base = -1, .name = "object"});
_all_types.push_back({.obj = _tp_type, .base = 0, .name = "type"});
_all_types.push_back({.obj = new Py_<Type>(Type(1), Type(0)), .base = -1, .name = "object"});
_all_types.push_back({.obj = new Py_<Type>(Type(1), Type(1)), .base = 0, .name = "type"});
tp_object = 0; tp_type = 1;
tp_int = _new_type_object("int");
@ -683,7 +678,6 @@ inline void VM::init_builtin_types(){
tp_module = _new_type_object("module");
tp_ref = _new_type_object("_ref");
tp_star_wrapper = _new_type_object("_star_wrapper");
tp_function = _new_type_object("function");
tp_native_function = _new_type_object("native_function");
tp_iterator = _new_type_object("iterator");
@ -697,6 +691,7 @@ inline void VM::init_builtin_types(){
this->False = new Py_<Dummy>(tp_bool, {});
this->_py_op_call = new Py_<Dummy>(_new_type_object("_py_op_call"), {});
this->_py_op_yield = new Py_<Dummy>(_new_type_object("_py_op_yield"), {});
this->builtins = new_module("builtins");
this->_main = new_module("__main__");
@ -712,19 +707,16 @@ inline void VM::init_builtin_types(){
builtins->attr().set("range", _t(tp_range));
post_init();
for(int i=0; i<_all_types.size(); i++){
auto& t = _all_types[i];
t.obj->attr()._try_perfect_rehash();
}
for(auto& t: _all_types) t.obj->attr()._try_perfect_rehash();
for(auto [k, v]: _modules.items()) v->attr()._try_perfect_rehash();
}
inline PyObject* VM::call(PyObject* callable, Args args, const Args& kwargs, bool opCall){
if(is_type(callable, tp_type)){
PyObject** new_f = callable->attr().try_get(__new__);
PyObject* new_f = callable->attr().try_get(__new__);
PyObject* obj;
if(new_f != nullptr){
obj = call(*new_f, std::move(args), kwargs, false);
obj = call(new_f, std::move(args), kwargs, false);
}else{
obj = gcnew<DummyInstance>(OBJ_GET(Type, callable), {});
PyObject* init_f = getattr(obj, __init__, false, true);
@ -745,7 +737,7 @@ inline PyObject* VM::call(PyObject* callable, Args args, const Args& kwargs, boo
return f(this, args);
} else if(is_type(callable, tp_function)){
const Function& fn = CAST(Function&, callable);
NameDict_ locals = make_sp<NameDict>(
NameDict_ locals = std::make_shared<NameDict>(
fn.code->perfect_locals_capacity,
kLocalsLoadFactor,
fn.code->perfect_hash_seed
@ -754,7 +746,7 @@ inline PyObject* VM::call(PyObject* callable, Args args, const Args& kwargs, boo
int i = 0;
for(StrName name : fn.args){
if(i < args.size()){
locals->set(name, std::move(args[i++]));
locals->set(name, args[i++]);
continue;
}
TypeError("missing positional argument " + name.str().escape(true));
@ -764,12 +756,12 @@ inline PyObject* VM::call(PyObject* callable, Args args, const Args& kwargs, boo
if(!fn.starred_arg.empty()){
List vargs; // handle *args
while(i < args.size()) vargs.push_back(std::move(args[i++]));
locals->set(fn.starred_arg, VAR(Tuple::from_list(std::move(vargs))));
while(i < args.size()) vargs.push_back(args[i++]);
locals->set(fn.starred_arg, VAR(Tuple(std::move(vargs))));
}else{
for(StrName key : fn.kwargs_order){
if(i < args.size()){
locals->set(key, std::move(args[i++]));
locals->set(key, args[i++]);
}else{
break;
}
@ -806,14 +798,13 @@ inline void VM::unpack_args(Args& args){
if(is_type(args[i], tp_star_wrapper)){
auto& star = _CAST(StarWrapper&, args[i]);
if(!star.rvalue) UNREACHABLE();
PyObject* list = asList(star.obj);
List& list_c = CAST(List&, list);
unpacked.insert(unpacked.end(), list_c.begin(), list_c.end());
List& list = CAST(List&, asList(star.obj));
unpacked.insert(unpacked.end(), list.begin(), list.end());
}else{
unpacked.push_back(args[i]);
}
}
args = Args::from_list(std::move(unpacked));
args = Args(std::move(unpacked));
}
using Super = std::pair<PyObject*, Type>;
@ -827,23 +818,23 @@ inline PyObject* VM::getattr(PyObject* obj, StrName name, bool throw_err, bool c
obj = super.first;
objtype = _t(super.second);
}
PyObject** cls_var = find_name_in_mro(objtype, name);
PyObject* cls_var = find_name_in_mro(objtype, name);
if(cls_var != nullptr){
// handle descriptor
PyObject** descr_get = _t(*cls_var)->attr().try_get(__get__);
if(descr_get != nullptr) return call(*descr_get, Args{*cls_var, obj});
PyObject* descr_get = _t(cls_var)->attr().try_get(__get__);
if(descr_get != nullptr) return call(descr_get, Args{cls_var, obj});
}
// handle instance __dict__
if(!class_only && !is_tagged(obj) && obj->is_attr_valid()){
PyObject** val = obj->attr().try_get(name);
if(val != nullptr) return *val;
PyObject* val = obj->attr().try_get(name);
if(val != nullptr) return val;
}
if(cls_var != nullptr){
// bound method is non-data descriptor
if(is_type(*cls_var, tp_function) || is_type(*cls_var, tp_native_function)){
return VAR(BoundMethod(obj, *cls_var));
if(is_type(cls_var, tp_function) || is_type(cls_var, tp_native_function)){
return VAR(BoundMethod(obj, cls_var));
}
return *cls_var;
return cls_var;
}
if(throw_err) AttributeError(obj, name);
return nullptr;
@ -859,14 +850,14 @@ inline void VM::setattr(PyObject* obj, StrName name, T&& value){
obj = super.first;
objtype = _t(super.second);
}
PyObject** cls_var = find_name_in_mro(objtype, name);
PyObject* cls_var = find_name_in_mro(objtype, name);
if(cls_var != nullptr){
// handle descriptor
PyObject* cls_var_t = _t(*cls_var);
PyObject* cls_var_t = _t(cls_var);
if(cls_var_t->attr().contains(__get__)){
PyObject** descr_set = cls_var_t->attr().try_get(__set__);
PyObject* descr_set = cls_var_t->attr().try_get(__set__);
if(descr_set != nullptr){
call(*descr_set, Args{*cls_var, obj, std::forward<T>(value)});
call(descr_set, Args{cls_var, obj, std::forward<T>(value)});
}else{
TypeError("readonly attribute: " + name.str().escape(true));
}