Zxc200611/pocketpy
1
0
Fork 0
mirror of https://github.com/pocketpy/pocketpy synced 2025-10-24 05:20:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

up

Browse Source
This commit is contained in:
blueloveTH 2023-04-02 22:33:47 +08:00
parent 718ba988d5
commit 6bb307734c
6 changed files with 178 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
src/compiler.h
View File

@ -582,20 +582,41 @@ class Compiler {
} }
bool try_compile_assignment(){ bool try_compile_assignment(){
// switch (op) { Expr_ lhs = ctx()->s_expr.popx();
// case TK("+="): emit(OP_BINARY_OP, 0); break; switch (curr().type) {
// case TK("-="): emit(OP_BINARY_OP, 1); break; // case TK("+="): lhs->emit(ctx()); advance(); emit(OP_BINARY_OP, 0); break;
// case TK("*="): emit(OP_BINARY_OP, 2); break; // case TK("-="): lhs->emit(ctx()); advance(); emit(OP_BINARY_OP, 1); break;
// case TK("/="): emit(OP_BINARY_OP, 3); break; // case TK("*="): lhs->emit(ctx()); advance(); emit(OP_BINARY_OP, 2); break;
// case TK("//="): emit(OP_BINARY_OP, 4); break; // case TK("/="): lhs->emit(ctx()); advance(); emit(OP_BINARY_OP, 3); break;
// case TK("%="): emit(OP_BINARY_OP, 5); break; // case TK("//="): lhs->emit(ctx()); advance(); emit(OP_BINARY_OP, 4); break;
// case TK("<<="): emit(OP_BITWISE_OP, 0); break; // case TK("%="): lhs->emit(ctx()); advance(); emit(OP_BINARY_OP, 5); break;
// case TK(">>="): emit(OP_BITWISE_OP, 1); break; // case TK("<<="): lhs->emit(ctx()); advance(); emit(OP_BITWISE_OP, 0); break;
// case TK("&="): emit(OP_BITWISE_OP, 2); break; // case TK(">>="): lhs->emit(ctx()); advance(); emit(OP_BITWISE_OP, 1); break;
// case TK("|="): emit(OP_BITWISE_OP, 3); break; // case TK("&="): lhs->emit(ctx()); advance(); emit(OP_BITWISE_OP, 2); break;
// case TK("^="): emit(OP_BITWISE_OP, 4); break; // case TK("|="): lhs->emit(ctx()); advance(); emit(OP_BITWISE_OP, 3); break;
// default: UNREACHABLE(); // case TK("^="): lhs->emit(ctx()); advance(); emit(OP_BITWISE_OP, 4); break;
// } // case TK("="): advance(); break;
case TK("+="): case TK("-="): case TK("*="): case TK("/="): case TK("//="): case TK("%="):
case TK("<<="): case TK(">>="): case TK("&="): case TK("|="): case TK("^="): {
advance();
auto e = make_expr<BinaryExpr>();
e->op = prev().type;
e->lhs = lhs; // here should be a copy
EXPR_TUPLE();
e->rhs = ctx()->s_expr.popx();
// ...
} break;
case TK("="): advance(); break;
default: return false;
}
if(prev().type == TK("=")){
EXPR_TUPLE();
Expr_ rhs = ctx()->s_expr.popx();
// do assign here
// lhs = rhs
return true;
}
return true;
} }
void compile_stmt() { void compile_stmt() {
@ -704,8 +725,9 @@ class Compiler {
/*************************************************/ /*************************************************/
// handle dangling expression or assignment // handle dangling expression or assignment
default: { default: {
EXPR_TUPLE(true); EXPR_TUPLE();
if(!try_compile_assignment()){ if(!try_compile_assignment()){
ctx()->emit_expr();
if(mode()==REPL_MODE && name_scope()==NAME_GLOBAL){ if(mode()==REPL_MODE && name_scope()==NAME_GLOBAL){
ctx()->emit(OP_PRINT_EXPR, BC_NOARG, BC_KEEPLINE); ctx()->emit(OP_PRINT_EXPR, BC_NOARG, BC_KEEPLINE);
}else{ }else{

94
src/expr.h
View File

@ -18,6 +18,7 @@ struct Expr{
virtual std::vector<const Expr*> children() const { return {}; } virtual std::vector<const Expr*> children() const { return {}; }
virtual bool is_starred() const { return false; } virtual bool is_starred() const { return false; }
virtual bool is_literal() const { return false; }
// for OP_DELETE_XXX // for OP_DELETE_XXX
virtual bool emit_del(CodeEmitContext* ctx) { return false; } virtual bool emit_del(CodeEmitContext* ctx) { return false; }
@ -162,20 +163,6 @@ struct StarredExpr: Expr{
} }
}; };
// PASS
struct NegatedExpr: Expr{
Expr_ child;
NegatedExpr(Expr_&& child): child(std::move(child)) {}
Str str() const override { return "-"; }
std::vector<const Expr*> children() const override { return {child.get()}; }
void emit(CodeEmitContext* ctx) override {
child->emit(ctx);
ctx->emit(OP_UNARY_NEGATIVE, BC_NOARG, line);
}
};
// PASS // PASS
struct NotExpr: Expr{ struct NotExpr: Expr{
Expr_ child; Expr_ child;
@ -265,19 +252,48 @@ struct LiteralExpr: Expr{
if(std::holds_alternative<i64>(value)){ if(std::holds_alternative<i64>(value)){
obj = VAR(std::get<i64>(value)); obj = VAR(std::get<i64>(value));
} }
if(std::holds_alternative<f64>(value)){ if(std::holds_alternative<f64>(value)){
obj = VAR(std::get<f64>(value)); obj = VAR(std::get<f64>(value));
} }
if(std::holds_alternative<Str>(value)){ if(std::holds_alternative<Str>(value)){
obj = VAR(std::get<Str>(value)); obj = VAR(std::get<Str>(value));
} }
if(obj == nullptr) UNREACHABLE();
if(!obj) UNREACHABLE();
int index = ctx->add_const(obj); int index = ctx->add_const(obj);
ctx->emit(OP_LOAD_CONST, index, line); ctx->emit(OP_LOAD_CONST, index, line);
} }
bool is_literal() const override { return true; }
};
// PASS
struct NegatedExpr: Expr{
Expr_ child;
NegatedExpr(Expr_&& child): child(std::move(child)) {}
Str str() const override { return "-"; }
std::vector<const Expr*> children() const override { return {child.get()}; }
void emit(CodeEmitContext* ctx) override {
VM* vm = ctx->vm;
// if child is a int of float, do constant folding
if(child->is_literal()){
LiteralExpr* lit = static_cast<LiteralExpr*>(child.get());
PyObject* obj = nullptr;
if(std::holds_alternative<i64>(lit->value)){
obj = VAR(std::get<i64>(lit->value));
}
if(std::holds_alternative<f64>(lit->value)){
obj = VAR(std::get<f64>(lit->value));
}
if(obj != nullptr){
ctx->emit(OP_LOAD_CONST, ctx()->add_const(obj), line);
return;
}
}
child->emit(ctx);
ctx->emit(OP_UNARY_NEGATIVE, BC_NOARG, line);
}
}; };
// PASS // PASS
@ -630,45 +646,3 @@ struct TernaryExpr: Expr{
} // namespace pkpy } // namespace pkpy
// struct TupleRef : BaseRef {
// Tuple objs;
// TupleRef(Tuple&& objs) : objs(std::move(objs)) {}
// PyObject* get(VM* vm, Frame* frame) const{
// Tuple args(objs.size());
// for (int i = 0; i < objs.size(); i++) {
// args[i] = vm->PyRef_AS_C(objs[i])->get(vm, frame);
// }
// return VAR(std::move(args));
// }
// void set(VM* vm, Frame* frame, PyObject* val) const{
// val = vm->asIter(val);
// BaseIter* iter = vm->PyIter_AS_C(val);
// for(int i=0; i<objs.size(); i++){
// PyObject* x;
// if(is_type(objs[i], vm->tp_star_wrapper)){
// auto& star = _CAST(StarWrapper&, objs[i]);
// if(star.rvalue) vm->ValueError("can't use starred expression here");
// if(i != objs.size()-1) vm->ValueError("* can only be used at the end");
// auto ref = vm->PyRef_AS_C(star.obj);
// List list;
// while((x = iter->next()) != nullptr) list.push_back(x);
// ref->set(vm, frame, VAR(std::move(list)));
// return;
// }else{
// x = iter->next();
// if(x == nullptr) vm->ValueError("not enough values to unpack");
// vm->PyRef_AS_C(objs[i])->set(vm, frame, x);
// }
// }
// PyObject* x = iter->next();
// if(x != nullptr) vm->ValueError("too many values to unpack");
// }
// void del(VM* vm, Frame* frame) const{
// for(int i=0; i<objs.size(); i++) vm->PyRef_AS_C(objs[i])->del(vm, frame);
// }
// };

41
src/iter.h
View File

@ -6,10 +6,10 @@ namespace pkpy{
class RangeIter : public BaseIter { class RangeIter : public BaseIter {
i64 current; i64 current;
Range r; Range r; // copy by value, so we don't need to keep ref
public: public:
RangeIter(VM* vm, PyObject* _ref) : BaseIter(vm, _ref) { RangeIter(VM* vm, PyObject* ref) : BaseIter(vm) {
this->r = OBJ_GET(Range, _ref); this->r = OBJ_GET(Range, ref);
this->current = r.start; this->current = r.start;
} }
@ -26,28 +26,37 @@ public:
template <typename T> template <typename T>
class ArrayIter : public BaseIter { class ArrayIter : public BaseIter {
size_t index = 0; int index;
const T* p; PyObject* ref;
public: public:
ArrayIter(VM* vm, PyObject* _ref) : BaseIter(vm, _ref) { p = &OBJ_GET(T, _ref);} ArrayIter(VM* vm, PyObject* ref) : BaseIter(vm), ref(ref), index(0) {}
PyObject* next(){
PyObject* next() override{
const T* p = &OBJ_GET(T, ref);
if(index == p->size()) return nullptr; if(index == p->size()) return nullptr;
return p->operator[](index++); return p->operator[](index++);
} }
void _mark() override {
OBJ_MARK(ref);
}
}; };
class StringIter : public BaseIter { class StringIter : public BaseIter {
int index = 0; int index = 0;
Str* str; PyObject* ref;
public: public:
StringIter(VM* vm, PyObject* _ref) : BaseIter(vm, _ref) { StringIter(VM* vm, PyObject* ref) : BaseIter(vm), ref(ref) {}
str = &OBJ_GET(Str, _ref);
}
PyObject* next() { PyObject* next() override{
Str* str = &OBJ_GET(Str, ref);
if(index == str->u8_length()) return nullptr; if(index == str->u8_length()) return nullptr;
return VAR(str->u8_getitem(index++)); return VAR(str->u8_getitem(index++));
} }
void _mark() override {
OBJ_MARK(ref);
}
}; };
inline PyObject* Generator::next(){ inline PyObject* Generator::next(){
@ -58,20 +67,14 @@ inline PyObject* Generator::next(){
frame = std::move(vm->callstack.top()); frame = std::move(vm->callstack.top());
vm->callstack.pop(); vm->callstack.pop();
state = 1; state = 1;
return frame->pop_value(vm); return frame->popx();
}else{ }else{
state = 2; state = 2;
return nullptr; return nullptr;
} }
} }
inline void BaseIter::_mark() {
if(_ref != nullptr) OBJ_MARK(_ref);
if(loop_var != nullptr) OBJ_MARK(loop_var);
}
inline void Generator::_mark(){ inline void Generator::_mark(){
BaseIter::_mark();
if(frame!=nullptr) frame->_mark(); if(frame!=nullptr) frame->_mark();
} }

27
src/lexer.h
View File

@ -9,21 +9,23 @@ namespace pkpy{
typedef uint8_t TokenIndex; typedef uint8_t TokenIndex;
constexpr const char* kTokens[] = { constexpr const char* kTokens[] = {
"is not", "not in",
"@eof", "@eol", "@sof", "@eof", "@eol", "@sof",
".", ",", ":", ";", "#", "(", ")", "[", "]", "{", "}", "%", "::", "@id", "@num", "@str", "@fstr",
"+", "-", "*", "/", "//", "**", "=", ">", "<", "...", "->", "@indent", "@dedent",
"<<", ">>", "&", "|", "^", "?", "@", /*****************************************/
"==", "!=", ">=", "<=", "+", "+=", "-", "-=", // (INPLACE_OP - 1) can get '=' removed
"+=", "-=", "*=", "/=", "//=", "%=", "&=", "|=", "^=", ">>=", "<<=", "*", "*=", "/", "/=", "//", "//=", "%", "%=",
"&", "&=", "|", "|=", "^", "^=",
"<<", "<<=", ">>", ">>=",
/*****************************************/
".", ",", ":", ";", "#", "(", ")", "[", "]", "{", "}", "::",
"**", "=", ">", "<", "...", "->", "?", "@", "==", "!=", ">=", "<=",
/** KW_BEGIN **/ /** KW_BEGIN **/
"class", "import", "as", "def", "lambda", "pass", "del", "from", "with", "yield", "class", "import", "as", "def", "lambda", "pass", "del", "from", "with", "yield",
"None", "in", "is", "and", "or", "not", "True", "False", "global", "try", "except", "finally", "None", "in", "is", "and", "or", "not", "True", "False", "global", "try", "except", "finally",
"goto", "label", // extended keywords, not available in cpython "goto", "label", // extended keywords, not available in cpython
"while", "for", "if", "elif", "else", "break", "continue", "return", "assert", "raise", "while", "for", "if", "elif", "else", "break", "continue", "return", "assert", "raise"
/** KW_END **/
"is not", "not in",
"@id", "@num", "@str", "@fstr",
"@indent", "@dedent"
}; };
using TokenValue = std::variant<std::monostate, i64, f64, Str>; using TokenValue = std::variant<std::monostate, i64, f64, Str>;
@ -40,12 +42,9 @@ constexpr TokenIndex TK(const char token[]) {
} }
#define TK_STR(t) kTokens[t] #define TK_STR(t) kTokens[t]
const TokenIndex kTokenKwBegin = TK("class");
const TokenIndex kTokenKwEnd = TK("raise");
const std::map<std::string_view, TokenIndex> kTokenKwMap = [](){ const std::map<std::string_view, TokenIndex> kTokenKwMap = [](){
std::map<std::string_view, TokenIndex> map; std::map<std::string_view, TokenIndex> map;
for(int k=kTokenKwBegin; k<=kTokenKwEnd; k++) map[kTokens[k]] = k; for(int k=TK("class"); k<kTokenCount; k++) map[kTokens[k]] = k;
return map; return map;
}(); }();

6
src/obj.h
View File

@ -82,12 +82,10 @@ struct Slice {
class BaseIter { class BaseIter {
protected: protected:
VM* vm; VM* vm;
PyObject* _ref; // keep a reference to the object so it will not be deleted while iterating
public: public:
BaseIter(VM* vm) : vm(vm) {}
virtual void _mark() {}
virtual PyObject* next() = 0; virtual PyObject* next() = 0;
PyObject* loop_var;
BaseIter(VM* vm, PyObject* _ref) : vm(vm), _ref(_ref) {}
virtual void _mark();
virtual ~BaseIter() = default; virtual ~BaseIter() = default;
}; };

146
src/vm.h
View File

@ -33,7 +33,7 @@ class Generator: public BaseIter {
int state; // 0,1,2 int state; // 0,1,2
public: public:
Generator(VM* vm, std::unique_ptr<Frame>&& frame) Generator(VM* vm, std::unique_ptr<Frame>&& frame)
: BaseIter(vm, nullptr), frame(std::move(frame)), state(0) {} : BaseIter(vm), frame(std::move(frame)), state(0) {}
PyObject* next() override; PyObject* next() override;
void _mark() override; void _mark() override;
@ -351,19 +351,11 @@ inline PyObject* NativeFunc::operator()(VM* vm, Args& args) const{
} }
inline void CodeObject::optimize(VM* vm){ inline void CodeObject::optimize(VM* vm){
std::vector<StrName> keys; // here we simple pass all names, but only some of them are NAME_LOCAL
for(auto& p: names) if(p.second == NAME_LOCAL) keys.push_back(p.first); // TODO: ...
uint32_t base_n = (uint32_t)(keys.size() / kLocalsLoadFactor + 0.5); uint32_t base_n = (uint32_t)(names.size() / kLocalsLoadFactor + 0.5);
perfect_locals_capacity = find_next_capacity(base_n); perfect_locals_capacity = find_next_capacity(base_n);
perfect_hash_seed = find_perfect_hash_seed(perfect_locals_capacity, keys); perfect_hash_seed = find_perfect_hash_seed(perfect_locals_capacity, names);
for(int i=1; i<codes.size(); i++){
if(codes[i].op == OP_UNARY_NEGATIVE && codes[i-1].op == OP_LOAD_CONST){
codes[i].op = OP_NO_OP;
int pos = codes[i-1].arg;
consts[pos] = vm->num_negated(consts[pos]);
}
}
// pre-compute sn in co_consts // pre-compute sn in co_consts
for(int i=0; i<consts.size(); i++){ for(int i=0; i<consts.size(); i++){
@ -559,73 +551,74 @@ inline PyObject* VM::new_module(StrName name) {
} }
inline Str VM::disassemble(CodeObject_ co){ inline Str VM::disassemble(CodeObject_ co){
auto pad = [](const Str& s, const int n){ return "";
if(s.size() >= n) return s.substr(0, n); // auto pad = [](const Str& s, const int n){
return s + std::string(n - s.size(), ' '); // if(s.size() >= n) return s.substr(0, n);
}; // return s + std::string(n - s.size(), ' ');
// };
std::vector<int> jumpTargets; // std::vector<int> jumpTargets;
for(auto byte : co->codes){ // for(auto byte : co->codes){
if(byte.op == OP_JUMP_ABSOLUTE || byte.op == OP_POP_JUMP_IF_FALSE){ // if(byte.op == OP_JUMP_ABSOLUTE || byte.op == OP_POP_JUMP_IF_FALSE){
jumpTargets.push_back(byte.arg); // jumpTargets.push_back(byte.arg);
} // }
} // }
StrStream ss; // StrStream ss;
ss << std::string(54, '-') << '\n'; // ss << std::string(54, '-') << '\n';
ss << co->name << ":\n"; // ss << co->name << ":\n";
int prev_line = -1; // int prev_line = -1;
for(int i=0; i<co->codes.size(); i++){ // for(int i=0; i<co->codes.size(); i++){
const Bytecode& byte = co->codes[i]; // const Bytecode& byte = co->codes[i];
if(byte.op == OP_NO_OP) continue; // if(byte.op == OP_NO_OP) continue;
Str line = std::to_string(byte.line); // Str line = std::to_string(byte.line);
if(byte.line == prev_line) line = ""; // if(byte.line == prev_line) line = "";
else{ // else{
if(prev_line != -1) ss << "\n"; // if(prev_line != -1) ss << "\n";
prev_line = byte.line; // prev_line = byte.line;
} // }
std::string pointer; // std::string pointer;
if(std::find(jumpTargets.begin(), jumpTargets.end(), i) != jumpTargets.end()){ // if(std::find(jumpTargets.begin(), jumpTargets.end(), i) != jumpTargets.end()){
pointer = "-> "; // pointer = "-> ";
}else{ // }else{
pointer = " "; // pointer = " ";
} // }
ss << pad(line, 8) << pointer << pad(std::to_string(i), 3); // ss << pad(line, 8) << pointer << pad(std::to_string(i), 3);
ss << " " << pad(OP_NAMES[byte.op], 20) << " "; // ss << " " << pad(OP_NAMES[byte.op], 20) << " ";
// ss << pad(byte.arg == -1 ? "" : std::to_string(byte.arg), 5); // // ss << pad(byte.arg == -1 ? "" : std::to_string(byte.arg), 5);
std::string argStr = byte.arg == -1 ? "" : std::to_string(byte.arg); // std::string argStr = byte.arg == -1 ? "" : std::to_string(byte.arg);
if(byte.op == OP_LOAD_CONST){ // if(byte.op == OP_LOAD_CONST){
argStr += " (" + CAST(Str, asRepr(co->consts[byte.arg])) + ")"; // argStr += " (" + CAST(Str, asRepr(co->consts[byte.arg])) + ")";
} // }
if(byte.op == OP_LOAD_NAME_REF || byte.op == OP_LOAD_NAME || byte.op == OP_RAISE || byte.op == OP_STORE_NAME){ // if(byte.op == OP_LOAD_NAME_REF || byte.op == OP_LOAD_NAME || byte.op == OP_RAISE || byte.op == OP_STORE_NAME){
argStr += " (" + co->names[byte.arg].first.str().escape(true) + ")"; // argStr += " (" + co->names[byte.arg].first.str().escape(true) + ")";
} // }
ss << argStr; // ss << argStr;
// ss << pad(argStr, 20); // may overflow // // ss << pad(argStr, 20); // may overflow
// ss << co->blocks[byte.block].to_string(); // // ss << co->blocks[byte.block].to_string();
if(i != co->codes.size() - 1) ss << '\n'; // if(i != co->codes.size() - 1) ss << '\n';
} // }
StrStream consts; // StrStream consts;
consts << "co_consts: "; // consts << "co_consts: ";
consts << CAST(Str, asRepr(VAR(co->consts))); // consts << CAST(Str, asRepr(VAR(co->consts)));
StrStream names; // StrStream names;
names << "co_names: "; // names << "co_names: ";
List list; // List list;
for(int i=0; i<co->names.size(); i++){ // for(int i=0; i<co->names.size(); i++){
list.push_back(VAR(co->names[i].first.str())); // list.push_back(VAR(co->names[i].first.str()));
} // }
names << CAST(Str, asRepr(VAR(list))); // names << CAST(Str, asRepr(VAR(list)));
ss << '\n' << consts.str() << '\n' << names.str() << '\n'; // ss << '\n' << consts.str() << '\n' << names.str() << '\n';
for(int i=0; i<co->consts.size(); i++){ // for(int i=0; i<co->consts.size(); i++){
PyObject* obj = co->consts[i]; // PyObject* obj = co->consts[i];
if(is_type(obj, tp_function)){ // if(is_type(obj, tp_function)){
const auto& f = CAST(Function&, obj); // const auto& f = CAST(Function&, obj);
ss << disassemble(f.code); // ss << disassemble(f.code);
} // }
} // }
return Str(ss.str()); // return Str(ss.str());
} }
inline void VM::init_builtin_types(){ inline void VM::init_builtin_types(){
@ -769,7 +762,6 @@ inline void VM::unpack_args(Args& args){
for(int i=0; i<args.size(); i++){ for(int i=0; i<args.size(); i++){
if(is_type(args[i], tp_star_wrapper)){ if(is_type(args[i], tp_star_wrapper)){
auto& star = _CAST(StarWrapper&, args[i]); auto& star = _CAST(StarWrapper&, args[i]);
if(!star.rvalue) UNREACHABLE();
List& list = CAST(List&, asList(star.obj)); List& list = CAST(List&, asList(star.obj));
unpacked.insert(unpacked.end(), list.begin(), list.end()); unpacked.insert(unpacked.end(), list.begin(), list.end());
}else{ }else{