Zxc200611/pocketpy
1
0
Fork 0
mirror of https://github.com/pocketpy/pocketpy synced 2025-10-20 11:30:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

some fix

Browse Source
This commit is contained in:
blueloveTH 2024-06-29 14:34:38 +08:00
parent 3610e87244
commit a03a3bdbf8
5 changed files with 211 additions and 879 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
include/pocketpy/compiler/compiler.hpp
View File

@ -1,141 +0,0 @@
#pragma once
#include "pocketpy/compiler/expr.hpp"
#include "pocketpy/objects/error.hpp"
namespace pkpy {
struct Compiler;
typedef Error* (Compiler::*PrattCallback)() noexcept;
struct PrattRule {
PrattCallback prefix;
PrattCallback infix;
Precedence precedence;
};
struct Compiler {
PK_ALWAYS_PASS_BY_POINTER(Compiler)
static PrattRule rules[TK__COUNT__];
Lexer lexer;
vector<CodeEmitContext> contexts;
VM* vm;
bool unknown_global_scope; // for eval/exec() call
// for parsing token stream
int __i = 0;
const Token& tk(int i) const noexcept{ return lexer.nexts[i]; }
const Token& prev() const noexcept{ return tk(__i - 1); }
const Token& curr() const noexcept{ return tk(__i); }
const Token& next() const noexcept{ return tk(__i + 1); }
const Token& err() const noexcept{
if(__i >= lexer.nexts.size()) return prev();
return curr();
}
void advance(int delta = 1) noexcept{
__i += delta;
#if PK_DEBUG_COMPILER
if(__i>=0 && __i<lexer.nexts.size()){
printf("%s:%d %s %s\n",
lexer.src.filename().c_str(),
curr().line,
pk_TokenSymbols(curr().type),
curr().str().escape().c_str()
);
}
#endif
}
CodeEmitContext* ctx() noexcept{ return &contexts.back(); }
CompileMode mode() const noexcept{ return lexer.src->mode; }
NameScope name_scope() const noexcept;
CodeObject* push_global_context() noexcept;
FuncDecl_ push_f_context(c11_string name, int* out_index) noexcept;
static void init_pratt_rules() noexcept;
bool match(TokenIndex expected) noexcept;
bool match_end_stmt() noexcept;
bool match_newlines(bool* need_more_lines = NULL) noexcept;
/*************************************************/
[[nodiscard]] Error* EXPR() noexcept{ return parse_expression(PREC_LOWEST + 1); }
[[nodiscard]] Error* EXPR_TUPLE(bool allow_slice = false) noexcept;
[[nodiscard]] Error* EXPR_VARS() noexcept; // special case for `for loop` and `comp`
template <typename T, typename... Args>
T* make_expr(Args&&... args) noexcept{
static_assert(sizeof(T) <= kPoolExprBlockSize);
static_assert(std::is_base_of_v<Expr, T>);
void* p = PoolExpr_alloc();
T* expr = new (p) T(std::forward<Args>(args)...);
expr->line = prev().line;
return expr;
}
[[nodiscard]] Error* consume_comp(Opcode op0, Opcode op1) noexcept;
[[nodiscard]] Error* pop_context() noexcept;
Error* exprLiteral() noexcept;
Error* exprLong() noexcept;
Error* exprImag() noexcept;
Error* exprBytes() noexcept;
Error* exprFString() noexcept;
Error* exprLambda() noexcept;
Error* exprOr() noexcept;
Error* exprAnd() noexcept;
Error* exprTernary() noexcept;
Error* exprBinaryOp() noexcept;
Error* exprNot() noexcept;
Error* exprUnaryOp() noexcept;
Error* exprGroup() noexcept;
Error* exprList() noexcept;
Error* exprMap() noexcept;
Error* exprCall() noexcept;
Error* exprName() noexcept;
Error* exprAttrib() noexcept;
Error* exprSlice0() noexcept;
Error* exprSlice1() noexcept;
Error* exprSubscr() noexcept;
Error* exprLiteral0() noexcept;
bool is_expression(bool allow_slice = false) noexcept;
[[nodiscard]] Error* compile_block_body(PrattCallback callback = NULL) noexcept;
[[nodiscard]] Error* compile_normal_import() noexcept;
[[nodiscard]] Error* compile_from_import() noexcept;
[[nodiscard]] Error* parse_expression(int precedence, bool allow_slice = false) noexcept;
[[nodiscard]] Error* compile_if_stmt() noexcept;
[[nodiscard]] Error* compile_while_loop() noexcept;
[[nodiscard]] Error* compile_for_loop() noexcept;
[[nodiscard]] Error* compile_try_except() noexcept;
[[nodiscard]] Error* compile_decorated() noexcept;
[[nodiscard]] Error* try_compile_assignment(bool* is_assign) noexcept;
[[nodiscard]] Error* compile_stmt() noexcept;
[[nodiscard]] Error* consume_type_hints() noexcept;
[[nodiscard]] Error* _compile_f_args(FuncDecl* decl, bool enable_type_hints) noexcept;
[[nodiscard]] Error* compile_function(int decorators = 0) noexcept;
[[nodiscard]] Error* compile_class(int decorators = 0) noexcept;
PyVar to_object(const TokenValue& value) noexcept;
[[nodiscard]] Error* read_literal(PyVar* out) noexcept;
[[nodiscard]] Error* SyntaxError(const char* msg = "invalid syntax", ...) noexcept;
[[nodiscard]] Error* IndentationError(const char* msg) noexcept{ return lexer._error(false, "IndentationError", msg, {}); }
[[nodiscard]] Error* NeedMoreLines() noexcept{
return lexer._error(false, "NeedMoreLines", "", {}, (i64)ctx()->is_compiling_class);
}
public:
Compiler(VM* vm, std::string_view source, const Str& filename, CompileMode mode, bool unknown_global_scope = false) noexcept;
[[nodiscard]] Error* compile(CodeObject** out) noexcept;
~Compiler();
};
} // namespace pkpy

17
include/pocketpy/compiler/expr.h
View File

@ -1,17 +0,0 @@
#pragma once
#include <stdbool.h>
#include "pocketpy/common/memorypool.h"
#include "pocketpy/compiler/lexer.h"
#include "pocketpy/common/strname.h"
#include "pocketpy/objects/codeobject.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif

464
include/pocketpy/compiler/expr.hpp
View File

@ -1,464 +0,0 @@
#pragma once
#include "pocketpy/objects/codeobject.hpp"
#include "pocketpy/compiler/lexer.hpp"
namespace pkpy {
struct CodeEmitContext;
struct Expr;
typedef small_vector<Expr*, 4> Expr_vector;
struct Expr {
int line = 0;
virtual ~Expr() = default;
virtual void emit_(CodeEmitContext* ctx) = 0;
Expr() = default;
Expr(const Expr&) = delete;
Expr(Expr&&) = delete;
Expr& operator=(const Expr&) = delete;
Expr& operator=(Expr&&) = delete;
virtual bool is_literal() const { return false; }
virtual bool is_json_object() const { return false; }
virtual bool is_attrib() const { return false; }
virtual bool is_subscr() const { return false; }
virtual bool is_compare() const { return false; }
virtual int star_level() const { return 0; }
virtual bool is_tuple() const { return false; }
virtual bool is_name() const { return false; }
bool is_starred() const { return star_level() > 0; }
// for OP_DELETE_XXX
[[nodiscard]] virtual bool emit_del(CodeEmitContext* ctx) { return false; }
// for OP_STORE_XXX
[[nodiscard]] virtual bool emit_store(CodeEmitContext* ctx) { return false; }
virtual void emit_inplace(CodeEmitContext* ctx) { emit_(ctx); }
[[nodiscard]] virtual bool emit_store_inplace(CodeEmitContext* ctx) { return emit_store(ctx); }
};
inline void delete_expr(Expr* p) noexcept{
if(!p) return;
p->~Expr();
PoolExpr_dealloc(p);
}
struct CodeEmitContext{
VM* vm;
FuncDecl* func; // optional, weakref
CodeObject* co; // 1 CodeEmitContext <=> 1 CodeObject*
vector<Expr*> _s_expr;
int level;
vector<StrName> global_names;
int curr_iblock = 0;
bool is_compiling_class = false;
c11_smallmap_s2n _co_consts_string_dedup_map;
CodeEmitContext(VM* vm, CodeObject* co, int level) : vm(vm), co(co), level(level) {
func = NULL;
c11_smallmap_s2n__ctor(&_co_consts_string_dedup_map);
}
int get_loop() const noexcept;
CodeBlock* enter_block(CodeBlockType type) noexcept;
void exit_block() noexcept;
int emit_(Opcode opcode, uint16_t arg, int line, bool is_virtual = false) noexcept;
void revert_last_emit_() noexcept;
int emit_int(i64 value, int line) noexcept;
void patch_jump(int index) noexcept;
bool add_label(StrName name) noexcept;
int add_varname(StrName name) noexcept;
int add_const(PyVar) noexcept;
int add_const_string(std::string_view) noexcept;
void emit_store_name(NameScope scope, StrName name, int line) noexcept;
void try_merge_for_iter_store(int) noexcept;
// emit top -> pop -> delete
void s_emit_top() noexcept{
s_debug_info("s_emit_top");
Expr* e = _s_expr.popx_back();
e->emit_(this);
delete_expr(e);
}
// push
void s_push(Expr* expr) noexcept{
s_debug_info("s_push");
_s_expr.push_back(expr);
}
// top
Expr* s_top() noexcept{
return _s_expr.back();
}
// size
int s_size() const noexcept{
return _s_expr.size();
}
// pop -> delete
void s_pop() noexcept{
s_debug_info("s_pop");
Expr* e = _s_expr.popx_back();
delete_expr(e);
}
// pop move
Expr* s_popx() noexcept{
s_debug_info("s_popx");
return _s_expr.popx_back();
}
// clean
void s_clean() noexcept{
s_debug_info("s_clean");
c11_smallmap_s2n__dtor(&_co_consts_string_dedup_map);
for(Expr* e: _s_expr) delete_expr(e);
_s_expr.clear();
}
// emit decorators
void s_emit_decorators(int count) noexcept;
// debug stack
#if PK_DEBUG_COMPILER
void s_debug_info(const char* op) noexcept{
SStream ss;
for(int i=0; i<s_size(); i++) {
Expr* e = _s_expr[i];
ss << typeid(*e).name();
if(i != s_size() - 1) ss << ", ";
}
printf("[%s] %s\n", ss.str().c_str(), op);
}
#else
void s_debug_info(const char*) noexcept{}
#endif
};
struct NameExpr : Expr {
StrName name;
NameScope scope;
NameExpr(StrName name, NameScope scope) : name(name), scope(scope) {}
void emit_(CodeEmitContext* ctx) override;
bool emit_del(CodeEmitContext* ctx) override;
bool emit_store(CodeEmitContext* ctx) override;
bool is_name() const override { return true; }
};
struct _UnaryExpr : Expr {
Expr* child;
_UnaryExpr(Expr* child) : child(child) {}
_UnaryExpr() : child(nullptr) {}
~_UnaryExpr() { delete_expr(child); }
};
struct _BinaryExpr : Expr {
Expr* lhs;
Expr* rhs;
_BinaryExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {}
_BinaryExpr() : lhs(nullptr), rhs(nullptr) {}
~_BinaryExpr() {
delete_expr(lhs);
delete_expr(rhs);
}
};
struct InvertExpr : _UnaryExpr {
using _UnaryExpr::_UnaryExpr;
void emit_(CodeEmitContext* ctx) override;
};
struct StarredExpr : _UnaryExpr {
int level;
StarredExpr(Expr* child, int level) : _UnaryExpr(child), level(level) {}
int star_level() const override { return level; }
void emit_(CodeEmitContext* ctx) override;
bool emit_store(CodeEmitContext* ctx) override;
};
struct NotExpr : _UnaryExpr {
using _UnaryExpr::_UnaryExpr;
void emit_(CodeEmitContext* ctx) override;
};
struct AndExpr : _BinaryExpr {
using _BinaryExpr::_BinaryExpr;
void emit_(CodeEmitContext* ctx) override;
};
struct OrExpr : _BinaryExpr {
using _BinaryExpr::_BinaryExpr;
void emit_(CodeEmitContext* ctx) override;
};
// [None, True, False, ...]
struct Literal0Expr : Expr {
TokenIndex token;
Literal0Expr(TokenIndex token) : token(token) {}
bool is_json_object() const override { return true; }
void emit_(CodeEmitContext* ctx) override;
};
struct LongExpr : Expr {
Str s;
LongExpr(const Str& s) : s(s) {}
void emit_(CodeEmitContext* ctx) override;
};
struct BytesExpr : Expr {
Str s;
BytesExpr(const Str& s) : s(s) {}
void emit_(CodeEmitContext* ctx) override;
};
struct ImagExpr : Expr {
f64 value;
ImagExpr(f64 value) : value(value) {}
void emit_(CodeEmitContext* ctx) override;
};
// @num, @str which needs to invoke OP_LOAD_CONST
struct LiteralExpr : Expr {
TokenValue value;
LiteralExpr(TokenValue value) : value(value) {}
void emit_(CodeEmitContext* ctx) override;
bool is_literal() const override { return true; }
bool is_json_object() const override { return true; }
};
struct NegatedExpr : _UnaryExpr {
using _UnaryExpr::_UnaryExpr;
void emit_(CodeEmitContext* ctx) override;
bool is_json_object() const override { return child->is_literal(); }
};
struct SliceExpr : Expr {
Expr* start = nullptr;
Expr* stop = nullptr;
Expr* step = nullptr;
void emit_(CodeEmitContext* ctx) override;
~SliceExpr() {
delete_expr(start);
delete_expr(stop);
delete_expr(step);
}
};
struct DictItemExpr : Expr {
Expr* key; // maybe nullptr if it is **kwargs
Expr* value;
DictItemExpr(): key(nullptr), value(nullptr) {}
int star_level() const override { return value->star_level(); }
void emit_(CodeEmitContext* ctx) override;
~DictItemExpr() {
delete_expr(key);
delete_expr(value);
}
};
struct SequenceExpr : Expr {
array<Expr*> items;
SequenceExpr(int count) : items(count) {}
virtual Opcode opcode() const = 0;
void emit_(CodeEmitContext* ctx) override {
for(auto& item: items) item->emit_(ctx);
ctx->emit_(opcode(), items.size(), line);
}
~SequenceExpr() {
for(Expr* item: items) delete_expr(item);
}
};
struct ListExpr : SequenceExpr {
using SequenceExpr::SequenceExpr;
Opcode opcode() const override {
for(auto& e: items)
if(e->is_starred()) return OP_BUILD_LIST_UNPACK;
return OP_BUILD_LIST;
}
bool is_json_object() const override { return true; }
};
struct DictExpr : SequenceExpr {
using SequenceExpr::SequenceExpr;
Opcode opcode() const override {
for(auto& e: items)
if(e->is_starred()) return OP_BUILD_DICT_UNPACK;
return OP_BUILD_DICT;
}
bool is_json_object() const override { return true; }
};
struct SetExpr : SequenceExpr {
using SequenceExpr::SequenceExpr;
Opcode opcode() const override {
for(auto& e: items)
if(e->is_starred()) return OP_BUILD_SET_UNPACK;
return OP_BUILD_SET;
}
};
struct TupleExpr : SequenceExpr {
using SequenceExpr::SequenceExpr;
bool is_tuple() const override { return true; }
Opcode opcode() const override {
for(auto& e: items)
if(e->is_starred()) return OP_BUILD_TUPLE_UNPACK;
return OP_BUILD_TUPLE;
}
bool emit_store(CodeEmitContext* ctx) override;
bool emit_del(CodeEmitContext* ctx) override;
};
struct CompExpr : Expr {
Expr* expr = nullptr; // loop expr
Expr* vars = nullptr; // loop vars
Expr* iter = nullptr; // loop iter
Expr* cond = nullptr; // optional if condition
Opcode op0;
Opcode op1;
CompExpr(Opcode op0, Opcode op1) : op0(op0), op1(op1) {}
void emit_(CodeEmitContext* ctx) override;
~CompExpr() {
delete_expr(expr);
delete_expr(vars);
delete_expr(iter);
delete_expr(cond);
}
};
struct LambdaExpr : Expr {
int index;
LambdaExpr(int index) : index(index) {}
void emit_(CodeEmitContext* ctx) override {
ctx->emit_(OP_LOAD_FUNCTION, index, line);
}
};
struct FStringExpr : Expr {
Str src;
FStringExpr(const Str& src) : src(src) {}
void _load_simple_expr(CodeEmitContext* ctx, Str expr);
void emit_(CodeEmitContext* ctx) override;
};
struct SubscrExpr : _BinaryExpr {
using _BinaryExpr::_BinaryExpr;
bool is_subscr() const override { return true; }
void emit_(CodeEmitContext* ctx) override;
bool emit_del(CodeEmitContext* ctx) override;
bool emit_store(CodeEmitContext* ctx) override;
void emit_inplace(CodeEmitContext* ctx) override;
bool emit_store_inplace(CodeEmitContext* ctx) override;
};
struct AttribExpr : _UnaryExpr {
StrName name;
AttribExpr(Expr* child, StrName name) : _UnaryExpr(child), name(name) {}
void emit_(CodeEmitContext* ctx) override;
bool emit_del(CodeEmitContext* ctx) override;
bool emit_store(CodeEmitContext* ctx) override;
void emit_method(CodeEmitContext* ctx);
bool is_attrib() const override { return true; }
void emit_inplace(CodeEmitContext* ctx) override;
bool emit_store_inplace(CodeEmitContext* ctx) override;
};
struct CallExpr : Expr {
Expr* callable;
Expr_vector args;
// **a will be interpreted as a special keyword argument: {"**": a}
vector<pair<StrName, Expr*>> kwargs;
void emit_(CodeEmitContext* ctx) override;
~CallExpr() {
delete_expr(callable);
for(Expr* arg: args) delete_expr(arg);
for(auto [_, arg]: kwargs) delete_expr(arg);
}
};
struct GroupedExpr : _UnaryExpr {
using _UnaryExpr::_UnaryExpr;
void emit_(CodeEmitContext* ctx) override { child->emit_(ctx); }
bool emit_del(CodeEmitContext* ctx) override { return child->emit_del(ctx); }
bool emit_store(CodeEmitContext* ctx) override { return child->emit_store(ctx); }
};
struct BinaryExpr : _BinaryExpr {
TokenIndex op;
bool inplace;
BinaryExpr(TokenIndex op, bool inplace = false)
: _BinaryExpr(), op(op), inplace(inplace) {}
bool is_compare() const override;
void _emit_compare(CodeEmitContext*, small_vector_2<int, 8>&);
void emit_(CodeEmitContext* ctx) override;
};
struct TernaryExpr : Expr {
Expr* cond = nullptr;
Expr* true_expr = nullptr;
Expr* false_expr = nullptr;
void emit_(CodeEmitContext* ctx) override;
~TernaryExpr() {
delete_expr(cond);
delete_expr(true_expr);
delete_expr(false_expr);
}
};
} // namespace pkpy

204
src/compiler/compiler.c
View File

@ -1,6 +1,8 @@
#include "pocketpy/compiler/compiler.h" #include "pocketpy/compiler/compiler.h"
#include "pocketpy/compiler/expr.h"
#include "pocketpy/compiler/lexer.h" #include "pocketpy/compiler/lexer.h"
#include "pocketpy/objects/sourcedata.h"
#include "pocketpy/common/strname.h"
#include "pocketpy/common/memorypool.h"
#include <ctype.h> #include <ctype.h>
/* expr.h */ /* expr.h */
@ -33,8 +35,10 @@ typedef struct ExprVt {
#define vtemit_(self, ctx) vtcall(emit_, (self), (ctx)) #define vtemit_(self, ctx) vtcall(emit_, (self), (ctx))
#define vtemit_del(self, ctx) ((self)->vt->emit_del ? vtcall(emit_del, self, ctx) : false) #define vtemit_del(self, ctx) ((self)->vt->emit_del ? vtcall(emit_del, self, ctx) : false)
#define vtemit_store(self, ctx) ((self)->vt->emit_store ? vtcall(emit_store, self, ctx) : false) #define vtemit_store(self, ctx) ((self)->vt->emit_store ? vtcall(emit_store, self, ctx) : false)
#define vtemit_inplace(self, ctx) ((self)->vt->emit_inplace ? vtcall(emit_inplace, self, ctx) : vtemit_(self, ctx)) #define vtemit_inplace(self, ctx) \
#define vtemit_istore(self, ctx) ((self)->vt->emit_istore ? vtcall(emit_istore, self, ctx) : vtemit_store(self, ctx)) ((self)->vt->emit_inplace ? vtcall(emit_inplace, self, ctx) : vtemit_(self, ctx))
#define vtemit_istore(self, ctx) \
((self)->vt->emit_istore ? vtcall(emit_istore, self, ctx) : vtemit_store(self, ctx))
#define COMMON_HEADER \ #define COMMON_HEADER \
const ExprVt* vt; \ const ExprVt* vt; \
@ -149,12 +153,10 @@ bool NameExpr__emit_store(Expr* self_, Ctx* ctx) {
} }
NameExpr* NameExpr__new(StrName name, NameScope scope) { NameExpr* NameExpr__new(StrName name, NameScope scope) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = NameExpr__emit_,
.emit_ = NameExpr__emit_,
.emit_del = NameExpr__emit_del, .emit_del = NameExpr__emit_del,
.emit_store = NameExpr__emit_store, .emit_store = NameExpr__emit_store,
.is_name = true .is_name = true};
};
static_assert_expr_size(NameExpr); static_assert_expr_size(NameExpr);
NameExpr* self = PoolExpr_alloc(); NameExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -184,11 +186,9 @@ bool StarredExpr__emit_store(Expr* self_, Ctx* ctx) {
} }
StarredExpr* StarredExpr__new(Expr* child, int level) { StarredExpr* StarredExpr__new(Expr* child, int level) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = StarredExpr__emit_,
.emit_ = StarredExpr__emit_,
.emit_store = StarredExpr__emit_store, .emit_store = StarredExpr__emit_store,
.is_starred = true .is_starred = true};
};
static_assert_expr_size(StarredExpr); static_assert_expr_size(StarredExpr);
StarredExpr* self = PoolExpr_alloc(); StarredExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -218,10 +218,7 @@ static void UnaryExpr__emit_(Expr* self_, Ctx* ctx) {
} }
UnaryExpr* UnaryExpr__new(Expr* child, Opcode opcode) { UnaryExpr* UnaryExpr__new(Expr* child, Opcode opcode) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = UnaryExpr__emit_, .dtor = UnaryExpr__dtor};
.emit_ = UnaryExpr__emit_,
.dtor = UnaryExpr__dtor
};
static_assert_expr_size(UnaryExpr); static_assert_expr_size(UnaryExpr);
UnaryExpr* self = PoolExpr_alloc(); UnaryExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -246,9 +243,7 @@ void RawStringExpr__emit_(Expr* self_, Ctx* ctx) {
} }
RawStringExpr* RawStringExpr__new(c11_string value, Opcode opcode) { RawStringExpr* RawStringExpr__new(c11_string value, Opcode opcode) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = RawStringExpr__emit_};
.emit_ = RawStringExpr__emit_
};
static_assert_expr_size(RawStringExpr); static_assert_expr_size(RawStringExpr);
RawStringExpr* self = PoolExpr_alloc(); RawStringExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -273,9 +268,7 @@ void ImagExpr__emit_(Expr* self_, Ctx* ctx) {
} }
ImagExpr* ImagExpr__new(double value) { ImagExpr* ImagExpr__new(double value) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = ImagExpr__emit_};
.emit_ = ImagExpr__emit_
};
static_assert_expr_size(ImagExpr); static_assert_expr_size(ImagExpr);
ImagExpr* self = PoolExpr_alloc(); ImagExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -315,11 +308,9 @@ void LiteralExpr__emit_(Expr* self_, Ctx* ctx) {
} }
LiteralExpr* LiteralExpr__new(const TokenValue* value) { LiteralExpr* LiteralExpr__new(const TokenValue* value) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = LiteralExpr__emit_,
.emit_ = LiteralExpr__emit_,
.is_literal = true, .is_literal = true,
.is_json_object = true .is_json_object = true};
};
static_assert_expr_size(LiteralExpr); static_assert_expr_size(LiteralExpr);
LiteralExpr* self = PoolExpr_alloc(); LiteralExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -344,20 +335,23 @@ void SliceExpr__dtor(Expr* self_){
void SliceExpr__emit_(Expr* self_, Ctx* ctx) { void SliceExpr__emit_(Expr* self_, Ctx* ctx) {
SliceExpr* self = (SliceExpr*)self_; SliceExpr* self = (SliceExpr*)self_;
if(self->start) vtemit_(self->start, ctx); if(self->start)
else Ctx__emit_(ctx, OP_LOAD_NONE, BC_NOARG, self->line); vtemit_(self->start, ctx);
if(self->stop) vtemit_(self->stop, ctx); else
else Ctx__emit_(ctx, OP_LOAD_NONE, BC_NOARG, self->line); Ctx__emit_(ctx, OP_LOAD_NONE, BC_NOARG, self->line);
if(self->step) vtemit_(self->step, ctx); if(self->stop)
else Ctx__emit_(ctx, OP_LOAD_NONE, BC_NOARG, self->line); vtemit_(self->stop, ctx);
else
Ctx__emit_(ctx, OP_LOAD_NONE, BC_NOARG, self->line);
if(self->step)
vtemit_(self->step, ctx);
else
Ctx__emit_(ctx, OP_LOAD_NONE, BC_NOARG, self->line);
Ctx__emit_(ctx, OP_BUILD_SLICE, BC_NOARG, self->line); Ctx__emit_(ctx, OP_BUILD_SLICE, BC_NOARG, self->line);
} }
SliceExpr* SliceExpr__new() { SliceExpr* SliceExpr__new() {
const static ExprVt Vt = { const static ExprVt Vt = {.dtor = SliceExpr__dtor, .emit_ = SliceExpr__emit_};
.dtor = SliceExpr__dtor,
.emit_ = SliceExpr__emit_
};
static_assert_expr_size(SliceExpr); static_assert_expr_size(SliceExpr);
SliceExpr* self = PoolExpr_alloc(); SliceExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -399,8 +393,10 @@ bool TupleExpr__emit_store(Expr* self_, Ctx* ctx) {
Expr* e = c11__getitem(Expr*, &self->items, i); Expr* e = c11__getitem(Expr*, &self->items, i);
if(e->vt->is_starred) { if(e->vt->is_starred) {
if(((StarredExpr*)e)->level > 0) { if(((StarredExpr*)e)->level > 0) {
if(starred_i == -1) starred_i = i; if(starred_i == -1)
else return false; // multiple StarredExpr not allowed starred_i = i;
else
return false; // multiple StarredExpr not allowed
} }
} }
} }
@ -456,20 +452,16 @@ static SequenceExpr* SequenceExpr__new(const ExprVt* vt, int count, Opcode opcod
} }
SequenceExpr* ListExpr__new(int count) { SequenceExpr* ListExpr__new(int count) {
const static ExprVt ListExprVt = { const static ExprVt ListExprVt = {.dtor = SequenceExpr__dtor,
.dtor = SequenceExpr__dtor,
.emit_ = SequenceExpr__emit_, .emit_ = SequenceExpr__emit_,
.is_json_object = true .is_json_object = true};
};
return SequenceExpr__new(&ListExprVt, count, OP_BUILD_LIST); return SequenceExpr__new(&ListExprVt, count, OP_BUILD_LIST);
} }
SequenceExpr* DictExpr__new(int count) { SequenceExpr* DictExpr__new(int count) {
const static ExprVt DictExprVt = { const static ExprVt DictExprVt = {.dtor = SequenceExpr__dtor,
.dtor = SequenceExpr__dtor,
.emit_ = SequenceExpr__emit_, .emit_ = SequenceExpr__emit_,
.is_json_object = true .is_json_object = true};
};
return SequenceExpr__new(&DictExprVt, count, OP_BUILD_DICT); return SequenceExpr__new(&DictExprVt, count, OP_BUILD_DICT);
} }
@ -482,13 +474,11 @@ SequenceExpr* SetExpr__new(int count){
} }
SequenceExpr* TupleExpr__new(int count) { SequenceExpr* TupleExpr__new(int count) {
const static ExprVt TupleExprVt = { const static ExprVt TupleExprVt = {.dtor = SequenceExpr__dtor,
.dtor = SequenceExpr__dtor,
.emit_ = SequenceExpr__emit_, .emit_ = SequenceExpr__emit_,
.is_tuple = true, .is_tuple = true,
.emit_store = TupleExpr__emit_store, .emit_store = TupleExpr__emit_store,
.emit_del = TupleExpr__emit_del .emit_del = TupleExpr__emit_del};
};
return SequenceExpr__new(&TupleExprVt, count, OP_BUILD_TUPLE); return SequenceExpr__new(&TupleExprVt, count, OP_BUILD_TUPLE);
} }
@ -538,10 +528,7 @@ void CompExpr__emit_(Expr* self_, Ctx* ctx) {
} }
CompExpr* CompExpr__new(Opcode op0, Opcode op1) { CompExpr* CompExpr__new(Opcode op0, Opcode op1) {
const static ExprVt Vt = { const static ExprVt Vt = {.dtor = CompExpr__dtor, .emit_ = CompExpr__emit_};
.dtor = CompExpr__dtor,
.emit_ = CompExpr__emit_
};
static_assert_expr_size(CompExpr); static_assert_expr_size(CompExpr);
CompExpr* self = PoolExpr_alloc(); CompExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -566,9 +553,7 @@ static void LambdaExpr__emit_(Expr* self_, Ctx* ctx) {
} }
LambdaExpr* LambdaExpr__new(int index) { LambdaExpr* LambdaExpr__new(int index) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = LambdaExpr__emit_};
.emit_ = LambdaExpr__emit_
};
static_assert_expr_size(LambdaExpr); static_assert_expr_size(LambdaExpr);
LambdaExpr* self = PoolExpr_alloc(); LambdaExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -625,12 +610,7 @@ static void _load_simple_expr(Ctx* ctx, c11_string expr, int line) {
bool is_fastpath = false; bool is_fastpath = false;
if(is_identifier(expr)) { if(is_identifier(expr)) {
// ctx->emit_(OP_LOAD_NAME, StrName(expr.sv()).index, line); // ctx->emit_(OP_LOAD_NAME, StrName(expr.sv()).index, line);
Ctx__emit_( Ctx__emit_(ctx, OP_LOAD_NAME, pk_StrName__map2(expr), line);
ctx,
OP_LOAD_NAME,
pk_StrName__map2(expr),
line
);
is_fastpath = true; is_fastpath = true;
} else { } else {
int dot = c11_string__index(expr, '.'); int dot = c11_string__index(expr, '.');
@ -650,9 +630,7 @@ static void _load_simple_expr(Ctx* ctx, c11_string expr, int line) {
Ctx__emit_(ctx, OP_FSTRING_EVAL, index, line); Ctx__emit_(ctx, OP_FSTRING_EVAL, index, line);
} }
if(repr) { if(repr) { Ctx__emit_(ctx, OP_REPR, BC_NOARG, line); }
Ctx__emit_(ctx, OP_REPR, BC_NOARG, line);
}
} }
static void FStringExpr__emit_(Expr* self_, Ctx* ctx) { static void FStringExpr__emit_(Expr* self_, Ctx* ctx) {
@ -671,7 +649,8 @@ static void FStringExpr__emit_(Expr* self_, Ctx* ctx) {
// BUG: ':' is not a format specifier in f"{stack[2:]}" // BUG: ':' is not a format specifier in f"{stack[2:]}"
int conon = c11_string__index(expr, ':'); int conon = c11_string__index(expr, ':');
if(conon >= 0) { if(conon >= 0) {
c11_string spec = {expr.data+(conon+1), expr.size-(conon+1)}; // expr[conon+1:] c11_string spec = {expr.data + (conon + 1),
expr.size - (conon + 1)}; // expr[conon+1:]
// filter some invalid spec // filter some invalid spec
bool ok = true; bool ok = true;
for(int k = 0; k < spec.size; k++) { for(int k = 0; k < spec.size; k++) {
@ -685,7 +664,10 @@ static void FStringExpr__emit_(Expr* self_, Ctx* ctx) {
expr.size = conon; // expr[:conon] expr.size = conon; // expr[:conon]
_load_simple_expr(ctx, expr, self->line); _load_simple_expr(ctx, expr, self->line);
// ctx->emit_(OP_FORMAT_STRING, ctx->add_const_string(spec.sv()), line); // ctx->emit_(OP_FORMAT_STRING, ctx->add_const_string(spec.sv()), line);
Ctx__emit_(ctx, OP_FORMAT_STRING, Ctx__add_const_string(ctx, spec), self->line); Ctx__emit_(ctx,
OP_FORMAT_STRING,
Ctx__add_const_string(ctx, spec),
self->line);
} else { } else {
// ':' is not a spec indicator // ':' is not a spec indicator
_load_simple_expr(ctx, expr, self->line); _load_simple_expr(ctx, expr, self->line);
@ -702,12 +684,10 @@ static void FStringExpr__emit_(Expr* self_, Ctx* ctx) {
if(j + 1 < self->src.size && src[j + 1] == '{') { if(j + 1 < self->src.size && src[j + 1] == '{') {
// {{ -> { // {{ -> {
j++; j++;
Ctx__emit_( Ctx__emit_(ctx,
ctx,
OP_LOAD_CONST, OP_LOAD_CONST,
Ctx__add_const_string(ctx, (c11_string){"{", 1}), Ctx__add_const_string(ctx, (c11_string){"{", 1}),
self->line self->line);
);
count++; count++;
} else { } else {
// { -> } // { -> }
@ -719,12 +699,10 @@ static void FStringExpr__emit_(Expr* self_, Ctx* ctx) {
if(j + 1 < self->src.size && src[j + 1] == '}') { if(j + 1 < self->src.size && src[j + 1] == '}') {
// }} -> } // }} -> }
j++; j++;
Ctx__emit_( Ctx__emit_(ctx,
ctx,
OP_LOAD_CONST, OP_LOAD_CONST,
Ctx__add_const_string(ctx, (c11_string){"}", 1}), Ctx__add_const_string(ctx, (c11_string){"}", 1}),
self->line self->line);
);
count++; count++;
} else { } else {
// } -> error // } -> error
@ -737,12 +715,7 @@ static void FStringExpr__emit_(Expr* self_, Ctx* ctx) {
while(j < self->src.size && src[j] != '{' && src[j] != '}') while(j < self->src.size && src[j] != '{' && src[j] != '}')
j++; j++;
c11_string literal = {src + i, j - i}; // src[i:j] c11_string literal = {src + i, j - i}; // src[i:j]
Ctx__emit_( Ctx__emit_(ctx, OP_LOAD_CONST, Ctx__add_const_string(ctx, literal), self->line);
ctx,
OP_LOAD_CONST,
Ctx__add_const_string(ctx, literal),
self->line
);
count++; count++;
continue; // skip j++ continue; // skip j++
} }
@ -760,9 +733,7 @@ static void FStringExpr__emit_(Expr* self_, Ctx* ctx) {
} }
FStringExpr* FStringExpr__new(c11_string src) { FStringExpr* FStringExpr__new(c11_string src) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = FStringExpr__emit_};
.emit_ = FStringExpr__emit_
};
static_assert_expr_size(FStringExpr); static_assert_expr_size(FStringExpr);
FStringExpr* self = PoolExpr_alloc(); FStringExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -794,10 +765,7 @@ void LogicBinaryExpr__emit_(Expr* self_, Ctx* ctx) {
} }
LogicBinaryExpr* LogicBinaryExpr__new(Expr* lhs, Expr* rhs, Opcode opcode) { LogicBinaryExpr* LogicBinaryExpr__new(Expr* lhs, Expr* rhs, Opcode opcode) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = LogicBinaryExpr__emit_, .dtor = LogicBinaryExpr__dtor};
.emit_ = LogicBinaryExpr__emit_,
.dtor = LogicBinaryExpr__dtor
};
static_assert_expr_size(LogicBinaryExpr); static_assert_expr_size(LogicBinaryExpr);
LogicBinaryExpr* self = PoolExpr_alloc(); LogicBinaryExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -834,12 +802,10 @@ bool GroupedExpr__emit_store(Expr* self_, Ctx* ctx) {
} }
GroupedExpr* GroupedExpr__new(Expr* child) { GroupedExpr* GroupedExpr__new(Expr* child) {
const static ExprVt Vt = { const static ExprVt Vt = {.dtor = GroupedExpr__dtor,
.dtor = GroupedExpr__dtor,
.emit_ = GroupedExpr__emit_, .emit_ = GroupedExpr__emit_,
.emit_del = GroupedExpr__emit_del, .emit_del = GroupedExpr__emit_del,
.emit_store = GroupedExpr__emit_store .emit_store = GroupedExpr__emit_store};
};
static_assert_expr_size(GroupedExpr); static_assert_expr_size(GroupedExpr);
GroupedExpr* self = PoolExpr_alloc(); GroupedExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -945,9 +911,7 @@ static void BinaryExpr__emit_(Expr* self_, Ctx* ctx) {
Ctx__emit_(ctx, opcode, BC_NOARG, self->line); Ctx__emit_(ctx, opcode, BC_NOARG, self->line);
c11__foreach(int, &jmps, i) { c11__foreach(int, &jmps, i) { Ctx__patch_jump(ctx, *i); }
Ctx__patch_jump(ctx, *i);
}
} }
typedef struct TernaryExpr { typedef struct TernaryExpr {
@ -976,10 +940,7 @@ void TernaryExpr__emit_(Expr* self_, Ctx* ctx) {
} }
TernaryExpr* TernaryExpr__new() { TernaryExpr* TernaryExpr__new() {
const static ExprVt Vt = { const static ExprVt Vt = {.dtor = TernaryExpr__dtor, .emit_ = TernaryExpr__emit_};
.dtor = TernaryExpr__dtor,
.emit_ = TernaryExpr__emit_
};
static_assert_expr_size(TernaryExpr); static_assert_expr_size(TernaryExpr);
TernaryExpr* self = PoolExpr_alloc(); TernaryExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -1114,14 +1075,12 @@ bool AttribExpr__emit_istore(Expr* self_, Ctx* ctx) {
} }
AttribExpr* AttribExpr__new(Expr* child, StrName name) { AttribExpr* AttribExpr__new(Expr* child, StrName name) {
const static ExprVt Vt = { const static ExprVt Vt = {.emit_ = AttribExpr__emit_,
.emit_ = AttribExpr__emit_,
.emit_del = AttribExpr__emit_del, .emit_del = AttribExpr__emit_del,
.emit_store = AttribExpr__emit_store, .emit_store = AttribExpr__emit_store,
.emit_inplace = AttribExpr__emit_inplace, .emit_inplace = AttribExpr__emit_inplace,
.emit_istore = AttribExpr__emit_istore, .emit_istore = AttribExpr__emit_istore,
.is_attrib = true .is_attrib = true};
};
static_assert_expr_size(AttribExpr); static_assert_expr_size(AttribExpr);
AttribExpr* self = PoolExpr_alloc(); AttribExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -1165,7 +1124,8 @@ void CallExpr__emit_(Expr* self_, Ctx* ctx) {
if(e->val->vt->is_starred) vkwargs = true; if(e->val->vt->is_starred) vkwargs = true;
} }
// if callable is a AttrExpr, we should try to use `fast_call` instead of use `boundmethod` proxy // if callable is a AttrExpr, we should try to use `fast_call` instead of use `boundmethod`
// proxy
if(self->callable->vt->is_attrib) { if(self->callable->vt->is_attrib) {
AttribExpr* p = (AttribExpr*)self->callable; AttribExpr* p = (AttribExpr*)self->callable;
vtemit_(p->child, ctx); vtemit_(p->child, ctx);
@ -1175,9 +1135,7 @@ void CallExpr__emit_(Expr* self_, Ctx* ctx) {
Ctx__emit_(ctx, OP_LOAD_NULL, BC_NOARG, BC_KEEPLINE); Ctx__emit_(ctx, OP_LOAD_NULL, BC_NOARG, BC_KEEPLINE);
} }
c11__foreach(Expr*, &self->args, e){ c11__foreach(Expr*, &self->args, e) { vtemit_(*e, ctx); }
vtemit_(*e, ctx);
}
if(vargs || vkwargs) { if(vargs || vkwargs) {
Ctx__emit_(ctx, OP_BUILD_TUPLE_UNPACK, (uint16_t)self->args.count, self->line); Ctx__emit_(ctx, OP_BUILD_TUPLE_UNPACK, (uint16_t)self->args.count, self->line);
@ -1215,10 +1173,7 @@ void CallExpr__emit_(Expr* self_, Ctx* ctx) {
} }
CallExpr* CallExpr__new(Expr* callable) { CallExpr* CallExpr__new(Expr* callable) {
const static ExprVt Vt = { const static ExprVt Vt = {.dtor = CallExpr__dtor, .emit_ = CallExpr__emit_};
.dtor = CallExpr__dtor,
.emit_ = CallExpr__emit_
};
static_assert_expr_size(CallExpr); static_assert_expr_size(CallExpr);
CallExpr* self = PoolExpr_alloc(); CallExpr* self = PoolExpr_alloc();
self->vt = &Vt; self->vt = &Vt;
@ -1254,29 +1209,29 @@ void Ctx__s_emit_top(Ctx* self) {
c11_vector__pop(&self->s_expr); c11_vector__pop(&self->s_expr);
Expr__delete(top); Expr__delete(top);
} }
// push // push
void Ctx__s_push(Ctx* self, Expr* expr) { void Ctx__s_push(Ctx* self, Expr* expr) { c11_vector__push(Expr*, &self->s_expr, expr); }
c11_vector__push(Expr*, &self->s_expr, expr);
}
// top // top
Expr* Ctx__s_top(Ctx* self){ Expr* Ctx__s_top(Ctx* self) { return c11_vector__back(Expr*, &self->s_expr); }
return c11_vector__back(Expr*, &self->s_expr);
}
// size // size
int Ctx__s_size(Ctx* self) { int Ctx__s_size(Ctx* self) { return self->s_expr.count; }
return self->s_expr.count;
}
// pop -> delete // pop -> delete
void Ctx__s_pop(Ctx* self) { void Ctx__s_pop(Ctx* self) {
Expr__delete(c11_vector__back(Expr*, &self->s_expr)); Expr__delete(c11_vector__back(Expr*, &self->s_expr));
c11_vector__pop(&self->s_expr); c11_vector__pop(&self->s_expr);
} }
// pop move // pop move
Expr* Ctx__s_popx(Ctx* self) { Expr* Ctx__s_popx(Ctx* self) {
Expr* e = c11_vector__back(Expr*, &self->s_expr); Expr* e = c11_vector__back(Expr*, &self->s_expr);
c11_vector__pop(&self->s_expr); c11_vector__pop(&self->s_expr);
return e; return e;
} }
// clean // clean
void Ctx__s_clean(Ctx* self) { void Ctx__s_clean(Ctx* self) {
c11_smallmap_s2n__dtor(&self->co_consts_string_dedup_map); // ?? c11_smallmap_s2n__dtor(&self->co_consts_string_dedup_map); // ??
@ -1287,7 +1242,6 @@ void Ctx__s_clean(Ctx* self){
} }
/* compiler.c */ /* compiler.c */
typedef struct Compiler Compiler; typedef struct Compiler Compiler;
typedef Error* (*PrattCallback)(Compiler* self); typedef Error* (*PrattCallback)(Compiler* self);
@ -1323,7 +1277,7 @@ static void Compiler__dtor(Compiler* self) {
#define prev() tk(self->i - 1) #define prev() tk(self->i - 1)
#define curr() tk(self->i) #define curr() tk(self->i)
#define next() tk(self->i + 1) #define next() tk(self->i + 1)
#define err() (self->i == self->tokens.count ? prev() : curr()) // #define err() (self->i == self->tokens.count ? prev() : curr())
#define advance() self->i++ #define advance() self->i++
#define mode() self->src->mode #define mode() self->src->mode
@ -1513,7 +1467,7 @@ Error* Compiler__compile(Compiler* self, CodeObject* out) {
return NULL; return NULL;
} }
Error* compile(pk_SourceData_ src, CodeObject* out) { Error* pk_compile(pk_SourceData_ src, CodeObject* out) {
pk_TokenArray tokens; pk_TokenArray tokens;
Error* err = pk_Lexer__process(src, &tokens); Error* err = pk_Lexer__process(src, &tokens);
if(err) return err; if(err) return err;
@ -1524,7 +1478,7 @@ Error* compile(pk_SourceData_ src, CodeObject* out) {
// Token* t = data + i; // Token* t = data + i;
// py_Str tmp; // py_Str tmp;
// py_Str__ctor2(&tmp, t->start, t->length); // py_Str__ctor2(&tmp, t->start, t->length);
// printf("[%d] %s: %s\n", t->line, TokenSymbols[t->type], py_Str__data(&tmp)); // printf("[%d] %s: %s\n", t->line, pk_TokenSymbols[t->type], py_Str__data(&tmp));
// py_Str__dtor(&tmp); // py_Str__dtor(&tmp);
// } // }
@ -1532,7 +1486,7 @@ Error* compile(pk_SourceData_ src, CodeObject* out) {
Compiler__ctor(&compiler, src, tokens); Compiler__ctor(&compiler, src, tokens);
CodeObject__ctor(out, src, py_Str__sv(&src->filename)); CodeObject__ctor(out, src, py_Str__sv(&src->filename));
err = Compiler__compile(&compiler, out); err = Compiler__compile(&compiler, out);
CodeObject__dtor(out); if(err) CodeObject__dtor(out);
Compiler__dtor(&compiler); Compiler__dtor(&compiler);
return err; return err;
} }

70
src/compiler/lexer.c
View File

@ -21,18 +21,18 @@ typedef struct pk_Lexer{
c11_vector/*T=int*/ indents; c11_vector/*T=int*/ indents;
} pk_Lexer; } pk_Lexer;
typedef struct pk_TokenDeserializer { typedef struct TokenDeserializer {
const char* curr; const char* curr;
const char* source; const char* source;
} pk_TokenDeserializer; } TokenDeserializer;
void pk_TokenDeserializer__ctor(pk_TokenDeserializer* self, const char* source); void TokenDeserializer__ctor(TokenDeserializer* self, const char* source);
bool pk_TokenDeserializer__match_char(pk_TokenDeserializer* self, char c); bool TokenDeserializer__match_char(TokenDeserializer* self, char c);
c11_string pk_TokenDeserializer__read_string(pk_TokenDeserializer* self, char c); c11_string TokenDeserializer__read_string(TokenDeserializer* self, char c);
py_Str pk_TokenDeserializer__read_string_from_hex(pk_TokenDeserializer* self, char c); py_Str TokenDeserializer__read_string_from_hex(TokenDeserializer* self, char c);
int pk_TokenDeserializer__read_count(pk_TokenDeserializer* self); int TokenDeserializer__read_count(TokenDeserializer* self);
int64_t pk_TokenDeserializer__read_uint(pk_TokenDeserializer* self, char c); int64_t TokenDeserializer__read_uint(TokenDeserializer* self, char c);
double pk_TokenDeserializer__read_float(pk_TokenDeserializer* self, char c); double TokenDeserializer__read_float(TokenDeserializer* self, char c);
const static TokenValue EmptyTokenValue; const static TokenValue EmptyTokenValue;
@ -553,34 +553,34 @@ static Error* lex_one_token(pk_Lexer* self, bool* eof){
} }
static Error* from_precompiled(pk_Lexer* self) { static Error* from_precompiled(pk_Lexer* self) {
pk_TokenDeserializer deserializer; TokenDeserializer deserializer;
pk_TokenDeserializer__ctor(&deserializer, py_Str__data(&self->src->source)); TokenDeserializer__ctor(&deserializer, py_Str__data(&self->src->source));
deserializer.curr += 5; // skip "pkpy:" deserializer.curr += 5; // skip "pkpy:"
c11_string version = pk_TokenDeserializer__read_string(&deserializer, '\n'); c11_string version = TokenDeserializer__read_string(&deserializer, '\n');
if(c11_string__cmp3(version, PK_VERSION) != 0) { if(c11_string__cmp3(version, PK_VERSION) != 0) {
return SyntaxError("precompiled version mismatch"); return SyntaxError("precompiled version mismatch");
} }
if(pk_TokenDeserializer__read_uint(&deserializer, '\n') != (int64_t)self->src->mode){ if(TokenDeserializer__read_uint(&deserializer, '\n') != (int64_t)self->src->mode){
return SyntaxError("precompiled mode mismatch"); return SyntaxError("precompiled mode mismatch");
} }
int count = pk_TokenDeserializer__read_count(&deserializer); int count = TokenDeserializer__read_count(&deserializer);
c11_vector* precompiled_tokens = &self->src->_precompiled_tokens; c11_vector* precompiled_tokens = &self->src->_precompiled_tokens;
for(int i = 0; i < count; i++) { for(int i = 0; i < count; i++) {
c11_string item = pk_TokenDeserializer__read_string(&deserializer, '\n'); c11_string item = TokenDeserializer__read_string(&deserializer, '\n');
py_Str copied_item; py_Str copied_item;
py_Str__ctor2(&copied_item, item.data, item.size); py_Str__ctor2(&copied_item, item.data, item.size);
c11_vector__push(py_Str, precompiled_tokens, copied_item); c11_vector__push(py_Str, precompiled_tokens, copied_item);
} }
count = pk_TokenDeserializer__read_count(&deserializer); count = TokenDeserializer__read_count(&deserializer);
for(int i = 0; i < count; i++) { for(int i = 0; i < count; i++) {
Token t; Token t;
t.type = (TokenIndex)pk_TokenDeserializer__read_uint(&deserializer, ','); t.type = (TokenIndex)TokenDeserializer__read_uint(&deserializer, ',');
if(is_raw_string_used(t.type)) { if(is_raw_string_used(t.type)) {
int64_t index = pk_TokenDeserializer__read_uint(&deserializer, ','); int64_t index = TokenDeserializer__read_uint(&deserializer, ',');
py_Str* p = c11__at(py_Str, precompiled_tokens, index); py_Str* p = c11__at(py_Str, precompiled_tokens, index);
t.start = py_Str__data(p); t.start = py_Str__data(p);
t.length = c11__getitem(py_Str, precompiled_tokens, index).size; t.length = c11__getitem(py_Str, precompiled_tokens, index).size;
@ -589,30 +589,30 @@ static Error* from_precompiled(pk_Lexer* self) {
t.length = 0; t.length = 0;
} }
if(pk_TokenDeserializer__match_char(&deserializer, ',')) { if(TokenDeserializer__match_char(&deserializer, ',')) {
t.line = c11_vector__back(Token, &self->nexts).line; t.line = c11_vector__back(Token, &self->nexts).line;
} else { } else {
t.line = (int)pk_TokenDeserializer__read_uint(&deserializer, ','); t.line = (int)TokenDeserializer__read_uint(&deserializer, ',');
} }
if(pk_TokenDeserializer__match_char(&deserializer, ',')) { if(TokenDeserializer__match_char(&deserializer, ',')) {
t.brackets_level = c11_vector__back(Token, &self->nexts).brackets_level; t.brackets_level = c11_vector__back(Token, &self->nexts).brackets_level;
} else { } else {
t.brackets_level = (int)pk_TokenDeserializer__read_uint(&deserializer, ','); t.brackets_level = (int)TokenDeserializer__read_uint(&deserializer, ',');
} }
char type = (*deserializer.curr++); // read_char char type = (*deserializer.curr++); // read_char
switch(type) { switch(type) {
case 'I': { case 'I': {
int64_t res = pk_TokenDeserializer__read_uint(&deserializer, '\n'); int64_t res = TokenDeserializer__read_uint(&deserializer, '\n');
t.value = (TokenValue){TokenValue_I64, ._i64 = res}; t.value = (TokenValue){TokenValue_I64, ._i64 = res};
} break; } break;
case 'F': { case 'F': {
double res = pk_TokenDeserializer__read_float(&deserializer, '\n'); double res = TokenDeserializer__read_float(&deserializer, '\n');
t.value = (TokenValue){TokenValue_F64, ._f64 = res}; t.value = (TokenValue){TokenValue_F64, ._f64 = res};
} break; } break;
case 'S': { case 'S': {
py_Str res = pk_TokenDeserializer__read_string_from_hex(&deserializer, '\n'); py_Str res = TokenDeserializer__read_string_from_hex(&deserializer, '\n');
t.value = (TokenValue){TokenValue_STR, ._str = res}; t.value = (TokenValue){TokenValue_STR, ._str = res};
} break; } break;
default: default:
@ -875,12 +875,12 @@ const char* pk_TokenSymbols[] = {
"try", "while", "with", "yield", "try", "while", "with", "yield",
}; };
void pk_TokenDeserializer__ctor(pk_TokenDeserializer* self, const char* source){ void TokenDeserializer__ctor(TokenDeserializer* self, const char* source){
self->curr = source; self->curr = source;
self->source = source; self->source = source;
} }
bool pk_TokenDeserializer__match_char(pk_TokenDeserializer* self, char c){ bool TokenDeserializer__match_char(TokenDeserializer* self, char c){
if(*self->curr == c) { if(*self->curr == c) {
self->curr++; self->curr++;
return true; return true;
@ -888,7 +888,7 @@ bool pk_TokenDeserializer__match_char(pk_TokenDeserializer* self, char c){
return false; return false;
} }
c11_string pk_TokenDeserializer__read_string(pk_TokenDeserializer* self, char c){ c11_string TokenDeserializer__read_string(TokenDeserializer* self, char c){
const char* start = self->curr; const char* start = self->curr;
while(*self->curr != c) while(*self->curr != c)
self->curr++; self->curr++;
@ -897,8 +897,8 @@ c11_string pk_TokenDeserializer__read_string(pk_TokenDeserializer* self, char c)
return retval; return retval;
} }
py_Str pk_TokenDeserializer__read_string_from_hex(pk_TokenDeserializer* self, char c){ py_Str TokenDeserializer__read_string_from_hex(TokenDeserializer* self, char c){
c11_string sv = pk_TokenDeserializer__read_string(self, c); c11_string sv = TokenDeserializer__read_string(self, c);
const char* s = sv.data; const char* s = sv.data;
char* buffer = (char*)malloc(sv.size / 2 + 1); char* buffer = (char*)malloc(sv.size / 2 + 1);
for(int i = 0; i < sv.size; i += 2) { for(int i = 0; i < sv.size; i += 2) {
@ -927,13 +927,13 @@ py_Str pk_TokenDeserializer__read_string_from_hex(pk_TokenDeserializer* self, ch
}; };
} }
int pk_TokenDeserializer__read_count(pk_TokenDeserializer* self){ int TokenDeserializer__read_count(TokenDeserializer* self){
assert(*self->curr == '='); assert(*self->curr == '=');
self->curr++; self->curr++;
return pk_TokenDeserializer__read_uint(self, '\n'); return TokenDeserializer__read_uint(self, '\n');
} }
int64_t pk_TokenDeserializer__read_uint(pk_TokenDeserializer* self, char c){ int64_t TokenDeserializer__read_uint(TokenDeserializer* self, char c){
int64_t out = 0; int64_t out = 0;
while(*self->curr != c) { while(*self->curr != c) {
out = out * 10 + (*self->curr - '0'); out = out * 10 + (*self->curr - '0');
@ -943,8 +943,8 @@ int64_t pk_TokenDeserializer__read_uint(pk_TokenDeserializer* self, char c){
return out; return out;
} }
double pk_TokenDeserializer__read_float(pk_TokenDeserializer* self, char c){ double TokenDeserializer__read_float(TokenDeserializer* self, char c){
c11_string sv = pk_TokenDeserializer__read_string(self, c); c11_string sv = TokenDeserializer__read_string(self, c);
py_Str nullterm; py_Str nullterm;
py_Str__ctor2(&nullterm, sv.data, sv.size); py_Str__ctor2(&nullterm, sv.data, sv.size);
char* end; char* end;