mirror of
https://github.com/pocketpy/pocketpy
synced 2025-10-28 15:30:17 +00:00
Compare commits
4 Commits
23ffa73f4c
...
92fcc116af
| Author | SHA1 | Date | |
|---|---|---|---|
|
92fcc116af | ||
|
|
dbd75615d9 | ||
|
|
1431cf8cde | ||
|
|
eb1806deaa |
@ -17,7 +17,7 @@ struct PrattRule {
|
||||
struct Compiler {
|
||||
PK_ALWAYS_PASS_BY_POINTER(Compiler)
|
||||
|
||||
static PrattRule rules[kTokenCount];
|
||||
static PrattRule rules[TK__COUNT__];
|
||||
|
||||
Lexer lexer;
|
||||
vector<CodeEmitContext> contexts;
|
||||
@ -43,7 +43,7 @@ struct Compiler {
|
||||
printf("%s:%d %s %s\n",
|
||||
lexer.src.filename().c_str(),
|
||||
curr().line,
|
||||
TK_STR(curr().type),
|
||||
pk_TokenSymbols(curr().type),
|
||||
curr().str().escape().c_str()
|
||||
);
|
||||
}
|
||||
|
||||
@ -10,7 +10,48 @@ struct Expr;
|
||||
|
||||
typedef small_vector<Expr*, 4> Expr_vector;
|
||||
|
||||
static bool default_false(const Expr*) { return false; }
|
||||
static int default_zero(const Expr*) { return 0; }
|
||||
static void default_dtor(Expr*) {}
|
||||
|
||||
struct ExprVt{
|
||||
void (*dtor)(Expr*);
|
||||
/* reflections */
|
||||
bool (*is_literal)(const Expr*);
|
||||
bool (*is_json_object)(const Expr*);
|
||||
bool (*is_attrib)(const Expr*);
|
||||
bool (*is_subscr)(const Expr*);
|
||||
bool (*is_compare)(const Expr*);
|
||||
int (*star_level)(const Expr*);
|
||||
bool (*is_tuple)(const Expr*);
|
||||
bool (*is_name)(const Expr*);
|
||||
/* emit */
|
||||
void (*emit_)(Expr*, CodeEmitContext*);
|
||||
bool (*emit_del)(Expr*, CodeEmitContext*);
|
||||
bool (*emit_store)(Expr*, CodeEmitContext*);
|
||||
void (*emit_inplace)(Expr*, CodeEmitContext*);
|
||||
bool (*emit_store_inplace)(Expr*, CodeEmitContext*);
|
||||
};
|
||||
|
||||
void ExprVt__ctor(ExprVt* vt){
|
||||
vt->dtor = default_dtor;
|
||||
vt->is_literal = default_false;
|
||||
vt->is_json_object = default_false;
|
||||
vt->is_attrib = default_false;
|
||||
vt->is_subscr = default_false;
|
||||
vt->is_compare = default_false;
|
||||
vt->star_level = default_zero;
|
||||
vt->is_tuple = default_false;
|
||||
vt->is_name = default_false;
|
||||
vt->emit_ = NULL; // must be set
|
||||
vt->emit_del = NULL;
|
||||
vt->emit_store = NULL;
|
||||
vt->emit_inplace = NULL;
|
||||
vt->emit_store_inplace = NULL;
|
||||
}
|
||||
|
||||
struct Expr {
|
||||
ExprVt* vt;
|
||||
int line = 0;
|
||||
virtual ~Expr() = default;
|
||||
virtual void emit_(CodeEmitContext* ctx) = 0;
|
||||
@ -22,34 +63,53 @@ struct Expr {
|
||||
Expr& operator=(Expr&&) = delete;
|
||||
|
||||
virtual bool is_literal() const { return false; }
|
||||
|
||||
virtual bool is_json_object() const { return false; }
|
||||
|
||||
virtual bool is_attrib() const { return false; }
|
||||
|
||||
virtual bool is_subscr() const { return false; }
|
||||
|
||||
virtual bool is_compare() const { return false; }
|
||||
|
||||
virtual int star_level() const { return 0; }
|
||||
|
||||
virtual bool is_tuple() const { return false; }
|
||||
|
||||
virtual bool is_name() const { return false; }
|
||||
|
||||
bool is_starred() const { return star_level() > 0; }
|
||||
|
||||
// for OP_DELETE_XXX
|
||||
[[nodiscard]] virtual bool emit_del(CodeEmitContext* ctx) { return false; }
|
||||
|
||||
// for OP_STORE_XXX
|
||||
[[nodiscard]] virtual bool emit_store(CodeEmitContext* ctx) { return false; }
|
||||
|
||||
virtual void emit_inplace(CodeEmitContext* ctx) { emit_(ctx); }
|
||||
|
||||
[[nodiscard]] virtual bool emit_store_inplace(CodeEmitContext* ctx) { return emit_store(ctx); }
|
||||
};
|
||||
|
||||
void pk_Expr__emit_(Expr* self, CodeEmitContext* ctx){
|
||||
assert(self->vt->emit_);
|
||||
self->vt->emit_(self, ctx);
|
||||
}
|
||||
|
||||
bool pk_Expr__emit_del(Expr* self, CodeEmitContext* ctx){
|
||||
if(!self->vt->emit_del) return false;
|
||||
return self->vt->emit_del(self, ctx);
|
||||
}
|
||||
|
||||
bool pk_Expr__emit_store(Expr* self, CodeEmitContext* ctx){
|
||||
if(!self->vt->emit_store) return false;
|
||||
return self->vt->emit_store(self, ctx);
|
||||
}
|
||||
|
||||
void pk_Expr__emit_inplace(Expr* self, CodeEmitContext* ctx){
|
||||
if(!self->vt->emit_inplace){
|
||||
pk_Expr__emit_(self, ctx);
|
||||
return;
|
||||
}
|
||||
self->vt->emit_inplace(self, ctx);
|
||||
}
|
||||
|
||||
bool pk_Expr__emit_store_inplace(Expr* self, CodeEmitContext* ctx){
|
||||
if(!self->vt->emit_store_inplace){
|
||||
return pk_Expr__emit_store(self, ctx);
|
||||
}
|
||||
return self->vt->emit_store_inplace(self, ctx);
|
||||
}
|
||||
|
||||
inline void delete_expr(Expr* p) noexcept{
|
||||
if(!p) return;
|
||||
p->~Expr();
|
||||
|
||||
@ -1,24 +1,51 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdbool.h>
|
||||
#include "pocketpy/common/str.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct pkpy_TokenDeserializer {
|
||||
extern const char* pk_TokenSymbols[];
|
||||
|
||||
typedef struct pk_TokenDeserializer {
|
||||
const char* curr;
|
||||
const char* source;
|
||||
} pkpy_TokenDeserializer;
|
||||
} pk_TokenDeserializer;
|
||||
|
||||
void pkpy_TokenDeserializer__ctor(pkpy_TokenDeserializer* self, const char* source);
|
||||
bool pkpy_TokenDeserializer__match_char(pkpy_TokenDeserializer* self, char c);
|
||||
c11_string pkpy_TokenDeserializer__read_string(pkpy_TokenDeserializer* self, char c);
|
||||
pkpy_Str pkpy_TokenDeserializer__read_string_from_hex(pkpy_TokenDeserializer* self, char c);
|
||||
int pkpy_TokenDeserializer__read_count(pkpy_TokenDeserializer* self);
|
||||
int64_t pkpy_TokenDeserializer__read_uint(pkpy_TokenDeserializer* self, char c);
|
||||
double pkpy_TokenDeserializer__read_float(pkpy_TokenDeserializer* self, char c);
|
||||
typedef enum TokenIndex{
|
||||
TK_EOF, TK_EOL, TK_SOF,
|
||||
TK_ID, TK_NUM, TK_STR, TK_FSTR, TK_LONG, TK_BYTES, TK_IMAG,
|
||||
TK_INDENT, TK_DEDENT,
|
||||
/***************/
|
||||
TK_IS_NOT, TK_NOT_IN, TK_YIELD_FROM,
|
||||
/***************/
|
||||
TK_ADD, TK_IADD, TK_SUB, TK_ISUB,
|
||||
TK_MUL, TK_IMUL, TK_DIV, TK_IDIV, TK_FLOORDIV, TK_IFLOORDIV, TK_MOD, TK_IMOD,
|
||||
TK_AND, TK_IAND, TK_OR, TK_IOR, TK_XOR, TK_IXOR,
|
||||
TK_LSHIFT, TK_ILSHIFT, TK_RSHIFT, TK_IRSHIFT,
|
||||
/***************/
|
||||
TK_LPAREN, TK_RPAREN, TK_LBRACKET, TK_RBRACKET, TK_LBRACE, TK_RBRACE,
|
||||
TK_DOT, TK_DOTDOT, TK_DOTDOTDOT, TK_COMMA, TK_COLON, TK_SEMICOLON,
|
||||
TK_POW, TK_ARROW, TK_HASH, TK_DECORATOR,
|
||||
TK_GT, TK_LT, TK_ASSIGN, TK_EQ, TK_NE, TK_GE, TK_LE, TK_INVERT,
|
||||
/***************/
|
||||
TK_FALSE, TK_NONE, TK_TRUE, TK_AND_KW, TK_AS, TK_ASSERT, TK_BREAK, TK_CLASS, TK_CONTINUE,
|
||||
TK_DEF, TK_DEL, TK_ELIF, TK_ELSE, TK_EXCEPT, TK_FINALLY, TK_FOR, TK_FROM, TK_GLOBAL,
|
||||
TK_IF, TK_IMPORT, TK_IN, TK_IS, TK_LAMBDA, TK_NOT_KW, TK_OR_KW, TK_PASS, TK_RAISE, TK_RETURN,
|
||||
TK_TRY, TK_WHILE, TK_WITH, TK_YIELD,
|
||||
/***************/
|
||||
TK__COUNT__
|
||||
} TokenIndex;
|
||||
|
||||
void pk_TokenDeserializer__ctor(pk_TokenDeserializer* self, const char* source);
|
||||
bool pk_TokenDeserializer__match_char(pk_TokenDeserializer* self, char c);
|
||||
c11_string pk_TokenDeserializer__read_string(pk_TokenDeserializer* self, char c);
|
||||
pkpy_Str pk_TokenDeserializer__read_string_from_hex(pk_TokenDeserializer* self, char c);
|
||||
int pk_TokenDeserializer__read_count(pk_TokenDeserializer* self);
|
||||
int64_t pk_TokenDeserializer__read_uint(pk_TokenDeserializer* self, char c);
|
||||
double pk_TokenDeserializer__read_float(pk_TokenDeserializer* self, char c);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@ -2,56 +2,15 @@
|
||||
|
||||
#include "pocketpy/objects/error.hpp"
|
||||
#include "pocketpy/objects/sourcedata.h"
|
||||
#include "pocketpy/compiler/lexer.h"
|
||||
|
||||
#include <variant>
|
||||
|
||||
namespace pkpy {
|
||||
|
||||
typedef uint8_t TokenIndex;
|
||||
|
||||
// clang-format off
|
||||
constexpr const char* kTokens[] = {
|
||||
"is not", "not in", "yield from",
|
||||
"@eof", "@eol", "@sof",
|
||||
"@id", "@num", "@str", "@fstr", "@long", "@bytes", "@imag",
|
||||
"@indent", "@dedent",
|
||||
/*****************************************/
|
||||
"+", "+=", "-", "-=", // (INPLACE_OP - 1) can get '=' removed
|
||||
"*", "*=", "/", "/=", "//", "//=", "%", "%=",
|
||||
"&", "&=", "|", "|=", "^", "^=",
|
||||
"<<", "<<=", ">>", ">>=",
|
||||
/*****************************************/
|
||||
".", ",", ":", ";", "#", "(", ")", "[", "]", "{", "}",
|
||||
"**", "=", ">", "<", "..", "...", "->", "@", "==", "!=", ">=", "<=",
|
||||
"++", "--", "~",
|
||||
/** KW_BEGIN **/
|
||||
// NOTE: These keywords should be sorted in ascending order!!
|
||||
"False", "None", "True", "and", "as", "assert", "break", "class", "continue",
|
||||
"def", "del", "elif", "else", "except", "finally", "for", "from", "global",
|
||||
"if", "import", "in", "is", "lambda", "not", "or", "pass", "raise", "return",
|
||||
"try", "while", "with", "yield",
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
using TokenValue = std::variant<std::monostate, i64, f64, Str>;
|
||||
const int kTokenCount = sizeof(kTokens) / sizeof(kTokens[0]);
|
||||
|
||||
constexpr TokenIndex TK(const char token[]) {
|
||||
for(int k = 0; k < kTokenCount; k++) {
|
||||
const char* i = kTokens[k];
|
||||
const char* j = token;
|
||||
while(*i && *j && *i == *j) {
|
||||
i++;
|
||||
j++;
|
||||
}
|
||||
if(*i == *j) return k;
|
||||
}
|
||||
return 255;
|
||||
}
|
||||
|
||||
constexpr inline bool is_raw_string_used(TokenIndex t) noexcept{ return t == TK("@id") || t == TK("@long"); }
|
||||
|
||||
#define TK_STR(t) kTokens[t]
|
||||
constexpr inline bool is_raw_string_used(TokenIndex t) noexcept{ return t == TK_ID || t == TK_LONG; }
|
||||
|
||||
struct Token {
|
||||
TokenIndex type;
|
||||
|
||||
@ -34,6 +34,15 @@ void pkpy_Exception__dtor(pkpy_Exception* self);
|
||||
void pkpy_Exception__stpush(pkpy_Exception* self, pkpy_SourceData_ src, int lineno, const char* cursor, const char* name);
|
||||
pkpy_Str pkpy_Exception__summary(pkpy_Exception* self);
|
||||
|
||||
struct Error{
|
||||
const char* type;
|
||||
pkpy_SourceData_ src;
|
||||
int lineno;
|
||||
const char* cursor;
|
||||
char msg[100];
|
||||
int64_t userdata;
|
||||
};
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@ -59,13 +59,6 @@ struct TopLevelException : std::exception {
|
||||
}
|
||||
};
|
||||
|
||||
struct Error{
|
||||
const char* type;
|
||||
pkpy_SourceData_ src;
|
||||
int lineno;
|
||||
const char* cursor;
|
||||
char msg[100];
|
||||
i64 userdata;
|
||||
};
|
||||
|
||||
|
||||
} // namespace pkpy
|
||||
|
||||
@ -406,6 +406,7 @@ bool c11__is_unicode_Lo_char(int c){
|
||||
if(c == 0x1f955) return true;
|
||||
int index;
|
||||
c11__lower_bound(const int, kLoRangeA, 476, c, c11__less, &index);
|
||||
if(index == 476) return false;
|
||||
if(c == kLoRangeA[index]) return true;
|
||||
index -= 1;
|
||||
if(index < 0) return false;
|
||||
|
||||
@ -6,12 +6,12 @@
|
||||
|
||||
namespace pkpy {
|
||||
|
||||
#define consume(expected) if(!match(expected)) return SyntaxError("expected '%s', got '%s'", TK_STR(expected), TK_STR(curr().type));
|
||||
#define consume(expected) if(!match(expected)) return SyntaxError("expected '%s', got '%s'", pk_TokenSymbols[expected], pk_TokenSymbols[curr().type]);
|
||||
#define consume_end_stmt() if(!match_end_stmt()) return SyntaxError("expected statement end")
|
||||
#define check_newlines_repl() { bool __nml; match_newlines(&__nml); if(__nml) return NeedMoreLines(); }
|
||||
#define check(B) if((err = B)) return err
|
||||
|
||||
PrattRule Compiler::rules[kTokenCount];
|
||||
PrattRule Compiler::rules[TK__COUNT__];
|
||||
|
||||
NameScope Compiler::name_scope() const noexcept{
|
||||
auto s = contexts.size() > 1 ? NAME_LOCAL : NAME_GLOBAL;
|
||||
@ -44,7 +44,7 @@ Error* Compiler::pop_context() noexcept{
|
||||
ctx()->emit_(OP_RETURN_VALUE, 1, BC_KEEPLINE, true);
|
||||
// find the last valid token
|
||||
int j = __i - 1;
|
||||
while(tk(j).type == TK("@eol") || tk(j).type == TK("@dedent") || tk(j).type == TK("@eof"))
|
||||
while(tk(j).type == TK_EOL || tk(j).type == TK_DEDENT || tk(j).type == TK_EOF)
|
||||
j--;
|
||||
ctx()->co->end_line = tk(j).line;
|
||||
|
||||
@ -115,52 +115,52 @@ void Compiler::init_pratt_rules() noexcept{
|
||||
// http://journal.stuffwithstuff.com/2011/03/19/pratt-parsers-expression-parsing-made-easy/
|
||||
#define PK_METHOD(name) &Compiler::name
|
||||
#define PK_NO_INFIX nullptr, PREC_LOWEST
|
||||
for(TokenIndex i = 0; i < kTokenCount; i++) rules[i] = { nullptr, PK_NO_INFIX };
|
||||
rules[TK(".")] = { nullptr, PK_METHOD(exprAttrib), PREC_PRIMARY };
|
||||
rules[TK("(")] = { PK_METHOD(exprGroup), PK_METHOD(exprCall), PREC_PRIMARY };
|
||||
rules[TK("[")] = { PK_METHOD(exprList), PK_METHOD(exprSubscr), PREC_PRIMARY };
|
||||
rules[TK("{")] = { PK_METHOD(exprMap), PK_NO_INFIX };
|
||||
rules[TK("%")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK("+")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_TERM };
|
||||
rules[TK("-")] = { PK_METHOD(exprUnaryOp), PK_METHOD(exprBinaryOp), PREC_TERM };
|
||||
rules[TK("*")] = { PK_METHOD(exprUnaryOp), PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK("~")] = { PK_METHOD(exprUnaryOp), nullptr, PREC_UNARY };
|
||||
rules[TK("/")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK("//")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK("**")] = { PK_METHOD(exprUnaryOp), PK_METHOD(exprBinaryOp), PREC_EXPONENT };
|
||||
rules[TK(">")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("<")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("==")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("!=")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK(">=")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("<=")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("in")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("is")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("<<")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_SHIFT };
|
||||
rules[TK(">>")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_SHIFT };
|
||||
rules[TK("&")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_AND };
|
||||
rules[TK("|")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_OR };
|
||||
rules[TK("^")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_XOR };
|
||||
rules[TK("@")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK("if")] = { nullptr, PK_METHOD(exprTernary), PREC_TERNARY };
|
||||
rules[TK("not in")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("is not")] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK("and") ] = { nullptr, PK_METHOD(exprAnd), PREC_LOGICAL_AND };
|
||||
rules[TK("or")] = { nullptr, PK_METHOD(exprOr), PREC_LOGICAL_OR };
|
||||
rules[TK("not")] = { PK_METHOD(exprNot), nullptr, PREC_LOGICAL_NOT };
|
||||
rules[TK("True")] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK("False")] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK("None")] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK("...")] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK("lambda")] = { PK_METHOD(exprLambda), PK_NO_INFIX };
|
||||
rules[TK("@id")] = { PK_METHOD(exprName), PK_NO_INFIX };
|
||||
rules[TK("@num")] = { PK_METHOD(exprLiteral), PK_NO_INFIX };
|
||||
rules[TK("@str")] = { PK_METHOD(exprLiteral), PK_NO_INFIX };
|
||||
rules[TK("@fstr")] = { PK_METHOD(exprFString), PK_NO_INFIX };
|
||||
rules[TK("@long")] = { PK_METHOD(exprLong), PK_NO_INFIX };
|
||||
rules[TK("@imag")] = { PK_METHOD(exprImag), PK_NO_INFIX };
|
||||
rules[TK("@bytes")] = { PK_METHOD(exprBytes), PK_NO_INFIX };
|
||||
rules[TK(":")] = { PK_METHOD(exprSlice0), PK_METHOD(exprSlice1), PREC_PRIMARY };
|
||||
for(int i = 0; i < TK__COUNT__; i++) rules[i] = { nullptr, PK_NO_INFIX };
|
||||
rules[TK_DOT] = { nullptr, PK_METHOD(exprAttrib), PREC_PRIMARY };
|
||||
rules[TK_LPAREN] = { PK_METHOD(exprGroup), PK_METHOD(exprCall), PREC_PRIMARY };
|
||||
rules[TK_LBRACKET] = { PK_METHOD(exprList), PK_METHOD(exprSubscr), PREC_PRIMARY };
|
||||
rules[TK_LBRACE] = { PK_METHOD(exprMap), PK_NO_INFIX };
|
||||
rules[TK_MOD] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK_ADD] = { nullptr, PK_METHOD(exprBinaryOp), PREC_TERM };
|
||||
rules[TK_SUB] = { PK_METHOD(exprUnaryOp), PK_METHOD(exprBinaryOp), PREC_TERM };
|
||||
rules[TK_MUL] = { PK_METHOD(exprUnaryOp), PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK_INVERT] = { PK_METHOD(exprUnaryOp), nullptr, PREC_UNARY };
|
||||
rules[TK_DIV] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK_FLOORDIV] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK_POW] = { PK_METHOD(exprUnaryOp), PK_METHOD(exprBinaryOp), PREC_EXPONENT };
|
||||
rules[TK_GT] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_LT] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_EQ] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_NE] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_GE] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_LE] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_IN] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_IS] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_LSHIFT] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_SHIFT };
|
||||
rules[TK_RSHIFT] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_SHIFT };
|
||||
rules[TK_AND] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_AND };
|
||||
rules[TK_OR] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_OR };
|
||||
rules[TK_XOR] = { nullptr, PK_METHOD(exprBinaryOp), PREC_BITWISE_XOR };
|
||||
rules[TK_DECORATOR] = { nullptr, PK_METHOD(exprBinaryOp), PREC_FACTOR };
|
||||
rules[TK_IF] = { nullptr, PK_METHOD(exprTernary), PREC_TERNARY };
|
||||
rules[TK_NOT_IN] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_IS_NOT] = { nullptr, PK_METHOD(exprBinaryOp), PREC_COMPARISION };
|
||||
rules[TK_AND_KW ] = { nullptr, PK_METHOD(exprAnd), PREC_LOGICAL_AND };
|
||||
rules[TK_OR_KW] = { nullptr, PK_METHOD(exprOr), PREC_LOGICAL_OR };
|
||||
rules[TK_NOT_KW] = { PK_METHOD(exprNot), nullptr, PREC_LOGICAL_NOT };
|
||||
rules[TK_TRUE] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK_FALSE] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK_NONE] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK_DOTDOTDOT] = { PK_METHOD(exprLiteral0), PK_NO_INFIX };
|
||||
rules[TK_LAMBDA] = { PK_METHOD(exprLambda), PK_NO_INFIX };
|
||||
rules[TK_ID] = { PK_METHOD(exprName), PK_NO_INFIX };
|
||||
rules[TK_NUM] = { PK_METHOD(exprLiteral), PK_NO_INFIX };
|
||||
rules[TK_STR] = { PK_METHOD(exprLiteral), PK_NO_INFIX };
|
||||
rules[TK_FSTR] = { PK_METHOD(exprFString), PK_NO_INFIX };
|
||||
rules[TK_LONG] = { PK_METHOD(exprLong), PK_NO_INFIX };
|
||||
rules[TK_IMAG] = { PK_METHOD(exprImag), PK_NO_INFIX };
|
||||
rules[TK_BYTES] = { PK_METHOD(exprBytes), PK_NO_INFIX };
|
||||
rules[TK_COLON] = { PK_METHOD(exprSlice0), PK_METHOD(exprSlice1), PREC_PRIMARY };
|
||||
|
||||
#undef PK_METHOD
|
||||
#undef PK_NO_INFIX
|
||||
@ -175,30 +175,30 @@ bool Compiler::match(TokenIndex expected) noexcept{
|
||||
|
||||
bool Compiler::match_newlines(bool* need_more_lines) noexcept{
|
||||
bool consumed = false;
|
||||
if(curr().type == TK("@eol")) {
|
||||
while(curr().type == TK("@eol")) advance();
|
||||
if(curr().type == TK_EOL) {
|
||||
while(curr().type == TK_EOL) advance();
|
||||
consumed = true;
|
||||
}
|
||||
if(need_more_lines) {
|
||||
*need_more_lines = (mode() == REPL_MODE && curr().type == TK("@eof"));
|
||||
*need_more_lines = (mode() == REPL_MODE && curr().type == TK_EOF);
|
||||
}
|
||||
return consumed;
|
||||
}
|
||||
|
||||
bool Compiler::match_end_stmt() noexcept{
|
||||
if(match(TK(";"))) {
|
||||
if(match(TK_SEMICOLON)) {
|
||||
match_newlines();
|
||||
return true;
|
||||
}
|
||||
if(match_newlines() || curr().type == TK("@eof")) return true;
|
||||
if(curr().type == TK("@dedent")) return true;
|
||||
if(match_newlines() || curr().type == TK_EOF) return true;
|
||||
if(curr().type == TK_DEDENT) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
Error* Compiler::EXPR_TUPLE(bool allow_slice) noexcept{
|
||||
Error* err;
|
||||
check(parse_expression(PREC_LOWEST + 1, allow_slice));
|
||||
if(!match(TK(","))) return NULL;
|
||||
if(!match(TK_COMMA)) return NULL;
|
||||
// tuple expression
|
||||
int count = 1;
|
||||
do {
|
||||
@ -207,7 +207,7 @@ Error* Compiler::EXPR_TUPLE(bool allow_slice) noexcept{
|
||||
check(parse_expression(PREC_LOWEST + 1, allow_slice));
|
||||
count += 1;
|
||||
if(curr().brackets_level) check_newlines_repl();
|
||||
} while(match(TK(",")));
|
||||
} while(match(TK_COMMA));
|
||||
TupleExpr* e = make_expr<TupleExpr>(count);
|
||||
for(int i=count-1; i>=0; i--)
|
||||
e->items[i] = ctx()->s_popx();
|
||||
@ -218,10 +218,10 @@ Error* Compiler::EXPR_TUPLE(bool allow_slice) noexcept{
|
||||
Error* Compiler::EXPR_VARS() noexcept{
|
||||
int count = 0;
|
||||
do {
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
ctx()->s_push(make_expr<NameExpr>(prev().str(), name_scope()));
|
||||
count += 1;
|
||||
} while(match(TK(",")));
|
||||
} while(match(TK_COMMA));
|
||||
if(count > 1){
|
||||
TupleExpr* e = make_expr<TupleExpr>(count);
|
||||
for(int i=count-1; i>=0; i--)
|
||||
@ -260,9 +260,9 @@ Error* Compiler::exprLambda() noexcept{
|
||||
Error* err;
|
||||
FuncDecl_ decl = push_f_context("<lambda>");
|
||||
int line = prev().line; // backup line
|
||||
if(!match(TK(":"))) {
|
||||
if(!match(TK_COLON)) {
|
||||
check(_compile_f_args(decl, false));
|
||||
consume(TK(":"));
|
||||
consume(TK_COLON);
|
||||
}
|
||||
// https://github.com/pocketpy/pocketpy/issues/37
|
||||
check(parse_expression(PREC_LAMBDA + 1));
|
||||
@ -304,7 +304,7 @@ Error* Compiler::exprTernary() noexcept{
|
||||
Error* err;
|
||||
int line = prev().line;
|
||||
check(parse_expression(PREC_TERNARY + 1)); // [true_expr, cond]
|
||||
consume(TK("else"));
|
||||
consume(TK_ELSE);
|
||||
check(parse_expression(PREC_TERNARY + 1)); // [true_expr, cond, false_expr]
|
||||
auto e = make_expr<TernaryExpr>();
|
||||
e->line = line;
|
||||
@ -341,10 +341,10 @@ Error* Compiler::exprUnaryOp() noexcept{
|
||||
TokenIndex op = prev().type;
|
||||
check(parse_expression(PREC_UNARY + 1));
|
||||
switch(op) {
|
||||
case TK("-"): ctx()->s_push(make_expr<NegatedExpr>(ctx()->s_popx())); break;
|
||||
case TK("~"): ctx()->s_push(make_expr<InvertExpr>(ctx()->s_popx())); break;
|
||||
case TK("*"): ctx()->s_push(make_expr<StarredExpr>(ctx()->s_popx(), 1)); break;
|
||||
case TK("**"): ctx()->s_push(make_expr<StarredExpr>(ctx()->s_popx(), 2)); break;
|
||||
case TK_SUB: ctx()->s_push(make_expr<NegatedExpr>(ctx()->s_popx())); break;
|
||||
case TK_INVERT: ctx()->s_push(make_expr<InvertExpr>(ctx()->s_popx())); break;
|
||||
case TK_MUL: ctx()->s_push(make_expr<StarredExpr>(ctx()->s_popx(), 1)); break;
|
||||
case TK_POW: ctx()->s_push(make_expr<StarredExpr>(ctx()->s_popx(), 2)); break;
|
||||
default: assert(false);
|
||||
}
|
||||
return NULL;
|
||||
@ -355,7 +355,7 @@ Error* Compiler::exprGroup() noexcept{
|
||||
check_newlines_repl()
|
||||
check(EXPR_TUPLE()); // () is just for change precedence
|
||||
check_newlines_repl()
|
||||
consume(TK(")"));
|
||||
consume(TK_RPAREN);
|
||||
if(ctx()->s_top()->is_tuple()) return NULL;
|
||||
Expr* g = make_expr<GroupedExpr>(ctx()->s_popx());
|
||||
ctx()->s_push(g);
|
||||
@ -367,10 +367,10 @@ Error* Compiler::consume_comp(Opcode op0, Opcode op1) noexcept{
|
||||
Error* err;
|
||||
bool has_cond = false;
|
||||
check(EXPR_VARS()); // [expr, vars]
|
||||
consume(TK("in"));
|
||||
consume(TK_IN);
|
||||
check(parse_expression(PREC_TERNARY + 1)); // [expr, vars, iter]
|
||||
check_newlines_repl()
|
||||
if(match(TK("if"))) {
|
||||
if(match(TK_IF)) {
|
||||
check(parse_expression(PREC_TERNARY + 1)); // [expr, vars, iter, cond]
|
||||
has_cond = true;
|
||||
}
|
||||
@ -390,17 +390,17 @@ Error* Compiler::exprList() noexcept{
|
||||
int count = 0;
|
||||
do {
|
||||
check_newlines_repl()
|
||||
if(curr().type == TK("]")) break;
|
||||
if(curr().type == TK_RBRACKET) break;
|
||||
check(EXPR()); count += 1;
|
||||
check_newlines_repl()
|
||||
if(count == 1 && match(TK("for"))) {
|
||||
if(count == 1 && match(TK_FOR)) {
|
||||
check(consume_comp(OP_BUILD_LIST, OP_LIST_APPEND));
|
||||
consume(TK("]"));
|
||||
consume(TK_RBRACKET);
|
||||
return NULL;
|
||||
}
|
||||
check_newlines_repl()
|
||||
} while(match(TK(",")));
|
||||
consume(TK("]"));
|
||||
} while(match(TK_COMMA));
|
||||
consume(TK_RBRACKET);
|
||||
ListExpr* e = make_expr<ListExpr>(count);
|
||||
e->line = line; // override line
|
||||
for(int i=count-1; i>=0; i--)
|
||||
@ -415,10 +415,10 @@ Error* Compiler::exprMap() noexcept{
|
||||
int count = 0;
|
||||
do {
|
||||
check_newlines_repl()
|
||||
if(curr().type == TK("}")) break;
|
||||
if(curr().type == TK_RBRACE) break;
|
||||
check(EXPR()); // [key]
|
||||
int star_level = ctx()->s_top()->star_level();
|
||||
if(star_level == 2 || curr().type == TK(":")) { parsing_dict = true; }
|
||||
if(star_level == 2 || curr().type == TK_COLON) { parsing_dict = true; }
|
||||
if(parsing_dict) {
|
||||
if(star_level == 2) {
|
||||
DictItemExpr* dict_item = make_expr<DictItemExpr>();
|
||||
@ -426,7 +426,7 @@ Error* Compiler::exprMap() noexcept{
|
||||
dict_item->value = ctx()->s_popx();
|
||||
ctx()->s_push(dict_item);
|
||||
} else {
|
||||
consume(TK(":"));
|
||||
consume(TK_COLON);
|
||||
check(EXPR());
|
||||
DictItemExpr* dict_item = make_expr<DictItemExpr>();
|
||||
dict_item->value = ctx()->s_popx();
|
||||
@ -436,18 +436,18 @@ Error* Compiler::exprMap() noexcept{
|
||||
}
|
||||
count += 1;
|
||||
check_newlines_repl()
|
||||
if(count == 1 && match(TK("for"))) {
|
||||
if(count == 1 && match(TK_FOR)) {
|
||||
if(parsing_dict){
|
||||
check(consume_comp(OP_BUILD_DICT, OP_DICT_ADD));
|
||||
}else{
|
||||
check(consume_comp(OP_BUILD_SET, OP_SET_ADD));
|
||||
}
|
||||
consume(TK("}"));
|
||||
consume(TK_RBRACE);
|
||||
return NULL;
|
||||
}
|
||||
check_newlines_repl()
|
||||
} while(match(TK(",")));
|
||||
consume(TK("}"));
|
||||
} while(match(TK_COMMA));
|
||||
consume(TK_RBRACE);
|
||||
|
||||
SequenceExpr* se;
|
||||
if(count == 0 || parsing_dict) {
|
||||
@ -468,11 +468,11 @@ Error* Compiler::exprCall() noexcept{
|
||||
ctx()->s_push(e); // push onto the stack in advance
|
||||
do {
|
||||
check_newlines_repl()
|
||||
if(curr().type == TK(")")) break;
|
||||
if(curr().type == TK("@id") && next().type == TK("=")) {
|
||||
consume(TK("@id"));
|
||||
if(curr().type == TK_RPAREN) break;
|
||||
if(curr().type == TK_ID && next().type == TK_ASSIGN) {
|
||||
consume(TK_ID);
|
||||
StrName key(prev().sv());
|
||||
consume(TK("="));
|
||||
consume(TK_ASSIGN);
|
||||
check(EXPR());
|
||||
e->kwargs.push_back({key, ctx()->s_popx()});
|
||||
} else {
|
||||
@ -487,8 +487,8 @@ Error* Compiler::exprCall() noexcept{
|
||||
}
|
||||
}
|
||||
check_newlines_repl()
|
||||
} while(match(TK(",")));
|
||||
consume(TK(")"));
|
||||
} while(match(TK_COMMA));
|
||||
consume(TK_RPAREN);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -501,7 +501,7 @@ Error* Compiler::exprName() noexcept{
|
||||
}
|
||||
|
||||
Error* Compiler::exprAttrib() noexcept{
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
ctx()->s_push(make_expr<AttribExpr>(ctx()->s_popx(), StrName::get(prev().sv())));
|
||||
return NULL;
|
||||
}
|
||||
@ -514,11 +514,11 @@ Error* Compiler::exprSlice0() noexcept{
|
||||
check(EXPR());
|
||||
slice->stop = ctx()->s_popx();
|
||||
// try optional step
|
||||
if(match(TK(":"))) { // :<stop>:<step>
|
||||
if(match(TK_COLON)) { // :<stop>:<step>
|
||||
check(EXPR());
|
||||
slice->step = ctx()->s_popx();
|
||||
}
|
||||
} else if(match(TK(":"))) {
|
||||
} else if(match(TK_COLON)) {
|
||||
if(is_expression()) { // ::<step>
|
||||
check(EXPR());
|
||||
slice->step = ctx()->s_popx();
|
||||
@ -536,11 +536,11 @@ Error* Compiler::exprSlice1() noexcept{
|
||||
check(EXPR());
|
||||
slice->stop = ctx()->s_popx();
|
||||
// try optional step
|
||||
if(match(TK(":"))) { // <start>:<stop>:<step>
|
||||
if(match(TK_COLON)) { // <start>:<stop>:<step>
|
||||
check(EXPR());
|
||||
slice->step = ctx()->s_popx();
|
||||
}
|
||||
} else if(match(TK(":"))) { // <start>::<step>
|
||||
} else if(match(TK_COLON)) { // <start>::<step>
|
||||
check(EXPR());
|
||||
slice->step = ctx()->s_popx();
|
||||
} // else <start>:
|
||||
@ -553,7 +553,7 @@ Error* Compiler::exprSubscr() noexcept{
|
||||
check_newlines_repl()
|
||||
check(EXPR_TUPLE(true));
|
||||
check_newlines_repl()
|
||||
consume(TK("]")); // [lhs, rhs]
|
||||
consume(TK_RBRACKET); // [lhs, rhs]
|
||||
SubscrExpr* e = make_expr<SubscrExpr>();
|
||||
e->line = line;
|
||||
e->rhs = ctx()->s_popx(); // [lhs]
|
||||
@ -570,12 +570,12 @@ Error* Compiler::exprLiteral0() noexcept{
|
||||
Error* Compiler::compile_block_body(PrattCallback callback) noexcept{
|
||||
Error* err;
|
||||
if(!callback) callback = &Compiler::compile_stmt;
|
||||
consume(TK(":"));
|
||||
if(curr().type != TK("@eol") && curr().type != TK("@eof")) {
|
||||
consume(TK_COLON);
|
||||
if(curr().type != TK_EOL && curr().type != TK_EOF) {
|
||||
while(true) {
|
||||
check(compile_stmt());
|
||||
bool possible = curr().type != TK("@eol") && curr().type != TK("@eof");
|
||||
if(prev().type != TK(";") || !possible) break;
|
||||
bool possible = curr().type != TK_EOL && curr().type != TK_EOF;
|
||||
if(prev().type != TK_SEMICOLON || !possible) break;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
@ -585,13 +585,13 @@ Error* Compiler::compile_block_body(PrattCallback callback) noexcept{
|
||||
if(need_more_lines) return NeedMoreLines();
|
||||
if(!consumed) return SyntaxError("expected a new line after ':'");
|
||||
|
||||
consume(TK("@indent"));
|
||||
while(curr().type != TK("@dedent")) {
|
||||
consume(TK_INDENT);
|
||||
while(curr().type != TK_DEDENT) {
|
||||
match_newlines();
|
||||
check((this->*callback)());
|
||||
match_newlines();
|
||||
}
|
||||
consume(TK("@dedent"));
|
||||
consume(TK_DEDENT);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -599,15 +599,15 @@ Error* Compiler::compile_block_body(PrattCallback callback) noexcept{
|
||||
// import a [as b], c [as d]
|
||||
Error* Compiler::compile_normal_import() noexcept{
|
||||
do {
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
Str name = prev().str();
|
||||
ctx()->emit_(OP_IMPORT_PATH, ctx()->add_const_string(name.sv()), prev().line);
|
||||
if(match(TK("as"))) {
|
||||
consume(TK("@id"));
|
||||
if(match(TK_AS)) {
|
||||
consume(TK_ID);
|
||||
name = prev().str();
|
||||
}
|
||||
ctx()->emit_store_name(name_scope(), StrName(name), prev().line);
|
||||
} while(match(TK(",")));
|
||||
} while(match(TK_COMMA));
|
||||
consume_end_stmt();
|
||||
return NULL;
|
||||
}
|
||||
@ -624,9 +624,9 @@ Error* Compiler::compile_from_import() noexcept{
|
||||
|
||||
while(true) {
|
||||
switch(curr().type) {
|
||||
case TK("."): dots += 1; break;
|
||||
case TK(".."): dots += 2; break;
|
||||
case TK("..."): dots += 3; break;
|
||||
case TK_DOT: dots += 1; break;
|
||||
case TK_DOTDOT: dots += 2; break;
|
||||
case TK_DOTDOTDOT: dots += 3; break;
|
||||
default: goto __EAT_DOTS_END;
|
||||
}
|
||||
advance();
|
||||
@ -638,27 +638,27 @@ __EAT_DOTS_END:
|
||||
|
||||
if(dots > 0) {
|
||||
// @id is optional if dots > 0
|
||||
if(match(TK("@id"))) {
|
||||
if(match(TK_ID)) {
|
||||
ss << prev().sv();
|
||||
while(match(TK("."))) {
|
||||
consume(TK("@id"));
|
||||
while(match(TK_DOT)) {
|
||||
consume(TK_ID);
|
||||
ss << "." << prev().sv();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// @id is required if dots == 0
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
ss << prev().sv();
|
||||
while(match(TK("."))) {
|
||||
consume(TK("@id"));
|
||||
while(match(TK_DOT)) {
|
||||
consume(TK_ID);
|
||||
ss << "." << prev().sv();
|
||||
}
|
||||
}
|
||||
|
||||
ctx()->emit_(OP_IMPORT_PATH, ctx()->add_const_string(ss.str().sv()), prev().line);
|
||||
consume(TK("import"));
|
||||
consume(TK_IMPORT);
|
||||
|
||||
if(match(TK("*"))) {
|
||||
if(match(TK_MUL)) {
|
||||
if(name_scope() != NAME_GLOBAL) return SyntaxError("from <module> import * can only be used in global scope");
|
||||
// pop the module and import __all__
|
||||
ctx()->emit_(OP_POP_IMPORT_STAR, BC_NOARG, prev().line);
|
||||
@ -668,15 +668,15 @@ __EAT_DOTS_END:
|
||||
|
||||
do {
|
||||
ctx()->emit_(OP_DUP_TOP, BC_NOARG, BC_KEEPLINE);
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
Str name = prev().str();
|
||||
ctx()->emit_(OP_LOAD_ATTR, StrName(name).index, prev().line);
|
||||
if(match(TK("as"))) {
|
||||
consume(TK("@id"));
|
||||
if(match(TK_AS)) {
|
||||
consume(TK_ID);
|
||||
name = prev().str();
|
||||
}
|
||||
ctx()->emit_store_name(name_scope(), StrName(name), prev().line);
|
||||
} while(match(TK(",")));
|
||||
} while(match(TK_COMMA));
|
||||
ctx()->emit_(OP_POP_TOP, BC_NOARG, BC_KEEPLINE);
|
||||
consume_end_stmt();
|
||||
return NULL;
|
||||
@ -684,18 +684,18 @@ __EAT_DOTS_END:
|
||||
|
||||
bool Compiler::is_expression(bool allow_slice) noexcept{
|
||||
PrattCallback prefix = rules[curr().type].prefix;
|
||||
return prefix != nullptr && (allow_slice || curr().type != TK(":"));
|
||||
return prefix != nullptr && (allow_slice || curr().type != TK_COLON);
|
||||
}
|
||||
|
||||
Error* Compiler::parse_expression(int precedence, bool allow_slice) noexcept{
|
||||
PrattCallback prefix = rules[curr().type].prefix;
|
||||
if(prefix == nullptr || (curr().type == TK(":") && !allow_slice)) {
|
||||
return SyntaxError("expected an expression, got %s", TK_STR(curr().type));
|
||||
if(prefix == nullptr || (curr().type == TK_COLON && !allow_slice)) {
|
||||
return SyntaxError("expected an expression, got %s", pk_TokenSymbols[curr().type]);
|
||||
}
|
||||
advance();
|
||||
Error* err;
|
||||
check((this->*prefix)());
|
||||
while(rules[curr().type].precedence >= precedence && (allow_slice || curr().type != TK(":"))) {
|
||||
while(rules[curr().type].precedence >= precedence && (allow_slice || curr().type != TK_COLON)) {
|
||||
TokenIndex op = curr().type;
|
||||
advance();
|
||||
PrattCallback infix = rules[op].infix;
|
||||
@ -712,12 +712,12 @@ Error* Compiler::compile_if_stmt() noexcept{
|
||||
int patch = ctx()->emit_(OP_POP_JUMP_IF_FALSE, BC_NOARG, prev().line);
|
||||
err = compile_block_body();
|
||||
if(err) return err;
|
||||
if(match(TK("elif"))) {
|
||||
if(match(TK_ELIF)) {
|
||||
int exit_patch = ctx()->emit_(OP_JUMP_FORWARD, BC_NOARG, prev().line);
|
||||
ctx()->patch_jump(patch);
|
||||
check(compile_if_stmt());
|
||||
ctx()->patch_jump(exit_patch);
|
||||
} else if(match(TK("else"))) {
|
||||
} else if(match(TK_ELSE)) {
|
||||
int exit_patch = ctx()->emit_(OP_JUMP_FORWARD, BC_NOARG, prev().line);
|
||||
ctx()->patch_jump(patch);
|
||||
check(compile_block_body());
|
||||
@ -739,7 +739,7 @@ Error* Compiler::compile_while_loop() noexcept{
|
||||
ctx()->patch_jump(patch);
|
||||
ctx()->exit_block();
|
||||
// optional else clause
|
||||
if(match(TK("else"))) {
|
||||
if(match(TK_ELSE)) {
|
||||
check(compile_block_body());
|
||||
block->end2 = ctx()->co->codes.size();
|
||||
}
|
||||
@ -749,7 +749,7 @@ Error* Compiler::compile_while_loop() noexcept{
|
||||
Error* Compiler::compile_for_loop() noexcept{
|
||||
Error* err;
|
||||
check(EXPR_VARS()); // [vars]
|
||||
consume(TK("in"));
|
||||
consume(TK_IN);
|
||||
check(EXPR_TUPLE()); // [vars, iter]
|
||||
ctx()->s_emit_top(); // [vars]
|
||||
ctx()->emit_(OP_GET_ITER_NEW, BC_NOARG, BC_KEEPLINE);
|
||||
@ -764,7 +764,7 @@ Error* Compiler::compile_for_loop() noexcept{
|
||||
ctx()->emit_(OP_LOOP_CONTINUE, ctx()->get_loop(), BC_KEEPLINE, true);
|
||||
ctx()->exit_block();
|
||||
// optional else clause
|
||||
if(match(TK("else"))) {
|
||||
if(match(TK_ELSE)) {
|
||||
check(compile_block_body());
|
||||
block->end2 = ctx()->co->codes.size();
|
||||
}
|
||||
@ -781,16 +781,16 @@ Error* Compiler::compile_try_except() noexcept{
|
||||
ctx()->exit_block();
|
||||
|
||||
int finally_entry = -1;
|
||||
if(curr().type != TK("finally")) {
|
||||
if(curr().type != TK_FINALLY) {
|
||||
do {
|
||||
StrName as_name;
|
||||
consume(TK("except"));
|
||||
consume(TK_EXCEPT);
|
||||
if(is_expression()) {
|
||||
check(EXPR()); // push assumed type on to the stack
|
||||
ctx()->s_emit_top();
|
||||
ctx()->emit_(OP_EXCEPTION_MATCH, BC_NOARG, prev().line);
|
||||
if(match(TK("as"))) {
|
||||
consume(TK("@id"));
|
||||
if(match(TK_AS)) {
|
||||
consume(TK_ID);
|
||||
as_name = StrName(prev().sv());
|
||||
}
|
||||
} else {
|
||||
@ -807,10 +807,10 @@ Error* Compiler::compile_try_except() noexcept{
|
||||
check(compile_block_body());
|
||||
patches.push_back(ctx()->emit_(OP_JUMP_FORWARD, BC_NOARG, BC_KEEPLINE));
|
||||
ctx()->patch_jump(patch);
|
||||
} while(curr().type == TK("except"));
|
||||
} while(curr().type == TK_EXCEPT);
|
||||
}
|
||||
|
||||
if(match(TK("finally"))) {
|
||||
if(match(TK_FINALLY)) {
|
||||
int patch = ctx()->emit_(OP_JUMP_FORWARD, BC_NOARG, BC_KEEPLINE);
|
||||
finally_entry = ctx()->co->codes.size();
|
||||
check(compile_block_body());
|
||||
@ -848,12 +848,12 @@ Error* Compiler::compile_decorated() noexcept{
|
||||
bool consumed = match_newlines(&need_more_lines);
|
||||
if(need_more_lines) return NeedMoreLines();
|
||||
if(!consumed) return SyntaxError("expected a newline after '@'");
|
||||
} while(match(TK("@")));
|
||||
} while(match(TK_DECORATOR));
|
||||
|
||||
if(match(TK("class"))) {
|
||||
if(match(TK_CLASS)) {
|
||||
check(compile_class(count));
|
||||
} else {
|
||||
consume(TK("def"));
|
||||
consume(TK_DEF);
|
||||
check(compile_function(count));
|
||||
}
|
||||
return NULL;
|
||||
@ -862,17 +862,17 @@ Error* Compiler::compile_decorated() noexcept{
|
||||
Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{
|
||||
Error* err;
|
||||
switch(curr().type) {
|
||||
case TK("+="):
|
||||
case TK("-="):
|
||||
case TK("*="):
|
||||
case TK("/="):
|
||||
case TK("//="):
|
||||
case TK("%="):
|
||||
case TK("<<="):
|
||||
case TK(">>="):
|
||||
case TK("&="):
|
||||
case TK("|="):
|
||||
case TK("^="): {
|
||||
case TK_IADD:
|
||||
case TK_ISUB:
|
||||
case TK_IMUL:
|
||||
case TK_IDIV:
|
||||
case TK_IFLOORDIV:
|
||||
case TK_IMOD:
|
||||
case TK_ILSHIFT:
|
||||
case TK_IRSHIFT:
|
||||
case TK_IAND:
|
||||
case TK_IOR:
|
||||
case TK_IXOR: {
|
||||
if(ctx()->s_top()->is_starred()) return SyntaxError();
|
||||
if(ctx()->is_compiling_class){
|
||||
return SyntaxError("can't use inplace operator in class definition");
|
||||
@ -882,7 +882,7 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{
|
||||
// a.x += 1; a should be evaluated only once
|
||||
// -1 to remove =; inplace=true
|
||||
int line = prev().line;
|
||||
TokenIndex op = prev().type-1;
|
||||
TokenIndex op = (TokenIndex)(prev().type - 1);
|
||||
// [lhs]
|
||||
check(EXPR_TUPLE()); // [lhs, rhs]
|
||||
if(ctx()->s_top()->is_starred()) return SyntaxError();
|
||||
@ -897,9 +897,9 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{
|
||||
*is_assign = true;
|
||||
return NULL;
|
||||
}
|
||||
case TK("="): {
|
||||
case TK_ASSIGN: {
|
||||
int n = 0;
|
||||
while(match(TK("="))) {
|
||||
while(match(TK_ASSIGN)) {
|
||||
check(EXPR_TUPLE());
|
||||
n += 1;
|
||||
}
|
||||
@ -923,7 +923,7 @@ Error* Compiler::try_compile_assignment(bool* is_assign) noexcept{
|
||||
|
||||
Error* Compiler::compile_stmt() noexcept{
|
||||
Error* err;
|
||||
if(match(TK("class"))) {
|
||||
if(match(TK_CLASS)) {
|
||||
check(compile_class());
|
||||
return NULL;
|
||||
}
|
||||
@ -931,24 +931,24 @@ Error* Compiler::compile_stmt() noexcept{
|
||||
int kw_line = prev().line; // backup line number
|
||||
int curr_loop_block = ctx()->get_loop();
|
||||
switch(prev().type) {
|
||||
case TK("break"):
|
||||
case TK_BREAK:
|
||||
if(curr_loop_block < 0) return SyntaxError("'break' outside loop");
|
||||
ctx()->emit_(OP_LOOP_BREAK, curr_loop_block, kw_line);
|
||||
consume_end_stmt();
|
||||
break;
|
||||
case TK("continue"):
|
||||
case TK_CONTINUE:
|
||||
if(curr_loop_block < 0) return SyntaxError("'continue' not properly in loop");
|
||||
ctx()->emit_(OP_LOOP_CONTINUE, curr_loop_block, kw_line);
|
||||
consume_end_stmt();
|
||||
break;
|
||||
case TK("yield"):
|
||||
case TK_YIELD:
|
||||
if(contexts.size() <= 1) return SyntaxError("'yield' outside function");
|
||||
check(EXPR_TUPLE());
|
||||
ctx()->s_emit_top();
|
||||
ctx()->emit_(OP_YIELD_VALUE, BC_NOARG, kw_line);
|
||||
consume_end_stmt();
|
||||
break;
|
||||
case TK("yield from"):
|
||||
case TK_YIELD_FROM:
|
||||
if(contexts.size() <= 1) return SyntaxError("'yield from' outside function");
|
||||
check(EXPR_TUPLE());
|
||||
ctx()->s_emit_top();
|
||||
@ -960,7 +960,7 @@ Error* Compiler::compile_stmt() noexcept{
|
||||
ctx()->exit_block();
|
||||
consume_end_stmt();
|
||||
break;
|
||||
case TK("return"):
|
||||
case TK_RETURN:
|
||||
if(contexts.size() <= 1) return SyntaxError("'return' outside function");
|
||||
if(match_end_stmt()) {
|
||||
ctx()->emit_(OP_RETURN_VALUE, 1, kw_line);
|
||||
@ -972,22 +972,22 @@ Error* Compiler::compile_stmt() noexcept{
|
||||
}
|
||||
break;
|
||||
/*************************************************/
|
||||
case TK("if"): check(compile_if_stmt()); break;
|
||||
case TK("while"): check(compile_while_loop()); break;
|
||||
case TK("for"): check(compile_for_loop()); break;
|
||||
case TK("import"): check(compile_normal_import()); break;
|
||||
case TK("from"): check(compile_from_import()); break;
|
||||
case TK("def"): check(compile_function()); break;
|
||||
case TK("@"): check(compile_decorated()); break;
|
||||
case TK("try"): check(compile_try_except()); break;
|
||||
case TK("pass"): consume_end_stmt(); break;
|
||||
case TK_IF: check(compile_if_stmt()); break;
|
||||
case TK_WHILE: check(compile_while_loop()); break;
|
||||
case TK_FOR: check(compile_for_loop()); break;
|
||||
case TK_IMPORT: check(compile_normal_import()); break;
|
||||
case TK_FROM: check(compile_from_import()); break;
|
||||
case TK_DEF: check(compile_function()); break;
|
||||
case TK_DECORATOR: check(compile_decorated()); break;
|
||||
case TK_TRY: check(compile_try_except()); break;
|
||||
case TK_PASS: consume_end_stmt(); break;
|
||||
/*************************************************/
|
||||
case TK("assert"): {
|
||||
case TK_ASSERT: {
|
||||
check(EXPR()); // condition
|
||||
ctx()->s_emit_top();
|
||||
int index = ctx()->emit_(OP_POP_JUMP_IF_TRUE, BC_NOARG, kw_line);
|
||||
int has_msg = 0;
|
||||
if(match(TK(","))) {
|
||||
if(match(TK_COMMA)) {
|
||||
check(EXPR()); // message
|
||||
ctx()->s_emit_top();
|
||||
has_msg = 1;
|
||||
@ -997,32 +997,32 @@ Error* Compiler::compile_stmt() noexcept{
|
||||
consume_end_stmt();
|
||||
break;
|
||||
}
|
||||
case TK("global"):
|
||||
case TK_GLOBAL:
|
||||
do {
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
ctx()->global_names.push_back(StrName(prev().sv()));
|
||||
} while(match(TK(",")));
|
||||
} while(match(TK_COMMA));
|
||||
consume_end_stmt();
|
||||
break;
|
||||
case TK("raise"): {
|
||||
case TK_RAISE: {
|
||||
check(EXPR());
|
||||
ctx()->s_emit_top();
|
||||
ctx()->emit_(OP_RAISE, BC_NOARG, kw_line);
|
||||
consume_end_stmt();
|
||||
} break;
|
||||
case TK("del"): {
|
||||
case TK_DEL: {
|
||||
check(EXPR_TUPLE());
|
||||
if(!ctx()->s_top()->emit_del(ctx())) return SyntaxError();
|
||||
ctx()->s_pop();
|
||||
consume_end_stmt();
|
||||
} break;
|
||||
case TK("with"): {
|
||||
case TK_WITH: {
|
||||
check(EXPR()); // [ <expr> ]
|
||||
ctx()->s_emit_top();
|
||||
ctx()->enter_block(CodeBlockType::CONTEXT_MANAGER);
|
||||
Expr* as_name = nullptr;
|
||||
if(match(TK("as"))) {
|
||||
consume(TK("@id"));
|
||||
if(match(TK_AS)) {
|
||||
consume(TK_ID);
|
||||
as_name = make_expr<NameExpr>(prev().str(), name_scope());
|
||||
}
|
||||
ctx()->emit_(OP_WITH_ENTER, BC_NOARG, prev().line);
|
||||
@ -1039,18 +1039,18 @@ Error* Compiler::compile_stmt() noexcept{
|
||||
ctx()->exit_block();
|
||||
} break;
|
||||
/*************************************************/
|
||||
case TK("=="): {
|
||||
consume(TK("@id"));
|
||||
case TK_EQ: {
|
||||
consume(TK_ID);
|
||||
if(mode() != EXEC_MODE) return SyntaxError("'label' is only available in EXEC_MODE");
|
||||
if(!ctx()->add_label(prev().str())) {
|
||||
Str escaped(prev().str().escape());
|
||||
return SyntaxError("label %s already exists", escaped.c_str());
|
||||
}
|
||||
consume(TK("=="));
|
||||
consume(TK_EQ);
|
||||
consume_end_stmt();
|
||||
} break;
|
||||
case TK("->"):
|
||||
consume(TK("@id"));
|
||||
case TK_ARROW:
|
||||
consume(TK_ID);
|
||||
if(mode() != EXEC_MODE) return SyntaxError("'goto' is only available in EXEC_MODE");
|
||||
ctx()->emit_(OP_GOTO, StrName(prev().sv()).index, prev().line);
|
||||
consume_end_stmt();
|
||||
@ -1064,7 +1064,7 @@ Error* Compiler::compile_stmt() noexcept{
|
||||
bool is_typed_name = false; // e.g. x: int
|
||||
// eat variable's type hint if it is a single name
|
||||
if(ctx()->s_top()->is_name()) {
|
||||
if(match(TK(":"))) {
|
||||
if(match(TK_COLON)) {
|
||||
check(consume_type_hints());
|
||||
is_typed_name = true;
|
||||
|
||||
@ -1107,15 +1107,15 @@ Error* Compiler::consume_type_hints() noexcept{
|
||||
|
||||
Error* Compiler::compile_class(int decorators) noexcept{
|
||||
Error* err;
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
int namei = StrName(prev().sv()).index;
|
||||
bool has_base = false;
|
||||
if(match(TK("("))) {
|
||||
if(match(TK_LPAREN)) {
|
||||
if(is_expression()) {
|
||||
check(EXPR());
|
||||
has_base = true; // [base]
|
||||
}
|
||||
consume(TK(")"));
|
||||
consume(TK_RPAREN);
|
||||
}
|
||||
if(!has_base) {
|
||||
ctx()->emit_(OP_LOAD_NONE, BC_NOARG, prev().line);
|
||||
@ -1148,15 +1148,15 @@ Error* Compiler::_compile_f_args(FuncDecl_ decl, bool enable_type_hints) noexcep
|
||||
if(state > 3) return SyntaxError();
|
||||
if(state == 3) return SyntaxError("**kwargs should be the last argument");
|
||||
match_newlines();
|
||||
if(match(TK("*"))) {
|
||||
if(match(TK_MUL)) {
|
||||
if(state < 1)
|
||||
state = 1;
|
||||
else
|
||||
return SyntaxError("*args should be placed before **kwargs");
|
||||
} else if(match(TK("**"))) {
|
||||
} else if(match(TK_POW)) {
|
||||
state = 3;
|
||||
}
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
StrName name(prev().sv());
|
||||
|
||||
// check duplicate argument name
|
||||
@ -1174,8 +1174,8 @@ Error* Compiler::_compile_f_args(FuncDecl_ decl, bool enable_type_hints) noexcep
|
||||
}
|
||||
|
||||
// eat type hints
|
||||
if(enable_type_hints && match(TK(":"))) check(consume_type_hints());
|
||||
if(state == 0 && curr().type == TK("=")) state = 2;
|
||||
if(enable_type_hints && match(TK_COLON)) check(consume_type_hints());
|
||||
if(state == 0 && curr().type == TK_ASSIGN) state = 2;
|
||||
int index = ctx()->add_varname(name);
|
||||
switch(state) {
|
||||
case 0: decl->args.push_back(index); break;
|
||||
@ -1184,7 +1184,7 @@ Error* Compiler::_compile_f_args(FuncDecl_ decl, bool enable_type_hints) noexcep
|
||||
state += 1;
|
||||
break;
|
||||
case 2: {
|
||||
consume(TK("="));
|
||||
consume(TK_ASSIGN);
|
||||
PyVar value;
|
||||
check(read_literal(&value));
|
||||
if(value == nullptr) return SyntaxError("default argument must be a literal");
|
||||
@ -1195,21 +1195,21 @@ Error* Compiler::_compile_f_args(FuncDecl_ decl, bool enable_type_hints) noexcep
|
||||
state += 1;
|
||||
break;
|
||||
}
|
||||
} while(match(TK(",")));
|
||||
} while(match(TK_COMMA));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Error* Compiler::compile_function(int decorators) noexcept{
|
||||
Error* err;
|
||||
consume(TK("@id"));
|
||||
consume(TK_ID);
|
||||
Str decl_name = prev().str();
|
||||
FuncDecl_ decl = push_f_context(decl_name);
|
||||
consume(TK("("));
|
||||
if(!match(TK(")"))) {
|
||||
consume(TK_LPAREN);
|
||||
if(!match(TK_RPAREN)) {
|
||||
check(_compile_f_args(decl, true));
|
||||
consume(TK(")"));
|
||||
consume(TK_RPAREN);
|
||||
}
|
||||
if(match(TK("->"))) check(consume_type_hints());
|
||||
if(match(TK_ARROW)) check(consume_type_hints());
|
||||
check(compile_block_body());
|
||||
check(pop_context());
|
||||
|
||||
@ -1251,29 +1251,29 @@ Error* Compiler::read_literal(PyVar* out) noexcept{
|
||||
Error* err;
|
||||
advance();
|
||||
switch(prev().type) {
|
||||
case TK("-"): {
|
||||
consume(TK("@num"));
|
||||
case TK_SUB: {
|
||||
consume(TK_NUM);
|
||||
PyVar val = to_object(prev().value);
|
||||
*out = vm->py_negate(val);
|
||||
return NULL;
|
||||
}
|
||||
case TK("@num"): *out = to_object(prev().value); return NULL;
|
||||
case TK("@str"): *out = to_object(prev().value); return NULL;
|
||||
case TK("True"): *out = VAR(true); return NULL;
|
||||
case TK("False"): *out = VAR(false); return NULL;
|
||||
case TK("None"): *out = vm->None; return NULL;
|
||||
case TK("..."): *out = vm->Ellipsis; return NULL;
|
||||
case TK("("): {
|
||||
case TK_NUM: *out = to_object(prev().value); return NULL;
|
||||
case TK_STR: *out = to_object(prev().value); return NULL;
|
||||
case TK_TRUE: *out = VAR(true); return NULL;
|
||||
case TK_FALSE: *out = VAR(false); return NULL;
|
||||
case TK_NONE: *out = vm->None; return NULL;
|
||||
case TK_DOTDOTDOT: *out = vm->Ellipsis; return NULL;
|
||||
case TK_LPAREN: {
|
||||
List cpnts;
|
||||
while(true) {
|
||||
PyVar elem;
|
||||
check(read_literal(&elem));
|
||||
cpnts.push_back(elem);
|
||||
if(curr().type == TK(")")) break;
|
||||
consume(TK(","));
|
||||
if(curr().type == TK(")")) break;
|
||||
if(curr().type == TK_RPAREN) break;
|
||||
consume(TK_COMMA);
|
||||
if(curr().type == TK_RPAREN) break;
|
||||
}
|
||||
consume(TK(")"));
|
||||
consume(TK_RPAREN);
|
||||
*out = VAR(cpnts.to_tuple());
|
||||
return NULL;
|
||||
}
|
||||
@ -1297,20 +1297,20 @@ Error* Compiler::compile(CodeObject_* out) noexcept{
|
||||
// if(lexer.src.filename()[0] != '<'){
|
||||
// printf("%s\n", lexer.src.filename().c_str());
|
||||
// for(int i=0; i<lexer.nexts.size(); i++){
|
||||
// printf("%s: %s\n", TK_STR(tk(i).type), tk(i).str().escape().c_str());
|
||||
// printf("%s: %s\n", pk_TokenSymbols[tk(i).type], tk(i).str().escape().c_str());
|
||||
// }
|
||||
// }
|
||||
|
||||
CodeObject_ code = push_global_context();
|
||||
|
||||
assert(curr().type == TK("@sof"));
|
||||
assert(curr().type == TK_SOF);
|
||||
advance(); // skip @sof, so prev() is always valid
|
||||
match_newlines(); // skip possible leading '\n'
|
||||
|
||||
if(mode() == EVAL_MODE) {
|
||||
check(EXPR_TUPLE());
|
||||
ctx()->s_emit_top();
|
||||
consume(TK("@eof"));
|
||||
consume(TK_EOF);
|
||||
ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE);
|
||||
check(pop_context());
|
||||
*out = code;
|
||||
@ -1319,7 +1319,7 @@ Error* Compiler::compile(CodeObject_* out) noexcept{
|
||||
check(EXPR());
|
||||
Expr* e = ctx()->s_popx();
|
||||
if(!e->is_json_object()) return SyntaxError("expect a JSON object, literal or array");
|
||||
consume(TK("@eof"));
|
||||
consume(TK_EOF);
|
||||
e->emit_(ctx());
|
||||
ctx()->emit_(OP_RETURN_VALUE, BC_NOARG, BC_KEEPLINE);
|
||||
check(pop_context());
|
||||
@ -1327,7 +1327,7 @@ Error* Compiler::compile(CodeObject_* out) noexcept{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
while(!match(TK("@eof"))) {
|
||||
while(!match(TK_EOF)) {
|
||||
check(compile_stmt());
|
||||
match_newlines();
|
||||
}
|
||||
|
||||
@ -231,10 +231,10 @@ void OrExpr::emit_(CodeEmitContext* ctx) {
|
||||
|
||||
void Literal0Expr::emit_(CodeEmitContext* ctx) {
|
||||
switch(token) {
|
||||
case TK("None"): ctx->emit_(OP_LOAD_NONE, BC_NOARG, line); break;
|
||||
case TK("True"): ctx->emit_(OP_LOAD_TRUE, BC_NOARG, line); break;
|
||||
case TK("False"): ctx->emit_(OP_LOAD_FALSE, BC_NOARG, line); break;
|
||||
case TK("..."): ctx->emit_(OP_LOAD_ELLIPSIS, BC_NOARG, line); break;
|
||||
case TK_NONE: ctx->emit_(OP_LOAD_NONE, BC_NOARG, line); break;
|
||||
case TK_TRUE: ctx->emit_(OP_LOAD_TRUE, BC_NOARG, line); break;
|
||||
case TK_FALSE: ctx->emit_(OP_LOAD_FALSE, BC_NOARG, line); break;
|
||||
case TK_DOTDOTDOT: ctx->emit_(OP_LOAD_ELLIPSIS, BC_NOARG, line); break;
|
||||
default: assert(false);
|
||||
}
|
||||
}
|
||||
@ -681,12 +681,12 @@ void CallExpr::emit_(CodeEmitContext* ctx) {
|
||||
|
||||
bool BinaryExpr::is_compare() const {
|
||||
switch(op) {
|
||||
case TK("<"):
|
||||
case TK("<="):
|
||||
case TK("=="):
|
||||
case TK("!="):
|
||||
case TK(">"):
|
||||
case TK(">="): return true;
|
||||
case TK_LT:
|
||||
case TK_LE:
|
||||
case TK_EQ:
|
||||
case TK_NE:
|
||||
case TK_GT:
|
||||
case TK_GE: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
@ -701,12 +701,12 @@ void BinaryExpr::_emit_compare(CodeEmitContext* ctx, small_vector_2<int, 8>& jmp
|
||||
ctx->emit_(OP_DUP_TOP, BC_NOARG, line); // [a, b, b]
|
||||
ctx->emit_(OP_ROT_THREE, BC_NOARG, line); // [b, a, b]
|
||||
switch(op) {
|
||||
case TK("<"): ctx->emit_(OP_COMPARE_LT, BC_NOARG, line); break;
|
||||
case TK("<="): ctx->emit_(OP_COMPARE_LE, BC_NOARG, line); break;
|
||||
case TK("=="): ctx->emit_(OP_COMPARE_EQ, BC_NOARG, line); break;
|
||||
case TK("!="): ctx->emit_(OP_COMPARE_NE, BC_NOARG, line); break;
|
||||
case TK(">"): ctx->emit_(OP_COMPARE_GT, BC_NOARG, line); break;
|
||||
case TK(">="): ctx->emit_(OP_COMPARE_GE, BC_NOARG, line); break;
|
||||
case TK_LT: ctx->emit_(OP_COMPARE_LT, BC_NOARG, line); break;
|
||||
case TK_LE: ctx->emit_(OP_COMPARE_LE, BC_NOARG, line); break;
|
||||
case TK_EQ: ctx->emit_(OP_COMPARE_EQ, BC_NOARG, line); break;
|
||||
case TK_NE: ctx->emit_(OP_COMPARE_NE, BC_NOARG, line); break;
|
||||
case TK_GT: ctx->emit_(OP_COMPARE_GT, BC_NOARG, line); break;
|
||||
case TK_GE: ctx->emit_(OP_COMPARE_GE, BC_NOARG, line); break;
|
||||
default: PK_UNREACHABLE()
|
||||
}
|
||||
// [b, RES]
|
||||
@ -731,34 +731,34 @@ void BinaryExpr::emit_(CodeEmitContext* ctx) {
|
||||
|
||||
rhs->emit_(ctx);
|
||||
switch(op) {
|
||||
case TK("+"): ctx->emit_(OP_BINARY_ADD, BC_NOARG, line); break;
|
||||
case TK("-"): ctx->emit_(OP_BINARY_SUB, BC_NOARG, line); break;
|
||||
case TK("*"): ctx->emit_(OP_BINARY_MUL, BC_NOARG, line); break;
|
||||
case TK("/"): ctx->emit_(OP_BINARY_TRUEDIV, BC_NOARG, line); break;
|
||||
case TK("//"): ctx->emit_(OP_BINARY_FLOORDIV, BC_NOARG, line); break;
|
||||
case TK("%"): ctx->emit_(OP_BINARY_MOD, BC_NOARG, line); break;
|
||||
case TK("**"): ctx->emit_(OP_BINARY_POW, BC_NOARG, line); break;
|
||||
case TK_ADD: ctx->emit_(OP_BINARY_ADD, BC_NOARG, line); break;
|
||||
case TK_SUB: ctx->emit_(OP_BINARY_SUB, BC_NOARG, line); break;
|
||||
case TK_MUL: ctx->emit_(OP_BINARY_MUL, BC_NOARG, line); break;
|
||||
case TK_DIV: ctx->emit_(OP_BINARY_TRUEDIV, BC_NOARG, line); break;
|
||||
case TK_FLOORDIV: ctx->emit_(OP_BINARY_FLOORDIV, BC_NOARG, line); break;
|
||||
case TK_MOD: ctx->emit_(OP_BINARY_MOD, BC_NOARG, line); break;
|
||||
case TK_POW: ctx->emit_(OP_BINARY_POW, BC_NOARG, line); break;
|
||||
|
||||
case TK("<"): ctx->emit_(OP_COMPARE_LT, BC_NOARG, line); break;
|
||||
case TK("<="): ctx->emit_(OP_COMPARE_LE, BC_NOARG, line); break;
|
||||
case TK("=="): ctx->emit_(OP_COMPARE_EQ, BC_NOARG, line); break;
|
||||
case TK("!="): ctx->emit_(OP_COMPARE_NE, BC_NOARG, line); break;
|
||||
case TK(">"): ctx->emit_(OP_COMPARE_GT, BC_NOARG, line); break;
|
||||
case TK(">="): ctx->emit_(OP_COMPARE_GE, BC_NOARG, line); break;
|
||||
case TK_LT: ctx->emit_(OP_COMPARE_LT, BC_NOARG, line); break;
|
||||
case TK_LE: ctx->emit_(OP_COMPARE_LE, BC_NOARG, line); break;
|
||||
case TK_EQ: ctx->emit_(OP_COMPARE_EQ, BC_NOARG, line); break;
|
||||
case TK_NE: ctx->emit_(OP_COMPARE_NE, BC_NOARG, line); break;
|
||||
case TK_GT: ctx->emit_(OP_COMPARE_GT, BC_NOARG, line); break;
|
||||
case TK_GE: ctx->emit_(OP_COMPARE_GE, BC_NOARG, line); break;
|
||||
|
||||
case TK("in"): ctx->emit_(OP_CONTAINS_OP, 0, line); break;
|
||||
case TK("not in"): ctx->emit_(OP_CONTAINS_OP, 1, line); break;
|
||||
case TK("is"): ctx->emit_(OP_IS_OP, BC_NOARG, line); break;
|
||||
case TK("is not"): ctx->emit_(OP_IS_NOT_OP, BC_NOARG, line); break;
|
||||
case TK_IN: ctx->emit_(OP_CONTAINS_OP, 0, line); break;
|
||||
case TK_NOT_IN: ctx->emit_(OP_CONTAINS_OP, 1, line); break;
|
||||
case TK_IS: ctx->emit_(OP_IS_OP, BC_NOARG, line); break;
|
||||
case TK_IS_NOT: ctx->emit_(OP_IS_NOT_OP, BC_NOARG, line); break;
|
||||
|
||||
case TK("<<"): ctx->emit_(OP_BITWISE_LSHIFT, BC_NOARG, line); break;
|
||||
case TK(">>"): ctx->emit_(OP_BITWISE_RSHIFT, BC_NOARG, line); break;
|
||||
case TK("&"): ctx->emit_(OP_BITWISE_AND, BC_NOARG, line); break;
|
||||
case TK("|"): ctx->emit_(OP_BITWISE_OR, BC_NOARG, line); break;
|
||||
case TK("^"): ctx->emit_(OP_BITWISE_XOR, BC_NOARG, line); break;
|
||||
case TK_LSHIFT: ctx->emit_(OP_BITWISE_LSHIFT, BC_NOARG, line); break;
|
||||
case TK_RSHIFT: ctx->emit_(OP_BITWISE_RSHIFT, BC_NOARG, line); break;
|
||||
case TK_AND: ctx->emit_(OP_BITWISE_AND, BC_NOARG, line); break;
|
||||
case TK_OR: ctx->emit_(OP_BITWISE_OR, BC_NOARG, line); break;
|
||||
case TK_XOR: ctx->emit_(OP_BITWISE_XOR, BC_NOARG, line); break;
|
||||
|
||||
case TK("@"): ctx->emit_(OP_BINARY_MATMUL, BC_NOARG, line); break;
|
||||
default: PK_FATAL_ERROR("unknown binary operator: %s\n", TK_STR(op));
|
||||
case TK_DECORATOR: ctx->emit_(OP_BINARY_MATMUL, BC_NOARG, line); break;
|
||||
default: PK_FATAL_ERROR("unknown binary operator: %s\n", pk_TokenSymbols[op]);
|
||||
}
|
||||
|
||||
for(int i: jmps)
|
||||
|
||||
@ -3,12 +3,36 @@
|
||||
#include "pocketpy/common/smallmap.h"
|
||||
#include "pocketpy/compiler/lexer.h"
|
||||
|
||||
void pkpy_TokenDeserializer__ctor(pkpy_TokenDeserializer* self, const char* source){
|
||||
const char* pk_TokenSymbols[] = {
|
||||
"@eof", "@eol", "@sof",
|
||||
"@id", "@num", "@str", "@fstr", "@long", "@bytes", "@imag",
|
||||
"@indent", "@dedent",
|
||||
// These 3 are compound keywords which are generated on the fly
|
||||
"is not", "not in", "yield from",
|
||||
/*****************************************/
|
||||
"+", "+=", "-", "-=", // (INPLACE_OP - 1) can get '=' removed
|
||||
"*", "*=", "/", "/=", "//", "//=", "%", "%=",
|
||||
"&", "&=", "|", "|=", "^", "^=",
|
||||
"<<", "<<=", ">>", ">>=",
|
||||
/*****************************************/
|
||||
"(", ")", "[", "]", "{", "}",
|
||||
".", "..", "...", ",", ":", ";",
|
||||
"**", "->", "#", "@",
|
||||
">", "<", "=", "==", "!=", ">=", "<=", "~",
|
||||
/** KW_BEGIN **/
|
||||
// NOTE: These keywords should be sorted in ascending order!!
|
||||
"False", "None", "True", "and", "as", "assert", "break", "class", "continue",
|
||||
"def", "del", "elif", "else", "except", "finally", "for", "from", "global",
|
||||
"if", "import", "in", "is", "lambda", "not", "or", "pass", "raise", "return",
|
||||
"try", "while", "with", "yield",
|
||||
};
|
||||
|
||||
void pk_TokenDeserializer__ctor(pk_TokenDeserializer* self, const char* source){
|
||||
self->curr = source;
|
||||
self->source = source;
|
||||
}
|
||||
|
||||
bool pkpy_TokenDeserializer__match_char(pkpy_TokenDeserializer* self, char c){
|
||||
bool pk_TokenDeserializer__match_char(pk_TokenDeserializer* self, char c){
|
||||
if(*self->curr == c) {
|
||||
self->curr++;
|
||||
return true;
|
||||
@ -16,17 +40,17 @@ bool pkpy_TokenDeserializer__match_char(pkpy_TokenDeserializer* self, char c){
|
||||
return false;
|
||||
}
|
||||
|
||||
c11_string pkpy_TokenDeserializer__read_string(pkpy_TokenDeserializer* self, char c){
|
||||
c11_string pk_TokenDeserializer__read_string(pk_TokenDeserializer* self, char c){
|
||||
const char* start = self->curr;
|
||||
while(*self->curr != c)
|
||||
self->curr++;
|
||||
c11_string retval = {start, self->curr - start};
|
||||
c11_string retval = {start, (int)(self->curr-start)};
|
||||
self->curr++; // skip the delimiter
|
||||
return retval;
|
||||
}
|
||||
|
||||
pkpy_Str pkpy_TokenDeserializer__read_string_from_hex(pkpy_TokenDeserializer* self, char c){
|
||||
c11_string sv = pkpy_TokenDeserializer__read_string(self, c);
|
||||
pkpy_Str pk_TokenDeserializer__read_string_from_hex(pk_TokenDeserializer* self, char c){
|
||||
c11_string sv = pk_TokenDeserializer__read_string(self, c);
|
||||
const char* s = sv.data;
|
||||
char* buffer = (char*)malloc(sv.size / 2 + 1);
|
||||
for(int i = 0; i < sv.size; i += 2) {
|
||||
@ -55,13 +79,13 @@ pkpy_Str pkpy_TokenDeserializer__read_string_from_hex(pkpy_TokenDeserializer* se
|
||||
};
|
||||
}
|
||||
|
||||
int pkpy_TokenDeserializer__read_count(pkpy_TokenDeserializer* self){
|
||||
int pk_TokenDeserializer__read_count(pk_TokenDeserializer* self){
|
||||
assert(*self->curr == '=');
|
||||
self->curr++;
|
||||
return pkpy_TokenDeserializer__read_uint(self, '\n');
|
||||
return pk_TokenDeserializer__read_uint(self, '\n');
|
||||
}
|
||||
|
||||
int64_t pkpy_TokenDeserializer__read_uint(pkpy_TokenDeserializer* self, char c){
|
||||
int64_t pk_TokenDeserializer__read_uint(pk_TokenDeserializer* self, char c){
|
||||
int64_t out = 0;
|
||||
while(*self->curr != c) {
|
||||
out = out * 10 + (*self->curr - '0');
|
||||
@ -71,8 +95,8 @@ int64_t pkpy_TokenDeserializer__read_uint(pkpy_TokenDeserializer* self, char c){
|
||||
return out;
|
||||
}
|
||||
|
||||
double pkpy_TokenDeserializer__read_float(pkpy_TokenDeserializer* self, char c){
|
||||
c11_string sv = pkpy_TokenDeserializer__read_string(self, c);
|
||||
double pk_TokenDeserializer__read_float(pk_TokenDeserializer* self, char c){
|
||||
c11_string sv = pk_TokenDeserializer__read_string(self, c);
|
||||
pkpy_Str nullterm;
|
||||
pkpy_Str__ctor2(&nullterm, sv.data, sv.size);
|
||||
char* end;
|
||||
|
||||
@ -62,11 +62,11 @@ bool Lexer::eat_indentation() noexcept{
|
||||
// https://docs.python.org/3/reference/lexical_analysis.html#indentation
|
||||
if(spaces > indents.back()) {
|
||||
indents.push_back(spaces);
|
||||
nexts.push_back(Token{TK("@indent"), token_start, 0, current_line, brackets_level, {}});
|
||||
nexts.push_back(Token{TK_INDENT, token_start, 0, current_line, brackets_level, {}});
|
||||
} else if(spaces < indents.back()) {
|
||||
while(spaces < indents.back()) {
|
||||
indents.pop_back();
|
||||
nexts.push_back(Token{TK("@dedent"), token_start, 0, current_line, brackets_level, {}});
|
||||
nexts.push_back(Token{TK_DEDENT, token_start, 0, current_line, brackets_level, {}});
|
||||
}
|
||||
if(spaces != indents.back()) { return false; }
|
||||
}
|
||||
@ -129,29 +129,32 @@ Error* Lexer::eat_name() noexcept{
|
||||
|
||||
int length = (int)(curr_char - token_start);
|
||||
if(length == 0) return SyntaxError("@id contains invalid char");
|
||||
std::string_view name(token_start, length);
|
||||
c11_string name = {token_start, length};
|
||||
|
||||
if(src->mode == JSON_MODE) {
|
||||
if(name == "true") {
|
||||
add_token(TK("True"));
|
||||
} else if(name == "false") {
|
||||
add_token(TK("False"));
|
||||
} else if(name == "null") {
|
||||
add_token(TK("None"));
|
||||
if(c11_string__cmp3(name, "true") == 0) {
|
||||
add_token(TK_TRUE);
|
||||
} else if(c11_string__cmp3(name, "false") == 0) {
|
||||
add_token(TK_FALSE);
|
||||
} else if(c11_string__cmp3(name, "null") == 0) {
|
||||
add_token(TK_NONE);
|
||||
} else {
|
||||
return SyntaxError("invalid JSON token");
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const auto KW_BEGIN = kTokens + TK("False");
|
||||
const auto KW_END = kTokens + kTokenCount;
|
||||
const char** KW_BEGIN = pk_TokenSymbols + TK_FALSE;
|
||||
int KW_COUNT = TK__COUNT__ - TK_FALSE;
|
||||
#define less(a, b) (c11_string__cmp3(b, a) > 0)
|
||||
int out;
|
||||
c11__lower_bound(const char*, KW_BEGIN, KW_COUNT, name, less, &out);
|
||||
#undef less
|
||||
|
||||
auto it = lower_bound(KW_BEGIN, KW_END, name);
|
||||
if(it != KW_END && *it == name) {
|
||||
add_token(it - kTokens);
|
||||
if(out != KW_COUNT && c11_string__cmp3(name, KW_BEGIN[out]) == 0) {
|
||||
add_token((TokenIndex)(out + TK_FALSE));
|
||||
} else {
|
||||
add_token(TK("@id"));
|
||||
add_token(TK_ID);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
@ -172,32 +175,33 @@ bool Lexer::matchchar(char c) noexcept{
|
||||
|
||||
void Lexer::add_token(TokenIndex type, TokenValue value) noexcept{
|
||||
switch(type) {
|
||||
case TK("{"):
|
||||
case TK("["):
|
||||
case TK("("): brackets_level++; break;
|
||||
case TK(")"):
|
||||
case TK("]"):
|
||||
case TK("}"): brackets_level--; break;
|
||||
case TK_LBRACE:
|
||||
case TK_LBRACKET:
|
||||
case TK_LPAREN: brackets_level++; break;
|
||||
case TK_RPAREN:
|
||||
case TK_RBRACKET:
|
||||
case TK_RBRACE: brackets_level--; break;
|
||||
default: break;
|
||||
}
|
||||
auto token = Token{type,
|
||||
token_start,
|
||||
(int)(curr_char - token_start),
|
||||
current_line - ((type == TK("@eol")) ? 1 : 0),
|
||||
current_line - ((type == TK_EOL) ? 1 : 0),
|
||||
brackets_level,
|
||||
value};
|
||||
// handle "not in", "is not", "yield from"
|
||||
if(!nexts.empty()) {
|
||||
auto& back = nexts.back();
|
||||
if(back.type == TK("not") && type == TK("in")) {
|
||||
back.type = TK("not in");
|
||||
if(back.type == TK_NOT_KW && type == TK_IN) {
|
||||
back.type = TK_NOT_IN;
|
||||
return;
|
||||
}
|
||||
if(back.type == TK("is") && type == TK("not")) {
|
||||
back.type = TK("is not");
|
||||
if(back.type == TK_IS && type == TK_NOT_KW) {
|
||||
back.type = TK_IS_NOT;
|
||||
return;
|
||||
}
|
||||
if(back.type == TK("yield") && type == TK("from")) {
|
||||
back.type = TK("yield from");
|
||||
if(back.type == TK_YIELD && type == TK_FROM) {
|
||||
back.type = TK_YIELD_FROM;
|
||||
return;
|
||||
}
|
||||
nexts.push_back(token);
|
||||
@ -271,11 +275,11 @@ Error* Lexer::eat_string(char quote, StringType type) noexcept{
|
||||
Error* err = eat_string_until(quote, type == StringType::RAW_STRING, &s);
|
||||
if(err) return err;
|
||||
if(type == StringType::F_STRING) {
|
||||
add_token(TK("@fstr"), s);
|
||||
add_token(TK_FSTR, s);
|
||||
}else if(type == StringType::NORMAL_BYTES) {
|
||||
add_token(TK("@bytes"), s);
|
||||
add_token(TK_BYTES, s);
|
||||
}else{
|
||||
add_token(TK("@str"), s);
|
||||
add_token(TK_STR, s);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
@ -299,13 +303,13 @@ Error* Lexer::eat_number() noexcept{
|
||||
if(text[0] != '.' && !is_scientific_notation) {
|
||||
// try long
|
||||
if(i[-1] == 'L') {
|
||||
add_token(TK("@long"));
|
||||
add_token(TK_LONG);
|
||||
return NULL;
|
||||
}
|
||||
// try integer
|
||||
i64 int_out;
|
||||
switch(parse_uint(text, &int_out, -1)) {
|
||||
case IntParsingResult::Success: add_token(TK("@num"), int_out); return NULL;
|
||||
case IntParsingResult::Success: add_token(TK_NUM, int_out); return NULL;
|
||||
case IntParsingResult::Overflow: return SyntaxError("int literal is too large");
|
||||
case IntParsingResult::Failure: break; // do nothing
|
||||
}
|
||||
@ -321,12 +325,12 @@ Error* Lexer::eat_number() noexcept{
|
||||
}
|
||||
|
||||
if(p_end == text.data() + text.size()) {
|
||||
add_token(TK("@num"), (f64)float_out);
|
||||
add_token(TK_NUM, (f64)float_out);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if(i[-1] == 'j' && p_end == text.data() + text.size() - 1) {
|
||||
add_token(TK("@imag"), (f64)float_out);
|
||||
add_token(TK_IMAG, (f64)float_out);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -346,17 +350,17 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
|
||||
return NULL;
|
||||
}
|
||||
case '#': skip_line_comment(); break;
|
||||
case '~': add_token(TK("~")); return NULL;
|
||||
case '{': add_token(TK("{")); return NULL;
|
||||
case '}': add_token(TK("}")); return NULL;
|
||||
case ',': add_token(TK(",")); return NULL;
|
||||
case ':': add_token(TK(":")); return NULL;
|
||||
case ';': add_token(TK(";")); return NULL;
|
||||
case '(': add_token(TK("(")); return NULL;
|
||||
case ')': add_token(TK(")")); return NULL;
|
||||
case '[': add_token(TK("[")); return NULL;
|
||||
case ']': add_token(TK("]")); return NULL;
|
||||
case '@': add_token(TK("@")); return NULL;
|
||||
case '~': add_token(TK_INVERT); return NULL;
|
||||
case '{': add_token(TK_LBRACE); return NULL;
|
||||
case '}': add_token(TK_RBRACE); return NULL;
|
||||
case ',': add_token(TK_COMMA); return NULL;
|
||||
case ':': add_token(TK_COLON); return NULL;
|
||||
case ';': add_token(TK_SEMICOLON); return NULL;
|
||||
case '(': add_token(TK_LPAREN); return NULL;
|
||||
case ')': add_token(TK_RPAREN); return NULL;
|
||||
case '[': add_token(TK_LBRACKET); return NULL;
|
||||
case ']': add_token(TK_RBRACKET); return NULL;
|
||||
case '@': add_token(TK_DECORATOR); return NULL;
|
||||
case '\\': {
|
||||
// line continuation character
|
||||
char c = eatchar_include_newline();
|
||||
@ -367,16 +371,16 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
|
||||
eat_spaces();
|
||||
return NULL;
|
||||
}
|
||||
case '%': add_token_2('=', TK("%"), TK("%=")); return NULL;
|
||||
case '&': add_token_2('=', TK("&"), TK("&=")); return NULL;
|
||||
case '|': add_token_2('=', TK("|"), TK("|=")); return NULL;
|
||||
case '^': add_token_2('=', TK("^"), TK("^=")); return NULL;
|
||||
case '%': add_token_2('=', TK_MOD, TK_IMOD); return NULL;
|
||||
case '&': add_token_2('=', TK_AND, TK_IAND); return NULL;
|
||||
case '|': add_token_2('=', TK_OR, TK_IOR); return NULL;
|
||||
case '^': add_token_2('=', TK_XOR, TK_IXOR); return NULL;
|
||||
case '.': {
|
||||
if(matchchar('.')) {
|
||||
if(matchchar('.')) {
|
||||
add_token(TK("..."));
|
||||
add_token(TK_DOTDOTDOT);
|
||||
} else {
|
||||
add_token(TK(".."));
|
||||
add_token(TK_DOTDOT);
|
||||
}
|
||||
} else {
|
||||
char next_char = peekchar();
|
||||
@ -384,53 +388,43 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
|
||||
Error* err = eat_number();
|
||||
if(err) return err;
|
||||
} else {
|
||||
add_token(TK("."));
|
||||
add_token(TK_DOT);
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
case '=': add_token_2('=', TK("="), TK("==")); return NULL;
|
||||
case '+':
|
||||
if(matchchar('+')) {
|
||||
add_token(TK("++"));
|
||||
} else {
|
||||
add_token_2('=', TK("+"), TK("+="));
|
||||
}
|
||||
return NULL;
|
||||
case '=': add_token_2('=', TK_ASSIGN, TK_EQ); return NULL;
|
||||
case '+': add_token_2('=', TK_ADD, TK_IADD); return NULL;
|
||||
case '>': {
|
||||
if(matchchar('='))
|
||||
add_token(TK(">="));
|
||||
add_token(TK_GE);
|
||||
else if(matchchar('>'))
|
||||
add_token_2('=', TK(">>"), TK(">>="));
|
||||
add_token_2('=', TK_RSHIFT, TK_IRSHIFT);
|
||||
else
|
||||
add_token(TK(">"));
|
||||
add_token(TK_GT);
|
||||
return NULL;
|
||||
}
|
||||
case '<': {
|
||||
if(matchchar('='))
|
||||
add_token(TK("<="));
|
||||
add_token(TK_LE);
|
||||
else if(matchchar('<'))
|
||||
add_token_2('=', TK("<<"), TK("<<="));
|
||||
add_token_2('=', TK_LSHIFT, TK_ILSHIFT);
|
||||
else
|
||||
add_token(TK("<"));
|
||||
add_token(TK_LT);
|
||||
return NULL;
|
||||
}
|
||||
case '-': {
|
||||
if(matchchar('-')) {
|
||||
add_token(TK("--"));
|
||||
} else {
|
||||
if(matchchar('='))
|
||||
add_token(TK("-="));
|
||||
add_token(TK_ISUB);
|
||||
else if(matchchar('>'))
|
||||
add_token(TK("->"));
|
||||
add_token(TK_ARROW);
|
||||
else
|
||||
add_token(TK("-"));
|
||||
}
|
||||
add_token(TK_SUB);
|
||||
return NULL;
|
||||
}
|
||||
case '!':
|
||||
if(matchchar('=')){
|
||||
add_token(TK("!="));
|
||||
add_token(TK_NE);
|
||||
}else{
|
||||
Error* err = SyntaxError("expected '=' after '!'");
|
||||
if(err) return err;
|
||||
@ -438,22 +432,22 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
|
||||
break;
|
||||
case '*':
|
||||
if(matchchar('*')) {
|
||||
add_token(TK("**")); // '**'
|
||||
add_token(TK_POW); // '**'
|
||||
} else {
|
||||
add_token_2('=', TK("*"), TK("*="));
|
||||
add_token_2('=', TK_MUL, TK_IMUL);
|
||||
}
|
||||
return NULL;
|
||||
case '/':
|
||||
if(matchchar('/')) {
|
||||
add_token_2('=', TK("//"), TK("//="));
|
||||
add_token_2('=', TK_FLOORDIV, TK_IFLOORDIV);
|
||||
} else {
|
||||
add_token_2('=', TK("/"), TK("/="));
|
||||
add_token_2('=', TK_DIV, TK_IDIV);
|
||||
}
|
||||
return NULL;
|
||||
case ' ':
|
||||
case '\t': eat_spaces(); break;
|
||||
case '\n': {
|
||||
add_token(TK("@eol"));
|
||||
add_token(TK_EOL);
|
||||
if(!eat_indentation()){
|
||||
return IndentationError("unindent does not match any outer indentation level");
|
||||
}
|
||||
@ -479,10 +473,10 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
|
||||
token_start = curr_char;
|
||||
while(indents.size() > 1) {
|
||||
indents.pop_back();
|
||||
add_token(TK("@dedent"));
|
||||
add_token(TK_DEDENT);
|
||||
return NULL;
|
||||
}
|
||||
add_token(TK("@eof"));
|
||||
add_token(TK_EOF);
|
||||
*eof = true;
|
||||
return NULL;
|
||||
}
|
||||
@ -506,7 +500,7 @@ Error* Lexer::_error(bool lexer_err, const char* type, const char* msg, va_list*
|
||||
if(args){
|
||||
vsnprintf(err->msg, sizeof(err->msg), msg, *args);
|
||||
}else{
|
||||
std::strncpy(err->msg, msg, sizeof(err->msg));
|
||||
strncpy(err->msg, msg, sizeof(err->msg));
|
||||
}
|
||||
err->userdata = userdata;
|
||||
return err;
|
||||
@ -527,7 +521,7 @@ Error* Lexer::run() noexcept{
|
||||
return from_precompiled();
|
||||
}
|
||||
// push initial tokens
|
||||
this->nexts.push_back(Token{TK("@sof"), token_start, 0, current_line, brackets_level, {}});
|
||||
this->nexts.push_back(Token{TK_SOF, token_start, 0, current_line, brackets_level, {}});
|
||||
this->indents.push_back(0);
|
||||
|
||||
bool eof = false;
|
||||
@ -539,34 +533,34 @@ Error* Lexer::run() noexcept{
|
||||
}
|
||||
|
||||
Error* Lexer::from_precompiled() noexcept{
|
||||
pkpy_TokenDeserializer deserializer;
|
||||
pkpy_TokenDeserializer__ctor(&deserializer, pkpy_Str__data(&src->source));
|
||||
pk_TokenDeserializer deserializer;
|
||||
pk_TokenDeserializer__ctor(&deserializer, pkpy_Str__data(&src->source));
|
||||
|
||||
deserializer.curr += 5; // skip "pkpy:"
|
||||
c11_string version = pkpy_TokenDeserializer__read_string(&deserializer, '\n');
|
||||
c11_string version = pk_TokenDeserializer__read_string(&deserializer, '\n');
|
||||
|
||||
if(c11_string__cmp3(version, PK_VERSION) != 0) {
|
||||
return SyntaxError("precompiled version mismatch");
|
||||
}
|
||||
if(pkpy_TokenDeserializer__read_uint(&deserializer, '\n') != (i64)src->mode){
|
||||
if(pk_TokenDeserializer__read_uint(&deserializer, '\n') != (i64)src->mode){
|
||||
return SyntaxError("precompiled mode mismatch");
|
||||
}
|
||||
|
||||
int count = pkpy_TokenDeserializer__read_count(&deserializer);
|
||||
int count = pk_TokenDeserializer__read_count(&deserializer);
|
||||
c11_vector* precompiled_tokens = &src->_precompiled_tokens;
|
||||
for(int i = 0; i < count; i++) {
|
||||
c11_string item = pkpy_TokenDeserializer__read_string(&deserializer, '\n');
|
||||
c11_string item = pk_TokenDeserializer__read_string(&deserializer, '\n');
|
||||
pkpy_Str copied_item;
|
||||
pkpy_Str__ctor2(&copied_item, item.data, item.size);
|
||||
c11_vector__push(pkpy_Str, precompiled_tokens, copied_item);
|
||||
}
|
||||
|
||||
count = pkpy_TokenDeserializer__read_count(&deserializer);
|
||||
count = pk_TokenDeserializer__read_count(&deserializer);
|
||||
for(int i = 0; i < count; i++) {
|
||||
Token t;
|
||||
t.type = (unsigned char)pkpy_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
t.type = (TokenIndex)pk_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
if(is_raw_string_used(t.type)) {
|
||||
i64 index = pkpy_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
i64 index = pk_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
pkpy_Str* p = c11__at(pkpy_Str, precompiled_tokens, index);
|
||||
t.start = pkpy_Str__data(p);
|
||||
t.length = c11__getitem(pkpy_Str, precompiled_tokens, index).size;
|
||||
@ -575,28 +569,28 @@ Error* Lexer::from_precompiled() noexcept{
|
||||
t.length = 0;
|
||||
}
|
||||
|
||||
if(pkpy_TokenDeserializer__match_char(&deserializer, ',')) {
|
||||
if(pk_TokenDeserializer__match_char(&deserializer, ',')) {
|
||||
t.line = nexts.back().line;
|
||||
} else {
|
||||
t.line = (int)pkpy_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
t.line = (int)pk_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
}
|
||||
|
||||
if(pkpy_TokenDeserializer__match_char(&deserializer, ',')) {
|
||||
if(pk_TokenDeserializer__match_char(&deserializer, ',')) {
|
||||
t.brackets_level = nexts.back().brackets_level;
|
||||
} else {
|
||||
t.brackets_level = (int)pkpy_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
t.brackets_level = (int)pk_TokenDeserializer__read_uint(&deserializer, ',');
|
||||
}
|
||||
|
||||
char type = (*deserializer.curr++); // read_char
|
||||
switch(type) {
|
||||
case 'I':
|
||||
t.value = pkpy_TokenDeserializer__read_uint(&deserializer, '\n');
|
||||
t.value = pk_TokenDeserializer__read_uint(&deserializer, '\n');
|
||||
break;
|
||||
case 'F':
|
||||
t.value = pkpy_TokenDeserializer__read_float(&deserializer, '\n');
|
||||
t.value = pk_TokenDeserializer__read_float(&deserializer, '\n');
|
||||
break;
|
||||
case 'S': {
|
||||
pkpy_Str res = pkpy_TokenDeserializer__read_string_from_hex(&deserializer, '\n');
|
||||
pkpy_Str res = pk_TokenDeserializer__read_string_from_hex(&deserializer, '\n');
|
||||
t.value = Str(std::move(res));
|
||||
} break;
|
||||
default:
|
||||
|
||||
@ -201,6 +201,7 @@ struct Random {
|
||||
f64 key = self.gen.uniform(0.0, cum_weights[size - 1]);
|
||||
int index;
|
||||
c11__lower_bound(f64, cum_weights.begin(), cum_weights.size(), key, c11__less, &index);
|
||||
assert(index != cum_weights.size());
|
||||
result[i] = data[index];
|
||||
}
|
||||
return VAR(std::move(result));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user