Zxc200611/pocketpy
1
0
Fork 0
mirror of https://github.com/pocketpy/pocketpy synced 2025-10-24 13:30:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

move lexer

Browse Source
This commit is contained in:
blueloveTH 2024-06-28 15:02:55 +08:00
parent fa31f4c5df
commit 104c266bc0
3 changed files with 697 additions and 809 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
include/pocketpy/compiler/lexer.h
View File

@ -78,13 +78,6 @@ enum Precedence {
PREC_HIGHEST,
};
enum StringType {
NORMAL_STRING,
RAW_STRING,
F_STRING,
NORMAL_BYTES
};
#define is_raw_string_used(t) ((t) == TK_ID || (t) == TK_LONG)
typedef struct pk_TokenDeserializer {
@ -100,6 +93,13 @@ int pk_TokenDeserializer__read_count(pk_TokenDeserializer* self);
int64_t pk_TokenDeserializer__read_uint(pk_TokenDeserializer* self, char c);
double pk_TokenDeserializer__read_float(pk_TokenDeserializer* self, char c);
typedef enum IntParsingResult{
IntParsing_SUCCESS,
IntParsing_FAILURE,
IntParsing_OVERFLOW,
} IntParsingResult;
IntParsingResult parse_uint(c11_string text, int64_t* out, int base);
#ifdef __cplusplus
}

751
src/compiler/lexer.cpp
View File

@ -1,751 +0,0 @@
#include "pocketpy/compiler/lexer.hpp"
#include "pocketpy/common/config.h"
#include "pocketpy/common/str.h"
#include "pocketpy/common/smallmap.h"
#include "pocketpy/compiler/lexer.h"
#include <cstdarg>
namespace pkpy {
static bool is_possible_number_char(char c) noexcept{
switch(c) {
// clang-format off
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
case '.': case 'L': case 'x': case 'o': case 'j':
return true;
default: return false;
// clang-format on
}
}
bool Lexer::match_n_chars(int n, char c0) noexcept{
const char* c = curr_char;
for(int i = 0; i < n; i++) {
if(*c == '\0') return false;
if(*c != c0) return false;
c++;
}
for(int i = 0; i < n; i++)
eatchar_include_newline();
return true;
}
bool Lexer::match_string(const char* s) noexcept{
int s_len = strlen(s);
bool ok = strncmp(curr_char, s, s_len) == 0;
if(ok)
for(int i = 0; i < s_len; i++)
eatchar_include_newline();
return ok;
}
int Lexer::eat_spaces() noexcept{
int count = 0;
while(true) {
switch(peekchar()) {
case ' ': count += 1; break;
case '\t': count += 4; break;
default: return count;
}
eatchar();
}
}
bool Lexer::eat_indentation() noexcept{
if(brackets_level > 0) return true;
int spaces = eat_spaces();
if(peekchar() == '#') skip_line_comment();
if(peekchar() == '\0' || peekchar() == '\n') return true;
// https://docs.python.org/3/reference/lexical_analysis.html#indentation
if(spaces > indents.back()) {
indents.push_back(spaces);
nexts.push_back(Token{TK_INDENT, token_start, 0, current_line, brackets_level, {}});
} else if(spaces < indents.back()) {
while(spaces < indents.back()) {
indents.pop_back();
nexts.push_back(Token{TK_DEDENT, token_start, 0, current_line, brackets_level, {}});
}
if(spaces != indents.back()) { return false; }
}
return true;
}
char Lexer::eatchar() noexcept{
char c = peekchar();
assert(c != '\n'); // eatchar() cannot consume a newline
curr_char++;
return c;
}
char Lexer::eatchar_include_newline() noexcept{
char c = peekchar();
curr_char++;
if(c == '\n') {
current_line++;
c11_vector__push(const char*, &src->line_starts, curr_char);
}
return c;
}
Error* Lexer::eat_name() noexcept{
curr_char--;
while(true) {
unsigned char c = peekchar();
int u8bytes = c11__u8_header(c, true);
if(u8bytes == 0) return SyntaxError("invalid char: %c", c);
if(u8bytes == 1) {
if(isalpha(c) || c == '_' || isdigit(c)) {
curr_char++;
continue;
} else {
break;
}
}
// handle multibyte char
Str u8str(curr_char, u8bytes);
if(u8str.size != u8bytes) return SyntaxError("invalid utf8 sequence: %s", u8str.c_str());
uint32_t value = 0;
for(int k = 0; k < u8bytes; k++) {
uint8_t b = u8str[k];
if(k == 0) {
if(u8bytes == 2)
value = (b & 0b00011111) << 6;
else if(u8bytes == 3)
value = (b & 0b00001111) << 12;
else if(u8bytes == 4)
value = (b & 0b00000111) << 18;
} else {
value |= (b & 0b00111111) << (6 * (u8bytes - k - 1));
}
}
if(c11__is_unicode_Lo_char(value))
curr_char += u8bytes;
else
break;
}
int length = (int)(curr_char - token_start);
if(length == 0) return SyntaxError("@id contains invalid char");
c11_string name = {token_start, length};
if(src->mode == JSON_MODE) {
if(c11_string__cmp3(name, "true") == 0) {
add_token(TK_TRUE);
} else if(c11_string__cmp3(name, "false") == 0) {
add_token(TK_FALSE);
} else if(c11_string__cmp3(name, "null") == 0) {
add_token(TK_NONE);
} else {
return SyntaxError("invalid JSON token");
}
return NULL;
}
const char** KW_BEGIN = pk_TokenSymbols + TK_FALSE;
int KW_COUNT = TK__COUNT__ - TK_FALSE;
#define less(a, b) (c11_string__cmp3(b, a) > 0)
int out;
c11__lower_bound(const char*, KW_BEGIN, KW_COUNT, name, less, &out);
#undef less
if(out != KW_COUNT && c11_string__cmp3(name, KW_BEGIN[out]) == 0) {
add_token((TokenIndex)(out + TK_FALSE));
} else {
add_token(TK_ID);
}
return NULL;
}
void Lexer::skip_line_comment() noexcept{
char c;
while((c = peekchar()) != '\0') {
if(c == '\n') return;
eatchar();
}
}
bool Lexer::matchchar(char c) noexcept{
if(peekchar() != c) return false;
eatchar_include_newline();
return true;
}
void Lexer::add_token(TokenIndex type, TokenValue value) noexcept{
switch(type) {
case TK_LBRACE:
case TK_LBRACKET:
case TK_LPAREN: brackets_level++; break;
case TK_RPAREN:
case TK_RBRACKET:
case TK_RBRACE: brackets_level--; break;
default: break;
}
auto token = Token{type,
token_start,
(int)(curr_char - token_start),
current_line - ((type == TK_EOL) ? 1 : 0),
brackets_level,
value};
// handle "not in", "is not", "yield from"
if(!nexts.empty()) {
auto& back = nexts.back();
if(back.type == TK_NOT_KW && type == TK_IN) {
back.type = TK_NOT_IN;
return;
}
if(back.type == TK_IS && type == TK_NOT_KW) {
back.type = TK_IS_NOT;
return;
}
if(back.type == TK_YIELD && type == TK_FROM) {
back.type = TK_YIELD_FROM;
return;
}
nexts.push_back(token);
}
}
void Lexer::add_token_2(char c, TokenIndex one, TokenIndex two) noexcept{
if(matchchar(c))
add_token(two);
else
add_token(one);
}
Error* Lexer::eat_string_until(char quote, bool raw, Str* out) noexcept{
bool quote3 = match_n_chars(2, quote);
small_vector_2<char, 32> buff;
while(true) {
char c = eatchar_include_newline();
if(c == quote) {
if(quote3 && !match_n_chars(2, quote)) {
buff.push_back(c);
continue;
}
break;
}
if(c == '\0') {
if(quote3 && src->mode == REPL_MODE) return NeedMoreLines();
return SyntaxError("EOL while scanning string literal");
}
if(c == '\n') {
if(!quote3)
return SyntaxError("EOL while scanning string literal");
else {
buff.push_back(c);
continue;
}
}
if(!raw && c == '\\') {
switch(eatchar_include_newline()) {
case '"': buff.push_back('"'); break;
case '\'': buff.push_back('\''); break;
case '\\': buff.push_back('\\'); break;
case 'n': buff.push_back('\n'); break;
case 'r': buff.push_back('\r'); break;
case 't': buff.push_back('\t'); break;
case 'b': buff.push_back('\b'); break;
case 'x': {
char hex[3] = {eatchar(), eatchar(), '\0'};
size_t parsed;
char code;
try {
code = (char)std::stoi(hex, &parsed, 16);
} catch(...) {
return SyntaxError("invalid hex char");
}
if(parsed != 2) return SyntaxError("invalid hex char");
buff.push_back(code);
} break;
default: return SyntaxError("invalid escape char");
}
} else {
buff.push_back(c);
}
}
*out = Str(buff.data(), buff.size());
return nullptr;
}
Error* Lexer::eat_string(char quote, StringType type) noexcept{
Str s;
Error* err = eat_string_until(quote, type == StringType::RAW_STRING, &s);
if(err) return err;
if(type == StringType::F_STRING) {
add_token(TK_FSTR, s);
}else if(type == StringType::NORMAL_BYTES) {
add_token(TK_BYTES, s);
}else{
add_token(TK_STR, s);
}
return NULL;
}
Error* Lexer::eat_number() noexcept{
const char* i = token_start;
while(is_possible_number_char(*i))
i++;
bool is_scientific_notation = false;
if(*(i - 1) == 'e' && (*i == '+' || *i == '-')) {
i++;
while(isdigit(*i) || *i == 'j')
i++;
is_scientific_notation = true;
}
std::string_view text(token_start, i - token_start);
this->curr_char = i;
if(text[0] != '.' && !is_scientific_notation) {
// try long
if(i[-1] == 'L') {
add_token(TK_LONG);
return NULL;
}
// try integer
i64 int_out;
switch(parse_uint(text, &int_out, -1)) {
case IntParsingResult::Success: add_token(TK_NUM, int_out); return NULL;
case IntParsingResult::Overflow: return SyntaxError("int literal is too large");
case IntParsingResult::Failure: break; // do nothing
}
}
// try float
double float_out;
char* p_end;
try {
float_out = std::strtod(text.data(), &p_end);
} catch(...) {
return SyntaxError("invalid number literal");
}
if(p_end == text.data() + text.size()) {
add_token(TK_NUM, (f64)float_out);
return NULL;
}
if(i[-1] == 'j' && p_end == text.data() + text.size() - 1) {
add_token(TK_IMAG, (f64)float_out);
return NULL;
}
return SyntaxError("invalid number literal");
}
Error* Lexer::lex_one_token(bool* eof) noexcept{
*eof = false;
while(peekchar() != '\0') {
token_start = curr_char;
char c = eatchar_include_newline();
switch(c) {
case '\'':
case '"': {
Error* err = eat_string(c, StringType::NORMAL_STRING);
if(err) return err;
return NULL;
}
case '#': skip_line_comment(); break;
case '~': add_token(TK_INVERT); return NULL;
case '{': add_token(TK_LBRACE); return NULL;
case '}': add_token(TK_RBRACE); return NULL;
case ',': add_token(TK_COMMA); return NULL;
case ':': add_token(TK_COLON); return NULL;
case ';': add_token(TK_SEMICOLON); return NULL;
case '(': add_token(TK_LPAREN); return NULL;
case ')': add_token(TK_RPAREN); return NULL;
case '[': add_token(TK_LBRACKET); return NULL;
case ']': add_token(TK_RBRACKET); return NULL;
case '@': add_token(TK_DECORATOR); return NULL;
case '\\': {
// line continuation character
char c = eatchar_include_newline();
if(c != '\n') {
if(src->mode == REPL_MODE && c == '\0') return NeedMoreLines();
return SyntaxError("expected newline after line continuation character");
}
eat_spaces();
return NULL;
}
case '%': add_token_2('=', TK_MOD, TK_IMOD); return NULL;
case '&': add_token_2('=', TK_AND, TK_IAND); return NULL;
case '|': add_token_2('=', TK_OR, TK_IOR); return NULL;
case '^': add_token_2('=', TK_XOR, TK_IXOR); return NULL;
case '.': {
if(matchchar('.')) {
if(matchchar('.')) {
add_token(TK_DOTDOTDOT);
} else {
add_token(TK_DOTDOT);
}
} else {
char next_char = peekchar();
if(next_char >= '0' && next_char <= '9') {
Error* err = eat_number();
if(err) return err;
} else {
add_token(TK_DOT);
}
}
return NULL;
}
case '=': add_token_2('=', TK_ASSIGN, TK_EQ); return NULL;
case '+': add_token_2('=', TK_ADD, TK_IADD); return NULL;
case '>': {
if(matchchar('='))
add_token(TK_GE);
else if(matchchar('>'))
add_token_2('=', TK_RSHIFT, TK_IRSHIFT);
else
add_token(TK_GT);
return NULL;
}
case '<': {
if(matchchar('='))
add_token(TK_LE);
else if(matchchar('<'))
add_token_2('=', TK_LSHIFT, TK_ILSHIFT);
else
add_token(TK_LT);
return NULL;
}
case '-': {
if(matchchar('='))
add_token(TK_ISUB);
else if(matchchar('>'))
add_token(TK_ARROW);
else
add_token(TK_SUB);
return NULL;
}
case '!':
if(matchchar('=')){
add_token(TK_NE);
}else{
Error* err = SyntaxError("expected '=' after '!'");
if(err) return err;
}
break;
case '*':
if(matchchar('*')) {
add_token(TK_POW); // '**'
} else {
add_token_2('=', TK_MUL, TK_IMUL);
}
return NULL;
case '/':
if(matchchar('/')) {
add_token_2('=', TK_FLOORDIV, TK_IFLOORDIV);
} else {
add_token_2('=', TK_DIV, TK_IDIV);
}
return NULL;
case ' ':
case '\t': eat_spaces(); break;
case '\n': {
add_token(TK_EOL);
if(!eat_indentation()){
return IndentationError("unindent does not match any outer indentation level");
}
return NULL;
}
default: {
if(c == 'f') {
if(matchchar('\'')) return eat_string('\'', StringType::F_STRING);
if(matchchar('"')) return eat_string('"', StringType::F_STRING);
} else if(c == 'r') {
if(matchchar('\'')) return eat_string('\'', StringType::RAW_STRING);
if(matchchar('"')) return eat_string('"', StringType::RAW_STRING);
} else if(c == 'b') {
if(matchchar('\'')) return eat_string('\'', StringType::NORMAL_BYTES);
if(matchchar('"')) return eat_string('"', StringType::NORMAL_BYTES);
}
if(c >= '0' && c <= '9') return eat_number();
return eat_name();
}
}
}
token_start = curr_char;
while(indents.size() > 1) {
indents.pop_back();
add_token(TK_DEDENT);
return NULL;
}
add_token(TK_EOF);
*eof = true;
return NULL;
}
Error* Lexer::_error(bool lexer_err, const char* type, const char* msg, va_list* args, i64 userdata) noexcept{
Error* err = (Error*)malloc(sizeof(Error));
err->type = type;
err->src = src;
PK_INCREF(src);
if(lexer_err){
err->lineno = current_line;
err->cursor = curr_char;
if(*curr_char == '\n') {
err->lineno--;
err->cursor--;
}
}else{
err->lineno = -1;
err->cursor = NULL;
}
if(args){
vsnprintf(err->msg, sizeof(err->msg), msg, *args);
}else{
strncpy(err->msg, msg, sizeof(err->msg));
}
err->userdata = userdata;
return err;
}
Error* Lexer::SyntaxError(const char* fmt, ...) noexcept{
va_list args;
va_start(args, fmt);
Error* err = _error(true, "SyntaxError", fmt, &args);
va_end(args);
return err;
}
Error* Lexer::run() noexcept{
assert(!this->used);
this->used = true;
if(src->is_precompiled) {
return from_precompiled();
}
// push initial tokens
this->nexts.push_back(Token{TK_SOF, token_start, 0, current_line, brackets_level, {}});
this->indents.push_back(0);
bool eof = false;
while(!eof) {
Error* err = lex_one_token(&eof);
if(err) return err;
}
return NULL;
}
Error* Lexer::from_precompiled() noexcept{
pk_TokenDeserializer deserializer;
pk_TokenDeserializer__ctor(&deserializer, py_Str__data(&src->source));
deserializer.curr += 5; // skip "pkpy:"
c11_string version = pk_TokenDeserializer__read_string(&deserializer, '\n');
if(c11_string__cmp3(version, PK_VERSION) != 0) {
return SyntaxError("precompiled version mismatch");
}
if(pk_TokenDeserializer__read_uint(&deserializer, '\n') != (i64)src->mode){
return SyntaxError("precompiled mode mismatch");
}
int count = pk_TokenDeserializer__read_count(&deserializer);
c11_vector* precompiled_tokens = &src->_precompiled_tokens;
for(int i = 0; i < count; i++) {
c11_string item = pk_TokenDeserializer__read_string(&deserializer, '\n');
py_Str copied_item;
py_Str__ctor2(&copied_item, item.data, item.size);
c11_vector__push(py_Str, precompiled_tokens, copied_item);
}
count = pk_TokenDeserializer__read_count(&deserializer);
for(int i = 0; i < count; i++) {
Token t;
t.type = (TokenIndex)pk_TokenDeserializer__read_uint(&deserializer, ',');
if(is_raw_string_used(t.type)) {
i64 index = pk_TokenDeserializer__read_uint(&deserializer, ',');
py_Str* p = c11__at(py_Str, precompiled_tokens, index);
t.start = py_Str__data(p);
t.length = c11__getitem(py_Str, precompiled_tokens, index).size;
} else {
t.start = NULL;
t.length = 0;
}
if(pk_TokenDeserializer__match_char(&deserializer, ',')) {
t.line = nexts.back().line;
} else {
t.line = (int)pk_TokenDeserializer__read_uint(&deserializer, ',');
}
if(pk_TokenDeserializer__match_char(&deserializer, ',')) {
t.brackets_level = nexts.back().brackets_level;
} else {
t.brackets_level = (int)pk_TokenDeserializer__read_uint(&deserializer, ',');
}
char type = (*deserializer.curr++); // read_char
switch(type) {
case 'I':
t.value = pk_TokenDeserializer__read_uint(&deserializer, '\n');
break;
case 'F':
t.value = pk_TokenDeserializer__read_float(&deserializer, '\n');
break;
case 'S': {
py_Str res = pk_TokenDeserializer__read_string_from_hex(&deserializer, '\n');
t.value = Str(std::move(res));
} break;
default:
t.value = {};
break;
}
nexts.push_back(t);
}
return NULL;
}
Error* Lexer::precompile(Str* out) noexcept{
assert(!src->is_precompiled);
Error* err = run();
if(err) return err;
SStream ss;
ss << "pkpy:" PK_VERSION << '\n'; // L1: version string
ss << (int)src->mode << '\n'; // L2: mode
c11_smallmap_s2n token_indices;
c11_smallmap_s2n__ctor(&token_indices);
for(auto token: nexts) {
if(is_raw_string_used(token.type)) {
c11_string token_sv = {token.start, token.length};
if(!c11_smallmap_s2n__contains(&token_indices, token_sv)) {
c11_smallmap_s2n__set(&token_indices, token_sv, 0);
// assert no '\n' in token.sv()
for(char c: token.sv())
assert(c != '\n');
}
}
}
ss << "=" << (int)token_indices.count << '\n'; // L3: raw string count
uint16_t index = 0;
for(int i=0; i<token_indices.count; i++){
auto kv = c11__at(c11_smallmap_s2n_KV, &token_indices, i);
ss << kv->key << '\n'; // L4: raw strings
kv->value = index++;
}
ss << "=" << (int)nexts.size() << '\n'; // L5: token count
for(int i = 0; i < nexts.size(); i++) {
const Token& token = nexts[i];
ss << (int)token.type << ',';
if(is_raw_string_used(token.type)) {
uint16_t *p = c11_smallmap_s2n__try_get(&token_indices, {token.start, token.length});
assert(p != NULL);
ss << (int)*p << ',';
}
if(i > 0 && nexts[i - 1].line == token.line)
ss << ',';
else
ss << token.line << ',';
if(i > 0 && nexts[i - 1].brackets_level == token.brackets_level)
ss << ',';
else
ss << token.brackets_level << ',';
// visit token value
std::visit(
[&ss](auto&& arg) {
using T = std::decay_t<decltype(arg)>;
if constexpr(std::is_same_v<T, i64>) {
ss << 'I' << arg;
} else if constexpr(std::is_same_v<T, f64>) {
ss << 'F' << arg;
} else if constexpr(std::is_same_v<T, Str>) {
ss << 'S';
for(char c: arg)
ss.write_hex((unsigned char)c);
}
ss << '\n';
},
token.value);
}
*out = ss.str();
c11_smallmap_s2n__dtor(&token_indices);
return NULL;
}
IntParsingResult parse_uint(std::string_view text, i64* out, int base) noexcept{
*out = 0;
if(base == -1) {
if(text.substr(0, 2) == "0b")
base = 2;
else if(text.substr(0, 2) == "0o")
base = 8;
else if(text.substr(0, 2) == "0x")
base = 16;
else
base = 10;
}
if(base == 10) {
// 10-base 12334
if(text.length() == 0) return IntParsingResult::Failure;
for(char c: text) {
if(c >= '0' && c <= '9') {
*out = (*out * 10) + (c - '0');
} else {
return IntParsingResult::Failure;
}
}
const std::string_view INT64_MAX_S = "9223372036854775807";
if(text.length() > INT64_MAX_S.length()) return IntParsingResult::Overflow;
return IntParsingResult::Success;
} else if(base == 2) {
// 2-base 0b101010
if(text.substr(0, 2) == "0b") text.remove_prefix(2);
if(text.length() == 0) return IntParsingResult::Failure;
for(char c: text) {
if(c == '0' || c == '1') {
*out = (*out << 1) | (c - '0');
} else {
return IntParsingResult::Failure;
}
}
const std::string_view INT64_MAX_S = "111111111111111111111111111111111111111111111111111111111111111";
if(text.length() > INT64_MAX_S.length()) return IntParsingResult::Overflow;
return IntParsingResult::Success;
} else if(base == 8) {
// 8-base 0o123
if(text.substr(0, 2) == "0o") text.remove_prefix(2);
if(text.length() == 0) return IntParsingResult::Failure;
for(char c: text) {
if(c >= '0' && c <= '7') {
*out = (*out << 3) | (c - '0');
} else {
return IntParsingResult::Failure;
}
}
const std::string_view INT64_MAX_S = "777777777777777777777";
if(text.length() > INT64_MAX_S.length()) return IntParsingResult::Overflow;
return IntParsingResult::Success;
} else if(base == 16) {
// 16-base 0x123
if(text.substr(0, 2) == "0x") text.remove_prefix(2);
if(text.length() == 0) return IntParsingResult::Failure;
for(char c: text) {
if(c >= '0' && c <= '9') {
*out = (*out << 4) | (c - '0');
} else if(c >= 'a' && c <= 'f') {
*out = (*out << 4) | (c - 'a' + 10);
} else if(c >= 'A' && c <= 'F') {
*out = (*out << 4) | (c - 'A' + 10);
} else {
return IntParsingResult::Failure;
}
}
const std::string_view INT64_MAX_S = "7fffffffffffffff";
if(text.length() > INT64_MAX_S.length()) return IntParsingResult::Overflow;
return IntParsingResult::Success;
}
return IntParsingResult::Failure;
}
} // namespace pkpy

741
src/compiler/lexer2.c
View File

@ -1,5 +1,12 @@
#include "pocketpy/common/smallmap.h"
#include "pocketpy/common/config.h"
#include "pocketpy/common/sstream.h"
#include "pocketpy/common/vector.h"
#include "pocketpy/compiler/lexer.h"
#include "pocketpy/objects/sourcedata.h"
#include <ctype.h>
#include <stdarg.h>
#include <stdbool.h>
typedef struct pk_Lexer{
pk_SourceData_ src;
@ -14,7 +21,7 @@ typedef struct pk_Lexer{
const static TokenValue EmptyTokenValue;
void pk_Lexer__ctor(pk_Lexer* self, pk_SourceData_ src){
static void pk_Lexer__ctor(pk_Lexer* self, pk_SourceData_ src){
PK_INCREF(src);
self->src = src;
self->curr_char = self->token_start = py_Str__data(&src->source);
@ -24,45 +31,20 @@ void pk_Lexer__ctor(pk_Lexer* self, pk_SourceData_ src){
c11_vector__ctor(&self->indents, sizeof(int));
}
void pk_Lexer__dtor(pk_Lexer* self){
static void pk_Lexer__dtor(pk_Lexer* self){
PK_DECREF(self->src);
c11_vector__dtor(&self->nexts);
c11_vector__dtor(&self->indents);
}
void* pk_Lexer__run(pk_SourceData_ src, void** out_tokens){
pk_Lexer lexer;
pk_Lexer__ctor(&lexer, src);
if(src->is_precompiled) {
pk_Lexer__dtor(&lexer);
return from_precompiled();
}
// push initial tokens
Token sof = {TK_SOF, lexer.token_start, 0, lexer.current_line, lexer.brackets_level, EmptyTokenValue};
c11_vector__push(Token, &lexer.nexts, sof);
c11_vector__push(int, &lexer.indents, 0);
bool eof = false;
while(!eof) {
void* err = lex_one_token(&eof);
if(err){
pk_Lexer__dtor(&lexer);
return err;
}
}
pk_Lexer__dtor(&lexer);
return NULL;
}
char eatchar(pk_Lexer* self){
static char eatchar(pk_Lexer* self){
char c = *self->curr_char;
assert(c != '\n'); // eatchar() cannot consume a newline
self->curr_char++;
return c;
}
char eatchar_include_newline(pk_Lexer* self){
static char eatchar_include_newline(pk_Lexer* self){
char c = *self->curr_char;
self->curr_char++;
if(c == '\n') {
@ -72,7 +54,7 @@ char eatchar_include_newline(pk_Lexer* self){
return c;
}
int eat_spaces(pk_Lexer* self){
static int eat_spaces(pk_Lexer* self){
int count = 0;
while(true) {
switch(*self->curr_char) {
@ -84,13 +66,13 @@ int eat_spaces(pk_Lexer* self){
}
}
bool matchchar(pk_Lexer* self, char c){
static bool matchchar(pk_Lexer* self, char c){
if(*self->curr_char != c) return false;
eatchar_include_newline(self);
return true;
}
bool match_n_chars(pk_Lexer* self, int n, char c0){
static bool match_n_chars(pk_Lexer* self, int n, char c0){
const char* c = self->curr_char;
for(int i = 0; i < n; i++) {
if(*c == '\0') return false;
@ -102,23 +84,14 @@ bool match_n_chars(pk_Lexer* self, int n, char c0){
return true;
}
bool match_string(pk_Lexer* self, const char* s){
int s_len = strlen(s);
if(strncmp(self->curr_char, s, s_len) == 0){
for(int i = 0; i < s_len; i++)
eatchar_include_newline(self);
}
return ok;
}
void skip_line_comment(pk_Lexer* self){
static void skip_line_comment(pk_Lexer* self){
while(*self->curr_char) {
if(*self->curr_char == '\n') return;
eatchar(self);
}
}
void add_token(pk_Lexer* self, TokenIndex type, TokenValue value){
static void add_token_with_value(pk_Lexer* self, TokenIndex type, TokenValue value){
switch(type) {
case TK_LBRACE:
case TK_LBRACKET:
@ -153,18 +126,21 @@ void add_token(pk_Lexer* self, TokenIndex type, TokenValue value){
}
}
void add_token_2(pk_Lexer* self, char c, TokenIndex one, TokenIndex two){
if(matchchar(self, c))
add_token(self, two, EmptyTokenValue);
else
add_token(self, one, EmptyTokenValue);
static void add_token(pk_Lexer* self, TokenIndex type){
add_token_with_value(self, type, EmptyTokenValue);
}
bool eat_indentation(pk_Lexer* self){
static void add_token_2(pk_Lexer* self, char c, TokenIndex one, TokenIndex two){
if(matchchar(self, c))
add_token(self, two);
else
add_token(self, one);
}
static bool eat_indentation(pk_Lexer* self){
if(self->brackets_level > 0) return true;
int spaces = eat_spaces(self);
if(*self->curr_char == '#') skip_line_comment();
if(*self->curr_char == '#') skip_line_comment(self);
if(*self->curr_char == '\0' || *self->curr_char == '\n'){
return true;
}
@ -185,3 +161,666 @@ bool eat_indentation(pk_Lexer* self){
}
return true;
}
static bool is_possible_number_char(char c){
switch(c) {
// clang-format off
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
case '.': case 'L': case 'x': case 'o': case 'j':
return true;
default: return false;
// clang-format on
}
}
/******************************/
typedef struct Error Error;
static Error* SyntaxError(const char* fmt, ...){
// va_list args;
// va_start(args, fmt);
// Error* err = _error(true, "SyntaxError", fmt, &args);
// va_end(args);
// return err;
return NULL;
}
static Error* NeedMoreLines(){
return NULL;
}
static Error* eat_name(pk_Lexer* self){
self->curr_char--;
while(true) {
unsigned char c = *self->curr_char;
int u8bytes = c11__u8_header(c, true);
if(u8bytes == 0) return SyntaxError("invalid char: %c", c);
if(u8bytes == 1) {
if(isalnum(c) || c == '_') {
self->curr_char++;
continue;
} else {
break;
}
}
// handle multibyte char
py_Str u8str;
py_Str__ctor2(&u8str, self->curr_char, u8bytes);
if(u8str.size != u8bytes){
py_Str__dtor(&u8str);
return SyntaxError("invalid utf8 sequence: %s", py_Str__data(&u8str));
}
uint32_t value = 0;
for(int k = 0; k < u8bytes; k++) {
uint8_t b = py_Str__data(&u8str)[k];
if(k == 0) {
if(u8bytes == 2)
value = (b & 0b00011111) << 6;
else if(u8bytes == 3)
value = (b & 0b00001111) << 12;
else if(u8bytes == 4)
value = (b & 0b00000111) << 18;
} else {
value |= (b & 0b00111111) << (6 * (u8bytes - k - 1));
}
}
if(c11__is_unicode_Lo_char(value))
self->curr_char += u8bytes;
else
break;
}
int length = (int)(self->curr_char - self->token_start);
if(length == 0) return SyntaxError("@id contains invalid char");
c11_string name = {self->token_start, length};
if(self->src->mode == JSON_MODE) {
if(c11_string__cmp3(name, "true") == 0) {
add_token(self, TK_TRUE);
} else if(c11_string__cmp3(name, "false") == 0) {
add_token(self, TK_FALSE);
} else if(c11_string__cmp3(name, "null") == 0) {
add_token(self, TK_NONE);
} else {
return SyntaxError("invalid JSON token");
}
return NULL;
}
const char** KW_BEGIN = pk_TokenSymbols + TK_FALSE;
int KW_COUNT = TK__COUNT__ - TK_FALSE;
#define less(a, b) (c11_string__cmp3(b, a) > 0)
int out;
c11__lower_bound(const char*, KW_BEGIN, KW_COUNT, name, less, &out);
#undef less
if(out != KW_COUNT && c11_string__cmp3(name, KW_BEGIN[out]) == 0) {
add_token(self, (TokenIndex)(out + TK_FALSE));
} else {
add_token(self, TK_ID);
}
return NULL;
}
static Error* eat_string_until(pk_Lexer* self, char quote, bool raw, py_Str* out) {
// previous char is quote
bool quote3 = match_n_chars(self, 2, quote);
// small_vector_2<char, 32> buff;
pk_SStream buff;
while(true) {
char c = eatchar_include_newline(self);
if(c == quote) {
if(quote3 && !match_n_chars(self, 2, quote)) {
pk_SStream__write_char(&buff, c);
continue;
}
break;
}
if(c == '\0') {
if(quote3 && self->src->mode == REPL_MODE){
return NeedMoreLines();
}
return SyntaxError("EOL while scanning string literal");
}
if(c == '\n') {
if(!quote3)
return SyntaxError("EOL while scanning string literal");
else {
pk_SStream__write_char(&buff, c);
continue;
}
}
if(!raw && c == '\\') {
switch(eatchar_include_newline(self)) {
case '"': pk_SStream__write_char(&buff, '"'); break;
case '\'': pk_SStream__write_char(&buff, '\''); break;
case '\\': pk_SStream__write_char(&buff, '\\'); break;
case 'n': pk_SStream__write_char(&buff, '\n'); break;
case 'r': pk_SStream__write_char(&buff, '\r'); break;
case 't': pk_SStream__write_char(&buff, '\t'); break;
case 'b': pk_SStream__write_char(&buff, '\b'); break;
case 'x': {
char hex[3] = {eatchar(self), eatchar(self), '\0'};
int code;
if(sscanf(hex, "%x", &code) != 1) {
return SyntaxError("invalid hex char");
}
pk_SStream__write_char(&buff, (char)code);
} break;
default: return SyntaxError("invalid escape char");
}
} else {
pk_SStream__write_char(&buff, c);
}
}
*out = pk_SStream__submit(&buff);
return NULL;
}
enum StringType {
NORMAL_STRING,
RAW_STRING,
F_STRING,
NORMAL_BYTES
};
static Error* eat_string(pk_Lexer* self, char quote, enum StringType type){
py_Str s;
Error* err = eat_string_until(self, quote, type == RAW_STRING, &s);
if(err) return err;
TokenValue value = {2, ._str = s};
if(type == F_STRING) {
add_token_with_value(self, TK_FSTR, value);
}else if(type == NORMAL_BYTES) {
add_token_with_value(self, TK_BYTES, value);
}else{
add_token_with_value(self, TK_STR, value);
}
return NULL;
}
static Error* eat_number(pk_Lexer* self){
const char* i = self->token_start;
while(is_possible_number_char(*i)) i++;
bool is_scientific_notation = false;
if(*(i - 1) == 'e' && (*i == '+' || *i == '-')) {
i++;
while(isdigit(*i) || *i == 'j') i++;
is_scientific_notation = true;
}
c11_string text = {self->token_start, i - self->token_start};
self->curr_char = i;
if(text.data[0] != '.' && !is_scientific_notation) {
// try long
if(i[-1] == 'L') {
add_token(self, TK_LONG);
return NULL;
}
// try integer
TokenValue value = {.index = 0};
switch(parse_uint(text, &value._i64, -1)) {
case IntParsing_SUCCESS:
add_token_with_value(self, TK_NUM, value);
return NULL;
case IntParsing_OVERFLOW:
return SyntaxError("int literal is too large");
case IntParsing_FAILURE:
break; // do nothing
}
}
// try float
double float_out;
char* p_end;
float_out = strtod(text.data, &p_end);
if(p_end == text.data + text.size){
TokenValue value = {.index = 1, ._f64 = float_out};
add_token_with_value(self, TK_NUM, value);
return NULL;
}
if(i[-1] == 'j' && p_end == text.data + text.size - 1) {
TokenValue value = {.index = 1, ._f64 = float_out};
add_token_with_value(self, TK_IMAG, value);
return NULL;
}
return SyntaxError("invalid number literal");
}
static Error* lex_one_token(pk_Lexer* self, bool* eof){
*eof = false;
while(*self->curr_char) {
self->token_start = self->curr_char;
char c = eatchar_include_newline(self);
switch(c) {
case '\'':
case '"': {
Error* err = eat_string(self, c, NORMAL_STRING);
if(err) return err;
return NULL;
}
case '#': skip_line_comment(self); break;
case '~': add_token(self, TK_INVERT); return NULL;
case '{': add_token(self, TK_LBRACE); return NULL;
case '}': add_token(self, TK_RBRACE); return NULL;
case ',': add_token(self, TK_COMMA); return NULL;
case ':': add_token(self, TK_COLON); return NULL;
case ';': add_token(self, TK_SEMICOLON); return NULL;
case '(': add_token(self, TK_LPAREN); return NULL;
case ')': add_token(self, TK_RPAREN); return NULL;
case '[': add_token(self, TK_LBRACKET); return NULL;
case ']': add_token(self, TK_RBRACKET); return NULL;
case '@': add_token(self, TK_DECORATOR); return NULL;
case '\\': {
// line continuation character
char c = eatchar_include_newline(self);
if(c != '\n') {
if(self->src->mode == REPL_MODE && c == '\0') return NeedMoreLines();
return SyntaxError("expected newline after line continuation character");
}
eat_spaces(self);
return NULL;
}
case '%': add_token_2(self, '=', TK_MOD, TK_IMOD); return NULL;
case '&': add_token_2(self, '=', TK_AND, TK_IAND); return NULL;
case '|': add_token_2(self, '=', TK_OR, TK_IOR); return NULL;
case '^': add_token_2(self, '=', TK_XOR, TK_IXOR); return NULL;
case '.': {
if(matchchar(self, '.')) {
if(matchchar(self, '.')) {
add_token(self, TK_DOTDOTDOT);
} else {
add_token(self, TK_DOTDOT);
}
} else {
char next_char = *self->curr_char;
if(next_char >= '0' && next_char <= '9') {
Error* err = eat_number(self);
if(err) return err;
} else {
add_token(self, TK_DOT);
}
}
return NULL;
}
case '=': add_token_2(self, '=', TK_ASSIGN, TK_EQ); return NULL;
case '+': add_token_2(self, '=', TK_ADD, TK_IADD); return NULL;
case '>': {
if(matchchar(self, '='))
add_token(self, TK_GE);
else if(matchchar(self, '>'))
add_token_2(self, '=', TK_RSHIFT, TK_IRSHIFT);
else
add_token(self, TK_GT);
return NULL;
}
case '<': {
if(matchchar(self, '='))
add_token(self, TK_LE);
else if(matchchar(self, '<'))
add_token_2(self, '=', TK_LSHIFT, TK_ILSHIFT);
else
add_token(self, TK_LT);
return NULL;
}
case '-': {
if(matchchar(self, '='))
add_token(self, TK_ISUB);
else if(matchchar(self, '>'))
add_token(self, TK_ARROW);
else
add_token(self, TK_SUB);
return NULL;
}
case '!':
if(matchchar(self, '=')){
add_token(self, TK_NE);
}else{
Error* err = SyntaxError("expected '=' after '!'");
if(err) return err;
}
break;
case '*':
if(matchchar(self, '*')) {
add_token(self, TK_POW); // '**'
} else {
add_token_2(self, '=', TK_MUL, TK_IMUL);
}
return NULL;
case '/':
if(matchchar(self, '/')) {
add_token_2(self, '=', TK_FLOORDIV, TK_IFLOORDIV);
} else {
add_token_2(self, '=', TK_DIV, TK_IDIV);
}
return NULL;
case ' ':
case '\t': eat_spaces(self); break;
case '\n': {
add_token(self, TK_EOL);
if(!eat_indentation(self)){
return SyntaxError("unindent does not match any outer indentation level");
}
return NULL;
}
default: {
if(c == 'f') {
if(matchchar(self, '\'')) return eat_string(self, '\'', F_STRING);
if(matchchar(self, '"')) return eat_string(self, '"', F_STRING);
} else if(c == 'r') {
if(matchchar(self, '\'')) return eat_string(self, '\'', RAW_STRING);
if(matchchar(self, '"')) return eat_string(self, '"', RAW_STRING);
} else if(c == 'b') {
if(matchchar(self, '\'')) return eat_string(self, '\'', NORMAL_BYTES);
if(matchchar(self, '"')) return eat_string(self, '"', NORMAL_BYTES);
}
if(c >= '0' && c <= '9') return eat_number(self);
return eat_name(self);
}
}
}
self->token_start = self->curr_char;
while(self->indents.count > 1) {
c11_vector__pop(int, &self->indents);
add_token(self, TK_DEDENT);
return NULL;
}
add_token(self, TK_EOF);
*eof = true;
return NULL;
}
static Error* from_precompiled(pk_Lexer* self) {
pk_TokenDeserializer deserializer;
pk_TokenDeserializer__ctor(&deserializer, py_Str__data(&self->src->source));
deserializer.curr += 5; // skip "pkpy:"
c11_string version = pk_TokenDeserializer__read_string(&deserializer, '\n');
if(c11_string__cmp3(version, PK_VERSION) != 0) {
return SyntaxError("precompiled version mismatch");
}
if(pk_TokenDeserializer__read_uint(&deserializer, '\n') != (int64_t)self->src->mode){
return SyntaxError("precompiled mode mismatch");
}
int count = pk_TokenDeserializer__read_count(&deserializer);
c11_vector* precompiled_tokens = &self->src->_precompiled_tokens;
for(int i = 0; i < count; i++) {
c11_string item = pk_TokenDeserializer__read_string(&deserializer, '\n');
py_Str copied_item;
py_Str__ctor2(&copied_item, item.data, item.size);
c11_vector__push(py_Str, precompiled_tokens, copied_item);
}
count = pk_TokenDeserializer__read_count(&deserializer);
for(int i = 0; i < count; i++) {
Token t;
t.type = (TokenIndex)pk_TokenDeserializer__read_uint(&deserializer, ',');
if(is_raw_string_used(t.type)) {
int64_t index = pk_TokenDeserializer__read_uint(&deserializer, ',');
py_Str* p = c11__at(py_Str, precompiled_tokens, index);
t.start = py_Str__data(p);
t.length = c11__getitem(py_Str, precompiled_tokens, index).size;
} else {
t.start = NULL;
t.length = 0;
}
if(pk_TokenDeserializer__match_char(&deserializer, ',')) {
t.line = c11_vector__back(Token, &self->nexts).line;
} else {
t.line = (int)pk_TokenDeserializer__read_uint(&deserializer, ',');
}
if(pk_TokenDeserializer__match_char(&deserializer, ',')) {
t.brackets_level = c11_vector__back(Token, &self->nexts).brackets_level;
} else {
t.brackets_level = (int)pk_TokenDeserializer__read_uint(&deserializer, ',');
}
char type = (*deserializer.curr++); // read_char
switch(type) {
case 'I': {
int64_t res = pk_TokenDeserializer__read_uint(&deserializer, '\n');
t.value = (TokenValue){0, ._i64 = res};
} break;
case 'F': {
double res = pk_TokenDeserializer__read_float(&deserializer, '\n');
t.value = (TokenValue){1, ._f64 = res};
} break;
case 'S': {
py_Str res = pk_TokenDeserializer__read_string_from_hex(&deserializer, '\n');
t.value = (TokenValue){2, ._str = res};
} break;
default:
t.value = EmptyTokenValue;
break;
}
c11_vector__push(Token, &self->nexts, t);
}
return NULL;
}
IntParsingResult parse_uint(c11_string text, int64_t* out, int base) {
*out = 0;
c11_string prefix = {.data = text.data, .size = PK_MIN(2, text.size)};
if(base == -1) {
if(c11_string__cmp3(prefix, "0b") == 0)
base = 2;
else if(c11_string__cmp3(prefix, "0o") == 0)
base = 8;
else if(c11_string__cmp3(prefix, "0x") == 0)
base = 16;
else
base = 10;
}
if(base == 10) {
// 10-base 12334
if(text.size == 0) return IntParsing_FAILURE;
for(int i = 0; i < text.size; i++) {
char c = text.data[i];
if(c >= '0' && c <= '9') {
*out = (*out * 10) + (c - '0');
} else {
return IntParsing_FAILURE;
}
}
// "9223372036854775807".__len__() == 19
if(text.size > 19) return IntParsing_OVERFLOW;
return IntParsing_SUCCESS;
} else if(base == 2) {
// 2-base 0b101010
if(c11_string__cmp3(prefix, "0b") == 0) {
// text.remove_prefix(2);
text = (c11_string){text.data + 2, text.size - 2};
}
if(text.size == 0) return IntParsing_FAILURE;
for(int i = 0; i < text.size; i++) {
char c = text.data[i];
if(c == '0' || c == '1') {
*out = (*out << 1) | (c - '0');
} else {
return IntParsing_FAILURE;
}
}
// "111111111111111111111111111111111111111111111111111111111111111".__len__() == 63
if(text.size > 63) return IntParsing_OVERFLOW;
return IntParsing_SUCCESS;
} else if(base == 8) {
// 8-base 0o123
if(c11_string__cmp3(prefix, "0o") == 0) {
// text.remove_prefix(2);
text = (c11_string){text.data + 2, text.size - 2};
}
if(text.size == 0) return IntParsing_FAILURE;
for(int i = 0; i < text.size; i++) {
char c = text.data[i];
if(c >= '0' && c <= '7') {
*out = (*out << 3) | (c - '0');
} else {
return IntParsing_FAILURE;
}
}
// "777777777777777777777".__len__() == 21
if(text.size > 21) return IntParsing_OVERFLOW;
return IntParsing_SUCCESS;
} else if(base == 16) {
// 16-base 0x123
if(c11_string__cmp3(prefix, "0x") == 0) {
// text.remove_prefix(2);
text = (c11_string){text.data + 2, text.size - 2};
}
if(text.size == 0) return IntParsing_FAILURE;
for(int i = 0; i < text.size; i++) {
char c = text.data[i];
if(c >= '0' && c <= '9') {
*out = (*out << 4) | (c - '0');
} else if(c >= 'a' && c <= 'f') {
*out = (*out << 4) | (c - 'a' + 10);
} else if(c >= 'A' && c <= 'F') {
*out = (*out << 4) | (c - 'A' + 10);
} else {
return IntParsing_FAILURE;
}
}
// "7fffffffffffffff".__len__() == 16
if(text.size > 16) return IntParsing_OVERFLOW;
return IntParsing_SUCCESS;
}
return IntParsing_FAILURE;
}
Error* pk_Lexer__run(pk_SourceData_ src, c11_array* out_tokens){
pk_Lexer lexer;
pk_Lexer__ctor(&lexer, src);
if(src->is_precompiled) {
Error* err = from_precompiled(&lexer);
// set out tokens
pk_Lexer__dtor(&lexer);
return err;
}
// push initial tokens
Token sof = {TK_SOF, lexer.token_start, 0, lexer.current_line, lexer.brackets_level, EmptyTokenValue};
c11_vector__push(Token, &lexer.nexts, sof);
c11_vector__push(int, &lexer.indents, 0);
bool eof = false;
while(!eof) {
void* err = lex_one_token(&lexer, &eof);
if(err){
pk_Lexer__dtor(&lexer);
return err;
}
}
// set out_tokens
pk_Lexer__dtor(&lexer);
return NULL;
}
Error* pk_Lexer__precompile(pk_SourceData_ src, py_Str* out) {
assert(!src->is_precompiled);
c11_array/*T=Token*/ nexts; // output tokens
Error* err = pk_Lexer__run(src, &nexts);
if(err) return err;
pk_SStream ss;
pk_SStream__ctor(&ss);
// L1: version string
pk_SStream__write_cstr(&ss, "pkpy:" PK_VERSION "\n");
// L2: mode
pk_SStream__write_int(&ss, (int)src->mode);
pk_SStream__write_char(&ss, '\n');
c11_smallmap_s2n token_indices;
c11_smallmap_s2n__ctor(&token_indices);
c11_vector__foreach(Token, &nexts, token) {
if(is_raw_string_used(token->type)) {
c11_string token_sv = {token->start, token->length};
if(!c11_smallmap_s2n__contains(&token_indices, token_sv)) {
c11_smallmap_s2n__set(&token_indices, token_sv, 0);
}
}
}
// L3: raw string count
pk_SStream__write_char(&ss, '=');
pk_SStream__write_int(&ss, token_indices.count);
pk_SStream__write_char(&ss, '\n');
uint16_t index = 0;
for(int i=0; i<token_indices.count; i++){
c11_smallmap_s2n_KV* kv = c11__at(c11_smallmap_s2n_KV, &token_indices, i);
// L4: raw strings
pk_SStream__write_cstrn(&ss, kv->key.data, kv->key.size);
kv->value = index++;
}
// L5: token count
pk_SStream__write_char(&ss, '=');
pk_SStream__write_int(&ss, nexts.count);
pk_SStream__write_char(&ss, '\n');
for(int i = 0; i < nexts.count; i++) {
const Token* token = c11__at(Token, &nexts, i);
pk_SStream__write_int(&ss, (int)token->type);
pk_SStream__write_char(&ss, ',');
if(is_raw_string_used(token->type)) {
uint16_t *p = c11_smallmap_s2n__try_get(
&token_indices, (c11_string){token->start, token->length});
assert(p != NULL);
pk_SStream__write_int(&ss, (int)*p);
pk_SStream__write_char(&ss, ',');
}
if(i > 0 && c11__getitem(Token, &nexts, i-1).line == token->line){
pk_SStream__write_char(&ss, ',');
}else{
pk_SStream__write_int(&ss, token->line);
pk_SStream__write_char(&ss, ',');
}
if(i > 0 && c11__getitem(Token, &nexts, i-1).brackets_level == token->brackets_level){
pk_SStream__write_char(&ss, ',');
}else{
pk_SStream__write_int(&ss, token->brackets_level);
pk_SStream__write_char(&ss, ',');
}
// visit token value
switch(token->value.index){
case 0: break;
case 1:
pk_SStream__write_char(&ss, 'I');
pk_SStream__write_int(&ss, token->value._i64);
break;
case 2:
pk_SStream__write_char(&ss, 'F');
pk_SStream__write_float(&ss, token->value._f64, -1);
break;
case 3: {
pk_SStream__write_char(&ss, 'S');
c11_string sv = py_Str__sv(&token->value._str);
for(int i=0; i<sv.size; i++){
pk_SStream__write_hex(&ss, sv.data[i], false);
}
break;
}
}
pk_SStream__write_char(&ss, '\n');
}
*out = pk_SStream__submit(&ss);
c11_smallmap_s2n__dtor(&token_indices);
return NULL;
}