From c07bf67714f121ca1c9216e6bd67441b5b4885df Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sun, 14 Apr 2024 01:17:17 +0800 Subject: [PATCH] some fix --- include/pocketpy/error.h | 2 ++ include/pocketpy/lexer.h | 4 ++++ src/compiler.cpp | 47 +++++++++++++++++++++++++++++++++++----- tests/94_compile.py | 2 ++ 4 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 tests/94_compile.py diff --git a/include/pocketpy/error.h b/include/pocketpy/error.h index 00ebf97b..ff2f4fe1 100644 --- a/include/pocketpy/error.h +++ b/include/pocketpy/error.h @@ -31,7 +31,9 @@ struct SourceData { Str source; pod_vector line_starts; + bool is_precompiled; + Str _precompiled_tokens; SourceData(std::string_view source, const Str& filename, CompileMode mode); SourceData(const Str& filename, CompileMode mode); diff --git a/include/pocketpy/lexer.h b/include/pocketpy/lexer.h index 29c2b676..8cd51fbd 100644 --- a/include/pocketpy/lexer.h +++ b/include/pocketpy/lexer.h @@ -42,6 +42,10 @@ constexpr TokenIndex TK(const char token[]) { return 255; } +inline constexpr bool is_raw_string_used(TokenIndex t){ + return t == TK("@id") || t == TK("@long"); +} + #define TK_STR(t) kTokens[t] const std::map kTokenKwMap = [](){ std::map map; diff --git a/src/compiler.cpp b/src/compiler.cpp index 0b29e4b6..b2927219 100644 --- a/src/compiler.cpp +++ b/src/compiler.cpp @@ -1144,7 +1144,7 @@ __EAT_DOTS_END: } if(match(TK("->"))) consume_type_hints(); const char* _end = curr().start; - decl->signature = Str(_start, _end-_start); + if(_start && _end) decl->signature = Str(_start, _end-_start); compile_block_body(); pop_context(); @@ -1229,9 +1229,31 @@ __EAT_DOTS_END: auto tokens = lexer.run(); SStream ss; ss << "pkpy:" PK_VERSION << '\n'; // L1: version string - ss << "=" << (int)tokens.size() << '\n'; // L5: token count + ss << (int)mode() << '\n'; // L2: mode + + SStream token_ss; // no '\n' in token_ss + token_ss << '|'; + std::map token_offsets; + for(auto token: tokens){ + if(is_raw_string_used(token.type)){ + auto it = token_offsets.find(token.sv()); + if(it == token_offsets.end()){ + token_offsets[token.sv()] = token_ss.buffer.size(); + // assert no '\n' in token.sv() + for(char c: token.sv()) if(c=='\n') PK_FATAL_ERROR(); + token_ss << token.sv() << '|'; + } + } + } + ss << token_ss.str() << '\n'; // L3: raw string + + ss << "=" << (int)tokens.size() << '\n'; // L4: token count for(auto token: tokens){ ss << (int)token.type << ','; + if(is_raw_string_used(token.type)){ + ss << token_offsets[token.sv()] << ','; // offset + ss << token.sv().size() << ','; // length + } ss << token.line << ','; ss << token.brackets_level << ','; // visit token value @@ -1255,15 +1277,28 @@ __EAT_DOTS_END: TokenDeserializer deserializer(source); deserializer.curr += 5; // skip "pkpy:" std::string_view version = deserializer.read_string('\n'); - if(version != PK_VERSION) SyntaxError(_S("precompiled version mismatch: ", version, "!=" PK_VERSION)); + + if(version != PK_VERSION){ + SyntaxError(_S("precompiled version mismatch: ", version, "!=" PK_VERSION)); + } + if(deserializer.read_int('\n') != (i64)mode()){ + SyntaxError("precompiled mode mismatch"); + } + + lexer.src->_precompiled_tokens = deserializer.read_string('\n'); deserializer.curr += 1; // skip '=' i64 count = deserializer.read_int('\n'); - const char* null_start = lexer.src->source.c_str(); + const char* tokens_c_str = lexer.src->_precompiled_tokens.c_str(); for(int i=0; i", "eval") +assert eval(code) == 3