diff --git a/include/pocketpy/compiler.h b/include/pocketpy/compiler.h index 0b6d1134..4065c4c4 100644 --- a/include/pocketpy/compiler.h +++ b/include/pocketpy/compiler.h @@ -121,6 +121,7 @@ class Compiler { public: Compiler(VM* vm, std::string_view source, const Str& filename, CompileMode mode, bool unknown_global_scope=false); + void precompile(); CodeObject_ compile(); }; diff --git a/include/pocketpy/lexer.h b/include/pocketpy/lexer.h index f5428346..29c2b676 100644 --- a/include/pocketpy/lexer.h +++ b/include/pocketpy/lexer.h @@ -59,14 +59,6 @@ struct Token{ Str str() const { return Str(start, length);} std::string_view sv() const { return std::string_view(start, length);} - - // Str info() const { - // SStream ss; - // ss << line << ": " << TK_STR(type) << " '" << ( - // sv()=="\n" ? "\\n" : sv() - // ) << "'"; - // return ss.str(); - // } }; // https://docs.python.org/3/reference/expressions.html#operator-precedence diff --git a/src/compiler.cpp b/src/compiler.cpp index 92cff48e..c6200ac2 100644 --- a/src/compiler.cpp +++ b/src/compiler.cpp @@ -1225,6 +1225,35 @@ __EAT_DOTS_END: init_pratt_rules(); } + void Compiler::precompile(){ + SStream ss; + ss << PK_VERSION << '\n'; // L1: version string + ss << lexer.src->filename << '\n'; // L2: filename + ss << mode() << '\n'; // L3: compile mode + ss << (int)unknown_global_scope << '\n'; // L4: unknown global scope + auto tokens = lexer.run(); + ss << '=' << (int)tokens.size() << '\n'; // L5: token count + for(auto token: lexer.run()){ + ss << (int)token.type << '\n'; + int offset = token.start - lexer.src->source.c_str(); + ss << offset << '\n'; + ss << token.length << '\n'; + ss << token.line << '\n'; + ss << token.brackets_level << '\n'; + // visit token value + std::visit([&ss](auto&& arg){ + using T = std::decay_t; + if constexpr(std::is_same_v){ + ss << 'i' << arg << '\n'; + }else if constexpr(std::is_same_v){ + ss << 'f' << arg << '\n'; + }else if constexpr(std::is_same_v){ + ss << 's' << arg.escape() << '\n'; + } + }, token.value); + } + std::cout << ss.str() << std::endl; + } CodeObject_ Compiler::compile(){ PK_ASSERT(i == 0) // make sure it is the first time to compile diff --git a/src/lexer.cpp b/src/lexer.cpp index 4ecf9629..64384827 100644 --- a/src/lexer.cpp +++ b/src/lexer.cpp @@ -486,6 +486,7 @@ static bool is_unicode_Lo_char(uint32_t c) { } std::vector Lexer::run() { + PK_ASSERT(curr_char == src->source.c_str()); while (lex_one_token()); return std::move(nexts); }