From 31bf7f45c4583f2e513473ef28c59e0d1c69127f Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Mon, 17 Jun 2024 00:42:47 +0800 Subject: [PATCH] remove source data --- include/pocketpy/common/refcount.h | 18 ++++++++++++++ include/pocketpy/compiler/lexer.hpp | 18 ++++++++++---- include/pocketpy/objects/codeobject.hpp | 11 +++++---- include/pocketpy/objects/error.hpp | 30 ++++++++++++++++++----- include/pocketpy/objects/sourcedata.h | 7 +++++- include/pocketpy/objects/sourcedata.hpp | 31 ------------------------ scripts/run_tests.py | 10 ++++++-- src/common/sourcedata.c | 8 +++++++ src/compiler/compiler.cpp | 2 +- src/compiler/lexer.cpp | 32 +++++++++++-------------- src/objects/error.cpp | 4 ---- src/pocketpy.cpp | 7 +++++- 12 files changed, 106 insertions(+), 72 deletions(-) create mode 100644 include/pocketpy/common/refcount.h delete mode 100644 include/pocketpy/objects/sourcedata.hpp diff --git a/include/pocketpy/common/refcount.h b/include/pocketpy/common/refcount.h new file mode 100644 index 00000000..15780bf8 --- /dev/null +++ b/include/pocketpy/common/refcount.h @@ -0,0 +1,18 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +// ref counting +typedef struct RefCounted { + int count; + void (*dtor)(void*); +} RefCounted; + +#define PK_INCREF(obj) (obj)->rc.count++ +#define PK_DECREF(obj) if (--(obj)->rc.count == 0) (obj)->rc.dtor(obj) + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/include/pocketpy/compiler/lexer.hpp b/include/pocketpy/compiler/lexer.hpp index 96ff80f5..0721624b 100644 --- a/include/pocketpy/compiler/lexer.hpp +++ b/include/pocketpy/compiler/lexer.hpp @@ -1,7 +1,7 @@ #pragma once -#include "pocketpy/objects/sourcedata.hpp" #include "pocketpy/objects/error.hpp" +#include "pocketpy/objects/sourcedata.h" #include @@ -95,8 +95,10 @@ enum Precedence { enum class StringType { NORMAL_STRING, RAW_STRING, F_STRING, NORMAL_BYTES }; struct Lexer { + PK_ALWAYS_PASS_BY_POINTER(Lexer) + VM* vm; - std::shared_ptr src; + pkpy_SourceData_ src; const char* token_start; const char* curr_char; int current_line = 1; @@ -131,11 +133,19 @@ struct Lexer { [[nodiscard]] Error* IndentationError(const char* msg) noexcept { return _error(true, "IndentationError", msg, NULL); } [[nodiscard]] Error* NeedMoreLines() noexcept { return _error(true, "NeedMoreLines", "", NULL, 0); } - Lexer(VM* vm, std::shared_ptr src) noexcept; - [[nodiscard]] Error* run() noexcept; [[nodiscard]] Error* from_precompiled() noexcept; [[nodiscard]] Error* precompile(Str* out) noexcept; + + Lexer(VM* vm, std::string_view source, const Str& filename, CompileMode mode) noexcept{ + src = pkpy_SourceData__rcnew({source.data(), (int)source.size()}, &filename, mode); + this->token_start = pkpy_Str__data(&src->source); + this->curr_char = pkpy_Str__data(&src->source); + } + + ~Lexer(){ + PK_DECREF(src); + } }; enum class IntParsingResult { diff --git a/include/pocketpy/objects/codeobject.hpp b/include/pocketpy/objects/codeobject.hpp index 7b850542..56a03882 100644 --- a/include/pocketpy/objects/codeobject.hpp +++ b/include/pocketpy/objects/codeobject.hpp @@ -4,7 +4,7 @@ #include "pocketpy/common/traits.hpp" #include "pocketpy/objects/tuplelist.hpp" #include "pocketpy/objects/namedict.hpp" -#include "pocketpy/objects/sourcedata.hpp" +#include "pocketpy/objects/sourcedata.h" #include "pocketpy/common/smallmap.h" namespace pkpy { @@ -71,13 +71,15 @@ using CodeObject_ = std::shared_ptr; using FuncDecl_ = std::shared_ptr; struct CodeObject { + PK_ALWAYS_PASS_BY_POINTER(CodeObject) + struct LineInfo { int lineno; // line number for each bytecode bool is_virtual; // whether this bytecode is virtual (not in source code) int iblock; // block index }; - std::shared_ptr src; + pkpy_SourceData_ src; Str name; vector codes; @@ -99,17 +101,18 @@ struct CodeObject { void _gc_mark(VM*) const; - CodeObject(std::shared_ptr src, const Str& name) : + CodeObject(pkpy_SourceData_ src, const Str& name) : src(src), name(name), nlocals(0), start_line(-1), end_line(-1) { blocks.push_back(CodeBlock(CodeBlockType::NO_BLOCK, -1, 0)); - c11_smallmap_n2i__ctor(&varnames_inv); c11_smallmap_n2i__ctor(&labels); + PK_INCREF(src); } ~CodeObject() { c11_smallmap_n2i__dtor(&varnames_inv); c11_smallmap_n2i__dtor(&labels); + PK_DECREF(src); } }; diff --git a/include/pocketpy/objects/error.hpp b/include/pocketpy/objects/error.hpp index a125329d..14ee6169 100644 --- a/include/pocketpy/objects/error.hpp +++ b/include/pocketpy/objects/error.hpp @@ -1,7 +1,7 @@ #pragma once #include "pocketpy/common/str.hpp" -#include "pocketpy/objects/sourcedata.hpp" +#include "pocketpy/objects/sourcedata.h" namespace pkpy { @@ -33,15 +33,33 @@ struct Exception { PyObject* _self; // weak reference struct Frame { - std::shared_ptr src; // weak ref + pkpy_SourceData_ src; int lineno; const char* cursor; std::string name; - Str snapshot() const { return src->snapshot(lineno, cursor, name); } + Str snapshot() const { + return pkpy_SourceData__snapshot(src, lineno, cursor, name.empty() ? nullptr : name.c_str()); + } - Frame(std::shared_ptr src, int lineno, const char* cursor, std::string_view name) : - src(src), lineno(lineno), cursor(cursor), name(name) {} + Frame(pkpy_SourceData_ src, int lineno, const char* cursor, std::string_view name) : + src(src), lineno(lineno), cursor(cursor), name(name) { + PK_INCREF(src); + } + // disable copy + Frame(const Frame&) = delete; + // allow move + Frame(Frame&& other) noexcept{ + src = other.src; + lineno = other.lineno; + cursor = other.cursor; + name = std::move(other.name); + other.src = nullptr; + } + + ~Frame() { + if(src) PK_DECREF(src); + } }; vector stacktrace; @@ -79,7 +97,7 @@ struct TopLevelException : std::exception { struct Error{ const char* type; - std::shared_ptr src; + pkpy_SourceData_ src; int lineno; const char* cursor; char msg[100]; diff --git a/include/pocketpy/objects/sourcedata.h b/include/pocketpy/objects/sourcedata.h index ac0cf888..b524b48f 100644 --- a/include/pocketpy/objects/sourcedata.h +++ b/include/pocketpy/objects/sourcedata.h @@ -3,6 +3,7 @@ #include #include "pocketpy/common/str.h" #include "pocketpy/common/vector.h" +#include "pocketpy/common/refcount.h" #ifdef __cplusplus extern "C" { @@ -11,6 +12,7 @@ extern "C" { enum CompileMode { EXEC_MODE, EVAL_MODE, REPL_MODE, JSON_MODE, CELL_MODE }; struct pkpy_SourceData { + RefCounted rc; enum CompileMode mode; bool is_precompiled; @@ -21,10 +23,13 @@ struct pkpy_SourceData { c11_vector/*T=pkpy_Str*/ _precompiled_tokens; }; +typedef struct pkpy_SourceData* pkpy_SourceData_; + +pkpy_SourceData_ pkpy_SourceData__rcnew(c11_string source, const pkpy_Str *filename, enum CompileMode mode); void pkpy_SourceData__ctor(struct pkpy_SourceData *self, c11_string source, const pkpy_Str *filename, enum CompileMode mode); void pkpy_SourceData__dtor(struct pkpy_SourceData* self); -bool pkpy_SourceData__get_line(const struct pkpy_SourceData *self, int lineno, const char **st, const char **ed); +bool pkpy_SourceData__get_line(const struct pkpy_SourceData* self, int lineno, const char** st, const char** ed); pkpy_Str pkpy_SourceData__snapshot(const struct pkpy_SourceData *self, int lineno, const char *cursor, const char *name); #ifdef __cplusplus diff --git a/include/pocketpy/objects/sourcedata.hpp b/include/pocketpy/objects/sourcedata.hpp deleted file mode 100644 index f4ce2908..00000000 --- a/include/pocketpy/objects/sourcedata.hpp +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include "pocketpy/common/utils.h" -#include "pocketpy/common/str.hpp" -#include "pocketpy/objects/sourcedata.h" - -namespace pkpy { - -struct SourceData : public pkpy_SourceData { - SourceData(std::string_view source, const Str& filename, CompileMode mode) { - pkpy_SourceData__ctor(this, {source.data(), (int)source.size()}, &filename, mode); - } - - ~SourceData() { - pkpy_SourceData__dtor(this); - } - - std::string_view get_line(int lineno) const { - const char *st, *ed; - if (pkpy_SourceData__get_line(this, lineno, &st, &ed)) { - return std::string_view(st, ed - st); - } - return ""; - } - - Str snapshot(int lineno, const char* cursor, std::string_view name) const { - return pkpy_SourceData__snapshot(this, lineno, cursor, name.empty() ? nullptr : name.data()); - } -}; - -} // namespace pkpy diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 533ecfa9..e0e29b11 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -65,8 +65,14 @@ print('ans_2:', A('abc').get()) exit() ''', capture_output=True, check=True) res.check_returncode() - assert 'ans_1: 3' in res.stdout, res.stdout - assert 'ans_2: abc' in res.stdout, res.stdout + # assert 'ans_1: 3' in res.stdout, res.stdout + if 'ans_1: 3' not in res.stdout: + print(res.stdout) + exit(1) + # assert 'ans_2: abc' in res.stdout, res.stdout + if 'ans_2: abc' not in res.stdout: + print(res.stdout) + exit(1) if len(sys.argv) == 2: diff --git a/src/common/sourcedata.c b/src/common/sourcedata.c index 66ba607d..a6d9536f 100644 --- a/src/common/sourcedata.c +++ b/src/common/sourcedata.c @@ -4,6 +4,14 @@ #include #include +pkpy_SourceData_ pkpy_SourceData__rcnew(c11_string source, const pkpy_Str* filename, enum CompileMode mode) { + pkpy_SourceData_ self = malloc(sizeof(struct pkpy_SourceData)); + pkpy_SourceData__ctor(self, source, filename, mode); + self->rc.count = 1; + self->rc.dtor = (void(*)(void*))pkpy_SourceData__dtor; + return self; +} + void pkpy_SourceData__ctor(struct pkpy_SourceData* self, c11_string source, // may not be null-terminated const pkpy_Str* filename, diff --git a/src/compiler/compiler.cpp b/src/compiler/compiler.cpp index 78a231e7..26d0ebb5 100644 --- a/src/compiler/compiler.cpp +++ b/src/compiler/compiler.cpp @@ -1281,7 +1281,7 @@ Error* Compiler::read_literal(PyVar* out) noexcept{ } Compiler::Compiler(VM* vm, std::string_view source, const Str& filename, CompileMode mode, bool unknown_global_scope) noexcept: - lexer(vm, std::make_shared(source, filename, mode)){ + lexer(vm, source, filename, mode){ this->vm = vm; this->unknown_global_scope = unknown_global_scope; init_pratt_rules(); diff --git a/src/compiler/lexer.cpp b/src/compiler/lexer.cpp index 8eb2ee86..80f03fb1 100644 --- a/src/compiler/lexer.cpp +++ b/src/compiler/lexer.cpp @@ -488,27 +488,28 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{ } Error* Lexer::_error(bool lexer_err, const char* type, const char* msg, va_list* args, i64 userdata) noexcept{ - PK_THREAD_LOCAL Error err; - err.type = type; - err.src = src; + Error* err = (Error*)malloc(sizeof(Error)); + err->type = type; + err->src = src; + PK_INCREF(src); if(lexer_err){ - err.lineno = current_line; - err.cursor = curr_char; + err->lineno = current_line; + err->cursor = curr_char; if(*curr_char == '\n') { - err.lineno--; - err.cursor--; + err->lineno--; + err->cursor--; } }else{ - err.lineno = -1; - err.cursor = NULL; + err->lineno = -1; + err->cursor = NULL; } if(args){ - vsnprintf(err.msg, sizeof(err.msg), msg, *args); + vsnprintf(err->msg, sizeof(err->msg), msg, *args); }else{ - std::strncpy(err.msg, msg, sizeof(err.msg)); + std::strncpy(err->msg, msg, sizeof(err->msg)); } - err.userdata = userdata; - return &err; + err->userdata = userdata; + return err; } Error* Lexer::SyntaxError(const char* fmt, ...) noexcept{ @@ -519,11 +520,6 @@ Error* Lexer::SyntaxError(const char* fmt, ...) noexcept{ return err; } -Lexer::Lexer(VM* vm, std::shared_ptr src) noexcept : vm(vm), src(src){ - this->token_start = pkpy_Str__data(&src->source); - this->curr_char = pkpy_Str__data(&src->source); -} - Error* Lexer::run() noexcept{ assert(!this->used); this->used = true; diff --git a/src/objects/error.cpp b/src/objects/error.cpp index bc80da92..f7fe8a86 100644 --- a/src/objects/error.cpp +++ b/src/objects/error.cpp @@ -4,10 +4,6 @@ namespace pkpy { Str Exception::summary() const { SStream ss; if(is_re) ss << "Traceback (most recent call last):\n"; - // while(!st.empty()) { - // ss << st.top().snapshot() << '\n'; - // st.pop(); - // } for(int i = stacktrace.size() - 1; i >= 0; i--) { ss << stacktrace[i].snapshot() << '\n'; } diff --git a/src/pocketpy.cpp b/src/pocketpy.cpp index 69c5447d..e16cae27 100644 --- a/src/pocketpy.cpp +++ b/src/pocketpy.cpp @@ -1733,7 +1733,10 @@ CodeObject_ VM::compile(std::string_view source, const Str& filename, CompileMod void VM::__compile_error(Error* err){ assert(err != nullptr); if(err->type == std::string_view("NeedMoreLines")){ - throw NeedMoreLines((bool)err->userdata); + bool arg = (bool)err->userdata; + PK_DECREF(err->src); + std::free(err); + throw NeedMoreLines(arg); } __last_exception = vm->call( vm->builtins->attr(err->type), @@ -1741,6 +1744,8 @@ void VM::__compile_error(Error* err){ ).get(); Exception& e = __last_exception->as(); e.st_push(err->src, err->lineno, err->cursor, ""); + PK_DECREF(err->src); + std::free(err); _error(__last_exception); }