remove source data

include/pocketpy/common/refcount.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// ref counting
+typedef struct RefCounted {
+    int count;
+    void (*dtor)(void*);
+} RefCounted;
+
+#define PK_INCREF(obj) (obj)->rc.count++
+#define PK_DECREF(obj) if (--(obj)->rc.count == 0) (obj)->rc.dtor(obj)
+
+#ifdef __cplusplus
+}
+#endif

include/pocketpy/compiler/lexer.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "pocketpy/objects/sourcedata.hpp"
 #include "pocketpy/objects/error.hpp"
+#include "pocketpy/objects/sourcedata.h"
 
 #include <variant>
 
@@ -95,8 +95,10 @@ enum Precedence {
 enum class StringType { NORMAL_STRING, RAW_STRING, F_STRING, NORMAL_BYTES };
 
 struct Lexer {
+    PK_ALWAYS_PASS_BY_POINTER(Lexer)
+
     VM* vm;
-    std::shared_ptr<SourceData> src;
+    pkpy_SourceData_ src;
     const char* token_start;
     const char* curr_char;
     int current_line = 1;
@@ -131,11 +133,19 @@ struct Lexer {
     [[nodiscard]] Error* IndentationError(const char* msg) noexcept { return _error(true, "IndentationError", msg, NULL); }
     [[nodiscard]] Error* NeedMoreLines() noexcept { return _error(true, "NeedMoreLines", "", NULL, 0); }
 
-    Lexer(VM* vm, std::shared_ptr<SourceData> src) noexcept;
-    
     [[nodiscard]] Error* run() noexcept;
     [[nodiscard]] Error* from_precompiled() noexcept;
     [[nodiscard]] Error* precompile(Str* out) noexcept;
+
+    Lexer(VM* vm, std::string_view source, const Str& filename, CompileMode mode) noexcept{
+        src = pkpy_SourceData__rcnew({source.data(), (int)source.size()}, &filename, mode);
+        this->token_start = pkpy_Str__data(&src->source);
+        this->curr_char = pkpy_Str__data(&src->source);
+    }
+
+    ~Lexer(){
+        PK_DECREF(src);
+    }
 };
 
 enum class IntParsingResult {

include/pocketpy/objects/codeobject.hpp

@@ -4,7 +4,7 @@
 #include "pocketpy/common/traits.hpp"
 #include "pocketpy/objects/tuplelist.hpp"
 #include "pocketpy/objects/namedict.hpp"
-#include "pocketpy/objects/sourcedata.hpp"
+#include "pocketpy/objects/sourcedata.h"
 #include "pocketpy/common/smallmap.h"
 
 namespace pkpy {
@@ -71,13 +71,15 @@ using CodeObject_ = std::shared_ptr<CodeObject>;
 using FuncDecl_ = std::shared_ptr<FuncDecl>;
 
 struct CodeObject {
+    PK_ALWAYS_PASS_BY_POINTER(CodeObject)
+    
     struct LineInfo {
         int lineno;       // line number for each bytecode
         bool is_virtual;  // whether this bytecode is virtual (not in source code)
         int iblock;       // block index
     };
 
-    std::shared_ptr<SourceData> src;
+    pkpy_SourceData_ src;
     Str name;
 
     vector<Bytecode> codes;
@@ -99,17 +101,18 @@ struct CodeObject {
 
     void _gc_mark(VM*) const;
 
-    CodeObject(std::shared_ptr<SourceData> src, const Str& name) :
+    CodeObject(pkpy_SourceData_ src, const Str& name) :
         src(src), name(name), nlocals(0), start_line(-1), end_line(-1) {
         blocks.push_back(CodeBlock(CodeBlockType::NO_BLOCK, -1, 0));
-
         c11_smallmap_n2i__ctor(&varnames_inv);
         c11_smallmap_n2i__ctor(&labels);
+        PK_INCREF(src);
     }
 
     ~CodeObject() {
         c11_smallmap_n2i__dtor(&varnames_inv);
         c11_smallmap_n2i__dtor(&labels);
+        PK_DECREF(src);
     }
 };
 

include/pocketpy/objects/error.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "pocketpy/common/str.hpp"
-#include "pocketpy/objects/sourcedata.hpp"
+#include "pocketpy/objects/sourcedata.h"
 
 namespace pkpy {
 
@@ -33,15 +33,33 @@ struct Exception {
     PyObject* _self;  // weak reference
 
     struct Frame {
-        std::shared_ptr<SourceData> src;        // weak ref
+        pkpy_SourceData_ src;
         int lineno;
         const char* cursor;
         std::string name;
 
-        Str snapshot() const { return src->snapshot(lineno, cursor, name); }
+        Str snapshot() const {
+            return pkpy_SourceData__snapshot(src, lineno, cursor, name.empty() ? nullptr : name.c_str());
+        }
 
-        Frame(std::shared_ptr<SourceData> src, int lineno, const char* cursor, std::string_view name) :
+        Frame(pkpy_SourceData_ src, int lineno, const char* cursor, std::string_view name) :
-            src(src), lineno(lineno), cursor(cursor), name(name) {}
+            src(src), lineno(lineno), cursor(cursor), name(name) {
+                PK_INCREF(src);
+            }
+        // disable copy
+        Frame(const Frame&) = delete;
+        // allow move
+        Frame(Frame&& other) noexcept{
+            src = other.src;
+            lineno = other.lineno;
+            cursor = other.cursor;
+            name = std::move(other.name);
+            other.src = nullptr;
+        }
+
+        ~Frame() {
+            if(src) PK_DECREF(src);
+        }
     };
 
     vector<Frame> stacktrace;
@@ -79,7 +97,7 @@ struct TopLevelException : std::exception {
 
 struct Error{
     const char* type;
-    std::shared_ptr<SourceData> src;
+    pkpy_SourceData_ src;
     int lineno;
     const char* cursor;
     char msg[100];

include/pocketpy/objects/sourcedata.h

@@ -3,6 +3,7 @@
 #include <stdbool.h>
 #include "pocketpy/common/str.h"
 #include "pocketpy/common/vector.h"
+#include "pocketpy/common/refcount.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -11,6 +12,7 @@ extern "C" {
 enum CompileMode { EXEC_MODE, EVAL_MODE, REPL_MODE, JSON_MODE, CELL_MODE };
 
 struct pkpy_SourceData {
+    RefCounted rc;
     enum CompileMode mode;
     bool is_precompiled;
 
@@ -21,10 +23,13 @@ struct pkpy_SourceData {
     c11_vector/*T=pkpy_Str*/ _precompiled_tokens;
 };
 
+typedef struct pkpy_SourceData* pkpy_SourceData_;
+
+pkpy_SourceData_ pkpy_SourceData__rcnew(c11_string source, const pkpy_Str *filename, enum CompileMode mode);
 void pkpy_SourceData__ctor(struct pkpy_SourceData *self, c11_string source, const pkpy_Str *filename, enum CompileMode mode);
 void pkpy_SourceData__dtor(struct pkpy_SourceData* self);
 
-bool pkpy_SourceData__get_line(const struct pkpy_SourceData *self, int lineno, const char **st, const char **ed);
+bool pkpy_SourceData__get_line(const struct pkpy_SourceData* self, int lineno, const char** st, const char** ed);
 pkpy_Str pkpy_SourceData__snapshot(const struct pkpy_SourceData *self, int lineno, const char *cursor, const char *name);
 
 #ifdef __cplusplus

include/pocketpy/objects/sourcedata.hpp

@@ -1,31 +0,0 @@
-#pragma once
-
-#include "pocketpy/common/utils.h"
-#include "pocketpy/common/str.hpp"
-#include "pocketpy/objects/sourcedata.h"
-
-namespace pkpy {
-
-struct SourceData : public pkpy_SourceData {
-    SourceData(std::string_view source, const Str& filename, CompileMode mode) {
-        pkpy_SourceData__ctor(this, {source.data(), (int)source.size()}, &filename, mode);
-    }
-
-    ~SourceData() {
-        pkpy_SourceData__dtor(this);
-    }
-
-    std::string_view get_line(int lineno) const {
-        const char *st, *ed;
-        if (pkpy_SourceData__get_line(this, lineno, &st, &ed)) {
-            return std::string_view(st, ed - st);
-        }
-        return "<?>";
-    }
-
-    Str snapshot(int lineno, const char* cursor, std::string_view name) const {
-        return pkpy_SourceData__snapshot(this, lineno, cursor, name.empty() ? nullptr : name.data());
-    }
-};
-
-}  // namespace pkpy

scripts/run_tests.py

@@ -65,8 +65,14 @@ print('ans_2:', A('abc').get())
 exit()
 ''', capture_output=True, check=True)
         res.check_returncode()
-        assert 'ans_1: 3' in res.stdout, res.stdout
+        # assert 'ans_1: 3' in res.stdout, res.stdout
-        assert 'ans_2: abc' in res.stdout, res.stdout
+        if 'ans_1: 3' not in res.stdout:
+            print(res.stdout)
+            exit(1)
+        # assert 'ans_2: abc' in res.stdout, res.stdout
+        if 'ans_2: abc' not in res.stdout:
+            print(res.stdout)
+            exit(1)
 
 
 if len(sys.argv) == 2:

src/common/sourcedata.c

@@ -4,6 +4,14 @@
 #include <stdlib.h>
 #include <string.h>
 
+pkpy_SourceData_ pkpy_SourceData__rcnew(c11_string source, const pkpy_Str* filename, enum CompileMode mode) {
+    pkpy_SourceData_ self = malloc(sizeof(struct pkpy_SourceData));
+    pkpy_SourceData__ctor(self, source, filename, mode);
+    self->rc.count = 1;
+    self->rc.dtor = (void(*)(void*))pkpy_SourceData__dtor;
+    return self;
+}
+
 void pkpy_SourceData__ctor(struct pkpy_SourceData* self,
                            c11_string source,       // may not be null-terminated
                            const pkpy_Str* filename,

src/compiler/compiler.cpp

@@ -1281,7 +1281,7 @@ Error* Compiler::read_literal(PyVar* out) noexcept{
 }
 
 Compiler::Compiler(VM* vm, std::string_view source, const Str& filename, CompileMode mode, bool unknown_global_scope) noexcept:
-    lexer(vm, std::make_shared<SourceData>(source, filename, mode)){
+    lexer(vm, source, filename, mode){
     this->vm = vm;
     this->unknown_global_scope = unknown_global_scope;
     init_pratt_rules();

src/compiler/lexer.cpp

@@ -488,27 +488,28 @@ Error* Lexer::lex_one_token(bool* eof) noexcept{
 }
 
 Error* Lexer::_error(bool lexer_err, const char* type, const char* msg, va_list* args, i64 userdata) noexcept{
-    PK_THREAD_LOCAL Error err;
+    Error* err = (Error*)malloc(sizeof(Error));
-    err.type = type;
+    err->type = type;
-    err.src = src;
+    err->src = src;
+    PK_INCREF(src);
     if(lexer_err){
-        err.lineno = current_line;
+        err->lineno = current_line;
-        err.cursor = curr_char;
+        err->cursor = curr_char;
         if(*curr_char == '\n') {
-            err.lineno--;
+            err->lineno--;
-            err.cursor--;
+            err->cursor--;
         }
     }else{
-        err.lineno = -1;
+        err->lineno = -1;
-        err.cursor = NULL;
+        err->cursor = NULL;
     }
     if(args){
-        vsnprintf(err.msg, sizeof(err.msg), msg, *args);
+        vsnprintf(err->msg, sizeof(err->msg), msg, *args);
     }else{
-        std::strncpy(err.msg, msg, sizeof(err.msg));
+        std::strncpy(err->msg, msg, sizeof(err->msg));
     }
-    err.userdata = userdata;
+    err->userdata = userdata;
-    return &err;
+    return err;
 }
 
 Error* Lexer::SyntaxError(const char* fmt, ...) noexcept{
@@ -519,11 +520,6 @@ Error* Lexer::SyntaxError(const char* fmt, ...) noexcept{
     return err;
 }
 
-Lexer::Lexer(VM* vm, std::shared_ptr<SourceData> src) noexcept : vm(vm), src(src){
-    this->token_start = pkpy_Str__data(&src->source);
-    this->curr_char = pkpy_Str__data(&src->source);
-}
-
 Error* Lexer::run() noexcept{
     assert(!this->used);
     this->used = true;

src/objects/error.cpp

@@ -4,10 +4,6 @@ namespace pkpy {
 Str Exception::summary() const {
     SStream ss;
     if(is_re) ss << "Traceback (most recent call last):\n";
-    // while(!st.empty()) {
-    //     ss << st.top().snapshot() << '\n';
-    //     st.pop();
-    // }
     for(int i = stacktrace.size() - 1; i >= 0; i--) {
         ss << stacktrace[i].snapshot() << '\n';
     }

src/pocketpy.cpp

@@ -1733,7 +1733,10 @@ CodeObject_ VM::compile(std::string_view source, const Str& filename, CompileMod
 void VM::__compile_error(Error* err){
     assert(err != nullptr);
     if(err->type == std::string_view("NeedMoreLines")){
-        throw NeedMoreLines((bool)err->userdata);
+        bool arg = (bool)err->userdata;
+        PK_DECREF(err->src);
+        std::free(err);
+        throw NeedMoreLines(arg);
     }
     __last_exception = vm->call(
         vm->builtins->attr(err->type),
@@ -1741,6 +1744,8 @@ void VM::__compile_error(Error* err){
     ).get();
     Exception& e = __last_exception->as<Exception>();
     e.st_push(err->src, err->lineno, err->cursor, "");
+    PK_DECREF(err->src);
+    std::free(err);
     _error(__last_exception);
 }