From e5d5f09d83ba6e77e2950e0b23528cef656f316b Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sun, 30 Apr 2023 14:42:17 +0800 Subject: [PATCH 1/5] ... --- src/obj.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/obj.h b/src/obj.h index 5dd9ffab..9cc17c63 100644 --- a/src/obj.h +++ b/src/obj.h @@ -12,17 +12,23 @@ struct Function; class VM; typedef PyObject* (*NativeFuncC)(VM*, ArgsView); -typedef shared_ptr CodeObject_; +typedef int (*LuaStyleFuncC)(VM*); struct NativeFunc { NativeFuncC f; int argc; // DONOT include self bool method; + + // this is designed for lua style C bindings + // access it via `CAST(NativeFunc&, args[-2])._lua_f` + LuaStyleFuncC _lua_f; - NativeFunc(NativeFuncC f, int argc, bool method) : f(f), argc(argc), method(method) {} + NativeFunc(NativeFuncC f, int argc, bool method) : f(f), argc(argc), method(method), _lua_f(nullptr) {} PyObject* operator()(VM* vm, ArgsView args) const; }; +typedef shared_ptr CodeObject_; + struct FuncDecl { struct KwArg { int key; // index in co->varnames From a04cdb4cadc44713a424045b36a3ebb004dd9484 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sun, 30 Apr 2023 21:53:50 +0800 Subject: [PATCH 2/5] add `requests` module --- .gitignore | 1 + amalgamate.py | 2 +- python/_dict.py | 2 +- python/requests.py | 40 +++++++++++++++++ src/pocketpy.h | 12 +++--- src/requests.h | 105 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 python/requests.py create mode 100644 src/requests.h diff --git a/.gitignore b/.gitignore index 67e402d6..4631fb01 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ src/_generated.h profile.sh test tmp.rar +src/httplib.h diff --git a/amalgamate.py b/amalgamate.py index b6815710..73cf30da 100644 --- a/amalgamate.py +++ b/amalgamate.py @@ -9,7 +9,7 @@ pipeline = [ ["common.h", "memory.h", "vector.h", "str.h", "tuplelist.h", "namedict.h", "error.h", "lexer.h"], ["obj.h", "codeobject.h", "frame.h"], ["gc.h", "vm.h", "ceval.h", "expr.h", "compiler.h", "repl.h"], - ["iter.h", "cffi.h", "io.h", "_generated.h", "pocketpy.h"] + ["iter.h", "cffi.h", "requests.h", "io.h", "_generated.h", "pocketpy.h"] ] copied = set() diff --git a/python/_dict.py b/python/_dict.py index 24adfc9b..97a3515f 100644 --- a/python/_dict.py +++ b/python/_dict.py @@ -74,7 +74,7 @@ class dict: def items(self): for kv in self._a: if kv is not None: - yield kv + yield kv[0], kv[1] def clear(self): self._a = [None] * self._capacity diff --git a/python/requests.py b/python/requests.py new file mode 100644 index 00000000..2ebf35fb --- /dev/null +++ b/python/requests.py @@ -0,0 +1,40 @@ +class Response: + def __init__(self, status_code, reason, content): + self.status_code = status_code + self.reason = reason + self.content = content + + assert type(self.status_code) is int + assert type(self.reason) is str + assert type(self.content) is bytes + + @property + def text(self): + return self.content.decode() + + def __repr__(self): + code = self.status_code + return f'' + +def _parse_h(headers): + if headers is None: + return [] + if type(headers) is dict: + return list(headers.items()) + raise ValueError('headers must be dict or None') + +def get(url, headers=None): + headers = _parse_h(headers) + return _request('GET', url, headers, None) + +def post(url, data: bytes, headers=None): + headers = _parse_h(headers) + return _request('POST', url, headers, data) + +def put(url, data: bytes, headers=None): + headers = _parse_h(headers) + return _request('PUT', url, headers, data) + +def delete(url, headers=None): + headers = _parse_h(headers) + return _request('DELETE', url, headers, None) \ No newline at end of file diff --git a/src/pocketpy.h b/src/pocketpy.h index 2501c396..6b3f0b45 100644 --- a/src/pocketpy.h +++ b/src/pocketpy.h @@ -6,6 +6,7 @@ #include "repl.h" #include "iter.h" #include "cffi.h" +#include "requests.h" #include "io.h" #include "_generated.h" @@ -920,11 +921,6 @@ inline void VM::post_init(){ add_module_gc(this); add_module_random(this); - if(enable_os){ - add_module_io(this); - add_module_os(this); - } - for(const char* name: {"this", "functools", "collections", "heapq", "bisect"}){ _lazy_modules[name] = kPythonLibs[name]; } @@ -969,6 +965,12 @@ inline void VM::post_init(){ } return VAR(MappingProxy(args[0])); })); + + if(enable_os){ + add_module_io(this); + add_module_os(this); + add_module_requests(this); + } #endif } diff --git a/src/requests.h b/src/requests.h new file mode 100644 index 00000000..d510723c --- /dev/null +++ b/src/requests.h @@ -0,0 +1,105 @@ +#pragma once + +#include "common.h" +#include "obj.h" +#include "vm.h" +#include "_generated.h" + +#if __has_include("httplib.h") +#include "httplib.h" + +namespace pkpy { + +inline void add_module_requests(VM* vm){ + static StrName m_requests("requests"); + static StrName m_Response("Response"); + PyObject* mod = vm->new_module(m_requests); + CodeObject_ code = vm->compile(kPythonLibs["requests"], "requests.py", EXEC_MODE); + vm->_exec(code, mod); + + vm->bind_func<4>(mod, "_request", [](VM* vm, ArgsView args){ + Str method = CAST(Str&, args[0]); + Str url = CAST(Str&, args[1]); + PyObject* headers = args[2]; // a dict object + PyObject* body = args[3]; // a bytes object + + if(url.index("http://") != 0){ + vm->ValueError("url must start with http://"); + } + + for(char c: url){ + switch(c){ + case '.': + case '-': + case '_': + case '~': + case ':': + case '/': + break; + default: + if(!isalnum(c)){ + vm->ValueError(fmt("invalid character in url: '", c, "'")); + } + } + } + + int slash = url.index("/", 7); + Str path = "/"; + if(slash != -1){ + path = url.substr(slash); + url = url.substr(0, slash); + if(path.empty()) path = "/"; + } + + httplib::Client client(url.str()); + + httplib::Headers h; + if(headers != vm->None){ + List list = CAST(List&, headers); + for(auto& item : list){ + Tuple t = CAST(Tuple&, item); + Str key = CAST(Str&, t[0]); + Str value = CAST(Str&, t[1]); + h.emplace(key.str(), value.str()); + } + } + + auto _to_resp = [=](const httplib::Result& res){ + std::vector buf(res->body.size()); + for(int i=0; ibody.size(); i++) buf[i] = res->body[i]; + return vm->call( + vm->_modules[m_requests]->attr(m_Response), + VAR(res->status), + VAR(res->reason), + VAR(Bytes(std::move(buf))) + ); + }; + + if(method == "GET"){ + httplib::Result res = client.Get(path.str(), h); + return _to_resp(res); + }else if(method == "POST"){ + Bytes b = CAST(Bytes&, body); + httplib::Result res = client.Post(path.str(), h, b.data(), b.size(), "application/octet-stream"); + return _to_resp(res); + }else if(method == "PUT"){ + Bytes b = CAST(Bytes&, body); + httplib::Result res = client.Put(path.str(), h, b.data(), b.size(), "application/octet-stream"); + return _to_resp(res); + }else if(method == "DELETE"){ + httplib::Result res = client.Delete(path.str(), h); + return _to_resp(res); + }else{ + vm->ValueError("invalid method"); + } + UNREACHABLE(); + }); +} + +} // namespace pkpy + +#else + +inline void add_module_requests(VM* vm){ } + +#endif \ No newline at end of file From 70e8a60f8bf42012deac032fdda0d305d843ced7 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sun, 30 Apr 2023 22:01:28 +0800 Subject: [PATCH 3/5] ... --- amalgamate.py | 2 +- src/io.h | 30 ++++++++++++------------------ src/requests.h | 2 +- src/vm.h | 3 ++- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/amalgamate.py b/amalgamate.py index 73cf30da..e3614989 100644 --- a/amalgamate.py +++ b/amalgamate.py @@ -9,7 +9,7 @@ pipeline = [ ["common.h", "memory.h", "vector.h", "str.h", "tuplelist.h", "namedict.h", "error.h", "lexer.h"], ["obj.h", "codeobject.h", "frame.h"], ["gc.h", "vm.h", "ceval.h", "expr.h", "compiler.h", "repl.h"], - ["iter.h", "cffi.h", "requests.h", "io.h", "_generated.h", "pocketpy.h"] + ["_generated.h", "iter.h", "cffi.h", "requests.h", "io.h", "pocketpy.h"] ] copied = set() diff --git a/src/io.h b/src/io.h index b93803eb..2f686705 100644 --- a/src/io.h +++ b/src/io.h @@ -10,17 +10,6 @@ #include namespace pkpy{ - -inline Bytes _read_file_cwd(const Str& name){ - std::filesystem::path path(name.sv()); - bool exists = std::filesystem::exists(path); - if(!exists) return Bytes(); - std::ifstream ifs(path, std::ios::binary); - std::vector buffer(std::istreambuf_iterator(ifs), {}); - ifs.close(); - return Bytes(std::move(buffer)); -} - struct FileIO { PY_CLASS(FileIO, io, FileIO) @@ -94,6 +83,16 @@ struct FileIO { }; inline void add_module_io(VM* vm){ + _read_file_cwd = [](const Str& name){ + std::filesystem::path path(name.sv()); + bool exists = std::filesystem::exists(path); + if(!exists) return Bytes(); + std::ifstream ifs(path, std::ios::binary); + std::vector buffer(std::istreambuf_iterator(ifs), {}); + ifs.close(); + return Bytes(std::move(buffer)); + }; + PyObject* mod = vm->new_module("io"); FileIO::register_class(vm, mod); vm->bind_builtin_func<2>("open", [](VM* vm, ArgsView args){ @@ -182,13 +181,8 @@ inline void add_module_os(VM* vm){ #else namespace pkpy{ -inline void add_module_io(VM* vm){} -inline void add_module_os(VM* vm){} - -inline Bytes _read_file_cwd(const Str& name){ - return Bytes(); -} - +inline void add_module_io(void* vm){} +inline void add_module_os(void* vm){} } // namespace pkpy #endif \ No newline at end of file diff --git a/src/requests.h b/src/requests.h index d510723c..2574b77c 100644 --- a/src/requests.h +++ b/src/requests.h @@ -100,6 +100,6 @@ inline void add_module_requests(VM* vm){ #else -inline void add_module_requests(VM* vm){ } +inline void add_module_requests(void* vm){ } #endif \ No newline at end of file diff --git a/src/vm.h b/src/vm.h index c4ce61d0..90331cd2 100644 --- a/src/vm.h +++ b/src/vm.h @@ -24,7 +24,8 @@ namespace pkpy{ #define POPX() (s_data.popx()) #define STACK_VIEW(n) (s_data.view(n)) -Bytes _read_file_cwd(const Str& name); +typedef Bytes (*ReadFileCwdFunc)(const Str& name); +inline ReadFileCwdFunc _read_file_cwd = [](const Str& name) { return Bytes(); }; #define DEF_NATIVE_2(ctype, ptype) \ template<> inline ctype py_cast(VM* vm, PyObject* obj) { \ From f9ed405bf5657e4df1553097ef357a0ecb134750 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sun, 30 Apr 2023 22:08:42 +0800 Subject: [PATCH 4/5] ... --- src/io.h | 21 +++++++++++---------- src/vm.h | 1 + 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/io.h b/src/io.h index 2f686705..0c055718 100644 --- a/src/io.h +++ b/src/io.h @@ -10,6 +10,17 @@ #include namespace pkpy{ + +inline int _ = set_read_file_cwd([](const Str& name){ + std::filesystem::path path(name.sv()); + bool exists = std::filesystem::exists(path); + if(!exists) return Bytes(); + std::ifstream ifs(path, std::ios::binary); + std::vector buffer(std::istreambuf_iterator(ifs), {}); + ifs.close(); + return Bytes(std::move(buffer)); +}); + struct FileIO { PY_CLASS(FileIO, io, FileIO) @@ -83,16 +94,6 @@ struct FileIO { }; inline void add_module_io(VM* vm){ - _read_file_cwd = [](const Str& name){ - std::filesystem::path path(name.sv()); - bool exists = std::filesystem::exists(path); - if(!exists) return Bytes(); - std::ifstream ifs(path, std::ios::binary); - std::vector buffer(std::istreambuf_iterator(ifs), {}); - ifs.close(); - return Bytes(std::move(buffer)); - }; - PyObject* mod = vm->new_module("io"); FileIO::register_class(vm, mod); vm->bind_builtin_func<2>("open", [](VM* vm, ArgsView args){ diff --git a/src/vm.h b/src/vm.h index 90331cd2..3d6db7eb 100644 --- a/src/vm.h +++ b/src/vm.h @@ -26,6 +26,7 @@ namespace pkpy{ typedef Bytes (*ReadFileCwdFunc)(const Str& name); inline ReadFileCwdFunc _read_file_cwd = [](const Str& name) { return Bytes(); }; +inline int set_read_file_cwd(ReadFileCwdFunc func) { _read_file_cwd = func; return 0; } #define DEF_NATIVE_2(ctype, ptype) \ template<> inline ctype py_cast(VM* vm, PyObject* obj) { \ From fcc13cb42297c45037291a377d9357f76fde6a45 Mon Sep 17 00:00:00 2001 From: blueloveTH Date: Sun, 30 Apr 2023 22:29:04 +0800 Subject: [PATCH 5/5] ... --- src/obj.h | 1 + src/requests.h | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/obj.h b/src/obj.h index 9cc17c63..a504b597 100644 --- a/src/obj.h +++ b/src/obj.h @@ -82,6 +82,7 @@ struct Bytes{ Bytes() : _data(), _ok(false) {} Bytes(std::vector&& data) : _data(std::move(data)), _ok(true) {} + Bytes(const std::string& data) : _data(data.begin(), data.end()), _ok(true) {} operator bool() const noexcept { return _ok; } }; diff --git a/src/requests.h b/src/requests.h index 2574b77c..cc4b4423 100644 --- a/src/requests.h +++ b/src/requests.h @@ -65,13 +65,11 @@ inline void add_module_requests(VM* vm){ } auto _to_resp = [=](const httplib::Result& res){ - std::vector buf(res->body.size()); - for(int i=0; ibody.size(); i++) buf[i] = res->body[i]; return vm->call( vm->_modules[m_requests]->attr(m_Response), VAR(res->status), VAR(res->reason), - VAR(Bytes(std::move(buf))) + VAR(Bytes(res->body)) ); };