Zxc200611/pocketpy
1
0
Fork 0
mirror of https://github.com/pocketpy/pocketpy synced 2025-10-20 03:20:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1 from blueloveTH/new_str

Browse Source 
New _Str implementation
This commit is contained in:
BLUELOVETH 2022-11-19 17:20:22 +08:00 committed by GitHub
commit d0fa268245
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 208 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build_cpp.sh
View File

@ -1 +1 @@
g++ -o pocketpy src/main.cpp --std=c++17 -O1 -pthread
g++ -o pocketpy src/main.cpp --std=c++17 -O1 -pthread -Wno-literal-suffix

2
build_wasm.sh
View File

@ -1,3 +1,3 @@
rm -rf web/lib/
mkdir -p web/lib/
emcc src/main.cpp -fexceptions -sEXIT_RUNTIME -O3 -sEXPORTED_FUNCTIONS=_repl_input,_repl_start -sEXPORTED_RUNTIME_METHODS=ccall -o web/lib/pocketpy.js
em++ src/main.cpp -fexceptions -sEXIT_RUNTIME -O3 -sEXPORTED_FUNCTIONS=_repl_input,_repl_start -sEXPORTED_RUNTIME_METHODS=ccall -o web/lib/pocketpy.js

2
src/codeobject.h
View File

@ -112,7 +112,7 @@ struct CodeObject {
auto fn = std::get_if<_Func>(&co_consts[i]->_native);
if(fn) ss << '\n' << (*fn)->code->name << ":\n" << (*fn)->code->toString();
}
return _Str(ss);
return _Str(ss.str());
}
};

6
src/iter.h
View File

@ -44,14 +44,14 @@ public:
class StringIterator : public _Iterator {
private:
size_t index = 0;
const _Str* str;
_Str str;
public:
StringIterator(VM* vm, PyVar _ref) : _Iterator(vm, _ref) {
str = &std::get<_Str>(_ref->_native);
str = std::get<_Str>(_ref->_native);
}
bool hasNext(){
return index < str->u8_length();
return index < str.u8_length();
}
PyVar next();

26
src/main.cpp
View File

@ -69,17 +69,25 @@ int main(int argc, char** argv){
});
if(code == nullptr) return 1;
//std::cout << code->toString() << std::endl;
Timer("Running time").run([=]{
vm->startExec(code);
while(pkpy_tvm_get_state(vm) != THREAD_FINISHED){
if(pkpy_tvm_get_state(vm) == THREAD_SUSPENDED){
std::string line;
std::getline(std::cin, line);
pkpy_tvm_write_stdin(vm, line.c_str());
pkpy_tvm_resume(vm);
}
}
vm->exec(code);
});
// for(auto& kv : _strIntern)
// std::cout << kv.first << ", ";
// Timer("Running time").run([=]{
// vm->startExec(code);
// while(pkpy_tvm_get_state(vm) != THREAD_FINISHED){
// if(pkpy_tvm_get_state(vm) == THREAD_SUSPENDED){
// std::string line;
// std::getline(std::cin, line);
// pkpy_tvm_write_stdin(vm, line.c_str());
// pkpy_tvm_resume(vm);
// }
// }
// });
return 0;
}

12
src/obj.h
View File

@ -84,19 +84,25 @@ const int _SIZEOF_VALUE = sizeof(_Value);
struct PyObject {
PyVarDict attribs;
_Value _native;
PyVar _type;
inline bool isType(const PyVar& type){
return attribs[__class__] == type;
return this->_type == type;
}
inline void setType(const PyVar& type){
this->_type = type;
this->attribs[__class__] = type;
}
// currently __name__ is only used for 'type'
_Str getName(){
_Value val = attribs["__name__"]->_native;
_Value val = attribs[__name__]->_native;
return std::get<_Str>(val);
}
_Str getTypeName(){
return attribs[__class__]->getName();
return _type->getName();
}
PyObject(_Value val): _native(val) {}

10
src/pocketpy.h
View File

@ -126,7 +126,7 @@ void __initializeBuiltinFunctions(VM* _vm) {
_vm->bindMethod("type", "__new__", [](VM* vm, PyVarList args) {
vm->__checkArgSize(args, 1);
return args[0]->attribs[__class__];
return args[0]->_type;
});
_vm->bindMethod("range", "__new__", [](VM* vm, PyVarList args) {
@ -354,7 +354,7 @@ void __initializeBuiltinFunctions(VM* _vm) {
const _Str& _self (vm->PyStr_AS_C(args[0]));
_StrStream ss;
for(auto c : _self.str()) ss << (char)toupper(c);
return vm->PyStr(ss);
return vm->PyStr(ss.str());
});
_vm->bindMethod("str", "lower", [](VM* vm, PyVarList args) {
@ -362,7 +362,7 @@ void __initializeBuiltinFunctions(VM* _vm) {
const _Str& _self (vm->PyStr_AS_C(args[0]));
_StrStream ss;
for(auto c : _self.str()) ss << (char)tolower(c);
return vm->PyStr(ss);
return vm->PyStr(ss.str());
});
_vm->bindMethod("str", "replace", [](VM* vm, PyVarList args) {
@ -403,7 +403,7 @@ void __initializeBuiltinFunctions(VM* _vm) {
if(i > 0) ss << _self;
ss << vm->PyStr_AS_C(vm->asStr(_list[i]));
}
return vm->PyStr(ss);
return vm->PyStr(ss.str());
});
/************ PyList ************/
@ -713,7 +713,7 @@ extern "C" {
_StrStream* s_out = dynamic_cast<_StrStream*>(vm->_stdout);
_StrStream* s_err = dynamic_cast<_StrStream*>(vm->_stderr);
if(s_out == nullptr || s_err == nullptr) UNREACHABLE();
PyOutputDump* dump = new PyOutputDump(*s_out, *s_err);
PyOutputDump* dump = new PyOutputDump(s_out->str(), s_err->str());
s_out->str("");
s_err->str("");
return dump;

204
src/str.h
View File

@ -4,38 +4,27 @@
typedef std::stringstream _StrStream;
class _Str {
private:
mutable bool utf8_initialized = false;
mutable std::vector<uint16_t> _u8_index;
std::string _s;
class _StrMemory : public std::string {
mutable std::vector<uint16_t>* _u8_index = nullptr;
mutable bool hash_initialized = false;
mutable size_t _hash;
void utf8_lazy_init() const{
if(utf8_initialized) return;
if(_u8_index != nullptr) return;
_u8_index = new std::vector<uint16_t>();
_u8_index->reserve(size());
if(size() > 65535) throw std::runtime_error("String has more than 65535 bytes.");
for(uint16_t i = 0; i < size(); i++){
// https://stackoverflow.com/questions/3911536/utf-8-unicode-whats-with-0xc0-and-0x80
if((_s[i] & 0xC0) != 0x80)
_u8_index.push_back(i);
if((at(i) & 0xC0) != 0x80)
_u8_index->push_back(i);
}
utf8_initialized = true;
}
public:
_Str(const char* s): _s(s) {}
_Str(const char* s, size_t len): _s(s, len) {}
_Str(int n, char fill): _s(n, fill) {}
_Str(const std::string& s): _s(s) {}
_Str(std::string&& s): _s(std::move(s)) {}
_Str(const _StrStream& ss): _s(ss.str()) {}
_Str(){}
size_t hash() const{
if(!hash_initialized){
_hash = std::hash<std::string>()(_s);
_hash = std::hash<std::string>()(*this);
hash_initialized = true;
}
return _hash;
@ -43,93 +32,181 @@ public:
int u8_length() const {
utf8_lazy_init();
return _u8_index.size();
return _u8_index->size();
}
_Str u8_getitem(int i) const{
std::string u8_getitem(int i) const{
return u8_substr(i, i+1);
}
_Str u8_substr(int start, int end) const{
std::string u8_substr(int start, int end) const{
utf8_lazy_init();
if(start >= end) return _Str();
int c_end = end >= _u8_index.size() ? size() : _u8_index[end];
return _s.substr(_u8_index.at(start), c_end - _u8_index.at(start));
if(start >= end) return std::string();
int c_end = end >= _u8_index->size() ? size() : _u8_index->at(end);
return substr(_u8_index->at(start), c_end - _u8_index->at(start));
}
int size() const {
return _s.size();
_StrMemory(const std::string& s) : std::string(s) {}
_StrMemory(std::string&& s) : std::string(std::move(s)) {}
~_StrMemory(){
if(_u8_index != nullptr) delete _u8_index;
}
};
std::unordered_map<std::string, std::shared_ptr<_StrMemory>> _strIntern;
struct _StrLiteral {
const char* _str;
size_t _len;
constexpr _StrLiteral(const char* str, size_t len) : _str(str), _len(len) {}
};
inline constexpr _StrLiteral operator "" _c(const char* str, size_t len){
return _StrLiteral(str, len);
}
bool empty() const {
return _s.empty();
class _Str {
private:
std::shared_ptr<_StrMemory> _s;
bool interned = false;
public:
_Str(const _StrLiteral& s){
construct(std::string(s._str, s._len));
intern();
}
_Str(const char* s){
construct(std::string(s));
}
_Str(const char* s, size_t len){
construct(std::string(s, len));
}
_Str(int n, char fill){
construct(std::string(n, fill));
}
_Str(const std::string& s){
construct(s);
}
_Str(std::string&& s){
construct(std::move(s));
}
_Str(){
construct(std::string());
}
_Str(const _Str& s) : _s(s._s), interned(s.interned) {}
void construct(std::string s){
auto it = _strIntern.find(s);
if(it != _strIntern.end()){
this->_s = it->second;
interned = true;
}else{
this->_s = std::make_shared<_StrMemory>(std::move(s));
}
}
// force the string to be interned
void intern(){
if(interned) return;
auto it = _strIntern.find(*this->_s);
if(it == _strIntern.end()) _strIntern[*this->_s] = this->_s;
else this->_s = it->second;
interned = true;
}
inline int u8_length() const {
return this->_s->u8_length();
}
inline _Str u8_getitem(int i) const{
return _Str(this->_s->u8_getitem(i));
}
inline _Str u8_substr(int start, int end) const{
return _Str(this->_s->u8_substr(start, end));
}
inline size_t hash() const{
return _s->hash();
}
inline int size() const {
return _s->size();
}
inline bool empty() const {
return _s->empty();
}
bool operator==(const _Str& other) const {
return _s == other._s;
if(interned && other.interned) return _s == other._s;
return *_s == *other._s;
}
bool operator!=(const _Str& other) const {
return _s != other._s;
if(interned && other.interned) return _s != other._s;
return *_s != *other._s;
}
bool operator==(const char* other) const {
return _s == other;
return *_s == other;
}
bool operator!=(const char* other) const {
return _s != other;
return *_s != other;
}
bool operator<(const _Str& other) const {
return _s < other._s;
return *_s < *other._s;
}
bool operator>(const _Str& other) const {
return _s > other._s;
return *_s > *other._s;
}
char operator[](int i) const {
return _s[i];
return _s->at(i);
}
friend std::ostream& operator<<(std::ostream& os, const _Str& s) {
os << s._s;
os << *s._s;
return os;
}
_Str operator+(const _Str& other) const {
return _Str(_s + other._s);
return _Str(*_s + *other._s);
}
_Str operator+(const char* other) const {
return _Str(_s + other);
return _Str(*_s + other);
}
_Str operator+(const std::string& other) const {
return _Str(_s + other);
return _Str(*_s + other);
}
friend _Str operator+(const char* other, const _Str& s){
return _Str(other + s._s);
return _Str(other + *s._s);
}
friend _Str operator+(const std::string& other, const _Str& s){
return _Str(other + s._s);
return _Str(other + *s._s);
}
const std::string& str() const {
return _s;
return *_s;
}
const char* c_str() const {
return _s.c_str();
return _s->c_str();
}
static const std::size_t npos = std::string::npos;
_Str __lstrip() const {
std::string copy(_s);
std::string copy(*_s);
copy.erase(copy.begin(), std::find_if(copy.begin(), copy.end(), [](char c) {
return !std::isspace(c);
}));
@ -138,7 +215,7 @@ public:
_Str __escape(bool single_quote) const {
_StrStream ss;
for (auto c = _s.cbegin(); c != _s.cend(); c++) {
for (auto c = _s->cbegin(); c != _s->cend(); c++) {
switch (*c) {
case '"':
if(!single_quote) ss << '\\';
@ -165,7 +242,6 @@ public:
}
};
namespace std {
template<>
struct hash<_Str> {
@ -175,30 +251,32 @@ namespace std {
};
}
const _Str& __class__ = _Str("__class__");
const _Str& __base__ = _Str("__base__");
const _Str& __new__ = _Str("__new__");
const _Str& __iter__ = _Str("__iter__");
const _Str& __str__ = _Str("__str__");
const _Str& __repr__ = _Str("__repr__");
const _Str& __module__ = _Str("__module__");
const _Str& __getitem__ = _Str("__getitem__");
const _Str& __setitem__ = _Str("__setitem__");
const _Str& __delitem__ = _Str("__delitem__");
const _Str& __contains__ = _Str("__contains__");
const _Str& __init__ = _Str("__init__");
const _Str& __json__ = _Str("__json__");
const _Str& __class__ = _Str("__class__"_c);
const _Str& __base__ = _Str("__base__"_c);
const _Str& __new__ = _Str("__new__"_c);
const _Str& __iter__ = _Str("__iter__"_c);
const _Str& __str__ = _Str("__str__"_c);
const _Str& __repr__ = _Str("__repr__"_c);
const _Str& __module__ = _Str("__module__"_c);
const _Str& __getitem__ = _Str("__getitem__"_c);
const _Str& __setitem__ = _Str("__setitem__"_c);
const _Str& __delitem__ = _Str("__delitem__"_c);
const _Str& __contains__ = _Str("__contains__"_c);
const _Str& __init__ = _Str("__init__"_c);
const _Str& __json__ = _Str("__json__"_c);
const _Str& __name__ = _Str("__name__"_c);
const _Str& __len__ = _Str("__len__"_c);
const _Str CMP_SPECIAL_METHODS[] = {
"__lt__", "__le__", "__eq__", "__ne__", "__gt__", "__ge__"
"__lt__"_c, "__le__"_c, "__eq__"_c, "__ne__"_c, "__gt__"_c, "__ge__"_c
}; // __ne__ should not be used
const _Str BINARY_SPECIAL_METHODS[] = {
"__add__", "__sub__", "__mul__", "__truediv__", "__floordiv__", "__mod__", "__pow__"
"__add__"_c, "__sub__"_c, "__mul__"_c, "__truediv__"_c, "__floordiv__"_c, "__mod__"_c, "__pow__"_c
};
const _Str BITWISE_SPECIAL_METHODS[] = {
"__lshift__", "__rshift__", "__and__", "__or__", "__xor__"
"__lshift__"_c, "__rshift__"_c, "__and__"_c, "__or__"_c, "__xor__"_c
};
const uint32_t __LoRangeA[] = {170,186,443,448,660,1488,1519,1568,1601,1646,1649,1749,1774,1786,1791,1808,1810,1869,1969,1994,2048,2112,2144,2208,2230,2308,2365,2384,2392,2418,2437,2447,2451,2474,2482,2486,2493,2510,2524,2527,2544,2556,2565,2575,2579,2602,2610,2613,2616,2649,2654,2674,2693,2703,2707,2730,2738,2741,2749,2768,2784,2809,2821,2831,2835,2858,2866,2869,2877,2908,2911,2929,2947,2949,2958,2962,2969,2972,2974,2979,2984,2990,3024,3077,3086,3090,3114,3133,3160,3168,3200,3205,3214,3218,3242,3253,3261,3294,3296,3313,3333,3342,3346,3389,3406,3412,3423,3450,3461,3482,3507,3517,3520,3585,3634,3648,3713,3716,3718,3724,3749,3751,3762,3773,3776,3804,3840,3904,3913,3976,4096,4159,4176,4186,4193,4197,4206,4213,4238,4352,4682,4688,4696,4698,4704,4746,4752,4786,4792,4800,4802,4808,4824,4882,4888,4992,5121,5743,5761,5792,5873,5888,5902,5920,5952,5984,5998,6016,6108,6176,6212,6272,6279,6314,6320,6400,6480,6512,6528,6576,6656,6688,6917,6981,7043,7086,7098,7168,7245,7258,7401,7406,7413,7418,8501,11568,11648,11680,11688,11696,11704,11712,11720,11728,11736,12294,12348,12353,12447,12449,12543,12549,12593,12704,12784,13312,19968,40960,40982,42192,42240,42512,42538,42606,42656,42895,42999,43003,43011,43015,43020,43072,43138,43250,43259,43261,43274,43312,43360,43396,43488,43495,43514,43520,43584,43588,43616,43633,43642,43646,43697,43701,43705,43712,43714,43739,43744,43762,43777,43785,43793,43808,43816,43968,44032,55216,55243,63744,64112,64285,64287,64298,64312,64318,64320,64323,64326,64467,64848,64914,65008,65136,65142,65382,65393,65440,65474,65482,65490,65498,65536,65549,65576,65596,65599,65616,65664,66176,66208,66304,66349,66370,66384,66432,66464,66504,66640,66816,66864,67072,67392,67424,67584,67592,67594,67639,67644,67647,67680,67712,67808,67828,67840,67872,67968,68030,68096,68112,68117,68121,68192,68224,68288,68297,68352,68416,68448,68480,68608,68864,69376,69415,69424,69600,69635,69763,69840,69891,69956,69968,70006,70019,70081,70106,70108,70144,70163,70272,70280,70282,70287,70303,70320,70405,70415,70419,70442,70450,70453,70461,70480,70493,70656,70727,70751,70784,70852,70855,71040,71128,71168,71236,71296,71352,71424,71680,71935,72096,72106,72161,72163,72192,72203,72250,72272,72284,72349,72384,72704,72714,72768,72818,72960,72968,72971,73030,73056,73063,73066,73112,73440,73728,74880,77824,82944,92160,92736,92880,92928,93027,93053,93952,94032,94208,100352,110592,110928,110948,110960,113664,113776,113792,113808,123136,123214,123584,124928,126464,126469,126497,126500,126503,126505,126516,126521,126523,126530,126535,126537,126539,126541,126545,126548,126551,126553,126555,126557,126559,126561,126564,126567,126572,126580,126585,126590,126592,126603,126625,126629,126635,131072,173824,177984,178208,183984,194560};

40
src/vm.h
View File

@ -30,7 +30,7 @@
}else{ \
__checkType(ptype, _tp_type); \
_raw = new PyObject(_native); \
_raw->attribs[__class__] = ptype; \
_raw->setType(ptype); \
} \
PyVar obj = PyVar(_raw, [this](PyObject* p){\
if(_pool##name.size() < max_size){ \
@ -116,7 +116,7 @@ private:
PyVarList items = frame->popNValuesReversed(this, byte.arg);
_StrStream ss;
for(const auto& i : items) ss << PyStr_AS_C(asStr(i));
frame->push(PyStr(ss));
frame->push(PyStr(ss.str()));
} break;
case OP_LOAD_EVAL_FN: {
frame->push(builtins->attribs["eval"]);
@ -371,11 +371,10 @@ public:
PyVar asBool(const PyVar& obj){
if(obj == None) return False;
PyVar tp = obj->attribs[__class__];
if(tp == _tp_bool) return obj;
if(tp == _tp_int) return PyBool(PyInt_AS_C(obj) != 0);
if(tp == _tp_float) return PyBool(PyFloat_AS_C(obj) != 0.0);
PyVarOrNull len_fn = getAttr(obj, "__len__", false);
if(obj->_type == _tp_bool) return obj;
if(obj->_type == _tp_int) return PyBool(PyInt_AS_C(obj) != 0);
if(obj->_type == _tp_float) return PyBool(PyFloat_AS_C(obj) != 0.0);
PyVarOrNull len_fn = getAttr(obj, __len__, false);
if(len_fn != nullptr){
PyVar ret = call(len_fn, {});
return PyBool(PyInt_AS_C(ret) > 0);
@ -384,7 +383,7 @@ public:
}
PyVar fastCall(const PyVar& obj, const _Str& name, PyVarList args){
PyVar cls = obj->attribs[__class__];
PyVar cls = obj->_type;
while(cls != None) {
auto it = cls->attribs.find(name);
if(it != cls->attribs.end()){
@ -414,6 +413,7 @@ public:
if(callable->isType(_tp_bounded_method)){
auto& bm = PyBoundedMethod_AS_C(callable);
// TODO: avoid insertion here, bad performance
args.insert(args.begin(), bm.obj);
callable = bm.method;
}
@ -516,7 +516,7 @@ public:
PyVar newUserClassType(_Str name, PyVar base){
PyVar obj = newClassType(name, base);
setAttr(obj, "__name__", PyStr(name));
setAttr(obj, __name__, PyStr(name));
_types.erase(name);
return obj;
}
@ -524,7 +524,7 @@ public:
PyVar newClassType(_Str name, PyVar base=nullptr) {
if(base == nullptr) base = _tp_object;
PyVar obj = std::make_shared<PyObject>((_Int)0);
setAttr(obj, __class__, _tp_type);
obj->setType(_tp_type);
setAttr(obj, __base__, base);
_types[name] = obj;
return obj;
@ -533,13 +533,13 @@ public:
PyVar newObject(PyVar type, _Value _native) {
__checkType(type, _tp_type);
PyVar obj = std::make_shared<PyObject>(_native);
setAttr(obj, __class__, type);
obj->setType(type);
return obj;
}
PyVar newModule(_Str name, bool saveToPath=true) {
PyVar obj = newObject(_tp_module, (_Int)-2);
setAttr(obj, "__name__", PyStr(name));
setAttr(obj, __name__, PyStr(name));
if(saveToPath) _modules[name] = obj;
return obj;
}
@ -548,7 +548,7 @@ public:
auto it = obj->attribs.find(name);
if(it != obj->attribs.end()) return it->second;
PyVar cls = obj->attribs[__class__];
PyVar cls = obj->_type;
while(cls != None) {
it = cls->attribs.find(name);
if(it != cls->attribs.end()){
@ -570,6 +570,7 @@ public:
}
void bindMethod(_Str typeName, _Str funcName, _CppFunc fn) {
funcName.intern();
PyVar type = _types[typeName];
PyVar func = PyNativeFunction(fn);
setAttr(type, funcName, func);
@ -586,6 +587,7 @@ public:
}
void bindFunc(PyVar module, _Str funcName, _CppFunc fn) {
funcName.intern();
__checkType(module, _tp_module);
PyVar func = PyNativeFunction(fn);
setAttr(module, funcName, func);
@ -593,7 +595,7 @@ public:
bool isInstance(PyVar obj, PyVar type){
__checkType(type, _tp_type);
PyVar t = obj->attribs[__class__];
PyVar t = obj->_type;
while (t != None){
if (t == type) return true;
t = t->attribs[__base__];
@ -691,15 +693,15 @@ public:
this->True = newObject(_tp_bool, true);
this->False = newObject(_tp_bool, false);
this->builtins = newModule("builtins");
this->_main = newModule("__main__", false);
this->_main = newModule("__main__"_c, false);
setAttr(_tp_type, __base__, _tp_object);
setAttr(_tp_type, __class__, _tp_type);
_tp_type->setType(_tp_type);
setAttr(_tp_object, __base__, None);
setAttr(_tp_object, __class__, _tp_type);
_tp_object->setType(_tp_type);
for (auto& [name, type] : _types) {
setAttr(type, "__name__", PyStr(name));
setAttr(type, __name__, PyStr(name));
}
this->__py2py_call_signal = newObject(_tp_object, (_Int)7);
@ -916,7 +918,7 @@ PyVar RangeIterator::next(){
}
PyVar StringIterator::next(){
return vm->PyStr(str->u8_getitem(index++));
return vm->PyStr(str.u8_getitem(index++));
}
enum ThreadState {

8
test_cpp_1.sh Normal file
View File

@ -0,0 +1,8 @@
g++ -o pocketpy src/main.cpp --std=c++17 -pg -O1 -pthread -Wno-literal-suffix
./pocketpy tests/1.py
gprof pocketpy gmon.out > gprof.txt
#gprof pocketpy | gprof2dot | dot -Tsvg -o output.svg
rm gmon.out