From a5f9c21b0fdceff3a80d93dd9e3760aeb31abee9 Mon Sep 17 00:00:00 2001 From: Rob McMullen Date: Sat, 27 Jun 2026 12:57:50 -0700 Subject: [PATCH] Added str.splitlines * uses same algorithm as CPython, supports all CPython eol characters in https://docs.python.org/3/library/stdtypes.html#str.splitlines --- include/pocketpy/common/str.h | 1 + src/bindings/py_str.c | 22 ++++++++++++- src/common/str.c | 38 ++++++++++++++++++++++ tests/043_str_splitlines.py | 59 +++++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/043_str_splitlines.py diff --git a/include/pocketpy/common/str.h b/include/pocketpy/common/str.h index fb6a52db..aa738df0 100644 --- a/include/pocketpy/common/str.h +++ b/include/pocketpy/common/str.h @@ -69,6 +69,7 @@ c11_string* c11_sv__replace2(c11_sv self, c11_sv old, c11_sv new_); c11_vector /* T=c11_sv */ c11_sv__split(c11_sv self, char sep); c11_vector /* T=c11_sv */ c11_sv__split2(c11_sv self, c11_sv sep); c11_vector /* T=c11_sv */ c11_sv__splitwhitespace(c11_sv self); +c11_vector /* T=c11_sv */ c11_sv__splitlines(c11_sv self, bool keepends); // misc int c11__unicode_index_to_byte(const char* data, int i); diff --git a/src/bindings/py_str.c b/src/bindings/py_str.c index 8d8a21ec..a6d40c36 100644 --- a/src/bindings/py_str.c +++ b/src/bindings/py_str.c @@ -384,6 +384,25 @@ static bool str_split(int argc, py_Ref argv) { return true; } +static bool str_splitlines(int argc, py_Ref argv) { + c11_sv self = c11_string__sv(pk_tostr(&argv[0])); + c11_vector res; + bool keepends = false; + if(argc > 2) return TypeError("splitlines() takes at most 2 arguments"); + if(argc == 2) { + if(!py_checkbool(&argv[1])) return false; + keepends = py_tobool(&argv[1]); + } + res = c11_sv__splitlines(self, keepends); + py_newlist(py_retval()); + for(int i = 0; i < res.length; i++) { + c11_sv part = c11__getitem(c11_sv, &res, i); + py_newstrv(py_list_emplace(py_retval()), part); + } + c11_vector__dtor(&res); + return true; +} + static bool str_count(int argc, py_Ref argv) { PY_CHECK_ARGC(2); c11_string* self = pk_tostr(&argv[0]); @@ -640,6 +659,7 @@ py_Type pk_str__register() { py_bindmethod(tp_str, "join", str_join); py_bindmethod(tp_str, "replace", str_replace); py_bindmethod(tp_str, "split", str_split); + py_bindmethod(tp_str, "splitlines", str_splitlines); py_bindmethod(tp_str, "count", str_count); py_bindmethod(tp_str, "strip", str_strip); py_bindmethod(tp_str, "lstrip", str_lstrip); @@ -814,4 +834,4 @@ py_Type pk_bytes__register() { return type; } -#undef DEF_STR_CMP_OP \ No newline at end of file +#undef DEF_STR_CMP_OP diff --git a/src/common/str.c b/src/common/str.c index 6b858d56..240b1954 100644 --- a/src/common/str.c +++ b/src/common/str.c @@ -219,6 +219,44 @@ c11_vector /* T=c11_sv */ c11_sv__splitwhitespace(c11_sv self) { return retval; } +c11_vector /* T=c11_sv */ c11_sv__splitlines(c11_sv self, bool keepends) { + c11_vector retval; + c11_vector__ctor(&retval, sizeof(c11_sv)); + const char* data = self.data; + int i = 0; + int eol = 0; + int eol_size = 1; + for(int j = 0; j < self.size; ) { + while(j < self.size) { + const char c = data[j]; + eol_size = c11__u8_header(c, false); + if(c == '\n' || c == '\r' || c == '\v' || c == '\f' || c == '\x1c' || c == '\x1d' || c == '\x1e') + break; + if(eol_size == 3 && j + 2 < self.size) { + int val = c11__u8_value(eol_size, &data[j]); + if(val == 0x2028 || val == 0x2029) + break; + } + j += eol_size; + } + + eol = j; + if(j < self.size) { + // CRLF treated as one line break + if(data[j] == '\r' && j + 1 < self.size && data[j+1] == '\n') + j += 2; + else + j += eol_size; + if(keepends) + eol = j; + } + c11_sv tmp = {data + i, eol - i}; + c11_vector__push(c11_sv, &retval, tmp); + i = j; + } + return retval; +} + c11_vector /* T=c11_sv */ c11_sv__split(c11_sv self, char sep) { c11_vector retval; c11_vector__ctor(&retval, sizeof(c11_sv)); diff --git a/tests/043_str_splitlines.py b/tests/043_str_splitlines.py new file mode 100644 index 00000000..e1236066 --- /dev/null +++ b/tests/043_str_splitlines.py @@ -0,0 +1,59 @@ +try: + ''.splitlines(5) + exit(1) +except TypeError: + pass + +try: + ''.splitlines(5, 5) + exit(1) +except TypeError: + pass + +assert ''.splitlines() == [] +assert ''.splitlines(False) == [] +assert ''.splitlines(True) == [] + +assert '\n'.splitlines() == [''] +assert '\r'.splitlines() == [''] +assert '\r\n'.splitlines() == [''] +assert '\v'.splitlines() == [''] +assert '\f'.splitlines() == [''] +assert '\x1c'.splitlines() == [''] +assert '\x1d'.splitlines() == [''] +assert '\x1e'.splitlines() == [''] +assert b'\xe2\x80\xa8'.decode().splitlines() == [''] +assert b'\xe2\x80\xa9'.decode().splitlines() == [''] +assert '🥕'.splitlines() == ['🥕'] + +all_ends = ['\n', '\r', '\r\n', '\v', '\f', '\x1c', '\x1d', '\x1e', b'\xe2\x80\xa8'.decode(), b'\xe2\x80\xa9'.decode()] +for eol in all_ends: + assert (eol).splitlines() == [''] + assert (eol).splitlines(False) == [''] + assert (eol).splitlines(True) == [eol] + for text in ['a', 'a b', 'abc\tdef', '🥕 and 🍋', '测试123测试']: + assert (text).splitlines(False) == [text] + assert (text).splitlines(True) == [text] + assert (text + eol).splitlines(False) == [text] + assert (text + eol).splitlines(True) == [text + eol] + assert (eol + eol).splitlines(False) == ['', ''] + assert (eol + eol).splitlines(True) == [eol, eol] + assert (eol + text).splitlines(False) == ['', text] + assert (eol + text).splitlines(True) == [eol, text] + assert (text + eol + eol + eol + eol).splitlines(False) == [text, '', '', ''] + assert (text + eol + eol + eol + eol).splitlines(True) == [text + eol, eol, eol, eol] + assert (eol + eol + text + eol + eol + eol + eol).splitlines(False) == ['', '', text, '', '', ''] + assert (eol + eol + text + eol + eol + eol + eol).splitlines(True) == [eol, eol, text + eol, eol, eol, eol] + assert (text + eol + text).splitlines(False) == [text, text] + assert (text + eol + text).splitlines(True) == [text + eol, text] + assert (text + eol + text + eol).splitlines(False) == [text, text] + assert (text + eol + text + eol).splitlines(True) == [text + eol, text + eol] + +assert '\r\r\n'.splitlines() == ['', ''] +assert '\r\r\n'.splitlines(True) == ['\r', '\r\n'] +assert '\n\r\r\n'.splitlines() == ['', '', ''] +assert '\n\r\r\n'.splitlines(True) == ['\n', '\r', '\r\n'] +assert '\n\r\r\n\n'.splitlines() == ['', '', '', ''] +assert '\n\r\r\n\n'.splitlines(True) == ['\n', '\r', '\r\n', '\n'] +assert ''.join(all_ends).splitlines() == [''] * len(all_ends) +assert ''.join(all_ends).splitlines(True) == all_ends