Added str.splitlines (#523)

* uses same algorithm as CPython, supports all CPython eol characters
  in https://docs.python.org/3/library/stdtypes.html#str.splitlines

include/pocketpy/common/str.h

@@ -69,6 +69,7 @@ c11_string* c11_sv__replace2(c11_sv self, c11_sv old, c11_sv new_);
 c11_vector /* T=c11_sv */ c11_sv__split(c11_sv self, char sep);
 c11_vector /* T=c11_sv */ c11_sv__split2(c11_sv self, c11_sv sep);
 c11_vector /* T=c11_sv */ c11_sv__splitwhitespace(c11_sv self);
+c11_vector /* T=c11_sv */ c11_sv__splitlines(c11_sv self, bool keepends);
 
 // misc
 int c11__unicode_index_to_byte(const char* data, int i);

src/bindings/py_str.c

@@ -384,6 +384,25 @@ static bool str_split(int argc, py_Ref argv) {
     return true;
 }
 
+static bool str_splitlines(int argc, py_Ref argv) {
+    c11_sv self = c11_string__sv(pk_tostr(&argv[0]));
+    c11_vector res;
+    bool keepends = false;
+    if(argc > 2) return TypeError("splitlines() takes at most 2 arguments");
+    if(argc == 2) {
+        if(!py_checkbool(&argv[1])) return false;
+        keepends = py_tobool(&argv[1]);
+    }
+    res = c11_sv__splitlines(self, keepends);
+    py_newlist(py_retval());
+    for(int i = 0; i < res.length; i++) {
+        c11_sv part = c11__getitem(c11_sv, &res, i);
+        py_newstrv(py_list_emplace(py_retval()), part);
+    }
+    c11_vector__dtor(&res);
+    return true;
+}
+
 static bool str_count(int argc, py_Ref argv) {
     PY_CHECK_ARGC(2);
     c11_string* self = pk_tostr(&argv[0]);
@@ -640,6 +659,7 @@ py_Type pk_str__register() {
     py_bindmethod(tp_str, "join", str_join);
     py_bindmethod(tp_str, "replace", str_replace);
     py_bindmethod(tp_str, "split", str_split);
+    py_bindmethod(tp_str, "splitlines", str_splitlines);
     py_bindmethod(tp_str, "count", str_count);
     py_bindmethod(tp_str, "strip", str_strip);
     py_bindmethod(tp_str, "lstrip", str_lstrip);
@@ -814,4 +834,4 @@ py_Type pk_bytes__register() {
     return type;
 }
 
-#undef DEF_STR_CMP_OP
+#undef DEF_STR_CMP_OP

src/common/str.c

@@ -219,6 +219,44 @@ c11_vector /* T=c11_sv */ c11_sv__splitwhitespace(c11_sv self) {
     return retval;
 }
 
+c11_vector /* T=c11_sv */ c11_sv__splitlines(c11_sv self, bool keepends) {
+    c11_vector retval;
+    c11_vector__ctor(&retval, sizeof(c11_sv));
+    const char* data = self.data;
+    int i = 0;
+    int eol = 0;
+    int eol_size = 1;
+    for(int j = 0; j < self.size; ) {
+        while(j < self.size) {
+            const char c = data[j];
+            eol_size = c11__u8_header(c, false);
+            if(c == '\n' || c == '\r' || c == '\v' || c == '\f' || c == '\x1c' || c == '\x1d' || c == '\x1e')
+                break;
+            if(eol_size == 3 && j + 2 < self.size) {
+                int val = c11__u8_value(eol_size, &data[j]);
+                if(val == 0x2028 || val == 0x2029)
+                    break;
+            }
+            j += eol_size;
+        }
+
+        eol = j;
+        if(j < self.size) {
+            // CRLF treated as one line break
+            if(data[j] == '\r' && j + 1 < self.size && data[j+1] == '\n')
+                j += 2;
+            else
+                j += eol_size;
+            if(keepends)
+                eol = j;
+        }
+        c11_sv tmp = {data + i, eol - i};
+        c11_vector__push(c11_sv, &retval, tmp);
+        i = j;
+    }
+    return retval;
+}
+
 c11_vector /* T=c11_sv */ c11_sv__split(c11_sv self, char sep) {
     c11_vector retval;
     c11_vector__ctor(&retval, sizeof(c11_sv));

tests/043_str_splitlines.py

@@ -0,0 +1,59 @@
+try:
+    ''.splitlines(5)
+    exit(1)
+except TypeError:
+    pass
+
+try:
+    ''.splitlines(5, 5)
+    exit(1)
+except TypeError:
+    pass
+
+assert ''.splitlines() == []
+assert ''.splitlines(False) == []
+assert ''.splitlines(True) == []
+
+assert '\n'.splitlines() == ['']
+assert '\r'.splitlines() == ['']
+assert '\r\n'.splitlines() == ['']
+assert '\v'.splitlines() == ['']
+assert '\f'.splitlines() == ['']
+assert '\x1c'.splitlines() == ['']
+assert '\x1d'.splitlines() == ['']
+assert '\x1e'.splitlines() == ['']
+assert b'\xe2\x80\xa8'.decode().splitlines() == ['']
+assert b'\xe2\x80\xa9'.decode().splitlines() == ['']
+assert '🥕'.splitlines() == ['🥕']
+
+all_ends = ['\n', '\r', '\r\n', '\v', '\f', '\x1c', '\x1d', '\x1e', b'\xe2\x80\xa8'.decode(), b'\xe2\x80\xa9'.decode()]
+for eol in all_ends:
+    assert (eol).splitlines() == ['']
+    assert (eol).splitlines(False) == ['']
+    assert (eol).splitlines(True) == [eol]
+    for text in ['a', 'a b', 'abc\tdef', '🥕 and 🍋', '测试123测试']:
+        assert (text).splitlines(False) == [text]
+        assert (text).splitlines(True) == [text]
+        assert (text + eol).splitlines(False) == [text]
+        assert (text + eol).splitlines(True) == [text + eol]
+        assert (eol + eol).splitlines(False) == ['', '']
+        assert (eol + eol).splitlines(True) == [eol, eol]
+        assert (eol + text).splitlines(False) == ['', text]
+        assert (eol + text).splitlines(True) == [eol, text]
+        assert (text + eol + eol + eol + eol).splitlines(False) == [text, '', '', '']
+        assert (text + eol + eol + eol + eol).splitlines(True) == [text + eol, eol, eol, eol]
+        assert (eol + eol + text + eol + eol + eol + eol).splitlines(False) == ['', '', text, '', '', '']
+        assert (eol + eol + text + eol + eol + eol + eol).splitlines(True) == [eol, eol, text + eol, eol, eol, eol]
+        assert (text + eol + text).splitlines(False) == [text, text]
+        assert (text + eol + text).splitlines(True) == [text + eol, text]
+        assert (text + eol + text + eol).splitlines(False) == [text, text]
+        assert (text + eol + text + eol).splitlines(True) == [text + eol, text + eol]
+
+assert '\r\r\n'.splitlines() == ['', '']
+assert '\r\r\n'.splitlines(True) == ['\r', '\r\n']
+assert '\n\r\r\n'.splitlines() == ['', '', '']
+assert '\n\r\r\n'.splitlines(True) == ['\n', '\r', '\r\n']
+assert '\n\r\r\n\n'.splitlines() == ['', '', '', '']
+assert '\n\r\r\n\n'.splitlines(True) == ['\n', '\r', '\r\n', '\n']
+assert ''.join(all_ends).splitlines() == [''] * len(all_ends)
+assert ''.join(all_ends).splitlines(True) == all_ends