Skip to content

Commit 5d87971

Browse files
Match Python line boundaries in str.splitlines (#1106)
Signed-off-by: rudrakshkarpe <[email protected]>
1 parent 93227e8 commit 5d87971

4 files changed

Lines changed: 79 additions & 19 deletions

File tree

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Exercise long scans, ordinary line endings, and Python's Unicode line boundaries.
2+
3+
ascii_line = 'a' * 80
4+
unicode_line = 'αβγé😀' * 16
5+
samples = [
6+
('a' * 32768, 1),
7+
((ascii_line + '\n' + 'b' * 80 + '\r\n') * 128, 256),
8+
((unicode_line + '\r\n') * 128, 128),
9+
((unicode_line + '\x85' + ascii_line + '\u2028' + unicode_line + '\u2029') * 64, 192),
10+
]
11+
12+
for text, expected_lines in samples:
13+
for _ in range(20):
14+
lines = text.splitlines()
15+
assert len(lines) == expected_lines
16+
for _ in range(20):
17+
lines = text.splitlines(True)
18+
assert len(lines) == expected_lines
19+
assert ''.join(lines) == text

‎crates/monty/src/types/str.rs‎

Lines changed: 34 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,6 +1436,18 @@ fn rsplit_whitespace_n(s: &str, maxsplit: usize) -> Vec<&str> {
14361436
/// Returns a list of the lines in the string, breaking at line boundaries.
14371437
/// Accepts keepends as either positional or keyword argument.
14381438
fn str_splitlines<'h>(s: &impl HeapView<'h, str>, args: ArgValues, vm: &mut VM<'h>) -> RunResult<Value> {
1439+
// Skip ordinary bytes with one lookup; Unicode boundaries start with 0xc2 or 0xe2.
1440+
const LINE_BOUNDARY_START: [bool; 256] = {
1441+
let mut table = [false; 256];
1442+
let starts = b"\n\r\x0b\x0c\x1c\x1d\x1e\xc2\xe2";
1443+
let mut i = 0;
1444+
while i < starts.len() {
1445+
table[starts[i] as usize] = true;
1446+
i += 1;
1447+
}
1448+
table
1449+
};
1450+
14391451
let keepends = parse_splitlines_args(args, vm)?;
14401452
let s = s.get(vm.heap);
14411453

@@ -1452,26 +1464,29 @@ fn str_splitlines<'h>(s: &impl HeapView<'h, str>, args: ArgValues, vm: &mut VM<'
14521464
let mut line_end = start;
14531465

14541466
while end < len {
1455-
match bytes[end] {
1456-
b'\n' => {
1457-
line_end = end;
1458-
end += 1;
1459-
break;
1460-
}
1461-
b'\r' => {
1462-
line_end = end;
1463-
end += 1;
1464-
// Check for \r\n
1465-
if end < len && bytes[end] == b'\n' {
1466-
end += 1;
1467-
}
1468-
break;
1469-
}
1470-
_ => {
1471-
end += 1;
1472-
line_end = end;
1473-
}
1467+
end += bytes[end..]
1468+
.iter()
1469+
.position(|b| LINE_BOUNDARY_START[usize::from(*b)])
1470+
.unwrap_or(len - end);
1471+
if end == len {
1472+
line_end = end;
1473+
break;
1474+
}
1475+
// Match CPython's line boundaries as complete UTF-8 sequences.
1476+
let boundary_len = match bytes[end] {
1477+
b'\r' if bytes.get(end + 1) == Some(&b'\n') => 2,
1478+
b'\n' | b'\r' | b'\x0b' | b'\x0c' | b'\x1c'..=b'\x1e' => 1,
1479+
0xc2 if bytes.get(end + 1) == Some(&0x85) => 2, // U+0085 NEXT LINE
1480+
0xe2 if matches!(bytes.get(end + 1..end + 3), Some([0x80, 0xa8 | 0xa9])) => 3,
1481+
_ => 0,
1482+
};
1483+
if boundary_len > 0 {
1484+
line_end = end;
1485+
end += boundary_len;
1486+
break;
14741487
}
1488+
end += 1;
1489+
line_end = end;
14751490
}
14761491

14771492
let line = if keepends { &s[start..end] } else { &s[start..line_end] };

‎crates/monty/test_cases/bytes__methods.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,10 @@
224224
assert b''.splitlines() == []
225225
assert b'a\nb'.splitlines(keepends=True) == [b'a\n', b'b']
226226
assert b'a\nb'.splitlines(keepends=False) == [b'a', b'b']
227+
# Unlike str.splitlines(), bytes split only at CR, LF and CRLF.
228+
other_boundaries = b'a\vb\fc\x1cd\x1de\x1ef\x85g\xc2\x85h\xe2\x80\xa8i\xe2\x80\xa9j'
229+
assert other_boundaries.splitlines() == [other_boundaries]
230+
assert other_boundaries.splitlines(True) == [other_boundaries]
227231

228232
# === bytes.partition() ===
229233
assert b'hello world'.partition(b' ') == (b'hello', b' ', b'world')

‎crates/monty/test_cases/str__methods.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,28 @@
283283
assert 'a\r\nb'.splitlines() == ['a', 'b']
284284
assert ''.splitlines() == []
285285

286+
# All Python line boundaries, including multi-byte Unicode separators.
287+
for boundary in ['\n', '\r', '\r\n', '\v', '\f', '\x1c', '\x1d', '\x1e', '\x85', '\u2028', '\u2029']:
288+
for first, last in [('a', 'b'), ('é', '🐍')]:
289+
text = first + boundary + last
290+
assert text.splitlines() == [first, last]
291+
assert text.splitlines(keepends=True) == [first + boundary, last]
292+
assert boundary.splitlines() == ['']
293+
assert boundary.splitlines(True) == [boundary]
294+
assert (boundary * 2).splitlines() == ['', '']
295+
assert (boundary * 2).splitlines(True) == [boundary, boundary]
296+
assert ('a' + boundary).splitlines() == ['a']
297+
assert ('a' + boundary).splitlines(True) == ['a' + boundary]
298+
299+
mixed_lines = 'a\r\n\u2028b\x85c\vd\fe\x1cf\x1dg\x1eh\u2029'
300+
assert mixed_lines.splitlines() == ['a', '', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
301+
assert mixed_lines.splitlines(True) == ['a\r\n', '\u2028', 'b\x85', 'c\v', 'd\f', 'e\x1c', 'f\x1d', 'g\x1e', 'h\u2029']
302+
# Whitespace and UTF-8 bytes resembling separators are not themselves line boundaries.
303+
assert 'a\t b\x1fc\xa0d\u2003e\u200bf\u0084g\u0086h\u2027i\u202aj'.splitlines() == [
304+
'a\t b\x1fc\xa0d\u2003e\u200bf\u0084g\u0086h\u2027i\u202aj'
305+
]
306+
assert 'ą測'.splitlines() == ['ą測']
307+
286308
# partition()
287309
assert 'hello world'.partition(' ') == ('hello', ' ', 'world')
288310
assert 'hello'.partition('x') == ('hello', '', '')

0 commit comments

Comments
 (0)