Skip to content

Commit 18028cd

Browse files
committed
Count actual newline bytes in dictionary training samples
1 parent 0cb868f commit 18028cd

1 file changed

Lines changed: 6 additions & 2 deletions

File tree

‎zstd-safe/src/tests.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,12 @@ fn test_cctx_cycle() {
7979
fn test_dictionary() {
8080
// Prepare some content to train the dictionary.
8181
let bytes = LONG_CONTENT.as_bytes();
82-
let line_sizes: Vec<usize> =
83-
LONG_CONTENT.lines().map(|line| line.len() + 1).collect();
82+
// Include the actual newline bytes, including CRLF on Windows and a
83+
// possibly unterminated final line.
84+
let line_sizes: Vec<usize> = LONG_CONTENT
85+
.split_inclusive('\n')
86+
.map(|line| line.len())
87+
.collect();
8488

8589
// Train the dictionary
8690
let mut dict_buffer = std::vec![0u8; 100_000];

0 commit comments

Comments
 (0)