Skip to content

Commit 45c61c4

Browse files
committed
fix(tools): normalise CR and CRLF line endings before parsing
`remove_blank_lines` only stripped trailing \r/\n bytes, leaving internal \r\n or lone \r sequences intact. tree-sitter 0.26.8 only advances its row counter on \n, so CR-only files collapsed all content onto row 0, producing incorrect sloc/cloc/blank metrics. Rewrite as `normalize_line_endings`: an in-place write-pointer pass that converts every \r\n → \n and every lone \r → \n before trimming trailing newlines. Also normalise CRLF in `check_func_space` (test helper) so test strings with \r\n behave identically to real files. Adds four unit tests for normalize_line_endings and three LOC regression canaries (python_cloc_crlf_matches_lf, python_blank_crlf_matches_lf, rust_cloc_crlf_matches_lf) that will catch any future tree-sitter row-counting regression with \r bytes. Fixes the CR-only case of mozilla/rust-code-analysis#994.
1 parent b52a1c5 commit 45c61c4

2 files changed

Lines changed: 155 additions & 14 deletions

File tree

‎src/metrics/loc.rs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4927,4 +4927,76 @@ f",
49274927
},
49284928
);
49294929
}
4930+
4931+
// CRLF regression tests: metrics must be identical regardless of line ending style.
4932+
// These also serve as canaries for tree-sitter row-counting behaviour with \r bytes.
4933+
4934+
#[test]
4935+
fn python_cloc_crlf_matches_lf() {
4936+
check_metrics::<PythonParser>("# comment\nx = 1", "foo.py", |m| {
4937+
assert_eq!(m.loc.cloc(), 1.0);
4938+
assert_eq!(m.loc.ploc(), 1.0);
4939+
assert_eq!(m.loc.sloc(), 2.0);
4940+
assert_eq!(m.loc.blank(), 0.0);
4941+
});
4942+
check_metrics::<PythonParser>("# comment\r\nx = 1", "foo.py", |m| {
4943+
assert_eq!(m.loc.cloc(), 1.0);
4944+
assert_eq!(m.loc.ploc(), 1.0);
4945+
assert_eq!(m.loc.sloc(), 2.0);
4946+
assert_eq!(m.loc.blank(), 0.0);
4947+
});
4948+
// Lone-CR (old Mac line endings) is the true canary: without CR normalisation,
4949+
// tree-sitter 0.26.8 only advances its row counter on \n, collapsing all content
4950+
// onto row 0 and producing wrong sloc/cloc metrics.
4951+
check_metrics::<PythonParser>("# comment\rx = 1", "foo.py", |m| {
4952+
assert_eq!(m.loc.cloc(), 1.0);
4953+
assert_eq!(m.loc.ploc(), 1.0);
4954+
assert_eq!(m.loc.sloc(), 2.0);
4955+
assert_eq!(m.loc.blank(), 0.0);
4956+
});
4957+
}
4958+
4959+
#[test]
4960+
fn python_blank_crlf_matches_lf() {
4961+
check_metrics::<PythonParser>("# comment\n\nx = 1", "foo.py", |m| {
4962+
assert_eq!(m.loc.blank(), 1.0);
4963+
});
4964+
check_metrics::<PythonParser>("# comment\r\n\r\nx = 1", "foo.py", |m| {
4965+
assert_eq!(m.loc.blank(), 1.0);
4966+
});
4967+
// Lone-CR: without normalisation the blank \r line stays on row 0 and is not counted.
4968+
check_metrics::<PythonParser>("# comment\r\rx = 1", "foo.py", |m| {
4969+
assert_eq!(m.loc.blank(), 1.0);
4970+
});
4971+
}
4972+
4973+
#[test]
4974+
fn rust_cloc_crlf_matches_lf() {
4975+
check_metrics::<RustParser>(
4976+
"fn f() {\n // comment\n let x = 1;\n}",
4977+
"foo.rs",
4978+
|m| {
4979+
assert_eq!(m.loc.cloc(), 1.0);
4980+
assert_eq!(m.loc.sloc(), 4.0);
4981+
},
4982+
);
4983+
check_metrics::<RustParser>(
4984+
"fn f() {\r\n // comment\r\n let x = 1;\r\n}",
4985+
"foo.rs",
4986+
|m| {
4987+
assert_eq!(m.loc.cloc(), 1.0);
4988+
assert_eq!(m.loc.sloc(), 4.0);
4989+
},
4990+
);
4991+
// Lone-CR: without normalisation, tree-sitter 0.26.8 only advances its row counter on
4992+
// \n, so all content collapses onto row 0 and sloc becomes 1 instead of 4.
4993+
check_metrics::<RustParser>(
4994+
"fn f() {\r // comment\r let x = 1;\r}",
4995+
"foo.rs",
4996+
|m| {
4997+
assert_eq!(m.loc.cloc(), 1.0);
4998+
assert_eq!(m.loc.sloc(), 4.0);
4999+
},
5000+
);
5001+
}
49305002
}

‎src/tools.rs‎

Lines changed: 83 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use termcolor::{Color, ColorSpec, StandardStreamLock, WriteColor};
1111
use crate::langs::fake;
1212
use crate::langs::*;
1313

14-
/// Reads a file.
14+
/// Reads a file, normalising all CR-only and CRLF line endings to LF.
1515
///
1616
/// # Examples
1717
///
@@ -28,12 +28,14 @@ pub fn read_file(path: &Path) -> std::io::Result<Vec<u8>> {
2828
let mut data = Vec::new();
2929
file.read_to_end(&mut data)?;
3030

31-
remove_blank_lines(&mut data);
31+
normalize_line_endings(&mut data);
3232

3333
Ok(data)
3434
}
3535

36-
/// Reads a file and adds an `EOL` at its end.
36+
/// Reads a file, normalising all CR-only and CRLF line endings to LF, and ensures
37+
/// the buffer ends with exactly one `\n`. Returns `None` for files ≤ 3 bytes or
38+
/// files that appear to be non-UTF-8.
3739
///
3840
/// # Examples
3941
///
@@ -82,7 +84,7 @@ pub fn read_file_with_eol(path: &Path) -> std::io::Result<Option<Vec<u8>>> {
8284

8385
file.read_to_end(&mut data)?;
8486

85-
remove_blank_lines(&mut data);
87+
normalize_line_endings(&mut data);
8688

8789
Ok(Some(data))
8890
}
@@ -245,16 +247,30 @@ pub fn guess_language<'a, P: AsRef<Path>>(buf: &[u8], path: P) -> (Option<LANG>,
245247
}
246248
}
247249

248-
/// Replaces \n and \r ending characters with a single generic \n
249-
pub(crate) fn remove_blank_lines(data: &mut Vec<u8>) {
250-
let count_trailing = data
251-
.iter()
252-
.rev()
253-
.take_while(|&c| *c == b'\n' || *c == b'\r')
254-
.count();
255-
if count_trailing > 0 {
256-
data.truncate(data.len() - count_trailing);
250+
/// Normalises all CR-only and CRLF line endings to LF throughout the buffer,
251+
/// then ensures the buffer ends with exactly one `\n`.
252+
pub(crate) fn normalize_line_endings(data: &mut Vec<u8>) {
253+
// In-place compaction: write pointer stays ≤ read pointer, so no extra allocation.
254+
let mut w = 0;
255+
let mut r = 0;
256+
while r < data.len() {
257+
if data[r] == b'\r' {
258+
data[w] = b'\n';
259+
w += 1;
260+
r += if data.get(r + 1).copied() == Some(b'\n') {
261+
2
262+
} else {
263+
1
264+
};
265+
} else {
266+
data[w] = data[r];
267+
w += 1;
268+
r += 1;
269+
}
257270
}
271+
data.truncate(w);
272+
let trailing = data.iter().rev().take_while(|&&c| c == b'\n').count();
273+
data.truncate(data.len() - trailing);
258274
data.push(b'\n');
259275
}
260276

@@ -390,7 +406,9 @@ pub(crate) fn check_func_space<T: crate::ParserTrait, F: Fn(crate::FuncSpace)>(
390406
check: F,
391407
) {
392408
let path = std::path::PathBuf::from(filename);
393-
let mut trimmed_bytes = source.trim_end().trim_matches('\n').as_bytes().to_vec();
409+
// Mirror the CRLF/CR normalisation that read_file_with_eol applies via normalize_line_endings
410+
let normalized = source.replace("\r\n", "\n").replace('\r', "\n");
411+
let mut trimmed_bytes = normalized.trim_end().trim_matches('\n').as_bytes().to_vec();
394412
trimmed_bytes.push(b'\n');
395413
let parser = T::new(trimmed_bytes, &path, None);
396414
let func_space = crate::metrics(&parser, &path).unwrap();
@@ -425,6 +443,13 @@ mod tests {
425443
(b"\xEF\xBBabc\n".to_vec(), None),
426444
(b"abcdef\n".to_vec(), Some(b"abcdef\n".to_vec())),
427445
(b"abcdef".to_vec(), Some(b"abcdef\n".to_vec())),
446+
// CRLF throughout should be normalised to LF
447+
(b"abc\r\ndef\r\n".to_vec(), Some(b"abc\ndef\n".to_vec())),
448+
// UTF-8 BOM + CRLF
449+
(
450+
b"\xEF\xBB\xBFabc\r\ndef\r\n".to_vec(),
451+
Some(b"abc\ndef\n".to_vec()),
452+
),
428453
];
429454
for (d, expected) in data {
430455
write_file(&tmp_path, &d).unwrap();
@@ -492,4 +517,48 @@ mod tests {
492517
(Some(LANG::Cpp), "obj-c/c++")
493518
);
494519
}
520+
521+
#[test]
522+
fn normalize_line_endings_normalizes_crlf() {
523+
let mut d = b"code\r\n# comment\r\n".to_vec();
524+
normalize_line_endings(&mut d);
525+
assert_eq!(d, b"code\n# comment\n");
526+
}
527+
528+
#[test]
529+
fn normalize_line_endings_normalizes_lone_cr() {
530+
let mut d = b"code\r# comment\r".to_vec();
531+
normalize_line_endings(&mut d);
532+
assert_eq!(d, b"code\n# comment\n");
533+
}
534+
535+
#[test]
536+
fn normalize_line_endings_normalizes_cr_before_crlf() {
537+
// lone CR followed immediately by CRLF → two separate line breaks
538+
let mut d = b"a\r\r\nb".to_vec();
539+
normalize_line_endings(&mut d);
540+
assert_eq!(d, b"a\n\nb\n");
541+
}
542+
543+
#[test]
544+
fn normalize_line_endings_normalizes_crlf_blank_line() {
545+
let mut d = b"a\r\n\r\nb\r\n".to_vec();
546+
normalize_line_endings(&mut d);
547+
assert_eq!(d, b"a\n\nb\n");
548+
}
549+
550+
#[test]
551+
fn normalize_line_endings_empty_buffer() {
552+
let mut d = b"".to_vec();
553+
normalize_line_endings(&mut d);
554+
assert_eq!(d, b"\n");
555+
}
556+
557+
#[test]
558+
fn normalize_line_endings_mixed_endings() {
559+
// LF + lone-CR + CRLF in one buffer — each is converted independently.
560+
let mut d = b"a\nb\rc\r\nd".to_vec();
561+
normalize_line_endings(&mut d);
562+
assert_eq!(d, b"a\nb\nc\nd\n");
563+
}
495564
}

0 commit comments

Comments
 (0)