Skip to content

Commit 6f53d57

Browse files
authored
fix: avoid exact scan row counts for SSTs with unknown counts (#9759)
fix: avoid exact scan row counts for legacy SSTs Treat selected SSTs with unknown row counts as non-exact before checking region partition visibility. Preserve exact statistics for known counts and cover legacy metadata mixed with known-count files. Fixes #9718 Signed-off-by: Aumnertic <[email protected]>
1 parent a7230f7 commit 6f53d57

1 file changed

Lines changed: 41 additions & 2 deletions

File tree

‎src/mito2/src/read/scan_region.rs‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1703,9 +1703,14 @@ impl ScanInput {
17031703
);
17041704
}
17051705

1706-
/// Whether physical source row counts equal the rows visible to this region,
1707-
/// before applying query predicates.
1706+
/// Whether physical source row counts are known and equal the rows visible
1707+
/// to this region, before applying query predicates.
17081708
pub(crate) fn total_rows_is_exact(&self) -> bool {
1709+
// Legacy SST metadata uses zero for unknown row counts.
1710+
if self.files.iter().any(|file| file.num_rows() == 0) {
1711+
return false;
1712+
}
1713+
17091714
if self.region_partition_expr.is_none() {
17101715
return true;
17111716
}
@@ -2542,6 +2547,39 @@ mod tests {
25422547
);
25432548
}
25442549

2550+
#[tokio::test]
2551+
async fn test_total_rows_is_exact_with_legacy_file() {
2552+
let file_meta = FileMeta {
2553+
num_rows: 10,
2554+
..Default::default()
2555+
};
2556+
let mut json = serde_json::to_value(&file_meta).unwrap();
2557+
json.as_object_mut().unwrap().remove("num_rows");
2558+
let legacy_meta = serde_json::from_value::<FileMeta>(json).unwrap();
2559+
assert_eq!(legacy_meta.num_rows, 0);
2560+
2561+
let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
2562+
for (file_metas, expected_rows, exact) in [
2563+
(vec![file_meta.clone()], 10, true),
2564+
(vec![legacy_meta.clone()], 0, false),
2565+
(vec![file_meta, legacy_meta], 10, false),
2566+
] {
2567+
let files = file_metas
2568+
.into_iter()
2569+
.map(|meta| {
2570+
FileHandle::new(meta, Arc::new(crate::sst::file_purger::NoopFilePurger))
2571+
})
2572+
.collect();
2573+
let input = new_scan_input(metadata.clone(), vec![])
2574+
.await
2575+
.with_append_mode(true)
2576+
.with_files(files)
2577+
.build();
2578+
assert_eq!(input.total_rows(), expected_rows);
2579+
assert_eq!(input.total_rows_is_exact(), exact);
2580+
}
2581+
}
2582+
25452583
#[tokio::test]
25462584
async fn test_total_rows_is_exact_after_partition_filter() {
25472585
let expr = partition_col("k0").gt_eq(Value::String("foo".into()));
@@ -2564,6 +2602,7 @@ mod tests {
25642602
.map(|partition_expr| {
25652603
FileHandle::new(
25662604
FileMeta {
2605+
num_rows: 1,
25672606
partition_expr,
25682607
..Default::default()
25692608
},

0 commit comments

Comments
 (0)