Skip to content

Commit d33d7f1

Browse files
mkleenfred1268
authored andcommitted
fix: Avoid panicing when stats are not available for a file group split (apache#23277)
## Which issue does this PR close? - Closes apache#23219. ## Rationale for this change The query from the issue: ```sql SELECT (((Cast(id AS BIGINT) % 1024) + 1024) % 1024) AS computed_bucket FROM profile ORDER BY computed_bucket, Cast(id AS BIGINT) limit 10; ``` panics: ``` thread 'main' panicked at .../datafusion-datasource-54.0.0/src/statistics.rs:100:48: index out of bounds: the len is 0 but the index is 0 ``` The underlying issue is that the current code panics when files are split by statistics and there are no statistics available for the column where the sort order is defined in this case `computed_bucket`. ## What changes are included in this PR? - Fix in `MinMaxStatistics` to check if there are stats available for a given column - Test ## Are these changes tested? Yes ## Are there any user-facing changes? No
1 parent e146411 commit d33d7f1

1 file changed

Lines changed: 37 additions & 2 deletions

File tree

‎datafusion/datasource/src/statistics.rs‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,16 @@ impl MinMaxStatistics {
9797
.zip(s.column_statistics[i].max_value.get_value().cloned())
9898
.ok_or_else(|| plan_datafusion_err!("statistics not found"))
9999
} else {
100-
let partition_value = &pv[i - s.column_statistics.len()];
101-
Ok((partition_value.clone(), partition_value.clone()))
100+
if let Some(partition_value) =
101+
pv.get(i - s.column_statistics.len())
102+
{
103+
Ok((partition_value.clone(), partition_value.clone()))
104+
} else {
105+
Err(plan_datafusion_err!(
106+
"statistics not found for partition, expected at most {}",
107+
s.column_statistics.len()
108+
))
109+
}
102110
}
103111
})
104112
.collect::<Result<Vec<_>>>()?
@@ -890,4 +898,31 @@ mod tests {
890898

891899
Ok(())
892900
}
901+
902+
#[test]
903+
fn min_max_statistics_missing_column_stats_returns_error() {
904+
let schema = test_schema();
905+
let sort_order =
906+
[PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into();
907+
let files = [
908+
file_with_stats("f1.parquet", Statistics::default()),
909+
file_with_stats("f2.parquet", Statistics::default()),
910+
];
911+
912+
let err = match MinMaxStatistics::new_from_files(
913+
&sort_order,
914+
&schema,
915+
None,
916+
files.iter(),
917+
) {
918+
Ok(_) => panic!("expected missing statistics error"),
919+
Err(err) => err,
920+
};
921+
922+
assert!(
923+
err.to_string()
924+
.contains("statistics not found for partition"),
925+
"unexpected error: {err:?}"
926+
);
927+
}
893928
}

0 commit comments

Comments
 (0)