Skip to content

Commit 2ec55f9

Browse files
[branch-54] fix: isolate anonymous file statistics cache (backport #22950, adapted) (#23573)
## Which issue does this PR close? - Backport of #22950 to `branch-54` (for 54.1.0, tracked in #22547). - Fixes panic #22935 (reporter bisected to a regression between 53.1.0 and 54.0.0; introduced by #20047). ## Rationale for this change Anonymous file reads can read the same path with different explicit schemas in one session. The shared file-statistics cache did not validate that cached statistics matched the read schema, so a narrower cached entry could be reused for a later wider-schema read and panic during statistics projection (`index out of bounds` in projection). No API change, so it fits the backport criteria. ## What changes are included in this PR? This is an **adapted** backport. The core fix (`table.rs`, `options.rs`, `context/mod.rs`) is applied; two conflicts were resolved for `branch-54`'s API shape: - `ListingTable::statistics_cache()` returns `Option<&Arc<dyn FileStatisticsCache>>` (on `branch-54`, `FileStatisticsCache` is a trait, not a concrete type as on `main`). - Kept `branch-54`'s import paths for `TableScopedPath` / `FileStatisticsCache`. - The regression test `anonymous_parquet_stats_cache_with_explicit_wider_schema` computes statistics via `plan.partition_statistics(None)` instead of `statistics_with_args`, which does not exist on `branch-54`. ## Are these changes tested? Yes. The regression test passes with the fix and, without it, reproduces the exact reported panic (`index out of bounds: the len is 2 but the index is 2`), confirming it exercises the regression. ## Are there any user-facing changes? No API change. Fixes a panic on anonymous multi-schema reads of the same path. Co-authored-by: Kumar Ujjawal <[email protected]>
1 parent 70bdc95 commit 2ec55f9

4 files changed

Lines changed: 133 additions & 10 deletions

File tree

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,21 @@ impl ListingTable {
264264
self
265265
}
266266

267+
fn statistics_cache(
268+
&self,
269+
has_table_reference: bool,
270+
) -> Option<&Arc<dyn FileStatisticsCache>> {
271+
let shared_cache = self.collected_statistics.as_ref()?;
272+
if has_table_reference || self.schema_source == SchemaSource::Inferred {
273+
Some(shared_cache)
274+
} else {
275+
// Anonymous specified-schema reads can use the same file path with
276+
// different logical schemas. File statistics are schema-dependent,
277+
// so avoid reusing stats computed for a different read schema.
278+
None
279+
}
280+
}
281+
267282
/// Specify the SQL definition for this table, if any
268283
pub fn with_definition(mut self, definition: Option<String>) -> Self {
269284
self.definition = definition;
@@ -806,7 +821,7 @@ impl ListingTable {
806821
let meta = &part_file.object_meta;
807822

808823
// Check cache first - if we have valid cached statistics and ordering
809-
if let Some(cache) = &self.collected_statistics
824+
if let Some(cache) = self.statistics_cache(path.table.is_some())
810825
&& let Some(cached) = cache.get(&path)
811826
&& cached.is_valid_for(meta)
812827
{
@@ -824,7 +839,7 @@ impl ListingTable {
824839
let statistics = Arc::new(file_meta.statistics);
825840

826841
// Store in cache
827-
if let Some(cache) = &self.collected_statistics {
842+
if let Some(cache) = self.statistics_cache(path.table.is_some()) {
828843
cache.put(
829844
&path,
830845
CachedFileMetadata::new(

‎datafusion/core/src/datasource/file_format/options.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ use crate::error::Result;
3434
use crate::execution::context::{SessionConfig, SessionState};
3535

3636
use arrow::datatypes::{DataType, Schema, SchemaRef};
37+
use datafusion_catalog_listing::SchemaSource;
3738
use datafusion_common::config::{ConfigFileDecryptionProperties, TableOptions};
3839
use datafusion_common::{
3940
DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION,
@@ -595,6 +596,11 @@ pub trait ReadOptions<'a> {
595596
table_path: ListingTableUrl,
596597
) -> Result<SchemaRef>;
597598

599+
/// Returns whether the read schema was inferred or specified.
600+
fn schema_source(&self) -> SchemaSource {
601+
SchemaSource::Specified
602+
}
603+
598604
/// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.
599605
async fn _get_resolved_schema(
600606
&'a self,
@@ -653,6 +659,10 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {
653659
self._get_resolved_schema(config, state, table_path, self.schema)
654660
.await
655661
}
662+
663+
fn schema_source(&self) -> SchemaSource {
664+
schema_source_from_option(self.schema)
665+
}
656666
}
657667

658668
#[cfg(feature = "parquet")]
@@ -697,6 +707,10 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {
697707
self._get_resolved_schema(config, state, table_path, self.schema)
698708
.await
699709
}
710+
711+
fn schema_source(&self) -> SchemaSource {
712+
schema_source_from_option(self.schema)
713+
}
700714
}
701715

702716
#[async_trait]
@@ -728,6 +742,10 @@ impl ReadOptions<'_> for JsonReadOptions<'_> {
728742
self._get_resolved_schema(config, state, table_path, self.schema)
729743
.await
730744
}
745+
746+
fn schema_source(&self) -> SchemaSource {
747+
schema_source_from_option(self.schema)
748+
}
731749
}
732750

733751
#[cfg(feature = "avro")]
@@ -755,6 +773,10 @@ impl ReadOptions<'_> for AvroReadOptions<'_> {
755773
self._get_resolved_schema(config, state, table_path, self.schema)
756774
.await
757775
}
776+
777+
fn schema_source(&self) -> SchemaSource {
778+
schema_source_from_option(self.schema)
779+
}
758780
}
759781

760782
#[async_trait]
@@ -781,4 +803,16 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> {
781803
self._get_resolved_schema(config, state, table_path, self.schema)
782804
.await
783805
}
806+
807+
fn schema_source(&self) -> SchemaSource {
808+
schema_source_from_option(self.schema)
809+
}
810+
}
811+
812+
fn schema_source_from_option(schema: Option<&Schema>) -> SchemaSource {
813+
if schema.is_some() {
814+
SchemaSource::Specified
815+
} else {
816+
SchemaSource::Inferred
817+
}
784818
}

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ use datafusion_catalog::memory::MemorySchemaProvider;
6464
use datafusion_catalog::{
6565
DynamicFileCatalog, TableFunction, TableFunctionImpl, UrlTableFactory,
6666
};
67+
use datafusion_catalog_listing::SchemaSource;
6768
use datafusion_common::config::{ConfigField, ConfigOptions};
6869
use datafusion_common::metadata::ScalarAndMetadata;
6970
use datafusion_common::{
@@ -1725,12 +1726,20 @@ impl SessionContext {
17251726
}
17261727
}
17271728

1728-
let resolved_schema = options
1729-
.get_resolved_schema(&session_config, self.state(), table_paths[0].clone())
1730-
.await?;
1729+
let schema_table_path = table_paths[0].clone();
17311730
let config = ListingTableConfig::new_with_multi_paths(table_paths)
1732-
.with_listing_options(listing_options)
1733-
.with_schema(resolved_schema);
1731+
.with_listing_options(listing_options);
1732+
let config = match options.schema_source() {
1733+
SchemaSource::Inferred | SchemaSource::Unset => {
1734+
config.infer_schema(&self.state()).await?
1735+
}
1736+
SchemaSource::Specified => {
1737+
let resolved_schema = options
1738+
.get_resolved_schema(&session_config, self.state(), schema_table_path)
1739+
.await?;
1740+
config.with_schema(resolved_schema)
1741+
}
1742+
};
17341743
let provider = ListingTable::try_new(config)?
17351744
.with_cache(self.runtime_env().cache_manager.get_file_statistic_cache());
17361745
self.read_table(Arc::new(provider))

‎datafusion/core/tests/parquet/file_statistics.rs‎

Lines changed: 68 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use std::fs;
1919
use std::sync::Arc;
2020

21+
use arrow::datatypes::{DataType, Field, Schema};
2122
use datafusion::datasource::TableProvider;
2223
use datafusion::datasource::file_format::parquet::ParquetFormat;
2324
use datafusion::datasource::listing::{
@@ -26,9 +27,9 @@ use datafusion::datasource::listing::{
2627
use datafusion::datasource::source::DataSourceExec;
2728
use datafusion::execution::context::SessionState;
2829
use datafusion::execution::session_state::SessionStateBuilder;
29-
use datafusion::prelude::SessionContext;
30-
use datafusion_common::DFSchema;
30+
use datafusion::prelude::{ParquetReadOptions, SessionContext};
3131
use datafusion_common::stats::Precision;
32+
use datafusion_common::{DFSchema, TableReference};
3233
use datafusion_execution::cache::DefaultListFilesCache;
3334
use datafusion_execution::cache::cache_manager::{
3435
CacheManagerConfig, FileStatisticsCache,
@@ -105,7 +106,9 @@ async fn check_stats_precision_with_filter_pushdown() {
105106
async fn load_table_stats_with_session_level_cache() {
106107
let testdata = datafusion::test_util::parquet_test_data();
107108
let filename = format!("{}/{}", testdata, "alltypes_plain.parquet");
108-
let table_path = ListingTableUrl::parse(filename).unwrap();
109+
let table_path = ListingTableUrl::parse(filename)
110+
.unwrap()
111+
.with_table_ref(TableReference::bare("alltypes_plain"));
109112

110113
let (cache1, _, state1) = get_cache_runtime_state();
111114

@@ -166,6 +169,68 @@ async fn load_table_stats_with_session_level_cache() {
166169
assert_eq!(get_static_cache_size(&state1), 1);
167170
}
168171

172+
#[tokio::test]
173+
async fn anonymous_parquet_stats_cache_with_explicit_wider_schema() {
174+
let temp_dir = tempdir().unwrap();
175+
let parquet_path = temp_dir.path().join("data.parquet");
176+
let parquet_path = parquet_path.to_string_lossy().to_string();
177+
178+
let ctx = SessionContext::new_with_config(
179+
SessionConfig::new().with_collect_statistics(true),
180+
);
181+
let cache = ctx
182+
.runtime_env()
183+
.cache_manager
184+
.get_file_statistic_cache()
185+
.unwrap();
186+
187+
ctx.sql(&format!(
188+
"COPY (
189+
SELECT 1::BIGINT AS id, 1000::BIGINT AS population
190+
) TO '{parquet_path}' STORED AS PARQUET"
191+
))
192+
.await
193+
.unwrap()
194+
.collect()
195+
.await
196+
.unwrap();
197+
198+
assert_eq!(cache.len(), 0);
199+
200+
ctx.read_parquet(&parquet_path, ParquetReadOptions::default())
201+
.await
202+
.unwrap()
203+
.collect()
204+
.await
205+
.unwrap();
206+
207+
assert_eq!(cache.len(), 1);
208+
209+
let wider_schema = Schema::new(vec![
210+
Field::new("id", DataType::Int64, true),
211+
Field::new("population", DataType::Int64, true),
212+
Field::new("extra", DataType::Int64, true),
213+
]);
214+
215+
let plan = ctx
216+
.read_parquet(
217+
&parquet_path,
218+
ParquetReadOptions::default().schema(&wider_schema),
219+
)
220+
.await
221+
.unwrap()
222+
.select_columns(&["id", "extra"])
223+
.unwrap()
224+
.create_physical_plan()
225+
.await
226+
.unwrap();
227+
228+
let stats = plan.partition_statistics(None).unwrap();
229+
assert_eq!(stats.column_statistics.len(), 2);
230+
assert_eq!(stats.column_statistics[1].null_count, Precision::Exact(1));
231+
assert_eq!(cache.len(), 1);
232+
}
233+
169234
#[tokio::test]
170235
async fn list_files_with_session_level_cache() {
171236
let p_name = "alltypes_plain.parquet";

0 commit comments

Comments
 (0)