Skip to content

Commit 88273eb

Browse files
authored
fix: isolate anonymous file statistics cache (apache#22950)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#22935. ## Rationale for this change Anonymous file reads can read the same path with different explicit schemas in the same session. The shared file statistics cache was keyed by table/path metadata, but did not validate that cached statistics matched the schema used to compute them. This could reuse narrower cached statistics for a later wider schema read and panic during statistics projection. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? This PR routes anonymous listing table statistics through a per-table cache instead of the shared session cache. Named tables still use the shared session cache, since their table reference gives the cache a stable identity. It also adds a regression test that first warms statistics with the physical schema, then reads the same Parquet file with a wider explicit schema. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? Yes <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? No API Change <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent efc7b3e commit 88273eb

4 files changed

Lines changed: 134 additions & 12 deletions

File tree

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,7 @@ use datafusion_datasource::schema_adapter::SchemaAdapterFactory;
3434
use datafusion_datasource::{
3535
ListingTableUrl, PartitionedFile, TableSchemaBuilder, compute_all_files_statistics,
3636
};
37-
use datafusion_execution::cache::cache_manager::FileStatisticsCache;
38-
use datafusion_execution::cache::cache_manager::TableScopedPath;
37+
use datafusion_execution::cache::cache_manager::{FileStatisticsCache, TableScopedPath};
3938
use datafusion_expr::dml::InsertOp;
4039
use datafusion_expr::execution_props::ExecutionProps;
4140
use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType};
@@ -264,6 +263,21 @@ impl ListingTable {
264263
self
265264
}
266265

266+
fn statistics_cache(
267+
&self,
268+
has_table_reference: bool,
269+
) -> Option<&Arc<FileStatisticsCache>> {
270+
let shared_cache = self.collected_statistics.as_ref()?;
271+
if has_table_reference || self.schema_source == SchemaSource::Inferred {
272+
Some(shared_cache)
273+
} else {
274+
// Anonymous specified-schema reads can use the same file path with
275+
// different logical schemas. File statistics are schema-dependent,
276+
// so avoid reusing stats computed for a different read schema.
277+
None
278+
}
279+
}
280+
267281
/// Specify the SQL definition for this table, if any
268282
pub fn with_definition(mut self, definition: Option<String>) -> Self {
269283
self.definition = definition;
@@ -807,7 +821,7 @@ impl ListingTable {
807821
let meta = &part_file.object_meta;
808822

809823
// Check cache first - if we have valid cached statistics and ordering
810-
if let Some(cache) = &self.collected_statistics
824+
if let Some(cache) = self.statistics_cache(path.table.is_some())
811825
&& let Some(cached) = cache.get(&path)
812826
&& cached.is_valid_for(meta)
813827
{
@@ -825,7 +839,7 @@ impl ListingTable {
825839
let statistics = Arc::new(file_meta.statistics);
826840

827841
// Store in cache
828-
if let Some(cache) = &self.collected_statistics {
842+
if let Some(cache) = self.statistics_cache(path.table.is_some()) {
829843
cache.put(
830844
&path,
831845
CachedFileMetadata::new(

‎datafusion/core/src/datasource/file_format/options.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ use crate::error::Result;
3434
use crate::execution::context::{SessionConfig, SessionState};
3535

3636
use arrow::datatypes::{DataType, Schema, SchemaRef};
37+
use datafusion_catalog_listing::SchemaSource;
3738
use datafusion_common::config::{ConfigFileDecryptionProperties, TableOptions};
3839
use datafusion_common::{
3940
DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION,
@@ -595,6 +596,11 @@ pub trait ReadOptions<'a> {
595596
table_path: ListingTableUrl,
596597
) -> Result<SchemaRef>;
597598

599+
/// Returns whether the read schema was inferred or specified.
600+
fn schema_source(&self) -> SchemaSource {
601+
SchemaSource::Specified
602+
}
603+
598604
/// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.
599605
async fn _get_resolved_schema(
600606
&'a self,
@@ -652,6 +658,10 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {
652658
self._get_resolved_schema(config, state, table_path, self.schema)
653659
.await
654660
}
661+
662+
fn schema_source(&self) -> SchemaSource {
663+
schema_source_from_option(self.schema)
664+
}
655665
}
656666

657667
#[cfg(feature = "parquet")]
@@ -695,6 +705,10 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {
695705
self._get_resolved_schema(config, state, table_path, self.schema)
696706
.await
697707
}
708+
709+
fn schema_source(&self) -> SchemaSource {
710+
schema_source_from_option(self.schema)
711+
}
698712
}
699713

700714
#[async_trait]
@@ -725,6 +739,10 @@ impl ReadOptions<'_> for JsonReadOptions<'_> {
725739
self._get_resolved_schema(config, state, table_path, self.schema)
726740
.await
727741
}
742+
743+
fn schema_source(&self) -> SchemaSource {
744+
schema_source_from_option(self.schema)
745+
}
728746
}
729747

730748
#[cfg(feature = "avro")]
@@ -751,6 +769,10 @@ impl ReadOptions<'_> for AvroReadOptions<'_> {
751769
self._get_resolved_schema(config, state, table_path, self.schema)
752770
.await
753771
}
772+
773+
fn schema_source(&self) -> SchemaSource {
774+
schema_source_from_option(self.schema)
775+
}
754776
}
755777

756778
#[async_trait]
@@ -776,4 +798,16 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> {
776798
self._get_resolved_schema(config, state, table_path, self.schema)
777799
.await
778800
}
801+
802+
fn schema_source(&self) -> SchemaSource {
803+
schema_source_from_option(self.schema)
804+
}
805+
}
806+
807+
fn schema_source_from_option(schema: Option<&Schema>) -> SchemaSource {
808+
if schema.is_some() {
809+
SchemaSource::Specified
810+
} else {
811+
SchemaSource::Inferred
812+
}
779813
}

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ use datafusion_catalog::memory::MemorySchemaProvider;
6565
use datafusion_catalog::{
6666
DynamicFileCatalog, TableFunction, TableFunctionImpl, UrlTableFactory,
6767
};
68+
use datafusion_catalog_listing::SchemaSource;
6869
use datafusion_common::config::{ConfigField, ConfigOptions};
6970
use datafusion_common::metadata::ScalarAndMetadata;
7071
use datafusion_common::{
@@ -1725,12 +1726,20 @@ impl SessionContext {
17251726
}
17261727
}
17271728

1728-
let resolved_schema = options
1729-
.get_resolved_schema(&session_config, self.state(), table_paths[0].clone())
1730-
.await?;
1729+
let schema_table_path = table_paths[0].clone();
17311730
let config = ListingTableConfig::new_with_multi_paths(table_paths)
1732-
.with_listing_options(listing_options)
1733-
.with_schema(resolved_schema);
1731+
.with_listing_options(listing_options);
1732+
let config = match options.schema_source() {
1733+
SchemaSource::Inferred | SchemaSource::Unset => {
1734+
config.infer_schema(&self.state()).await?
1735+
}
1736+
SchemaSource::Specified => {
1737+
let resolved_schema = options
1738+
.get_resolved_schema(&session_config, self.state(), schema_table_path)
1739+
.await?;
1740+
config.with_schema(resolved_schema)
1741+
}
1742+
};
17341743
let provider = ListingTable::try_new(config)?
17351744
.with_cache(self.runtime_env().cache_manager.get_file_statistic_cache());
17361745
self.read_table(Arc::new(provider))

‎datafusion/core/tests/parquet/file_statistics.rs‎

Lines changed: 68 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use std::fs;
1919
use std::sync::Arc;
2020

21+
use arrow::datatypes::{DataType, Field, Schema};
2122
use datafusion::datasource::TableProvider;
2223
use datafusion::datasource::file_format::parquet::ParquetFormat;
2324
use datafusion::datasource::listing::{
@@ -26,9 +27,9 @@ use datafusion::datasource::listing::{
2627
use datafusion::datasource::source::DataSourceExec;
2728
use datafusion::execution::context::SessionState;
2829
use datafusion::execution::session_state::SessionStateBuilder;
29-
use datafusion::prelude::SessionContext;
30-
use datafusion_common::DFSchema;
30+
use datafusion::prelude::{ParquetReadOptions, SessionContext};
3131
use datafusion_common::stats::Precision;
32+
use datafusion_common::{DFSchema, TableReference};
3233
use datafusion_execution::cache::cache_manager::{
3334
CacheManagerConfig, DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,
3435
DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, FileStatisticsCache, ListFilesCache,
@@ -111,7 +112,9 @@ async fn check_stats_precision_with_filter_pushdown() {
111112
async fn load_table_stats_with_session_level_cache() {
112113
let testdata = datafusion::test_util::parquet_test_data();
113114
let filename = format!("{}/{}", testdata, "alltypes_plain.parquet");
114-
let table_path = ListingTableUrl::parse(filename).unwrap();
115+
let table_path = ListingTableUrl::parse(filename)
116+
.unwrap()
117+
.with_table_ref(TableReference::bare("alltypes_plain"));
115118

116119
let (cache1, _, mut state1) = get_cache_runtime_state();
117120
let cfg_1 = state1.config_mut();
@@ -193,6 +196,68 @@ async fn load_table_stats_with_session_level_cache() {
193196
assert_eq!(get_static_cache_size(&state1), 1);
194197
}
195198

199+
#[tokio::test]
200+
async fn anonymous_parquet_stats_cache_with_explicit_wider_schema() {
201+
let temp_dir = tempdir().unwrap();
202+
let parquet_path = temp_dir.path().join("data.parquet");
203+
let parquet_path = parquet_path.to_string_lossy().to_string();
204+
205+
let ctx = SessionContext::new_with_config(
206+
SessionConfig::new().with_collect_statistics(true),
207+
);
208+
let cache = ctx
209+
.runtime_env()
210+
.cache_manager
211+
.get_file_statistic_cache()
212+
.unwrap();
213+
214+
ctx.sql(&format!(
215+
"COPY (
216+
SELECT 1::BIGINT AS id, 1000::BIGINT AS population
217+
) TO '{parquet_path}' STORED AS PARQUET"
218+
))
219+
.await
220+
.unwrap()
221+
.collect()
222+
.await
223+
.unwrap();
224+
225+
assert_eq!(cache.len(), 0);
226+
227+
ctx.read_parquet(&parquet_path, ParquetReadOptions::default())
228+
.await
229+
.unwrap()
230+
.collect()
231+
.await
232+
.unwrap();
233+
234+
assert_eq!(cache.len(), 1);
235+
236+
let wider_schema = Schema::new(vec![
237+
Field::new("id", DataType::Int64, true),
238+
Field::new("population", DataType::Int64, true),
239+
Field::new("extra", DataType::Int64, true),
240+
]);
241+
242+
let plan = ctx
243+
.read_parquet(
244+
&parquet_path,
245+
ParquetReadOptions::default().schema(&wider_schema),
246+
)
247+
.await
248+
.unwrap()
249+
.select_columns(&["id", "extra"])
250+
.unwrap()
251+
.create_physical_plan()
252+
.await
253+
.unwrap();
254+
255+
let stats = plan.statistics_with_args(&StatisticsArgs::new()).unwrap();
256+
assert_eq!(stats.column_statistics.len(), 2);
257+
assert_eq!(stats.column_statistics[1].null_count, Precision::Exact(1));
258+
assert_eq!(cache.len(), 1);
259+
}
260+
196261
#[tokio::test]
197262
async fn list_files_with_session_level_cache() {
198263
let p_name = "alltypes_plain.parquet";

0 commit comments

Comments
 (0)