Skip to content

Commit de0e5d9

Browse files
claudeadriangb
authored andcommitted
fix: apply struct field filters when the file schema needs adaptation
When the declared table schema differs from the physical file schema for a struct column, the expression adapter wraps the whole struct column in a cast, so `s['x']` becomes `get_field(cast(s AS Struct<..>), 'x')`. That hides the column from consumers that pattern match on `get_field(column, 'f')`. The Parquet scan is one such consumer: it decides at planning time (against the table schema) that a struct-field predicate can be evaluated as a row filter and reports it as fully handled, so `FilterExec` is removed from the plan. At runtime the row filter builder no longer recognizes the adapted expression, silently drops the predicate, and the query returns unfiltered rows. Narrow the cast to the field that is actually read: `get_field(cast(s AS Struct<..>), 'x')` becomes `cast(get_field(s, 'x') AS <type of x>)`. This keeps the column visible under the `get_field`, and also avoids materializing a whole cast struct just to read one field. Fields that are missing from the file collapse to a typed null literal, matching what the struct cast would have produced. `get_field` on a Map column is a runtime key lookup rather than a schema-level field access, so those keep the whole-column cast. Closes apache#24109. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01MebN5PsVnYvXUeVKju5K7P
1 parent e6b4221 commit de0e5d9

3 files changed

Lines changed: 437 additions & 0 deletions

File tree

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3775,4 +3775,109 @@ mod test {
37753775
assert_eq!(rows, 5);
37763776
}
37773777
}
3778+
3779+
/// Filters on struct fields (`s['x'] = 200`) must still be applied when the
3780+
/// table schema disagrees with the physical file schema, which forces the
3781+
/// expression adapter to insert a cast.
3782+
///
3783+
/// See <https://github.com/apache/datafusion/issues/24109>.
3784+
mod struct_field_pushdown {
3785+
use super::*;
3786+
use arrow::array::{Int32Array, StructArray};
3787+
use arrow::datatypes::Fields;
3788+
use datafusion_functions::core::get_field;
3789+
3790+
/// Writes `s: Struct<x: Int32>` with `x` values 100, 200, 300 and
3791+
/// returns the physical file schema plus the written file.
3792+
async fn write_struct_file(
3793+
store: &Arc<dyn ObjectStore>,
3794+
path: &str,
3795+
) -> (SchemaRef, PartitionedFile) {
3796+
let physical_struct_fields: Fields =
3797+
vec![Field::new("x", DataType::Int32, true)].into();
3798+
let struct_array = StructArray::new(
3799+
physical_struct_fields.clone(),
3800+
vec![Arc::new(Int32Array::from(vec![100, 200, 300])) as _],
3801+
None,
3802+
);
3803+
let schema = Arc::new(Schema::new(vec![Field::new(
3804+
"s",
3805+
DataType::Struct(physical_struct_fields),
3806+
true,
3807+
)]));
3808+
let batch = RecordBatch::try_new(
3809+
Arc::clone(&schema),
3810+
vec![Arc::new(struct_array) as _],
3811+
)
3812+
.unwrap();
3813+
let data_size = write_parquet(Arc::clone(store), path, batch).await;
3814+
let file =
3815+
PartitionedFile::new(path.to_string(), u64::try_from(data_size).unwrap());
3816+
(schema, file)
3817+
}
3818+
3819+
/// `s: Struct<x: Int64>` — the same column as on disk, but with a
3820+
/// wider leaf type so the adapter has to insert a cast.
3821+
fn logical_schema() -> SchemaRef {
3822+
Arc::new(Schema::new(vec![Field::new(
3823+
"s",
3824+
DataType::Struct(vec![Field::new("x", DataType::Int64, true)].into()),
3825+
true,
3826+
)]))
3827+
}
3828+
3829+
fn struct_field_eq(
3830+
schema: &SchemaRef,
3831+
value: ScalarValue,
3832+
) -> Arc<dyn PhysicalExpr> {
3833+
let expr = get_field().call(vec![col("s"), lit("x")]).eq(lit(value));
3834+
logical2physical(&expr, schema)
3835+
}
3836+
3837+
#[tokio::test]
3838+
async fn test_struct_field_filter_with_schema_adaptation() {
3839+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
3840+
let (_physical_schema, file) =
3841+
write_struct_file(&store, "struct_cast.parquet").await;
3842+
let logical_schema = logical_schema();
3843+
3844+
let opener = ParquetMorselizerBuilder::new()
3845+
.with_store(Arc::clone(&store))
3846+
.with_schema(Arc::clone(&logical_schema))
3847+
.with_projection_indices(&[0])
3848+
.with_predicate(struct_field_eq(
3849+
&logical_schema,
3850+
ScalarValue::Int64(Some(200)),
3851+
))
3852+
.with_pushdown_filters(true)
3853+
.build();
3854+
3855+
let stream = open_file(&opener, file).await.unwrap();
3856+
let (_batches, rows) = count_batches_and_rows(stream).await;
3857+
assert_eq!(rows, 1, "predicate on struct field must be applied");
3858+
}
3859+
3860+
/// Control: with matching schemas the filter has always worked.
3861+
#[tokio::test]
3862+
async fn test_struct_field_filter_without_schema_adaptation() {
3863+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
3864+
let (physical_schema, file) =
3865+
write_struct_file(&store, "struct_nocast.parquet").await;
3866+
3867+
let opener = ParquetMorselizerBuilder::new()
3868+
.with_store(Arc::clone(&store))
3869+
.with_schema(Arc::clone(&physical_schema))
3870+
.with_projection_indices(&[0])
3871+
.with_predicate(struct_field_eq(
3872+
&physical_schema,
3873+
ScalarValue::Int32(Some(200)),
3874+
))
3875+
.with_pushdown_filters(true)
3876+
.build();
3877+
3878+
let stream = open_file(&opener, file).await.unwrap();
3879+
let (_batches, rows) = count_batches_and_rows(stream).await;
3880+
assert_eq!(rows, 1);
3881+
}
3882+
}
37783883
}

0 commit comments

Comments
 (0)