Skip to content

Commit a4f0025

Browse files
committed
test: filter on the evolved column in the schema evolution fuzz test
Adds deterministic queries that read and filter on `b` (Int64 in the table, Int32 in one file), check the filter is pushed into the Parquet scan, and compare both pushdown settings against an in memory oracle. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01J7cZ4YPYj4nMWbNAd8ovtT
1 parent cec21a4 commit a4f0025

1 file changed

Lines changed: 48 additions & 0 deletions

File tree

‎datafusion/core/tests/fuzz_cases/leaf_pushdown_fuzz.rs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -821,6 +821,7 @@ async fn leaf_pushdown_parquet_schema_evolution() {
821821
register_parquet(&ctx, &dir, NARROW, &[narrow_batch()], None).await;
822822
contexts.push(ctx);
823823
}
824+
check_evolved_filters(&contexts[0], &contexts[1], &wide).await;
824825
run(
825826
&contexts[0],
826827
&contexts[1],
@@ -831,3 +832,50 @@ async fn leaf_pushdown_parquet_schema_evolution() {
831832
)
832833
.await;
833834
}
835+
836+
/// Filters on the evolved column `b` of `t1`. The fuzz cases mostly carry `b`
837+
/// through projections, where pruning can remove it. These read `b` and filter
838+
/// on it, so the pushed filter must be adapted from Int64 to the Int32 file.
839+
/// The oracle is an in memory `t1` that holds the same rows, all as Int64.
840+
async fn check_evolved_filters(on: &SessionContext, off: &SessionContext, wide: &Schema) {
841+
let narrow = table_batch(0);
842+
let mut columns = narrow.columns().to_vec();
843+
columns[1] = arrow::compute::cast(&columns[1], &DataType::Int64).unwrap();
844+
let widened = RecordBatch::try_new(Arc::new(wide.clone()), columns).unwrap();
845+
let oracle = SessionContext::new_with_config(config(true, false));
846+
oracle
847+
.register_batch(
848+
"t1",
849+
arrow::compute::concat_batches(&widened.schema(), [&widened, &widened])
850+
.unwrap(),
851+
)
852+
.unwrap();
853+
for sql in [
854+
"SELECT b, s['a'] FROM t1 WHERE b > 0",
855+
"SELECT s['a'] FROM t1 WHERE b > 0",
856+
"SELECT a, b FROM t1 WHERE b = 1 OR b IS NULL",
857+
"SELECT b, s['b'] FROM t1 WHERE b < 0 AND s['a'] IS NOT NULL",
858+
"SELECT b FROM t1 WHERE b + 1 > 1",
859+
] {
860+
let expected = collect(&oracle, sql).await.unwrap();
861+
let rows: usize = expected.iter().map(RecordBatch::num_rows).sum();
862+
assert!(
863+
rows > 0 && rows < 2 * ROWS as usize,
864+
"{sql}: the filter must keep some rows and drop some, keeps {rows}"
865+
);
866+
let plan = collect(on, &format!("EXPLAIN {sql}")).await.unwrap();
867+
let plan = pretty_format_batches(&plan).unwrap().to_string();
868+
assert!(
869+
plan.contains("predicate=") && !plan.contains("FilterExec"),
870+
"{sql}: the filter is not pushed into the Parquet scan:\n{plan}"
871+
);
872+
let expected = normalize(&expected);
873+
for (name, ctx) in [("parquet pushdown on", on), ("parquet pushdown off", off)] {
874+
let actual = normalize(&collect(ctx, sql).await.unwrap());
875+
assert_eq!(
876+
actual, expected,
877+
"{sql}: {name} differs from the in memory oracle"
878+
);
879+
}
880+
}
881+
}

0 commit comments

Comments
 (0)