@@ -821,6 +821,7 @@ async fn leaf_pushdown_parquet_schema_evolution() {
821821 register_parquet ( & ctx, & dir, NARROW , & [ narrow_batch ( ) ] , None ) . await ;
822822 contexts. push ( ctx) ;
823823 }
824+ check_evolved_filters ( & contexts[ 0 ] , & contexts[ 1 ] , & wide) . await ;
824825 run (
825826 & contexts[ 0 ] ,
826827 & contexts[ 1 ] ,
@@ -831,3 +832,50 @@ async fn leaf_pushdown_parquet_schema_evolution() {
831832 )
832833 . await ;
833834}
835+
836+ /// Filters on the evolved column `b` of `t1`. The fuzz cases mostly carry `b`
837+ /// through projections, where pruning can remove it. These read `b` and filter
838+ /// on it, so the pushed filter must be adapted from Int64 to the Int32 file.
839+ /// The oracle is an in memory `t1` that holds the same rows, all as Int64.
840+ async fn check_evolved_filters ( on : & SessionContext , off : & SessionContext , wide : & Schema ) {
841+ let narrow = table_batch ( 0 ) ;
842+ let mut columns = narrow. columns ( ) . to_vec ( ) ;
843+ columns[ 1 ] = arrow:: compute:: cast ( & columns[ 1 ] , & DataType :: Int64 ) . unwrap ( ) ;
844+ let widened = RecordBatch :: try_new ( Arc :: new ( wide. clone ( ) ) , columns) . unwrap ( ) ;
845+ let oracle = SessionContext :: new_with_config ( config ( true , false ) ) ;
846+ oracle
847+ . register_batch (
848+ "t1" ,
849+ arrow:: compute:: concat_batches ( & widened. schema ( ) , [ & widened, & widened] )
850+ . unwrap ( ) ,
851+ )
852+ . unwrap ( ) ;
853+ for sql in [
854+ "SELECT b, s['a'] FROM t1 WHERE b > 0" ,
855+ "SELECT s['a'] FROM t1 WHERE b > 0" ,
856+ "SELECT a, b FROM t1 WHERE b = 1 OR b IS NULL" ,
857+ "SELECT b, s['b'] FROM t1 WHERE b < 0 AND s['a'] IS NOT NULL" ,
858+ "SELECT b FROM t1 WHERE b + 1 > 1" ,
859+ ] {
860+ let expected = collect ( & oracle, sql) . await . unwrap ( ) ;
861+ let rows: usize = expected. iter ( ) . map ( RecordBatch :: num_rows) . sum ( ) ;
862+ assert ! (
863+ rows > 0 && rows < 2 * ROWS as usize ,
864+ "{sql}: the filter must keep some rows and drop some, keeps {rows}"
865+ ) ;
866+ let plan = collect ( on, & format ! ( "EXPLAIN {sql}" ) ) . await . unwrap ( ) ;
867+ let plan = pretty_format_batches ( & plan) . unwrap ( ) . to_string ( ) ;
868+ assert ! (
869+ plan. contains( "predicate=" ) && !plan. contains( "FilterExec" ) ,
870+ "{sql}: the filter is not pushed into the Parquet scan:\n {plan}"
871+ ) ;
872+ let expected = normalize ( & expected) ;
873+ for ( name, ctx) in [ ( "parquet pushdown on" , on) , ( "parquet pushdown off" , off) ] {
874+ let actual = normalize ( & collect ( ctx, sql) . await . unwrap ( ) ) ;
875+ assert_eq ! (
876+ actual, expected,
877+ "{sql}: {name} differs from the in memory oracle"
878+ ) ;
879+ }
880+ }
881+ }
0 commit comments