@@ -7001,15 +7001,16 @@ mod tests {
70017001 ( declared, acc. state ( ) . unwrap ( ) [ 0 ] . data_type ( ) )
70027002 }
70037003
7004- /// The plan declares nullable leaves while the batch that actually arrives carries
7005- /// non-nullable ones. `collect_list` / `collect_set` derive their state type from the
7006- /// declared type but emit the runtime type, so without the coercion the drift reaches
7007- /// `AggregateExec` and it fails validating its own output batch.
7004+ /// `collect_list` / `collect_set` derive their accumulator state type from the declared
7005+ /// argument type but can emit the runtime type, so a plan that declares nullable leaves while
7006+ /// the arriving batch carries non-nullable ones drifts, and the drift reaches `AggregateExec`
7007+ /// and fails its output-batch validation. `coerce_collect_child_nullability` normalizes the
7008+ /// argument to the declared type before it reaches the accumulator.
70087009 ///
7009- /// The first assertion deliberately pins the upstream behaviour this coercion defends
7010- /// against. If `ScalarValue::new_list` is ever fixed to honour its `data_type` for non-empty
7011- /// input, the drift disappears and that assertion starts failing — that is the signal to drop
7012- /// `coerce_collect_child_nullability` entirely, not a regression .
7010+ /// DataFusion 55's `ScalarValue::new_list` fix removed the drift for `collect_set`, which
7011+ /// rebuilds its state list honouring `data_type`. `collect_list` still rebuilds from the
7012+ /// runtime array and drifts, so the coercion remains necessary. This test pins that
7013+ /// `collect_list` still drifts without the coercion, and that the coercion normalizes both .
70137014 #[ test]
70147015 fn test_collect_agg_absorbs_nested_nullability_drift ( ) {
70157016 let plan_schema = collect_agg_schema ( collect_agg_struct_type ( true ) ) ;
@@ -7023,17 +7024,21 @@ mod tests {
70237024 AggregateUDF :: new_from_impl ( SparkCollectSet :: new ( ) ) ,
70247025 AggregateUDF :: new_from_impl ( SparkCollectList :: new ( ) ) ,
70257026 ] {
7026- // Without the coercion, declared and produced disagree on nested nullability.
7027+ // Without the coercion, collect_list still drifts (declared nullable leaves vs
7028+ // produced non-null ones). collect_set no longer drifts after DataFusion 55's new_list
7029+ // fix, so only assert the drift for the func that still exhibits it.
70277030 let ( declared, produced) =
70287031 collect_agg_declared_vs_produced ( & raw, & plan_schema, & batch_schema, func. clone ( ) ) ;
7029- assert ! (
7030- !declared. equals_datatype( & produced) ,
7031- "expected drift for {}: declared {declared}, produced {produced}" ,
7032- func. name( )
7033- ) ;
7032+ if func. name ( ) == "collect_list" {
7033+ assert ! (
7034+ !declared. equals_datatype( & produced) ,
7035+ "expected drift for {}: declared {declared}, produced {produced}" ,
7036+ func. name( )
7037+ ) ;
7038+ }
70347039
7035- // With it , the argument is normalized to the declared type before it reaches
7036- // the accumulator.
7040+ // With the coercion , the argument is normalized to the declared type before it
7041+ // reaches the accumulator, so both funcs emit a state matching the declared type .
70377042 let ( declared, produced) =
70387043 collect_agg_declared_vs_produced ( & coerced, & plan_schema, & batch_schema, func) ;
70397044 assert ! (
@@ -7157,11 +7162,15 @@ mod tests {
71577162 ..Default :: default ( )
71587163 } ;
71597164
7165+ // No partition_spec_idx/partition_data_idx: this test exercises unified-type merging,
7166+ // which is driven by the type/spec pools rather than the task's own spec. `FileScanTask`
7167+ // validation rejects a partitioned spec without matching partition data, and the JVM
7168+ // serializer always sends spec and data together, so a spec-without-data task never
7169+ // occurs in practice.
71607170 let proto_task = spark_operator:: IcebergFileScanTask {
71617171 data_file_path : "file:///tmp/data.parquet" . to_string ( ) ,
71627172 file_size_in_bytes : 100 ,
71637173 schema_idx : 0 ,
7164- partition_spec_idx : Some ( 0 ) ,
71657174 project_field_ids_idx : 0 ,
71667175 ..Default :: default ( )
71677176 } ;
0 commit comments