@@ -6934,15 +6934,16 @@ mod tests {
69346934 ( declared, acc. state ( ) . unwrap ( ) [ 0 ] . data_type ( ) )
69356935 }
69366936
6937- /// The plan declares nullable leaves while the batch that actually arrives carries
6938- /// non-nullable ones. `collect_list` / `collect_set` derive their state type from the
6939- /// declared type but emit the runtime type, so without the coercion the drift reaches
6940- /// `AggregateExec` and it fails validating its own output batch.
6937+ /// `collect_list` / `collect_set` derive their accumulator state type from the declared
6938+ /// argument type but can emit the runtime type, so a plan that declares nullable leaves while
6939+ /// the arriving batch carries non-nullable ones drifts, and the drift reaches `AggregateExec`
6940+ /// and fails its output-batch validation. `coerce_collect_child_nullability` normalizes the
6941+ /// argument to the declared type before it reaches the accumulator.
69416942 ///
6942- /// The first assertion deliberately pins the upstream behaviour this coercion defends
6943- /// against. If `ScalarValue::new_list` is ever fixed to honour its `data_type` for non-empty
6944- /// input, the drift disappears and that assertion starts failing — that is the signal to drop
6945- /// `coerce_collect_child_nullability` entirely, not a regression .
6943+ /// DataFusion 55's `ScalarValue::new_list` fix removed the drift for `collect_set`, which
6944+ /// rebuilds its state list honouring `data_type`. `collect_list` still rebuilds from the
6945+ /// runtime array and drifts, so the coercion remains necessary. This test pins that
6946+ /// `collect_list` still drifts without the coercion, and that the coercion normalizes both .
69466947 #[ test]
69476948 fn test_collect_agg_absorbs_nested_nullability_drift ( ) {
69486949 let plan_schema = collect_agg_schema ( collect_agg_struct_type ( true ) ) ;
@@ -6956,17 +6957,21 @@ mod tests {
69566957 AggregateUDF :: new_from_impl ( SparkCollectSet :: new ( ) ) ,
69576958 AggregateUDF :: new_from_impl ( SparkCollectList :: new ( ) ) ,
69586959 ] {
6959- // Without the coercion, declared and produced disagree on nested nullability.
6960+ // Without the coercion, collect_list still drifts (declared nullable leaves vs
6961+ // produced non-null ones). collect_set no longer drifts after DataFusion 55's new_list
6962+ // fix, so only assert the drift for the func that still exhibits it.
69606963 let ( declared, produced) =
69616964 collect_agg_declared_vs_produced ( & raw, & plan_schema, & batch_schema, func. clone ( ) ) ;
6962- assert ! (
6963- !declared. equals_datatype( & produced) ,
6964- "expected drift for {}: declared {declared}, produced {produced}" ,
6965- func. name( )
6966- ) ;
6965+ if func. name ( ) == "collect_list" {
6966+ assert ! (
6967+ !declared. equals_datatype( & produced) ,
6968+ "expected drift for {}: declared {declared}, produced {produced}" ,
6969+ func. name( )
6970+ ) ;
6971+ }
69676972
6968- // With it , the argument is normalized to the declared type before it reaches
6969- // the accumulator.
6973+ // With the coercion , the argument is normalized to the declared type before it
6974+ // reaches the accumulator, so both funcs emit a state matching the declared type .
69706975 let ( declared, produced) =
69716976 collect_agg_declared_vs_produced ( & coerced, & plan_schema, & batch_schema, func) ;
69726977 assert ! (
@@ -7090,11 +7095,15 @@ mod tests {
70907095 ..Default :: default ( )
70917096 } ;
70927097
7098+ // No partition_spec_idx/partition_data_idx: this test exercises unified-type merging,
7099+ // which is driven by the type/spec pools rather than the task's own spec. `FileScanTask`
7100+ // validation rejects a partitioned spec without matching partition data, and the JVM
7101+ // serializer always sends spec and data together, so a spec-without-data task never
7102+ // occurs in practice.
70937103 let proto_task = spark_operator:: IcebergFileScanTask {
70947104 data_file_path : "file:///tmp/data.parquet" . to_string ( ) ,
70957105 file_size_in_bytes : 100 ,
70967106 schema_idx : 0 ,
7097- partition_spec_idx : Some ( 0 ) ,
70987107 project_field_ids_idx : 0 ,
70997108 ..Default :: default ( )
71007109 } ;
0 commit comments