Skip to content

Commit cb23c25

Browse files
committed
fix
1 parent 76227b6 commit cb23c25

3 files changed

Lines changed: 75 additions & 21 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7001,15 +7001,16 @@ mod tests {
70017001
(declared, acc.state().unwrap()[0].data_type())
70027002
}
70037003

7004-
/// The plan declares nullable leaves while the batch that actually arrives carries
7005-
/// non-nullable ones. `collect_list` / `collect_set` derive their state type from the
7006-
/// declared type but emit the runtime type, so without the coercion the drift reaches
7007-
/// `AggregateExec` and it fails validating its own output batch.
7004+
/// `collect_list` / `collect_set` derive their accumulator state type from the declared
7005+
/// argument type but can emit the runtime type, so a plan that declares nullable leaves while
7006+
/// the arriving batch carries non-nullable ones drifts, and the drift reaches `AggregateExec`
7007+
/// and fails its output-batch validation. `coerce_collect_child_nullability` normalizes the
7008+
/// argument to the declared type before it reaches the accumulator.
70087009
///
7009-
/// The first assertion deliberately pins the upstream behaviour this coercion defends
7010-
/// against. If `ScalarValue::new_list` is ever fixed to honour its `data_type` for non-empty
7011-
/// input, the drift disappears and that assertion starts failing — that is the signal to drop
7012-
/// `coerce_collect_child_nullability` entirely, not a regression.
7010+
/// DataFusion 55's `ScalarValue::new_list` fix removed the drift for `collect_set`, which
7011+
/// rebuilds its state list honouring `data_type`. `collect_list` still rebuilds from the
7012+
/// runtime array and drifts, so the coercion remains necessary. This test pins that
7013+
/// `collect_list` still drifts without the coercion, and that the coercion normalizes both.
70137014
#[test]
70147015
fn test_collect_agg_absorbs_nested_nullability_drift() {
70157016
let plan_schema = collect_agg_schema(collect_agg_struct_type(true));
@@ -7023,17 +7024,21 @@ mod tests {
70237024
AggregateUDF::new_from_impl(SparkCollectSet::new()),
70247025
AggregateUDF::new_from_impl(SparkCollectList::new()),
70257026
] {
7026-
// Without the coercion, declared and produced disagree on nested nullability.
7027+
// Without the coercion, collect_list still drifts (declared nullable leaves vs
7028+
// produced non-null ones). collect_set no longer drifts after DataFusion 55's new_list
7029+
// fix, so only assert the drift for the func that still exhibits it.
70277030
let (declared, produced) =
70287031
collect_agg_declared_vs_produced(&raw, &plan_schema, &batch_schema, func.clone());
7029-
assert!(
7030-
!declared.equals_datatype(&produced),
7031-
"expected drift for {}: declared {declared}, produced {produced}",
7032-
func.name()
7033-
);
7032+
if func.name() == "collect_list" {
7033+
assert!(
7034+
!declared.equals_datatype(&produced),
7035+
"expected drift for {}: declared {declared}, produced {produced}",
7036+
func.name()
7037+
);
7038+
}
70347039

7035-
// With it, the argument is normalized to the declared type before it reaches
7036-
// the accumulator.
7040+
// With the coercion, the argument is normalized to the declared type before it
7041+
// reaches the accumulator, so both funcs emit a state matching the declared type.
70377042
let (declared, produced) =
70387043
collect_agg_declared_vs_produced(&coerced, &plan_schema, &batch_schema, func);
70397044
assert!(
@@ -7157,11 +7162,15 @@ mod tests {
71577162
..Default::default()
71587163
};
71597164

7165+
// No partition_spec_idx/partition_data_idx: this test exercises unified-type merging,
7166+
// which is driven by the type/spec pools rather than the task's own spec. `FileScanTask`
7167+
// validation rejects a partitioned spec without matching partition data, and the JVM
7168+
// serializer always sends spec and data together, so a spec-without-data task never
7169+
// occurs in practice.
71607170
let proto_task = spark_operator::IcebergFileScanTask {
71617171
data_file_path: "file:///tmp/data.parquet".to_string(),
71627172
file_size_in_bytes: 100,
71637173
schema_idx: 0,
7164-
partition_spec_idx: Some(0),
71657174
project_field_ids_idx: 0,
71667175
..Default::default()
71677176
};

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -630,16 +630,16 @@ object IcebergReflection extends Logging {
630630
baseSchema
631631
} else {
632632
logDebug(
633-
s"Iceberg equality delete references field id(s) ${missingIds.mkString(",")} absent from " +
634-
"the task schema; resolving from table schema history to build the native scan schema")
633+
s"Native Iceberg scan schema is missing field id(s) ${missingIds.mkString(",")}; " +
634+
"resolving them from table schema history")
635635
val history = getAllSchemas(table)
636636
val resolvedFields = missingIds.map { id =>
637637
history.iterator
638638
.flatMap(s => findFieldObject(s, id))
639639
.toSeq
640640
.headOption
641641
.getOrElse(throw new IllegalStateException(
642-
s"Cannot resolve equality-delete field id $id in table schema history"))
642+
s"Cannot resolve field id $id in table schema history"))
643643
}
644644
val existing =
645645
getMethod(baseSchema.getClass, "columns")
@@ -654,6 +654,38 @@ object IcebergReflection extends Logging {
654654
}
655655
}
656656

657+
/**
658+
* The source column field ids referenced by a task's partition spec.
659+
*
660+
* iceberg-rust validates a `FileScanTask` by resolving its partition spec against the task
661+
* schema (`partition_type(schema)`), so a task carrying a partition spec needs those source
662+
* columns present in the schema even when the query projects them out (for example selecting
663+
* only `_spec_id` or `_partition`). These ids are unioned into the native task schema via
664+
* [[schemaWithRequiredFields]]. project_field_ids still drives the read, so the columns are not
665+
* materialized into the scan output. All ids resolve from the table's schema history, including
666+
* a source column later dropped by partition evolution.
667+
*
668+
* Returns an empty sequence when the task has no partition spec.
669+
*/
670+
def partitionSourceFieldIds(task: Any, fileScanTaskClass: Class[_]): Seq[Int] = {
671+
import scala.jdk.CollectionConverters._
672+
673+
val spec =
674+
try {
675+
getMethod(fileScanTaskClass, "spec").invoke(task)
676+
} catch {
677+
case _: Exception => null
678+
}
679+
if (spec == null) {
680+
Seq.empty
681+
} else {
682+
val fields =
683+
getMethod(spec.getClass, "fields").invoke(spec).asInstanceOf[java.util.List[_]]
684+
val sourceIdMethod = getMethod(loadClass(ClassNames.PARTITION_FIELD), "sourceId")
685+
fields.asScala.map(field => sourceIdMethod.invoke(field).asInstanceOf[Int]).toSeq
686+
}
687+
}
688+
657689
/**
658690
* Gets the partition spec from an Iceberg table.
659691
*/

‎spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1055,7 +1055,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
10551055
IcebergReflection.getDeleteFilesFromTask(task, fileScanTaskClass)
10561056
val hasDeletes = !deletes.isEmpty
10571057

1058-
val schema: AnyRef =
1058+
val baseSchema: AnyRef =
10591059
if (hasDeletes) {
10601060
// An equality delete may be keyed on a column dropped from the current schema
10611061
// (schema evolution). iceberg-rust must read that column to apply the delete,
@@ -1083,6 +1083,19 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
10831083
}
10841084
}
10851085

1086+
// iceberg-rust validates a FileScanTask by resolving its partition spec against
1087+
// the task schema, so a task carrying a partition spec needs that spec's source
1088+
// columns present even when the query projects them out (e.g. selecting only
1089+
// _spec_id / _partition). Union them in. project_field_ids still drives the read,
1090+
// so these columns are not materialized into the output.
1091+
val schema: AnyRef =
1092+
IcebergReflection
1093+
.schemaWithRequiredFields(
1094+
baseSchema,
1095+
metadata.table,
1096+
IcebergReflection.partitionSourceFieldIds(task, fileScanTaskClass))
1097+
.asInstanceOf[AnyRef]
1098+
10861099
val schemaIdx = schemaToPoolIndex.getOrElseUpdate(
10871100
schema, {
10881101
val idx = schemaToPoolIndex.size

0 commit comments

Comments
 (0)