Skip to content

Commit 3ee7fa6

Browse files
committed
fix
1 parent 598f5e9 commit 3ee7fa6

3 files changed

Lines changed: 75 additions & 21 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6934,15 +6934,16 @@ mod tests {
69346934
(declared, acc.state().unwrap()[0].data_type())
69356935
}
69366936

6937-
/// The plan declares nullable leaves while the batch that actually arrives carries
6938-
/// non-nullable ones. `collect_list` / `collect_set` derive their state type from the
6939-
/// declared type but emit the runtime type, so without the coercion the drift reaches
6940-
/// `AggregateExec` and it fails validating its own output batch.
6937+
/// `collect_list` / `collect_set` derive their accumulator state type from the declared
6938+
/// argument type but can emit the runtime type, so a plan that declares nullable leaves while
6939+
/// the arriving batch carries non-nullable ones drifts, and the drift reaches `AggregateExec`
6940+
/// and fails its output-batch validation. `coerce_collect_child_nullability` normalizes the
6941+
/// argument to the declared type before it reaches the accumulator.
69416942
///
6942-
/// The first assertion deliberately pins the upstream behaviour this coercion defends
6943-
/// against. If `ScalarValue::new_list` is ever fixed to honour its `data_type` for non-empty
6944-
/// input, the drift disappears and that assertion starts failing — that is the signal to drop
6945-
/// `coerce_collect_child_nullability` entirely, not a regression.
6943+
/// DataFusion 55's `ScalarValue::new_list` fix removed the drift for `collect_set`, which
6944+
/// rebuilds its state list honouring `data_type`. `collect_list` still rebuilds from the
6945+
/// runtime array and drifts, so the coercion remains necessary. This test pins that
6946+
/// `collect_list` still drifts without the coercion, and that the coercion normalizes both.
69466947
#[test]
69476948
fn test_collect_agg_absorbs_nested_nullability_drift() {
69486949
let plan_schema = collect_agg_schema(collect_agg_struct_type(true));
@@ -6956,17 +6957,21 @@ mod tests {
69566957
AggregateUDF::new_from_impl(SparkCollectSet::new()),
69576958
AggregateUDF::new_from_impl(SparkCollectList::new()),
69586959
] {
6959-
// Without the coercion, declared and produced disagree on nested nullability.
6960+
// Without the coercion, collect_list still drifts (declared nullable leaves vs
6961+
// produced non-null ones). collect_set no longer drifts after DataFusion 55's new_list
6962+
// fix, so only assert the drift for the func that still exhibits it.
69606963
let (declared, produced) =
69616964
collect_agg_declared_vs_produced(&raw, &plan_schema, &batch_schema, func.clone());
6962-
assert!(
6963-
!declared.equals_datatype(&produced),
6964-
"expected drift for {}: declared {declared}, produced {produced}",
6965-
func.name()
6966-
);
6965+
if func.name() == "collect_list" {
6966+
assert!(
6967+
!declared.equals_datatype(&produced),
6968+
"expected drift for {}: declared {declared}, produced {produced}",
6969+
func.name()
6970+
);
6971+
}
69676972

6968-
// With it, the argument is normalized to the declared type before it reaches
6969-
// the accumulator.
6973+
// With the coercion, the argument is normalized to the declared type before it
6974+
// reaches the accumulator, so both funcs emit a state matching the declared type.
69706975
let (declared, produced) =
69716976
collect_agg_declared_vs_produced(&coerced, &plan_schema, &batch_schema, func);
69726977
assert!(
@@ -7090,11 +7095,15 @@ mod tests {
70907095
..Default::default()
70917096
};
70927097

7098+
// No partition_spec_idx/partition_data_idx: this test exercises unified-type merging,
7099+
// which is driven by the type/spec pools rather than the task's own spec. `FileScanTask`
7100+
// validation rejects a partitioned spec without matching partition data, and the JVM
7101+
// serializer always sends spec and data together, so a spec-without-data task never
7102+
// occurs in practice.
70937103
let proto_task = spark_operator::IcebergFileScanTask {
70947104
data_file_path: "file:///tmp/data.parquet".to_string(),
70957105
file_size_in_bytes: 100,
70967106
schema_idx: 0,
7097-
partition_spec_idx: Some(0),
70987107
project_field_ids_idx: 0,
70997108
..Default::default()
71007109
};

‎spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -630,16 +630,16 @@ object IcebergReflection extends Logging {
630630
baseSchema
631631
} else {
632632
logDebug(
633-
s"Iceberg equality delete references field id(s) ${missingIds.mkString(",")} absent from " +
634-
"the task schema; resolving from table schema history to build the native scan schema")
633+
s"Native Iceberg scan schema is missing field id(s) ${missingIds.mkString(",")}; " +
634+
"resolving them from table schema history")
635635
val history = getAllSchemas(table)
636636
val resolvedFields = missingIds.map { id =>
637637
history.iterator
638638
.flatMap(s => findFieldObject(s, id))
639639
.toSeq
640640
.headOption
641641
.getOrElse(throw new IllegalStateException(
642-
s"Cannot resolve equality-delete field id $id in table schema history"))
642+
s"Cannot resolve field id $id in table schema history"))
643643
}
644644
val existing =
645645
getMethod(baseSchema.getClass, "columns")
@@ -654,6 +654,38 @@ object IcebergReflection extends Logging {
654654
}
655655
}
656656

657+
/**
658+
* The source column field ids referenced by a task's partition spec.
659+
*
660+
* iceberg-rust validates a `FileScanTask` by resolving its partition spec against the task
661+
* schema (`partition_type(schema)`), so a task carrying a partition spec needs those source
662+
* columns present in the schema even when the query projects them out (for example selecting
663+
* only `_spec_id` or `_partition`). These ids are unioned into the native task schema via
664+
* [[schemaWithRequiredFields]]. project_field_ids still drives the read, so the columns are not
665+
* materialized into the scan output. All ids resolve from the table's schema history, including
666+
* a source column later dropped by partition evolution.
667+
*
668+
* Returns an empty sequence when the task has no partition spec.
669+
*/
670+
def partitionSourceFieldIds(task: Any, fileScanTaskClass: Class[_]): Seq[Int] = {
671+
import scala.jdk.CollectionConverters._
672+
673+
val spec =
674+
try {
675+
getMethod(fileScanTaskClass, "spec").invoke(task)
676+
} catch {
677+
case _: Exception => null
678+
}
679+
if (spec == null) {
680+
Seq.empty
681+
} else {
682+
val fields =
683+
getMethod(spec.getClass, "fields").invoke(spec).asInstanceOf[java.util.List[_]]
684+
val sourceIdMethod = getMethod(loadClass(ClassNames.PARTITION_FIELD), "sourceId")
685+
fields.asScala.map(field => sourceIdMethod.invoke(field).asInstanceOf[Int]).toSeq
686+
}
687+
}
688+
657689
/**
658690
* Gets the partition spec from an Iceberg table.
659691
*/

‎spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1054,7 +1054,7 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
10541054
IcebergReflection.getDeleteFilesFromTask(task, fileScanTaskClass)
10551055
val hasDeletes = !deletes.isEmpty
10561056

1057-
val schema: AnyRef =
1057+
val baseSchema: AnyRef =
10581058
if (hasDeletes) {
10591059
// An equality delete may be keyed on a column dropped from the current schema
10601060
// (schema evolution). iceberg-rust must read that column to apply the delete,
@@ -1082,6 +1082,19 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
10821082
}
10831083
}
10841084

1085+
// iceberg-rust validates a FileScanTask by resolving its partition spec against
1086+
// the task schema, so a task carrying a partition spec needs that spec's source
1087+
// columns present even when the query projects them out (e.g. selecting only
1088+
// _spec_id / _partition). Union them in. project_field_ids still drives the read,
1089+
// so these columns are not materialized into the output.
1090+
val schema: AnyRef =
1091+
IcebergReflection
1092+
.schemaWithRequiredFields(
1093+
baseSchema,
1094+
metadata.table,
1095+
IcebergReflection.partitionSourceFieldIds(task, fileScanTaskClass))
1096+
.asInstanceOf[AnyRef]
1097+
10851098
val schemaIdx = schemaToPoolIndex.getOrElseUpdate(
10861099
schema, {
10871100
val idx = schemaToPoolIndex.size

0 commit comments

Comments
 (0)