Skip to content

Commit 5550f42

Browse files
committed
fix: preserve array field dispatch and reuse metadata
Keep field lookup within list dispatch so invalid children retain the intended error, and reuse field metadata unless nullability must widen. Simplify the complex-null guard and document why post-scan filters stay. Move the required-field regression into the native Iceberg suite, cover null lists and SQL null elements, and retain explicit expected-row checks.
1 parent 016c0f6 commit 5550f42

7 files changed

Lines changed: 112 additions & 73 deletions

File tree

‎docs/source/user-guide/latest/iceberg.md‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,6 @@ The native Iceberg reader supports the following features:
9898
- `IN` and `NOT IN` list operations
9999
- `BETWEEN` operations
100100

101-
Native scanning does not imply that every predicate is evaluated inside iceberg-rust.
102-
List, map, and struct NULL checks can use native scans. Their residuals are omitted
103-
before native planning; the retained post-scan filter enforces them. Empty collections
104-
and collections containing null elements are non-null, matching Spark. A conjunction
105-
containing one of these residuals currently loses native row-group pruning for primitive conjuncts;
106-
safe partial pruning is tracked in [#5883](https://github.com/apache/datafusion-comet/issues/5883).
107-
108101
**Partitioning:**
109102

110103
- Standard partitioning with partition pruning
@@ -119,6 +112,15 @@ safe partial pruning is tracked in [#5883](https://github.com/apache/datafusion-
119112
- Hadoop Distributed File System (HDFS)
120113
- S3-compatible storage (AWS S3, MinIO)
121114

115+
### Predicate pushdown vs native scanning
116+
117+
Native scanning does not imply that every predicate is evaluated inside iceberg-rust.
118+
List, map, and struct NULL checks can use native scans. Their residuals are omitted
119+
at serialization time; the retained post-scan filter enforces them. Empty collections
120+
and collections containing null elements are non-null, matching Spark. A conjunction
121+
containing one of these residuals currently loses native row-group pruning for primitive conjuncts;
122+
safe partial pruning is tracked in [#5883](https://github.com/apache/datafusion-comet/issues/5883).
123+
122124
### REST Catalog
123125

124126
Comet's native Iceberg reader also supports REST catalogs. The following example shows how to

‎native/spark-expr/src/array_funcs/get_array_struct_fields.rs‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,11 @@ impl GetArrayStructFields {
6969
DataType::Struct(fields) => {
7070
let field = &fields[self.ordinal];
7171
// A null struct element yields null even when the field itself is required.
72-
Ok(Arc::new(field.as_ref().clone().with_nullable(
73-
list_field.is_nullable() || field.is_nullable(),
74-
)))
72+
if list_field.is_nullable() && !field.is_nullable() {
73+
Ok(Arc::new(field.as_ref().clone().with_nullable(true)))
74+
} else {
75+
Ok(Arc::clone(field))
76+
}
7577
}
7678
data_type => Err(DataFusionError::Internal(format!(
7779
"Unexpected data type in GetArrayStructFields: {data_type:?}"
@@ -98,15 +100,16 @@ impl PhysicalExpr for GetArrayStructFields {
98100

99101
fn evaluate(&self, batch: &RecordBatch) -> DataFusionResult<ColumnarValue> {
100102
let child_value = self.child.evaluate(batch)?.into_array(batch.num_rows())?;
101-
let field = self.child_field(batch.schema().as_ref())?;
102103

103104
match child_value.data_type() {
104105
DataType::List(_) => {
106+
let field = self.child_field(batch.schema().as_ref())?;
105107
let list_array = as_list_array(&child_value)?;
106108

107109
get_array_struct_fields(list_array, self.ordinal, field)
108110
}
109111
DataType::LargeList(_) => {
112+
let field = self.child_field(batch.schema().as_ref())?;
110113
let list_array = as_large_list_array(&child_value)?;
111114

112115
get_array_struct_fields(list_array, self.ordinal, field)
@@ -247,6 +250,21 @@ mod tests {
247250
}
248251
}
249252

253+
#[test]
254+
fn non_list_child_returns_dispatch_error() {
255+
let batch = RecordBatch::try_from_iter(vec![(
256+
"value",
257+
Arc::new(Int32Array::from(vec![1])) as arrow::array::ArrayRef,
258+
)])
259+
.unwrap();
260+
let expr = GetArrayStructFields::new(Arc::new(Column::new("value", 0)), 0);
261+
assert!(expr
262+
.evaluate(&batch)
263+
.unwrap_err()
264+
.to_string()
265+
.contains("Unexpected child type for ListExtract: Int32"));
266+
}
267+
250268
#[test]
251269
fn mismatched_output_field_returns_error() {
252270
let values = Arc::new(StructArray::new(

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -969,8 +969,7 @@ case class CometScanRule(session: SparkSession)
969969
defaultValuesSupported && schemaTypesSupported && encryptionKeyLengthSupported &&
970970
taskValidation.allParquet && allSupportedFilesystems && allLocationsOpenable &&
971971
metadataSchemeSupported && partitionTypesSupported && unifiedPartitionTypeSupported &&
972-
transformFunctionsSupported &&
973-
deleteFileTypesSupported && dppSubqueriesSupported) {
972+
transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported) {
974973
CometBatchScanExec(
975974
scanExec.clone().asInstanceOf[BatchScanExec],
976975
runtimeFilters = scanExec.runtimeFilters,

‎spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import org.apache.spark.sql.types._
4040
import com.google.protobuf.ByteString
4141

4242
import org.apache.comet.{CometConf, ConfigEntry}
43+
import org.apache.comet.DataTypeSupport.isComplexType
4344
import org.apache.comet.iceberg.{CometIcebergNativeScanMetadata, IcebergReflection}
4445
import org.apache.comet.objectstore.NativeConfig
4546
import org.apache.comet.serde.{CometOperatorSerde, OperatorOuterClass}
@@ -630,10 +631,9 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit
630631
operation match {
631632
// iceberg-rust has accessors only for primitive fields, not containers. Keep
632633
// the post-scan filter without sending residuals that would warn on every task.
633-
case IS_NULL | IS_NOT_NULL | NOT_NULL
634-
if attribute.dataType.isInstanceOf[ArrayType] ||
635-
attribute.dataType.isInstanceOf[MapType] ||
636-
attribute.dataType.isInstanceOf[StructType] =>
634+
// Containers cannot be partition columns, so Iceberg cannot remove their null
635+
// checks from the post-scan filter through exact partition selection.
636+
case IS_NULL | IS_NOT_NULL | NOT_NULL if isComplexType(attribute.dataType) =>
637637
None
638638
case IS_NULL => Some(unaryPredicate(columnName, IcebergPredicateOperator.IsNull))
639639
case IS_NOT_NULL | NOT_NULL =>

‎spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,8 @@ CREATE TABLE test_arr_struct(arr array<struct<name: string, value: int>>) USING
2121
statement
2222
INSERT INTO test_arr_struct VALUES (array(named_struct('name', 'a', 'value', 1), named_struct('name', 'b', 'value', 2))), (array(named_struct('name', 'x', 'value', 10))), (NULL)
2323

24+
statement
25+
INSERT INTO test_arr_struct VALUES (array(NULL, named_struct('name', 'b', 'value', 2)))
26+
2427
query spark_answer_only
2528
SELECT arr.name, arr.value FROM test_arr_struct

‎spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala‎

Lines changed: 2 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ package org.apache.comet
2222
import scala.util.Random
2323

2424
import org.apache.spark.sql.{Column, Row}
25-
import org.apache.spark.sql.comet.CometProjectExec
2625
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
2726
import org.apache.spark.sql.functions.{col, lit}
2827
import org.apache.spark.sql.types._
@@ -287,8 +286,8 @@ class CometFuzzIcebergSuite extends CometFuzzIcebergBase {
287286
}
288287

289288
// Spark infers IS NOT NULL below ordinary generators, but not outer generators.
290-
// Compare complete rows to preserve null elements and distinguish null struct parents
291-
// from non-null structs with null fields. Native scanning does not imply residual pushdown.
289+
// Compare generated rows, including null elements, with Spark for both generator forms.
290+
// Native scanning does not imply residual pushdown.
292291
for (column <- Seq("l", "m"); generator <- Seq("explode", "explode_outer")) {
293292
val query = s"SELECT id, $generator($column) FROM $tableName"
294293
withClue(query) {
@@ -300,51 +299,6 @@ class CometFuzzIcebergSuite extends CometFuzzIcebergBase {
300299
spark.sql(s"DROP TABLE IF EXISTS $tableName")
301300
}
302301
}
303-
test("filter pushdown - required field projection preserves null array elements") {
304-
import org.apache.iceberg.Schema
305-
import org.apache.iceberg.catalog.TableIdentifier
306-
import org.apache.iceberg.spark.SparkCatalog
307-
import org.apache.iceberg.types.Types
308-
309-
val tableName = "hadoop_catalog.db.required_array_field_test"
310-
val catalog = spark.sessionState.catalogManager
311-
.catalog("hadoop_catalog")
312-
.asInstanceOf[SparkCatalog]
313-
val element = Types.StructType.of(Types.NestedField.required(4, "a", Types.IntegerType.get()))
314-
val schema = new Schema(
315-
Types.NestedField.required(1, "id", Types.IntegerType.get()),
316-
Types.NestedField.optional(2, "l", Types.ListType.ofOptional(3, element)))
317-
try {
318-
catalog.icebergCatalog.createTable(
319-
TableIdentifier.of("db", "required_array_field_test"),
320-
schema)
321-
spark.sql(s"""
322-
INSERT INTO $tableName VALUES
323-
(1, array(named_struct('a', 1))),
324-
(2, NULL),
325-
(3, array()),
326-
(4, array(NULL)),
327-
(5, array(NULL, named_struct('a', 2)))
328-
""")
329-
assert(
330-
catalog.icebergCatalog
331-
.loadTable(TableIdentifier.of("db", "required_array_field_test"))
332-
.schema()
333-
.findField("l.element.a")
334-
.isRequired)
335-
val query = s"SELECT id, l.a FROM $tableName WHERE l IS NOT NULL"
336-
val (_, cometPlan) = checkSparkAnswer(query)
337-
assert(collectIcebergNativeScans(cometPlan).length == 1, s"$cometPlan")
338-
assert(
339-
collect(cometPlan) { case project: CometProjectExec => project }.nonEmpty,
340-
s"$cometPlan")
341-
checkAnswer(
342-
spark.sql(query),
343-
Seq(Row(1, Seq(1)), Row(3, Seq.empty[Int]), Row(4, Seq(null)), Row(5, Seq(null, 2))))
344-
} finally {
345-
spark.sql(s"DROP TABLE IF EXISTS $tableName")
346-
}
347-
}
348302

349303
def collectCometShuffleExchanges(plan: org.apache.spark.sql.execution.SparkPlan)
350304
: Seq[org.apache.spark.sql.execution.SparkPlan] = {

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 71 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,24 +26,28 @@ import java.nio.charset.StandardCharsets.UTF_8
2626
import scala.collection.mutable
2727
import scala.jdk.CollectionConverters._
2828

29+
import org.apache.iceberg.Schema
30+
import org.apache.iceberg.catalog.TableIdentifier
2931
import org.apache.iceberg.data.IcebergGenerics
3032
import org.apache.iceberg.expressions.Expressions
31-
import org.apache.iceberg.spark.Spark3Util
33+
import org.apache.iceberg.spark.{Spark3Util, SparkCatalog}
34+
import org.apache.iceberg.types.Types
3235
import org.apache.spark.CometListenerBusUtils
3336
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
3437
import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
35-
import org.apache.spark.sql.catalyst.expressions.DynamicPruningExpression
38+
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPruningExpression}
3639
import org.apache.spark.sql.comet._
3740
import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
3841
import org.apache.spark.sql.execution.{InSubqueryExec, ReusedSubqueryExec, SparkPlan, SubqueryExec}
3942
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, BroadcastQueryStageExec}
4043
import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec}
4144
import org.apache.spark.sql.internal.SQLConf
42-
import org.apache.spark.sql.types.{StringType, TimestampType}
45+
import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType, TimestampType}
4346

4447
import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus}
4548
import org.apache.comet.iceberg.{IcebergReflection, RESTCatalogHelper}
4649
import org.apache.comet.serde.OperatorOuterClass
50+
import org.apache.comet.serde.operator.CometIcebergNativeScan
4751
import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions}
4852

4953
/**
@@ -2215,18 +2219,18 @@ class CometIcebergNativeSuite
22152219
}
22162220

22172221
test("complex type null residuals are not serialized") {
2218-
import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StructType}
2219-
import org.apache.comet.serde.operator.CometIcebergNativeScan
2220-
import org.apache.spark.sql.catalyst.expressions.AttributeReference
2222+
assume(icebergAvailable, "Iceberg not available in classpath")
2223+
22212224
for (dataType <- Seq(
22222225
ArrayType(IntegerType),
22232226
MapType(StringType, IntegerType),
22242227
new StructType().add("value", IntegerType));
22252228
predicate <- Seq(Expressions.isNull("value"), Expressions.notNull("value"))) {
2226-
assert(
2227-
CometIcebergNativeScan
2229+
withClue(s"$dataType: $predicate") {
2230+
assert(CometIcebergNativeScan
22282231
.icebergExprToProto(predicate, Seq(AttributeReference("value", dataType)()), Set.empty)
22292232
.isEmpty)
2233+
}
22302234
}
22312235
assert(
22322236
CometIcebergNativeScan
@@ -2237,6 +2241,65 @@ class CometIcebergNativeSuite
22372241
.nonEmpty)
22382242
}
22392243

2244+
test("required field projection preserves null array elements") {
2245+
assume(icebergAvailable, "Iceberg not available in classpath")
2246+
2247+
withTempIcebergDir { warehouseDir =>
2248+
withSQLConf(
2249+
"spark.sql.catalog.test_cat" -> "org.apache.iceberg.spark.SparkCatalog",
2250+
"spark.sql.catalog.test_cat.type" -> "hadoop",
2251+
"spark.sql.catalog.test_cat.warehouse" -> warehouseDir.getAbsolutePath,
2252+
CometConf.COMET_ENABLED.key -> "true",
2253+
CometConf.COMET_EXEC_ENABLED.key -> "true",
2254+
CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") {
2255+
val tableName = "test_cat.db.required_array_field_test"
2256+
val catalog = spark.sessionState.catalogManager
2257+
.catalog("test_cat")
2258+
.asInstanceOf[SparkCatalog]
2259+
val element =
2260+
Types.StructType.of(Types.NestedField.required(4, "a", Types.IntegerType.get()))
2261+
val schema = new Schema(
2262+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
2263+
Types.NestedField.optional(2, "l", Types.ListType.ofOptional(3, element)))
2264+
try {
2265+
catalog.icebergCatalog.createTable(
2266+
TableIdentifier.of("db", "required_array_field_test"),
2267+
schema)
2268+
spark.sql(s"""
2269+
INSERT INTO $tableName VALUES
2270+
(1, array(named_struct('a', 1))),
2271+
(2, NULL),
2272+
(3, array()),
2273+
(4, array(NULL)),
2274+
(5, array(NULL, named_struct('a', 2)))
2275+
""")
2276+
assert(
2277+
catalog.icebergCatalog
2278+
.loadTable(TableIdentifier.of("db", "required_array_field_test"))
2279+
.schema()
2280+
.findField("l.element.a")
2281+
.isRequired)
2282+
val query = s"SELECT id, l.a FROM $tableName"
2283+
val (_, cometPlan) = checkSparkAnswer(query)
2284+
assertSingleNativeScan(cometPlan)
2285+
assert(
2286+
collect(cometPlan) { case project: CometProjectExec => project }.nonEmpty,
2287+
s"$cometPlan")
2288+
checkAnswer(
2289+
spark.sql(query),
2290+
Seq(
2291+
Row(1, Seq(1)),
2292+
Row(2, null),
2293+
Row(3, Seq.empty[Int]),
2294+
Row(4, Seq(null)),
2295+
Row(5, Seq(null, 2))))
2296+
} finally {
2297+
spark.sql(s"DROP TABLE IF EXISTS $tableName")
2298+
}
2299+
}
2300+
}
2301+
}
2302+
22402303
// Complex type filter tests
22412304
test("complex type filter - struct column IS NULL and IS NOT NULL") {
22422305
assume(icebergAvailable, "Iceberg not available in classpath")

0 commit comments

Comments
 (0)