Skip to content

Commit 266c540

Browse files
committed
fix: preserve nullability when extracting array struct fields
1 parent 3700467 commit 266c540

2 files changed

Lines changed: 133 additions & 7 deletions

File tree

‎native/spark-expr/src/array_funcs/get_array_struct_fields.rs‎

Lines changed: 86 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,15 @@ impl GetArrayStructFields {
6464
}
6565

6666
fn child_field(&self, input_schema: &Schema) -> DataFusionResult<FieldRef> {
67-
match self.list_field(input_schema)?.data_type() {
68-
DataType::Struct(fields) => Ok(Arc::clone(&fields[self.ordinal])),
67+
let list_field = self.list_field(input_schema)?;
68+
match list_field.data_type() {
69+
DataType::Struct(fields) => {
70+
let field = &fields[self.ordinal];
71+
// A null struct element yields null even when the field itself is required.
72+
Ok(Arc::new(field.as_ref().clone().with_nullable(
73+
list_field.is_nullable() || field.is_nullable(),
74+
)))
75+
}
6976
data_type => Err(DataFusionError::Internal(format!(
7077
"Unexpected data type in GetArrayStructFields: {data_type:?}"
7178
))),
@@ -86,23 +93,23 @@ impl PhysicalExpr for GetArrayStructFields {
8693
}
8794

8895
fn nullable(&self, input_schema: &Schema) -> DataFusionResult<bool> {
89-
Ok(self.list_field(input_schema)?.is_nullable()
90-
|| self.child_field(input_schema)?.is_nullable())
96+
self.child.nullable(input_schema)
9197
}
9298

9399
fn evaluate(&self, batch: &RecordBatch) -> DataFusionResult<ColumnarValue> {
94100
let child_value = self.child.evaluate(batch)?.into_array(batch.num_rows())?;
101+
let field = self.child_field(batch.schema().as_ref())?;
95102

96103
match child_value.data_type() {
97104
DataType::List(_) => {
98105
let list_array = as_list_array(&child_value)?;
99106

100-
get_array_struct_fields(list_array, self.ordinal)
107+
get_array_struct_fields(list_array, self.ordinal, field)
101108
}
102109
DataType::LargeList(_) => {
103110
let list_array = as_large_list_array(&child_value)?;
104111

105-
get_array_struct_fields(list_array, self.ordinal)
112+
get_array_struct_fields(list_array, self.ordinal, field)
106113
}
107114
data_type => Err(DataFusionError::Internal(format!(
108115
"Unexpected child type for ListExtract: {data_type:?}"
@@ -135,14 +142,14 @@ impl PhysicalExpr for GetArrayStructFields {
135142
fn get_array_struct_fields<O: OffsetSizeTrait>(
136143
list_array: &GenericListArray<O>,
137144
ordinal: usize,
145+
field: FieldRef,
138146
) -> DataFusionResult<ColumnarValue> {
139147
let values = list_array
140148
.values()
141149
.as_any()
142150
.downcast_ref::<StructArray>()
143151
.expect("A StructType is expected");
144152

145-
let field = Arc::clone(&values.fields()[ordinal]);
146153
// A field of a null struct is null, and Arrow keeps the children's validity independent of the
147154
// parent's, so the parent's nulls have to be unioned in. See
148155
// `datafusion_comet_common::struct_nulls`.
@@ -171,3 +178,75 @@ impl Display for GetArrayStructFields {
171178
)
172179
}
173180
}
181+
182+
#[cfg(test)]
183+
mod tests {
184+
use super::*;
185+
use arrow::array::Int32Array;
186+
use arrow::buffer::{NullBuffer, OffsetBuffer};
187+
use arrow::datatypes::Field;
188+
use datafusion::physical_expr::expressions::Column;
189+
190+
fn check_nullability<O: OffsetSizeTrait>() {
191+
for list_nullable in [true, false] {
192+
for element_nullable in [true, false] {
193+
for field_nullable in [false, true] {
194+
let field = Arc::new(Field::new("a", DataType::Int32, field_nullable));
195+
let values = Arc::new(StructArray::new(
196+
vec![field].into(),
197+
vec![Arc::new(Int32Array::from(vec![1, 99]))],
198+
element_nullable.then(|| NullBuffer::from(vec![true, false])),
199+
));
200+
let list = Arc::new(GenericListArray::<O>::new(
201+
Arc::new(Field::new(
202+
"element",
203+
values.data_type().clone(),
204+
element_nullable,
205+
)),
206+
OffsetBuffer::from_lengths([1, 1, 0, 0]),
207+
values,
208+
list_nullable.then(|| NullBuffer::from(vec![true, true, true, false])),
209+
));
210+
let schema = Arc::new(Schema::new(vec![Field::new(
211+
"l",
212+
list.data_type().clone(),
213+
list_nullable,
214+
)]));
215+
let batch =
216+
RecordBatch::try_new(Arc::clone(&schema), vec![list.clone()]).unwrap();
217+
let expr = GetArrayStructFields::new(Arc::new(Column::new("l", 0)), 0);
218+
assert_eq!(expr.nullable(&schema).unwrap(), list_nullable);
219+
let result = expr.evaluate(&batch).unwrap().into_array(4).unwrap();
220+
assert_eq!(expr.data_type(&schema).unwrap(), *result.data_type());
221+
let result = result
222+
.as_any()
223+
.downcast_ref::<GenericListArray<O>>()
224+
.unwrap();
225+
let output_field = match result.data_type() {
226+
DataType::List(field) | DataType::LargeList(field) => field,
227+
_ => unreachable!(),
228+
};
229+
assert_eq!(
230+
output_field.is_nullable(),
231+
element_nullable || field_nullable
232+
);
233+
let values = result
234+
.values()
235+
.as_any()
236+
.downcast_ref::<Int32Array>()
237+
.unwrap();
238+
assert_eq!(values.value(0), 1);
239+
assert_eq!(values.is_null(1), element_nullable);
240+
assert_eq!(result.offsets(), list.offsets());
241+
assert_eq!(result.nulls(), list.nulls());
242+
}
243+
}
244+
}
245+
}
246+
247+
#[test]
248+
fn get_array_struct_fields_nullability() {
249+
check_nullability::<i32>();
250+
check_nullability::<i64>();
251+
}
252+
}

‎spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ package org.apache.comet
2222
import scala.util.Random
2323

2424
import org.apache.spark.sql.{Column, Row}
25+
import org.apache.spark.sql.comet.CometProjectExec
2526
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
2627
import org.apache.spark.sql.functions.{col, lit}
2728
import org.apache.spark.sql.types._
@@ -280,6 +281,52 @@ class CometFuzzIcebergSuite extends CometFuzzIcebergBase {
280281
spark.sql(s"DROP TABLE IF EXISTS $tableName")
281282
}
282283
}
284+
test("filter pushdown - required field projection preserves null array elements") {
285+
import org.apache.iceberg.Schema
286+
import org.apache.iceberg.catalog.TableIdentifier
287+
import org.apache.iceberg.spark.SparkCatalog
288+
import org.apache.iceberg.types.Types
289+
290+
val tableName = "hadoop_catalog.db.required_array_field_test"
291+
val catalog = spark.sessionState.catalogManager
292+
.catalog("hadoop_catalog")
293+
.asInstanceOf[SparkCatalog]
294+
val element = Types.StructType.of(Types.NestedField.required(4, "a", Types.IntegerType.get()))
295+
val schema = new Schema(
296+
Types.NestedField.required(1, "id", Types.IntegerType.get()),
297+
Types.NestedField.optional(2, "l", Types.ListType.ofOptional(3, element)))
298+
try {
299+
catalog.icebergCatalog.createTable(
300+
TableIdentifier.of("db", "required_array_field_test"),
301+
schema)
302+
spark.sql(s"""
303+
INSERT INTO $tableName VALUES
304+
(1, array(named_struct('a', 1))),
305+
(2, NULL),
306+
(3, array()),
307+
(4, array(NULL)),
308+
(5, array(NULL, named_struct('a', 2)))
309+
""")
310+
assert(
311+
catalog.icebergCatalog
312+
.loadTable(TableIdentifier.of("db", "required_array_field_test"))
313+
.schema()
314+
.findField("l.element.a")
315+
.isRequired)
316+
val query = s"SELECT id, l.a FROM $tableName WHERE l IS NOT NULL"
317+
val (_, cometPlan) = checkSparkAnswer(query)
318+
assert(collectIcebergNativeScans(cometPlan).length == 1, s"$cometPlan")
319+
assert(
320+
collect(cometPlan) { case project: CometProjectExec => project }.nonEmpty,
321+
s"$cometPlan")
322+
checkAnswer(
323+
spark.sql(query),
324+
Seq(Row(1, Seq(1)), Row(3, Seq.empty[Int]), Row(4, Seq(null)), Row(5, Seq(null, 2))))
325+
} finally {
326+
spark.sql(s"DROP TABLE IF EXISTS $tableName")
327+
}
328+
}
329+
283330
def collectCometShuffleExchanges(plan: org.apache.spark.sql.execution.SparkPlan)
284331
: Seq[org.apache.spark.sql.execution.SparkPlan] = {
285332
collect(plan) {

0 commit comments

Comments
 (0)