Skip to content

Commit ec98323

Browse files
committed
feat: support Map for CreateArray literal
1 parent 215ab70 commit ec98323

20 files changed

Lines changed: 1173 additions & 109 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,9 @@ use crate::execution::{
4949
};
5050
use crate::jvm_bridge::{jni_call, JVMClasses, ShufflePartitionPusher};
5151
use arrow::compute::CastOptions;
52-
use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit, DECIMAL128_MAX_PRECISION};
52+
use arrow::datatypes::{
53+
DataType, Field, FieldRef, Fields, Schema, TimeUnit, DECIMAL128_MAX_PRECISION,
54+
};
5355
use arrow::ffi_stream::FFI_ArrowArrayStream;
5456
use datafusion::functions_aggregate::bit_and_or_xor::{bit_and_udaf, bit_or_udaf, bit_xor_udaf};
5557
use datafusion::functions_aggregate::count::count_udaf;
@@ -218,6 +220,39 @@ fn make_all_fields_nullable(data_type: &DataType) -> DataType {
218220
}
219221
}
220222

223+
/// Return a copy of a `Map` type with only the outer entries `value` field marked nullable, keeping
224+
/// the key field non-nullable and every nested key/value type byte-for-byte unchanged. Any non-`Map`
225+
/// type is returned unchanged.
226+
///
227+
/// This is the shallow counterpart of `make_all_fields_nullable`, used for `map_entries`.
228+
/// `map_entries` reuses the input map's entries array as its output list values but declares that
229+
/// element's `value` field nullable (`Struct(key non-null, value nullable)`), deriving both nested
230+
/// types verbatim from the input. Arrow's `GenericListArray::try_new` compares the declared element
231+
/// field's type against the reused values array's type in full, so the ONLY field that can mismatch
232+
/// is the entries `value` field's own `nullable` flag; widening just that field is sufficient.
233+
/// Recursing into nested types (as `make_all_fields_nullable` does) would additionally flip, say, a
234+
/// nested `Map`'s `valueContainsNull`, which then diverges from the Spark-serialized `return_type`
235+
/// and makes a downstream `make_array` see unequal map types and panic.
236+
fn widen_map_entry_value_nullable(data_type: &DataType) -> DataType {
237+
match data_type {
238+
DataType::Map(entries, sorted) => match entries.data_type() {
239+
DataType::Struct(kv) if kv.len() == 2 && !kv[1].is_nullable() => {
240+
let new_value = Arc::new(kv[1].as_ref().clone().with_nullable(true));
241+
let new_kv: Fields = vec![Arc::clone(&kv[0]), new_value].into();
242+
let new_entries = Arc::new(
243+
entries
244+
.as_ref()
245+
.clone()
246+
.with_data_type(DataType::Struct(new_kv)),
247+
);
248+
DataType::Map(new_entries, *sorted)
249+
}
250+
_ => data_type.clone(),
251+
},
252+
other => other.clone(),
253+
}
254+
}
255+
221256
/// If `expr` evaluates to `Timestamp(_, Some(_))` against `schema`, wrap it in a
222257
/// metadata-only cast to `Timestamp(_, None)`. This is required because
223258
/// DataFusion's `SortMergeJoinExec` comparator only supports timezone-less
@@ -3490,6 +3525,17 @@ impl PhysicalPlanner {
34903525
.collect::<Result<Vec<_>, _>>()?;
34913526

34923527
let fun_name = &expr.func;
3528+
// `map_entries` needs its argument's entry `value` field widened to nullable first (only
3529+
// that outer field). See `widen_map_entry_value_nullable`.
3530+
let args = if fun_name == "map_entries" {
3531+
args.into_iter()
3532+
.map(|arg| {
3533+
Self::coerce_child_to(arg, &input_schema, widen_map_entry_value_nullable)
3534+
})
3535+
.collect::<Result<Vec<_>, ExecutionError>>()?
3536+
} else {
3537+
args
3538+
};
34933539
let input_expr_types = args
34943540
.iter()
34953541
.map(|x| x.data_type(input_schema.as_ref()))
@@ -3653,6 +3699,26 @@ impl PhysicalPlanner {
36533699
Ok(Arc::new(CastExpr::new(child, nullable_type, None)))
36543700
}
36553701

3702+
/// Casts `child` so its type matches `widen(child_type)`, wrapping it in a `CastExpr` only when
3703+
/// the widened type differs. Used for the `map_entries` argument widening (with
3704+
/// `widen_map_entry_value_nullable`). Unlike `coerce_collect_child_nullability`, which casts
3705+
/// unconditionally as a normalization barrier for aggregate accumulators, this casts only when
3706+
/// the type actually changes. That is sufficient for the scalar `map_entries`, whose reused
3707+
/// entries array only needs its declared element type to line up.
3708+
fn coerce_child_to(
3709+
child: Arc<dyn PhysicalExpr>,
3710+
schema: &SchemaRef,
3711+
widen: impl Fn(&DataType) -> DataType,
3712+
) -> Result<Arc<dyn PhysicalExpr>, ExecutionError> {
3713+
let child_type = child.data_type(schema.as_ref())?;
3714+
let widened = widen(&child_type);
3715+
if child_type.equals_datatype(&widened) {
3716+
Ok(child)
3717+
} else {
3718+
Ok(Arc::new(CastExpr::new(child, widened, None)))
3719+
}
3720+
}
3721+
36563722
fn create_aggr_func_expr(
36573723
name: &str,
36583724
schema: SchemaRef,

‎native/spark-expr/src/struct_funcs/create_named_struct.rs‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
use arrow::array::StructArray;
1919
use arrow::datatypes::{DataType, Field, Schema};
2020
use arrow::record_batch::RecordBatch;
21-
use datafusion::common::Result as DataFusionResult;
21+
use datafusion::common::{Result as DataFusionResult, ScalarValue};
2222
use datafusion::logical_expr::ColumnarValue;
2323
use datafusion::physical_expr::PhysicalExpr;
2424
use std::{
@@ -71,13 +71,25 @@ impl PhysicalExpr for CreateNamedStruct {
7171
.iter()
7272
.map(|expr| expr.evaluate(batch))
7373
.collect::<datafusion::common::Result<Vec<_>>>()?;
74+
// When every field value is a scalar (e.g. an all-literal `named_struct` that reaches native
75+
// as a `CreateNamedStruct` because constant folding is disabled), return a scalar struct
76+
// rather than a length-1 `StructArray`. A downstream consumer such as `make_array` then
77+
// broadcasts the constant struct to the batch row count instead of failing with a
78+
// mixed-length error when a sibling child is a full-length column. This matches what a
79+
// constant-folded struct literal produces, and `GetStructField::evaluate` already handles a
80+
// scalar struct input.
81+
let all_scalar =
82+
!values.is_empty() && values.iter().all(|v| matches!(v, ColumnarValue::Scalar(_)));
7483
let arrays = ColumnarValue::values_to_arrays(&values)?;
7584
let fields = self.fields(&batch.schema())?;
76-
Ok(ColumnarValue::Array(Arc::new(StructArray::new(
77-
fields.into(),
78-
arrays,
79-
None,
80-
))))
85+
let struct_array = StructArray::new(fields.into(), arrays, None);
86+
if all_scalar {
87+
Ok(ColumnarValue::Scalar(ScalarValue::Struct(Arc::new(
88+
struct_array,
89+
))))
90+
} else {
91+
Ok(ColumnarValue::Array(Arc::new(struct_array)))
92+
}
8193
}
8294

8395
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {

‎spark/src/main/java/org/apache/comet/codegen/CometBatchKernel.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,13 @@
3131
*/
3232
public abstract class CometBatchKernel extends CometInternalRow {
3333

34-
protected final Object[] references;
34+
// `public` (not `protected`) so that the nested helper classes Spark's codegen emits when it
35+
// splits a large expression (e.g. a folded map rebuilt as a big `CreateMap`) can read it. Those
36+
// helpers are separate classes in the generated package, not subclasses of this one, so a
37+
// `protected` field would raise `IllegalAccessError` at runtime under cross-package protected
38+
// access rules. Spark's own generated classes avoid this by declaring `references` on the
39+
// generated class itself; Comet inherits it here instead, so it must be public.
40+
public final Object[] references;
3541

3642
protected CometBatchKernel(Object[] references) {
3743
this.references = references;

‎spark/src/main/scala/org/apache/comet/DataTypeSupport.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,21 @@ object DataTypeSupport {
8080
case _ => false
8181
}
8282

83+
/**
84+
* `dt` with every array/map/struct nullability flag forced to `true` at all nesting levels (map
85+
* key fields stay non-null per Arrow's map invariant). Re-derives Spark's `private[spark]`
86+
* `DataType.asNullable`, used as a common cast target to unify types whose Comet runtime
87+
* nullability exceeds Spark's Catalyst nullability.
88+
*/
89+
def deepNullable(dt: DataType): DataType = dt match {
90+
case ArrayType(et, _) => ArrayType(deepNullable(et), containsNull = true)
91+
case MapType(kt, vt, _) =>
92+
MapType(deepNullable(kt), deepNullable(vt), valueContainsNull = true)
93+
case StructType(fields) =>
94+
StructType(fields.map(f => f.copy(dataType = deepNullable(f.dataType), nullable = true)))
95+
case other => other
96+
}
97+
8398
def hasTemporalType(t: DataType): Boolean = t match {
8499
case DataTypes.DateType | DataTypes.TimestampType | DataTypes.TimestampNTZType =>
85100
true

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,22 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
531531
}
532532
}
533533

534+
/**
535+
* Attach a fresh `expr_id` and, when the expression's origin carries one, its `QueryContext` to
536+
* `protoExpr`. Native ANSI errors resolve their context by the `expr_id` of the `Expr` that
537+
* throws (see `register_query_context` and `ListExtract` in the native planner), so the
538+
* metadata has to sit on that `Expr`. The generic serde path applies this to the top-level
539+
* `Expr` it returns; a serde that nests a throwing expression inside a wrapper (e.g.
540+
* `CometElementAt`'s CASE-WHEN NULL guard) must also call it on the inner `Expr`, or that
541+
* expression's error renders without Spark's `== SQL ... ==` query context.
542+
*/
543+
private[serde] def attachExprIdAndContext(expr: Expression, protoExpr: Expr): Expr = {
544+
val builder = protoExpr.toBuilder
545+
builder.setExprId(nextExprId())
546+
extractQueryContext(expr).foreach(builder.setQueryContext)
547+
builder.build()
548+
}
549+
534550
def supportedDataType(dt: DataType, allowComplex: Boolean = false): Boolean = dt match {
535551
case _: ByteType | _: ShortType | _: IntegerType | _: LongType | _: FloatType |
536552
_: DoubleType | _: StringType | _: BinaryType | _: TimestampType | _: TimestampNTZType |
@@ -1004,12 +1020,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
10041020
withNativeExpr(expr, CometExplainInfo.exprDisplayName(expr))
10051021
}
10061022
// Attach QueryContext and expr_id to the expression
1007-
val builder = protoExpr.toBuilder
1008-
builder.setExprId(nextExprId())
1009-
extractQueryContext(expr).foreach { ctx =>
1010-
builder.setQueryContext(ctx)
1011-
}
1012-
builder.build()
1023+
attachExprIdAndContext(expr, protoExpr)
10131024
}
10141025
}
10151026

0 commit comments

Comments
 (0)