Skip to content

Commit 701e38a

Browse files
committed
fix: preserve map_from_arrays Spark error parity
1 parent a85d8a6 commit 701e38a

3 files changed

Lines changed: 22 additions & 7 deletions

File tree

‎docs/source/contributor-guide/expression-audits/map_funcs.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
## map_from_arrays
4646

4747
- Spark 3.4.3 (audited 2026-05-27): identical to 3.5.8.
48-
- Spark 3.5.8 (audited 2026-05-27): baseline. `MapFromArrays(left, right) extends BinaryExpression with NullIntolerant`; Spark uses `ArrayBasedMapBuilder` to detect duplicate keys (subject to `spark.sql.mapKeyDedupPolicy`) and rejects null keys with `RuntimeException("Cannot use null as map key")`. Comet `CometMapFromArrays` wraps the inputs in `CaseWhen(IsNotNull(left) AND IsNotNull(right), map(left, right), null)` so NULL-array inputs return NULL rather than triggering the previously reported native crash ([#3327](https://github.com/apache/datafusion-comet/issues/3327)).
48+
- Spark 3.5.8 (audited 2026-05-27): baseline. `MapFromArrays(left, right) extends BinaryExpression with NullIntolerant`; Spark uses `ArrayBasedMapBuilder` to detect duplicate keys (subject to `spark.sql.mapKeyDedupPolicy`) and rejects null keys with `RuntimeException("Cannot use null as map key")`. Comet `CometMapFromArrays` emits nested `CaseWhen(IsNotNull(left), CaseWhen(IsNotNull(right), map(left, right), null), null)` guards so NULL-array inputs return NULL rather than triggering the previously reported native crash ([#3327](https://github.com/apache/datafusion-comet/issues/3327)). The outer key guard preserves Spark's left-before-right evaluation: one `IsNotNull(left) AND IsNotNull(right)` predicate can still evaluate the values child for a NULL-key row in a mixed batch. The native `SparkMapFromArrays` wrapper also validates key/value lengths per non-null row before DataFusion constructs the map, so matching flattened totals cannot pair values across row boundaries.
4949
- Spark 4.0.1 (audited 2026-05-27): semantics unchanged; `NullIntolerant` trait replaced by `nullIntolerant: Boolean`.
5050
- Spark 4.1.1 (audited 2026-05-27): identical to 4.0.1.
5151

‎native/spark-expr/src/map_funcs/map_from_arrays.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ use datafusion::common::{exec_err, utils::take_function_args, Result};
2121
use datafusion::functions_nested::map::MapFunc;
2222
use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature};
2323

24+
use crate::SparkError;
25+
2426
/// Checks row boundaries before the DataFusion `map` used by CometMapFromArrays.
2527
#[derive(Debug, Default, PartialEq, Eq, Hash)]
2628
pub(crate) struct SparkMapFromArrays {
@@ -63,6 +65,13 @@ impl ScalarUDFImpl for SparkMapFromArrays {
6365
}
6466
}
6567

68+
/// Verifies that each visible row where both arrays are non-null has matching entry counts.
69+
///
70+
/// The caller expands scalar operands into arrays of the requested batch size before calling this
71+
/// function, so `keys` and `values` are row-aligned list-like arrays. Rows with either null
72+
/// array are skipped because Spark returns a null map before enforcing key/value lengths. A
73+
/// mismatched non-null row returns Spark's structured map-size error; unsupported list types still
74+
/// return the DataFusion execution error produced by `list_length`.
6675
fn validate_list_lengths(keys: &ArrayRef, values: &ArrayRef) -> Result<()> {
6776
// The upstream array path uses key offsets for both flattened children without checking
6877
// each row's lengths. Do not let batched operands silently pair across map rows.
@@ -72,12 +81,16 @@ fn validate_list_lengths(keys: &ArrayRef, values: &ArrayRef) -> Result<()> {
7281
&& values.is_valid(row)
7382
&& list_length(keys, row)? != list_length(values, row)?
7483
{
75-
return exec_err!("map requires key and value lists to have the same length");
84+
return Err(SparkError::MapKeyValueDiffSizes.into());
7685
}
7786
}
7887
Ok(())
7988
}
8089

90+
/// Returns the element count for one row of a supported Arrow list array.
91+
///
92+
/// This accepts variable-width and fixed-width list encodings. It returns a DataFusion execution
93+
/// error when the caller provides a non-list array rather than guessing an incompatible shape.
8194
fn list_length(array: &ArrayRef, row: usize) -> Result<i64> {
8295
match array.data_type() {
8396
DataType::List(_) => Ok(i64::from(array.as_list::<i32>().value_length(row))),

‎spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,9 @@ class CometMapExpressionSuite extends CometTestBase {
134134
CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> codegenEnabled) {
135135
withTable("map_null_keys") {
136136
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
137-
// Keep null and non-null keys in one batch. Only the null-key row divides by zero.
137+
// Keep null and non-null keys in one batch. Only the null-key row divides by zero,
138+
// and only that row has two values for no keys. Spark skips both evaluation and
139+
// length validation before constructing the non-null rows' maps.
138140
spark
139141
.range(0, 3, 1, 1)
140142
.selectExpr("CAST(id AS INT) AS k")
@@ -144,7 +146,7 @@ class CometMapExpressionSuite extends CometTestBase {
144146
}
145147
val query = """SELECT map_from_arrays(
146148
| CASE WHEN k = 0 THEN CAST(NULL AS ARRAY<INT>) ELSE array(1) END,
147-
| array(1 / k))
149+
| CASE WHEN k = 0 THEN array(1 / k, 2) ELSE array(1 / k) END)
148150
|FROM map_null_keys""".stripMargin
149151
val plan = sql(query).queryExecution.executedPlan
150152
assert(new ExtendedExplainInfo().getNativeExpressions(plan).contains("map_from_arrays"))
@@ -176,9 +178,9 @@ class CometMapExpressionSuite extends CometTestBase {
176178
val plan = sql(query).queryExecution.executedPlan
177179
assert(
178180
new ExtendedExplainInfo().getNativeExpressions(plan).contains("map_from_arrays"))
179-
val (sparkError, cometError) = checkSparkAnswerMaybeThrows(sql(query))
180-
assert(sparkError.exists(_.getMessage.contains("same length")))
181-
assert(cometError.exists(_.getMessage.contains("same length")))
181+
// Spark exposes this through the same legacy condition on every supported version.
182+
// checkSparkError also verifies the exception class and SQLSTATE match Spark.
183+
checkSparkError(sql(query), "_LEGACY_ERROR_TEMP_2128")
182184
}
183185
}
184186
}

0 commit comments

Comments
 (0)