Skip to content

Commit 9866221

Browse files
authored
fix: keep a folded single-entry map literal with a null key on Spark (#6585)
1 parent ff468b1 commit 9866221

2 files changed

Lines changed: 14 additions & 1 deletion

File tree

‎spark/src/main/scala/org/apache/comet/serde/literals.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -465,10 +465,10 @@ object CometLiteral extends CometExpressionSerde[Literal] with CometTypeShim wit
465465
* instead of asking the ordering to compare it.
466466
*/
467467
private def hasDuplicateMapKeys(keys: ArrayData, keyType: DataType): Boolean = {
468-
if (keys.numElements() < 2) return false
469468
// Single pass out of the ArrayData; a null slot comes back as null.
470469
val sorted = keys.toObjectArray(keyType)
471470
if (sorted.contains(null)) return true
471+
if (sorted.length < 2) return false
472472
val ordering = TypeUtils.getInterpretedOrdering(keyType)
473473
java.util.Arrays.sort(sorted, ordering.asInstanceOf[Ordering[Object]])
474474
(1 until sorted.length).exists(i => ordering.compare(sorted(i - 1), sorted(i)) == 0)

‎spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,19 @@ class CometMapExpressionSuite extends CometTestBase {
496496
}
497497
}
498498

499+
// A TRY cast over a foldable map folds to a literal whose failing key is null. With a single
500+
// entry it has to stay on Spark too, rather than being rebuilt as a `CreateMap` that rejects the
501+
// null key. Spark returns NULL for both rows.
502+
// https://github.com/apache/datafusion-comet/issues/6584
503+
test("folded single-entry map literal with a null key falls back (multirow)") {
504+
withParquetTable((1 until 3).map(i => (i, i.toLong)), "tbl") {
505+
checkSparkAnswerAndFallbackReason(
506+
"SELECT _1 AS id, element_at(try_cast(map(9999999999L, 20) AS map<int, int>), _1) AS v " +
507+
"FROM tbl",
508+
"Unsupported data type MapType")
509+
}
510+
}
511+
499512
// Direct single-level folded map with collated string keys. Native lookup is bytewise, so a
500513
// case-insensitive `a1` lookup against a stored `A1` cannot match; `MapKeySupport` declines it.
501514
test("folded map literal with collated string keys in element_at falls back (multirow)") {

0 commit comments

Comments
 (0)