Skip to content

Commit f80042f

Browse files
authored
fix: Route collated array element comparisons through the codegen dispatcher (#6471)
* fix: Route collated array element comparisons through the codegen dispatcher array_contains, arrays_overlap, array_distinct and array_union compare array elements by raw bytes in their native kernels, so they returned wrong results for strings with a non-UTF8_BINARY collation (for example UTF8_LCASE or UTF8_BINARY_RTRIM) when a dispatched collated cast fed them. Report these cases as Incompatible and mix in CodegenDispatchFallback so they run Spark's collation-aware comparison through the JVM codegen dispatcher by default. ArrayDistinct gets its own serde for this. * docs: Mark collation-routed array functions as Hybrid in expressions.md
1 parent e76c35f commit f80042f

6 files changed

Lines changed: 265 additions & 19 deletions

File tree

‎docs/source/user-guide/latest/expressions.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,8 @@ The tables below list every Spark built-in expression with its current status.
155155
| `array` | ✅ | Native | |
156156
| `array_append` | ✅ | Native | |
157157
| `array_compact` | ✅ | — | |
158-
| `array_contains` | ✅ | Native | Float/double element arrays route through the JVM codegen dispatcher by default; the native path is opt-in via allowIncompatible |
159-
| `array_distinct` | ✅ | Native | Floating-point elements fall back on Spark versions other than 4.2.0; signed-zero and NaN results may differ with native opt-in ([details](compatibility/floating-point.md)) |
158+
| `array_contains` | ✅ | Hybrid | Float/double element arrays and non-default string collations route through the JVM codegen dispatcher by default; the native path is opt-in via allowIncompatible |
159+
| `array_distinct` | ✅ | Native | Floating-point elements fall back on Spark versions other than 4.2.0; signed-zero and NaN results may differ with native opt-in ([details](compatibility/floating-point.md)). Non-default string collations fall back |
160160
| `array_except` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default; the incompatible native path is opt-in via allowIncompatible ([details](compatibility/expressions/array.md)) |
161161
| `array_insert` | ✅ | Native | |
162162
| `array_intersect` | ✅ | Hybrid | Routes through the JVM codegen dispatcher by default; the incompatible native path is opt-in via allowIncompatible ([details](compatibility/expressions/array.md)) |
@@ -167,8 +167,8 @@ The tables below list every Spark built-in expression with its current status.
167167
| `array_prepend` | ✅ | — | |
168168
| `array_remove` | ✅ | Native | |
169169
| `array_repeat` | ✅ | Native | |
170-
| `array_union` | ✅ | Native | Floating-point elements fall back on Spark versions other than 4.2.0; signed-zero and NaN results may differ with native opt-in ([details](compatibility/floating-point.md)) |
171-
| `arrays_overlap` | ✅ | Native | |
170+
| `array_union` | ✅ | Native | Floating-point elements fall back on Spark versions other than 4.2.0; signed-zero and NaN results may differ with native opt-in ([details](compatibility/floating-point.md)). Non-default string collations fall back |
171+
| `arrays_overlap` | ✅ | Hybrid | Non-default string collations use the JVM codegen dispatcher |
172172
| `arrays_zip` | ✅ | Native | |
173173
| `element_at` | ✅ | Native | |
174174
| `flatten` | ✅ | Native | Binary/struct/map elements fall back |

‎spark/src/main/scala/org/apache/comet/serde/arrays.scala‎

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -115,15 +115,24 @@ object CometArrayAppend extends CometExpressionSerde[ArrayAppend] with ArraysBas
115115

116116
object CometArrayContains
117117
extends CometExpressionSerde[ArrayContains]
118+
with CometTypeShim
118119
with CodegenDispatchFallback {
119120

121+
override def hasConditionalNativeDefault: Boolean = true
122+
120123
private val floatingPointReason: String =
121124
"Spark compares array elements with ordering.equiv, so -0.0 matches +0.0 and all NaNs match " +
122125
"each other; Comet's native array_contains compares the raw Arrow values bitwise"
123126

124-
override def getIncompatibleReasons(): Seq[String] = Seq(floatingPointReason)
127+
private val collationReason: String =
128+
ArrayElementEqualitySupport.collationReason("array_contains")
129+
130+
override def getIncompatibleReasons(): Seq[String] = Seq(floatingPointReason, collationReason)
125131

126132
override def getSupportLevel(expr: ArrayContains): SupportLevel = expr.left.dataType match {
133+
// See ArrayElementEqualitySupport: bytewise string comparison ignores the collation.
134+
case ArrayType(elementType, _) if hasNonDefaultStringCollation(elementType) =>
135+
Incompatible(Some(collationReason))
127136
// Native array_contains compares floating-point elements bitwise, disagreeing with Spark for
128137
// -0.0/+0.0 and NaN. Report Incompatible (not Unsupported) for float/double element types (at
129138
// any nesting level) so the expression routes through the JVM codegen dispatcher (Spark's own
@@ -309,7 +318,19 @@ object CometArrayMin extends CometExpressionSerde[ArrayMin] with CodegenDispatch
309318
}
310319
}
311320

312-
object CometArraysOverlap extends CometExpressionSerde[ArraysOverlap] {
321+
object CometArraysOverlap
322+
extends CometExpressionSerde[ArraysOverlap]
323+
with CodegenDispatchFallback {
324+
325+
override def hasConditionalNativeDefault: Boolean = true
326+
327+
override def getIncompatibleReasons(): Seq[String] =
328+
Seq(ArrayElementEqualitySupport.collationReason("arrays_overlap"))
329+
330+
// Both inputs share one element type, so checking the left side is enough.
331+
override def getSupportLevel(expr: ArraysOverlap): SupportLevel =
332+
ArrayElementEqualitySupport.getSupportLevel("arrays_overlap", expr.left.dataType)
333+
313334
override def convert(
314335
expr: ArraysOverlap,
315336
inputs: Seq[Attribute],
@@ -538,6 +559,30 @@ object CometSlice extends CometExpressionSerde[Slice] {
538559
}
539560
}
540561

562+
/**
563+
* Support level for native array kernels that compare elements for equality (membership, overlap,
564+
* dedup). They compare strings by raw bytes, so a non-UTF8_BINARY collation (for example
565+
* UTF8_LCASE, where 'a' equals 'A') gives wrong answers, and these cases report Incompatible.
566+
* Serdes with a boolean result mix in CodegenDispatchFallback, so the JVM codegen dispatcher runs
567+
* Spark's collation-aware comparison. Array-valued serdes fall back instead (see
568+
* ArraySetSupport).
569+
*/
570+
private object ArrayElementEqualitySupport extends CometTypeShim {
571+
def collationReason(name: String): String =
572+
"Spark compares non-UTF8_BINARY collated string elements under their collation, while " +
573+
s"Comet's native $name compares raw bytes"
574+
575+
def collationSupportLevel(name: String, arrayType: DataType): Option[SupportLevel] =
576+
if (hasNonDefaultStringCollation(arrayType)) {
577+
Some(Incompatible(Some(collationReason(name))))
578+
} else {
579+
None
580+
}
581+
582+
def getSupportLevel(name: String, arrayType: DataType): SupportLevel =
583+
collationSupportLevel(name, arrayType).getOrElse(Compatible())
584+
}
585+
541586
private[comet] object ArraySetSupport {
542587
val floatingPointReason: String =
543588
"Floating-point elements match Spark's signed-zero and NaN semantics natively only on " +
@@ -560,22 +605,33 @@ private[comet] object ArraySetSupport {
560605
Compatible()
561606
}
562607
}
608+
609+
/** Collated string elements are also Incompatible; see ArrayElementEqualitySupport. */
610+
def supportLevel(name: String, dataType: DataType): SupportLevel =
611+
ArrayElementEqualitySupport
612+
.collationSupportLevel(name, dataType)
613+
.getOrElse(supportLevel(dataType))
614+
615+
def incompatibleReasons(name: String): Seq[String] =
616+
Seq(floatingPointReason, ArrayElementEqualitySupport.collationReason(name))
563617
}
564618

565619
// Use projection fallback to avoid codegen dispatch overhead for array-valued results.
566620
// The native implementation remains available through opt-in.
567621
object CometArrayDistinct extends CometScalarFunction[ArrayDistinct]("array_distinct") {
568-
override def getIncompatibleReasons(): Seq[String] = Seq(ArraySetSupport.floatingPointReason)
622+
override def getIncompatibleReasons(): Seq[String] =
623+
ArraySetSupport.incompatibleReasons("array_distinct")
569624

570625
override def getSupportLevel(expr: ArrayDistinct): SupportLevel =
571-
ArraySetSupport.supportLevel(expr.dataType)
626+
ArraySetSupport.supportLevel("array_distinct", expr.dataType)
572627
}
573628

574629
object CometArrayUnion extends CometExpressionSerde[ArrayUnion] {
575-
override def getIncompatibleReasons(): Seq[String] = Seq(ArraySetSupport.floatingPointReason)
630+
override def getIncompatibleReasons(): Seq[String] =
631+
ArraySetSupport.incompatibleReasons("array_union")
576632

577633
override def getSupportLevel(expr: ArrayUnion): SupportLevel =
578-
ArraySetSupport.supportLevel(expr.dataType)
634+
ArraySetSupport.supportLevel("array_union", expr.dataType)
579635

580636
override def convert(
581637
expr: ArrayUnion,
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- MinSparkVersion: 4.0
19+
-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
20+
-- Config: spark.comet.expression.ArrayContains.allowIncompatible=false
21+
-- Config: spark.comet.expression.ArraysOverlap.allowIncompatible=false
22+
-- Config: spark.comet.expression.ArrayDistinct.allowIncompatible=false
23+
-- Config: spark.comet.expression.ArrayUnion.allowIncompatible=false
24+
25+
-- array_contains, arrays_overlap, array_distinct and array_union compare array elements for
26+
-- equality. The native kernels compare strings by raw bytes, so under a non-UTF8_BINARY collation
27+
-- they would miss 'a' = 'A' (UTF8_LCASE) or 'x ' = 'x' (UTF8_BINARY_RTRIM). Collated
28+
-- array_contains and arrays_overlap route through the JVM codegen dispatcher, which runs Spark's
29+
-- collation-aware comparison. array_distinct and array_union return arrays, where dispatch costs
30+
-- more than a projection fallback (see ArraySetSupport), so they fall back to Spark instead.
31+
32+
statement
33+
CREATE TABLE test_array_eq_collation(id int, a string, b string) USING parquet
34+
35+
statement
36+
INSERT INTO test_array_eq_collation VALUES
37+
(1, 'a', 'A'),
38+
(2, 'x ', 'x'),
39+
(3, 'b', 'c'),
40+
(4, NULL, 'a'),
41+
(5, 'A', NULL)
42+
43+
-- UTF8_BINARY strings stay on the native kernels.
44+
query expect_native(array_contains,arrays_overlap,array_distinct,array_union)
45+
SELECT id, array_contains(array(a), b), arrays_overlap(array(a), array(b)),
46+
array_distinct(array(a, b)), array_union(array(a), array(b))
47+
FROM test_array_eq_collation
48+
49+
-- UTF8_LCASE: 'a' and 'A' are equal.
50+
query expect_dispatch(array_contains)
51+
SELECT id, array_contains(array(CAST(a AS STRING COLLATE UTF8_LCASE)), CAST(b AS STRING COLLATE UTF8_LCASE))
52+
FROM test_array_eq_collation
53+
54+
query expect_dispatch(arrays_overlap)
55+
SELECT id, arrays_overlap(array(CAST(a AS STRING COLLATE UTF8_LCASE)), array(CAST(b AS STRING COLLATE UTF8_LCASE)))
56+
FROM test_array_eq_collation
57+
58+
query expect_fallback(native array_distinct compares raw bytes)
59+
SELECT id, array_distinct(array(CAST(a AS STRING COLLATE UTF8_LCASE), CAST(b AS STRING COLLATE UTF8_LCASE)))
60+
FROM test_array_eq_collation
61+
62+
query expect_fallback(native array_union compares raw bytes)
63+
SELECT id, array_union(array(CAST(a AS STRING COLLATE UTF8_LCASE)), array(CAST(b AS STRING COLLATE UTF8_LCASE)))
64+
FROM test_array_eq_collation
65+
66+
-- UTF8_BINARY_RTRIM: 'x ' and 'x' are equal.
67+
query expect_dispatch(array_contains)
68+
SELECT id, array_contains(array(CAST(a AS STRING COLLATE UTF8_BINARY_RTRIM)), CAST(b AS STRING COLLATE UTF8_BINARY_RTRIM))
69+
FROM test_array_eq_collation
70+
71+
query expect_dispatch(arrays_overlap)
72+
SELECT id, arrays_overlap(array(CAST(a AS STRING COLLATE UTF8_BINARY_RTRIM)), array(CAST(b AS STRING COLLATE UTF8_BINARY_RTRIM)))
73+
FROM test_array_eq_collation
74+
75+
query expect_fallback(native array_distinct compares raw bytes)
76+
SELECT id, array_distinct(array(CAST(a AS STRING COLLATE UTF8_BINARY_RTRIM), CAST(b AS STRING COLLATE UTF8_BINARY_RTRIM)))
77+
FROM test_array_eq_collation
78+
79+
query expect_fallback(native array_union compares raw bytes)
80+
SELECT id, array_union(array(CAST(a AS STRING COLLATE UTF8_BINARY_RTRIM)), array(CAST(b AS STRING COLLATE UTF8_BINARY_RTRIM)))
81+
FROM test_array_eq_collation
82+
83+
-- Collation detection also recurses into struct fields of the array elements.
84+
query expect_dispatch(array_contains,arrays_overlap)
85+
SELECT id,
86+
array_contains(array(named_struct('s', CAST(a AS STRING COLLATE UTF8_LCASE))),
87+
named_struct('s', CAST(b AS STRING COLLATE UTF8_LCASE))),
88+
arrays_overlap(array(named_struct('s', CAST(a AS STRING COLLATE UTF8_LCASE))),
89+
array(named_struct('s', CAST(b AS STRING COLLATE UTF8_LCASE))))
90+
FROM test_array_eq_collation
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- MinSparkVersion: 4.0
19+
-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
20+
-- Config: spark.comet.expression.ArrayContains.allowIncompatible=true
21+
-- Config: spark.comet.expression.ArraysOverlap.allowIncompatible=true
22+
-- Config: spark.comet.expression.ArrayDistinct.allowIncompatible=true
23+
-- Config: spark.comet.expression.ArrayUnion.allowIncompatible=true
24+
25+
-- allowIncompatible=true opts collated inputs into the native kernels, which compare raw bytes.
26+
-- The data only holds values whose bytewise and collation-aware equality agree, so the answers
27+
-- still match Spark; the point is that the native path is taken.
28+
29+
statement
30+
CREATE TABLE test_array_eq_collation(id int, a string, b string) USING parquet
31+
32+
statement
33+
INSERT INTO test_array_eq_collation VALUES (1, 'a', 'a'), (2, 'b', 'c'), (3, NULL, 'a')
34+
35+
query expect_native(array_contains,arrays_overlap)
36+
SELECT id,
37+
array_contains(array(CAST(a AS STRING COLLATE UTF8_LCASE)), CAST(b AS STRING COLLATE UTF8_LCASE)),
38+
arrays_overlap(array(CAST(a AS STRING COLLATE UTF8_LCASE)), array(CAST(b AS STRING COLLATE UTF8_LCASE)))
39+
FROM test_array_eq_collation
40+
41+
query expect_native(array_distinct,array_union)
42+
SELECT id,
43+
array_distinct(array(CAST(a AS STRING COLLATE UTF8_LCASE), CAST(b AS STRING COLLATE UTF8_LCASE))),
44+
array_union(array(CAST(a AS STRING COLLATE UTF8_LCASE)), array(CAST(b AS STRING COLLATE UTF8_LCASE)))
45+
FROM test_array_eq_collation
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- MinSparkVersion: 4.0
19+
-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
20+
-- Config: spark.comet.expression.ArrayContains.allowIncompatible=false
21+
-- Config: spark.comet.expression.ArraysOverlap.allowIncompatible=false
22+
-- Config: spark.comet.expression.ArrayDistinct.allowIncompatible=false
23+
-- Config: spark.comet.expression.ArrayUnion.allowIncompatible=false
24+
25+
-- With the JVM codegen dispatcher disabled, collated array_contains and arrays_overlap have no
26+
-- Spark-compatible Comet path and fall back to Spark. array_distinct and array_union fall back
27+
-- whether or not the dispatcher is enabled.
28+
29+
statement
30+
CREATE TABLE test_array_eq_collation(id int, a string, b string) USING parquet
31+
32+
statement
33+
INSERT INTO test_array_eq_collation VALUES (1, 'a', 'A'), (2, 'x ', 'x'), (3, 'b', 'c'), (4, NULL, 'a')
34+
35+
query expect_fallback(array_contains: spark.comet.exec.scalaUDF.codegen.enabled=false)
36+
SELECT id, array_contains(array(CAST(a AS STRING COLLATE UTF8_LCASE)), CAST(b AS STRING COLLATE UTF8_LCASE))
37+
FROM test_array_eq_collation
38+
39+
query expect_fallback(arrays_overlap: spark.comet.exec.scalaUDF.codegen.enabled=false)
40+
SELECT id, arrays_overlap(array(CAST(a AS STRING COLLATE UTF8_BINARY_RTRIM)), array(CAST(b AS STRING COLLATE UTF8_BINARY_RTRIM)))
41+
FROM test_array_eq_collation
42+
43+
query expect_fallback(native array_distinct compares raw bytes)
44+
SELECT id, array_distinct(array(CAST(a AS STRING COLLATE UTF8_LCASE), CAST(b AS STRING COLLATE UTF8_LCASE)))
45+
FROM test_array_eq_collation
46+
47+
query expect_fallback(native array_union compares raw bytes)
48+
SELECT id, array_union(array(CAST(a AS STRING COLLATE UTF8_BINARY_RTRIM)), array(CAST(b AS STRING COLLATE UTF8_BINARY_RTRIM)))
49+
FROM test_array_eq_collation

‎spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -467,18 +467,24 @@ class CometMapExpressionSuite extends CometTestBase {
467467
}
468468

469469
// The collated counterpart of the double-key `map_contains_key` case above: a nested
470-
// `UTF8_LCASE`-keyed map would reach `array_contains` with bytewise comparison. Expansion declines
471-
// the folded literal so the case-insensitive lookup stays on Spark. The outer lookup key is the
472-
// dynamic `_1` so the inner map survives as a literal (a constant key would let Spark fold
473-
// `map_keys` into a collated-string array literal instead, which never reaches this guard).
470+
// `UTF8_LCASE`-keyed map would reach `array_contains` with bytewise comparison. Like the
471+
// floating-point case, `CometArrayContains` reports collated element types Incompatible, so the
472+
// default config codegen-dispatches it; `allowIncompatible=true` forces the native kernel so the
473+
// expansion guard runs. Expansion declines the folded literal so the case-insensitive lookup
474+
// stays on Spark. The outer lookup key is the dynamic `_1` so the inner map survives as a literal
475+
// (a constant key would let Spark fold `map_keys` into a collated-string array literal instead,
476+
// which never reaches this guard).
474477
test("map_contains_key over nested collated map keys falls back (multirow)") {
475478
assume(isSpark40Plus)
479+
val query = "SELECT _1 AS id, map_contains_key(" +
480+
"element_at(map(1, map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7)), _1), " +
481+
"CAST('a1' AS STRING COLLATE UTF8_LCASE)) AS present FROM tbl"
476482
withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") {
477-
checkSparkAnswerAndFallbackReason(
478-
"SELECT _1 AS id, map_contains_key(" +
479-
"element_at(map(1, map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7)), _1), " +
480-
"CAST('a1' AS STRING COLLATE UTF8_LCASE)) AS present FROM tbl",
481-
"Unsupported data type MapType")
483+
withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[ArrayContains]) -> "true") {
484+
checkSparkAnswerAndFallbackReason(query, "Unsupported data type MapType")
485+
}
486+
// Under the default config the collated lookup is dispatched and matches Spark.
487+
checkSparkAnswer(query)
482488
}
483489
}
484490

0 commit comments

Comments
 (0)