Skip to content

Commit 41fc083

Browse files
committed
fix: keep collated array_join and array_intersect native via codegen dispatch
Address review feedback on #4713: report Incompatible instead of Unsupported for non-default collations so both route through the JVM codegen dispatcher (Spark's own doGenCode) and stay native while matching Spark, reconciling the three #2190 collation cases with CometReverse. array_join concatenation is collation-independent, so only the output string's collation metadata is dropped. array_intersect membership is collation-aware, so the dispatcher (not the byte-based native path) produces the correct result. Add collation SQL tests for both.
1 parent 0571632 commit 41fc083

3 files changed

Lines changed: 57 additions & 20 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/arrays.scala‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -193,16 +193,20 @@ object CometArrayIntersect
193193
"Result array element order may differ from Spark when the right array is longer " +
194194
"than the left (DataFusion probes the longer side)."
195195

196-
private val unsupportedCollationReason: String =
197-
"array_intersect on collated strings is not supported."
198-
199-
override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason)
196+
private val collationReason: String =
197+
"array_intersect does not propagate non-UTF8_BINARY collations to the output array elements " +
198+
"(https://github.com/apache/datafusion-comet/issues/2190)"
200199

201-
override def getUnsupportedReasons(): Seq[String] = Seq(unsupportedCollationReason)
200+
override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason, collationReason)
202201

203202
override def getSupportLevel(expr: ArrayIntersect): SupportLevel = {
203+
// The native array_intersect dedups by raw bytes, which is wrong under non-default collations,
204+
// so report Incompatible rather than Unsupported: the JVM codegen dispatcher (Spark's own
205+
// doGenCode) performs collation-aware set membership and keeps execution native, matching
206+
// Spark. Only the output elements' collation metadata is dropped, consistent with CometReverse
207+
// and CometArrayJoin.
204208
if (hasNonDefaultStringCollation(expr.dataType)) {
205-
Unsupported(Some(unsupportedCollationReason))
209+
Incompatible(Some(collationReason))
206210
} else {
207211
Incompatible(Some(incompatReason))
208212
}
@@ -393,20 +397,20 @@ object CometArrayJoin
393397

394398
private val incompatReason = "Null handling may differ from Spark"
395399

396-
private val unsupportedCollationReason =
397-
"array_join on collated strings is not supported " +
400+
private val collationReason =
401+
"array_join does not propagate non-UTF8_BINARY collations to the output string " +
398402
"(https://github.com/apache/datafusion-comet/issues/2190)"
399403

400-
override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason)
401-
402-
override def getUnsupportedReasons(): Seq[String] = Seq(unsupportedCollationReason)
404+
override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason, collationReason)
403405

404406
override def getSupportLevel(expr: ArrayJoin): SupportLevel = {
405-
// Spark 4.0 widens ArrayJoin's input to StringTypeWithCollation; the native array_to_string
406-
// produces UTF8_BINARY semantics and does not propagate non-default collations, so surface
407-
// that as a fallback reason in EXPLAIN, mirroring CometArrayIntersect.
407+
// Spark 4.0 widens ArrayJoin's input to StringTypeWithCollation. Concatenation itself is
408+
// collation-independent, so the joined value is always correct; only the output string's
409+
// collation metadata is dropped (Comet columns are UTF8_BINARY). Report Incompatible rather
410+
// than Unsupported so the JVM codegen dispatcher (Spark's own doGenCode) keeps collated
411+
// array_join native and matching Spark, consistent with CometReverse's #2190 handling.
408412
if (hasNonDefaultStringCollation(expr.array.dataType)) {
409-
Unsupported(Some(unsupportedCollationReason))
413+
Incompatible(Some(collationReason))
410414
} else {
411415
Incompatible(Some(incompatReason))
412416
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- MinSparkVersion: 4.0
19+
20+
-- Spark 4.0+ widens ArrayIntersect's input to collated strings. Set membership is collation-aware,
21+
-- so Comet's native byte-based dedup would be wrong. Comet instead routes collated array_intersect
22+
-- through the JVM codegen dispatcher (Spark's own doGenCode), which performs collation-aware
23+
-- membership and matches Spark. Only the output elements' collation metadata is dropped (#2190).
24+
25+
-- UTF8_LCASE: 'A' and 'a' are equal, so the intersection is non-empty and case is taken from the
26+
-- left array.
27+
query
28+
SELECT array_intersect(array('A' COLLATE UTF8_LCASE, 'b' COLLATE UTF8_LCASE), array('a' COLLATE UTF8_LCASE))
29+
30+
-- UTF8_LCASE with duplicates and a non-matching element
31+
query
32+
SELECT array_intersect(array('Hello' COLLATE UTF8_LCASE, 'WORLD' COLLATE UTF8_LCASE), array('hello' COLLATE UTF8_LCASE, 'x' COLLATE UTF8_LCASE))

‎spark/src/test/resources/sql-tests/expressions/array/array_join_collation.sql‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,15 @@
1717

1818
-- MinSparkVersion: 4.0
1919

20-
-- Spark 4.0+ widens ArrayJoin's input to collated strings. Comet's native array_to_string matches
21-
-- raw UTF8_BINARY bytes and does not propagate non-default collations, so collated inputs fall
22-
-- back to Spark.
20+
-- Spark 4.0+ widens ArrayJoin's input to collated strings. Concatenation is collation-independent,
21+
-- so the joined value is always correct; Comet routes collated array_join through the JVM codegen
22+
-- dispatcher (Spark's own doGenCode), keeping it native and matching Spark. Only the output
23+
-- string's collation metadata is dropped (tracked by #2190).
2324

2425
-- collated array elements
25-
query expect_fallback(array_join on collated strings is not supported)
26+
query
2627
SELECT array_join(array('a' COLLATE UTF8_LCASE, 'b' COLLATE UTF8_LCASE), ',')
2728

2829
-- collated elements with null replacement
29-
query expect_fallback(array_join on collated strings is not supported)
30+
query
3031
SELECT array_join(array('a' COLLATE UTF8_LCASE, NULL, 'c' COLLATE UTF8_LCASE), ',', 'NULL')

0 commit comments

Comments
 (0)