Skip to content

Commit 76b9d2e

Browse files
authored
feat: add codegen dispatch fallback for CometCast incompatible/unsupported cases (including legacy config paths) (#5079)
1 parent 038fd6e commit 76b9d2e

9 files changed

Lines changed: 137 additions & 54 deletions

File tree

‎dev/diffs/4.0.2.diff‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1380,6 +1380,28 @@ index 2e33f6505ab..be967565303 100644
13801380
}
13811381

13821382
withTable("t1", "t2") {
1383+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala
1384+
index a40e34d94d0..abc1f035d15 100644
1385+
--- a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala
1386+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala
1387+
@@ -21,6 +21,7 @@ import org.apache.spark.sql.QueryTest.sameRows
1388+
import org.apache.spark.sql.catalyst.InternalRow
1389+
import org.apache.spark.sql.catalyst.expressions.{Cast, Literal}
1390+
import org.apache.spark.sql.catalyst.expressions.variant.{ToVariantObject, VariantExpressionEvalUtils}
1391+
+import org.apache.spark.sql.comet.CometNativeColumnarToRowExec
1392+
import org.apache.spark.sql.execution.WholeStageCodegenExec
1393+
import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
1394+
import org.apache.spark.sql.functions._
1395+
@@ -358,7 +359,8 @@ class VariantEndToEndSuite extends QueryTest with SharedSparkSession {
1396+
s"cast(to_variant_object(s) as ${schema(2).dataType.sql})")
1397+
checkAnswer(df, input)
1398+
val plan = df.queryExecution.executedPlan
1399+
- assert(plan.isInstanceOf[WholeStageCodegenExec] == (codegenMode == "CODEGEN_ONLY"))
1400+
+ assert(plan.isInstanceOf[WholeStageCodegenExec] == (codegenMode == "CODEGEN_ONLY")
1401+
+ || plan.isInstanceOf[CometNativeColumnarToRowExec])
1402+
}
1403+
}
1404+
}
13831405
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala
13841406
index 11e9547dfc5..637411056ae 100644
13851407
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala

‎dev/diffs/4.1.2.diff‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1479,6 +1479,28 @@ index 3ba48da0e32..0313aaa9ec6 100644
14791479
}
14801480

14811481
withTable("t1", "t2") {
1482+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala
1483+
index 8a0e2c29653..d276a51cbc6 100644
1484+
--- a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala
1485+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala
1486+
@@ -21,6 +21,7 @@ import org.apache.spark.sql.QueryTest.sameRows
1487+
import org.apache.spark.sql.catalyst.InternalRow
1488+
import org.apache.spark.sql.catalyst.expressions.{Cast, Literal}
1489+
import org.apache.spark.sql.catalyst.expressions.variant.{ToVariantObject, VariantExpressionEvalUtils}
1490+
+import org.apache.spark.sql.comet.CometNativeColumnarToRowExec
1491+
import org.apache.spark.sql.execution.WholeStageCodegenExec
1492+
import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
1493+
import org.apache.spark.sql.functions._
1494+
@@ -358,7 +359,8 @@ class VariantEndToEndSuite extends QueryTest with SharedSparkSession {
1495+
s"cast(to_variant_object(s) as ${schema(2).dataType.sql})")
1496+
checkAnswer(df, input)
1497+
val plan = df.queryExecution.executedPlan
1498+
- assert(plan.isInstanceOf[WholeStageCodegenExec] == (codegenMode == "CODEGEN_ONLY"))
1499+
+ assert(plan.isInstanceOf[WholeStageCodegenExec] == (codegenMode == "CODEGEN_ONLY")
1500+
+ || plan.isInstanceOf[CometNativeColumnarToRowExec])
1501+
}
1502+
}
1503+
}
14821504
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantShreddingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/VariantShreddingSuite.scala
14831505
index fee375db10a..8c2c24e2c5f 100644
14841506
--- a/sql/core/src/test/scala/org/apache/spark/sql/VariantShreddingSuite.scala

‎spark/src/main/scala/org/apache/comet/expressions/CometCast.scala‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,22 +25,28 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType,
2525

2626
import org.apache.comet.CometConf
2727
import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason}
28-
import org.apache.comet.serde.{CometExpressionSerde, Compatible, ExprOuterClass, Incompatible, SupportLevel, Unsupported}
28+
import org.apache.comet.DataTypeSupport.isComplexType
29+
import org.apache.comet.serde.{CodegenDispatchFallback, CometExpressionSerde, Compatible, ExprOuterClass, Incompatible, SupportLevel, Unsupported}
2930
import org.apache.comet.serde.ExprOuterClass.Expr
3031
import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProtoInternal, serializeDataType}
31-
import org.apache.comet.shims.CometExprShim
32+
import org.apache.comet.shims.{CometExprShim, CometTypeShim}
3233

33-
object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
34+
object CometCast
35+
extends CometExpressionSerde[Cast]
36+
with CometExprShim
37+
with CometTypeShim
38+
with CodegenDispatchFallback {
3439

3540
// Shared with CometCastSuite so the asserted reason cannot drift from production.
3641
private[comet] val negativeScaleDecimalToStringReason: String =
3742
"Negative-scale decimal requires spark.sql.legacy.allowNegativeScaleOfDecimal=true"
3843

3944
// When `spark.sql.legacy.castComplexTypesToString.enabled` is true, Spark wraps maps and
4045
// structs with `[]` (instead of `{}`) when casting to string, and omits NULL elements of
41-
// structs/maps/arrays (instead of rendering them as the literal "null"). Comet only
42-
// implements the default formatting, so fall back to Spark for any array/map/struct to-string
43-
// cast when the flag is enabled. The flag is internal in Spark 4.0 and defaults to false.
46+
// structs/maps/arrays (instead of rendering them as the literal "null"). Comet's native
47+
// cast only implements the default formatting, so with the flag enabled we route
48+
// array/map/struct to-string casts through the JVM codegen dispatcher via
49+
// `CodegenDispatchFallback`. The flag is internal in Spark 4.0 and defaults to false.
4450
private[comet] val legacyCastComplexTypesToStringReason: String =
4551
"spark.sql.legacy.castComplexTypesToString.enabled=true is not supported"
4652

@@ -75,7 +81,12 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
7581
if (cast.child.isInstanceOf[Literal]) {
7682
// A cast whose child is a literal is folded by Spark at planning time via `cast.eval()`
7783
// (see `convert`), so the cast never executes natively and the result matches Spark by
78-
// definition. `CometLiteral` then validates the resulting literal's data type.
84+
// definition. `CometLiteral` then validates the resulting literal's data type, except
85+
// for `VariantType` which must be rejected here: the fold produces a `Literal[VariantType]`
86+
// that no downstream Comet serde can serialize.
87+
if (isVariantType(cast.child.dataType) || isVariantType(cast.dataType)) {
88+
return unsupported(cast.child.dataType, cast.dataType)
89+
}
7990
Compatible()
8091
} else {
8192
isSupported(cast.child.dataType, cast.dataType, cast.timeZoneId, evalMode(cast))
@@ -159,13 +170,20 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
159170
timeZoneId: Option[String],
160171
evalMode: CometEvalMode.Value): SupportLevel = {
161172

173+
// Spark 4's `VariantType` (SPARK-45827) has no native counterpart in Comet, and the codegen
174+
// dispatcher also cannot serialize `VariantType` in the data args or return type. The
175+
// version-shimmed `isVariantType` returns false on Spark 3.x. Reporting `Unsupported` lets the
176+
// `CodegenDispatchFallback` mixin try the dispatcher and then fall back to Spark cleanly.
177+
if (isVariantType(fromType) || isVariantType(toType)) {
178+
return unsupported(fromType, toType)
179+
}
180+
162181
if (fromType == toType) {
163182
return Compatible()
164183
}
165184

166-
if (toType == DataTypes.StringType && legacyCastComplexTypesToString && (fromType
167-
.isInstanceOf[ArrayType] || fromType.isInstanceOf[StructType] ||
168-
fromType.isInstanceOf[MapType])) {
185+
if (toType == DataTypes.StringType && isComplexType(fromType) &&
186+
legacyCastComplexTypesToString) {
169187
return Unsupported(Some(legacyCastComplexTypesToStringReason))
170188
}
171189

‎spark/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ trait CometTypeShim {
3636
@nowarn // Spark 4 feature; Variant shredding doesn't exist in Spark 3.x.
3737
def isVariantStruct(s: StructType): Boolean = false
3838

39+
@nowarn // Spark 4 feature; VariantType doesn't exist in Spark 3.x.
40+
def isVariantType(dt: DataType): Boolean = false
41+
3942
@nowarn // Spark 4.1 feature; TimeType doesn't exist in Spark 3.x.
4043
def isTimeType(dt: DataType): Boolean = false
4144
}

‎spark/src/main/spark-4.x/org/apache/comet/shims/CometTypeShim.scala‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
package org.apache.comet.shims
2121

2222
import org.apache.spark.sql.execution.datasources.VariantMetadata
23-
import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StringType, StructType}
23+
import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StringType, StructType, VariantType}
2424

2525
trait CometTypeShim {
2626
// A `StringType` carries collation metadata in Spark 4.0. Only non-default (non-UTF8_BINARY)
@@ -54,6 +54,12 @@ trait CometTypeShim {
5454
// and force scan fallback.
5555
def isVariantStruct(s: StructType): Boolean = VariantMetadata.isVariantStruct(s)
5656

57+
// Comet has no native execution path for Spark 4's `VariantType` (introduced in
58+
// SPARK-45827). Serdes call this to route casts/expressions touching the type back to Spark
59+
// rather than serializing an unsupported datatype into the native plan. Stubbed to `false` in
60+
// Spark 3.x where `VariantType` does not exist.
61+
def isVariantType(dt: DataType): Boolean = dt.isInstanceOf[VariantType]
62+
5763
def isTimeType(dt: DataType): Boolean =
5864
dt.getClass.getSimpleName.startsWith("TimeType")
5965

‎spark/src/test/resources/sql-tests/expressions/cast/cast_complex_types_to_string.sql‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,7 @@ SELECT cast(named_struct('a', named_struct('b', named_struct('c', 1, 'd', 'leaf'
149149
query
150150
SELECT cast(named_struct('s1', '', 's2', ' ', 's3', cast(null as string)) as string)
151151

152-
-- Map-valued field: not supported, falls back to Spark.
153-
query expect_fallback(to StringType is not supported)
152+
query
154153
SELECT cast(named_struct('m', map('k', 1)) as string)
155154

156155
-- ----------------------------------------------------------------------------
@@ -270,69 +269,70 @@ SELECT cast(array(cast(1.5 as double), cast('NaN' as double), cast('-Infinity' a
270269
query
271270
SELECT cast(array(array(array(1, 2), array(3)), array(array(cast(null as int)))) as string)
272271

273-
-- Array of map: not supported, falls back to Spark.
274-
query expect_fallback(to StringType is not supported)
272+
-- Array of map: map-to-string is routed through the codegen dispatcher via the outer array.
273+
query
275274
SELECT cast(array(map('k', 1)) as string)
276275

277276
-- ----------------------------------------------------------------------------
278277
-- Map → string
279278
-- ----------------------------------------------------------------------------
280-
-- Comet does not implement map-to-string casts, so every map → string falls back to Spark.
279+
-- Comet has no native map-to-string cast; `CometCast` mixes in `CodegenDispatchFallback`, so
280+
-- these stay native via the codegen dispatcher and match Spark exactly.
281281
-- Note: maps materialized through parquet have nondeterministic entry order, so map column
282282
-- tests use literal maps directly rather than reading from a parquet table.
283283

284284
-- Map with string keys, int values.
285-
query expect_fallback(Cast from MapType)
285+
query
286286
SELECT cast(map('a', 1, 'b', 2, 'c', 3) as string)
287287

288288
-- Map with NULL values rendered as "null".
289-
query expect_fallback(Cast from MapType)
289+
query
290290
SELECT cast(map('a', 1, 'b', cast(null as int), 'c', 3) as string)
291291

292292
-- Map with int keys, string values.
293-
query expect_fallback(Cast from MapType)
293+
query
294294
SELECT cast(map(1, 'one', 2, 'two', 3, 'three') as string)
295295

296296
-- Map with boolean values.
297-
query expect_fallback(Cast from MapType)
297+
query
298298
SELECT cast(map('t', true, 'f', false, 'n', cast(null as boolean)) as string)
299299

300300
-- Map with bigint values at min/max.
301-
query expect_fallback(Cast from MapType)
301+
query
302302
SELECT cast(map('max', 9223372036854775807, 'min', -9223372036854775808, 'zero', cast(0 as bigint)) as string)
303303

304304
-- Map with decimal values.
305-
query expect_fallback(Cast from MapType)
305+
query
306306
SELECT cast(map('pos', cast('1.234567890123456789' as decimal(38, 18)), 'neg', cast('-1.234567890123456789' as decimal(38, 18)), 'null', cast(null as decimal(38, 18))) as string)
307307

308308
-- Map with date and timestamp values.
309-
query expect_fallback(Cast from MapType)
309+
query
310310
SELECT cast(map('a', date '2024-01-15', 'b', date '1970-01-01', 'c', cast(null as date)) as string)
311311

312-
query expect_fallback(Cast from MapType)
312+
query
313313
SELECT cast(map('a', timestamp '2024-01-15 10:30:45', 'b', cast(null as timestamp)) as string)
314314

315315
-- Map with binary values.
316-
query expect_fallback(Cast from MapType)
316+
query
317317
SELECT cast(map('a', X'616263', 'b', X'', 'c', cast(null as binary)) as string)
318318

319319
-- Map with float / double values: NaN / ±0 / ±Infinity / NULL.
320-
query expect_fallback(Cast from MapType)
320+
query
321321
SELECT cast(map('nan', cast('NaN' as float), 'neg0', cast(-0.0 as float), 'null', cast(null as float)) as string)
322322

323-
query expect_fallback(Cast from MapType)
323+
query
324324
SELECT cast(map('nan', cast('NaN' as double), 'inf', cast('Infinity' as double), 'ninf', cast('-Infinity' as double), 'null', cast(null as double)) as string)
325325

326326
-- Map with struct values: each value rendered as `{f1, f2, ...}`.
327-
query expect_fallback(Cast from MapType)
327+
query
328328
SELECT cast(map('a', named_struct('x', 1, 'y', 'first'), 'b', cast(null as struct<x: int, y: string>)) as string)
329329

330330
-- Map with array values.
331-
query expect_fallback(Cast from MapType)
331+
query
332332
SELECT cast(map('a', array(1, 2, 3), 'b', array(cast(null as int)), 'c', cast(null as array<int>)) as string)
333333

334334
-- Empty map.
335-
query expect_fallback(Cast from MapType)
335+
query
336336
SELECT cast(map() as string)
337337

338338
-- NULL map: Spark constant-folds this to a literal NULL, so the cast never reaches Comet
@@ -341,5 +341,5 @@ query
341341
SELECT cast(cast(null as map<string, int>) as string)
342342

343343
-- Map of map.
344-
query expect_fallback(Cast from MapType)
344+
query
345345
SELECT cast(map('outer', map('inner', 1)) as string)

‎spark/src/test/resources/sql-tests/expressions/cast/cast_complex_types_to_string_legacy.sql‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,26 +15,25 @@
1515
-- specific language governing permissions and limitations
1616
-- under the License.
1717

18-
-- When `spark.sql.legacy.castComplexTypesToString.enabled` is true Spark wraps maps and
19-
-- structs with `[...]` (instead of `{...}`) and omits NULL elements of structs/maps/arrays
20-
-- (instead of rendering them as the literal "null"). Comet only implements the default
21-
-- formatting, so any array/map/struct → string cast must fall back to Spark.
22-
-- The flag is internal in Spark 4.0 and defaults to false.
18+
-- When `spark.sql.legacy.castComplexTypesToString.enabled` is true, Spark wraps maps and
19+
-- structs with `[...]` instead of `{...}` and omits NULL elements. `CometCast` mixes in
20+
-- `CodegenDispatchFallback`, so these casts stay native via the codegen dispatcher and
21+
-- match Spark exactly. The flag is internal in Spark 4.0 and defaults to false.
2322

2423
-- Config: spark.sql.legacy.castComplexTypesToString.enabled=true
2524

26-
-- Struct → string falls back.
27-
query expect_fallback(spark.sql.legacy.castComplexTypesToString.enabled=true is not supported)
25+
-- Struct → string.
26+
query
2827
SELECT CAST(struct(1, 2, null) AS STRING)
2928

30-
-- Array → string falls back (NULL elements rendered differently between modes).
31-
query expect_fallback(spark.sql.legacy.castComplexTypesToString.enabled=true is not supported)
29+
-- Array → string.
30+
query
3231
SELECT CAST(array(1, 2, null) AS STRING)
3332

34-
-- Map → string falls back (`[]` vs `{}` wrapping differs between modes).
35-
query expect_fallback(spark.sql.legacy.castComplexTypesToString.enabled=true is not supported)
33+
-- Map → string.
34+
query
3635
SELECT CAST(map('a', 1, 'b', null) AS STRING)
3736

38-
-- Nested complex types still fall back through the outer type.
39-
query expect_fallback(spark.sql.legacy.castComplexTypesToString.enabled=true is not supported)
37+
-- Nested complex types via the outer struct.
38+
query
4039
SELECT CAST(struct(array(1, null), map('k', null)) AS STRING)

‎spark/src/test/scala/org/apache/comet/CometCastSuite.scala‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1797,7 +1797,7 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
17971797
Unsupported(Some(expectedMessage)))
17981798
}
17991799

1800-
test("cast ArrayType(DateType) to unsupported ArrayType falls back") {
1800+
test("cast ArrayType(DateType) to unsupported ArrayType routes through codegen dispatch") {
18011801
val fromType = ArrayType(DateType)
18021802
val unsupportedElementTypes =
18031803
Seq(BooleanType, ByteType, ShortType, LongType, FloatType, DoubleType, DecimalType(10, 2))
@@ -1814,9 +1814,7 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
18141814
assert(
18151815
CometCast.isSupported(fromType, toType, None, CometEvalMode.LEGACY) ==
18161816
Unsupported(Some(expectedMessage)))
1817-
checkSparkAnswerAndFallbackReason(
1818-
data.select(col("a").cast(toType).as("converted")),
1819-
expectedMessage)
1817+
checkSparkAnswerAndOperator(data.select(col("a").cast(toType).as("converted")))
18201818
}
18211819
}
18221820
}
@@ -1870,6 +1868,26 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
18701868
}
18711869
}
18721870

1871+
test("cast to and from VariantType matches Spark") {
1872+
// VariantType has no native path in Comet and the codegen dispatcher cannot serialize it
1873+
// either, so the operator falls back to Spark. Guarded on Spark 4.0+ (parse_json / VARIANT
1874+
// are unavailable in 3.x).
1875+
assume(CometSparkSessionExtensions.isSpark40Plus, "VariantType requires Spark 4.0+")
1876+
withTable("variant_cast") {
1877+
sql("CREATE TABLE variant_cast(id INT, s STRING) USING parquet")
1878+
sql("""
1879+
|INSERT INTO variant_cast VALUES
1880+
| (1, '{"a": 1}'),
1881+
| (2, '{"b": [1, 2, 3]}'),
1882+
| (3, cast(null as string))
1883+
""".stripMargin)
1884+
checkSparkAnswer(
1885+
"SELECT id, CAST(parse_json(s) AS STRING) AS v FROM variant_cast ORDER BY id")
1886+
checkSparkAnswer(
1887+
"SELECT id, CAST(CAST(s AS VARIANT) AS STRING) AS v FROM variant_cast ORDER BY id")
1888+
}
1889+
}
1890+
18731891
private def testArrayCastMatrix(
18741892
elementTypes: Seq[DataType],
18751893
wrapType: DataType => DataType,

‎spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2060,9 +2060,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
20602060
sql(s"insert into $table values(0, 1, 100.000001)")
20612061

20622062
Seq(
2063-
(
2064-
s"SELECT cast(make_interval(c0, c1, c0, c1, c0, c0, c2) as string) as C from $table",
2065-
Set("Cast from CalendarIntervalType to StringType is not supported")),
20662063
(
20672064
"SELECT "
20682065
+ "date_part('YEAR', make_interval(c0, c1, c0, c1, c0, c0, c2))"
@@ -2080,9 +2077,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
20802077
+ s"(SELECT c1, sum(c0) as sum_c0, sum(c2) as sum_c2 from $table group by c1) as A, "
20812078
+ s"(SELECT c1, cast(make_interval(c0, c1, c0, c1, c0, c0, c2) as string) as casted from $table) as B "
20822079
+ "where A.c1 = B.c1 ",
2083-
Set(
2084-
"Cast from CalendarIntervalType to StringType is not supported",
2085-
"Comet shuffle is not enabled: spark.comet.shuffle.enabled is not enabled")),
2080+
Set("Comet shuffle is not enabled: spark.comet.shuffle.enabled is not enabled")),
20862081
(s"select * from $table LIMIT 10 OFFSET 3", Set("Comet shuffle is not enabled")))
20872082
.foreach(test => {
20882083
val qry = test._1

0 commit comments

Comments
 (0)