Skip to content

Commit be6c67c

Browse files
committed
test: cover multibyte strings, nested types, date and timestamp bounds and a non-UTC session over a shuffle-read write
1 parent 5fa47b2 commit be6c67c

2 files changed

Lines changed: 40 additions & 7 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -437,7 +437,9 @@ private class CometIcebergWriteRDD(
437437

438438
override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = {
439439
// Before the shuffle fetch that `resolveInputObjects` starts, as the probe's RDD does for a
440-
// child that is not read natively.
440+
// child that is not read natively. The probe is test-only, and only the scheduler-ordering
441+
// tests need this hook, so this path does not report `afterInput`: the JVM never sees the
442+
// rows of a shuffle that the native plan decodes.
441443
shuffleProbe.foreach(_.beforeInput())
442444
val (inputObjects, shuffleBlockIterators) =
443445
CometExecRDD.resolveInputObjects(Seq(input), Array(split), shuffleScanIndices, context)

‎spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,28 +1143,50 @@ class CometIcebergWriteActionSuite
11431143
withIcebergCatalog { _ =>
11441144
val columns = "id BIGINT, s STRING, d DOUBLE, st STRUCT<a: INT, b: STRING>, " +
11451145
"arr ARRAY<INT>, m MAP<STRING, INT>, dt DATE, ts TIMESTAMP, ntz TIMESTAMP_NTZ, " +
1146-
"dec DECIMAL(12, 3), bin BINARY, b BOOLEAN, f FLOAT"
1146+
"dec DECIMAL(12, 3), bin BINARY, b BOOLEAN, f FLOAT, " +
1147+
"nested ARRAY<STRUCT<x: INT, y: STRING>>, " +
1148+
"deep STRUCT<k: MAP<STRING, ARRAY<INT>>, v: INT>"
11471149
spark.sql(s"CREATE TABLE $catalog.$ns.shuffle_src ($columns) USING iceberg")
11481150
spark.sql(s"""
11491151
INSERT INTO $catalog.$ns.shuffle_src
11501152
SELECT id,
1151-
IF(id % 7 = 0, NULL, CONCAT('s', CAST(id AS STRING))),
1153+
CASE WHEN id % 7 = 0 THEN NULL
1154+
WHEN id % 5 = 0 THEN CONCAT('日本語-', CAST(id AS STRING))
1155+
WHEN id % 3 = 0 THEN CONCAT('é😀-', CAST(id AS STRING))
1156+
ELSE CONCAT('s', CAST(id AS STRING)) END,
11521157
CASE id % 11 WHEN 0 THEN NULL WHEN 1 THEN CAST('NaN' AS DOUBLE) WHEN 2 THEN -0.0D
11531158
WHEN 3 THEN CAST('Infinity' AS DOUBLE) WHEN 4 THEN CAST('-Infinity' AS DOUBLE)
11541159
ELSE id / 3.0 END,
11551160
IF(id % 13 = 0, NULL,
11561161
NAMED_STRUCT('a', CAST(id AS INT), 'b', IF(id % 2 = 0, NULL, CAST(id AS STRING)))),
11571162
IF(id % 17 = 0, NULL, ARRAY(CAST(id AS INT), NULL)),
11581163
IF(id % 19 = 0, NULL, MAP(CAST(id AS STRING), CAST(id AS INT))),
1159-
IF(id % 23 = 0, NULL, DATE_ADD(DATE '1970-01-01', CAST(id * 7 AS INT))),
1160-
IF(id % 29 = 0, NULL, TIMESTAMP_MICROS(id * 1000003)),
1161-
IF(id % 31 = 0, NULL, CAST(TIMESTAMP_MICROS(id * 999983) AS TIMESTAMP_NTZ)),
1164+
CASE WHEN id % 23 = 0 THEN NULL
1165+
WHEN id % 23 = 1 THEN DATE '1900-01-01'
1166+
WHEN id % 23 = 2 THEN DATE '1969-12-31'
1167+
WHEN id % 23 = 3 THEN DATE '9999-12-31'
1168+
ELSE DATE_ADD(DATE '1970-01-01', CAST(id * 7 AS INT)) END,
1169+
CASE WHEN id % 29 = 0 THEN NULL
1170+
WHEN id % 29 = 1 THEN TIMESTAMP_MICROS(-2208988800000000)
1171+
WHEN id % 29 = 2 THEN TIMESTAMP_MICROS(-1)
1172+
WHEN id % 29 = 3 THEN TIMESTAMP_MICROS(253402300799999999)
1173+
ELSE TIMESTAMP_MICROS(id * 1000003) END,
1174+
CASE WHEN id % 31 = 0 THEN NULL
1175+
WHEN id % 31 = 1 THEN CAST(TIMESTAMP_MICROS(-2208988800000000) AS TIMESTAMP_NTZ)
1176+
WHEN id % 31 = 2 THEN CAST(TIMESTAMP_MICROS(253402300799999999) AS TIMESTAMP_NTZ)
1177+
ELSE CAST(TIMESTAMP_MICROS(id * 999983) AS TIMESTAMP_NTZ) END,
11621178
IF(id % 37 = 0, NULL, CAST(CAST(id AS DECIMAL(12, 3)) / 7 AS DECIMAL(12, 3))),
11631179
IF(id % 41 = 0, NULL, CAST(CONCAT('b', CAST(id AS STRING)) AS BINARY)),
11641180
IF(id % 43 = 0, NULL, id % 2 = 0),
11651181
CASE id % 10 WHEN 0 THEN CAST('NaN' AS FLOAT) WHEN 1 THEN CAST(-0.0D AS FLOAT)
11661182
WHEN 2 THEN CAST('Infinity' AS FLOAT) WHEN 3 THEN CAST('-Infinity' AS FLOAT)
1167-
WHEN 4 THEN NULL ELSE CAST(id AS FLOAT) / 3 END
1183+
WHEN 4 THEN NULL ELSE CAST(id AS FLOAT) / 3 END,
1184+
IF(id % 47 = 0, NULL,
1185+
ARRAY(NAMED_STRUCT('x', CAST(id AS INT), 'y', CONCAT('日', CAST(id AS STRING))),
1186+
NULL, NAMED_STRUCT('x', CAST(NULL AS INT), 'y', CAST(NULL AS STRING)))),
1187+
IF(id % 53 = 0, NULL,
1188+
NAMED_STRUCT('k', IF(id % 2 = 0, MAP('a', ARRAY(CAST(id AS INT), NULL), 'b', ARRAY()),
1189+
CAST(NULL AS MAP<STRING, ARRAY<INT>>)), 'v', CAST(id AS INT)))
11681190
FROM range(0, 2000)""")
11691191

11701192
// (case, hint, filter, AQE on, the write's leaf when direct read is on, extra confs).
@@ -1183,6 +1205,15 @@ class CometIcebergWriteActionSuite
11831205
true,
11841206
"SHUFFLE_SCAN",
11851207
Seq(SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false")),
1208+
// The timestamp columns hold instants, so a session time zone must not change what the
1209+
// shuffle blocks decode to.
1210+
(
1211+
"non_utc_session",
1212+
"REPARTITION(8, id)",
1213+
"",
1214+
true,
1215+
"SHUFFLE_SCAN",
1216+
Seq(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "America/Los_Angeles")),
11861217
("no_aqe", "REPARTITION(8, id)", "", false, "SCAN", Seq.empty[(String, String)]))
11871218
cases.foreach { case (name, hint, filter, adaptive, directReadLeaf, extraConfs) =>
11881219
val expected =

0 commit comments

Comments
 (0)