@@ -1143,28 +1143,50 @@ class CometIcebergWriteActionSuite
11431143 withIcebergCatalog { _ =>
11441144 val columns = " id BIGINT, s STRING, d DOUBLE, st STRUCT<a: INT, b: STRING>, " +
11451145 " arr ARRAY<INT>, m MAP<STRING, INT>, dt DATE, ts TIMESTAMP, ntz TIMESTAMP_NTZ, " +
1146- " dec DECIMAL(12, 3), bin BINARY, b BOOLEAN, f FLOAT"
1146+ " dec DECIMAL(12, 3), bin BINARY, b BOOLEAN, f FLOAT, " +
1147+ " nested ARRAY<STRUCT<x: INT, y: STRING>>, " +
1148+ " deep STRUCT<k: MAP<STRING, ARRAY<INT>>, v: INT>"
11471149 spark.sql(s " CREATE TABLE $catalog. $ns.shuffle_src ( $columns) USING iceberg " )
11481150 spark.sql(s """
11491151 INSERT INTO $catalog. $ns.shuffle_src
11501152 SELECT id,
1151- IF(id % 7 = 0, NULL, CONCAT('s', CAST(id AS STRING))),
1153+ CASE WHEN id % 7 = 0 THEN NULL
1154+ WHEN id % 5 = 0 THEN CONCAT('日本語-', CAST(id AS STRING))
1155+ WHEN id % 3 = 0 THEN CONCAT('é😀-', CAST(id AS STRING))
1156+ ELSE CONCAT('s', CAST(id AS STRING)) END,
11521157 CASE id % 11 WHEN 0 THEN NULL WHEN 1 THEN CAST('NaN' AS DOUBLE) WHEN 2 THEN -0.0D
11531158 WHEN 3 THEN CAST('Infinity' AS DOUBLE) WHEN 4 THEN CAST('-Infinity' AS DOUBLE)
11541159 ELSE id / 3.0 END,
11551160 IF(id % 13 = 0, NULL,
11561161 NAMED_STRUCT('a', CAST(id AS INT), 'b', IF(id % 2 = 0, NULL, CAST(id AS STRING)))),
11571162 IF(id % 17 = 0, NULL, ARRAY(CAST(id AS INT), NULL)),
11581163 IF(id % 19 = 0, NULL, MAP(CAST(id AS STRING), CAST(id AS INT))),
1159- IF(id % 23 = 0, NULL, DATE_ADD(DATE '1970-01-01', CAST(id * 7 AS INT))),
1160- IF(id % 29 = 0, NULL, TIMESTAMP_MICROS(id * 1000003)),
1161- IF(id % 31 = 0, NULL, CAST(TIMESTAMP_MICROS(id * 999983) AS TIMESTAMP_NTZ)),
1164+ CASE WHEN id % 23 = 0 THEN NULL
1165+ WHEN id % 23 = 1 THEN DATE '1900-01-01'
1166+ WHEN id % 23 = 2 THEN DATE '1969-12-31'
1167+ WHEN id % 23 = 3 THEN DATE '9999-12-31'
1168+ ELSE DATE_ADD(DATE '1970-01-01', CAST(id * 7 AS INT)) END,
1169+ CASE WHEN id % 29 = 0 THEN NULL
1170+ WHEN id % 29 = 1 THEN TIMESTAMP_MICROS(-2208988800000000)
1171+ WHEN id % 29 = 2 THEN TIMESTAMP_MICROS(-1)
1172+ WHEN id % 29 = 3 THEN TIMESTAMP_MICROS(253402300799999999)
1173+ ELSE TIMESTAMP_MICROS(id * 1000003) END,
1174+ CASE WHEN id % 31 = 0 THEN NULL
1175+ WHEN id % 31 = 1 THEN CAST(TIMESTAMP_MICROS(-2208988800000000) AS TIMESTAMP_NTZ)
1176+ WHEN id % 31 = 2 THEN CAST(TIMESTAMP_MICROS(253402300799999999) AS TIMESTAMP_NTZ)
1177+ ELSE CAST(TIMESTAMP_MICROS(id * 999983) AS TIMESTAMP_NTZ) END,
11621178 IF(id % 37 = 0, NULL, CAST(CAST(id AS DECIMAL(12, 3)) / 7 AS DECIMAL(12, 3))),
11631179 IF(id % 41 = 0, NULL, CAST(CONCAT('b', CAST(id AS STRING)) AS BINARY)),
11641180 IF(id % 43 = 0, NULL, id % 2 = 0),
11651181 CASE id % 10 WHEN 0 THEN CAST('NaN' AS FLOAT) WHEN 1 THEN CAST(-0.0D AS FLOAT)
11661182 WHEN 2 THEN CAST('Infinity' AS FLOAT) WHEN 3 THEN CAST('-Infinity' AS FLOAT)
1167- WHEN 4 THEN NULL ELSE CAST(id AS FLOAT) / 3 END
1183+ WHEN 4 THEN NULL ELSE CAST(id AS FLOAT) / 3 END,
1184+ IF(id % 47 = 0, NULL,
1185+ ARRAY(NAMED_STRUCT('x', CAST(id AS INT), 'y', CONCAT('日', CAST(id AS STRING))),
1186+ NULL, NAMED_STRUCT('x', CAST(NULL AS INT), 'y', CAST(NULL AS STRING)))),
1187+ IF(id % 53 = 0, NULL,
1188+ NAMED_STRUCT('k', IF(id % 2 = 0, MAP('a', ARRAY(CAST(id AS INT), NULL), 'b', ARRAY()),
1189+ CAST(NULL AS MAP<STRING, ARRAY<INT>>)), 'v', CAST(id AS INT)))
11681190 FROM range(0, 2000) """ )
11691191
11701192 // (case, hint, filter, AQE on, the write's leaf when direct read is on, extra confs).
@@ -1183,6 +1205,15 @@ class CometIcebergWriteActionSuite
11831205 true ,
11841206 " SHUFFLE_SCAN" ,
11851207 Seq (SQLConf .COALESCE_PARTITIONS_ENABLED .key -> " false" )),
1208+ // The timestamp columns hold instants, so a session time zone must not change what the
1209+ // shuffle blocks decode to.
1210+ (
1211+ " non_utc_session" ,
1212+ " REPARTITION(8, id)" ,
1213+ " " ,
1214+ true ,
1215+ " SHUFFLE_SCAN" ,
1216+ Seq (SQLConf .SESSION_LOCAL_TIMEZONE .key -> " America/Los_Angeles" )),
11861217 (" no_aqe" , " REPARTITION(8, id)" , " " , false , " SCAN" , Seq .empty[(String , String )]))
11871218 cases.foreach { case (name, hint, filter, adaptive, directReadLeaf, extraConfs) =>
11881219 val expected =
0 commit comments