Skip to content

Commit 4535d1e

Browse files
committed
test: write complex and temporal types through the Hadoop route
1 parent cc74578 commit 4535d1e

1 file changed

Lines changed: 35 additions & 0 deletions

File tree

‎spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2270,6 +2270,41 @@ class CometParquetWriterSuite extends CometParquetWriterTestBase {
22702270
}
22712271
}
22722272

2273+
test("complex and temporal types are written through Hadoop") {
2274+
// The Hadoop route only forwards the encoded Parquet bytes, so a nested or temporal file must
2275+
// survive it unchanged. These are the shapes `nestedAndTemporalSource` does not have: an
2276+
// array of structs, a map with struct values, a decimal at the maximum precision, a date long
2277+
// before the epoch (still after the 1582 calendar cutover, which Spark 3.x refuses to write
2278+
// without a rebase mode), and a CJK string.
2279+
withRecordingFileSystem {
2280+
spark
2281+
.range(0, 5000)
2282+
.selectExpr(
2283+
"id",
2284+
// a struct holding an array of structs, a map, and a nullable field
2285+
"named_struct('arr', array(named_struct('k', id, 'v', CAST(id AS STRING))), " +
2286+
"'m', map('x', id), 'maybe', CASE WHEN id % 3 = 0 THEN NULL ELSE id END) AS nested",
2287+
"array(named_struct('a', id, 'b', id * 2)) AS arr_of_structs",
2288+
"map(CAST(id AS STRING), named_struct('p', id, 'q', id)) AS map_struct",
2289+
"CAST(id AS DECIMAL(38, 10)) AS dec",
2290+
"date_add(DATE'1600-01-01', CAST(id AS INT)) AS d",
2291+
"timestamp_micros(id * 1000000) AS ts",
2292+
"CAST(timestamp_micros(id * 1000000) AS TIMESTAMP_NTZ) AS ts_ntz",
2293+
"concat('名前-', CAST(id AS STRING), '-café-naïve') AS s")
2294+
} { (df, outputPath) =>
2295+
val plan =
2296+
captureWritePlan(p => df.write.option("compression", "none").parquet(p), outputPath)
2297+
assertHasCometNativeWriteExec(plan)
2298+
// The source has several partitions, so the files close in any order.
2299+
assert(RecordingFileSystem.created.size > 0)
2300+
assert(
2301+
RecordingFileSystem.closed.asScala.toSet == RecordingFileSystem.created.asScala.toSet)
2302+
2303+
checkAnswer(spark.read.parquet(outputPath), df)
2304+
assertMatchesSparkWriter(df, outputPath)
2305+
}
2306+
}
2307+
22732308
test("a write of nested and temporal columns that fails partway is aborted") {
22742309
withRecordingFileSystem(nestedAndTemporalSource(8000)) { (df, outputPath) =>
22752310
RecordingFileSystem.failAtWrite = 3

0 commit comments

Comments
 (0)