@@ -2270,6 +2270,41 @@ class CometParquetWriterSuite extends CometParquetWriterTestBase {
22702270 }
22712271 }
22722272
2273+ test(" complex and temporal types are written through Hadoop" ) {
2274+ // The Hadoop route only forwards the encoded Parquet bytes, so a nested or temporal file must
2275+ // survive it unchanged. These are the shapes `nestedAndTemporalSource` does not have: an
2276+ // array of structs, a map with struct values, a decimal at the maximum precision, a date long
2277+ // before the epoch (still after the 1582 calendar cutover, which Spark 3.x refuses to write
2278+ // without a rebase mode), and a CJK string.
2279+ withRecordingFileSystem {
2280+ spark
2281+ .range(0 , 5000 )
2282+ .selectExpr(
2283+ " id" ,
2284+ // a struct holding an array of structs, a map, and a nullable field
2285+ " named_struct('arr', array(named_struct('k', id, 'v', CAST(id AS STRING))), " +
2286+ " 'm', map('x', id), 'maybe', CASE WHEN id % 3 = 0 THEN NULL ELSE id END) AS nested" ,
2287+ " array(named_struct('a', id, 'b', id * 2)) AS arr_of_structs" ,
2288+ " map(CAST(id AS STRING), named_struct('p', id, 'q', id)) AS map_struct" ,
2289+ " CAST(id AS DECIMAL(38, 10)) AS dec" ,
2290+ " date_add(DATE'1600-01-01', CAST(id AS INT)) AS d" ,
2291+ " timestamp_micros(id * 1000000) AS ts" ,
2292+ " CAST(timestamp_micros(id * 1000000) AS TIMESTAMP_NTZ) AS ts_ntz" ,
2293+ " concat('名前-', CAST(id AS STRING), '-café-naïve') AS s" )
2294+ } { (df, outputPath) =>
2295+ val plan =
2296+ captureWritePlan(p => df.write.option(" compression" , " none" ).parquet(p), outputPath)
2297+ assertHasCometNativeWriteExec(plan)
2298+ // The source has several partitions, so the files close in any order.
2299+ assert(RecordingFileSystem .created.size > 0 )
2300+ assert(
2301+ RecordingFileSystem .closed.asScala.toSet == RecordingFileSystem .created.asScala.toSet)
2302+
2303+ checkAnswer(spark.read.parquet(outputPath), df)
2304+ assertMatchesSparkWriter(df, outputPath)
2305+ }
2306+ }
2307+
22732308 test(" a write of nested and temporal columns that fails partway is aborted" ) {
22742309 withRecordingFileSystem(nestedAndTemporalSource(8000 )) { (df, outputPath) =>
22752310 RecordingFileSystem .failAtWrite = 3
0 commit comments