@@ -42,8 +42,10 @@ import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
4242import org .apache .spark .sql .catalyst .optimizer .{ConvertToLocalRelation , OptimizeIn }
4343import org .apache .spark .sql .catalyst .util .DateTimeUtils
4444import org .apache .spark .sql .comet .{CometNativeScanExec , CometScanExec }
45+ import org .apache .spark .sql .comet .util .Utils
4546import org .apache .spark .sql .execution .adaptive .AdaptiveSparkPlanHelper
4647import org .apache .spark .sql .execution .datasources .parquet .ParquetUtils
48+ import org .apache .spark .sql .functions .col
4749import org .apache .spark .sql .internal .SQLConf
4850import org .apache .spark .sql .types ._
4951
@@ -2598,6 +2600,114 @@ abstract class ParquetReadSuite extends CometTestBase {
25982600 }
25992601 }
26002602
2603+ // Spark writes timestamps as INT96 by default, and the native scan's INT96 coercion rebuilds
2604+ // struct, list and map fields, which must keep their metadata or an id on the container is
2605+ // lost and the column reads as nulls. The read renames the column so only its id matches it.
2606+ private val int96Ts = Timestamp .valueOf(" 2020-01-01 00:00:00" )
2607+ private val int96Inner = new StructType ().add(" a" , IntegerType ).add(" ts" , TimestampType )
2608+
2609+ Seq [(String , DataType , Any )](
2610+ (" struct" , int96Inner, Row (1 , int96Ts)),
2611+ (" list" , ArrayType (int96Inner), Seq (Row (1 , int96Ts), Row (2 , int96Ts))),
2612+ (" map" , MapType (StringType , int96Inner), Map (" k" -> Row (1 , int96Ts)))).foreach {
2613+ case (label, dataType, value) =>
2614+ test(s " an id on a $label column holding an INT96 timestamp matches it natively " ) {
2615+ withSQLConf(SQLConf .PARQUET_FIELD_ID_READ_ENABLED .key -> " true" ) {
2616+ withTempPath { dir =>
2617+ val rows = Seq (Row (value), Row (null ))
2618+ withSQLConf(
2619+ CometConf .COMET_ENABLED .key -> " false" ,
2620+ SQLConf .PARQUET_OUTPUT_TIMESTAMP_TYPE .key -> " INT96" ) {
2621+ spark
2622+ .createDataFrame(
2623+ spark.sparkContext.parallelize(rows),
2624+ new StructType ().add(" c" , dataType, true , withId(1 )))
2625+ .write
2626+ .parquet(dir.getCanonicalPath)
2627+ }
2628+ val readSchema = new StructType ().add(" renamed" , dataType, true , withId(1 ))
2629+ val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath)
2630+ checkSparkAnswerAndOperator(df)
2631+ checkAnswer(df, rows)
2632+ }
2633+ }
2634+ }
2635+ }
2636+
2637+ // Ids on a struct below a struct or a list, and on its INT96 leaf, match renamed fields.
2638+ // Spark's vectorized reader before 4.0 rejects the renamed struct below the list, so the
2639+ // comparison with Spark runs from 4.0 on and the pinned rows hold everywhere.
2640+ test(" an id on a struct nested in a struct or list with an INT96 leaf matches it natively" ) {
2641+ def schema (innerName : String , tsName : String ): StructType = {
2642+ def inner (innerId : Int , tsId : Int ): StructType = new StructType ().add(
2643+ innerName,
2644+ new StructType ().add(" a" , IntegerType ).add(tsName, TimestampType , true , withId(tsId)),
2645+ true ,
2646+ withId(innerId))
2647+ new StructType ().add(" s" , inner(2 , 3 )).add(" l" , ArrayType (inner(4 , 5 )))
2648+ }
2649+ val rows = Seq (
2650+ Row (Row (Row (1 , int96Ts)), Seq (Row (Row (2 , int96Ts)), Row (Row (3 , int96Ts)))),
2651+ Row (Row (null ), Seq (Row (Row (4 , null )), null )))
2652+
2653+ withSQLConf(SQLConf .PARQUET_FIELD_ID_READ_ENABLED .key -> " true" ) {
2654+ withTempPath { dir =>
2655+ withSQLConf(
2656+ CometConf .COMET_ENABLED .key -> " false" ,
2657+ SQLConf .PARQUET_OUTPUT_TIMESTAMP_TYPE .key -> " INT96" ) {
2658+ spark
2659+ .createDataFrame(spark.sparkContext.parallelize(rows), schema(" inner" , " ts" ))
2660+ .write
2661+ .parquet(dir.getCanonicalPath)
2662+ }
2663+ val df = spark.read
2664+ .schema(schema(" renamed_inner" , " renamed_ts" ))
2665+ .parquet(dir.getCanonicalPath)
2666+ if (isSpark40Plus) {
2667+ checkSparkAnswerAndOperator(df)
2668+ }
2669+ val plan = stripAQEPlan(df.queryExecution.executedPlan)
2670+ assert(
2671+ collect(plan) { case scan : CometNativeScanExec => scan }.nonEmpty,
2672+ s " expected CometNativeScanExec in the plan: \n $plan" )
2673+ checkAnswer(df, rows)
2674+ }
2675+ }
2676+ }
2677+
2678+ // A Variant column is a struct of value and metadata in the Arrow schema, so the same coercion
2679+ // rebuilds it when the file holds an INT96 column anywhere, and its id has to survive.
2680+ test(" an id on a Variant column next to an INT96 timestamp matches it natively" ) {
2681+ assume(Utils .variantType.isDefined, " VariantType requires Spark 4.0+" )
2682+ withSQLConf(
2683+ SQLConf .PARQUET_FIELD_ID_READ_ENABLED .key -> " true" ,
2684+ " spark.sql.variant.allowReadingShredded" -> " true" ,
2685+ " spark.sql.variant.pushVariantIntoScan" -> " false" ) {
2686+ withTempPath { dir =>
2687+ withSQLConf(
2688+ CometConf .COMET_ENABLED .key -> " false" ,
2689+ SQLConf .PARQUET_OUTPUT_TIMESTAMP_TYPE .key -> " INT96" ) {
2690+ spark
2691+ .sql(""" SELECT parse_json(j) AS v, t AS ts FROM VALUES
2692+ | ('{"a":1}', TIMESTAMP'2020-01-01 00:00:00'), (NULL, NULL) AS input(j, t)
2693+ |""" .stripMargin)
2694+ .select(col(" v" ).as(" v" , withId(1 )), col(" ts" ))
2695+ .coalesce(1 )
2696+ .write
2697+ .parquet(dir.getCanonicalPath)
2698+ }
2699+ val readSchema = new StructType ()
2700+ .add(" renamed" , Utils .variantType.get, true , withId(1 ))
2701+ .add(" ts" , TimestampType )
2702+ val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath)
2703+ checkSparkAnswerAndOperator(df)
2704+ checkAnswer(
2705+ df.selectExpr(" to_json(renamed)" , " ts" ),
2706+ Row (""" {"a":1}""" , int96Ts) :: Row (null , null ) :: Nil )
2707+ }
2708+ }
2709+ }
2710+
26012711 // Spark checks each file on its own. A directory holding one file with ids and one without
26022712 // raises on the second, and with `ignoreMissing` the file without ids reads as nulls because
26032713 // no root field of it carries the requested id. Each side is written as one file. Spread over
0 commit comments