Skip to content

Commit d735166

Browse files
authored
test: cover container field ids next to INT96 timestamps in the native Parquet scan (apache#6862)
DataFusion 55.2.0 keeps field metadata when coercing INT96 timestamps, so a field id on a struct, list, map or Variant column now survives into the native scan's id matching. Add tests for root and nested containers that read the column under a new name, so only the id can match it.
1 parent 9e13649 commit d735166

1 file changed

Lines changed: 110 additions & 0 deletions

File tree

‎spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@ import org.apache.spark.sql.{CometTestBase, DataFrame, Row}
4242
import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, OptimizeIn}
4343
import org.apache.spark.sql.catalyst.util.DateTimeUtils
4444
import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec}
45+
import org.apache.spark.sql.comet.util.Utils
4546
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
4647
import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils
48+
import org.apache.spark.sql.functions.col
4749
import org.apache.spark.sql.internal.SQLConf
4850
import org.apache.spark.sql.types._
4951

@@ -2598,6 +2600,114 @@ abstract class ParquetReadSuite extends CometTestBase {
25982600
}
25992601
}
26002602

2603+
// Spark writes timestamps as INT96 by default, and the native scan's INT96 coercion rebuilds
2604+
// struct, list and map fields, which must keep their metadata or an id on the container is
2605+
// lost and the column reads as nulls. The read renames the column so only its id matches it.
2606+
private val int96Ts = Timestamp.valueOf("2020-01-01 00:00:00")
2607+
private val int96Inner = new StructType().add("a", IntegerType).add("ts", TimestampType)
2608+
2609+
Seq[(String, DataType, Any)](
2610+
("struct", int96Inner, Row(1, int96Ts)),
2611+
("list", ArrayType(int96Inner), Seq(Row(1, int96Ts), Row(2, int96Ts))),
2612+
("map", MapType(StringType, int96Inner), Map("k" -> Row(1, int96Ts)))).foreach {
2613+
case (label, dataType, value) =>
2614+
test(s"an id on a $label column holding an INT96 timestamp matches it natively") {
2615+
withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true") {
2616+
withTempPath { dir =>
2617+
val rows = Seq(Row(value), Row(null))
2618+
withSQLConf(
2619+
CometConf.COMET_ENABLED.key -> "false",
2620+
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") {
2621+
spark
2622+
.createDataFrame(
2623+
spark.sparkContext.parallelize(rows),
2624+
new StructType().add("c", dataType, true, withId(1)))
2625+
.write
2626+
.parquet(dir.getCanonicalPath)
2627+
}
2628+
val readSchema = new StructType().add("renamed", dataType, true, withId(1))
2629+
val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath)
2630+
checkSparkAnswerAndOperator(df)
2631+
checkAnswer(df, rows)
2632+
}
2633+
}
2634+
}
2635+
}
2636+
2637+
// Ids on a struct below a struct or a list, and on its INT96 leaf, match renamed fields.
2638+
// Spark's vectorized reader before 4.0 rejects the renamed struct below the list, so the
2639+
// comparison with Spark runs from 4.0 on and the pinned rows hold everywhere.
2640+
test("an id on a struct nested in a struct or list with an INT96 leaf matches it natively") {
2641+
def schema(innerName: String, tsName: String): StructType = {
2642+
def inner(innerId: Int, tsId: Int): StructType = new StructType().add(
2643+
innerName,
2644+
new StructType().add("a", IntegerType).add(tsName, TimestampType, true, withId(tsId)),
2645+
true,
2646+
withId(innerId))
2647+
new StructType().add("s", inner(2, 3)).add("l", ArrayType(inner(4, 5)))
2648+
}
2649+
val rows = Seq(
2650+
Row(Row(Row(1, int96Ts)), Seq(Row(Row(2, int96Ts)), Row(Row(3, int96Ts)))),
2651+
Row(Row(null), Seq(Row(Row(4, null)), null)))
2652+
2653+
withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true") {
2654+
withTempPath { dir =>
2655+
withSQLConf(
2656+
CometConf.COMET_ENABLED.key -> "false",
2657+
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") {
2658+
spark
2659+
.createDataFrame(spark.sparkContext.parallelize(rows), schema("inner", "ts"))
2660+
.write
2661+
.parquet(dir.getCanonicalPath)
2662+
}
2663+
val df = spark.read
2664+
.schema(schema("renamed_inner", "renamed_ts"))
2665+
.parquet(dir.getCanonicalPath)
2666+
if (isSpark40Plus) {
2667+
checkSparkAnswerAndOperator(df)
2668+
}
2669+
val plan = stripAQEPlan(df.queryExecution.executedPlan)
2670+
assert(
2671+
collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty,
2672+
s"expected CometNativeScanExec in the plan:\n$plan")
2673+
checkAnswer(df, rows)
2674+
}
2675+
}
2676+
}
2677+
2678+
// A Variant column is a struct of value and metadata in the Arrow schema, so the same coercion
2679+
// rebuilds it when the file holds an INT96 column anywhere, and its id has to survive.
2680+
test("an id on a Variant column next to an INT96 timestamp matches it natively") {
2681+
assume(Utils.variantType.isDefined, "VariantType requires Spark 4.0+")
2682+
withSQLConf(
2683+
SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true",
2684+
"spark.sql.variant.allowReadingShredded" -> "true",
2685+
"spark.sql.variant.pushVariantIntoScan" -> "false") {
2686+
withTempPath { dir =>
2687+
withSQLConf(
2688+
CometConf.COMET_ENABLED.key -> "false",
2689+
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") {
2690+
spark
2691+
.sql("""SELECT parse_json(j) AS v, t AS ts FROM VALUES
2692+
| ('{"a":1}', TIMESTAMP'2020-01-01 00:00:00'), (NULL, NULL) AS input(j, t)
2693+
|""".stripMargin)
2694+
.select(col("v").as("v", withId(1)), col("ts"))
2695+
.coalesce(1)
2696+
.write
2697+
.parquet(dir.getCanonicalPath)
2698+
}
2699+
val readSchema = new StructType()
2700+
.add("renamed", Utils.variantType.get, true, withId(1))
2701+
.add("ts", TimestampType)
2702+
val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath)
2703+
checkSparkAnswerAndOperator(df)
2704+
checkAnswer(
2705+
df.selectExpr("to_json(renamed)", "ts"),
2706+
Row("""{"a":1}""", int96Ts) :: Row(null, null) :: Nil)
2707+
}
2708+
}
2709+
}
2710+
26012711
// Spark checks each file on its own. A directory holding one file with ids and one without
26022712
// raises on the second, and with `ignoreMissing` the file without ids reads as nulls because
26032713
// no root field of it carries the requested id. Each side is written as one file. Spread over

0 commit comments

Comments
 (0)