Skip to content

Commit 7fe9328

Browse files
committed
fix: fall back when a schema repeats a Parquet field id
Under `spark.sql.parquet.fieldId.read.enabled` Spark resolves each requested field to the one Parquet field carrying its id, and raises FOUND_DUPLICATE_FIELD_IN_FIELD_ID_LOOKUP_MODE when more than one answers. Comet never looked at field ids, so a requested schema that repeats one was read positionally and returned rows where Spark raises. The ids ride in the requested schema's `StructField.metadata`, so this is decidable from the plan. `DataTypeSupport` gains a `hasDuplicateFieldIds` predicate beside the existing duplicate-name one, and `CometScanTypeChecker` declines a field list that trips it. Both entry points are overridden, because they see different things. `isTypeSupported` is handed each field's data type, so it sees nested structs as the trait's recursion reaches them but can never compare two top-level fields; `isSchemaSupported` is where the schema's own field list is available. Each list is inspected exactly once, so nothing is re-walked at an enclosing level. `ArrowCachedBatchSerializer.supportsType` also accepted a struct with duplicate child names, so caching such a relation stored it in Comet's Arrow format, which Java Arrow cannot import back because it keys struct children by name. One more schema check delegates it to Spark's default cache format, alongside the interval types already excluded there, and the two copies of that predicate in the shuffle gate now call the shared one rather than spelling it out inline. No Parquet decoding changes; see #5786 for that path. Closes #5801.
1 parent 58ab5f6 commit 7fe9328

7 files changed

Lines changed: 279 additions & 46 deletions

File tree

‎spark/src/main/scala/org/apache/comet/DataTypeSupport.scala‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@ package org.apache.comet
2121

2222
import scala.collection.mutable.ListBuffer
2323

24+
import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils
2425
import org.apache.spark.sql.types._
2526

26-
import org.apache.comet.DataTypeSupport.{ARRAY_ELEMENT, MAP_KEY, MAP_VALUE}
27+
import org.apache.comet.DataTypeSupport.{hasDuplicateFieldNames, ARRAY_ELEMENT, MAP_KEY, MAP_VALUE}
2728

2829
trait DataTypeSupport {
2930

@@ -53,7 +54,7 @@ trait DataTypeSupport {
5354
BinaryType | StringType | _: DecimalType | DateType | TimestampType | TimestampNTZType |
5455
CalendarIntervalType =>
5556
true
56-
case StructType(fields) if fields.map(_.name).distinct.length != fields.length =>
57+
case StructType(fields) if hasDuplicateFieldNames(fields) =>
5758
// Java Arrow keys struct children by name, so a struct with duplicate field names
5859
// cannot cross the JVM Arrow boundary intact
5960
fallbackReasons += s"Unsupported ${name}: struct with duplicate field names"
@@ -85,6 +86,37 @@ object DataTypeSupport {
8586
case _ => false
8687
}
8788

89+
/** True when two of `fields` carry byte-identical names. */
90+
def hasDuplicateFieldNames(fields: Array[StructField]): Boolean =
91+
fields.map(_.name).distinct.length != fields.length
92+
93+
/**
94+
* True when two of `fields` declare the same Parquet field id.
95+
*
96+
* Deliberately not Spark's check: `ParquetReadSupport.matchIdField` raises when one *requested*
97+
* id is carried by several fields *in the file*. The two coincide only when the requested
98+
* schema equals the file schema, which is exactly when DataFusion's opener skips the expression
99+
* adapter that would have validated the lookup (#5801). So this can decline a read Spark would
100+
* have accepted, costing native execution but not correctness; it cannot report a duplicate
101+
* Spark would not. File-side ambiguity is left to #5786.
102+
*
103+
* Only meaningful under `spark.sql.parquet.fieldId.read.enabled`; callers gate on that.
104+
*/
105+
def hasDuplicateFieldIds(fields: Array[StructField]): Boolean = {
106+
val ids = fields.flatMap(fieldId)
107+
ids.distinct.length != ids.length
108+
}
109+
110+
private def fieldId(field: StructField): Option[Int] = {
111+
if (!ParquetUtils.hasFieldId(field)) None
112+
else {
113+
// A malformed id is not this check's business to report -- getFieldId raises on one -- so
114+
// treat it as absent and let the reader complain about it.
115+
try Some(ParquetUtils.getFieldId(field))
116+
catch { case _: IllegalArgumentException => None }
117+
}
118+
}
119+
88120
/**
89121
* `dt` with every array/map/struct nullability flag forced to `true` at all nesting levels (map
90122
* key fields stay non-null per Arrow's map invariant). Re-derives Spark's `private[spark]`

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ import org.apache.comet.CometSparkSessionExtensions.{isCometLoaded, isSpark35Plu
4949
import org.apache.comet.DataTypeSupport.isComplexType
5050
import org.apache.comet.iceberg.{CometIcebergNativeScanMetadata, IcebergReflection}
5151
import org.apache.comet.objectstore.NativeConfig
52-
import org.apache.comet.parquet.CometParquetUtils.{encryptionEnabled, isEncryptionConfigSupported}
52+
import org.apache.comet.parquet.CometParquetUtils.{encryptionEnabled, isEncryptionConfigSupported, readFieldId}
5353
import org.apache.comet.serde.operator.{CometIcebergNativeScan, CometNativeScan}
5454
import org.apache.comet.shims.{CometTypeShim, ShimCometStreaming, ShimFileFormat, ShimSubqueryBroadcast}
5555

@@ -1094,6 +1094,22 @@ case class CometScanRule(session: SparkSession)
10941094

10951095
case class CometScanTypeChecker() extends DataTypeSupport with CometTypeShim {
10961096

1097+
/**
1098+
* `isTypeSupported` only ever sees a field's *data type*, so nothing there can compare two
1099+
* top-level fields. Spark's field id ambiguity applies at the schema root too, so check it
1100+
* here.
1101+
*/
1102+
override def isSchemaSupported(
1103+
schema: StructType,
1104+
fallbackReasons: ListBuffer[String]): Boolean = {
1105+
if (duplicateFieldIds(schema.fields)) {
1106+
fallbackReasons += "duplicate Parquet field ids among top-level fields"
1107+
false
1108+
} else {
1109+
super.isSchemaSupported(schema, fallbackReasons)
1110+
}
1111+
}
1112+
10971113
override def isTypeSupported(
10981114
dt: DataType,
10991115
name: String,
@@ -1118,10 +1134,20 @@ case class CometScanTypeChecker() extends DataTypeSupport with CometTypeShim {
11181134
false
11191135
case s: StructType if s.fields.isEmpty =>
11201136
false
1137+
case StructType(fields) if duplicateFieldIds(fields) =>
1138+
// Under field id matching Spark resolves each requested field to the one Parquet field
1139+
// carrying its id and raises when more than one answers. Comet reads such a struct
1140+
// positionally instead, so hand the read back to Spark and let it report the ambiguity.
1141+
fallbackReasons += s"Unsupported ${name}: struct with duplicate Parquet field ids"
1142+
false
11211143
case _ =>
11221144
super.isTypeSupported(dt, name, fallbackReasons)
11231145
}
11241146
}
1147+
1148+
/** True when the session resolves Parquet fields by id and `fields` repeat one. */
1149+
private def duplicateFieldIds(fields: Array[StructField]): Boolean =
1150+
readFieldId(SQLConf.get) && DataTypeSupport.hasDuplicateFieldIds(fields)
11251151
}
11261152

11271153
object CometScanRule extends Logging {

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowCachedBatchSerializer.scala‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import org.apache.spark.storage.StorageLevel
3939
import org.apache.spark.unsafe.types.UTF8String
4040
import org.apache.spark.util.io.ChunkedByteBuffer
4141

42-
import org.apache.comet.CometArrowAllocator
42+
import org.apache.comet.{CometArrowAllocator, DataTypeSupport}
4343

4444
/**
4545
* Cached batch format used when Comet writes Spark in-memory cache data.
@@ -663,6 +663,11 @@ object ArrowCachedBatchSerializer {
663663
* This mirrors the vectors `Utils.getFieldVector` accepts. A type missing from that list throws
664664
* during cache materialization, so it has to be delegated to Spark's default cache format
665665
* instead. Interval types are the notable omission.
666+
*
667+
* A struct with duplicate child names is rejected for a different reason: Java Arrow keys a
668+
* struct vector's children by name, so the batch cannot be imported back across the C data
669+
* interface (#5605) and the native scan over the cache could never read it. Storing it in
670+
* Comet's format would only be work thrown away.
666671
*/
667672
def supportsType(dt: DataType): Boolean = dt match {
668673
case BooleanType | ByteType | ShortType | IntegerType | LongType | FloatType | DoubleType |
@@ -672,6 +677,7 @@ object ArrowCachedBatchSerializer {
672677
case _: StringType => true
673678
case ArrayType(elementType, _) => supportsType(elementType)
674679
case MapType(keyType, valueType, _) => supportsType(keyType) && supportsType(valueType)
680+
case StructType(fields) if DataTypeSupport.hasDuplicateFieldNames(fields) => false
675681
case StructType(fields) => fields.forall(f => supportsType(f.dataType))
676682
case _ => false
677683
}

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ import org.apache.spark.util.random.XORShiftRandom
4949

5050
import com.google.common.base.Objects
5151

52-
import org.apache.comet.{CometConf, CometExplainInfo}
52+
import org.apache.comet.{CometConf, CometExplainInfo, DataTypeSupport}
5353
import org.apache.comet.CometConf.{COMET_SHUFFLE_ENABLED, COMET_SHUFFLE_MODE}
5454
import org.apache.comet.CometSparkSessionExtensions.{cometCelebornShuffleFallbackReason, hasFallbackReason, isCometCelebornShuffleManagerEnabled, isCometShuffleManagerEnabled, isSpark40Plus, withFallbackReasons}
5555
import org.apache.comet.serde.{Compatible, OperatorOuterClass, QueryPlanSerde, SupportLevel, Unsupported}
@@ -480,7 +480,7 @@ object CometShuffleExchangeExec
480480
fields.nonEmpty && fields.forall(f => supportedSerializableDataType(f.dataType)) &&
481481
// Java Arrow keys struct children by name, so the FFI import of a decoded batch
482482
// fails on duplicate field names
483-
fields.map(f => f.name).distinct.length == fields.length
483+
!DataTypeSupport.hasDuplicateFieldNames(fields)
484484
case ArrayType(elementType, _) =>
485485
supportedSerializableDataType(elementType)
486486
case MapType(keyType, valueType, _) =>
@@ -607,7 +607,7 @@ object CometShuffleExchangeExec
607607
case StructType(fields) =>
608608
fields.nonEmpty && fields.forall(f => supportedSerializableDataType(f.dataType)) &&
609609
// Java Arrow stream reader cannot work on duplicate field name
610-
fields.map(f => f.name).distinct.length == fields.length
610+
!DataTypeSupport.hasDuplicateFieldNames(fields)
611611
case ArrayType(elementType, _) =>
612612
supportedSerializableDataType(elementType)
613613
case MapType(keyType, valueType, _) =>

‎spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala‎

Lines changed: 43 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,19 @@ class CometInMemoryCacheSuite extends CometTestBase {
295295
}
296296
}
297297

298+
// Column expression and the reason the serializer has to decline it.
299+
private val unsupportedForArrowCache = Seq(
300+
// Interval types have no Arrow vector in Utils.getFieldVector. Without the schema check in
301+
// the serializer, caching this relation fails outright with "Unsupported Arrow Vector for
302+
// serialize: class org.apache.arrow.vector.DurationVector".
303+
"make_dt_interval(0, 0, 0, id) AS payload",
304+
// Java Arrow keys a struct vector's children by name, so the two `a` children collapse into
305+
// one and the batch fails its arity check coming back across the C data interface. Without
306+
// the check, this is cached in Comet's format and the read dies with
307+
// "ArrowArray struct has 2 children (expected 1)".
308+
// See https://github.com/apache/datafusion-comet/issues/5605.
309+
"named_struct('a', id, 'a', id + 1) AS payload")
310+
298311
test("Comet cache serializer delegates unsupported types to Spark's cache format") {
299312
withSQLConf(
300313
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
@@ -303,53 +316,45 @@ class CometInMemoryCacheSuite extends CometTestBase {
303316
CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true",
304317
"spark.comet.sparkToColumnar.enabled" -> "true") {
305318

306-
spark.catalog.clearCache()
307-
308-
// Interval types have no Arrow vector in Utils.getFieldVector. Without the schema check in
309-
// the serializer, caching this relation fails outright with "Unsupported Arrow Vector for
310-
// serialize: class org.apache.arrow.vector.DurationVector".
311-
spark
312-
.sql("""
313-
SELECT id AS key, make_dt_interval(0, 0, 0, id) AS dt
314-
FROM range(1000)
315-
""")
316-
.createOrReplaceTempView("default_cached_batch")
319+
for (column <- unsupportedForArrowCache) {
320+
spark.catalog.clearCache()
317321

318-
spark.catalog.cacheTable("default_cached_batch")
319-
spark.table("default_cached_batch").count()
322+
spark
323+
.sql(s"SELECT id AS key, $column FROM range(1000)")
324+
.createOrReplaceTempView("default_cached_batch")
320325

321-
assert(
322-
cachedBatchTypes("default_cached_batch").sameElements(
323-
Array("org.apache.spark.sql.execution.columnar.DefaultCachedBatch")))
324-
325-
// Columnar read path, delegated to Spark's serializer.
326-
val columnarDf = spark.sql("""
327-
SELECT key, dt
328-
FROM default_cached_batch
329-
WHERE key >= 10 AND key < 20
330-
""")
331-
assert(columnarDf.collect().length == 10)
332-
checkSparkAnswer(columnarDf)
326+
spark.catalog.cacheTable("default_cached_batch")
327+
spark.table("default_cached_batch").count()
333328

334-
val columnarPlan = columnarDf.queryExecution.executedPlan.toString()
335-
assert(!columnarPlan.contains("CometInMemoryTableScan"))
329+
assert(
330+
cachedBatchTypes("default_cached_batch").sameElements(
331+
Array("org.apache.spark.sql.execution.columnar.DefaultCachedBatch")),
332+
s"$column was cached in Comet's format")
336333

337-
// Row read path: disabling the vectorized cache reader makes Spark use
338-
// convertCachedBatchToInternalRow.
339-
withSQLConf(SQLConf.CACHE_VECTORIZED_READER_ENABLED.key -> "false") {
340-
val rowDf = spark.sql("""
341-
SELECT dt
334+
// Columnar read path, delegated to Spark's serializer.
335+
val columnarDf = spark.sql("""
336+
SELECT key, payload
342337
FROM default_cached_batch
343338
WHERE key >= 10 AND key < 20
344339
""")
345-
assert(rowDf.collect().length == 10)
346-
checkSparkAnswer(rowDf)
340+
checkSparkAnswer(columnarDf)
341+
assert(
342+
!columnarDf.queryExecution.executedPlan.toString().contains("CometInMemoryTableScan"))
343+
344+
// Row read path: disabling the vectorized cache reader makes Spark use
345+
// convertCachedBatchToInternalRow.
346+
withSQLConf(SQLConf.CACHE_VECTORIZED_READER_ENABLED.key -> "false") {
347+
val rowDf = spark.sql("""
348+
SELECT payload
349+
FROM default_cached_batch
350+
WHERE key >= 10 AND key < 20
351+
""")
352+
checkSparkAnswer(rowDf)
353+
assert(!rowDf.queryExecution.executedPlan.toString().contains("CometInMemoryTableScan"))
354+
}
347355

348-
val rowPlan = rowDf.queryExecution.executedPlan.toString()
349-
assert(!rowPlan.contains("CometInMemoryTableScan"))
356+
spark.catalog.clearCache()
350357
}
351-
352-
spark.catalog.clearCache()
353358
}
354359
}
355360

‎spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -929,6 +929,112 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper
929929
}
930930
}
931931

932+
private def withFieldId(name: String, id: Int): StructField =
933+
StructField(
934+
name,
935+
LongType,
936+
nullable = true,
937+
new MetadataBuilder().putLong("parquet.field.id", id.toLong).build())
938+
939+
/**
940+
* Write a one-row file and assert Comet hands the read back to Spark under field id matching,
941+
* and keeps it native without.
942+
*
943+
* The file must carry no key-value metadata: arrow-rs folds Spark's metadata into the physical
944+
* schema, so a Spark-written file never compares equal to the requested schema and always
945+
* reaches the expression adapter that would have caught the ambiguity. `writeDirect` is what
946+
* keeps the metadata out. See https://github.com/apache/datafusion-comet/issues/5801.
947+
*/
948+
private def checkDuplicateFieldIdsFallBack(
949+
messageType: String,
950+
writeRecord: RecordConsumer => Unit,
951+
readSchema: StructType): Unit = withTempPath { dir =>
952+
writeDirect(
953+
new Path(dir.getCanonicalPath, "duplicate-field-ids.parquet").toString,
954+
messageType,
955+
writeRecord)
956+
957+
withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true") {
958+
val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath)
959+
val plan = df.queryExecution.executedPlan
960+
assert(
961+
collect(plan) { case scan: CometNativeScanExec => scan }.isEmpty,
962+
s"expected no native scan, got:\n$plan")
963+
val error = intercept[Exception](df.collect())
964+
assert(
965+
causeChain(error).exists(e =>
966+
String.valueOf(e.getMessage).contains("""Found duplicate field(s) "1"""")),
967+
s"unexpected error: $error")
968+
}
969+
970+
withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "false") {
971+
val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath)
972+
assert(
973+
collect(df.queryExecution.executedPlan) { case scan: CometNativeScanExec =>
974+
scan
975+
}.nonEmpty,
976+
"expected a native scan with field id matching off")
977+
checkSparkAnswer(df)
978+
}
979+
}
980+
981+
test("native scan declines a nested struct whose fields repeat a Parquet field id") {
982+
// Spark resolves each requested field to the one Parquet field carrying its id and raises when
983+
// more than one answers. DataFusion's opener skips the expression adapter when the file's
984+
// physical schema compares equal to the logical schema and no predicate is pushed, so the
985+
// native scan read this positionally and returned rows where Spark raises.
986+
checkDuplicateFieldIdsFallBack(
987+
"""message spark_schema {
988+
| optional group s {
989+
| optional int64 x = 1;
990+
| optional int64 y = 1;
991+
| }
992+
|}
993+
""".stripMargin,
994+
{ rc: RecordConsumer =>
995+
rc.startMessage()
996+
rc.startField("s", 0)
997+
rc.startGroup()
998+
rc.startField("x", 0)
999+
rc.addLong(10L)
1000+
rc.endField("x", 0)
1001+
rc.startField("y", 1)
1002+
rc.addLong(20L)
1003+
rc.endField("y", 1)
1004+
rc.endGroup()
1005+
rc.endField("s", 0)
1006+
rc.endMessage()
1007+
},
1008+
StructType(
1009+
Seq(
1010+
StructField(
1011+
"s",
1012+
StructType(Seq(withFieldId("x", 1), withFieldId("y", 1))),
1013+
nullable = true))))
1014+
}
1015+
1016+
test("native scan declines top-level fields that repeat a Parquet field id") {
1017+
// The same ambiguity one level up. It reaches the guard through a different override, because
1018+
// `isTypeSupported` only sees field data types and so never compares two top-level fields.
1019+
checkDuplicateFieldIdsFallBack(
1020+
"""message spark_schema {
1021+
| optional int64 x = 1;
1022+
| optional int64 y = 1;
1023+
|}
1024+
""".stripMargin,
1025+
{ rc: RecordConsumer =>
1026+
rc.startMessage()
1027+
rc.startField("x", 0)
1028+
rc.addLong(10L)
1029+
rc.endField("x", 0)
1030+
rc.startField("y", 1)
1031+
rc.addLong(20L)
1032+
rc.endField("y", 1)
1033+
rc.endMessage()
1034+
},
1035+
StructType(Seq(withFieldId("x", 1), withFieldId("y", 1))))
1036+
}
1037+
9321038
/** Write a Parquet file using a raw RecordConsumer for full schema control. */
9331039
private def writeDirect(
9341040
path: String,

0 commit comments

Comments
 (0)