Skip to content

Commit e40b939

Browse files
peterxcliclaude
andcommitted
fix: exempt ID-resolved columns from name-duplicate validation
Spark resolves an ID-bearing requested field by unique field ID before any name matching (ParquetReadSupport.matchIdField), so a case-sensitive file with v/V siblings reads fine when v is requested by ID. Comet's per-column duplicate check ran before the field-ID exemption and raised _LEGACY_ERROR_TEMP_2093 for that scan. Hoist the exemption above both the duplicate check and the Unicode fold guard, mirroring Spark's dispatch; duplicate-ID validation stays in remap_physical_schema. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 53c58be commit e40b939

2 files changed

Lines changed: 85 additions & 19 deletions

File tree

‎native/core/src/parquet/schema_adapter.rs‎

Lines changed: 48 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -495,24 +495,30 @@ impl PhysicalExprAdapter for SparkPhysicalExprAdapter {
495495
let mut column_err: Option<DataFusionError> = None;
496496
let _ = Arc::<dyn PhysicalExpr>::clone(&expr).transform(|e| {
497497
if let Some(col) = e.downcast_ref::<Column>() {
498-
if let Some((req, matched)) = check_column_duplicate(col.name(), orig_physical)
499-
{
500-
column_err = Some(DataFusionError::External(Box::new(
501-
SparkError::DuplicateFieldCaseInsensitive {
502-
required_field_name: req,
503-
matched_fields: matched,
504-
},
505-
)));
506-
}
507-
if column_err.is_none() && variant_scan {
508-
let id_matched = should_match_by_id
509-
&& self
510-
.logical_file_schema
511-
.field_with_name(col.name())
512-
.ok()
513-
.and_then(parse_field_id)
514-
.is_some();
515-
if !id_matched {
498+
// Spark resolves an ID-bearing requested field by unique field ID before
499+
// any name matching (`ParquetReadSupport.matchIdField`), so file name
500+
// collisions and Unicode folds never affect it; duplicate-ID validation
501+
// runs in `remap_physical_schema`. All other fields fall through to the
502+
// name matcher and get both checks.
503+
let id_matched = should_match_by_id
504+
&& self
505+
.logical_file_schema
506+
.field_with_name(col.name())
507+
.ok()
508+
.and_then(parse_field_id)
509+
.is_some();
510+
if !id_matched {
511+
if let Some((req, matched)) =
512+
check_column_duplicate(col.name(), orig_physical)
513+
{
514+
column_err = Some(DataFusionError::External(Box::new(
515+
SparkError::DuplicateFieldCaseInsensitive {
516+
required_field_name: req,
517+
matched_fields: matched,
518+
},
519+
)));
520+
}
521+
if column_err.is_none() && variant_scan {
516522
if let Some(phys) =
517523
check_column_unicode_fold_mismatch(col.name(), orig_physical)
518524
{
@@ -2205,4 +2211,28 @@ mod test {
22052211
]));
22062212
rewrite_column(logical, physical, true, "k").unwrap();
22072213
}
2214+
2215+
#[test]
2216+
fn test_duplicate_name_validation_exempts_id_resolved_columns() {
2217+
// A case-sensitive Spark write can produce `v`/`V` siblings. With field-ID reads,
2218+
// Spark resolves the requested `v` by unique ID before any name matching
2219+
// (`ParquetReadSupport.matchIdField`), so the case-insensitive duplicate error must
2220+
// not fire for it.
2221+
let logical: SchemaRef = Arc::new(Schema::new(vec![variant_field("v", Some(1))]));
2222+
let physical: SchemaRef = Arc::new(Schema::new(vec![
2223+
variant_field("v", Some(1)),
2224+
int_field("V", Some(2)),
2225+
]));
2226+
rewrite_column(logical, physical, true, "v").unwrap();
2227+
2228+
// Without field IDs the requested name falls through to name matching, which keeps
2229+
// Spark's ambiguity error.
2230+
let logical: SchemaRef = Arc::new(Schema::new(vec![variant_field("v", None)]));
2231+
let physical: SchemaRef = Arc::new(Schema::new(vec![
2232+
variant_field("v", None),
2233+
int_field("V", None),
2234+
]));
2235+
let err = rewrite_column(logical, physical, false, "v").unwrap_err();
2236+
assert!(err.to_string().contains("Found duplicate field"), "{err}");
2237+
}
22082238
}

‎spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,10 @@ import org.apache.parquet.schema.MessageTypeParser
3434
import org.apache.spark.sql.{CometTestBase, Row}
3535
import org.apache.spark.sql.comet.CometNativeScanExec
3636
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
37+
import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils
3738
import org.apache.spark.sql.functions.{array, col}
3839
import org.apache.spark.sql.internal.SQLConf
39-
import org.apache.spark.sql.types.{IntegerType, LongType, NullType, StringType, StructType}
40+
import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, NullType, StringType, StructType}
4041

4142
import org.apache.comet.CometConf
4243
import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus}
@@ -211,6 +212,41 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper
211212
}
212213
}
213214

215+
test("case-insensitive Variant scan resolves duplicate names by field ID") {
216+
assume(isSpark40Plus, "VariantType requires Spark 4.0+")
217+
218+
def fieldId(id: Int) =
219+
new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id).build()
220+
221+
withTempPath { path =>
222+
withSQLConf(
223+
CometConf.COMET_ENABLED.key -> "false",
224+
SQLConf.CASE_SENSITIVE.key -> "true",
225+
"spark.sql.variant.writeShredding.enabled" -> "false") {
226+
sql("SELECT parse_json('42') AS v, 7 AS V")
227+
.select(col("v").as("v", fieldId(1)), col("V").as("V", fieldId(2)))
228+
.write
229+
.parquet(path.toString)
230+
}
231+
232+
val variantField = StructType.fromDDL("v VARIANT").fields.head
233+
val readSchema = StructType(Seq(variantField.copy(metadata = fieldId(1))))
234+
235+
withSQLConf(
236+
SQLConf.CASE_SENSITIVE.key -> "false",
237+
SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true",
238+
"spark.sql.parquet.filterPushdown" -> "false",
239+
"spark.sql.variant.allowReadingShredded" -> "true",
240+
"spark.sql.variant.pushVariantIntoScan" -> "false") {
241+
// Spark resolves the unique field ID before name matching, so the physical `v`/`V`
242+
// name collision must not raise the case-insensitive duplicate error for `v`.
243+
val (_, cometPlan) =
244+
checkSparkAnswer(spark.read.schema(readSchema).parquet(path.toString))
245+
assert(collect(cometPlan) { case s: CometNativeScanExec => s }.size == 1)
246+
}
247+
}
248+
}
249+
214250
test("Variant scan falls back when Parquet encryption is configured") {
215251
assume(isSpark40Plus, "VariantType requires Spark 4.0+")
216252

0 commit comments

Comments
 (0)