Skip to content

Commit 4925cc2

Browse files
peterxcliclaude
andcommitted
fix: match Spark's fold-ambiguity and field-ID shadowing semantics
Two residuals in the case-insensitive Variant guard. First, when several physical names Unicode-fold to the requested name, Spark rejects the read as ambiguous while the guard early-returned on the single ASCII match; the fold analysis now raises Spark's duplicate-field error for that set. Second, with field-ID reads a wrong-ID physical field could keep a name an ID-matched logical field resolves to, letting the adapter's first-name lookup select the wrong column; the remap now hides such fields behind fake names the way Spark's matchIdField hides unmatched requested fields. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent e40b939 commit 4925cc2

2 files changed

Lines changed: 228 additions & 40 deletions

File tree

‎native/core/src/parquet/schema_adapter.rs‎

Lines changed: 156 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,24 @@ fn remap_physical_schema(
166166
} else {
167167
std::collections::HashSet::new()
168168
};
169+
// Names of logical fields that resolve by ID (their ID is present in the file). Spark
170+
// never name-matches these (`matchIdField` runs first), so any other physical field
171+
// carrying such a name must be hidden from the adapter's name lookup, otherwise a
172+
// first-name match can select the wrong physical column ahead of the ID-matched one
173+
// (e.g. physical `v BINARY` ID 2 shadowing `other VARIANT` ID 1 renamed to `v`).
174+
let id_matched_logical_names: Vec<String> = if should_match_by_id {
175+
logical_schema
176+
.fields()
177+
.iter()
178+
.filter_map(|lf| {
179+
parse_field_id(lf)
180+
.filter(|id| id_to_phys_names.contains_key(id))
181+
.map(|_| lf.name().clone())
182+
})
183+
.collect()
184+
} else {
185+
Vec::new()
186+
};
169187
let mut fake_counter: usize = 0;
170188

171189
let mut name_map: HashMap<String, String> = HashMap::new();
@@ -208,6 +226,27 @@ fn remap_physical_schema(
208226
);
209227
}
210228

229+
// A physical field that was not ID-matched must not keep a name that an
230+
// ID-matched logical field resolves to; hide it like Spark's fake-name scheme
231+
// so the ID-matched column stays the only name match. Exact names collide in
232+
// case-sensitive mode; ASCII-case-insensitive names collide otherwise.
233+
if should_match_by_id
234+
&& id_matched_logical_names.iter().any(|name| {
235+
if case_sensitive {
236+
name == field.name()
237+
} else {
238+
name.eq_ignore_ascii_case(field.name())
239+
}
240+
})
241+
{
242+
fake_counter += 1;
243+
let fake_name = format!("__comet_id_shadowed_field_{}", fake_counter);
244+
return Arc::new(
245+
Field::new(fake_name, field.data_type().clone(), field.is_nullable())
246+
.with_metadata(field.metadata().clone()),
247+
);
248+
}
249+
211250
// Name match. Spark's `matchIdField` does not fall through to a name match for
212251
// ID-bearing logical fields, so skip those when the schema is ID-bearing.
213252
if !case_sensitive {
@@ -350,29 +389,50 @@ fn check_column_duplicate(col_name: &str, physical_schema: &SchemaRef) -> Option
350389
}
351390
}
352391

353-
/// Detect a physical Parquet column that Spark's case-insensitive resolver would match to
354-
/// `col_name` but the ASCII-only matcher cannot. Spark groups physical names by Unicode
355-
/// lowercasing (`toLowerCase(Locale.ROOT)` in `ParquetReadSupport`), so a physical name such
356-
/// as `K` (U+212A KELVIN SIGN) resolves to a requested ASCII `k` there while staying
357-
/// unmatched here, and the scan-planning gate cannot see physical file names. Returns the
358-
/// folded physical name when no ASCII-insensitive match exists but a non-ASCII physical name
359-
/// Unicode-lowercases to the same string as `col_name`.
360-
fn check_column_unicode_fold_mismatch(
392+
/// Outcome of Spark-parity Unicode fold analysis for one requested column name.
393+
enum UnicodeFoldIssue {
394+
/// More than one physical name folds to the requested name; Spark rejects the read as
395+
/// ambiguous, so the native scan must raise the same duplicate-field error. Carries the
396+
/// bracketed matched-name list in Spark's message format.
397+
Ambiguous(String),
398+
/// Exactly one physical name folds to the requested name, but the ASCII-only matcher
399+
/// cannot see it, so the column would be silently null-filled. Carries that name.
400+
FoldOnlyMatch(String),
401+
}
402+
403+
/// Compare Spark's case-insensitive resolution of `col_name` against the ASCII-only matcher.
404+
/// Spark groups physical names by Unicode lowercasing (`toLowerCase(Locale.ROOT)` in
405+
/// `ParquetReadSupport`), so a physical name such as `K` (U+212A KELVIN SIGN) resolves to a
406+
/// requested ASCII `k` there while staying unmatched here, and two physical names that both
407+
/// fold to the requested name are ambiguous there even when only one matches here. The
408+
/// scan-planning gate cannot see physical file names, so this runs per opened file.
409+
fn check_column_unicode_fold(
361410
col_name: &str,
362411
physical_schema: &SchemaRef,
363-
) -> Option<String> {
364-
if physical_schema
412+
) -> Option<UnicodeFoldIssue> {
413+
let folded = col_name.to_lowercase();
414+
let matches: Vec<&str> = physical_schema
365415
.fields()
366416
.iter()
367-
.any(|pf| pf.name().eq_ignore_ascii_case(col_name))
368-
{
369-
return None;
417+
.filter(|pf| pf.name().to_lowercase() == folded)
418+
.map(|pf| pf.name().as_str())
419+
.collect();
420+
match matches.as_slice() {
421+
[] => None,
422+
[single] => {
423+
// An ASCII-case-insensitive match always folds equal, so a single fold match
424+
// that the ASCII matcher also sees is the ordinary working case.
425+
if single.eq_ignore_ascii_case(col_name) {
426+
None
427+
} else {
428+
Some(UnicodeFoldIssue::FoldOnlyMatch(single.to_string()))
429+
}
430+
}
431+
_ => Some(UnicodeFoldIssue::Ambiguous(format!(
432+
"[{}]",
433+
matches.join(", ")
434+
))),
370435
}
371-
physical_schema
372-
.fields()
373-
.iter()
374-
.find(|pf| !pf.name().is_ascii() && pf.name().to_lowercase() == col_name.to_lowercase())
375-
.map(|pf| pf.name().clone())
376436
}
377437

378438
impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory {
@@ -519,19 +579,31 @@ impl PhysicalExprAdapter for SparkPhysicalExprAdapter {
519579
)));
520580
}
521581
if column_err.is_none() && variant_scan {
522-
if let Some(phys) =
523-
check_column_unicode_fold_mismatch(col.name(), orig_physical)
524-
{
525-
column_err = Some(DataFusionError::Execution(format!(
526-
"Case-insensitive scan of a Variant-bearing schema requires \
527-
Unicode column matching: requested column '{name}' resolves \
528-
to physical Parquet column '{phys}' only under Unicode case \
529-
folding, which Comet's native scan does not support yet \
530-
(https://github.com/apache/datafusion-comet/issues/5495). \
531-
Retry with spark.comet.enabled=false, or align the column \
532-
name case.",
533-
name = col.name(),
534-
)));
582+
match check_column_unicode_fold(col.name(), orig_physical) {
583+
Some(UnicodeFoldIssue::Ambiguous(matched)) => {
584+
// Spark rejects the fold-ambiguous read, so raise its
585+
// duplicate-field error rather than serving one match.
586+
column_err = Some(DataFusionError::External(Box::new(
587+
SparkError::DuplicateFieldCaseInsensitive {
588+
required_field_name: col.name().to_string(),
589+
matched_fields: matched,
590+
},
591+
)));
592+
}
593+
Some(UnicodeFoldIssue::FoldOnlyMatch(phys)) => {
594+
column_err = Some(DataFusionError::Execution(format!(
595+
"Case-insensitive scan of a Variant-bearing schema \
596+
requires Unicode column matching: requested column \
597+
'{name}' resolves to physical Parquet column '{phys}' \
598+
only under Unicode case folding, which Comet's native \
599+
scan does not support yet \
600+
(https://github.com/apache/datafusion-comet/issues/5495). \
601+
Retry with spark.comet.enabled=false, or align the \
602+
column name case.",
603+
name = col.name(),
604+
)));
605+
}
606+
None => {}
535607
}
536608
}
537609
}
@@ -2067,8 +2139,8 @@ mod test {
20672139
}
20682140

20692141
#[test]
2070-
fn test_check_column_unicode_fold_mismatch() {
2071-
use super::check_column_unicode_fold_mismatch;
2142+
fn test_check_column_unicode_fold() {
2143+
use super::{check_column_unicode_fold, UnicodeFoldIssue};
20722144

20732145
// Physical `K` (U+212A KELVIN SIGN) folds to `k` in Spark's Unicode resolver but not
20742146
// in the ASCII matcher: the guard must report it.
@@ -2077,14 +2149,25 @@ mod test {
20772149
DataType::Int32,
20782150
true,
20792151
)]));
2080-
assert_eq!(
2081-
check_column_unicode_fold_mismatch("k", &kelvin),
2082-
Some("\u{212A}".to_string())
2083-
);
2152+
assert!(matches!(
2153+
check_column_unicode_fold("k", &kelvin),
2154+
Some(UnicodeFoldIssue::FoldOnlyMatch(name)) if name == "\u{212A}"
2155+
));
20842156

20852157
// ASCII case differences are handled by the existing matcher.
20862158
let ascii: SchemaRef = Arc::new(Schema::new(vec![Field::new("K", DataType::Int32, true)]));
2087-
assert_eq!(check_column_unicode_fold_mismatch("k", &ascii), None);
2159+
assert!(check_column_unicode_fold("k", &ascii).is_none());
2160+
2161+
// ASCII `K` and Kelvin `K` both fold to `k`: Spark rejects the read as ambiguous,
2162+
// so the guard must report the duplicate instead of letting the ASCII match win.
2163+
let ambiguous: SchemaRef = Arc::new(Schema::new(vec![
2164+
Field::new("K", DataType::Int32, true),
2165+
Field::new("\u{212A}", DataType::Int32, true),
2166+
]));
2167+
assert!(matches!(
2168+
check_column_unicode_fold("k", &ambiguous),
2169+
Some(UnicodeFoldIssue::Ambiguous(matched)) if matched == "[K, \u{212A}]"
2170+
));
20882171

20892172
// `ſ` (U+017F) does not lowercase to `s`, so Spark also treats the requested column
20902173
// as missing; both engines agree on the missing-column path.
@@ -2093,15 +2176,15 @@ mod test {
20932176
DataType::Int32,
20942177
true,
20952178
)]));
2096-
assert_eq!(check_column_unicode_fold_mismatch("s", &long_s), None);
2179+
assert!(check_column_unicode_fold("s", &long_s).is_none());
20972180

20982181
// A genuinely missing column reports nothing.
20992182
let other: SchemaRef = Arc::new(Schema::new(vec![Field::new(
21002183
"other",
21012184
DataType::Int32,
21022185
true,
21032186
)]));
2104-
assert_eq!(check_column_unicode_fold_mismatch("k", &other), None);
2187+
assert!(check_column_unicode_fold("k", &other).is_none());
21052188
}
21062189

21072190
fn variant_field(name: &str, id: Option<i32>) -> Field {
@@ -2235,4 +2318,37 @@ mod test {
22352318
let err = rewrite_column(logical, physical, false, "v").unwrap_err();
22362319
assert!(err.to_string().contains("Found duplicate field"), "{err}");
22372320
}
2321+
2322+
#[test]
2323+
fn test_unicode_fold_ambiguity_raises_duplicate_error() {
2324+
// ASCII `K` and Kelvin `K` both fold to the requested `k` in Spark's resolver, which
2325+
// rejects the read as ambiguous; serving the single ASCII match would diverge.
2326+
let logical: SchemaRef = Arc::new(Schema::new(vec![variant_field("k", None)]));
2327+
let physical: SchemaRef = Arc::new(Schema::new(vec![
2328+
variant_field("K", None),
2329+
variant_field("\u{212A}", None),
2330+
]));
2331+
let err = rewrite_column(logical, physical, false, "k").unwrap_err();
2332+
assert!(err.to_string().contains("Found duplicate field"), "{err}");
2333+
}
2334+
2335+
#[test]
2336+
fn test_id_matched_name_shadowed_by_wrong_id_physical_field() {
2337+
// Physical `v BINARY` (ID 2) precedes `other VARIANT` (ID 1). Spark resolves the
2338+
// requested `v` (ID 1) by field ID to `other`; the remap renames `other` to `v` and
2339+
// must hide the wrong-ID physical `v` so the first-name lookup cannot select the
2340+
// Binary column ahead of the ID-matched one.
2341+
let logical: SchemaRef = Arc::new(Schema::new(vec![variant_field("v", Some(1))]));
2342+
let physical_v = {
2343+
let field = Field::new("v", DataType::Binary, true);
2344+
let mut metadata = std::collections::HashMap::new();
2345+
metadata.insert("PARQUET:field_id".to_string(), "2".to_string());
2346+
field.with_metadata(metadata)
2347+
};
2348+
let physical: SchemaRef = Arc::new(Schema::new(vec![
2349+
physical_v,
2350+
variant_field("other", Some(1)),
2351+
]));
2352+
rewrite_column(logical, physical, true, "v").unwrap();
2353+
}
22382354
}

‎spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,78 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper
247247
}
248248
}
249249

250+
test("case-insensitive Variant scan rejects Unicode fold-ambiguous names like Spark") {
251+
assume(isSpark40Plus, "VariantType requires Spark 4.0+")
252+
253+
withTempPath { path =>
254+
withSQLConf(
255+
CometConf.COMET_ENABLED.key -> "false",
256+
SQLConf.CASE_SENSITIVE.key -> "true",
257+
"spark.sql.variant.writeShredding.enabled" -> "false") {
258+
// ASCII `K` and Kelvin `K` (U+212A) both Unicode-lowercase to `k`.
259+
sql("SELECT parse_json('1') AS K, parse_json('2') AS `K`").write
260+
.parquet(path.toString)
261+
}
262+
263+
val table = s"variant_unicode_ambiguous_${System.currentTimeMillis()}"
264+
withTable(table) {
265+
sql(s"CREATE TABLE $table (`k` VARIANT) USING parquet OPTIONS (path '$path')")
266+
withSQLConf(
267+
SQLConf.CASE_SENSITIVE.key -> "false",
268+
"spark.sql.variant.allowReadingShredded" -> "true",
269+
"spark.sql.variant.pushVariantIntoScan" -> "false") {
270+
// Spark's resolver groups both physical names under `k` and rejects the read as
271+
// ambiguous; serving the single ASCII match would silently diverge, so the native
272+
// scan must raise the same duplicate-field error.
273+
val (sparkError, cometError) =
274+
checkSparkAnswerMaybeThrows(sql(s"SELECT `k` FROM $table"))
275+
assert(sparkError.isDefined)
276+
assert(cometError.isDefined)
277+
val messages = Iterator
278+
.iterate(cometError.get)(_.getCause)
279+
.takeWhile(_ != null)
280+
.map(_.getMessage)
281+
.mkString("\n")
282+
assert(messages.contains("duplicate field"), messages)
283+
}
284+
}
285+
}
286+
}
287+
288+
test("field-ID Variant read ignores a wrong-ID physical column with the requested name") {
289+
assume(isSpark40Plus, "VariantType requires Spark 4.0+")
290+
291+
def fieldId(id: Int) =
292+
new MetadataBuilder().putLong(ParquetUtils.FIELD_ID_METADATA_KEY, id).build()
293+
294+
withTempPath { path =>
295+
withSQLConf(
296+
CometConf.COMET_ENABLED.key -> "false",
297+
"spark.sql.variant.writeShredding.enabled" -> "false") {
298+
// Physical order: `v BINARY` (ID 2) before `other VARIANT` (ID 1).
299+
sql("SELECT cast('x' AS BINARY) AS v, parse_json('42') AS other")
300+
.select(col("v").as("v", fieldId(2)), col("other").as("other", fieldId(1)))
301+
.write
302+
.parquet(path.toString)
303+
}
304+
305+
val variantField = StructType.fromDDL("v VARIANT").fields.head
306+
val readSchema = StructType(Seq(variantField.copy(metadata = fieldId(1))))
307+
308+
withSQLConf(
309+
SQLConf.CASE_SENSITIVE.key -> "false",
310+
SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true",
311+
"spark.sql.variant.allowReadingShredded" -> "true",
312+
"spark.sql.variant.pushVariantIntoScan" -> "false") {
313+
// Spark resolves the requested `v` by field ID to `other`; the wrong-ID physical
314+
// `v` must not shadow that resolution in the native adapter's name lookup.
315+
val (_, cometPlan) =
316+
checkSparkAnswer(spark.read.schema(readSchema).parquet(path.toString))
317+
assert(collect(cometPlan) { case s: CometNativeScanExec => s }.size == 1)
318+
}
319+
}
320+
}
321+
250322
test("Variant scan falls back when Parquet encryption is configured") {
251323
assume(isSpark40Plus, "VariantType requires Spark 4.0+")
252324

0 commit comments

Comments
 (0)