Skip to content

Commit a69b5f5

Browse files
committed
fix: resolve parquet field id mapping once per file and validate it there
The schema adapter now resolves how every requested nested field reads from the file struct once per file, mirroring Spark's clipParquetSchema, and raises a duplicate field id or ambiguous name for any referenced column whether or not a cast is emitted. Identical file and requested schemas with a duplicated id are rejected as Spark rejects them. The resolved mapping is handed to CometCastColumnExpr and applied positionally per batch; the relabel shortcut runs only when the mapping is positional. Per id and per name lookups use a small Copy entry and gather matching names only when reporting an ambiguity, so a wide struct allocates nothing per field id. Placeholder names generated for shielded file columns are reserved against the folded logical and physical names that downstream lookups compare, so a requested column differing only by case keeps its default. The reservation set is built on the first placeholder only.
1 parent 58e67fe commit a69b5f5

4 files changed

Lines changed: 887 additions & 291 deletions

File tree

‎native/core/src/parquet/cast_column.rs‎

Lines changed: 84 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ use arrow::{
2424
record_batch::RecordBatch,
2525
};
2626

27-
use crate::parquet::parquet_support::{field_id, spark_parquet_convert, SparkParquetOptions};
27+
use crate::parquet::parquet_support::{
28+
spark_parquet_convert_with_mapping, FieldMapping, SparkParquetOptions,
29+
};
2830
use datafusion::common::format::DEFAULT_CAST_OPTIONS;
2931
use datafusion::common::Result as DataFusionResult;
3032
use datafusion::common::ScalarValue;
@@ -73,21 +75,6 @@ fn types_differ_only_in_field_names(physical: &DataType, logical: &DataType) ->
7375
}
7476
}
7577

76-
/// Returns true if any struct field in the type tree carries Parquet field-id
77-
/// metadata, mirroring the per-struct `should_match_by_id` check in
78-
/// `parquet_convert_struct_to_struct`.
79-
fn contains_field_id_metadata(data_type: &DataType) -> bool {
80-
match data_type {
81-
DataType::Struct(fields) => fields
82-
.iter()
83-
.any(|f| field_id(f).is_some() || contains_field_id_metadata(f.data_type())),
84-
DataType::List(f) | DataType::LargeList(f) | DataType::Map(f, _) => {
85-
contains_field_id_metadata(f.data_type())
86-
}
87-
_ => false,
88-
}
89-
}
90-
9178
/// Recursively relabel an array so its DataType matches `target_type`.
9279
/// This only changes metadata (field names, nullability flags in nested fields);
9380
/// it does NOT change the underlying buffer data.
@@ -202,8 +189,11 @@ pub struct CometCastColumnExpr {
202189
/// Options forwarded to [`cast_column`].
203190
cast_options: CastOptions<'static>,
204191
/// Spark parquet options for complex nested type conversions.
205-
/// When present, enables `spark_parquet_convert` as a fallback.
192+
/// When present, enables the nested conversion as a fallback.
206193
parquet_options: Option<SparkParquetOptions>,
194+
/// Which file field supplies each requested nested field, resolved once per file and
195+
/// reused for every batch. Set together with `parquet_options`.
196+
field_mapping: Option<Arc<FieldMapping>>,
207197
}
208198

209199
// Manually derive `PartialEq`/`Hash` as `Arc<dyn PhysicalExpr>` does not
@@ -215,6 +205,7 @@ impl PartialEq for CometCastColumnExpr {
215205
&& self.target_field.eq(&other.target_field)
216206
&& self.cast_options.eq(&other.cast_options)
217207
&& self.parquet_options.eq(&other.parquet_options)
208+
&& self.field_mapping.eq(&other.field_mapping)
218209
}
219210
}
220211

@@ -225,6 +216,7 @@ impl Hash for CometCastColumnExpr {
225216
self.target_field.hash(state);
226217
self.cast_options.hash(state);
227218
self.parquet_options.hash(state);
219+
self.field_mapping.hash(state);
228220
}
229221
}
230222

@@ -242,12 +234,19 @@ impl CometCastColumnExpr {
242234
target_field,
243235
cast_options: cast_options.unwrap_or(DEFAULT_CAST_OPTIONS),
244236
parquet_options: None,
237+
field_mapping: None,
245238
}
246239
}
247240

248-
/// Set Spark parquet options to enable complex nested type conversions.
249-
pub fn with_parquet_options(mut self, options: SparkParquetOptions) -> Self {
241+
/// Enable nested type conversions with Spark parquet options and the field mapping
242+
/// resolved for this expression's physical and target types.
243+
pub fn with_parquet_options(
244+
mut self,
245+
options: SparkParquetOptions,
246+
field_mapping: Arc<FieldMapping>,
247+
) -> Self {
250248
self.parquet_options = Some(options);
249+
self.field_mapping = Some(field_mapping);
251250
self
252251
}
253252
}
@@ -286,14 +285,13 @@ impl PhysicalExpr for CometCastColumnExpr {
286285
let input_physical_field = self.input_physical_field.data_type();
287286
let target_field = self.target_field.data_type();
288287

289-
// An id-based read must resolve struct fields through spark_parquet_convert,
290-
// which validates the requested ids (e.g. a requested id duplicated in the
291-
// file schema errors); metadata-only relabeling would skip that validation.
292-
let id_based_read = self
293-
.parquet_options
288+
// Relabeling only swaps metadata, so it is right when every requested field reads
289+
// the file field at its own position. A mapping that reorders fields (ids resolved
290+
// to other positions) has to go through the nested conversion below.
291+
let positional = self
292+
.field_mapping
294293
.as_ref()
295-
.is_some_and(|opts| opts.use_field_id)
296-
&& contains_field_id_metadata(target_field);
294+
.is_none_or(|mapping| mapping.is_positional());
297295

298296
// Handle specific type conversions with custom casts
299297
match (input_physical_field, target_field) {
@@ -315,9 +313,8 @@ impl PhysicalExpr for CometCastColumnExpr {
315313
// Nested types that differ only in field names (e.g., List element named
316314
// "item" vs "element", or Map entries named "key_value" vs "entries").
317315
// Re-label the array so the DataType metadata matches the logical schema.
318-
// Skipped for id-based reads so field-id validation still runs below.
319316
(physical, logical)
320-
if !id_based_read
317+
if positional
321318
&& physical != logical
322319
&& types_differ_only_in_field_names(physical, logical) =>
323320
{
@@ -329,16 +326,17 @@ impl PhysicalExpr for CometCastColumnExpr {
329326
other => Ok(other),
330327
}
331328
}
332-
// Fallback: use spark_parquet_convert for complex nested type conversions
333-
// (e.g., List<Struct{a,b,c}> → List<Struct{a,c}>, Map field selection, etc.)
334-
_ => {
335-
if let Some(parquet_options) = &self.parquet_options {
336-
let converted = spark_parquet_convert(value, target_field, parquet_options)?;
337-
Ok(converted)
338-
} else {
339-
Ok(value)
340-
}
341-
}
329+
// Fallback: nested conversion through the resolved mapping
330+
// (e.g., List<Struct{a,b,c}> -> List<Struct{a,c}>, Map field selection, etc.)
331+
_ => match (&self.parquet_options, &self.field_mapping) {
332+
(Some(parquet_options), Some(mapping)) => spark_parquet_convert_with_mapping(
333+
value,
334+
target_field,
335+
mapping,
336+
parquet_options,
337+
),
338+
_ => Ok(value),
339+
},
342340
}
343341
}
344342

@@ -362,8 +360,8 @@ impl PhysicalExpr for CometCastColumnExpr {
362360
Arc::clone(&self.target_field),
363361
Some(self.cast_options.clone()),
364362
);
365-
if let Some(opts) = &self.parquet_options {
366-
new_expr = new_expr.with_parquet_options(opts.clone());
363+
if let (Some(opts), Some(mapping)) = (&self.parquet_options, &self.field_mapping) {
364+
new_expr = new_expr.with_parquet_options(opts.clone(), Arc::clone(mapping));
367365
}
368366
Ok(Arc::new(new_expr))
369367
}
@@ -376,6 +374,7 @@ impl PhysicalExpr for CometCastColumnExpr {
376374
#[cfg(test)]
377375
mod tests {
378376
use super::*;
377+
use crate::parquet::parquet_support::resolve_field_mapping;
379378
use arrow::array::{Array, Int32Array, StringArray};
380379
use arrow::datatypes::{Field, Fields};
381380
use datafusion::physical_expr::expressions::Column;
@@ -390,53 +389,65 @@ mod tests {
390389
)]))
391390
}
392391

393-
/// A file struct with two children sharing field id 1 must raise Spark's
394-
/// duplicate-id error when the requested schema looks id 1 up, even though
395-
/// every child name matches and the relabel shortcut would otherwise apply.
392+
/// File struct `x` (id 1) = 42, `y` (id 2) = 43; requested struct names them the same
393+
/// but swaps the ids. Names and types match, so only the positional gate keeps the
394+
/// relabel shortcut from firing: the mapping reads by id and the result must be
395+
/// `x` = 43, `y` = 42.
396396
#[test]
397-
fn test_field_id_read_rejects_duplicate_ids_despite_matching_names() {
398-
let physical_fields = Fields::from(vec![
399-
int_field_with_id("x", 1),
400-
int_field_with_id("y", 1),
401-
int_field_with_id("z", 2),
402-
]);
403-
let logical_fields = Fields::from(vec![
404-
int_field_with_id("x", 1),
405-
int_field_with_id("y", 3),
406-
int_field_with_id("z", 2),
407-
]);
397+
fn test_swapped_field_ids_bypass_relabel_shortcut() {
398+
let physical_fields =
399+
Fields::from(vec![int_field_with_id("x", 1), int_field_with_id("y", 2)]);
400+
let logical_fields =
401+
Fields::from(vec![int_field_with_id("x", 2), int_field_with_id("y", 1)]);
408402

409403
let input_field = Arc::new(Field::new(
410404
"s",
411405
DataType::Struct(physical_fields.clone()),
412406
true,
413407
));
414-
let target_field = Arc::new(Field::new("s", DataType::Struct(logical_fields), true));
408+
let target_field = Arc::new(Field::new(
409+
"s",
410+
DataType::Struct(logical_fields.clone()),
411+
true,
412+
));
415413

416414
let columns: Vec<ArrayRef> = vec![
417415
Arc::new(Int32Array::from(vec![42])),
418416
Arc::new(Int32Array::from(vec![43])),
419-
Arc::new(Int32Array::from(vec![44])),
420417
];
421418
let struct_arr = StructArray::new(physical_fields, columns, None);
422419
let schema = Schema::new(vec![Arc::clone(&input_field)]);
423420
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_arr)]).unwrap();
424421

425422
let mut opts = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false);
426423
opts.use_field_id = true;
424+
let mapping = Arc::new(
425+
resolve_field_mapping(input_field.data_type(), target_field.data_type(), &opts)
426+
.unwrap(),
427+
);
428+
assert!(!mapping.is_positional());
427429

428430
let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("s", 0));
429431
let cast_expr = CometCastColumnExpr::new(col_expr, input_field, target_field, None)
430-
.with_parquet_options(opts);
431-
432-
let err = cast_expr
433-
.evaluate(&batch)
434-
.expect_err("requested field id 1 matches two file fields and must error");
435-
let msg = err.to_string();
436-
assert!(
437-
msg.contains("_LEGACY_ERROR_TEMP_2094") && msg.contains("id=1"),
438-
"expected duplicate field id error, got: {msg}"
439-
);
432+
.with_parquet_options(opts, mapping);
433+
434+
let ColumnarValue::Array(arr) = cast_expr.evaluate(&batch).unwrap() else {
435+
panic!("expected array result");
436+
};
437+
assert_eq!(arr.data_type(), &DataType::Struct(logical_fields));
438+
let result = arr.as_any().downcast_ref::<StructArray>().unwrap();
439+
let x = result
440+
.column(0)
441+
.as_any()
442+
.downcast_ref::<Int32Array>()
443+
.unwrap();
444+
let y = result
445+
.column(1)
446+
.as_any()
447+
.downcast_ref::<Int32Array>()
448+
.unwrap();
449+
assert_eq!(x.value(0), 43);
450+
assert_eq!(y.value(0), 42);
440451
}
441452

442453
/// Companion guard: without any field ids the relabel shortcut must keep
@@ -492,11 +503,16 @@ mod tests {
492503
);
493504

494505
// Enabling id read mode without any id metadata must not disable the
495-
// shortcut either.
506+
// shortcut either: the resolved mapping is positional.
496507
let mut opts = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false);
497508
opts.use_field_id = true;
509+
let mapping = Arc::new(
510+
resolve_field_mapping(input_field.data_type(), target_field.data_type(), &opts)
511+
.unwrap(),
512+
);
513+
assert!(mapping.is_positional());
498514
let id_mode_expr = CometCastColumnExpr::new(col_expr, input_field, target_field, None)
499-
.with_parquet_options(opts);
515+
.with_parquet_options(opts, mapping);
500516

501517
for cast_expr in [plain_expr, id_mode_expr] {
502518
let result = cast_expr.evaluate(&batch).unwrap();

0 commit comments

Comments
 (0)