Skip to content

Commit 3d68f22

Browse files
committed
fix: run duplicate field id validation before metadata-only relabeling
CometCastColumnExpr relabeled structs whose types differ only in field metadata, skipping spark_parquet_convert and its duplicate field id check. Guard the shortcut so id-based reads with field id metadata in the target type always take the validating conversion path.
1 parent 65039d8 commit 3d68f22

2 files changed

Lines changed: 164 additions & 3 deletions

File tree

‎native/core/src/parquet/cast_column.rs‎

Lines changed: 163 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use arrow::{
2424
record_batch::RecordBatch,
2525
};
2626

27-
use crate::parquet::parquet_support::{spark_parquet_convert, SparkParquetOptions};
27+
use crate::parquet::parquet_support::{field_id, spark_parquet_convert, SparkParquetOptions};
2828
use datafusion::common::format::DEFAULT_CAST_OPTIONS;
2929
use datafusion::common::Result as DataFusionResult;
3030
use datafusion::common::ScalarValue;
@@ -73,6 +73,21 @@ fn types_differ_only_in_field_names(physical: &DataType, logical: &DataType) ->
7373
}
7474
}
7575

76+
/// Returns true if any struct field in the type tree carries Parquet field-id
77+
/// metadata, mirroring the per-struct `should_match_by_id` check in
78+
/// `parquet_convert_struct_to_struct`.
79+
fn contains_field_id_metadata(data_type: &DataType) -> bool {
80+
match data_type {
81+
DataType::Struct(fields) => fields
82+
.iter()
83+
.any(|f| field_id(f).is_some() || contains_field_id_metadata(f.data_type())),
84+
DataType::List(f) | DataType::LargeList(f) | DataType::Map(f, _) => {
85+
contains_field_id_metadata(f.data_type())
86+
}
87+
_ => false,
88+
}
89+
}
90+
7691
/// Recursively relabel an array so its DataType matches `target_type`.
7792
/// This only changes metadata (field names, nullability flags in nested fields);
7893
/// it does NOT change the underlying buffer data.
@@ -271,6 +286,15 @@ impl PhysicalExpr for CometCastColumnExpr {
271286
let input_physical_field = self.input_physical_field.data_type();
272287
let target_field = self.target_field.data_type();
273288

289+
// An id-based read must resolve struct fields through spark_parquet_convert,
290+
// which validates the requested ids (e.g. a requested id duplicated in the
291+
// file schema errors); metadata-only relabeling would skip that validation.
292+
let id_based_read = self
293+
.parquet_options
294+
.as_ref()
295+
.is_some_and(|opts| opts.use_field_id)
296+
&& contains_field_id_metadata(target_field);
297+
274298
// Handle specific type conversions with custom casts
275299
match (input_physical_field, target_field) {
276300
// Timestamp(Microsecond) -> Timestamp(Millisecond)
@@ -291,8 +315,11 @@ impl PhysicalExpr for CometCastColumnExpr {
291315
// Nested types that differ only in field names (e.g., List element named
292316
// "item" vs "element", or Map entries named "key_value" vs "entries").
293317
// Re-label the array so the DataType metadata matches the logical schema.
318+
// Skipped for id-based reads so field-id validation still runs below.
294319
(physical, logical)
295-
if physical != logical && types_differ_only_in_field_names(physical, logical) =>
320+
if !id_based_read
321+
&& physical != logical
322+
&& types_differ_only_in_field_names(physical, logical) =>
296323
{
297324
match value {
298325
ColumnarValue::Array(array) => {
@@ -352,6 +379,140 @@ mod tests {
352379
use arrow::array::{Array, Int32Array, StringArray};
353380
use arrow::datatypes::{Field, Fields};
354381
use datafusion::physical_expr::expressions::Column;
382+
use datafusion_comet_spark_expr::EvalMode;
383+
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
384+
use std::collections::HashMap;
385+
386+
fn int_field_with_id(name: &str, id: i32) -> Field {
387+
Field::new(name, DataType::Int32, true).with_metadata(HashMap::from([(
388+
PARQUET_FIELD_ID_META_KEY.to_string(),
389+
id.to_string(),
390+
)]))
391+
}
392+
393+
/// A file struct with two children sharing field id 1 must raise Spark's
394+
/// duplicate-id error when the requested schema looks id 1 up, even though
395+
/// every child name matches and the relabel shortcut would otherwise apply.
396+
#[test]
397+
fn test_field_id_read_rejects_duplicate_ids_despite_matching_names() {
398+
let physical_fields = Fields::from(vec![
399+
int_field_with_id("x", 1),
400+
int_field_with_id("y", 1),
401+
int_field_with_id("z", 2),
402+
]);
403+
let logical_fields = Fields::from(vec![
404+
int_field_with_id("x", 1),
405+
int_field_with_id("y", 3),
406+
int_field_with_id("z", 2),
407+
]);
408+
409+
let input_field = Arc::new(Field::new(
410+
"s",
411+
DataType::Struct(physical_fields.clone()),
412+
true,
413+
));
414+
let target_field = Arc::new(Field::new("s", DataType::Struct(logical_fields), true));
415+
416+
let columns: Vec<ArrayRef> = vec![
417+
Arc::new(Int32Array::from(vec![42])),
418+
Arc::new(Int32Array::from(vec![43])),
419+
Arc::new(Int32Array::from(vec![44])),
420+
];
421+
let struct_arr = StructArray::new(physical_fields, columns, None);
422+
let schema = Schema::new(vec![Arc::clone(&input_field)]);
423+
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_arr)]).unwrap();
424+
425+
let mut opts = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false);
426+
opts.use_field_id = true;
427+
428+
let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("s", 0));
429+
let cast_expr = CometCastColumnExpr::new(col_expr, input_field, target_field, None)
430+
.with_parquet_options(opts);
431+
432+
let err = cast_expr
433+
.evaluate(&batch)
434+
.expect_err("requested field id 1 matches two file fields and must error");
435+
let msg = err.to_string();
436+
assert!(
437+
msg.contains("_LEGACY_ERROR_TEMP_2094") && msg.contains("id=1"),
438+
"expected duplicate field id error, got: {msg}"
439+
);
440+
}
441+
442+
/// Companion guard: without any field ids the relabel shortcut must keep
443+
/// handling name-only differences, whether or not id read mode is enabled.
444+
#[test]
445+
fn test_relabel_shortcut_kept_for_name_only_differences_without_ids() {
446+
// Physical: s { col: List(Field("item", Int32)) }
447+
// Logical: s { col: List(Field("element", Int32)) }
448+
let physical_list_field = Arc::new(Field::new("item", DataType::Int32, true));
449+
let logical_list_field = Arc::new(Field::new("element", DataType::Int32, true));
450+
let physical_fields = Fields::from(vec![Field::new(
451+
"col",
452+
DataType::List(Arc::clone(&physical_list_field)),
453+
true,
454+
)]);
455+
let logical_fields = Fields::from(vec![Field::new(
456+
"col",
457+
DataType::List(logical_list_field),
458+
true,
459+
)]);
460+
461+
let input_field = Arc::new(Field::new(
462+
"s",
463+
DataType::Struct(physical_fields.clone()),
464+
true,
465+
));
466+
let target_field = Arc::new(Field::new(
467+
"s",
468+
DataType::Struct(logical_fields.clone()),
469+
true,
470+
));
471+
472+
let values = Int32Array::from(vec![1, 2, 3]);
473+
let list = ListArray::new(
474+
physical_list_field,
475+
arrow::buffer::OffsetBuffer::new(vec![0, 2, 3].into()),
476+
Arc::new(values),
477+
None,
478+
);
479+
let struct_arr = StructArray::new(physical_fields, vec![Arc::new(list) as ArrayRef], None);
480+
let schema = Schema::new(vec![Arc::clone(&input_field)]);
481+
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_arr)]).unwrap();
482+
483+
let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("s", 0));
484+
485+
// Without parquet options the fallback arm would return the value
486+
// unchanged, so a relabeled result proves the shortcut itself fired.
487+
let plain_expr = CometCastColumnExpr::new(
488+
Arc::clone(&col_expr),
489+
Arc::clone(&input_field),
490+
Arc::clone(&target_field),
491+
None,
492+
);
493+
494+
// Enabling id read mode without any id metadata must not disable the
495+
// shortcut either.
496+
let mut opts = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false);
497+
opts.use_field_id = true;
498+
let id_mode_expr = CometCastColumnExpr::new(col_expr, input_field, target_field, None)
499+
.with_parquet_options(opts);
500+
501+
for cast_expr in [plain_expr, id_mode_expr] {
502+
let result = cast_expr.evaluate(&batch).unwrap();
503+
let ColumnarValue::Array(arr) = result else {
504+
panic!("expected array result");
505+
};
506+
assert_eq!(arr.data_type(), &DataType::Struct(logical_fields.clone()));
507+
let result_struct = arr.as_any().downcast_ref::<StructArray>().unwrap();
508+
let result_list = result_struct
509+
.column(0)
510+
.as_any()
511+
.downcast_ref::<ListArray>()
512+
.unwrap();
513+
assert_eq!(result_list.len(), 2);
514+
}
515+
}
355516

356517
#[test]
357518
fn test_cast_timestamp_micros_to_millis_array() {

‎native/core/src/parquet/parquet_support.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ fn parquet_convert_array(
239239
}
240240

241241
/// Read the Parquet field id stored under arrow-rs's `PARQUET_FIELD_ID_META_KEY`.
242-
fn field_id(field: &arrow::datatypes::Field) -> Option<i32> {
242+
pub(crate) fn field_id(field: &arrow::datatypes::Field) -> Option<i32> {
243243
field
244244
.metadata()
245245
.get(PARQUET_FIELD_ID_META_KEY)

0 commit comments

Comments
 (0)