Skip to content

Commit 8bb2406

Browse files
committed
Preserve field metadata when coercing INT96 timestamps
Fixes apache#24786 `Int96Coercer` rebuilds every struct, list and map field with `Field::new_struct`, `Field::new_list` and `Field::new`, which start from empty metadata. Leaf fields go through `field_with_new_type`, which clones the field, so only container fields are affected, and only when the file holds an INT96 column: the same file read without `coerce_int96` keeps its metadata. Carry `current_field.metadata()` across in the three constructors. Signed-off-by: Leonid Ryzhyk <[email protected]>
1 parent 4ada0dc commit 8bb2406

1 file changed

Lines changed: 56 additions & 3 deletions

File tree

‎datafusion/datasource-parquet/src/schema_coercion.rs‎

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,8 @@ fn coerce_int96_to_resolution_impl(
328328
current_field.name(),
329329
processed_children.as_slice(),
330330
current_field.is_nullable(),
331-
);
331+
)
332+
.with_metadata(current_field.metadata().clone());
332333
parent_fields.borrow_mut().push(Arc::new(processed_struct));
333334
}
334335
(DataType::List(unprocessed_child), None) => {
@@ -361,7 +362,8 @@ fn coerce_int96_to_resolution_impl(
361362
current_field.name(),
362363
Arc::clone(&processed_children[0]),
363364
current_field.is_nullable(),
364-
);
365+
)
366+
.with_metadata(current_field.metadata().clone());
365367
parent_fields.borrow_mut().push(Arc::new(processed_list));
366368
}
367369
(DataType::Map(unprocessed_child, _), None) => {
@@ -391,7 +393,8 @@ fn coerce_int96_to_resolution_impl(
391393
current_field.name(),
392394
DataType::Map(Arc::clone(&processed_children[0]), *sorted),
393395
current_field.is_nullable(),
394-
);
396+
)
397+
.with_metadata(current_field.metadata().clone());
395398
parent_fields.borrow_mut().push(Arc::new(processed_map));
396399
}
397400
(DataType::Timestamp(TimeUnit::Nanosecond, None), None)
@@ -460,6 +463,7 @@ pub fn transform_binary_to_string(schema: &Schema) -> Schema {
460463
}
461464
#[cfg(test)]
462465
mod tests {
466+
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
463467
use parquet::arrow::parquet_to_arrow_schema;
464468

465469
use super::*;
@@ -728,4 +732,53 @@ mod tests {
728732

729733
assert_eq!(result, expected_schema);
730734
}
735+
736+
#[test]
737+
fn coerce_int96_to_resolution_preserves_field_metadata() {
738+
// Spark and Delta Lake stamp a field id on every field. Coercion must
739+
// carry that metadata across on nested fields as well as leaves:
740+
// formats that identify a column by id rather than by name, such as
741+
// Delta Lake column mapping, cannot resolve a column that loses it.
742+
let spark_schema = "
743+
message spark_schema {
744+
REQUIRED INT64 c0 = 1;
745+
OPTIONAL group c1 = 2 {
746+
OPTIONAL INT96 c2 = 3;
747+
}
748+
OPTIONAL group c3 (LIST) = 4 {
749+
REPEATED group list {
750+
OPTIONAL INT96 element = 5;
751+
}
752+
}
753+
}
754+
";
755+
756+
let schema = parse_message_type(spark_schema).expect("should parse schema");
757+
let descr = SchemaDescriptor::new(Arc::new(schema));
758+
let arrow_schema = parquet_to_arrow_schema(&descr, None).unwrap();
759+
760+
let result = Int96Coercer::new(&descr, &arrow_schema, &TimeUnit::Microsecond)
761+
.coerce()
762+
.unwrap();
763+
764+
for (original, coerced) in arrow_schema.fields().iter().zip(result.fields()) {
765+
assert_eq!(
766+
original.metadata(),
767+
coerced.metadata(),
768+
"field {} lost its metadata",
769+
original.name(),
770+
);
771+
}
772+
773+
// The struct and the list are the fields the coercion rebuilds, so
774+
// spell out that they kept their ids rather than leaving it to the loop.
775+
assert_eq!(
776+
result.field_with_name("c1").unwrap().metadata()[PARQUET_FIELD_ID_META_KEY],
777+
"2",
778+
);
779+
assert_eq!(
780+
result.field_with_name("c3").unwrap().metadata()[PARQUET_FIELD_ID_META_KEY],
781+
"4",
782+
);
783+
}
731784
}

0 commit comments

Comments
 (0)