Skip to content

Commit bee5811

Browse files
committed
Preserve field metadata when coercing INT96 timestamps
Backport of the fix on main to the 53.1.0 release, for pinning via [patch.crates-io] until upstream lands it. Upstream issue: apache#24786 `coerce_int96_to_resolution` rebuilds every struct, list and map field with `Field::new_struct`, `Field::new_list` and `Field::new`, which start from empty metadata. Leaf fields go through `field_with_new_type`, which clones the field, so only container fields are affected, and only when the file holds an INT96 column: the same file read without `coerce_int96` keeps its metadata. `PARQUET:field_id` lives in that metadata, and formats that identify a column by id rather than by name depend on it. Delta Lake column mapping is one: the read schema names columns `col-<id>` and pairs them with the file by field id, so a struct column that loses its id is resolved by neither id nor name. It reads back as NULL, or errors if the table declares the column NOT NULL. Carry `current_field.metadata()` across in the three constructors.
1 parent eae7bf4 commit bee5811

1 file changed

Lines changed: 56 additions & 3 deletions

File tree

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -816,7 +816,8 @@ pub fn coerce_int96_to_resolution(
816816
current_field.name(),
817817
processed_children.as_slice(),
818818
current_field.is_nullable(),
819-
);
819+
)
820+
.with_metadata(current_field.metadata().clone());
820821
parent_fields.borrow_mut().push(Arc::new(processed_struct));
821822
}
822823
(DataType::List(unprocessed_child), None) => {
@@ -849,7 +850,8 @@ pub fn coerce_int96_to_resolution(
849850
current_field.name(),
850851
Arc::clone(&processed_children[0]),
851852
current_field.is_nullable(),
852-
);
853+
)
854+
.with_metadata(current_field.metadata().clone());
853855
parent_fields.borrow_mut().push(Arc::new(processed_list));
854856
}
855857
(DataType::Map(unprocessed_child, _), None) => {
@@ -879,7 +881,8 @@ pub fn coerce_int96_to_resolution(
879881
current_field.name(),
880882
DataType::Map(Arc::clone(&processed_children[0]), *sorted),
881883
current_field.is_nullable(),
882-
);
884+
)
885+
.with_metadata(current_field.metadata().clone());
883886
parent_fields.borrow_mut().push(Arc::new(processed_map));
884887
}
885888
(DataType::Timestamp(TimeUnit::Nanosecond, None), None)
@@ -1778,6 +1781,7 @@ async fn output_single_parquet_file_parallelized(
17781781

17791782
#[cfg(test)]
17801783
mod tests {
1784+
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
17811785
use parquet::arrow::parquet_to_arrow_schema;
17821786
use std::sync::Arc;
17831787

@@ -1992,4 +1996,53 @@ mod tests {
19921996

19931997
assert_eq!(result, expected_schema);
19941998
}
1999+
2000+
#[test]
2001+
fn coerce_int96_to_resolution_preserves_field_metadata() {
2002+
// Spark and Delta Lake stamp a field id on every field. Coercion must
2003+
// carry that metadata across, on nested fields as well as leaves:
2004+
// formats that identify a column by id rather than by name, such as
2005+
// Delta Lake column mapping, cannot resolve a column that loses it.
2006+
let spark_schema = "
2007+
message spark_schema {
2008+
REQUIRED INT64 c0 = 1;
2009+
OPTIONAL group c1 = 2 {
2010+
OPTIONAL INT96 c2 = 3;
2011+
}
2012+
OPTIONAL group c3 (LIST) = 4 {
2013+
REPEATED group list {
2014+
OPTIONAL INT96 element = 5;
2015+
}
2016+
}
2017+
}
2018+
";
2019+
2020+
let schema = parse_message_type(spark_schema).expect("should parse schema");
2021+
let descr = SchemaDescriptor::new(Arc::new(schema));
2022+
let arrow_schema = parquet_to_arrow_schema(&descr, None).unwrap();
2023+
2024+
let result =
2025+
coerce_int96_to_resolution(&descr, &arrow_schema, &TimeUnit::Microsecond)
2026+
.unwrap();
2027+
2028+
for (original, coerced) in arrow_schema.fields().iter().zip(result.fields()) {
2029+
assert_eq!(
2030+
original.metadata(),
2031+
coerced.metadata(),
2032+
"field {} lost its metadata",
2033+
original.name(),
2034+
);
2035+
}
2036+
2037+
// The struct and the list are the fields rebuilt by the coercion, so
2038+
// spell out that they kept their ids rather than relying on the loop.
2039+
assert_eq!(
2040+
result.field_with_name("c1").unwrap().metadata()[PARQUET_FIELD_ID_META_KEY],
2041+
"2",
2042+
);
2043+
assert_eq!(
2044+
result.field_with_name("c3").unwrap().metadata()[PARQUET_FIELD_ID_META_KEY],
2045+
"4",
2046+
);
2047+
}
19952048
}

0 commit comments

Comments
 (0)