Skip to content

Commit ba95460

Browse files
committed
fix: keep extension types when skip_metadata clears Parquet field metadata
With `skip_metadata = true` (the default), Parquet schema inference cleared all field metadata, including the `ARROW:extension:name` and `ARROW:extension:metadata` keys. Columns with an extension type were read as their storage type. Keep the extension type keys and clear the rest: the extension type is part of the type of a field, not a writer annotation. Only top-level fields are affected: the metadata of nested fields was never cleared. This is the fix proposed in apache#26517. It changes how existing files with extension types (for example `arrow.uuid`) are read.
1 parent 27b047b commit ba95460

1 file changed

Lines changed: 13 additions & 2 deletions

File tree

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ pub use crate::schema_coercion::{
2121
pub use crate::sink::ParquetSink;
2222

2323
use arrow::datatypes::{DataType, FieldRef, Fields, Metadata, Schema, SchemaRef};
24+
use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
2425
use datafusion_datasource::TableSchema;
2526
use datafusion_datasource::file_compression_type::FileCompressionType;
2627
use datafusion_datasource::file_sink_config::FileSinkConfig;
@@ -242,7 +243,8 @@ impl ParquetFormat {
242243
}
243244

244245
/// Clears all metadata (Schema level and field level) on an iterator
245-
/// of Schemas
246+
/// of Schemas, except for the extension type of the fields, which is part
247+
/// of the type of a field.
246248
fn clear_metadata(
247249
schemas: impl IntoIterator<Item = Schema>,
248250
) -> impl Iterator<Item = Schema> {
@@ -251,7 +253,16 @@ fn clear_metadata(
251253
.fields()
252254
.iter()
253255
.map(|field| {
254-
field.as_ref().clone().with_metadata(Metadata::new()) // clear meta
256+
let metadata = field
257+
.metadata()
258+
.iter()
259+
.filter(|(key, _)| {
260+
key.as_str() == EXTENSION_TYPE_NAME_KEY
261+
|| key.as_str() == EXTENSION_TYPE_METADATA_KEY
262+
})
263+
.map(|(key, value)| (key.clone(), value.clone()))
264+
.collect::<Metadata>();
265+
field.as_ref().clone().with_metadata(metadata)
255266
})
256267
.collect::<Fields>();
257268
Schema::new(fields)

0 commit comments

Comments
 (0)