Skip to content

Commit ff89288

Browse files
committed
fix: keep extension types when skip_metadata clears Parquet field metadata
With `skip_metadata = true` (the default), Parquet schema inference cleared all field metadata, including the `ARROW:extension:name` and `ARROW:extension:metadata` keys. Columns with an extension type were read as their storage type. Keep the extension type keys and clear the rest: the extension type is part of the type of a field, not a writer annotation. Only top-level fields are handled; nested fields still lose their metadata. This is the fix proposed in apache#26517. It changes how existing files with extension types (for example `arrow.uuid`) are read.
1 parent 4b8a6d5 commit ff89288

1 file changed

Lines changed: 13 additions & 2 deletions

File tree

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ pub use crate::schema_coercion::{
3434
pub use crate::sink::ParquetSink;
3535

3636
use arrow::datatypes::{DataType, FieldRef, Fields, Metadata, Schema, SchemaRef};
37+
use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
3738
use datafusion_datasource::TableSchema;
3839
use datafusion_datasource::file_compression_type::FileCompressionType;
3940
use datafusion_datasource::file_sink_config::FileSinkConfig;
@@ -255,7 +256,8 @@ impl ParquetFormat {
255256
}
256257

257258
/// Clears all metadata (Schema level and field level) on an iterator
258-
/// of Schemas
259+
/// of Schemas, except for the extension type of the fields, which is part
260+
/// of the type of a field.
259261
fn clear_metadata(
260262
schemas: impl IntoIterator<Item = Schema>,
261263
) -> impl Iterator<Item = Schema> {
@@ -264,7 +266,16 @@ fn clear_metadata(
264266
.fields()
265267
.iter()
266268
.map(|field| {
267-
field.as_ref().clone().with_metadata(Metadata::new()) // clear meta
269+
let metadata = field
270+
.metadata()
271+
.iter()
272+
.filter(|(key, _)| {
273+
key.as_str() == EXTENSION_TYPE_NAME_KEY
274+
|| key.as_str() == EXTENSION_TYPE_METADATA_KEY
275+
})
276+
.map(|(key, value)| (key.clone(), value.clone()))
277+
.collect::<Metadata>();
278+
field.as_ref().clone().with_metadata(metadata)
268279
})
269280
.collect::<Fields>();
270281
Schema::new(fields)

0 commit comments

Comments
 (0)