@@ -43,10 +43,7 @@ use arrow_schema::Field;
4343use arrow_schema:: FieldRef ;
4444use arrow_schema:: Fields ;
4545use arrow_schema:: Schema ;
46- use arrow_schema:: extension:: EXTENSION_TYPE_METADATA_KEY ;
4746use arrow_schema:: extension:: EXTENSION_TYPE_NAME_KEY ;
48- use base64:: Engine ;
49- use base64:: prelude:: BASE64_STANDARD ;
5047use tracing:: debug;
5148use tracing:: trace;
5249use vortex_array:: ArrayRef ;
@@ -93,6 +90,8 @@ use crate::convert::trim_offsets;
9390use crate :: dtype:: from_arrow_data_type;
9491use crate :: dtype:: to_data_type_naive;
9592use crate :: executor:: execute_arrow_naive;
93+ use crate :: ext_metadata:: decode_arrow_ext_metadata;
94+ use crate :: ext_metadata:: encode_arrow_ext_metadata;
9695use crate :: run_end_import:: run_end_from_arrow;
9796
9897/// Outcome of a successful call to [`ArrowExportVTable::execute_arrow`].
@@ -265,27 +264,27 @@ type ArrowImporterRegistry = ArcSwapMap<Id, Arc<[ArrowImportVTableRef]>>;
265264/// extension type (for example `vortex-uuid`); temporal extensions are handled by the canonical
266265/// Arrow ↔ Vortex path and do not need plugins.
267266///
268- /// An extension type that does not have an exporter exports its storage field, with two Arrow
269- /// metadata keys added: `ARROW:extension:name` holds its Vortex ID, and `ARROW:extension:metadata`
270- /// holds its serialized metadata in standard padded base64. The metadata is base64 because Arrow
271- /// field metadata must be UTF-8, which is also how Parquet stores `ARROW:schema`. DataFusion
272- /// compares these strings when it merges file schemas, so [`ExtDTypeRef::serialize_metadata`] must
273- /// give equal bytes for equal dtypes. The export fails if the storage field already has an Arrow
274- /// extension name, because one field cannot have two.
267+ /// An extension type that does not have an exporter exports its storage field, with field metadata
268+ /// that names its Vortex ID and holds its serialized metadata ([`ext_metadata`] describes the
269+ /// format). DataFusion compares this metadata when it merges file schemas, so
270+ /// [`ExtDTypeRef::serialize_metadata`] must give equal bytes for equal dtypes. The export fails if
271+ /// the storage field already has an Arrow extension name, because one field cannot have two.
275272///
276- /// A caller can also export to such a field directly. If no exporter accepts the target, but its
277- /// extension name is the Vortex ID of the array, the array exports its storage. This works for
278- /// every extension type, so a caller can export a timestamp column as `Int64` without losing its
279- /// type. However, the target must have the same metadata as the array, because different metadata
280- /// would give the values a different meaning, such as a different time unit for a timestamp. To
281- /// check this, a session that knows the Vortex ID rebuilds the dtype from the target and compares
282- /// the dtypes, and any other session compares the metadata bytes.
273+ /// A caller can also export to such a field directly, for example one from
274+ /// [`Self::to_arrow_storage_field`]. If no exporter accepts the target, but its extension name is
275+ /// the Vortex ID of the array, the array exports its storage. This works for every extension type,
276+ /// so a caller can export a timestamp column as `Int64` without losing its type. However, the
277+ /// target must have the same metadata as the array, because different metadata would give the
278+ /// values a different meaning, such as a different time unit for a timestamp. To check this, a
279+ /// session that knows the Vortex ID rebuilds the dtype from the target and compares the dtypes, and
280+ /// any other session compares the metadata bytes.
283281///
284282/// On import, the importers run first. If none of them accepts the field, but its extension name is
285283/// a Vortex ID that the session knows (see [`initialize`]), the field imports as that extension
286284/// type over its storage. In that case, the import fails if the type rejects the metadata or the
287285/// storage. A field with any other extension name imports as plain storage.
288286///
287+ /// [`ext_metadata`]: crate::ext_metadata
289288/// [`initialize`]: crate::initialize
290289#[ derive( Clone , Debug , Default ) ]
291290pub struct ArrowSession {
@@ -387,7 +386,7 @@ impl ArrowSession {
387386 DType :: Extension ( ext) if !ext. is :: < AnyTemporal > ( ) => {
388387 match self . to_arrow_field_with_exporter ( name, ext) ? {
389388 Some ( field) => Ok ( field) ,
390- None => self . to_arrow_field_without_exporter ( name, ext) ,
389+ None => self . to_arrow_storage_field ( name, ext) ,
391390 }
392391 }
393392 DType :: Variant ( _) => {
@@ -486,9 +485,16 @@ impl ArrowSession {
486485 Ok ( None )
487486 }
488487
489- /// Build the storage field of an extension type without an exporter, and add the Arrow
490- /// extension metadata that holds its Vortex ID and its metadata.
491- fn to_arrow_field_without_exporter (
488+ /// Returns the storage field of `ext_dtype`, with field metadata that holds the extension type
489+ /// ([`ext_metadata`] describes the format).
490+ ///
491+ /// A type with its own Arrow type can also export to this field, for example to export a
492+ /// timestamp column as `Int64` and import it back as a timestamp.
493+ ///
494+ /// Fails if the storage field already has an Arrow extension name.
495+ ///
496+ /// [`ext_metadata`]: crate::ext_metadata
497+ pub fn to_arrow_storage_field (
492498 & self ,
493499 name : & str ,
494500 ext_dtype : & ExtDTypeRef ,
@@ -500,14 +506,9 @@ impl ArrowSession {
500506 ) ;
501507 }
502508
503- let metadata = BASE64_STANDARD . encode ( ext_dtype. serialize_metadata ( ) ?) ;
504- field. metadata_mut ( ) . extend ( [
505- (
506- EXTENSION_TYPE_NAME_KEY . to_owned ( ) ,
507- ext_dtype. id ( ) . to_string ( ) ,
508- ) ,
509- ( EXTENSION_TYPE_METADATA_KEY . to_owned ( ) , metadata) ,
510- ] ) ;
509+ field
510+ . metadata_mut ( )
511+ . extend ( encode_arrow_ext_metadata ( ext_dtype) ?) ;
511512
512513 Ok ( field)
513514 }
@@ -531,7 +532,7 @@ impl ArrowSession {
531532 } ;
532533
533534 dtype_plugin
534- . deserialize ( & decode_ext_metadata ( field) ?, storage_dtype. clone ( ) )
535+ . deserialize ( & decode_arrow_ext_metadata ( field) ?, storage_dtype. clone ( ) )
535536 . map ( Some )
536537 }
537538
@@ -1036,20 +1037,6 @@ fn run_end_values_field(values_field: &FieldRef, nullability: Nullability) -> Fi
10361037 . with_nullable ( nullability. into ( ) )
10371038}
10381039
1039- /// Decode the base64 `ARROW:extension:metadata` of a field whose extension name is a Vortex ID.
1040- ///
1041- /// A field without that key has empty metadata, as Arrow allows.
1042- fn decode_ext_metadata ( field : & Field ) -> VortexResult < Vec < u8 > > {
1043- let encoded = field. extension_type_metadata ( ) . unwrap_or_default ( ) ;
1044-
1045- BASE64_STANDARD . decode ( encoded) . map_err ( |err| {
1046- vortex_err ! (
1047- "Arrow field {:?} must carry base64 extension metadata, got {encoded:?}: {err}" ,
1048- field. name( )
1049- )
1050- } )
1051- }
1052-
10531040impl SessionVar for ArrowSession {
10541041 fn as_any ( & self ) -> & dyn Any {
10551042 self
@@ -1164,7 +1151,7 @@ impl ArrowExporter<'_> {
11641151 target. name( )
11651152 ) ,
11661153 None => vortex_ensure ! (
1167- decode_ext_metadata ( target) ? == ext_dtype. serialize_metadata( ) ?,
1154+ decode_arrow_ext_metadata ( target) ? == ext_dtype. serialize_metadata( ) ?,
11681155 "Arrow field {:?} must carry the metadata of {ext_dtype}, got {:?}" ,
11691156 target. name( ) ,
11701157 target. extension_type_metadata( ) . unwrap_or_default( )
0 commit comments