@@ -256,33 +256,26 @@ type ArrowImporterRegistry = ArcSwapMap<Id, Arc<[ArrowImportVTableRef]>>;
256256
257257/// Session-scoped registry of Arrow extension plugins.
258258///
259- /// Exporters are stored in two indices: one keyed by Arrow extension Id (used for
260- /// `execute_arrow` dispatch) and one keyed by Vortex extension Id (used **only** by
261- /// `to_arrow_field` / `to_arrow_schema` inference, when callers need to translate a Vortex
262- /// extension `DType` into an Arrow `Field` with no target schema in hand). Importers are
263- /// keyed by Arrow extension name. Extension plugins are registered by the crate that owns the
264- /// extension type (for example `vortex-uuid`); temporal extensions are handled by the canonical
265- /// Arrow ↔ Vortex path and do not need plugins.
259+ /// A plugin converts one Vortex extension type to and from an Arrow extension type. The crate that
260+ /// owns the type registers its plugin, as `vortex-uuid` does. Temporal types need no plugin,
261+ /// because Arrow has native types for them.
266262///
267- /// An extension type that does not have an exporter exports its storage field, with field metadata
268- /// that names its Vortex ID and holds its serialized metadata ([`ext_metadata`] describes the
269- /// format). DataFusion compares this metadata when it merges file schemas, so
270- /// [`ExtDTypeRef::serialize_metadata `] must give equal bytes for equal dtypes. The export fails if
271- /// the storage field already has an Arrow extension name, because one field cannot have two .
263+ /// An extension type without a plugin exports as its storage, and the Arrow field records the type
264+ /// so that an import can restore it. `ARROW:extension:name` holds the Vortex ID, and
265+ /// `ARROW:extension:metadata` holds the metadata in base64, since Arrow metadata must be UTF-8
266+ /// ([`ext_metadata `] describes the format). DataFusion compares this metadata when it merges file
267+ /// schemas, so [`ExtDTypeRef::serialize_metadata`] must give equal bytes for equal types .
272268///
273- /// A caller can also export to such a field directly, for example one from
274- /// [`Self::to_arrow_storage_field`]. If no exporter accepts the target, but its extension name is
275- /// the Vortex ID of the array, the array exports its storage. This works for every extension type,
276- /// so a caller can export a timestamp column as `Int64` without losing its type. However, the
277- /// target must have the same metadata as the array, because different metadata would give the
278- /// values a different meaning, such as a different time unit for a timestamp. To check this, a
279- /// session that knows the Vortex ID rebuilds the dtype from the target and compares the dtypes, and
280- /// any other session compares the metadata bytes.
269+ /// A caller can also export an array to a field that names its Vortex ID, such as a field from
270+ /// [`Self::to_arrow_storage_field`]. The export then checks that an import of the result gives back
271+ /// the same type. The field must have the same metadata, and its Arrow type must convert to the
272+ /// storage type (or to the type itself, such as an Arrow timestamp). Without these checks, values
273+ /// could silently change meaning, such as milliseconds that read back as seconds. For the same
274+ /// reason, an extension array cannot export to a field that names a different known Vortex ID.
281275///
282- /// On import, the importers run first. If none of them accepts the field, but its extension name is
283- /// a Vortex ID that the session knows (see [`initialize`]), the field imports as that extension
284- /// type over its storage. In that case, the import fails if the type rejects the metadata or the
285- /// storage. A field with any other extension name imports as plain storage.
276+ /// On import, the plugins run first. If none applies, a field that names a known Vortex ID imports
277+ /// as that type over its storage, and a field with any other name imports as plain storage. A
278+ /// Vortex ID is known if its type is registered in the session (see [`initialize`]).
286279///
287280/// [`ext_metadata`]: crate::ext_metadata
288281/// [`initialize`]: crate::initialize
@@ -341,8 +334,7 @@ impl ArrowSession {
341334 /// For [`DType::Extension`]s, plugins registered against the extension's `Id`
342335 /// are tried in registration order; the first plugin to return `Some(field)` wins.
343336 ///
344- /// An extension type without a plugin exports its storage field, plus Arrow extension metadata
345- /// that holds its Vortex ID (see [`ArrowSession`]).
337+ /// An extension type without a plugin exports through [`Self::to_arrow_storage_field`].
346338 pub fn to_arrow_field ( & self , name : & str , dtype : & DType ) -> VortexResult < Field > {
347339 // Handle the structural encodings, which may have recursive types
348340 match dtype {
@@ -513,26 +505,40 @@ impl ArrowSession {
513505 Ok ( field)
514506 }
515507
516- /// Rebuild the extension dtype whose Vortex ID is the extension name of `field`, with
517- /// `storage_dtype` as its storage.
508+ /// Returns the extension type that `field` names by Vortex ID, to wrap around `storage_dtype`.
518509 ///
519- /// Returns `None` if the session does not know that Vortex ID, and an error if the metadata of
520- /// `field` is not valid for that dtype.
521- # [ expect ( clippy :: disallowed_methods , reason = "interning a dynamic id" ) ]
522- fn rebuild_ext_dtype_by_vortex_id (
510+ /// Returns `None` if the session does not know that Vortex ID, or if `storage_dtype` already is
511+ /// that type, such as an Arrow timestamp that names `vortex.timestamp`. Fails if the metadata
512+ /// of `field` does not fit the type.
513+ fn ext_dtype_to_wrap (
523514 & self ,
524515 field : & Field ,
525516 storage_dtype : & DType ,
526517 ) -> VortexResult < Option < ExtDTypeRef > > {
527- let Some ( dtype_plugin) = field
528- . extension_type_name ( )
529- . and_then ( |name| self . ext_dtypes . get ( & Id :: new ( name) ) )
530- else {
518+ let Some ( extension_name) = field. extension_type_name ( ) else {
519+ return Ok ( None ) ;
520+ } ;
521+ #[ expect( clippy:: disallowed_methods, reason = "interning a dynamic id" ) ]
522+ let Some ( ext_dtype_plugin) = self . ext_dtypes . get ( & Id :: new ( extension_name) ) else {
531523 return Ok ( None ) ;
532524 } ;
525+ let metadata = decode_arrow_ext_metadata ( field) ?;
526+
527+ // Wrapping a storage that already is the type would nest the type inside itself.
528+ let existing_ext_dtype = storage_dtype
529+ . as_extension_opt ( )
530+ . filter ( |ext_dtype| ext_dtype. id ( ) == ext_dtype_plugin. id ( ) ) ;
531+ if let Some ( existing_ext_dtype) = existing_ext_dtype {
532+ vortex_ensure ! (
533+ metadata == existing_ext_dtype. serialize_metadata( ) ?,
534+ "Arrow field {:?} must carry the metadata of {existing_ext_dtype}, got {metadata:?}" ,
535+ field. name( )
536+ ) ;
537+ return Ok ( None ) ;
538+ }
533539
534- dtype_plugin
535- . deserialize ( & decode_arrow_ext_metadata ( field ) ? , storage_dtype. clone ( ) )
540+ ext_dtype_plugin
541+ . deserialize ( & metadata , storage_dtype. clone ( ) )
536542 . map ( Some )
537543 }
538544
@@ -571,12 +577,10 @@ impl ArrowSession {
571577 let storage_dtype =
572578 self . from_arrow_datatype ( field. data_type ( ) , field. is_nullable ( ) . into ( ) ) ?;
573579
574- Ok (
575- match self . rebuild_ext_dtype_by_vortex_id ( field, & storage_dtype) ? {
576- Some ( ext_dtype) => DType :: Extension ( ext_dtype) ,
577- None => storage_dtype,
578- } ,
579- )
580+ Ok ( match self . ext_dtype_to_wrap ( field, & storage_dtype) ? {
581+ Some ( ext_dtype) => DType :: Extension ( ext_dtype) ,
582+ None => storage_dtype,
583+ } )
580584 }
581585
582586 /// Build the Vortex [`DType`] for an Arrow [`DataType`].
@@ -783,10 +787,9 @@ impl ArrowSession {
783787 }
784788
785789 /// [`Self::from_arrow_array`] with the Arrow [`Field`] already resolved: peel a dictionary,
786- /// probe the import plugins registered for the field's extension name, then fall back to the
787- /// canonical conversion. If the extension name of the field is a Vortex ID that the session
788- /// knows, the result is wrapped in that extension type. Also the recursion point for nested
789- /// fields, which already have a [`Field`].
790+ /// import through [`Self::from_arrow_array_with_importers`], and wrap the result in the
791+ /// extension type if the extension name of the field is a Vortex ID that the session knows.
792+ /// Also the recursion point for nested fields, which already have a [`Field`].
790793 #[ allow( clippy:: wrong_self_convention) ]
791794 fn from_arrow_array_inner (
792795 & self ,
@@ -797,6 +800,23 @@ impl ArrowSession {
797800 return self . dictionary_from_arrow ( array. as_ref ( ) , field) ;
798801 }
799802
803+ let imported = self . from_arrow_array_with_importers ( array, field) ?;
804+
805+ match self . ext_dtype_to_wrap ( field, imported. dtype ( ) ) ? {
806+ Some ( ext_dtype) => Ok ( ExtensionArray :: try_new ( ext_dtype, imported) ?. into_array ( ) ) ,
807+ None => Ok ( imported) ,
808+ }
809+ }
810+
811+ /// Probe the import plugins registered for the extension name of `field`, then fall back to
812+ /// the canonical conversion. Unlike [`Self::from_arrow_array_inner`], this never wraps the
813+ /// result in the extension type that a Vortex ID names.
814+ #[ allow( clippy:: wrong_self_convention) ]
815+ fn from_arrow_array_with_importers (
816+ & self ,
817+ array : ArrowArrayRef ,
818+ field : & Field ,
819+ ) -> VortexResult < ArrayRef > {
800820 let mut current = array;
801821
802822 if let Some ( extension_name) = field. metadata ( ) . get ( EXTENSION_TYPE_NAME_KEY ) {
@@ -838,12 +858,7 @@ impl ArrowSession {
838858 }
839859 }
840860
841- let imported = self . from_arrow_array_canonical ( current. as_ref ( ) , field) ?;
842-
843- match self . rebuild_ext_dtype_by_vortex_id ( field, imported. dtype ( ) ) ? {
844- Some ( ext_dtype) => Ok ( ExtensionArray :: try_new ( ext_dtype, imported) ?. into_array ( ) ) ,
845- None => Ok ( imported) ,
846- }
861+ self . from_arrow_array_canonical ( current. as_ref ( ) , field)
847862 }
848863
849864 /// Recurse into Arrow container arrays so nested fields with extension metadata reach
@@ -955,7 +970,11 @@ impl ArrowSession {
955970 let map = array. as_map ( ) ;
956971 let ( entries, offsets) = trim_map_entries ( map) ;
957972 let entries_array: ArrowArrayRef = Arc :: new ( entries) ;
958- let entries = self . from_arrow_array_inner ( entries_array, entries_field. as_ref ( ) ) ?;
973+ // Map entries must import as a plain struct, so a Vortex ID on their field is
974+ // ignored, as `from_arrow_datatype` also does. Wrapping the entries in that
975+ // extension type would only make `map_from_arrow_parts` fail.
976+ let entries =
977+ self . from_arrow_array_with_importers ( entries_array, entries_field. as_ref ( ) ) ?;
959978 map_from_arrow_parts (
960979 entries,
961980 & offsets,
@@ -1111,15 +1130,37 @@ impl ArrowExporter<'_> {
11111130 }
11121131 }
11131132
1133+ // The registry can hold an alias, so compare the array with the ID of the dtype plugin
1134+ // that the extension name resolves to.
1135+ let named_vortex_id = self
1136+ . session
1137+ . ext_dtypes
1138+ . get ( & arrow_ext_id)
1139+ . map ( |dtype_plugin| dtype_plugin. id ( ) ) ;
1140+
11141141 if let Some ( ext_dtype) = current
11151142 . dtype ( )
11161143 . as_extension_opt ( )
1117- . filter ( |ext_dtype| ext_dtype. id ( ) == arrow_ext_id)
1144+ . filter ( |ext_dtype| {
1145+ ext_dtype. id ( ) == arrow_ext_id || Some ( ext_dtype. id ( ) ) == named_vortex_id
1146+ } )
11181147 . cloned ( )
11191148 {
11201149 return self . execute_arrow_ext_storage ( current, & ext_dtype, target_field, ctx) ;
11211150 }
11221151
1152+ // Only an extension array fails on a target that names another Vortex ID. A plain
1153+ // array still exports to it, because plugins export their storage to the target that
1154+ // they receive.
1155+ if let ( Some ( ext_dtype) , Some ( named_vortex_id) ) =
1156+ ( current. dtype ( ) . as_extension_opt ( ) , named_vortex_id)
1157+ {
1158+ vortex_bail ! (
1159+ "Arrow field {:?} names the extension type {named_vortex_id}, so the array must have that type, got {ext_dtype}" ,
1160+ target_field. name( )
1161+ ) ;
1162+ }
1163+
11231164 debug ! (
11241165 extension_id = arrow_ext_name,
11251166 data_type = ?target_field. data_type( ) ,
@@ -1132,32 +1173,39 @@ impl ArrowExporter<'_> {
11321173 execute_arrow_naive ( array, target. map ( |field| field. data_type ( ) ) , self , ctx)
11331174 }
11341175
1135- /// Export the storage of `array` to `target`, whose extension name is the Vortex ID of
1136- /// `ext_dtype`. Fails if `target` does not have the metadata of `ext_dtype `.
1176+ /// Export `array`, whose type is `ext_dtype`, to `target`, whose extension name is the Vortex
1177+ /// ID of `ext_dtype`. [`ArrowSession`] describes the checks on `target `.
11371178 fn execute_arrow_ext_storage (
11381179 & self ,
11391180 array : ArrayRef ,
11401181 ext_dtype : & ExtDTypeRef ,
11411182 target : & Field ,
11421183 ctx : & mut ExecutionCtx ,
11431184 ) -> VortexResult < ArrowArrayRef > {
1144- match self
1185+ let target_metadata = decode_arrow_ext_metadata ( target) ?;
1186+ vortex_ensure ! (
1187+ target_metadata == ext_dtype. serialize_metadata( ) ?,
1188+ "Arrow field {:?} must carry the metadata of {ext_dtype}, got {target_metadata:?}" ,
1189+ target. name( )
1190+ ) ;
1191+
1192+ let target_dtype = self
11451193 . session
1146- . rebuild_ext_dtype_by_vortex_id ( target, ext_dtype. storage_dtype ( ) ) ?
1147- {
1148- Some ( target_dtype) => vortex_ensure ! (
1149- target_dtype == * ext_dtype,
1150- "Arrow field {:?} must name {ext_dtype}, got {target_dtype}" ,
1151- target. name( )
1152- ) ,
1153- None => vortex_ensure ! (
1154- decode_arrow_ext_metadata( target) ? == ext_dtype. serialize_metadata( ) ?,
1155- "Arrow field {:?} must carry the metadata of {ext_dtype}, got {:?}" ,
1156- target. name( ) ,
1157- target. extension_type_metadata( ) . unwrap_or_default( )
1158- ) ,
1194+ . from_arrow_datatype ( target. data_type ( ) , ext_dtype. nullability ( ) ) ?;
1195+
1196+ // An Arrow type that already holds the extension type, such as an Arrow timestamp, exports
1197+ // the array itself, so that the executor of that type checks its own parameters.
1198+ if target_dtype. eq_ignore_nullability ( array. dtype ( ) ) {
1199+ return execute_arrow_naive ( array, Some ( target. data_type ( ) ) , self , ctx) ;
11591200 }
11601201
1202+ vortex_ensure ! (
1203+ target_dtype. eq_ignore_nullability( ext_dtype. storage_dtype( ) ) ,
1204+ "Arrow field {:?} must have an Arrow type that imports as the storage of {ext_dtype}, got {}" ,
1205+ target. name( ) ,
1206+ target. data_type( )
1207+ ) ;
1208+
11611209 let storage = array
11621210 . execute :: < ExtensionArray > ( ctx) ?
11631211 . storage_array ( )
0 commit comments