@@ -267,11 +267,13 @@ type ArrowImporterRegistry = ArcSwapMap<Id, Arc<[ArrowImportVTableRef]>>;
267267/// schemas, so [`ExtDTypeRef::serialize_metadata`] must give equal bytes for equal types.
268268///
269269/// A caller can also export an array to a field that names its Vortex ID, such as a field from
270- /// [`Self::to_arrow_storage_field`]. The export then checks that an import of the result gives back
271- /// the same type. The field must have the same metadata, and its Arrow type must convert to the
272- /// storage type (or to the type itself, such as an Arrow timestamp). Without these checks, values
273- /// could silently change meaning, such as milliseconds that read back as seconds. For the same
274- /// reason, an extension array cannot export to a field that names a different known Vortex ID.
270+ /// [`Self::to_arrow_storage_field`]. The export then checks that the field describes the type of the
271+ /// array. The field must have the same metadata, and its Arrow type must import as the storage type
272+ /// (or as the type itself, such as an Arrow timestamp). Nested extension types count as their
273+ /// storage types in this check, and the export checks each nested field when it reaches it. Without
274+ /// these checks, values could silently change meaning, such as milliseconds that read back as
275+ /// seconds. For the same reason, an extension array cannot export to a field that names a different
276+ /// known Vortex ID.
275277///
276278/// On import, the plugins run first. If none applies, a field that names a known Vortex ID imports
277279/// as that type over its storage, and a field with any other name imports as plain storage. A
@@ -529,11 +531,7 @@ impl ArrowSession {
529531 . as_extension_opt ( )
530532 . filter ( |ext_dtype| ext_dtype. id ( ) == ext_dtype_plugin. id ( ) ) ;
531533 if let Some ( existing_ext_dtype) = existing_ext_dtype {
532- vortex_ensure ! (
533- metadata == existing_ext_dtype. serialize_metadata( ) ?,
534- "Arrow field {:?} must carry the metadata of {existing_ext_dtype}, got {metadata:?}" ,
535- field. name( )
536- ) ;
534+ ensure_ext_metadata ( field, & metadata, existing_ext_dtype) ?;
537535 return Ok ( None ) ;
538536 }
539537
@@ -969,10 +967,10 @@ impl ArrowSession {
969967 DataType :: Map ( entries_field, keys_sorted) => {
970968 let map = array. as_map ( ) ;
971969 let ( entries, offsets) = trim_map_entries ( map) ;
970+ // `map_from_arrow_parts` needs the entries as a plain struct, so a Vortex ID on the
971+ // entries field is ignored. Import plugins for its extension name still run, while
972+ // `from_arrow_datatype` ignores the entries field completely.
972973 let entries_array: ArrowArrayRef = Arc :: new ( entries) ;
973- // Map entries must import as a plain struct, so a Vortex ID on their field is
974- // ignored, as `from_arrow_datatype` also does. Wrapping the entries in that
975- // extension type would only make `map_from_arrow_parts` fail.
976974 let entries =
977975 self . from_arrow_array_with_importers ( entries_array, entries_field. as_ref ( ) ) ?;
978976 map_from_arrow_parts (
@@ -1130,8 +1128,8 @@ impl ArrowExporter<'_> {
11301128 }
11311129 }
11321130
1133- // The registry can hold an alias, so compare the array with the ID of the dtype plugin
1134- // that the extension name resolves to .
1131+ // The extension name can be an alias, which the dtype registry resolves to the Vortex ID
1132+ // of its type, so the array can match either of them .
11351133 let named_vortex_id = self
11361134 . session
11371135 . ext_dtypes
@@ -1149,9 +1147,9 @@ impl ArrowExporter<'_> {
11491147 return self . execute_arrow_ext_storage ( current, & ext_dtype, target_field, ctx) ;
11501148 }
11511149
1152- // Only an extension array fails on a target that names another Vortex ID. A plain
1153- // array still exports to it , because plugins export their storage to the target that
1154- // they receive .
1150+ // An extension array of another type must not export to a field that names this Vortex
1151+ // ID. A plain array can , because a plugin such as `Vector` exports its storage to the
1152+ // same field that it received .
11551153 if let ( Some ( ext_dtype) , Some ( named_vortex_id) ) =
11561154 ( current. dtype ( ) . as_extension_opt ( ) , named_vortex_id)
11571155 {
@@ -1182,25 +1180,21 @@ impl ArrowExporter<'_> {
11821180 target : & Field ,
11831181 ctx : & mut ExecutionCtx ,
11841182 ) -> VortexResult < ArrowArrayRef > {
1185- let target_metadata = decode_arrow_ext_metadata ( target) ?;
1186- vortex_ensure ! (
1187- target_metadata == ext_dtype. serialize_metadata( ) ?,
1188- "Arrow field {:?} must carry the metadata of {ext_dtype}, got {target_metadata:?}" ,
1189- target. name( )
1190- ) ;
1183+ ensure_ext_metadata ( target, & decode_arrow_ext_metadata ( target) ?, ext_dtype) ?;
11911184
11921185 let target_dtype = self
11931186 . session
11941187 . from_arrow_datatype ( target. data_type ( ) , ext_dtype. nullability ( ) ) ?;
11951188
1196- // An Arrow type that already holds the extension type, such as an Arrow timestamp, exports
1197- // the array itself, so that the executor of that type checks its own parameters.
1189+ // If the Arrow type of the target imports as the extension type itself, such as an Arrow
1190+ // timestamp, export the array to it. The executor of that Arrow type then checks the
1191+ // parameters of the type, such as the time unit of a timestamp.
11981192 if target_dtype. eq_ignore_nullability ( array. dtype ( ) ) {
11991193 return execute_arrow_naive ( array, Some ( target. data_type ( ) ) , self , ctx) ;
12001194 }
12011195
12021196 vortex_ensure ! (
1203- target_dtype . eq_ignore_nullability ( ext_dtype. storage_dtype( ) ) ,
1197+ eq_storage_layout ( & target_dtype , ext_dtype. storage_dtype( ) ) ,
12041198 "Arrow field {:?} must have an Arrow type that imports as the storage of {ext_dtype}, got {}" ,
12051199 target. name( ) ,
12061200 target. data_type( )
@@ -1215,6 +1209,65 @@ impl ArrowExporter<'_> {
12151209 }
12161210}
12171211
1212+ /// Fails unless `metadata`, the extension metadata of `field`, is the metadata of `ext_dtype`.
1213+ fn ensure_ext_metadata (
1214+ field : & Field ,
1215+ metadata : & [ u8 ] ,
1216+ ext_dtype : & ExtDTypeRef ,
1217+ ) -> VortexResult < ( ) > {
1218+ vortex_ensure ! (
1219+ metadata == ext_dtype. serialize_metadata( ) ?,
1220+ "Arrow field {:?} must carry the metadata of {ext_dtype}, got {metadata:?}" ,
1221+ field. name( )
1222+ ) ;
1223+
1224+ Ok ( ( ) )
1225+ }
1226+
1227+ /// Returns whether `lhs` and `rhs` have the same storage layout, which means that they are equal,
1228+ /// ignoring nullability, after every nested extension type is replaced by its storage type. The
1229+ /// types at the top level are compared as they are.
1230+ ///
1231+ /// An export compares the type that its target imports as with the storage type this way, because
1232+ /// an import can lose the metadata of a nested extension type, such as the version of a UUID, or
1233+ /// not know its Vortex ID. The export of each nested field checks that field when it reaches it.
1234+ fn eq_storage_layout ( lhs : & DType , rhs : & DType ) -> bool {
1235+ match ( lhs, rhs) {
1236+ ( DType :: List ( lhs, _) , DType :: List ( rhs, _) ) => eq_nested_storage_layout ( lhs, rhs) ,
1237+ ( DType :: FixedSizeList ( lhs, lhs_size, _) , DType :: FixedSizeList ( rhs, rhs_size, _) ) => {
1238+ lhs_size == rhs_size && eq_nested_storage_layout ( lhs, rhs)
1239+ }
1240+ ( DType :: Map ( lhs, _) , DType :: Map ( rhs, _) ) => {
1241+ lhs. keys_sorted ( ) == rhs. keys_sorted ( )
1242+ && eq_nested_storage_layout ( & lhs. key_dtype ( ) , & rhs. key_dtype ( ) )
1243+ && eq_nested_storage_layout ( & lhs. value_dtype ( ) , & rhs. value_dtype ( ) )
1244+ }
1245+ ( DType :: Struct ( lhs, _) , DType :: Struct ( rhs, _) ) => {
1246+ lhs. names ( ) == rhs. names ( )
1247+ && lhs
1248+ . fields ( )
1249+ . zip ( rhs. fields ( ) )
1250+ . all ( |( lhs, rhs) | eq_nested_storage_layout ( & lhs, & rhs) )
1251+ }
1252+ _ => lhs. eq_ignore_nullability ( rhs) ,
1253+ }
1254+ }
1255+
1256+ /// [`eq_storage_layout`] for two nested types, which replaces their own extension types by their
1257+ /// storage types too.
1258+ fn eq_nested_storage_layout ( lhs : & DType , rhs : & DType ) -> bool {
1259+ eq_storage_layout ( innermost_storage_dtype ( lhs) , innermost_storage_dtype ( rhs) )
1260+ }
1261+
1262+ /// Returns `dtype` without its extension types, which can wrap each other.
1263+ fn innermost_storage_dtype ( mut dtype : & DType ) -> & DType {
1264+ while let Some ( ext_dtype) = dtype. as_extension_opt ( ) {
1265+ dtype = ext_dtype. storage_dtype ( ) ;
1266+ }
1267+
1268+ dtype
1269+ }
1270+
12181271/// Extension trait for accessing the [`ArrowSession`] on a Vortex session.
12191272pub trait ArrowSessionExt : SessionExt {
12201273 /// Get the Arrow session.
0 commit comments