@@ -29,6 +29,7 @@ use arrow_array::Array as ArrowArray;
2929use arrow_array:: ArrayRef as ArrowArrayRef ;
3030use arrow_array:: RecordBatch ;
3131use arrow_array:: RunArray ;
32+ use arrow_array:: cast:: AsArray ;
3233use arrow_array:: make_array;
3334use arrow_array:: types:: Int16Type ;
3435use arrow_array:: types:: Int32Type ;
@@ -185,7 +186,10 @@ pub trait ArrowExportVTable: 'static + Send + Sync + Debug {
185186
186187/// Plugin layer for importing an Arrow extension-typed array into a Vortex array.
187188///
188- /// Plugins are dispatched by `arrow_ext_id`.
189+ /// Plugins are dispatched by `arrow_ext_id`. The session peels Arrow dictionaries before it
190+ /// dispatches, so a plugin never sees a [`DataType::Dictionary`] field or array. It sees the value
191+ /// field, with the dictionary field's metadata, and the values array. The session then wraps the
192+ /// plugin's result in a [`DictArray`] with the dictionary's codes.
189193///
190194/// This is purely an implementation trait, its methods should not be called directly. Instead,
191195/// use the methods on [`ArrowSession`].
@@ -391,6 +395,8 @@ impl ArrowSession {
391395
392396 /// Build the Vortex [`DType`] for an Arrow [`Field`].
393397 ///
398+ /// A [`DataType::Dictionary`] field converts as its value field with the same metadata.
399+ ///
394400 /// Plugins registered against the field's Arrow extension name are tried in
395401 /// registration order; the first plugin to return `Some(dtype)` wins. If none
396402 /// match (or all return `None`), the builtin `arrow.parquet.variant` extension maps
@@ -399,6 +405,11 @@ impl ArrowSession {
399405 /// fields is preserved.
400406 #[ expect( clippy:: disallowed_methods, reason = "interning a dynamic id" ) ]
401407 pub fn from_arrow_field ( & self , field : & Field ) -> VortexResult < DType > {
408+ if let DataType :: Dictionary ( _, value_type) = field. data_type ( ) {
409+ return self
410+ . from_arrow_field ( & field. clone ( ) . with_data_type ( value_type. as_ref ( ) . clone ( ) ) ) ;
411+ }
412+
402413 if let Some ( name) = field. metadata ( ) . get ( EXTENSION_TYPE_NAME_KEY ) {
403414 for plugin in self . importers ( & Id :: new ( name) ) . iter ( ) {
404415 if let Some ( dtype) = plugin. from_arrow_field ( field, self ) ? {
@@ -410,10 +421,6 @@ impl ArrowSession {
410421 return Ok ( DType :: Variant ( field. is_nullable ( ) . into ( ) ) ) ;
411422 }
412423 }
413- if let DataType :: Dictionary ( _, value_type) = field. data_type ( ) {
414- return self
415- . from_arrow_field ( & field. clone ( ) . with_data_type ( value_type. as_ref ( ) . clone ( ) ) ) ;
416- }
417424 self . from_arrow_datatype ( field. data_type ( ) , field. is_nullable ( ) . into ( ) )
418425 }
419426
@@ -596,6 +603,9 @@ impl ArrowSession {
596603 /// extension fields nested inside containers reach their importers; leaf types fall
597604 /// through to the canonical Arrow → Vortex array conversion.
598605 ///
606+ /// A dictionary array imports as a [`DictArray`]. Its values import as described above, with
607+ /// `field`'s metadata.
608+ ///
599609 /// # Errors
600610 ///
601611 /// Returns an error if the field (or requested nullability) is non-nullable but the array
@@ -609,15 +619,20 @@ impl ArrowSession {
609619 self . from_arrow_array_inner ( array, field. as_ref ( ) )
610620 }
611621
612- /// [`Self::from_arrow_array`] with the Arrow [`Field`] already resolved: probe the import
613- /// plugins registered for the field's extension name, then fall back to the canonical
614- /// conversion. Also the recursion point for nested fields, which already have a [`Field`].
622+ /// [`Self::from_arrow_array`] with the Arrow [`Field`] already resolved: peel a dictionary,
623+ /// probe the import plugins registered for the field's extension name, then fall back to the
624+ /// canonical conversion. Also the recursion point for nested fields, which already have a
625+ /// [`Field`].
615626 #[ allow( clippy:: wrong_self_convention) ]
616627 fn from_arrow_array_inner (
617628 & self ,
618629 array : ArrowArrayRef ,
619630 field : & Field ,
620631 ) -> VortexResult < ArrayRef > {
632+ if let DataType :: Dictionary ( ..) = field. data_type ( ) {
633+ return self . dictionary_from_arrow ( array. as_ref ( ) , field) ;
634+ }
635+
621636 if let Some ( extension_name) = field. metadata ( ) . get ( EXTENSION_TYPE_NAME_KEY ) {
622637 #[ expect( clippy:: disallowed_methods, reason = "interning a dynamic id" ) ]
623638 let importers = self . importers ( & Id :: new ( extension_name) ) ;
@@ -668,8 +683,6 @@ impl ArrowSession {
668683 array : & dyn ArrowArray ,
669684 field : & Field ,
670685 ) -> VortexResult < ArrayRef > {
671- use arrow_array:: cast:: AsArray ;
672-
673686 match field. data_type ( ) {
674687 DataType :: Struct ( fields) => {
675688 let arrow_struct = array. as_struct ( ) ;
@@ -790,20 +803,6 @@ impl ArrowSession {
790803 ) ,
791804 }
792805 }
793- DataType :: Dictionary ( ..) => {
794- let dict = array. as_any_dictionary ( ) ;
795- // Extension metadata on the dictionary field describes its values.
796- let values_field = field
797- . clone ( )
798- . with_data_type ( dict. values ( ) . data_type ( ) . clone ( ) ) ;
799- let values =
800- self . from_arrow_array ( ArrowArrayRef :: clone ( dict. values ( ) ) , & values_field) ?;
801- let codes = dict. keys ( ) ;
802- let codes = from_arrow_dyn ( codes, codes. is_nullable ( ) ) ?;
803- // SAFETY: arrow-rs enforces the dictionary invariants on construction, so the
804- // codes are in-bounds for the values.
805- Ok ( unsafe { DictArray :: new_unchecked ( codes, values) } . into_array ( ) )
806- }
807806 _ => from_arrow_dyn ( array, field. is_nullable ( ) ) ,
808807 }
809808 }
@@ -827,6 +826,34 @@ impl ArrowSession {
827826 self . from_arrow_array_inner ( ArrowArrayRef :: clone ( run_array. values ( ) ) , values_field) ?;
828827 run_end_from_arrow ( run_array, values)
829828 }
829+
830+ /// Decode an Arrow dictionary array into a [`DictArray`], importing its values with `field`'s
831+ /// metadata so that an extension name on the dictionary field reaches its importer.
832+ fn dictionary_from_arrow (
833+ & self ,
834+ array : & dyn ArrowArray ,
835+ field : & Field ,
836+ ) -> VortexResult < ArrayRef > {
837+ let dict = array. as_any_dictionary_opt ( ) . ok_or_else ( || {
838+ vortex_err ! (
839+ "expected an Arrow DictionaryArray, got {}" ,
840+ array. data_type( )
841+ )
842+ } ) ?;
843+
844+ let values_field = field
845+ . clone ( )
846+ . with_data_type ( dict. values ( ) . data_type ( ) . clone ( ) ) ;
847+ let values =
848+ self . from_arrow_array_inner ( ArrowArrayRef :: clone ( dict. values ( ) ) , & values_field) ?;
849+
850+ let codes = dict. keys ( ) ;
851+ let codes = from_arrow_dyn ( codes, codes. is_nullable ( ) ) ?;
852+
853+ // SAFETY: arrow-rs enforces the dictionary invariants on construction, so the codes are
854+ // in-bounds for the values.
855+ Ok ( unsafe { DictArray :: new_unchecked ( codes, values) } . into_array ( ) )
856+ }
830857}
831858
832859/// The values field of an Arrow [`DataType::RunEndEncoded`], re-stamped with the run-end array's
@@ -1072,24 +1099,76 @@ mod tests {
10721099 Ok ( ( ) )
10731100 }
10741101
1075- /// A plain Arrow dictionary imports as a Vortex `Dict` array over the dictionary values,
1076- /// matching the dtype the schema conversion reports.
1077- #[ test]
1078- fn dictionary_imports_as_dict_encoding ( ) -> VortexResult < ( ) > {
1102+ static REJECT_DICTIONARY_ID : CachedId = CachedId :: new ( "test.reject_dictionary" ) ;
1103+
1104+ /// An importer that fails on an Arrow dictionary and imports anything else as its storage.
1105+ #[ derive( Debug ) ]
1106+ struct DictionaryRejectingImporter ;
1107+
1108+ impl ArrowImportVTable for DictionaryRejectingImporter {
1109+ fn arrow_ext_id ( & self ) -> Id {
1110+ * REJECT_DICTIONARY_ID
1111+ }
1112+
1113+ fn from_arrow_field (
1114+ & self ,
1115+ field : & Field ,
1116+ session : & ArrowSession ,
1117+ ) -> VortexResult < Option < DType > > {
1118+ vortex_ensure ! (
1119+ !matches!( field. data_type( ) , DataType :: Dictionary ( ..) ) ,
1120+ "an importer must never see a dictionary field, got {}" ,
1121+ field. data_type( )
1122+ ) ;
1123+ session
1124+ . from_arrow_datatype ( field. data_type ( ) , field. is_nullable ( ) . into ( ) )
1125+ . map ( Some )
1126+ }
1127+
1128+ fn from_arrow_array (
1129+ & self ,
1130+ array : ArrowArrayRef ,
1131+ field : & Field ,
1132+ _dtype : & DType ,
1133+ session : & ArrowSession ,
1134+ ) -> VortexResult < ArrowImport > {
1135+ vortex_ensure ! (
1136+ !matches!( array. data_type( ) , DataType :: Dictionary ( ..) ) ,
1137+ "an importer must never see a dictionary array, got {}" ,
1138+ array. data_type( )
1139+ ) ;
1140+ session
1141+ . from_arrow_array ( array, field. is_nullable ( ) )
1142+ . map ( ArrowImport :: Imported )
1143+ }
1144+ }
1145+
1146+ /// An Arrow dictionary imports as a Vortex `Dict` array over the dictionary values, matching
1147+ /// the dtype the schema conversion reports. The session peels the dictionary before any
1148+ /// importer runs, so a dictionary field named for an importer that fails on dictionaries still
1149+ /// imports.
1150+ #[ rstest]
1151+ #[ case:: plain( None ) ]
1152+ #[ case:: extension_name( Some ( * REJECT_DICTIONARY_ID ) ) ]
1153+ fn dictionary_imports_as_dict_encoding ( #[ case] extension_name : Option < Id > ) -> VortexResult < ( ) > {
10791154 let session = ArrowSession :: default ( ) ;
1155+ session. register_importer ( Arc :: new ( DictionaryRejectingImporter ) ) ;
10801156 let dict: ArrowArrayRef = Arc :: new ( DictionaryArray :: < Int32Type > :: try_new (
10811157 Int32Array :: from ( vec ! [ 0 , 1 , 0 , 1 ] ) ,
10821158 Arc :: new ( StringArray :: from ( vec ! [ "a" , "b" ] ) ) ,
10831159 ) ?) ;
1084- let field = Field :: new ( "s" , dict. data_type ( ) . clone ( ) , false ) ;
1160+ let metadata = extension_name
1161+ . map ( |id| ( EXTENSION_TYPE_NAME_KEY . to_string ( ) , id. to_string ( ) ) )
1162+ . into_iter ( )
1163+ . collect ( ) ;
1164+ let field = Field :: new ( "s" , dict. data_type ( ) . clone ( ) , false ) . with_metadata ( metadata) ;
10851165
10861166 let dtype = session. from_arrow_field ( & field) ?;
10871167 assert_eq ! ( dtype, DType :: Utf8 ( Nullability :: NonNullable ) ) ;
10881168
10891169 let array = session. from_arrow_array ( dict, & field) ?;
10901170 assert ! ( array. is:: <Dict >( ) , "expected a Dict encoding, got {array}" ) ;
10911171 assert_eq ! ( array. dtype( ) , & dtype) ;
1092- assert_eq ! ( array. len( ) , 4 ) ;
10931172 assert_arrays_eq ! (
10941173 array,
10951174 VarBinViewArray :: from_iter_str( [ "a" , "b" , "a" , "b" ] ) ,
0 commit comments