Skip to content

Commit 8b01ab5

Browse files
authored
Peel Arrow dictionaries before Arrow importer plugins run (#11185)
## Summary - Tracking Issue: #11114 A dictionary-encoded extension column imports today only because its importer declines the `Dictionary` type, after which `ArrowSession` retries with the value type. The next PR in this stack (SN2 in #11114) makes the importers of known types fail on Arrow input outside their type instead of declining it, which would break these columns. So the session now peels the dictionary before any importer runs. The result is still a `Dict` over the imported values, so nothing is decompressed. ## Changes The existing dictionary test now also runs with an importer that fails on any dictionary. ## API Changes `ArrowImportVTable` implementations no longer receive `Dictionary` fields or arrays. They get the value field, with the dictionary field's metadata, and the values array. Note that a dictionary-encoded GeoArrow field with a CRS now keeps its CRS in the dtype from `ArrowSession::from_arrow_field` and `from_arrow_schema`, which matches what the array import already produced. Before, the name-only fallback of the spatial importers accepted the dictionary field itself and used default metadata. Signed-off-by: Connor Tsui <[email protected]>
1 parent 0e8442b commit 8b01ab5

1 file changed

Lines changed: 109 additions & 30 deletions

File tree

‎vortex-arrow/src/session.rs‎

Lines changed: 109 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use arrow_array::Array as ArrowArray;
2929
use arrow_array::ArrayRef as ArrowArrayRef;
3030
use arrow_array::RecordBatch;
3131
use arrow_array::RunArray;
32+
use arrow_array::cast::AsArray;
3233
use arrow_array::make_array;
3334
use arrow_array::types::Int16Type;
3435
use arrow_array::types::Int32Type;
@@ -185,7 +186,10 @@ pub trait ArrowExportVTable: 'static + Send + Sync + Debug {
185186

186187
/// Plugin layer for importing an Arrow extension-typed array into a Vortex array.
187188
///
188-
/// Plugins are dispatched by `arrow_ext_id`.
189+
/// Plugins are dispatched by `arrow_ext_id`. The session peels Arrow dictionaries before it
190+
/// dispatches, so a plugin never sees a [`DataType::Dictionary`] field or array. It sees the value
191+
/// field, with the dictionary field's metadata, and the values array. The session then wraps the
192+
/// plugin's result in a [`DictArray`] with the dictionary's codes.
189193
///
190194
/// This is purely an implementation trait, its methods should not be called directly. Instead,
191195
/// use the methods on [`ArrowSession`].
@@ -391,6 +395,8 @@ impl ArrowSession {
391395

392396
/// Build the Vortex [`DType`] for an Arrow [`Field`].
393397
///
398+
/// A [`DataType::Dictionary`] field converts as its value field with the same metadata.
399+
///
394400
/// Plugins registered against the field's Arrow extension name are tried in
395401
/// registration order; the first plugin to return `Some(dtype)` wins. If none
396402
/// match (or all return `None`), the builtin `arrow.parquet.variant` extension maps
@@ -399,6 +405,11 @@ impl ArrowSession {
399405
/// fields is preserved.
400406
#[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
401407
pub fn from_arrow_field(&self, field: &Field) -> VortexResult<DType> {
408+
if let DataType::Dictionary(_, value_type) = field.data_type() {
409+
return self
410+
.from_arrow_field(&field.clone().with_data_type(value_type.as_ref().clone()));
411+
}
412+
402413
if let Some(name) = field.metadata().get(EXTENSION_TYPE_NAME_KEY) {
403414
for plugin in self.importers(&Id::new(name)).iter() {
404415
if let Some(dtype) = plugin.from_arrow_field(field, self)? {
@@ -410,10 +421,6 @@ impl ArrowSession {
410421
return Ok(DType::Variant(field.is_nullable().into()));
411422
}
412423
}
413-
if let DataType::Dictionary(_, value_type) = field.data_type() {
414-
return self
415-
.from_arrow_field(&field.clone().with_data_type(value_type.as_ref().clone()));
416-
}
417424
self.from_arrow_datatype(field.data_type(), field.is_nullable().into())
418425
}
419426

@@ -596,6 +603,9 @@ impl ArrowSession {
596603
/// extension fields nested inside containers reach their importers; leaf types fall
597604
/// through to the canonical Arrow → Vortex array conversion.
598605
///
606+
/// A dictionary array imports as a [`DictArray`]. Its values import as described above, with
607+
/// `field`'s metadata.
608+
///
599609
/// # Errors
600610
///
601611
/// Returns an error if the field (or requested nullability) is non-nullable but the array
@@ -609,15 +619,20 @@ impl ArrowSession {
609619
self.from_arrow_array_inner(array, field.as_ref())
610620
}
611621

612-
/// [`Self::from_arrow_array`] with the Arrow [`Field`] already resolved: probe the import
613-
/// plugins registered for the field's extension name, then fall back to the canonical
614-
/// conversion. Also the recursion point for nested fields, which already have a [`Field`].
622+
/// [`Self::from_arrow_array`] with the Arrow [`Field`] already resolved: peel a dictionary,
623+
/// probe the import plugins registered for the field's extension name, then fall back to the
624+
/// canonical conversion. Also the recursion point for nested fields, which already have a
625+
/// [`Field`].
615626
#[allow(clippy::wrong_self_convention)]
616627
fn from_arrow_array_inner(
617628
&self,
618629
array: ArrowArrayRef,
619630
field: &Field,
620631
) -> VortexResult<ArrayRef> {
632+
if let DataType::Dictionary(..) = field.data_type() {
633+
return self.dictionary_from_arrow(array.as_ref(), field);
634+
}
635+
621636
if let Some(extension_name) = field.metadata().get(EXTENSION_TYPE_NAME_KEY) {
622637
#[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
623638
let importers = self.importers(&Id::new(extension_name));
@@ -668,8 +683,6 @@ impl ArrowSession {
668683
array: &dyn ArrowArray,
669684
field: &Field,
670685
) -> VortexResult<ArrayRef> {
671-
use arrow_array::cast::AsArray;
672-
673686
match field.data_type() {
674687
DataType::Struct(fields) => {
675688
let arrow_struct = array.as_struct();
@@ -790,20 +803,6 @@ impl ArrowSession {
790803
),
791804
}
792805
}
793-
DataType::Dictionary(..) => {
794-
let dict = array.as_any_dictionary();
795-
// Extension metadata on the dictionary field describes its values.
796-
let values_field = field
797-
.clone()
798-
.with_data_type(dict.values().data_type().clone());
799-
let values =
800-
self.from_arrow_array(ArrowArrayRef::clone(dict.values()), &values_field)?;
801-
let codes = dict.keys();
802-
let codes = from_arrow_dyn(codes, codes.is_nullable())?;
803-
// SAFETY: arrow-rs enforces the dictionary invariants on construction, so the
804-
// codes are in-bounds for the values.
805-
Ok(unsafe { DictArray::new_unchecked(codes, values) }.into_array())
806-
}
807806
_ => from_arrow_dyn(array, field.is_nullable()),
808807
}
809808
}
@@ -827,6 +826,34 @@ impl ArrowSession {
827826
self.from_arrow_array_inner(ArrowArrayRef::clone(run_array.values()), values_field)?;
828827
run_end_from_arrow(run_array, values)
829828
}
829+
830+
/// Decode an Arrow dictionary array into a [`DictArray`], importing its values with `field`'s
831+
/// metadata so that an extension name on the dictionary field reaches its importer.
832+
fn dictionary_from_arrow(
833+
&self,
834+
array: &dyn ArrowArray,
835+
field: &Field,
836+
) -> VortexResult<ArrayRef> {
837+
let dict = array.as_any_dictionary_opt().ok_or_else(|| {
838+
vortex_err!(
839+
"expected an Arrow DictionaryArray, got {}",
840+
array.data_type()
841+
)
842+
})?;
843+
844+
let values_field = field
845+
.clone()
846+
.with_data_type(dict.values().data_type().clone());
847+
let values =
848+
self.from_arrow_array_inner(ArrowArrayRef::clone(dict.values()), &values_field)?;
849+
850+
let codes = dict.keys();
851+
let codes = from_arrow_dyn(codes, codes.is_nullable())?;
852+
853+
// SAFETY: arrow-rs enforces the dictionary invariants on construction, so the codes are
854+
// in-bounds for the values.
855+
Ok(unsafe { DictArray::new_unchecked(codes, values) }.into_array())
856+
}
830857
}
831858

832859
/// The values field of an Arrow [`DataType::RunEndEncoded`], re-stamped with the run-end array's
@@ -1072,24 +1099,76 @@ mod tests {
10721099
Ok(())
10731100
}
10741101

1075-
/// A plain Arrow dictionary imports as a Vortex `Dict` array over the dictionary values,
1076-
/// matching the dtype the schema conversion reports.
1077-
#[test]
1078-
fn dictionary_imports_as_dict_encoding() -> VortexResult<()> {
1102+
static REJECT_DICTIONARY_ID: CachedId = CachedId::new("test.reject_dictionary");
1103+
1104+
/// An importer that fails on an Arrow dictionary and imports anything else as its storage.
1105+
#[derive(Debug)]
1106+
struct DictionaryRejectingImporter;
1107+
1108+
impl ArrowImportVTable for DictionaryRejectingImporter {
1109+
fn arrow_ext_id(&self) -> Id {
1110+
*REJECT_DICTIONARY_ID
1111+
}
1112+
1113+
fn from_arrow_field(
1114+
&self,
1115+
field: &Field,
1116+
session: &ArrowSession,
1117+
) -> VortexResult<Option<DType>> {
1118+
vortex_ensure!(
1119+
!matches!(field.data_type(), DataType::Dictionary(..)),
1120+
"an importer must never see a dictionary field, got {}",
1121+
field.data_type()
1122+
);
1123+
session
1124+
.from_arrow_datatype(field.data_type(), field.is_nullable().into())
1125+
.map(Some)
1126+
}
1127+
1128+
fn from_arrow_array(
1129+
&self,
1130+
array: ArrowArrayRef,
1131+
field: &Field,
1132+
_dtype: &DType,
1133+
session: &ArrowSession,
1134+
) -> VortexResult<ArrowImport> {
1135+
vortex_ensure!(
1136+
!matches!(array.data_type(), DataType::Dictionary(..)),
1137+
"an importer must never see a dictionary array, got {}",
1138+
array.data_type()
1139+
);
1140+
session
1141+
.from_arrow_array(array, field.is_nullable())
1142+
.map(ArrowImport::Imported)
1143+
}
1144+
}
1145+
1146+
/// An Arrow dictionary imports as a Vortex `Dict` array over the dictionary values, matching
1147+
/// the dtype the schema conversion reports. The session peels the dictionary before any
1148+
/// importer runs, so a dictionary field named for an importer that fails on dictionaries still
1149+
/// imports.
1150+
#[rstest]
1151+
#[case::plain(None)]
1152+
#[case::extension_name(Some(*REJECT_DICTIONARY_ID))]
1153+
fn dictionary_imports_as_dict_encoding(#[case] extension_name: Option<Id>) -> VortexResult<()> {
10791154
let session = ArrowSession::default();
1155+
session.register_importer(Arc::new(DictionaryRejectingImporter));
10801156
let dict: ArrowArrayRef = Arc::new(DictionaryArray::<Int32Type>::try_new(
10811157
Int32Array::from(vec![0, 1, 0, 1]),
10821158
Arc::new(StringArray::from(vec!["a", "b"])),
10831159
)?);
1084-
let field = Field::new("s", dict.data_type().clone(), false);
1160+
let metadata = extension_name
1161+
.map(|id| (EXTENSION_TYPE_NAME_KEY.to_string(), id.to_string()))
1162+
.into_iter()
1163+
.collect();
1164+
let field = Field::new("s", dict.data_type().clone(), false).with_metadata(metadata);
10851165

10861166
let dtype = session.from_arrow_field(&field)?;
10871167
assert_eq!(dtype, DType::Utf8(Nullability::NonNullable));
10881168

10891169
let array = session.from_arrow_array(dict, &field)?;
10901170
assert!(array.is::<Dict>(), "expected a Dict encoding, got {array}");
10911171
assert_eq!(array.dtype(), &dtype);
1092-
assert_eq!(array.len(), 4);
10931172
assert_arrays_eq!(
10941173
array,
10951174
VarBinViewArray::from_iter_str(["a", "b", "a", "b"]),

0 commit comments

Comments
 (0)