Skip to content

Commit d7aa24e

Browse files
authored
Make successful Arrow plugin imports final (#11464)
## Summary TurboQuant stores its Arrow metadata as base64 without an encoding marker. Vortex's fallback for registered Vortex IDs decodes base64 only when marked, so it reads TurboQuant's metadata as literal text and rejects valid arrays, even after a plugin has imported them successfully. An exporter that declines and delegates to storage conversion can hit the same check. ## Changes Successful plugin imports are final because the existing dtype and length checks already validate their result. Export fallback asks the registered importer to decode the field before comparing types, since the same metadata can use different byte encodings. The complete dtype check, apart from outer nullability, and the physical storage check remain. Without a matching importer, the field must contain the source type's serialized metadata. Map entries import as a plain struct because the map's dtype comes from its key and value fields. Running a plugin on the entries field could produce a different dtype from schema inference. Key and value plugins still run. Tests cover native metadata imports and export round trips, invalid metadata, and physical storage mismatches. A byte-collision case checks that equal metadata bytes cannot hide different type parameters. Map tests cover ignored entries plugins and length checks on value imports. ## API Changes Import plugins no longer run on a map's entries field. Extension types on the key and value fields still reach their plugins. Signed-off-by: Connor Tsui <[email protected]>
1 parent fc93194 commit d7aa24e

4 files changed

Lines changed: 213 additions & 59 deletions

File tree

‎vortex-arrow/src/executor/mod.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,20 @@ fn execute_arrow_ext_storage(
269269
options: &ArrowExportOptions,
270270
ctx: &mut ExecutionCtx,
271271
) -> VortexResult<ArrowArrayRef> {
272-
ensure_ext_metadata(target, &decode_arrow_ext_metadata(target)?, ext_dtype)?;
272+
if let Some(importer_dtype) = ctx
273+
.session()
274+
.arrow()
275+
.infer_field_dtype_with_importers(target)?
276+
{
277+
vortex_ensure_eq!(
278+
importer_dtype.as_nullable(),
279+
array.dtype().as_nullable(),
280+
"Arrow field {:?} must describe the source extension type",
281+
target.name()
282+
);
283+
} else {
284+
ensure_ext_metadata(target, &decode_arrow_ext_metadata(target)?, ext_dtype)?;
285+
}
273286

274287
let target_dtype = ctx
275288
.session()

‎vortex-arrow/src/extension/tests.rs‎

Lines changed: 72 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,18 @@ use std::sync::Arc;
55

66
use arrow_array::Int64Array;
77
use arrow_schema::DataType;
8+
use arrow_schema::extension::EXTENSION_TYPE_METADATA_KEY;
89
use rstest::rstest;
910
use vortex_array::IntoArray;
11+
use vortex_array::VortexSessionExecute;
12+
use vortex_array::arrays::ExtensionArray;
1013
use vortex_array::arrays::PrimitiveArray;
14+
use vortex_array::assert_arrays_eq;
1115
use vortex_array::dtype::DType;
1216
use vortex_array::dtype::PType;
1317
use vortex_array::dtype::session::DTypeSessionExt;
1418
use vortex_array::scalar::ScalarValue;
19+
use vortex_error::VortexError;
1520
use vortex_error::vortex_bail;
1621
use vortex_session::VortexSession;
1722
use vortex_session::registry::CachedId;
@@ -20,10 +25,11 @@ use super::*;
2025
use crate::ArrowExportVTable;
2126
use crate::ArrowImportVTable;
2227
use crate::ArrowSessionExt;
28+
use crate::ExecuteArrowExt;
2329
use crate::ext_metadata::VORTEX_EXTENSION_METADATA_KEY;
2430
use crate::ext_metadata::set_arrow_extension_name;
2531

26-
/// Exports a tagged field through its registered policy and checks its complete dtype on import.
32+
/// Checks a tagged field's complete dtype and round-trips its values through the registered policy.
2733
#[track_caller]
2834
fn round_trip_field<const PRESERVE_VORTEX: bool>(tag: u8, nullable: bool) -> VortexResult<Field> {
2935
let session = tagged_session::<PRESERVE_VORTEX>();
@@ -33,6 +39,22 @@ fn round_trip_field<const PRESERVE_VORTEX: bool>(tag: u8, nullable: bool) -> Vor
3339
let field = arrow.to_arrow_field("tag", &dtype)?;
3440
assert_eq!(arrow.from_arrow_field(&field)?, dtype);
3541

42+
let storage = if nullable {
43+
PrimitiveArray::from_option_iter([Some(1i64)]).into_array()
44+
} else {
45+
PrimitiveArray::from_iter([1i64]).into_array()
46+
};
47+
let expected = ExtensionArray::try_new(dtype.as_extension().clone(), storage)?.into_array();
48+
let mut ctx = session.create_execution_ctx();
49+
50+
let array: ArrowArrayRef = Arc::new(Int64Array::from(vec![1]));
51+
let imported = arrow.from_arrow_array(array, &field)?;
52+
assert_arrays_eq!(imported, expected, &mut ctx);
53+
54+
let exported = expected.clone().execute_arrow(Some(&field), &mut ctx)?;
55+
let round_tripped = arrow.from_arrow_array(exported, &field)?;
56+
assert_arrays_eq!(round_tripped, expected, &mut ctx);
57+
3658
Ok(field)
3759
}
3860

@@ -56,7 +78,9 @@ fn raw_tagged_session() -> VortexSession {
5678
session
5779
}
5880

59-
/// The native field infers tag 255. Tag zero deliberately has an empty serialized payload.
81+
/// The native field always carries the decimal text `255`, so it can express no other tag. This
82+
/// text differs from the one-byte wire metadata of the same tag, as a real plugin's native metadata
83+
/// can. Tag zero deliberately has an empty serialized payload.
6084
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
6185
struct Tagged<const PRESERVE_VORTEX: bool>;
6286

@@ -139,6 +163,9 @@ impl<const PRESERVE_VORTEX: bool> ArrowExtVTable for Tagged<PRESERVE_VORTEX> {
139163
ext_dtype.storage_dtype().is_nullable(),
140164
);
141165
set_arrow_extension_name(&mut field, self.arrow_ext_id().to_string());
166+
field
167+
.metadata_mut()
168+
.insert(EXTENSION_TYPE_METADATA_KEY.to_owned(), "255".to_owned());
142169

143170
Ok(Some(field))
144171
}
@@ -157,16 +184,25 @@ impl<const PRESERVE_VORTEX: bool> ArrowExtVTable for Tagged<PRESERVE_VORTEX> {
157184
field: &Field,
158185
_session: &ArrowSession,
159186
) -> VortexResult<Option<ExtDType<Self>>> {
160-
Ok(Some(Self::dtype(255, field.is_nullable())?))
187+
let tag = field
188+
.metadata()
189+
.get(EXTENSION_TYPE_METADATA_KEY)
190+
.map_or(Ok(255), |tag| tag.parse::<u8>())
191+
.map_err(|error| VortexError::external(error).with_context("invalid native tag"))?;
192+
193+
Ok(Some(Self::dtype(tag, field.is_nullable())?))
161194
}
162195

163196
fn from_arrow_array(
164197
array: ArrowArrayRef,
165-
_field: &Field,
166-
_ext_dtype: &ExtDType<Self>,
167-
_session: &ArrowSession,
198+
field: &Field,
199+
ext_dtype: &ExtDType<Self>,
200+
session: &ArrowSession,
168201
) -> VortexResult<ArrowImport> {
169-
Ok(ArrowImport::Unsupported(array))
202+
let storage = session.from_arrow_array(array, field.is_nullable())?;
203+
let imported = ExtensionArray::try_new(ext_dtype.clone().erased(), storage)?.into_array();
204+
205+
Ok(ArrowImport::Imported(imported))
170206
}
171207
}
172208

@@ -266,6 +302,35 @@ fn policy_preserves_dtype(
266302
Ok(())
267303
}
268304

305+
#[rstest]
306+
// Native text "7" means tag 7, but its byte is the Vortex wire encoding of tag 55.
307+
#[case::different_native_parameter(b'7', DataType::Int64, "7", "must describe")]
308+
#[case::different_physical_storage(255, DataType::Int32, "255", "must have an Arrow type")]
309+
fn declined_export_rejects_incompatible_target(
310+
#[case] tag: u8,
311+
#[case] data_type: DataType,
312+
#[case] native_metadata: &str,
313+
#[case] message: &str,
314+
) -> VortexResult<()> {
315+
let session = tagged_session::<false>();
316+
let dtype = Tagged::<false>::dtype(tag, false)?.erased();
317+
let array = ExtensionArray::try_new(dtype, PrimitiveArray::from_iter([1i64]).into_array())?
318+
.into_array();
319+
let mut field = Field::new("tag", data_type, false);
320+
set_arrow_extension_name(&mut field, Tagged::<false>.id().to_string());
321+
field.metadata_mut().insert(
322+
EXTENSION_TYPE_METADATA_KEY.to_owned(),
323+
native_metadata.to_owned(),
324+
);
325+
326+
let error = array
327+
.execute_arrow(Some(&field), &mut session.create_execution_ctx())
328+
.expect_err("native parameter and physical storage mismatches must be rejected");
329+
assert!(error.to_string().contains(message), "{error}");
330+
331+
Ok(())
332+
}
333+
269334
#[test]
270335
fn lossy_native_policy_is_rejected() -> VortexResult<()> {
271336
let error =

0 commit comments

Comments
 (0)