Skip to content

Commit 976d21c

Browse files
committed
Compare nested extension types by storage in Vortex ID exports
Signed-off-by: Connor Tsui <[email protected]>
1 parent 1c60cac commit 976d21c

2 files changed

Lines changed: 151 additions & 30 deletions

File tree

‎vortex-arrow/src/session.rs‎

Lines changed: 80 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -267,11 +267,13 @@ type ArrowImporterRegistry = ArcSwapMap<Id, Arc<[ArrowImportVTableRef]>>;
267267
/// schemas, so [`ExtDTypeRef::serialize_metadata`] must give equal bytes for equal types.
268268
///
269269
/// A caller can also export an array to a field that names its Vortex ID, such as a field from
270-
/// [`Self::to_arrow_storage_field`]. The export then checks that an import of the result gives back
271-
/// the same type. The field must have the same metadata, and its Arrow type must convert to the
272-
/// storage type (or to the type itself, such as an Arrow timestamp). Without these checks, values
273-
/// could silently change meaning, such as milliseconds that read back as seconds. For the same
274-
/// reason, an extension array cannot export to a field that names a different known Vortex ID.
270+
/// [`Self::to_arrow_storage_field`]. The export then checks that the field describes the type of the
271+
/// array. The field must have the same metadata, and its Arrow type must import as the storage type
272+
/// (or as the type itself, such as an Arrow timestamp). Nested extension types count as their
273+
/// storage types in this check, and the export checks each nested field when it reaches it. Without
274+
/// these checks, values could silently change meaning, such as milliseconds that read back as
275+
/// seconds. For the same reason, an extension array cannot export to a field that names a different
276+
/// known Vortex ID.
275277
///
276278
/// On import, the plugins run first. If none applies, a field that names a known Vortex ID imports
277279
/// as that type over its storage, and a field with any other name imports as plain storage. A
@@ -529,11 +531,7 @@ impl ArrowSession {
529531
.as_extension_opt()
530532
.filter(|ext_dtype| ext_dtype.id() == ext_dtype_plugin.id());
531533
if let Some(existing_ext_dtype) = existing_ext_dtype {
532-
vortex_ensure!(
533-
metadata == existing_ext_dtype.serialize_metadata()?,
534-
"Arrow field {:?} must carry the metadata of {existing_ext_dtype}, got {metadata:?}",
535-
field.name()
536-
);
534+
ensure_ext_metadata(field, &metadata, existing_ext_dtype)?;
537535
return Ok(None);
538536
}
539537

@@ -969,10 +967,10 @@ impl ArrowSession {
969967
DataType::Map(entries_field, keys_sorted) => {
970968
let map = array.as_map();
971969
let (entries, offsets) = trim_map_entries(map);
970+
// `map_from_arrow_parts` needs the entries as a plain struct, so a Vortex ID on the
971+
// entries field is ignored. Import plugins for its extension name still run, while
972+
// `from_arrow_datatype` ignores the entries field completely.
972973
let entries_array: ArrowArrayRef = Arc::new(entries);
973-
// Map entries must import as a plain struct, so a Vortex ID on their field is
974-
// ignored, as `from_arrow_datatype` also does. Wrapping the entries in that
975-
// extension type would only make `map_from_arrow_parts` fail.
976974
let entries =
977975
self.from_arrow_array_with_importers(entries_array, entries_field.as_ref())?;
978976
map_from_arrow_parts(
@@ -1130,8 +1128,8 @@ impl ArrowExporter<'_> {
11301128
}
11311129
}
11321130

1133-
// The registry can hold an alias, so compare the array with the ID of the dtype plugin
1134-
// that the extension name resolves to.
1131+
// The extension name can be an alias, which the dtype registry resolves to the Vortex ID
1132+
// of its type, so the array can match either of them.
11351133
let named_vortex_id = self
11361134
.session
11371135
.ext_dtypes
@@ -1149,9 +1147,9 @@ impl ArrowExporter<'_> {
11491147
return self.execute_arrow_ext_storage(current, &ext_dtype, target_field, ctx);
11501148
}
11511149

1152-
// Only an extension array fails on a target that names another Vortex ID. A plain
1153-
// array still exports to it, because plugins export their storage to the target that
1154-
// they receive.
1150+
// An extension array of another type must not export to a field that names this Vortex
1151+
// ID. A plain array can, because a plugin such as `Vector` exports its storage to the
1152+
// same field that it received.
11551153
if let (Some(ext_dtype), Some(named_vortex_id)) =
11561154
(current.dtype().as_extension_opt(), named_vortex_id)
11571155
{
@@ -1182,25 +1180,21 @@ impl ArrowExporter<'_> {
11821180
target: &Field,
11831181
ctx: &mut ExecutionCtx,
11841182
) -> VortexResult<ArrowArrayRef> {
1185-
let target_metadata = decode_arrow_ext_metadata(target)?;
1186-
vortex_ensure!(
1187-
target_metadata == ext_dtype.serialize_metadata()?,
1188-
"Arrow field {:?} must carry the metadata of {ext_dtype}, got {target_metadata:?}",
1189-
target.name()
1190-
);
1183+
ensure_ext_metadata(target, &decode_arrow_ext_metadata(target)?, ext_dtype)?;
11911184

11921185
let target_dtype = self
11931186
.session
11941187
.from_arrow_datatype(target.data_type(), ext_dtype.nullability())?;
11951188

1196-
// An Arrow type that already holds the extension type, such as an Arrow timestamp, exports
1197-
// the array itself, so that the executor of that type checks its own parameters.
1189+
// If the Arrow type of the target imports as the extension type itself, such as an Arrow
1190+
// timestamp, export the array to it. The executor of that Arrow type then checks the
1191+
// parameters of the type, such as the time unit of a timestamp.
11981192
if target_dtype.eq_ignore_nullability(array.dtype()) {
11991193
return execute_arrow_naive(array, Some(target.data_type()), self, ctx);
12001194
}
12011195

12021196
vortex_ensure!(
1203-
target_dtype.eq_ignore_nullability(ext_dtype.storage_dtype()),
1197+
eq_storage_layout(&target_dtype, ext_dtype.storage_dtype()),
12041198
"Arrow field {:?} must have an Arrow type that imports as the storage of {ext_dtype}, got {}",
12051199
target.name(),
12061200
target.data_type()
@@ -1215,6 +1209,65 @@ impl ArrowExporter<'_> {
12151209
}
12161210
}
12171211

1212+
/// Fails unless `metadata`, the extension metadata of `field`, is the metadata of `ext_dtype`.
1213+
fn ensure_ext_metadata(
1214+
field: &Field,
1215+
metadata: &[u8],
1216+
ext_dtype: &ExtDTypeRef,
1217+
) -> VortexResult<()> {
1218+
vortex_ensure!(
1219+
metadata == ext_dtype.serialize_metadata()?,
1220+
"Arrow field {:?} must carry the metadata of {ext_dtype}, got {metadata:?}",
1221+
field.name()
1222+
);
1223+
1224+
Ok(())
1225+
}
1226+
1227+
/// Returns whether `lhs` and `rhs` have the same storage layout, which means that they are equal,
1228+
/// ignoring nullability, after every nested extension type is replaced by its storage type. The
1229+
/// types at the top level are compared as they are.
1230+
///
1231+
/// An export compares the type that its target imports as with the storage type this way, because
1232+
/// an import can lose the metadata of a nested extension type, such as the version of a UUID, or
1233+
/// not know its Vortex ID. The export of each nested field checks that field when it reaches it.
1234+
fn eq_storage_layout(lhs: &DType, rhs: &DType) -> bool {
1235+
match (lhs, rhs) {
1236+
(DType::List(lhs, _), DType::List(rhs, _)) => eq_nested_storage_layout(lhs, rhs),
1237+
(DType::FixedSizeList(lhs, lhs_size, _), DType::FixedSizeList(rhs, rhs_size, _)) => {
1238+
lhs_size == rhs_size && eq_nested_storage_layout(lhs, rhs)
1239+
}
1240+
(DType::Map(lhs, _), DType::Map(rhs, _)) => {
1241+
lhs.keys_sorted() == rhs.keys_sorted()
1242+
&& eq_nested_storage_layout(&lhs.key_dtype(), &rhs.key_dtype())
1243+
&& eq_nested_storage_layout(&lhs.value_dtype(), &rhs.value_dtype())
1244+
}
1245+
(DType::Struct(lhs, _), DType::Struct(rhs, _)) => {
1246+
lhs.names() == rhs.names()
1247+
&& lhs
1248+
.fields()
1249+
.zip(rhs.fields())
1250+
.all(|(lhs, rhs)| eq_nested_storage_layout(&lhs, &rhs))
1251+
}
1252+
_ => lhs.eq_ignore_nullability(rhs),
1253+
}
1254+
}
1255+
1256+
/// [`eq_storage_layout`] for two nested types, which replaces their own extension types by their
1257+
/// storage types too.
1258+
fn eq_nested_storage_layout(lhs: &DType, rhs: &DType) -> bool {
1259+
eq_storage_layout(innermost_storage_dtype(lhs), innermost_storage_dtype(rhs))
1260+
}
1261+
1262+
/// Returns `dtype` without its extension types, which can wrap each other.
1263+
fn innermost_storage_dtype(mut dtype: &DType) -> &DType {
1264+
while let Some(ext_dtype) = dtype.as_extension_opt() {
1265+
dtype = ext_dtype.storage_dtype();
1266+
}
1267+
1268+
dtype
1269+
}
1270+
12181271
/// Extension trait for accessing the [`ArrowSession`] on a Vortex session.
12191272
pub trait ArrowSessionExt: SessionExt {
12201273
/// Get the Arrow session.

‎vortex-arrow/tests/vortex_id.rs‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,16 +58,24 @@ use vortex_error::vortex_bail;
5858
use vortex_session::VortexSession;
5959
use vortex_session::registry::CachedId;
6060

61-
/// A session that can look up [`Tagged`] by its Vortex ID.
61+
/// A session that can look up [`Tagged`] by its Vortex ID and by [`TAGGED_ALIAS`].
6262
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
6363
let session = VortexSession::empty();
6464
session.dtypes().register(Tagged);
65+
session
66+
.dtypes()
67+
.registry()
68+
.insert(*TAGGED_ALIAS_ID, Arc::new(Tagged));
6569
vortex_arrow::initialize(&session);
6670
session
6771
});
6872

6973
static TAGGED_ID: CachedId = CachedId::new("test.tagged");
7074

75+
/// Another name that the dtype registry of [`SESSION`] resolves to [`Tagged`].
76+
const TAGGED_ALIAS: &str = "test.tagged.alias";
77+
static TAGGED_ALIAS_ID: CachedId = CachedId::new(TAGGED_ALIAS);
78+
7179
/// An extension type without an Arrow plugin. It accepts any storage, and its metadata is one tag
7280
/// byte.
7381
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -334,6 +342,66 @@ fn timestamp_round_trips_through_a_vortex_id_field(
334342
Ok(())
335343
}
336344

345+
/// A field that names an alias of a Vortex ID exports and imports like a field that names the ID.
346+
#[test]
347+
fn alias_of_vortex_id_round_trips() -> VortexResult<()> {
348+
let array = tagged_i64s(0xff)?;
349+
let mut field = storage_field_as(Ok(array.clone()), DataType::Int64)?;
350+
field
351+
.metadata_mut()
352+
.insert(EXTENSION_TYPE_NAME_KEY.to_owned(), TAGGED_ALIAS.to_owned());
353+
let mut ctx = SESSION.create_execution_ctx();
354+
355+
let exported = SESSION
356+
.arrow()
357+
.execute_arrow(array.clone(), Some(&field), &mut ctx)?;
358+
let imported = SESSION.arrow().from_arrow_array(exported, &field)?;
359+
assert_arrays_eq!(imported, array, &mut ctx);
360+
361+
Ok(())
362+
}
363+
364+
/// A plain array exports to a field that names a Vortex ID, because a plugin such as `Vector`
365+
/// exports its storage to the same field that it received.
366+
#[test]
367+
fn plain_array_exports_to_a_vortex_id_field() -> VortexResult<()> {
368+
let storage = PrimitiveArray::from_option_iter([Some(1i64), None, Some(3)]).into_array();
369+
let field = storage_field_as(tagged_i64s(0xff), DataType::Int64)?;
370+
let mut ctx = SESSION.create_execution_ctx();
371+
372+
let exported = SESSION
373+
.arrow()
374+
.execute_arrow(storage.clone(), Some(&field), &mut ctx)?;
375+
let plain_field = Field::new("t", DataType::Int64, true);
376+
let imported = SESSION.arrow().from_arrow_array(exported, &plain_field)?;
377+
assert_arrays_eq!(imported, storage, &mut ctx);
378+
379+
Ok(())
380+
}
381+
382+
/// The export to a field that names a Vortex ID compares nested extension types by their storage,
383+
/// because the import of the field may not restore them. Here the session does not know the nested
384+
/// Vortex ID, and an import plugin that loses metadata, such as the version of a UUID, has the same
385+
/// effect.
386+
#[test]
387+
fn export_compares_nested_extension_types_by_storage() -> VortexResult<()> {
388+
let array = tagged_array(0x01, struct_of_tagged()?)?;
389+
let field = SESSION
390+
.arrow()
391+
.to_arrow_storage_field("t", array.dtype().as_extension())?;
392+
let session = VortexSession::empty();
393+
vortex_arrow::initialize(&session);
394+
let mut ctx = session.create_execution_ctx();
395+
396+
let exported = session
397+
.arrow()
398+
.execute_arrow(array.clone(), Some(&field), &mut ctx)?;
399+
let imported = SESSION.arrow().from_arrow_array(exported, &field)?;
400+
assert_arrays_eq!(imported, array, &mut ctx);
401+
402+
Ok(())
403+
}
404+
337405
/// A session that does not know the Vortex ID imports the field as plain storage.
338406
#[test]
339407
fn unknown_vortex_id_imports_as_storage() -> VortexResult<()> {
@@ -426,8 +494,8 @@ fn run_end_field_that_names_its_type_twice_wraps_it_once() -> VortexResult<()> {
426494
Ok(())
427495
}
428496

429-
/// The entries field of a map is part of the map type, so a Vortex ID on it is ignored by both the
430-
/// dtype import and the array import.
497+
/// The entries field of a map is part of the map type, so the dtype import and the array import
498+
/// both ignore a Vortex ID on it.
431499
#[test]
432500
fn map_entries_ignore_a_vortex_id() -> VortexResult<()> {
433501
let key_field = Arc::new(Field::new("key", DataType::Int64, false));

0 commit comments

Comments
 (0)