Skip to content

Commit 7854a19

Browse files
authored
Mark Vortex ID metadata as base64 (#11300)
## Summary - Tracking Issue: #11114 Since #11224, the metadata of an extension type without an Arrow plugin is unmarked base64, but other Arrow plugins write the same keys with plain text metadata. So `"/w=="` means the byte `0xff` to Vortex and the four bytes `/w==` to them. 0.92.0 and 0.93.0 already write this format, and every release that writes it into Parquet or IPC files makes it harder to change. ## Changes The metadata now carries `vortex:extension:metadata:encoding = base64`, as #10651 did, and unmarked metadata imports as its text. Writing text whenever the bytes are valid UTF-8 was rejected, because binary metadata such as `02 00 00` would then be written as text. Note that fields written by 0.92.0 and 0.93.0 no longer import. ## API Changes - New `vortex_arrow::ext_metadata` module and `ArrowSession::to_arrow_storage_field`. - Exported metadata gains the encoding key, and unmarked metadata imports as its text. Signed-off-by: Connor Tsui <[email protected]>
1 parent c1439e6 commit 7854a19

4 files changed

Lines changed: 161 additions & 71 deletions

File tree

‎vortex-arrow/src/ext_metadata.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! The Arrow field metadata that holds a Vortex extension type without an Arrow plugin.
5+
//!
6+
//! Arrow field metadata must be UTF-8, but Vortex metadata is arbitrary bytes, so the metadata is
7+
//! always base64, and [`EXTENSION_METADATA_ENCODING_KEY`] marks it as base64. The marker is needed
8+
//! because other producers write the same Arrow extension keys with plain text metadata.
9+
10+
use arrow_schema::Field;
11+
use arrow_schema::extension::EXTENSION_TYPE_METADATA_KEY;
12+
use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
13+
use base64::Engine;
14+
use base64::prelude::BASE64_STANDARD;
15+
use vortex_array::dtype::extension::ExtDTypeRef;
16+
use vortex_error::VortexResult;
17+
use vortex_error::vortex_bail;
18+
use vortex_error::vortex_err;
19+
20+
/// The Arrow field metadata key that names the encoding of `ARROW:extension:metadata`.
21+
///
22+
/// The only encoding is `base64`. Without this key, the metadata is its own UTF-8 bytes.
23+
pub const EXTENSION_METADATA_ENCODING_KEY: &str = "vortex:extension:metadata:encoding";
24+
25+
/// The value of [`EXTENSION_METADATA_ENCODING_KEY`] for standard padded base64.
26+
const BASE64_ENCODING: &str = "base64";
27+
28+
/// Returns the Arrow field metadata that holds `ext_dtype`.
29+
pub fn encode_arrow_ext_metadata(ext_dtype: &ExtDTypeRef) -> VortexResult<[(String, String); 3]> {
30+
let metadata = BASE64_STANDARD.encode(ext_dtype.serialize_metadata()?);
31+
32+
Ok([
33+
(
34+
EXTENSION_TYPE_NAME_KEY.to_owned(),
35+
ext_dtype.id().to_string(),
36+
),
37+
(EXTENSION_TYPE_METADATA_KEY.to_owned(), metadata),
38+
(
39+
EXTENSION_METADATA_ENCODING_KEY.to_owned(),
40+
BASE64_ENCODING.to_owned(),
41+
),
42+
])
43+
}
44+
45+
/// Returns the extension metadata bytes that `field` holds, which are empty if it has none.
46+
///
47+
/// An unmarked value is its own UTF-8 bytes. Fails if the encoding is not `base64`, or if the value
48+
/// is not valid base64.
49+
pub fn decode_arrow_ext_metadata(field: &Field) -> VortexResult<Vec<u8>> {
50+
let Some(metadata) = field.extension_type_metadata() else {
51+
return Ok(Vec::new());
52+
};
53+
54+
match field
55+
.metadata()
56+
.get(EXTENSION_METADATA_ENCODING_KEY)
57+
.map(String::as_str)
58+
{
59+
None => Ok(metadata.as_bytes().to_vec()),
60+
Some(BASE64_ENCODING) => BASE64_STANDARD.decode(metadata).map_err(|err| {
61+
vortex_err!(
62+
"Arrow field {:?} must carry base64 extension metadata, got {metadata:?}: {err}",
63+
field.name()
64+
)
65+
}),
66+
Some(encoding) => vortex_bail!(
67+
"Arrow field {:?} must have the extension metadata encoding {BASE64_ENCODING:?}, got {encoding:?}",
68+
field.name()
69+
),
70+
}
71+
}

‎vortex-arrow/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ pub mod convert;
2525
mod datum;
2626
pub mod dtype;
2727
mod executor;
28+
pub mod ext_metadata;
2829
mod iter;
2930
mod null_buffer;
3031
mod options;

‎vortex-arrow/src/session.rs‎

Lines changed: 32 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,7 @@ use arrow_schema::Field;
4343
use arrow_schema::FieldRef;
4444
use arrow_schema::Fields;
4545
use arrow_schema::Schema;
46-
use arrow_schema::extension::EXTENSION_TYPE_METADATA_KEY;
4746
use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
48-
use base64::Engine;
49-
use base64::prelude::BASE64_STANDARD;
5047
use tracing::debug;
5148
use tracing::trace;
5249
use vortex_array::ArrayRef;
@@ -93,6 +90,8 @@ use crate::convert::trim_offsets;
9390
use crate::dtype::from_arrow_data_type;
9491
use crate::dtype::to_data_type_naive;
9592
use crate::executor::execute_arrow_naive;
93+
use crate::ext_metadata::decode_arrow_ext_metadata;
94+
use crate::ext_metadata::encode_arrow_ext_metadata;
9695
use crate::run_end_import::run_end_from_arrow;
9796

9897
/// Outcome of a successful call to [`ArrowExportVTable::execute_arrow`].
@@ -265,27 +264,27 @@ type ArrowImporterRegistry = ArcSwapMap<Id, Arc<[ArrowImportVTableRef]>>;
265264
/// extension type (for example `vortex-uuid`); temporal extensions are handled by the canonical
266265
/// Arrow ↔ Vortex path and do not need plugins.
267266
///
268-
/// An extension type that does not have an exporter exports its storage field, with two Arrow
269-
/// metadata keys added: `ARROW:extension:name` holds its Vortex ID, and `ARROW:extension:metadata`
270-
/// holds its serialized metadata in standard padded base64. The metadata is base64 because Arrow
271-
/// field metadata must be UTF-8, which is also how Parquet stores `ARROW:schema`. DataFusion
272-
/// compares these strings when it merges file schemas, so [`ExtDTypeRef::serialize_metadata`] must
273-
/// give equal bytes for equal dtypes. The export fails if the storage field already has an Arrow
274-
/// extension name, because one field cannot have two.
267+
/// An extension type that does not have an exporter exports its storage field, with field metadata
268+
/// that names its Vortex ID and holds its serialized metadata ([`ext_metadata`] describes the
269+
/// format). DataFusion compares this metadata when it merges file schemas, so
270+
/// [`ExtDTypeRef::serialize_metadata`] must give equal bytes for equal dtypes. The export fails if
271+
/// the storage field already has an Arrow extension name, because one field cannot have two.
275272
///
276-
/// A caller can also export to such a field directly. If no exporter accepts the target, but its
277-
/// extension name is the Vortex ID of the array, the array exports its storage. This works for
278-
/// every extension type, so a caller can export a timestamp column as `Int64` without losing its
279-
/// type. However, the target must have the same metadata as the array, because different metadata
280-
/// would give the values a different meaning, such as a different time unit for a timestamp. To
281-
/// check this, a session that knows the Vortex ID rebuilds the dtype from the target and compares
282-
/// the dtypes, and any other session compares the metadata bytes.
273+
/// A caller can also export to such a field directly, for example one from
274+
/// [`Self::to_arrow_storage_field`]. If no exporter accepts the target, but its extension name is
275+
/// the Vortex ID of the array, the array exports its storage. This works for every extension type,
276+
/// so a caller can export a timestamp column as `Int64` without losing its type. However, the
277+
/// target must have the same metadata as the array, because different metadata would give the
278+
/// values a different meaning, such as a different time unit for a timestamp. To check this, a
279+
/// session that knows the Vortex ID rebuilds the dtype from the target and compares the dtypes, and
280+
/// any other session compares the metadata bytes.
283281
///
284282
/// On import, the importers run first. If none of them accepts the field, but its extension name is
285283
/// a Vortex ID that the session knows (see [`initialize`]), the field imports as that extension
286284
/// type over its storage. In that case, the import fails if the type rejects the metadata or the
287285
/// storage. A field with any other extension name imports as plain storage.
288286
///
287+
/// [`ext_metadata`]: crate::ext_metadata
289288
/// [`initialize`]: crate::initialize
290289
#[derive(Clone, Debug, Default)]
291290
pub struct ArrowSession {
@@ -387,7 +386,7 @@ impl ArrowSession {
387386
DType::Extension(ext) if !ext.is::<AnyTemporal>() => {
388387
match self.to_arrow_field_with_exporter(name, ext)? {
389388
Some(field) => Ok(field),
390-
None => self.to_arrow_field_without_exporter(name, ext),
389+
None => self.to_arrow_storage_field(name, ext),
391390
}
392391
}
393392
DType::Variant(_) => {
@@ -486,9 +485,16 @@ impl ArrowSession {
486485
Ok(None)
487486
}
488487

489-
/// Build the storage field of an extension type without an exporter, and add the Arrow
490-
/// extension metadata that holds its Vortex ID and its metadata.
491-
fn to_arrow_field_without_exporter(
488+
/// Returns the storage field of `ext_dtype`, with field metadata that holds the extension type
489+
/// ([`ext_metadata`] describes the format).
490+
///
491+
/// A type with its own Arrow type can also export to this field, for example to export a
492+
/// timestamp column as `Int64` and import it back as a timestamp.
493+
///
494+
/// Fails if the storage field already has an Arrow extension name.
495+
///
496+
/// [`ext_metadata`]: crate::ext_metadata
497+
pub fn to_arrow_storage_field(
492498
&self,
493499
name: &str,
494500
ext_dtype: &ExtDTypeRef,
@@ -500,14 +506,9 @@ impl ArrowSession {
500506
);
501507
}
502508

503-
let metadata = BASE64_STANDARD.encode(ext_dtype.serialize_metadata()?);
504-
field.metadata_mut().extend([
505-
(
506-
EXTENSION_TYPE_NAME_KEY.to_owned(),
507-
ext_dtype.id().to_string(),
508-
),
509-
(EXTENSION_TYPE_METADATA_KEY.to_owned(), metadata),
510-
]);
509+
field
510+
.metadata_mut()
511+
.extend(encode_arrow_ext_metadata(ext_dtype)?);
511512

512513
Ok(field)
513514
}
@@ -531,7 +532,7 @@ impl ArrowSession {
531532
};
532533

533534
dtype_plugin
534-
.deserialize(&decode_ext_metadata(field)?, storage_dtype.clone())
535+
.deserialize(&decode_arrow_ext_metadata(field)?, storage_dtype.clone())
535536
.map(Some)
536537
}
537538

@@ -1036,20 +1037,6 @@ fn run_end_values_field(values_field: &FieldRef, nullability: Nullability) -> Fi
10361037
.with_nullable(nullability.into())
10371038
}
10381039

1039-
/// Decode the base64 `ARROW:extension:metadata` of a field whose extension name is a Vortex ID.
1040-
///
1041-
/// A field without that key has empty metadata, as Arrow allows.
1042-
fn decode_ext_metadata(field: &Field) -> VortexResult<Vec<u8>> {
1043-
let encoded = field.extension_type_metadata().unwrap_or_default();
1044-
1045-
BASE64_STANDARD.decode(encoded).map_err(|err| {
1046-
vortex_err!(
1047-
"Arrow field {:?} must carry base64 extension metadata, got {encoded:?}: {err}",
1048-
field.name()
1049-
)
1050-
})
1051-
}
1052-
10531040
impl SessionVar for ArrowSession {
10541041
fn as_any(&self) -> &dyn Any {
10551042
self
@@ -1164,7 +1151,7 @@ impl ArrowExporter<'_> {
11641151
target.name()
11651152
),
11661153
None => vortex_ensure!(
1167-
decode_ext_metadata(target)? == ext_dtype.serialize_metadata()?,
1154+
decode_arrow_ext_metadata(target)? == ext_dtype.serialize_metadata()?,
11681155
"Arrow field {:?} must carry the metadata of {ext_dtype}, got {:?}",
11691156
target.name(),
11701157
target.extension_type_metadata().unwrap_or_default()

‎vortex-arrow/tests/vortex_id.rs‎

Lines changed: 57 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ use arrow_schema::DataType;
1919
use arrow_schema::Field;
2020
use arrow_schema::extension::EXTENSION_TYPE_METADATA_KEY;
2121
use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
22-
use base64::Engine;
23-
use base64::prelude::BASE64_STANDARD;
2422
use rstest::rstest;
2523
use vortex_array::ArrayRef;
2624
use vortex_array::IntoArray;
@@ -46,6 +44,7 @@ use vortex_array::validity::Validity;
4644
use vortex_arrow::ArrowSession;
4745
use vortex_arrow::ArrowSessionExt;
4846
use vortex_arrow::Datum;
47+
use vortex_arrow::ext_metadata::EXTENSION_METADATA_ENCODING_KEY;
4948
use vortex_buffer::buffer;
5049
use vortex_error::VortexResult;
5150
use vortex_error::vortex_bail;
@@ -140,26 +139,35 @@ fn tagged_timestamps() -> VortexResult<ArrayRef> {
140139
tagged_array(0xff, timestamps()?)
141140
}
142141

143-
/// Build an Arrow field of type `storage` by hand, with the extension metadata of `ext_dtype`. A
144-
/// caller does this to export the storage of a type that has its own Arrow type.
142+
/// The field from [`ArrowSession::to_arrow_storage_field`] for `ext_dtype`, with the Arrow type
143+
/// `storage`.
145144
fn field_with_ext_metadata(
146145
name: &str,
147146
storage: DataType,
148147
ext_dtype: &ExtDTypeRef,
149148
) -> VortexResult<Field> {
150-
let id = ext_dtype.id().to_string();
151-
let metadata = BASE64_STANDARD.encode(ext_dtype.serialize_metadata()?);
152-
let field_metadata = [
153-
(EXTENSION_TYPE_NAME_KEY.to_owned(), id),
154-
(EXTENSION_TYPE_METADATA_KEY.to_owned(), metadata),
155-
];
156-
let field = Field::new(name, storage, ext_dtype.is_nullable());
149+
let field = SESSION.arrow().to_arrow_storage_field(name, ext_dtype)?;
150+
151+
Ok(field.with_data_type(storage))
152+
}
153+
154+
/// An Arrow field of type `Int64` that names [`Tagged`], plus the given field metadata.
155+
fn tagged_int64_field(metadata: &[(&str, &str)]) -> Field {
156+
let mut field = Field::new("t", DataType::Int64, true);
157+
field
158+
.metadata_mut()
159+
.insert(EXTENSION_TYPE_NAME_KEY.to_owned(), "test.tagged".to_owned());
160+
field.metadata_mut().extend(
161+
metadata
162+
.iter()
163+
.map(|(key, value)| ((*key).to_owned(), (*value).to_owned())),
164+
);
157165

158-
Ok(field.with_metadata(field_metadata.into()))
166+
field
159167
}
160168

161169
#[test]
162-
fn export_field_holds_vortex_id_and_base64_metadata() -> VortexResult<()> {
170+
fn export_field_holds_vortex_id_and_marked_base64_metadata() -> VortexResult<()> {
163171
let field = SESSION
164172
.arrow()
165173
.to_arrow_field("t", tagged_i64s(0xff)?.dtype())?;
@@ -169,6 +177,10 @@ fn export_field_holds_vortex_id_and_base64_metadata() -> VortexResult<()> {
169177
[
170178
(EXTENSION_TYPE_NAME_KEY.to_owned(), "test.tagged".to_owned()),
171179
(EXTENSION_TYPE_METADATA_KEY.to_owned(), "/w==".to_owned()),
180+
(
181+
EXTENSION_METADATA_ENCODING_KEY.to_owned(),
182+
"base64".to_owned(),
183+
),
172184
]
173185
.into(),
174186
);
@@ -318,6 +330,21 @@ fn unknown_vortex_id_imports_as_storage() -> VortexResult<()> {
318330
Ok(())
319331
}
320332

333+
/// Metadata without the encoding key is its own UTF-8 bytes, as other producers of these fields
334+
/// write it.
335+
#[test]
336+
fn unmarked_metadata_imports_as_its_text() -> VortexResult<()> {
337+
let field = tagged_int64_field(&[(EXTENSION_TYPE_METADATA_KEY, "x")]);
338+
let expected = tagged(b'x', DType::Primitive(PType::I64, Nullability::Nullable))?;
339+
340+
assert_eq!(
341+
SESSION.arrow().from_arrow_field(&field)?,
342+
DType::Extension(expected)
343+
);
344+
345+
Ok(())
346+
}
347+
321348
/// A dictionary imports as a [`Dict`] whose values have the extension type, which matches what the
322349
/// importer plugins produce.
323350
#[test]
@@ -346,23 +373,27 @@ fn dictionary_values_keep_the_extension_type() -> VortexResult<()> {
346373
}
347374

348375
/// If the extension name of a field is a known Vortex ID, its metadata must be valid for that type.
349-
/// A missing `ARROW:extension:metadata` means empty metadata, which [`Tagged`] rejects.
376+
/// A missing `ARROW:extension:metadata` means empty metadata, and unmarked metadata is its own
377+
/// text. [`Tagged`] rejects both here, because neither is one byte.
350378
#[rstest]
351-
#[case::not_base64(Some("%"), "must carry base64 extension metadata")]
352-
#[case::missing(None, "Tagged metadata must be one byte, got []")]
379+
#[case::not_base64(
380+
&[(EXTENSION_TYPE_METADATA_KEY, "%"), (EXTENSION_METADATA_ENCODING_KEY, "base64")],
381+
"must carry base64 extension metadata"
382+
)]
383+
#[case::unknown_encoding(
384+
&[(EXTENSION_TYPE_METADATA_KEY, "ff"), (EXTENSION_METADATA_ENCODING_KEY, "hex")],
385+
r#"must have the extension metadata encoding "base64", got "hex""#
386+
)]
387+
#[case::missing(&[], "Tagged metadata must be one byte, got []")]
388+
#[case::unmarked_base64(
389+
&[(EXTENSION_TYPE_METADATA_KEY, "/w==")],
390+
"Tagged metadata must be one byte, got [47, 119, 61, 61]"
391+
)]
353392
fn import_rejects_bad_metadata(
354-
#[case] metadata: Option<&str>,
393+
#[case] metadata: &[(&str, &str)],
355394
#[case] message: &str,
356395
) -> VortexResult<()> {
357-
let mut field = Field::new("t", DataType::Int64, true);
358-
field
359-
.metadata_mut()
360-
.insert(EXTENSION_TYPE_NAME_KEY.to_owned(), "test.tagged".to_owned());
361-
if let Some(metadata) = metadata {
362-
field
363-
.metadata_mut()
364-
.insert(EXTENSION_TYPE_METADATA_KEY.to_owned(), metadata.to_owned());
365-
}
396+
let field = tagged_int64_field(metadata);
366397

367398
let err = SESSION
368399
.arrow()

0 commit comments

Comments
 (0)