Skip to content

Commit 8693977

Browse files
committed
fixes
Signed-off-by: Robert Kruszewski <[email protected]>
1 parent 1accaaa commit 8693977

3 files changed

Lines changed: 164 additions & 31 deletions

File tree

‎vortex-arrow/src/convert.rs‎

Lines changed: 57 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -648,21 +648,27 @@ pub(crate) fn map_from_arrow_parts(
648648
Ok(MapArray::try_new(map_dtype, entries)?.into_array())
649649
}
650650

651+
/// Conversion of an Arrow map array into a Vortex `Map` array.
652+
pub fn from_arrow_map(array: &ArrowMapArray, nullable: bool) -> VortexResult<ArrayRef> {
653+
let DataType::Map(_, keys_sorted) = array.data_type() else {
654+
vortex_panic!("Invalid data type for MapArray: {}", array.data_type());
655+
};
656+
let entries = from_arrow_struct(array.entries(), false)?;
657+
map_from_arrow_parts(
658+
entries,
659+
array.offsets(),
660+
array.nulls(),
661+
*keys_sorted,
662+
nullable,
663+
)
664+
}
665+
651666
impl FromArrowArray<&ArrowMapArray> for ArrayRef {
652667
fn from_arrow(array: &ArrowMapArray, nullable: bool) -> VortexResult<Self> {
653-
let DataType::Map(_, keys_sorted) = array.data_type() else {
654-
vortex_panic!("Invalid data type for MapArray: {}", array.data_type());
655-
};
656-
let entries = Self::from_arrow(array.entries(), false)?;
657-
map_from_arrow_parts(
658-
entries,
659-
array.offsets(),
660-
array.nulls(),
661-
*keys_sorted,
662-
nullable,
663-
)
668+
from_arrow_map(array, nullable)
664669
}
665670
}
671+
666672
/// Conversion of an Arrow null array into a Vortex `Null` array.
667673
pub fn from_arrow_null(value: &ArrowNullArray, nullable: bool) -> VortexResult<ArrayRef> {
668674
vortex_ensure!(
@@ -748,6 +754,7 @@ pub fn from_arrow_dyn(array: &dyn ArrowArray, nullable: bool) -> VortexResult<Ar
748754
DataType::FixedSizeList(..) => {
749755
from_arrow_fixed_size_list(array.as_fixed_size_list(), nullable)
750756
}
757+
DataType::Map(..) => from_arrow_map(array.as_map(), nullable),
751758
DataType::Null => from_arrow_null(as_null_array(array), nullable),
752759
DataType::Timestamp(u, _) => match u {
753760
ArrowTimeUnit::Second => {
@@ -906,6 +913,8 @@ mod tests {
906913
use arrow_array::builder::Int32Builder;
907914
use arrow_array::builder::LargeListBuilder;
908915
use arrow_array::builder::ListBuilder;
916+
use arrow_array::builder::MapBuilder as ArrowMapBuilder;
917+
use arrow_array::builder::StringBuilder;
909918
use arrow_array::builder::StringViewBuilder;
910919
use arrow_array::new_null_array;
911920
use arrow_array::types::ArrowPrimitiveType;
@@ -937,9 +946,12 @@ mod tests {
937946
use vortex_array::dtype::PType;
938947
use vortex_array::extension::datetime::TimeUnit;
939948
use vortex_array::extension::datetime::Timestamp;
949+
use vortex_error::VortexResult;
940950

941951
use crate::FromArrowArray as _;
942952
use crate::IntoVortexArray as _;
953+
use crate::convert::from_arrow_batch;
954+
use crate::convert::from_arrow_dyn;
943955

944956
#[rstest]
945957
#[case::i8(
@@ -1726,6 +1738,40 @@ mod tests {
17261738
ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true).unwrap();
17271739
}
17281740

1741+
#[test]
1742+
fn dyn_dispatch_imports_map() -> VortexResult<()> {
1743+
let mut builder = ArrowMapBuilder::new(None, StringBuilder::new(), Int32Builder::new());
1744+
builder.keys().append_value("joe");
1745+
builder.values().append_value(1);
1746+
builder.append(true)?;
1747+
builder.append(false)?;
1748+
let arrow = builder.finish();
1749+
1750+
let expected = DType::map(
1751+
DType::Utf8(Nullability::NonNullable),
1752+
DType::Primitive(PType::I32, Nullability::Nullable),
1753+
false,
1754+
Nullability::Nullable,
1755+
)?;
1756+
1757+
// Dispatching on the Arrow data type must cover maps...
1758+
assert_eq!(from_arrow_dyn(&arrow, true)?.dtype(), &expected);
1759+
1760+
// ...including maps reached by recursing into a container.
1761+
let batch = RecordBatch::try_from_iter_with_nullable([(
1762+
"maps",
1763+
Arc::new(arrow) as Arc<dyn ArrowArray>,
1764+
true,
1765+
)])?;
1766+
let imported = from_arrow_batch(&batch, false)?;
1767+
assert_eq!(
1768+
imported.as_::<Struct>().unmasked_field(0).dtype(),
1769+
&expected
1770+
);
1771+
1772+
Ok(())
1773+
}
1774+
17291775
#[test]
17301776
fn non_nullable_request_rejects_nulls() {
17311777
// Requesting `nullable = false` on an Arrow array that physically contains nulls is a

‎vortex-arrow/src/datum.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -144,20 +144,20 @@ where
144144
/// This is useful for compute functions that delegate to Arrow using [Datum],
145145
/// which will return a scalar (length 1 Arrow array) if the input array is constant.
146146
///
147+
/// The array is imported through the [`ArrowSession`](crate::ArrowSession) of `ctx`, so nested
148+
/// extension fields reach their registered import plugins.
149+
///
147150
/// # Error
148151
///
149152
/// The provided array must have length `len` or `1`.
150-
#[allow(deprecated)]
151-
pub fn from_arrow_columnar<A>(
152-
array: A,
153+
pub fn from_arrow_columnar(
154+
array: &dyn ArrowArray,
153155
len: usize,
154156
nullable: bool,
155157
ctx: &mut ExecutionCtx,
156-
) -> VortexResult<ArrayRef>
157-
where
158-
ArrayRef: FromArrowArray<A>,
159-
{
160-
let array = ArrayRef::from_arrow(array, nullable)?;
158+
) -> VortexResult<ArrayRef> {
159+
let session = ctx.session().clone();
160+
let array = session.arrow().from_arrow_array_nullable(array, nullable)?;
161161
if array.len() == len {
162162
return Ok(array);
163163
}

‎vortex-arrow/src/iter.rs‎

Lines changed: 99 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,50 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
use arrow_array::RecordBatchReader;
45
use arrow_array::ffi_stream;
6+
use arrow_schema::SchemaRef;
57
use vortex_array::ArrayRef;
68
use vortex_array::dtype::DType;
79
use vortex_array::iter::ArrayIterator;
810
use vortex_error::VortexError;
9-
use vortex_error::VortexExpect;
1011
use vortex_error::VortexResult;
1112

12-
use crate::convert::from_arrow_batch;
13-
use crate::dtype::from_arrow_schema_naive;
13+
use crate::ArrowSession;
1414

15-
/// An adapter for converting an `ArrowArrayStreamReader` into a Vortex `ArrayStream`.
15+
/// An adapter for converting an `ArrowArrayStreamReader` into a Vortex `ArrayIterator`.
16+
///
17+
/// The stream's schema and batches are imported through the [`ArrowSession`], so Arrow extension
18+
/// types are routed to their registered import plugins instead of being flattened into their
19+
/// storage types.
1620
pub struct ArrowArrayStreamAdapter {
1721
stream: ffi_stream::ArrowArrayStreamReader,
22+
session: ArrowSession,
23+
schema: SchemaRef,
1824
dtype: DType,
1925
}
2026

2127
impl ArrowArrayStreamAdapter {
22-
pub fn new(stream: ffi_stream::ArrowArrayStreamReader, dtype: DType) -> Self {
23-
Self { stream, dtype }
28+
/// Adapt `stream`, importing its schema and each of its batches through `session`.
29+
///
30+
/// The adapter holds a clone of `session`, which shares the plugin registries with it, so
31+
/// plugins registered after construction are still observed.
32+
///
33+
/// The schema declared by the stream is the authoritative schema for the import: every batch
34+
/// is converted against it, so extension metadata survives even if an individual batch has
35+
/// lost it.
36+
pub fn try_new(
37+
session: &ArrowSession,
38+
stream: ffi_stream::ArrowArrayStreamReader,
39+
) -> VortexResult<Self> {
40+
let schema = stream.schema();
41+
let dtype = session.from_arrow_schema(schema.as_ref())?;
42+
Ok(Self {
43+
stream,
44+
session: session.clone(),
45+
schema,
46+
dtype,
47+
})
2448
}
2549
}
2650

@@ -37,12 +61,75 @@ impl Iterator for ArrowArrayStreamAdapter {
3761
let batch = self.stream.next()?;
3862

3963
Some(batch.map_err(VortexError::from).and_then(|b| {
40-
debug_assert_eq!(
41-
&self.dtype,
42-
&from_arrow_schema_naive(b.schema().as_ref())
43-
.vortex_expect("arrow schema to dtype")
44-
);
45-
from_arrow_batch(&b, false)
64+
self.session
65+
.from_arrow_record_batch(b, self.schema.as_ref())
4666
}))
4767
}
4868
}
69+
70+
#[cfg(test)]
71+
mod tests {
72+
use std::sync::Arc;
73+
74+
use arrow_array::FixedSizeBinaryArray;
75+
use arrow_array::RecordBatch;
76+
use arrow_array::RecordBatchIterator;
77+
use arrow_schema::DataType;
78+
use arrow_schema::Field;
79+
use arrow_schema::Schema;
80+
use arrow_schema::extension::Uuid as ArrowUuid;
81+
use vortex_array::array_session;
82+
use vortex_array::arrays::Struct;
83+
use vortex_array::arrays::struct_::StructArrayExt;
84+
use vortex_array::extension::uuid::Uuid;
85+
use vortex_error::VortexExpect;
86+
87+
use super::*;
88+
use crate::ArrowSessionExt;
89+
90+
/// The adapter imports through the [`ArrowSession`], so a UUID column arrives as a Vortex
91+
/// extension array rather than its `FixedSizeList` storage.
92+
#[test]
93+
fn stream_preserves_extension_types() -> VortexResult<()> {
94+
let mut field = Field::new("id", DataType::FixedSizeBinary(16), false);
95+
field.try_with_extension_type(ArrowUuid)?;
96+
let schema = Arc::new(Schema::new(vec![field]));
97+
let ids = FixedSizeBinaryArray::try_from_iter(
98+
[*b"0123456789abcdef", *b"fedcba9876543210"].into_iter(),
99+
)?;
100+
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(ids)])?;
101+
102+
let reader = RecordBatchIterator::new([Ok(batch)], schema);
103+
let stream = ffi_stream::ArrowArrayStreamReader::try_new(
104+
ffi_stream::FFI_ArrowArrayStream::new(Box::new(reader)),
105+
)?;
106+
107+
let vortex_session = array_session();
108+
let mut adapter = ArrowArrayStreamAdapter::try_new(&vortex_session.arrow(), stream)?;
109+
110+
let DType::Struct(fields, _) = adapter.dtype().clone() else {
111+
panic!("expected a struct dtype, got {}", adapter.dtype());
112+
};
113+
assert!(
114+
fields
115+
.field_by_index(0)
116+
.vortex_expect("one field")
117+
.as_extension()
118+
.is::<Uuid>()
119+
);
120+
121+
let array = adapter.next().vortex_expect("one batch")?;
122+
assert_eq!(array.dtype(), adapter.dtype());
123+
assert!(
124+
array
125+
.as_::<Struct>()
126+
.unmasked_field(0)
127+
.dtype()
128+
.as_extension()
129+
.is::<Uuid>()
130+
);
131+
assert!(adapter.next().is_none());
132+
133+
Ok(())
134+
}
135+
}

0 commit comments

Comments
 (0)