11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
4+ use arrow_array:: RecordBatchReader ;
45use arrow_array:: ffi_stream;
6+ use arrow_schema:: SchemaRef ;
57use vortex_array:: ArrayRef ;
68use vortex_array:: dtype:: DType ;
79use vortex_array:: iter:: ArrayIterator ;
810use vortex_error:: VortexError ;
9- use vortex_error:: VortexExpect ;
1011use vortex_error:: VortexResult ;
1112
12- use crate :: convert:: from_arrow_batch;
13- use crate :: dtype:: from_arrow_schema_naive;
13+ use crate :: ArrowSession ;
1414
15- /// An adapter for converting an `ArrowArrayStreamReader` into a Vortex `ArrayStream`.
15+ /// An adapter for converting an `ArrowArrayStreamReader` into a Vortex `ArrayIterator`.
16+ ///
17+ /// The stream's schema and batches are imported through the [`ArrowSession`], so Arrow extension
18+ /// types are routed to their registered import plugins instead of being flattened into their
19+ /// storage types.
1620pub struct ArrowArrayStreamAdapter {
1721 stream : ffi_stream:: ArrowArrayStreamReader ,
22+ session : ArrowSession ,
23+ schema : SchemaRef ,
1824 dtype : DType ,
1925}
2026
2127impl ArrowArrayStreamAdapter {
22- pub fn new ( stream : ffi_stream:: ArrowArrayStreamReader , dtype : DType ) -> Self {
23- Self { stream, dtype }
28+ /// Adapt `stream`, importing its schema and each of its batches through `session`.
29+ ///
30+ /// The adapter holds a clone of `session`, which shares the plugin registries with it, so
31+ /// plugins registered after construction are still observed.
32+ ///
33+ /// The schema declared by the stream is the authoritative schema for the import: every batch
34+ /// is converted against it, so extension metadata survives even if an individual batch has
35+ /// lost it.
36+ pub fn try_new (
37+ session : & ArrowSession ,
38+ stream : ffi_stream:: ArrowArrayStreamReader ,
39+ ) -> VortexResult < Self > {
40+ let schema = stream. schema ( ) ;
41+ let dtype = session. from_arrow_schema ( schema. as_ref ( ) ) ?;
42+ Ok ( Self {
43+ stream,
44+ session : session. clone ( ) ,
45+ schema,
46+ dtype,
47+ } )
2448 }
2549}
2650
@@ -37,12 +61,75 @@ impl Iterator for ArrowArrayStreamAdapter {
3761 let batch = self . stream . next ( ) ?;
3862
3963 Some ( batch. map_err ( VortexError :: from) . and_then ( |b| {
40- debug_assert_eq ! (
41- & self . dtype,
42- & from_arrow_schema_naive( b. schema( ) . as_ref( ) )
43- . vortex_expect( "arrow schema to dtype" )
44- ) ;
45- from_arrow_batch ( & b, false )
64+ self . session
65+ . from_arrow_record_batch ( b, self . schema . as_ref ( ) )
4666 } ) )
4767 }
4868}
69+
70+ #[ cfg( test) ]
71+ mod tests {
72+ use std:: sync:: Arc ;
73+
74+ use arrow_array:: FixedSizeBinaryArray ;
75+ use arrow_array:: RecordBatch ;
76+ use arrow_array:: RecordBatchIterator ;
77+ use arrow_schema:: DataType ;
78+ use arrow_schema:: Field ;
79+ use arrow_schema:: Schema ;
80+ use arrow_schema:: extension:: Uuid as ArrowUuid ;
81+ use vortex_array:: array_session;
82+ use vortex_array:: arrays:: Struct ;
83+ use vortex_array:: arrays:: struct_:: StructArrayExt ;
84+ use vortex_array:: extension:: uuid:: Uuid ;
85+ use vortex_error:: VortexExpect ;
86+
87+ use super :: * ;
88+ use crate :: ArrowSessionExt ;
89+
90+ /// The adapter imports through the [`ArrowSession`], so a UUID column arrives as a Vortex
91+ /// extension array rather than its `FixedSizeList` storage.
92+ #[ test]
93+ fn stream_preserves_extension_types ( ) -> VortexResult < ( ) > {
94+ let mut field = Field :: new ( "id" , DataType :: FixedSizeBinary ( 16 ) , false ) ;
95+ field. try_with_extension_type ( ArrowUuid ) ?;
96+ let schema = Arc :: new ( Schema :: new ( vec ! [ field] ) ) ;
97+ let ids = FixedSizeBinaryArray :: try_from_iter (
98+ [ * b"0123456789abcdef" , * b"fedcba9876543210" ] . into_iter ( ) ,
99+ ) ?;
100+ let batch = RecordBatch :: try_new ( Arc :: clone ( & schema) , vec ! [ Arc :: new( ids) ] ) ?;
101+
102+ let reader = RecordBatchIterator :: new ( [ Ok ( batch) ] , schema) ;
103+ let stream = ffi_stream:: ArrowArrayStreamReader :: try_new (
104+ ffi_stream:: FFI_ArrowArrayStream :: new ( Box :: new ( reader) ) ,
105+ ) ?;
106+
107+ let vortex_session = array_session ( ) ;
108+ let mut adapter = ArrowArrayStreamAdapter :: try_new ( & vortex_session. arrow ( ) , stream) ?;
109+
110+ let DType :: Struct ( fields, _) = adapter. dtype ( ) . clone ( ) else {
111+ panic ! ( "expected a struct dtype, got {}" , adapter. dtype( ) ) ;
112+ } ;
113+ assert ! (
114+ fields
115+ . field_by_index( 0 )
116+ . vortex_expect( "one field" )
117+ . as_extension( )
118+ . is:: <Uuid >( )
119+ ) ;
120+
121+ let array = adapter. next ( ) . vortex_expect ( "one batch" ) ?;
122+ assert_eq ! ( array. dtype( ) , adapter. dtype( ) ) ;
123+ assert ! (
124+ array
125+ . as_:: <Struct >( )
126+ . unmasked_field( 0 )
127+ . dtype( )
128+ . as_extension( )
129+ . is:: <Uuid >( )
130+ ) ;
131+ assert ! ( adapter. next( ) . is_none( ) ) ;
132+
133+ Ok ( ( ) )
134+ }
135+ }
0 commit comments