Skip to content

Commit 9748b74

Browse files
authored
Scan extension columns over list, map, and string storage in DataFusion (#11225)
## Summary - Tracking Issue: #11114 DataFusion can't scan a `Vector` column today. For a logical `List`, `FixedSizeList`, `Map`, `ListView`, `Utf8`, or `Binary` type, the physical schema compares the layout with the extension dtype instead of its storage, so the scan fails before it reads a row. The same bug breaks a `Json` column that the table schema declares as `Utf8`. ## Changes The layout checks now look through extension types, as the struct check already did. An extension column keeps its logical layout rather than the type that `ArrowSession` would export for it, because the V2 source has no schema adapter, so its batches must match the plan schema exactly. The export type would make the V2 source emit `Utf8View` batches for a `Json` column that the table schema declares as `Utf8`. This only helps types with an Arrow export plugin, such as `Vector` and `Json`. A type without one still can't be scanned until Vortex can export it as its storage with extension metadata (AX in #11114). The first commit fixes a panic that this change would otherwise expose. The Arrow map export panicked on any dtype other than a map, and an extension column over map storage can now reach it. It returns an error instead, as the list exporters already do. Tests cover a SQL scan of a file with a `Vector` column, a V2 projection over a `Json` column, each layout check of the physical schema, and the map export of a non-map array. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent f0b14cd commit 9748b74

7 files changed

Lines changed: 215 additions & 11 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vortex-arrow/src/executor/map.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ use vortex_array::arrays::Map;
1515
use vortex_array::arrays::MapArray;
1616
use vortex_array::arrays::map::MapArrayExt;
1717
use vortex_array::arrays::map::MapArraySlotsExt;
18+
use vortex_array::dtype::DType;
1819
use vortex_error::VortexResult;
1920
use vortex_error::vortex_ensure;
2021
use vortex_error::vortex_ensure_eq;
@@ -34,6 +35,12 @@ pub(super) fn to_arrow_map(
3435
exporter: &ArrowExporter<'_>,
3536
ctx: &mut ExecutionCtx,
3637
) -> VortexResult<ArrowArrayRef> {
38+
vortex_ensure!(
39+
matches!(array.dtype(), DType::Map(..)),
40+
"Arrow map export requires a map dtype, got {}",
41+
array.dtype()
42+
);
43+
3744
let array = match array.try_downcast::<Map>() {
3845
Ok(map) => map,
3946
Err(array) => array.execute::<MapArray>(ctx)?,

‎vortex-arrow/src/executor/mod.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,18 @@ mod tests {
335335
DataType::List(Arc::new(Field::new("item", DataType::Int32, true)))
336336
}
337337

338+
fn map_target() -> DataType {
339+
let entries = Field::new_struct(
340+
"entries",
341+
vec![
342+
Field::new("key", DataType::Int32, false),
343+
Field::new("value", DataType::Int32, true),
344+
],
345+
false,
346+
);
347+
DataType::Map(Arc::new(entries), false)
348+
}
349+
338350
/// Must error rather than panic inside `execute::<T>`.
339351
#[rstest]
340352
#[case::bool_from_utf8(utf8(), DataType::Boolean)]
@@ -353,6 +365,7 @@ mod tests {
353365
utf8(),
354366
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int32, true)), 2)
355367
)]
368+
#[case::map_from_primitive(primitive(), map_target())]
356369
#[case::byte_from_primitive(primitive(), DataType::Utf8)]
357370
#[case::byte_from_bool(boolean(), DataType::Binary)]
358371
fn incompatible_target_returns_error(#[case] array: ArrayRef, #[case] target: DataType) {

‎vortex-datafusion/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "fs"] }
4949
url = { workspace = true }
5050
vortex-array = { workspace = true, features = ["_test-harness"] }
5151
vortex-json = { workspace = true }
52+
vortex-tensor = { workspace = true }
5253

5354
[lints]
5455
workspace = true

‎vortex-datafusion/src/convert/schema.rs‎

Lines changed: 115 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,14 +79,23 @@ fn calculate_physical_field_type(
7979
logical_type: &DataType,
8080
arrow_session: &ArrowSession,
8181
) -> DFResult<DataType> {
82+
// An extension dtype has the layout of its storage dtype, and the field metadata that names the
83+
// extension comes from the logical field. So the layout checks below use the storage dtype.
84+
// Temporal dtypes export as native Arrow types (see `ArrowSession::to_arrow_field`), which only
85+
// the default arm matches.
86+
let mut storage_dtype = dtype;
87+
while let DType::Extension(ext) = storage_dtype {
88+
storage_dtype = ext.storage_dtype();
89+
}
90+
8291
// Check if the logical type is one that doesn't roundtrip through DType
8392
Ok(match logical_type {
8493
// Dictionary types lose their encoding when converted to DType
8594
DataType::Dictionary(..) => logical_type.clone(),
8695

8796
// Non-view string/binary types become view types after roundtrip
8897
DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary => {
89-
if dtype.is_binary() || dtype.is_utf8() {
98+
if storage_dtype.is_binary() || storage_dtype.is_utf8() {
9099
logical_type.clone()
91100
} else {
92101
return Err(exec_datafusion_err!(
@@ -100,12 +109,7 @@ fn calculate_physical_field_type(
100109

101110
// For struct types, recursively check each field.
102111
DataType::Struct(logical_fields) => {
103-
// Walk through any extension layers to reach the underlying struct fields.
104-
let mut inner = dtype;
105-
while let DType::Extension(ext) = inner {
106-
inner = ext.storage_dtype();
107-
}
108-
if let DType::Struct(struct_dtype, _) = inner {
112+
if let DType::Struct(struct_dtype, _) = storage_dtype {
109113
let physical_fields: Vec<Field> = struct_dtype
110114
.names()
111115
.iter()
@@ -146,7 +150,7 @@ fn calculate_physical_field_type(
146150

147151
// For list types, recursively check the element type
148152
DataType::List(logical_elem) | DataType::LargeList(logical_elem) => {
149-
if let DType::List(elem_dtype, _) = dtype {
153+
if let DType::List(elem_dtype, _) = storage_dtype {
150154
let physical_elem_type = calculate_physical_field_type(
151155
elem_dtype,
152156
logical_elem.data_type(),
@@ -172,7 +176,7 @@ fn calculate_physical_field_type(
172176

173177
// For fixed-size list types, recursively check the element type
174178
DataType::FixedSizeList(logical_elem, size) => {
175-
if let DType::FixedSizeList(elem_dtype, ..) = dtype {
179+
if let DType::FixedSizeList(elem_dtype, ..) = storage_dtype {
176180
let physical_elem_type = calculate_physical_field_type(
177181
elem_dtype,
178182
logical_elem.data_type(),
@@ -195,7 +199,7 @@ fn calculate_physical_field_type(
195199
// Map field names and child metadata come from the reference schema, while child
196200
// types are recursively reconciled so nested extension metadata is preserved.
197201
DataType::Map(logical_entries, keys_sorted) => {
198-
let DType::Map(map_dtype, _) = dtype else {
202+
let DType::Map(map_dtype, _) = storage_dtype else {
199203
return Err(exec_datafusion_err!(
200204
"Failed to convert dtype to arrow: Vortex DType is {dtype} which is not compatible with {logical_type}"
201205
));
@@ -240,7 +244,7 @@ fn calculate_physical_field_type(
240244

241245
// For list view types, recursively check the element type
242246
DataType::ListView(logical_elem) | DataType::LargeListView(logical_elem) => {
243-
if let DType::List(elem_dtype, _) = dtype {
247+
if let DType::List(elem_dtype, _) = storage_dtype {
244248
let physical_elem_type = calculate_physical_field_type(
245249
elem_dtype,
246250
logical_elem.data_type(),
@@ -278,9 +282,16 @@ mod tests {
278282
use std::sync::Arc;
279283

280284
use arrow_schema::Fields;
285+
use vortex::array::EmptyMetadata;
281286
use vortex::dtype::Nullability;
282287
use vortex::dtype::PType;
283288
use vortex::dtype::StructFields;
289+
use vortex::dtype::extension::ExtDType;
290+
use vortex::dtype::extension::ExtId;
291+
use vortex::dtype::extension::ExtVTable;
292+
use vortex::error::VortexResult;
293+
use vortex::scalar::ScalarValue;
294+
use vortex::session::registry::CachedId;
284295

285296
use super::*;
286297

@@ -620,4 +631,97 @@ mod tests {
620631
assert_eq!(physical_schema, logical_schema);
621632
Ok(())
622633
}
634+
635+
/// An extension type that accepts every storage dtype, so one type can wrap each layout.
636+
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
637+
struct AnyStorage;
638+
639+
impl ExtVTable for AnyStorage {
640+
type Metadata = EmptyMetadata;
641+
type NativeValue<'a> = &'a ScalarValue;
642+
643+
fn id(&self) -> ExtId {
644+
static ID: CachedId = CachedId::new("test.any_storage");
645+
*ID
646+
}
647+
648+
fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult<Vec<u8>> {
649+
Ok(Vec::new())
650+
}
651+
652+
fn deserialize_metadata(&self, _metadata: &[u8]) -> VortexResult<Self::Metadata> {
653+
Ok(EmptyMetadata)
654+
}
655+
656+
fn validate_dtype(_ext_dtype: &ExtDType<Self>) -> VortexResult<()> {
657+
Ok(())
658+
}
659+
660+
fn unpack_native<'a>(
661+
_ext_dtype: &'a ExtDType<Self>,
662+
storage_value: &'a ScalarValue,
663+
) -> VortexResult<Self::NativeValue<'a>> {
664+
Ok(storage_value)
665+
}
666+
}
667+
668+
#[test]
669+
fn test_extension_storage_layouts_preserved() -> anyhow::Result<()> {
670+
let item = || Arc::new(Field::new("item", DataType::Int32, false));
671+
let entries = Field::new_struct(
672+
"entries",
673+
vec![
674+
Field::new("key", DataType::Int32, false),
675+
Field::new("value", DataType::Int32, false),
676+
],
677+
false,
678+
);
679+
let logical_schema = Schema::new(vec![
680+
Field::new("utf8_col", DataType::Utf8, true),
681+
Field::new("list_col", DataType::List(item()), true),
682+
Field::new("fsl_col", DataType::FixedSizeList(item(), 3), true),
683+
Field::new("map_col", DataType::Map(Arc::new(entries), false), true),
684+
Field::new("listview_col", DataType::ListView(item()), true),
685+
]);
686+
687+
let int32 = || DType::Primitive(PType::I32, Nullability::NonNullable);
688+
let extension = |storage_dtype| -> VortexResult<DType> {
689+
Ok(DType::Extension(
690+
ExtDType::<AnyStorage>::try_new(EmptyMetadata, storage_dtype)?.erased(),
691+
))
692+
};
693+
let dtype = DType::Struct(
694+
StructFields::from_iter([
695+
("utf8_col", extension(DType::Utf8(Nullability::Nullable))?),
696+
(
697+
"list_col",
698+
extension(DType::list(int32(), Nullability::Nullable))?,
699+
),
700+
(
701+
"fsl_col",
702+
extension(DType::FixedSizeList(
703+
Arc::new(int32()),
704+
3,
705+
Nullability::Nullable,
706+
))?,
707+
),
708+
(
709+
"map_col",
710+
extension(DType::map(int32(), int32(), false, Nullability::Nullable)?)?,
711+
),
712+
// Vortex has one list dtype, which the table can also read as a `ListView`.
713+
(
714+
"listview_col",
715+
extension(DType::list(int32(), Nullability::Nullable))?,
716+
),
717+
]),
718+
Nullability::NonNullable,
719+
);
720+
721+
let physical_schema =
722+
calculate_physical_schema(&dtype, &logical_schema, &ArrowSession::default())?;
723+
assert_eq!(physical_schema, logical_schema);
724+
725+
Ok(())
726+
}
623727
}

‎vortex-datafusion/src/persistent/tests.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ use rstest::rstest;
3535
use vortex::VortexSessionDefault;
3636
use vortex::array::IntoArray;
3737
use vortex::array::arrays::ChunkedArray;
38+
use vortex::array::arrays::FixedSizeListArray;
3839
use vortex::array::arrays::StructArray;
3940
use vortex::array::arrays::VarBinArray;
4041
use vortex::array::validity::Validity;
@@ -53,6 +54,8 @@ use vortex::layout::layouts::chunked::writer::ChunkedLayoutStrategy;
5354
use vortex::layout::layouts::flat::writer::FlatLayoutStrategy;
5455
use vortex::layout::layouts::table::TableStrategy;
5556
use vortex::session::VortexSession;
57+
use vortex_tensor::vector::ARROW_VECTOR_EXTENSION_NAME;
58+
use vortex_tensor::vector::Vector;
5659

5760
use crate::VortexFormatFactory;
5861
use crate::VortexSource;
@@ -719,6 +722,56 @@ async fn arrow_uuid_extension_roundtrip_nested_struct() -> anyhow::Result<()> {
719722
Ok(())
720723
}
721724

725+
/// `Vector` is an extension type over `FixedSizeList` storage, so the physical schema must look
726+
/// through the extension type to its storage.
727+
#[tokio::test]
728+
async fn test_scan_vector_column() -> anyhow::Result<()> {
729+
let session = VortexSession::default();
730+
vortex_tensor::initialize(&session);
731+
let ctx = TestSessionContext::new_with_factory(Arc::new(
732+
VortexFormatFactory::new_with_session(session.clone()),
733+
));
734+
735+
let elements = buffer![
736+
1.0f32, 2.0, 3.0, //
737+
4.0, 5.0, 6.0, //
738+
]
739+
.into_array();
740+
let storage = FixedSizeListArray::try_new(elements, 3, Validity::NonNullable, 2)?;
741+
let vectors = Vector::try_new_vector_array(storage.into_array())?;
742+
let array = StructArray::from_fields(&[("embedding", vectors)])?.into_array();
743+
744+
let mut writer =
745+
ObjectStoreWrite::new(Arc::clone(&ctx.store), &"vectors.vortex".into()).await?;
746+
session
747+
.write_options()
748+
.write(&mut writer, array.to_array_stream())
749+
.await?;
750+
writer.shutdown().await?;
751+
752+
let result = ctx
753+
.session
754+
.sql("SELECT * FROM '/vectors.vortex'")
755+
.await?
756+
.collect()
757+
.await?;
758+
759+
assert_eq!(
760+
result[0].schema().field(0).extension_type_name(),
761+
Some(ARROW_VECTOR_EXTENSION_NAME)
762+
);
763+
assert_snapshot!(pretty_format_batches(&result)?, @r"
764+
+-----------------+
765+
| embedding |
766+
+-----------------+
767+
| [1.0, 2.0, 3.0] |
768+
| [4.0, 5.0, 6.0] |
769+
+-----------------+
770+
");
771+
772+
Ok(())
773+
}
774+
722775
#[rstest]
723776
#[case::modulo("id, a", "CAST(a % 2 AS BIGINT) = 0")]
724777
#[case::in_list("id, a IN (1, 4, 5, 6, 7, NULL) AS member", "TRUE")]

‎vortex-datafusion/src/v2/tests.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,3 +518,28 @@ async fn test_dynamic_extension_literals_refresh_and_keep_residuals(
518518

519519
Ok(())
520520
}
521+
522+
/// The V2 source has no schema adapter, so a projection over an extension column must produce
523+
/// batches with exactly the plan schema.
524+
#[tokio::test]
525+
async fn test_extension_projection_preserves_arrow_type() -> anyhow::Result<()> {
526+
let fixture = ExtensionFilterFixture::new(ExtensionFilterType::Json)?;
527+
let projection = ProjectionExprs::from(vec![ProjectionExpr::new(fixture.column(), "value")]);
528+
let projected = fixture
529+
.source()
530+
.await?
531+
.try_swapping_with_projection(&projection)?
532+
.ok_or_else(|| anyhow!("projection was not swapped"))?;
533+
let plan = Arc::new(DataSourceExec::new(projected)) as Arc<dyn ExecutionPlan>;
534+
let plan_schema = plan.schema();
535+
536+
let batches = datafusion_physical_plan::collect(plan, SessionContext::new().task_ctx()).await?;
537+
for batch in &batches {
538+
assert_eq!(batch.schema(), plan_schema);
539+
}
540+
541+
let actual = concat_batches(&plan_schema, &batches)?;
542+
assert_eq!(actual.column(0), fixture.batch.column(1));
543+
544+
Ok(())
545+
}

0 commit comments

Comments
 (0)