Skip to content

Commit 136c0a6

Browse files
committed
fix(array): don't panic on unsupported arrow types
`DType::from_arrow` hit `unimplemented!` for unsupported arrow types (duration, interval, fixed-size binary), surfacing as a panic. add a fallible `TryFromArrowType` conversion that returns a `VortexError`, and route the session and python entry points through it so unsupported types raise a clean `ValueError` instead. Closes #8346 Signed-off-by: Han Damin <[email protected]>
1 parent 4668b6e commit 136c0a6

6 files changed

Lines changed: 116 additions & 38 deletions

File tree

‎vortex-array/src/arrow/session.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ use crate::dtype::FieldName;
5757
use crate::dtype::FieldNames;
5858
use crate::dtype::Nullability;
5959
use crate::dtype::StructFields;
60-
use crate::dtype::arrow::FromArrowType;
60+
use crate::dtype::arrow::TryFromArrowType;
6161
use crate::dtype::arrow::to_data_type_naive;
6262
use crate::dtype::extension::ExtId;
6363
use crate::extension::datetime::AnyTemporal;
@@ -313,7 +313,7 @@ impl ArrowSession {
313313
/// match (or all return `None`), recurses into container types ([`DataType::List`]
314314
/// family, [`DataType::FixedSizeList`], [`DataType::Struct`]) so extension metadata
315315
/// on nested element/struct fields is preserved. Leaf types use the canonical
316-
/// Arrow → Vortex mapping via [`DType::from_arrow`].
316+
/// Arrow → Vortex mapping via [`DType::try_from_arrow`].
317317
pub fn from_arrow_field(&self, field: &Field) -> VortexResult<DType> {
318318
if let Some(name) = field.metadata().get(EXTENSION_TYPE_NAME_KEY) {
319319
for plugin in self.importers(&Id::new(name)).iter() {
@@ -345,7 +345,7 @@ impl ArrowSession {
345345
.collect::<VortexResult<Vec<_>>>()?;
346346
DType::Struct(StructFields::from_iter(entries), nullability)
347347
}
348-
_ => DType::from_arrow(field),
348+
_ => DType::try_from_arrow(field)?,
349349
})
350350
}
351351

‎vortex-array/src/dtype/arrow.rs‎

Lines changed: 82 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -130,36 +130,63 @@ impl FromArrowType<SchemaRef> for DType {
130130
}
131131
}
132132

133+
impl TryFromArrowType<SchemaRef> for DType {
134+
fn try_from_arrow(value: SchemaRef) -> VortexResult<Self> {
135+
Self::try_from_arrow(value.as_ref())
136+
}
137+
}
138+
133139
impl FromArrowType<&Schema> for DType {
134140
fn from_arrow(value: &Schema) -> Self {
135-
Self::Struct(
136-
StructFields::from_arrow(value.fields()),
141+
Self::try_from_arrow(value).vortex_expect("arrow schema to dtype")
142+
}
143+
}
144+
145+
impl TryFromArrowType<&Schema> for DType {
146+
fn try_from_arrow(value: &Schema) -> VortexResult<Self> {
147+
Ok(Self::Struct(
148+
StructFields::try_from_arrow(value.fields())?,
137149
Nullability::NonNullable, // Must match From<RecordBatch> for Array
138-
)
150+
))
139151
}
140152
}
141153

142154
impl FromArrowType<&Fields> for StructFields {
143155
fn from_arrow(value: &Fields) -> Self {
144-
StructFields::from_iter(value.into_iter().map(|f| {
145-
(
146-
FieldName::from(f.name().as_str()),
147-
DType::from_arrow(f.as_ref()),
148-
)
149-
}))
156+
Self::try_from_arrow(value).vortex_expect("arrow fields to struct fields")
157+
}
158+
}
159+
160+
impl TryFromArrowType<&Fields> for StructFields {
161+
fn try_from_arrow(value: &Fields) -> VortexResult<Self> {
162+
value
163+
.into_iter()
164+
.map(|f| {
165+
Ok((
166+
FieldName::from(f.name().as_str()),
167+
DType::try_from_arrow(f.as_ref())?,
168+
))
169+
})
170+
.collect::<VortexResult<StructFields>>()
150171
}
151172
}
152173

153174
impl FromArrowType<(&DataType, Nullability)> for DType {
154-
fn from_arrow((data_type, nullability): (&DataType, Nullability)) -> Self {
175+
fn from_arrow(value: (&DataType, Nullability)) -> Self {
176+
Self::try_from_arrow(value).vortex_expect("arrow data type to dtype")
177+
}
178+
}
179+
180+
impl TryFromArrowType<(&DataType, Nullability)> for DType {
181+
fn try_from_arrow((data_type, nullability): (&DataType, Nullability)) -> VortexResult<Self> {
155182
if data_type.is_integer() || data_type.is_floating() {
156-
return DType::Primitive(
157-
PType::try_from_arrow(data_type).vortex_expect("arrow float/integer to ptype"),
183+
return Ok(DType::Primitive(
184+
PType::try_from_arrow(data_type)?,
158185
nullability,
159-
);
186+
));
160187
}
161188

162-
match data_type {
189+
Ok(match data_type {
163190
DataType::Null => DType::Null,
164191
DataType::Decimal32(precision, scale)
165192
| DataType::Decimal64(precision, scale)
@@ -189,36 +216,42 @@ impl FromArrowType<(&DataType, Nullability)> for DType {
189216
| DataType::LargeList(e)
190217
| DataType::ListView(e)
191218
| DataType::LargeListView(e) => {
192-
DType::List(Arc::new(Self::from_arrow(e.as_ref())), nullability)
219+
DType::List(Arc::new(Self::try_from_arrow(e.as_ref())?), nullability)
193220
}
194221
DataType::FixedSizeList(e, size) => DType::FixedSizeList(
195-
Arc::new(Self::from_arrow(e.as_ref())),
222+
Arc::new(Self::try_from_arrow(e.as_ref())?),
196223
*size as u32,
197224
nullability,
198225
),
199-
DataType::Struct(f) => DType::Struct(StructFields::from_arrow(f), nullability),
226+
DataType::Struct(f) => DType::Struct(StructFields::try_from_arrow(f)?, nullability),
200227
DataType::Dictionary(_, value_type) => {
201-
Self::from_arrow((value_type.as_ref(), nullability))
228+
Self::try_from_arrow((value_type.as_ref(), nullability))?
202229
}
203230
DataType::RunEndEncoded(_, value_type) => {
204-
Self::from_arrow((value_type.data_type(), nullability))
231+
Self::try_from_arrow((value_type.data_type(), nullability))?
205232
}
206-
_ => unimplemented!("Arrow data type not yet supported: {:?}", data_type),
207-
}
233+
_ => vortex_bail!("Arrow data type not supported: {data_type:?}"),
234+
})
208235
}
209236
}
210237

211238
impl FromArrowType<&Field> for DType {
212239
fn from_arrow(field: &Field) -> Self {
240+
Self::try_from_arrow(field).vortex_expect("arrow field to dtype")
241+
}
242+
}
243+
244+
impl TryFromArrowType<&Field> for DType {
245+
fn try_from_arrow(field: &Field) -> VortexResult<Self> {
213246
if field
214247
.metadata()
215248
.get("ARROW:extension:name")
216249
.map(|s| s.as_str())
217250
== Some("arrow.parquet.variant")
218251
{
219-
return DType::Variant(field.is_nullable().into());
252+
return Ok(DType::Variant(field.is_nullable().into()));
220253
}
221-
Self::from_arrow((field.data_type(), field.is_nullable().into()))
254+
Self::try_from_arrow((field.data_type(), field.is_nullable().into()))
222255
}
223256
}
224257

@@ -594,4 +627,30 @@ mod test {
594627

595628
assert_eq!(original_dtype, roundtripped_dtype);
596629
}
630+
631+
// Regression test for https://github.com/vortex-data/vortex/issues/8346: unsupported Arrow
632+
// types must return an error instead of panicking with `unimplemented!`.
633+
#[rstest]
634+
#[case::duration(DataType::Duration(ArrowTimeUnit::Microsecond))]
635+
#[case::interval(DataType::Interval(arrow_schema::IntervalUnit::DayTime))]
636+
#[case::fixed_size_binary(DataType::FixedSizeBinary(3))]
637+
fn test_try_from_arrow_unsupported_type_errors(#[case] data_type: DataType) {
638+
let err = DType::try_from_arrow((&data_type, Nullability::NonNullable))
639+
.expect_err("unsupported Arrow type should not convert")
640+
.to_string();
641+
assert!(err.contains("not supported"), "unexpected error: {err}");
642+
643+
// The same unsupported type nested in a field or schema must also error cleanly.
644+
let field = Field::new("c0", data_type, true);
645+
assert!(DType::try_from_arrow(&field).is_err());
646+
let schema = Schema::new(vec![field]);
647+
assert!(DType::try_from_arrow(&schema).is_err());
648+
}
649+
650+
#[test]
651+
fn test_try_from_arrow_supported_type_succeeds() -> VortexResult<()> {
652+
let dtype = DType::try_from_arrow((&DataType::Int32, Nullability::Nullable))?;
653+
assert_eq!(dtype, DType::Primitive(PType::I32, Nullability::Nullable));
654+
Ok(())
655+
}
597656
}

‎vortex-python/src/arrays/from_arrow.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use vortex::array::IntoArray;
1515
use vortex::array::arrays::ChunkedArray;
1616
use vortex::array::arrow::FromArrowArray;
1717
use vortex::dtype::DType;
18-
use vortex::dtype::arrow::FromArrowType;
18+
use vortex::dtype::arrow::TryFromArrowType;
1919
use vortex::error::VortexError;
2020
use vortex::error::VortexResult;
2121

@@ -48,16 +48,18 @@ pub(super) fn from_arrow(obj: &Borrowed<'_, '_, PyAny>) -> PyVortexResult<PyArra
4848
ArrayRef::from_arrow(arrow_array.as_ref(), false).map_err(PyVortexError::from)
4949
})
5050
.collect::<PyVortexResult<Vec<_>>>()?;
51-
let dtype: DType = obj
51+
let arrow_dtype = obj
5252
.getattr(intern!(py, "type"))
53-
.and_then(|v| DataType::from_pyarrow(&v.as_borrowed()))
54-
.map(|dt| DType::from_arrow(&Field::new("_", dt, false)))?;
53+
.and_then(|v| DataType::from_pyarrow(&v.as_borrowed()))?;
54+
let dtype = DType::try_from_arrow(&Field::new("_", arrow_dtype, false))
55+
.map_err(|e| PyValueError::new_err(e.to_string()))?;
5556
Ok(PyArrayRef::from(
5657
ChunkedArray::try_new(encoded_chunks, dtype)?.into_array(),
5758
))
5859
} else if obj.is_instance(table)? {
5960
let array_stream = ArrowArrayStreamReader::from_pyarrow(&obj.as_borrowed())?;
60-
let dtype = DType::from_arrow(array_stream.schema());
61+
let dtype = DType::try_from_arrow(array_stream.schema())
62+
.map_err(|e| PyValueError::new_err(e.to_string()))?;
6163
let chunks = array_stream
6264
.into_iter()
6365
.map(|b| {

‎vortex-python/src/dtype/mod.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use pyo3::types::PyType;
3535
use pyo3::wrap_pyfunction;
3636
use vortex::array::arrow::ArrowSessionExt;
3737
use vortex::dtype::DType;
38-
use vortex::dtype::arrow::FromArrowType;
38+
use vortex::dtype::arrow::TryFromArrowType;
3939

4040
use crate::arrow::FromPyArrow;
4141
use crate::arrow::ToPyArrow;
@@ -198,10 +198,9 @@ impl PyDType {
198198
#[pyo3(from_py_with = import_arrow_dtype)] arrow_dtype: DataType,
199199
non_nullable: bool,
200200
) -> PyResult<Bound<'py, PyDType>> {
201-
Self::init(
202-
cls.py(),
203-
DType::from_arrow(&Field::new("_", arrow_dtype, !non_nullable)),
204-
)
201+
let dtype = DType::try_from_arrow(&Field::new("_", arrow_dtype, !non_nullable))
202+
.map_err(|e| PyValueError::new_err(e.to_string()))?;
203+
Self::init(cls.py(), dtype)
205204
}
206205
}
207206

‎vortex-python/src/io.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ use arrow_array::RecordBatchReader;
55
use arrow_array::ffi_stream::ArrowArrayStreamReader;
66
use async_fs::File;
77
use pyo3::exceptions::PyTypeError;
8+
use pyo3::exceptions::PyValueError;
89
use pyo3::prelude::*;
910
use pyo3::pyfunction;
1011
use pyo3_object_store::PyObjectStore;
@@ -17,7 +18,7 @@ use vortex::array::iter::ArrayIteratorAdapter;
1718
use vortex::array::iter::ArrayIteratorExt;
1819
use vortex::compressor::BtrBlocksCompressorBuilder;
1920
use vortex::dtype::DType;
20-
use vortex::dtype::arrow::FromArrowType;
21+
use vortex::dtype::arrow::TryFromArrowType;
2122
use vortex::error::VortexError;
2223
use vortex::error::VortexResult;
2324
use vortex::file::WriteOptionsSessionExt;
@@ -407,7 +408,8 @@ fn try_arrow_stream_to_iterator(
407408
if ob.is_instance(pa_table)? || ob.is_instance(pa_record_batch_reader)? {
408409
// Convert to Arrow stream using FFI
409410
let arrow_stream = ArrowArrayStreamReader::from_pyarrow(ob)?;
410-
let dtype = DType::from_arrow(arrow_stream.schema());
411+
let dtype = DType::try_from_arrow(arrow_stream.schema())
412+
.map_err(|e| PyValueError::new_err(e.to_string()))?;
411413

412414
// Convert Arrow RecordBatch stream to Vortex ArrayIterator
413415
let vortex_iter = arrow_stream

‎vortex-python/test/test_array.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,19 @@ def test_empty_array():
4343
def test_scalar_at_out_of_bounds():
4444
a = vortex.array([10, 42, 999, 1992])
4545
_s = a.scalar_at(10)
46+
47+
48+
@pytest.mark.parametrize(
49+
"arrow_type",
50+
[
51+
pa.duration("us"),
52+
pa.month_day_nano_interval(),
53+
pa.binary(3),
54+
],
55+
)
56+
def test_unsupported_arrow_type_raises_value_error(arrow_type: pa.DataType):
57+
# Regression test for https://github.com/vortex-data/vortex/issues/8346:
58+
# unsupported Arrow types must surface as a clean ValueError, not a PanicException.
59+
table = pa.table({"c0": pa.array([], type=arrow_type)})
60+
with pytest.raises(ValueError):
61+
_ = vortex.array(table)

0 commit comments

Comments
 (0)