Skip to content

Commit 1d7951a

Browse files
authored
Split arrow dependency from vortex-array (#8717)
Vortex-array doesn't need arrow for any particular functionality. We want to be arrow compatible and support conversion from one to another but that logic doesn't need to live with array implementations. Instead we move all arrow related logic to vortex-arrow crate that users can depend on which can depend on other crates not just vortex-array N.B. we still have arrow-buffer dependency in vortex-array since we use their i256. Having that dependency doesn't seem particularly problematic
1 parent c27a612 commit 1d7951a

127 files changed

Lines changed: 1134 additions & 1048 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 42 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ members = [
1313
"vortex-io",
1414
"vortex-proto",
1515
"vortex-array",
16+
"vortex-arrow",
1617
"vortex-row",
1718
"vortex-tensor",
1819
"vortex-json",
@@ -285,6 +286,7 @@ zstd = { version = "0.13.3", default-features = false, features = [
285286
vortex = { version = "0.1.0", path = "./vortex" }
286287
vortex-alp = { version = "0.1.0", path = "./encodings/alp", default-features = false }
287288
vortex-array = { version = "0.1.0", path = "./vortex-array", default-features = false }
289+
vortex-arrow = { version = "0.1.0", path = "./vortex-arrow", default-features = false }
288290
vortex-btrblocks = { version = "0.1.0", path = "./vortex-btrblocks", default-features = false }
289291
vortex-buffer = { version = "0.1.0", path = "./vortex-buffer", default-features = false }
290292
vortex-bytebool = { version = "0.1.0", path = "./encodings/bytebool", default-features = false }

‎benchmarks/compress-bench/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ regex = { workspace = true }
3030
tokio = { workspace = true, features = ["full"] }
3131
tracing = { workspace = true }
3232
vortex = { workspace = true }
33+
vortex-arrow = { workspace = true }
3334
vortex-bench = { workspace = true }
3435

3536
[features]

‎benchmarks/compress-bench/src/vortex.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use vortex::expr::root;
1818
use vortex::expr::select;
1919
use vortex::file::OpenOptionsSessionExt;
2020
use vortex::file::WriteOptionsSessionExt;
21+
use vortex_arrow::ToArrowType;
2122
use vortex_bench::Format;
2223
use vortex_bench::SESSION;
2324
use vortex_bench::compress::Compressor;

‎benchmarks/datafusion-bench/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ parking_lot = { workspace = true }
3939
tokio = { workspace = true, features = ["full"] }
4040
url = { workspace = true }
4141
vortex = { workspace = true, features = ["object_store", "files", "tokio"] }
42+
vortex-arrow = { workspace = true }
4243
vortex-bench = { workspace = true }
4344
vortex-cuda = { workspace = true, optional = true }
4445
vortex-datafusion = { workspace = true }

‎benchmarks/datafusion-bench/src/main.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use parking_lot::Mutex;
2929
use tokio::fs::File;
3030
use vortex::io::filesystem::FileSystemRef;
3131
use vortex::scan::DataSourceRef;
32+
use vortex_arrow::ToArrowType;
3233
use vortex_bench::Benchmark;
3334
use vortex_bench::BenchmarkArg;
3435
use vortex_bench::CompactionStrategy;

‎encodings/parquet-variant/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ parquet-variant = { workspace = true }
2525
parquet-variant-compute = { workspace = true }
2626
prost = { workspace = true }
2727
vortex-array = { workspace = true }
28+
vortex-arrow = { workspace = true }
2829
vortex-buffer = { workspace = true }
2930
vortex-error = { workspace = true }
3031
vortex-json = { workspace = true }

‎encodings/parquet-variant/src/array.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,6 @@ use vortex_array::arrays::VariantArray;
2424
use vortex_array::arrays::list::ListArrayExt;
2525
use vortex_array::arrays::struct_::StructArrayExt;
2626
use vortex_array::arrays::variant::VariantArrayExt;
27-
#[expect(
28-
deprecated,
29-
reason = "TODO(aduffy): figure out what to do with Parquet Variant"
30-
)]
31-
use vortex_array::arrow::ArrowArrayExecutor;
32-
use vortex_array::arrow::FromArrowArray;
33-
use vortex_array::arrow::to_arrow_null_buffer;
3427
use vortex_array::builtins::ArrayBuiltins;
3528
use vortex_array::dtype::DType;
3629
use vortex_array::dtype::FieldName;
@@ -41,6 +34,13 @@ use vortex_array::smallvec::smallvec;
4134
use vortex_array::validity::Validity;
4235
use vortex_array::vtable::child_to_validity;
4336
use vortex_array::vtable::validity_to_child;
37+
#[expect(
38+
deprecated,
39+
reason = "TODO(aduffy): figure out what to do with Parquet Variant"
40+
)]
41+
use vortex_arrow::ArrowArrayExecutor;
42+
use vortex_arrow::FromArrowArray;
43+
use vortex_arrow::to_arrow_null_buffer;
4444
use vortex_buffer::BitBuffer;
4545
use vortex_error::VortexExpect;
4646
use vortex_error::VortexResult;

‎encodings/parquet-variant/src/arrow.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ use vortex_array::IntoArray;
2020
use vortex_array::VTable;
2121
use vortex_array::arrays::Variant;
2222
use vortex_array::arrays::variant::VariantArrayExt;
23-
use vortex_array::arrow::ArrowExport;
24-
use vortex_array::arrow::ArrowExportVTable;
25-
use vortex_array::arrow::ArrowImport;
26-
use vortex_array::arrow::ArrowImportVTable;
27-
use vortex_array::arrow::ArrowSession;
28-
use vortex_array::arrow::ArrowSessionExt;
29-
use vortex_array::arrow::to_arrow_null_buffer;
3023
use vortex_array::dtype::DType;
24+
use vortex_arrow::ArrowExport;
25+
use vortex_arrow::ArrowExportVTable;
26+
use vortex_arrow::ArrowImport;
27+
use vortex_arrow::ArrowImportVTable;
28+
use vortex_arrow::ArrowSession;
29+
use vortex_arrow::ArrowSessionExt;
30+
use vortex_arrow::to_arrow_null_buffer;
3131
use vortex_error::VortexResult;
3232
use vortex_error::vortex_bail;
3333
use vortex_error::vortex_err;
@@ -300,11 +300,11 @@ mod tests {
300300
use vortex_array::arrays::PrimitiveArray;
301301
use vortex_array::arrays::VarBinViewArray;
302302
use vortex_array::arrays::VariantArray;
303-
use vortex_array::arrow::ArrowSessionExt;
304303
use vortex_array::assert_arrays_eq;
305304
use vortex_array::dtype::DType;
306305
use vortex_array::dtype::Nullability;
307306
use vortex_array::validity::Validity;
307+
use vortex_arrow::ArrowSessionExt;
308308
use vortex_buffer::buffer;
309309
use vortex_error::VortexResult;
310310
use vortex_session::VortexSession;

‎encodings/parquet-variant/src/kernel.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,16 @@ use vortex_array::arrays::scalar_fn::ExactScalarFn;
3737
use vortex_array::arrays::scalar_fn::ScalarFnArrayView;
3838
use vortex_array::arrays::slice::SliceExecuteAdaptor;
3939
use vortex_array::arrays::slice::SliceKernel;
40-
use vortex_array::arrow::ArrowSessionExt;
41-
use vortex_array::arrow::FromArrowArray;
4240
use vortex_array::dtype::DType;
4341
use vortex_array::kernel::ExecuteParentKernel;
4442
use vortex_array::optimizer::kernels::ArrayKernelsExt;
4543
use vortex_array::scalar_fn::ScalarFnVTable;
4644
use vortex_array::scalar_fn::fns::variant_get::VariantGet;
4745
use vortex_array::scalar_fn::fns::variant_get::VariantPath;
4846
use vortex_array::scalar_fn::fns::variant_get::VariantPathElement;
47+
use vortex_arrow::ArrowSessionExt;
48+
use vortex_arrow::FromArrowArray;
49+
use vortex_arrow::ToArrowType;
4950
use vortex_error::VortexResult;
5051
use vortex_error::vortex_ensure_eq;
5152
use vortex_error::vortex_err;
@@ -318,7 +319,6 @@ mod tests {
318319
use vortex_array::arrays::VariantArray;
319320
use vortex_array::arrays::struct_::StructArrayExt;
320321
use vortex_array::arrays::variant::VariantArrayExt;
321-
use vortex_array::arrow::FromArrowArray;
322322
use vortex_array::assert_arrays_eq;
323323
use vortex_array::assert_nth_scalar_is_null;
324324
use vortex_array::dtype::DType as VortexDType;
@@ -329,6 +329,7 @@ mod tests {
329329
use vortex_array::scalar_fn::fns::variant_get::VariantPath;
330330
use vortex_array::scalar_fn::fns::variant_get::VariantPathElement;
331331
use vortex_array::validity::Validity;
332+
use vortex_arrow::FromArrowArray;
332333
use vortex_error::VortexResult;
333334
use vortex_error::vortex_bail;
334335
use vortex_error::vortex_ensure;

0 commit comments

Comments
 (0)