Skip to content

Commit beb7c15

Browse files
AdamGSrobert3005
andauthored
Bump Datafusion to 55 (#9334)
## Rationale for this change Periodic update to DataFusion ## What changes are included in this PR? 1. Bump Arrow to 59.2 1. Bump DF to 55.0.0 (apache/datafusion#22393) 2. Bump geoarrow dependency (requires release to support Arrow 59) ~ Currently this PR also includes bumping our MSRV to 1.95 because `geoarrow` requires that (DataFusion only requires 1.94) ~ Now #9334. ## What APIs are changed? Are there any user-facing changes? Just the dependency changes --------- Signed-off-by: Adam Gutglick <[email protected]> Signed-off-by: Robert Kruszewski <[email protected]> Signed-off-by: Robert Kruszewski <[email protected]> Co-authored-by: Robert Kruszewski <[email protected]>
1 parent 28f7cf0 commit beb7c15

24 files changed

Lines changed: 1492 additions & 573 deletions

File tree

‎Cargo.lock‎

Lines changed: 1278 additions & 383 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 45 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -102,16 +102,16 @@ anyhow = "1.0.100"
102102
arbitrary = "1.3.2"
103103
arc-swap = "1.9"
104104
arcref = "0.2.0"
105-
arrow-arith = "58.3"
106-
arrow-array = "58.3"
107-
arrow-buffer = "58.3"
108-
arrow-cast = "58.3"
109-
arrow-data = "58.3"
110-
arrow-ipc = "58.3"
111-
arrow-row = "58.3"
112-
arrow-schema = "58.3"
113-
arrow-select = "58.3"
114-
arrow-string = "58.3"
105+
arrow-arith = "59.2"
106+
arrow-array = "59.2"
107+
arrow-buffer = "59.2"
108+
arrow-cast = "59.2"
109+
arrow-data = "59.2"
110+
arrow-ipc = "59.2"
111+
arrow-row = "59.2"
112+
arrow-schema = "59.2"
113+
arrow-select = "59.2"
114+
arrow-string = "59.2"
115115
async-executor = "1.14.0"
116116
async-fs = "2.2.0"
117117
async-lock = "3.4"
@@ -126,8 +126,8 @@ bytes = "1.11.1"
126126
bzip2 = "0.6.0"
127127
cbindgen = "0.29.0"
128128
cc = "1.2"
129-
cfg-if = "1.0.1"
130-
chrono = "0.4.44"
129+
cfg-if = "1.0.4"
130+
chrono = "0.4.45"
131131
clap = "4.5"
132132
criterion = { package = "codspeed-criterion-compat-walltime", version = "5.0.0" }
133133
crossbeam-channel = "0.5.16"
@@ -139,22 +139,24 @@ cudarc = { version = "0.19.0", features = [
139139
"cuda-12050",
140140
] }
141141
custom-labels = "0.4.4"
142-
dashmap = "6.1.0"
143-
datafusion = { version = "54", default-features = false, features = ["sql"] }
144-
datafusion-catalog = { version = "54" }
145-
datafusion-common = { version = "54" }
146-
datafusion-common-runtime = { version = "54" }
147-
datafusion-datasource = { version = "54", default-features = false }
148-
datafusion-execution = { version = "54" }
149-
datafusion-expr = { version = "54" }
150-
datafusion-functions = { version = "54" }
151-
datafusion-functions-nested = { version = "54" }
152-
datafusion-physical-expr = { version = "54" }
153-
datafusion-physical-expr-adapter = { version = "54" }
154-
datafusion-physical-expr-common = { version = "54" }
155-
datafusion-physical-plan = { version = "54" }
156-
datafusion-pruning = { version = "54" }
157-
datafusion-sqllogictest = { version = "54" }
142+
dashmap = "6.2.1"
143+
datafusion = { version = "55.0.0", default-features = false, features = [
144+
"sql",
145+
] }
146+
datafusion-catalog = { version = "55.0.0" }
147+
datafusion-common = { version = "55.0.0" }
148+
datafusion-common-runtime = { version = "55.0.0" }
149+
datafusion-datasource = { version = "55.0.0", default-features = false }
150+
datafusion-execution = { version = "55.0.0" }
151+
datafusion-expr = { version = "55.0.0" }
152+
datafusion-functions = { version = "55.0.0" }
153+
datafusion-functions-nested = { version = "55.0.0" }
154+
datafusion-physical-expr = { version = "55.0.0" }
155+
datafusion-physical-expr-adapter = { version = "55.0.0" }
156+
datafusion-physical-expr-common = { version = "55.0.0" }
157+
datafusion-physical-plan = { version = "55.0.0" }
158+
datafusion-pruning = { version = "55.0.0" }
159+
datafusion-sqllogictest = { version = "55.0.0" }
158160
divan = { package = "codspeed-divan-compat", version = "5.0.0" }
159161
enum-iterator = "2.0.0"
160162
env_logger = "0.11"
@@ -189,7 +191,7 @@ libfuzzer-sys = "0.4"
189191
libloading = "0.8"
190192
liblzma = "0.4"
191193
log = { version = "0.4.21" }
192-
memchr = "2.8.0"
194+
memchr = "2.8.1"
193195
memmap2 = "0.9.5"
194196
mimalloc = "0.1.42"
195197
moka = { version = "0.12.10", default-features = false }
@@ -199,16 +201,16 @@ num-traits = "0.2.19"
199201
num_enum = { version = "0.7.3", default-features = false }
200202
object_store = { version = "0.13.2", default-features = false }
201203
object_store_opendal = "0.58.0"
202-
once_cell = "1.21"
204+
once_cell = "1.21.4"
203205
onpair = "0.2.0"
204206
opendal = { version = "0.58.1", default-features = false }
205207
opentelemetry = "0.32.0"
206208
opentelemetry-otlp = "0.32.0"
207209
opentelemetry_sdk = "0.32.0"
208210
parking_lot = { version = "0.12.3", features = ["nightly"] }
209-
parquet = "58.3"
210-
parquet-variant = "58.3"
211-
parquet-variant-compute = "58.3"
211+
parquet = "59.2"
212+
parquet-variant = "59.2"
213+
parquet-variant-compute = "59.2"
212214
paste = "1.0.15"
213215
pco = "1.0.1"
214216
percent-encoding = "2.3.2"
@@ -260,7 +262,7 @@ spatialbench-parquet = { package = "parquet", version = "56", features = [
260262
] }
261263
static_assertions = "1.1"
262264
strum = "0.28"
263-
syn = { version = "3.0.0", features = ["full"] }
265+
syn = { version = "3.0.2", features = ["full"] }
264266
sysinfo = "0.38.0"
265267
tabled = { version = "0.21.0", default-features = false }
266268
taffy = "0.12"
@@ -275,10 +277,8 @@ thiserror = "2.0.3"
275277
tokio = { version = "1.52" }
276278
tokio-stream = "0.1.17"
277279
tokio-util = "0.7.17"
278-
vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" }
279-
# Pull these into non-public crates to support DF 58
280-
tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
281-
tpchgen-arrow = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
280+
tpchgen = "3.0.0"
281+
tpchgen-arrow = "3.0.0"
282282
tracing = { version = "0.1.41", default-features = false }
283283
tracing-perfetto = "0.1.5"
284284
tracing-subscriber = "0.3"
@@ -297,6 +297,7 @@ zstd = { version = "0.13.3", default-features = false, features = [
297297
vortex = { version = "0.1.0", path = "./vortex" }
298298
vortex-alp = { version = "0.1.0", path = "./encodings/alp", default-features = false }
299299
vortex-array = { version = "0.1.0", path = "./vortex-array", default-features = false }
300+
vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" }
300301
vortex-arrow = { version = "0.1.0", path = "./vortex-arrow", default-features = false }
301302
vortex-btrblocks = { version = "0.1.0", path = "./vortex-btrblocks", default-features = false }
302303
vortex-buffer = { version = "0.1.0", path = "./vortex-buffer", default-features = false }
@@ -443,3 +444,9 @@ incremental = false
443444
# This improved build times significantly for default common cases that we use locally
444445
[profile.dev.package.vortex-fastlanes]
445446
debug = false
447+
448+
[patch.crates-io]
449+
geoarrow = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
450+
geoarrow-array = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
451+
geoarrow-cast = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
452+
geoarrow-schema = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }

‎benchmarks/datafusion-bench/src/lib.rs‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,7 @@ use datafusion::datasource::file_format::csv::CsvFormat;
1111
use datafusion::datasource::file_format::parquet::ParquetFormat;
1212
use datafusion::datasource::provider::DefaultTableFactory;
1313
use datafusion::execution::SessionStateBuilder;
14-
use datafusion::execution::cache::DefaultListFilesCache;
1514
use datafusion::execution::cache::cache_manager::CacheManagerConfig;
16-
use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache;
1715
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
1816
use datafusion::prelude::SessionConfig;
1917
use datafusion::prelude::SessionContext;
@@ -33,12 +31,7 @@ use vortex_datafusion::VortexTableOptions;
3331
pub fn get_session_context() -> SessionContext {
3432
let mut rt_builder = RuntimeEnvBuilder::new();
3533

36-
let file_static_cache = Arc::new(DefaultFileStatisticsCache::default());
37-
let list_file_cache = Arc::new(DefaultListFilesCache::default());
38-
let cache_config = CacheManagerConfig::default()
39-
.with_file_statistics_cache(Some(file_static_cache))
40-
.with_list_files_cache(Some(list_file_cache));
41-
rt_builder = rt_builder.with_cache_manager(cache_config);
34+
rt_builder = rt_builder.with_cache_manager(CacheManagerConfig::default());
4235

4336
let rt = rt_builder
4437
.build_arc()

‎benchmarks/datafusion-bench/src/main.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -260,8 +260,7 @@ async fn register_benchmark_tables<B: Benchmark + ?Sized>(
260260
let table_url = ListingTableUrl::try_new(benchmark_base.clone(), pattern)?
261261
.with_table_ref(table_ref.clone());
262262

263-
let listing_options = ListingOptions::new(Arc::clone(&file_format))
264-
.with_session_config_options(session.state().config());
263+
let listing_options = ListingOptions::new(Arc::clone(&file_format));
265264
let mut config =
266265
ListingTableConfig::new(table_url).with_listing_options(listing_options);
267266

‎benchmarks/lance-bench/src/random_access.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,7 @@ impl RandomAccessor for LanceRandomAccessor {
114114
}
115115

116116
async fn take(&self, indices: &[u64]) -> anyhow::Result<RandomAccessorRet> {
117-
Ok(RandomAccessorRet::RecordBatch(
118-
self.dataset.take(indices, self.projection.clone()).await?,
119-
))
117+
let batch = self.dataset.take(indices, self.projection.clone()).await?;
118+
Ok(RandomAccessorRet::Native(Box::new(batch)))
120119
}
121120
}

‎encodings/parquet-variant/src/array.rs‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -137,15 +137,15 @@ impl ParquetVariant {
137137
Validity::NonNullable
138138
});
139139
let metadata = session
140-
.from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_field()), false)?;
140+
.from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_column()), false)?;
141141

142-
let value = arrow_variant
143-
.value_field()
144-
.map(|v| session.from_arrow_array(ArrowArrayRef::clone(v), value_nullable))
145-
.transpose()?;
142+
let value = Some(session.from_arrow_array(
143+
ArrowArrayRef::clone(arrow_variant.value_column()),
144+
value_nullable,
145+
)?);
146146

147147
let typed_value = arrow_variant
148-
.typed_value_field()
148+
.typed_value_column()
149149
.map(|tv| session.from_arrow_array(ArrowArrayRef::clone(tv), typed_value_nullable))
150150
.transpose()?;
151151
ParquetVariant::try_new(validity, metadata, value, typed_value).map(IntoArray::into_array)
@@ -692,7 +692,7 @@ mod tests {
692692
}
693693

694694
#[test]
695-
fn test_arrow_variant_import_typed_value_only_preserves_storage() -> VortexResult<()> {
695+
fn test_arrow_variant_import_typed_value_only_normalizes_storage() -> VortexResult<()> {
696696
let metadata = binary_view_array([b"\x01\x00", b"\x01\x00", b"\x01\x00"]);
697697
let typed_value: ArrowArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30]));
698698

@@ -711,17 +711,21 @@ mod tests {
711711
let parquet_array = vortex_arr
712712
.as_opt::<ParquetVariant>()
713713
.ok_or_else(|| vortex_err!("expected parquet variant array"))?;
714-
assert!(parquet_array.value().is_none());
714+
let mut ctx = SESSION.create_execution_ctx();
715+
let value = parquet_array
716+
.value()
717+
.ok_or_else(|| vortex_err!("expected synthesized value child"))?;
718+
assert!(value.all_invalid(&mut ctx)?);
715719

716720
let typed_value = parquet_array
717721
.typed_value()
718722
.ok_or_else(|| vortex_err!("expected typed_value child"))?
719723
.clone()
720-
.execute::<PrimitiveArray>(&mut SESSION.create_execution_ctx())?;
724+
.execute::<PrimitiveArray>(&mut ctx)?;
721725
assert_arrays_eq!(
722726
typed_value,
723-
PrimitiveArray::from_iter([10i32, 20, 30]),
724-
&mut SESSION.create_execution_ctx()
727+
PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]),
728+
&mut ctx
725729
);
726730
Ok(())
727731
}

‎encodings/parquet-variant/src/arrow.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -715,7 +715,6 @@ mod tests {
715715

716716
let actual = session.arrow().from_arrow_array(exported, &field)?;
717717

718-
assert_arrays_eq!(actual, expected, &mut ctx);
719-
Ok(())
718+
assert_variant_scalars_eq(&actual, &expected, &session)
720719
}
721720
}

‎encodings/parquet-variant/src/kernel.rs‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -841,17 +841,13 @@ mod tests {
841841

842842
let metadata = SESSION
843843
.arrow()
844-
.from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_field()), false)?;
845-
let value = arrow_variant
846-
.value_field()
847-
.map(|value| {
848-
SESSION
849-
.arrow()
850-
.from_arrow_array(ArrowArrayRef::clone(value), value_nullable)
851-
})
852-
.transpose()?;
844+
.from_arrow_array(ArrowArrayRef::clone(arrow_variant.metadata_column()), false)?;
845+
let value = Some(SESSION.arrow().from_arrow_array(
846+
ArrowArrayRef::clone(arrow_variant.value_column()),
847+
value_nullable,
848+
)?);
853849
let typed_value = arrow_variant
854-
.typed_value_field()
850+
.typed_value_column()
855851
.map(|typed_value| {
856852
SESSION
857853
.arrow()

‎vortex-bench/src/random_access/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ pub trait BenchDataset: Send + Sync {
9090
pub enum RandomAccessorRet {
9191
RecordBatch(RecordBatch),
9292
ArrayRef(ArrayRef),
93+
Native(Box<dyn std::any::Any + Send>),
9394
}
9495

9596
/// Trait for format-specific random access (take) operations.

‎vortex-cuda/build.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,9 @@ fn main() {
7474
.is_some_and(|n| n.starts_with("bit_unpack_"));
7575

7676
match path.extension().and_then(|e| e.to_str()) {
77+
// Only watch hand-written .cuh/.h files, not generated ones
78+
// (generated files are rebuilt when cuda_kernel_generator changes)
7779
Some("cuh") | Some("h") if !is_generated => {
78-
// Only watch hand-written .cuh/.h files, not generated ones
79-
// (generated files are rebuilt when cuda_kernel_generator changes)
80-
8180
println!("cargo:rerun-if-changed={}", path.display());
8281
}
8382
Some("cu") => {

0 commit comments

Comments
 (0)