Skip to content

Commit 1ed57cb

Browse files
authored
[execution] Remove deprecated disk manager configuration API (apache#23139)
### Which issue does this PR close? - Part of apache#23080 ### Rationale for this change `DiskManagerConfig` and `DiskManager::try_new` were deprecated in version 48.0.0 in favor of the new `DiskManagerBuilder`. This PR remove this deprecated methods and update Runtime env initialization logic ### What changes are included in this PR? - Removed `RuntimeEnvBuilder::with_disk_manager` depercated method. - Removed deprecated `DiskManagerConfig` enum and its associated constructor methods. - Removed deprecated `DiskManager::try_new` method. - Refactored `RuntimeEnvBuilder` to store `Option<Arc<DiskManager>>` instead of the old configuration enum. - Update `RuntimeEnvBuilder::build` logic by using matching to handle the existing manager, builder, or default initialization. ### Are these changes tested? verifed by running local tests ### Are there any user-facing changes? Yes. This removes the deprecated public Rust APIs `RuntimeEnvBuilder::with_disk_manager`, `DiskManagerConfig` and `DiskManager::try_new`. Downstream users who need to configure the disk manager should migrate to using `DiskManager::builder`. This is an API change and should be labeled as `api change`.
1 parent 5fcc550 commit 1ed57cb

2 files changed

Lines changed: 10 additions & 97 deletions

File tree

‎datafusion/execution/src/disk_manager.rs‎

Lines changed: 0 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -117,46 +117,6 @@ pub enum DiskManagerMode {
117117
Disabled,
118118
}
119119

120-
/// Configuration for temporary disk access
121-
#[deprecated(since = "48.0.0", note = "Use DiskManagerBuilder instead")]
122-
#[derive(Debug, Clone, Default)]
123-
#[allow(clippy::allow_attributes)]
124-
#[allow(deprecated)]
125-
pub enum DiskManagerConfig {
126-
/// Use the provided [DiskManager] instance
127-
Existing(Arc<DiskManager>),
128-
129-
/// Create a new [DiskManager] that creates temporary files within
130-
/// a temporary directory chosen by the OS
131-
#[default]
132-
NewOs,
133-
134-
/// Create a new [DiskManager] that creates temporary files within
135-
/// the specified directories
136-
NewSpecified(Vec<PathBuf>),
137-
138-
/// Disable disk manager, attempts to create temporary files will error
139-
Disabled,
140-
}
141-
142-
#[expect(deprecated)]
143-
impl DiskManagerConfig {
144-
/// Create temporary files in a temporary directory chosen by the OS
145-
pub fn new() -> Self {
146-
Self::default()
147-
}
148-
149-
/// Create temporary files using the provided disk manager
150-
pub fn new_existing(existing: Arc<DiskManager>) -> Self {
151-
Self::Existing(existing)
152-
}
153-
154-
/// Create temporary files in the specified directories
155-
pub fn new_specified(paths: Vec<PathBuf>) -> Self {
156-
Self::NewSpecified(paths)
157-
}
158-
}
159-
160120
/// Manages files generated during query execution, e.g. spill files generated
161121
/// while processing dataset larger than available memory.
162122
#[derive(Debug)]
@@ -192,41 +152,6 @@ impl DiskManager {
192152
DiskManagerBuilder::default()
193153
}
194154

195-
/// Create a DiskManager given the configuration
196-
#[expect(deprecated)]
197-
#[deprecated(since = "48.0.0", note = "Use DiskManager::builder() instead")]
198-
pub fn try_new(config: DiskManagerConfig) -> Result<Arc<Self>> {
199-
match config {
200-
DiskManagerConfig::Existing(manager) => Ok(manager),
201-
DiskManagerConfig::NewOs => Ok(Arc::new(Self {
202-
local_dirs: Mutex::new(Some(vec![])),
203-
max_temp_directory_size: AtomicU64::new(DEFAULT_MAX_TEMP_DIRECTORY_SIZE),
204-
used_disk_space: Arc::new(AtomicU64::new(0)),
205-
active_files_count: Arc::new(AtomicUsize::new(0)),
206-
})),
207-
DiskManagerConfig::NewSpecified(conf_dirs) => {
208-
let local_dirs = create_local_dirs(&conf_dirs)?;
209-
debug!(
210-
"Created local dirs {local_dirs:?} as DataFusion working directory"
211-
);
212-
Ok(Arc::new(Self {
213-
local_dirs: Mutex::new(Some(local_dirs)),
214-
max_temp_directory_size: AtomicU64::new(
215-
DEFAULT_MAX_TEMP_DIRECTORY_SIZE,
216-
),
217-
used_disk_space: Arc::new(AtomicU64::new(0)),
218-
active_files_count: Arc::new(AtomicUsize::new(0)),
219-
}))
220-
}
221-
DiskManagerConfig::Disabled => Ok(Arc::new(Self {
222-
local_dirs: Mutex::new(None),
223-
max_temp_directory_size: AtomicU64::new(DEFAULT_MAX_TEMP_DIRECTORY_SIZE),
224-
used_disk_space: Arc::new(AtomicU64::new(0)),
225-
active_files_count: Arc::new(AtomicUsize::new(0)),
226-
})),
227-
}
228-
}
229-
230155
/// Atomically set the max temp directory size at runtime.
231156
///
232157
/// Takes `&self`, so it works through `Arc<DiskManager>` without requiring

‎datafusion/execution/src/runtime_env.rs‎

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
//! Execution [`RuntimeEnv`] environment that manages access to object
1919
//! store, memory manager, disk manager.
2020
21-
#[expect(deprecated)]
22-
use crate::disk_manager::{DiskManagerConfig, SpillingProgress};
21+
use crate::disk_manager::SpillingProgress;
2322
use crate::{
2423
disk_manager::{DiskManager, DiskManagerBuilder, DiskManagerMode},
2524
memory_pool::{
@@ -333,9 +332,8 @@ impl Default for RuntimeEnv {
333332
/// See example on [`RuntimeEnv`]
334333
#[derive(Clone)]
335334
pub struct RuntimeEnvBuilder {
336-
#[expect(deprecated)]
337335
/// DiskManager to manage temporary disk file usage
338-
pub disk_manager: DiskManagerConfig,
336+
pub disk_manager: Option<Arc<DiskManager>>,
339337
/// DiskManager builder to manager temporary disk file usage
340338
pub disk_manager_builder: Option<DiskManagerBuilder>,
341339
/// [`MemoryPool`] from which to allocate memory
@@ -371,14 +369,6 @@ impl RuntimeEnvBuilder {
371369
}
372370
}
373371

374-
#[expect(deprecated)]
375-
#[deprecated(since = "48.0.0", note = "Use with_disk_manager_builder instead")]
376-
/// Customize disk manager
377-
pub fn with_disk_manager(mut self, disk_manager: DiskManagerConfig) -> Self {
378-
self.disk_manager = disk_manager;
379-
self
380-
}
381-
382372
/// Customize the disk manager builder
383373
pub fn with_disk_manager_builder(mut self, disk_manager: DiskManagerBuilder) -> Self {
384374
self.disk_manager_builder = Some(disk_manager);
@@ -472,14 +462,15 @@ impl RuntimeEnvBuilder {
472462
let memory_pool =
473463
memory_pool.unwrap_or_else(|| Arc::new(UnboundedMemoryPool::default()));
474464

465+
let disk_manager: Arc<DiskManager> = match (disk_manager, disk_manager_builder) {
466+
(_, Some(builder)) => Arc::new(builder.build()?),
467+
(Some(manager), None) => manager,
468+
(None, None) => Arc::new(DiskManagerBuilder::default().build()?),
469+
};
470+
475471
Ok(RuntimeEnv {
476472
memory_pool,
477-
disk_manager: if let Some(builder) = disk_manager_builder {
478-
Arc::new(builder.build()?)
479-
} else {
480-
#[expect(deprecated)]
481-
DiskManager::try_new(disk_manager)?
482-
},
473+
disk_manager,
483474
cache_manager: CacheManager::try_new(&cache_manager)?,
484475
object_store_registry,
485476
#[cfg(feature = "parquet_encryption")]
@@ -511,10 +502,7 @@ impl RuntimeEnvBuilder {
511502
};
512503

513504
Self {
514-
#[expect(deprecated)]
515-
disk_manager: DiskManagerConfig::Existing(Arc::clone(
516-
&runtime_env.disk_manager,
517-
)),
505+
disk_manager: Some(Arc::clone(&runtime_env.disk_manager)),
518506
disk_manager_builder: None,
519507
memory_pool: Some(Arc::clone(&runtime_env.memory_pool)),
520508
cache_manager: cache_config,

0 commit comments

Comments
 (0)