Skip to content

Commit c8390c4

Browse files
committed
perf: create DiskManager Directories-mode working dirs lazily
DiskManagerBuilder::build in DiskManagerMode::Directories called create_local_dirs, so a datafusion-* directory was created in every configured directory when the DiskManager was built, and removed recursively when it was dropped, whether or not anything spilled. An engine that builds a RuntimeEnv per query or per task paid for that every time. Carry the configured paths on the DiskManager instead and create the working directories on the first create_tmp_file, as DiskManagerMode::OsTmpDirectory already does. Two visible differences, both noted on the issue: temp_dir_paths() returns nothing until the first spill, and an unusable configured directory now fails the first spill rather than build(). Closes apache#26446
1 parent 4f2dbf7 commit c8390c4

1 file changed

Lines changed: 64 additions & 22 deletions

File tree

‎datafusion/execution/src/disk_manager.rs‎

Lines changed: 64 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -117,28 +117,28 @@ impl DiskManagerBuilder {
117117
match self.mode {
118118
DiskManagerMode::OsTmpDirectory => Ok(DiskManager {
119119
local_dirs: Mutex::new(Some(vec![])),
120+
conf_dirs: vec![],
121+
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
122+
max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
123+
used_disk_space: Arc::new(AtomicU64::new(0)),
124+
active_files_count: Arc::new(AtomicUsize::new(0)),
125+
factory: None,
126+
}),
127+
DiskManagerMode::Directories(conf_dirs) => Ok(DiskManager {
128+
// Created on the first `create_tmp_file`, as `OsTmpDirectory`
129+
// already does, so building a `DiskManager` that never spills
130+
// costs no directory syscalls.
131+
local_dirs: Mutex::new(Some(vec![])),
132+
conf_dirs,
120133
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
121134
max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
122135
used_disk_space: Arc::new(AtomicU64::new(0)),
123136
active_files_count: Arc::new(AtomicUsize::new(0)),
124137
factory: None,
125138
}),
126-
DiskManagerMode::Directories(conf_dirs) => {
127-
let local_dirs = create_local_dirs(&conf_dirs)?;
128-
debug!(
129-
"Created local dirs {local_dirs:?} as DataFusion working directory"
130-
);
131-
Ok(DiskManager {
132-
local_dirs: Mutex::new(Some(local_dirs)),
133-
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
134-
max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
135-
used_disk_space: Arc::new(AtomicU64::new(0)),
136-
active_files_count: Arc::new(AtomicUsize::new(0)),
137-
factory: None,
138-
})
139-
}
140139
DiskManagerMode::Disabled => Ok(DiskManager {
141140
local_dirs: Mutex::new(None),
141+
conf_dirs: vec![],
142142
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
143143
max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
144144
used_disk_space: Arc::new(AtomicU64::new(0)),
@@ -147,6 +147,7 @@ impl DiskManagerBuilder {
147147
}),
148148
DiskManagerMode::Custom(factory) => Ok(DiskManager {
149149
local_dirs: Mutex::new(None),
150+
conf_dirs: vec![],
150151
max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
151152
max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),
152153
used_disk_space: Arc::new(AtomicU64::new(0)),
@@ -195,6 +196,10 @@ pub struct DiskManager {
195196
/// If `Some(vec![])` a new OS specified temporary directory will be created
196197
/// If `None` an error will be returned (configured not to spill)
197198
local_dirs: Mutex<Option<Vec<Arc<TempDir>>>>,
199+
/// Directories configured by [`DiskManagerMode::Directories`], whose working
200+
/// directories are created on the first `create_tmp_file`. Empty in every
201+
/// other mode, where an OS temporary directory is used instead.
202+
conf_dirs: Vec<PathBuf>,
198203
/// The maximum amount of data (in bytes) stored inside the temporary directories.
199204
/// Default to 100GB. Stored as `AtomicU64` so it can be adjusted at runtime
200205
/// without requiring exclusive (`&mut`) access to the `DiskManager`.
@@ -354,17 +359,23 @@ impl DiskManager {
354359
)
355360
})?;
356361

357-
// Create a temporary directory if needed
362+
// Create the working directories if needed
358363
if local_dirs.is_empty() {
359-
let tempdir = tempfile::tempdir().map_err(DataFusionError::IoError)?;
364+
if self.conf_dirs.is_empty() {
365+
let tempdir = tempfile::tempdir().map_err(DataFusionError::IoError)?;
360366

361-
debug!(
362-
"Created directory '{:?}' as DataFusion tempfile directory for {}",
363-
tempdir.path().to_string_lossy(),
364-
request_description,
365-
);
367+
debug!(
368+
"Created directory '{:?}' as DataFusion tempfile directory for {}",
369+
tempdir.path().to_string_lossy(),
370+
request_description,
371+
);
366372

367-
local_dirs.push(Arc::new(tempdir));
373+
local_dirs.push(Arc::new(tempdir));
374+
} else {
375+
let created = create_local_dirs(&self.conf_dirs)?;
376+
debug!("Created local dirs {created:?} as DataFusion working directory");
377+
local_dirs.extend(created);
378+
}
368379
}
369380

370381
let dir_index = rng().random_range(0..local_dirs.len());
@@ -719,6 +730,37 @@ mod tests {
719730
.unwrap();
720731
}
721732

733+
#[test]
734+
fn test_directories_mode_creates_dirs_lazily() {
735+
let dir = TempDir::new().unwrap();
736+
let root = dir.path().to_path_buf();
737+
738+
let manager = Arc::new(
739+
DiskManagerBuilder::default()
740+
.with_mode(DiskManagerMode::Directories(vec![root.clone()]))
741+
.build()
742+
.unwrap(),
743+
);
744+
745+
// Nothing has spilled, so no working directory should exist yet.
746+
let entries: Vec<_> = std::fs::read_dir(&root).unwrap().collect();
747+
assert!(
748+
entries.is_empty(),
749+
"expected no directories before the first spill, found {entries:?}"
750+
);
751+
assert!(manager.temp_dir_paths().is_empty());
752+
753+
// The first temporary file creates the working directory under the
754+
// configured path.
755+
let spill_file = manager.create_tmp_file("lazy dir test").unwrap();
756+
let spill_path = spill_file.path().expect("spill file should have a path");
757+
assert_path_in_dirs(spill_path, std::iter::once(root.as_path()));
758+
759+
let temp_dir_paths = manager.temp_dir_paths();
760+
assert_eq!(temp_dir_paths.len(), 1);
761+
assert!(temp_dir_paths[0].starts_with(&root));
762+
}
763+
722764
/// Asserts that `file_path` is found anywhere in any of `dir` directories
723765
fn assert_path_in_dirs<'a>(
724766
file_path: &'a Path,

0 commit comments

Comments
 (0)