@@ -117,28 +117,28 @@ impl DiskManagerBuilder {
117117 match self . mode {
118118 DiskManagerMode :: OsTmpDirectory => Ok ( DiskManager {
119119 local_dirs : Mutex :: new ( Some ( vec ! [ ] ) ) ,
120+ conf_dirs : vec ! [ ] ,
121+ max_temp_directory_size : AtomicU64 :: new ( self . max_temp_directory_size ) ,
122+ max_spill_merge_fan_in : AtomicUsize :: new ( self . max_spill_merge_fan_in ) ,
123+ used_disk_space : Arc :: new ( AtomicU64 :: new ( 0 ) ) ,
124+ active_files_count : Arc :: new ( AtomicUsize :: new ( 0 ) ) ,
125+ factory : None ,
126+ } ) ,
127+ DiskManagerMode :: Directories ( conf_dirs) => Ok ( DiskManager {
128+ // Created on the first `create_tmp_file`, as `OsTmpDirectory`
129+ // already does, so building a `DiskManager` that never spills
130+ // costs no directory syscalls.
131+ local_dirs : Mutex :: new ( Some ( vec ! [ ] ) ) ,
132+ conf_dirs,
120133 max_temp_directory_size : AtomicU64 :: new ( self . max_temp_directory_size ) ,
121134 max_spill_merge_fan_in : AtomicUsize :: new ( self . max_spill_merge_fan_in ) ,
122135 used_disk_space : Arc :: new ( AtomicU64 :: new ( 0 ) ) ,
123136 active_files_count : Arc :: new ( AtomicUsize :: new ( 0 ) ) ,
124137 factory : None ,
125138 } ) ,
126- DiskManagerMode :: Directories ( conf_dirs) => {
127- let local_dirs = create_local_dirs ( & conf_dirs) ?;
128- debug ! (
129- "Created local dirs {local_dirs:?} as DataFusion working directory"
130- ) ;
131- Ok ( DiskManager {
132- local_dirs : Mutex :: new ( Some ( local_dirs) ) ,
133- max_temp_directory_size : AtomicU64 :: new ( self . max_temp_directory_size ) ,
134- max_spill_merge_fan_in : AtomicUsize :: new ( self . max_spill_merge_fan_in ) ,
135- used_disk_space : Arc :: new ( AtomicU64 :: new ( 0 ) ) ,
136- active_files_count : Arc :: new ( AtomicUsize :: new ( 0 ) ) ,
137- factory : None ,
138- } )
139- }
140139 DiskManagerMode :: Disabled => Ok ( DiskManager {
141140 local_dirs : Mutex :: new ( None ) ,
141+ conf_dirs : vec ! [ ] ,
142142 max_temp_directory_size : AtomicU64 :: new ( self . max_temp_directory_size ) ,
143143 max_spill_merge_fan_in : AtomicUsize :: new ( self . max_spill_merge_fan_in ) ,
144144 used_disk_space : Arc :: new ( AtomicU64 :: new ( 0 ) ) ,
@@ -147,6 +147,7 @@ impl DiskManagerBuilder {
147147 } ) ,
148148 DiskManagerMode :: Custom ( factory) => Ok ( DiskManager {
149149 local_dirs : Mutex :: new ( None ) ,
150+ conf_dirs : vec ! [ ] ,
150151 max_temp_directory_size : AtomicU64 :: new ( self . max_temp_directory_size ) ,
151152 max_spill_merge_fan_in : AtomicUsize :: new ( self . max_spill_merge_fan_in ) ,
152153 used_disk_space : Arc :: new ( AtomicU64 :: new ( 0 ) ) ,
@@ -195,6 +196,10 @@ pub struct DiskManager {
195196 /// If `Some(vec![])` a new OS specified temporary directory will be created
196197 /// If `None` an error will be returned (configured not to spill)
197198 local_dirs : Mutex < Option < Vec < Arc < TempDir > > > > ,
199+ /// Directories configured by [`DiskManagerMode::Directories`], whose working
200+ /// directories are created on the first `create_tmp_file`. Empty in every
201+ /// other mode, where an OS temporary directory is used instead.
202+ conf_dirs : Vec < PathBuf > ,
198203 /// The maximum amount of data (in bytes) stored inside the temporary directories.
199204 /// Default to 100GB. Stored as `AtomicU64` so it can be adjusted at runtime
200205 /// without requiring exclusive (`&mut`) access to the `DiskManager`.
@@ -354,17 +359,23 @@ impl DiskManager {
354359 )
355360 } ) ?;
356361
357- // Create a temporary directory if needed
362+ // Create the working directories if needed
358363 if local_dirs. is_empty ( ) {
359- let tempdir = tempfile:: tempdir ( ) . map_err ( DataFusionError :: IoError ) ?;
364+ if self . conf_dirs . is_empty ( ) {
365+ let tempdir = tempfile:: tempdir ( ) . map_err ( DataFusionError :: IoError ) ?;
360366
361- debug ! (
362- "Created directory '{:?}' as DataFusion tempfile directory for {}" ,
363- tempdir. path( ) . to_string_lossy( ) ,
364- request_description,
365- ) ;
367+ debug ! (
368+ "Created directory '{:?}' as DataFusion tempfile directory for {}" ,
369+ tempdir. path( ) . to_string_lossy( ) ,
370+ request_description,
371+ ) ;
366372
367- local_dirs. push ( Arc :: new ( tempdir) ) ;
373+ local_dirs. push ( Arc :: new ( tempdir) ) ;
374+ } else {
375+ let created = create_local_dirs ( & self . conf_dirs ) ?;
376+ debug ! ( "Created local dirs {created:?} as DataFusion working directory" ) ;
377+ local_dirs. extend ( created) ;
378+ }
368379 }
369380
370381 let dir_index = rng ( ) . random_range ( 0 ..local_dirs. len ( ) ) ;
@@ -719,6 +730,37 @@ mod tests {
719730 . unwrap ( ) ;
720731 }
721732
733+ #[ test]
734+ fn test_directories_mode_creates_dirs_lazily ( ) {
735+ let dir = TempDir :: new ( ) . unwrap ( ) ;
736+ let root = dir. path ( ) . to_path_buf ( ) ;
737+
738+ let manager = Arc :: new (
739+ DiskManagerBuilder :: default ( )
740+ . with_mode ( DiskManagerMode :: Directories ( vec ! [ root. clone( ) ] ) )
741+ . build ( )
742+ . unwrap ( ) ,
743+ ) ;
744+
745+ // Nothing has spilled, so no working directory should exist yet.
746+ let entries: Vec < _ > = std:: fs:: read_dir ( & root) . unwrap ( ) . collect ( ) ;
747+ assert ! (
748+ entries. is_empty( ) ,
749+ "expected no directories before the first spill, found {entries:?}"
750+ ) ;
751+ assert ! ( manager. temp_dir_paths( ) . is_empty( ) ) ;
752+
753+ // The first temporary file creates the working directory under the
754+ // configured path.
755+ let spill_file = manager. create_tmp_file ( "lazy dir test" ) . unwrap ( ) ;
756+ let spill_path = spill_file. path ( ) . expect ( "spill file should have a path" ) ;
757+ assert_path_in_dirs ( spill_path, std:: iter:: once ( root. as_path ( ) ) ) ;
758+
759+ let temp_dir_paths = manager. temp_dir_paths ( ) ;
760+ assert_eq ! ( temp_dir_paths. len( ) , 1 ) ;
761+ assert ! ( temp_dir_paths[ 0 ] . starts_with( & root) ) ;
762+ }
763+
722764 /// Asserts that `file_path` is found anywhere in any of `dir` directories
723765 fn assert_path_in_dirs < ' a > (
724766 file_path : & ' a Path ,
0 commit comments