3737//! tasks own it through their `RuntimeEnv`. It additionally holds strong references to the last
3838//! [`MAX_IDLE_CACHES`] caches it handed out. Those matter when an executor runs one task at a
3939//! time: the next task of the stage starts after the previous one released its plan. Whatever the
40- //! tasks do, at most [`MAX_IDLE_CACHES`] caches outlive the tasks that used them. A retained cache
41- //! is freed when tasks of [`MAX_IDLE_CACHES`] other stages have asked for theirs since, or when
42- //! the plugin shuts down and [`clear_shared_metadata_caches `] runs. Nothing frees it sooner: an
43- //! executor is not told that a stage has finished, so one that goes idle after a query keeps the
44- //! caches of its last stages until it shuts down .
40+ //! tasks do, at most [`MAX_IDLE_CACHES`] caches outlive the tasks that used them. An executor is
41+ //! not told that a stage has finished, so a retained cache is freed once no task of its stage has
42+ //! run for [`IDLE_CACHE_TIMEOUT`], once tasks of [`MAX_IDLE_CACHES `] other stages have asked for
43+ //! theirs, or when the plugin shuts down and [`clear_shared_metadata_caches`] runs, whichever comes
44+ //! first. An executor that goes idle after a query frees them within the timeout .
4545//!
4646//! A stage's cache holds DataFusion's per-task limit for each of its tasks that is running, and one
4747//! task's limit once none is, so that sharing does not shrink the room each task had with a cache
5656//! the file's path within its object store, which drops the bucket, so [`scope_to_store`] gives
5757//! each scan a view of the cache that also keys by the scan's object store.
5858
59+ use crate :: execution:: jni_api:: get_runtime;
5960use crate :: execution:: spark_config:: {
6061 SparkConfig , COMET_APP_ID , COMET_PARQUET_SHARED_METADATA_CACHE_ENABLED , COMET_STAGE_ATTEMPT ,
6162 COMET_STAGE_ID ,
@@ -69,23 +70,35 @@ use datafusion::execution::cache::default_cache::DefaultCache;
6970use datafusion:: execution:: cache:: { Cache , CacheEntryInfo } ;
7071use object_store:: path:: Path ;
7172use std:: collections:: { HashMap , VecDeque } ;
73+ use std:: sync:: atomic:: { AtomicBool , Ordering } ;
7274use std:: sync:: { Arc , LazyLock , Mutex , Weak } ;
73- use std:: time:: Duration ;
75+ use std:: time:: { Duration , Instant } ;
7476
7577/// How many caches of stages with no running task the registry holds on to.
7678const MAX_IDLE_CACHES : usize = 2 ;
7779
80+ /// How long the registry holds on to the cache of a stage with no running task. The next task of a
81+ /// stage that runs one task at a time on an executor starts well within it.
82+ const IDLE_CACHE_TIMEOUT : Duration = Duration :: from_secs ( 10 ) ;
83+
7884#[ derive( Debug , Clone , PartialEq , Eq , Hash ) ]
7985pub ( crate ) struct StageKey {
8086 pub app_id : String ,
8187 pub stage_id : i32 ,
8288 pub stage_attempt : i32 ,
8389}
8490
91+ /// How many tasks of a stage are running, and since when none has been.
92+ #[ derive( Default ) ]
93+ struct Running {
94+ tasks : usize ,
95+ idle_since : Option < Instant > ,
96+ }
97+
8598/// The cache of one stage, and how many of its tasks are running.
8699struct StageCache {
87100 cache : Arc < DefaultCache < Path , CachedFileMetadataEntry > > ,
88- running : Mutex < usize > ,
101+ running : Mutex < Running > ,
89102}
90103
91104impl StageCache {
@@ -95,17 +108,23 @@ impl StageCache {
95108 DefaultCache :: new ( DEFAULT_METADATA_CACHE_LIMIT )
96109 . with_name ( "SharedFileMetadataCache" ) ,
97110 ) ,
98- running : Mutex :: new ( 0 ) ,
111+ running : Mutex :: new ( Running :: default ( ) ) ,
99112 }
100113 }
101114
102115 /// Counts a task in or out, and sizes the cache for the tasks left running. Lowering the limit
103116 /// evicts the least recently used entries down to it.
104117 fn update_running ( & self , update : impl FnOnce ( usize ) -> usize ) {
105118 let mut running = self . running . lock ( ) . unwrap ( ) ;
106- * running = update ( * running) ;
119+ running. tasks = update ( running. tasks ) ;
120+ running. idle_since = ( running. tasks == 0 ) . then ( Instant :: now) ;
107121 self . cache
108- . update_cache_limit ( DEFAULT_METADATA_CACHE_LIMIT . saturating_mul ( ( * running) . max ( 1 ) ) ) ;
122+ . update_cache_limit ( DEFAULT_METADATA_CACHE_LIMIT . saturating_mul ( running. tasks . max ( 1 ) ) ) ;
123+ }
124+
125+ /// When the cache has gone unused for `timeout`, or `None` while a task of the stage runs.
126+ fn expires_at ( & self , timeout : Duration ) -> Option < Instant > {
127+ Some ( self . running . lock ( ) . unwrap ( ) . idle_since ? + timeout)
109128 }
110129}
111130
@@ -138,7 +157,7 @@ impl Drop for TaskCache {
138157impl std:: fmt:: Debug for TaskCache {
139158 fn fmt ( & self , f : & mut std:: fmt:: Formatter < ' _ > ) -> std:: fmt:: Result {
140159 f. debug_struct ( "TaskCache" )
141- . field ( "running" , & * self . stage . running . lock ( ) . unwrap ( ) )
160+ . field ( "running" , & self . stage . running . lock ( ) . unwrap ( ) . tasks )
142161 . finish ( )
143162 }
144163}
@@ -208,6 +227,8 @@ struct Registry {
208227#[ derive( Default ) ]
209228pub ( crate ) struct SharedMetadataCaches {
210229 registry : Mutex < Registry > ,
230+ /// Whether a task that frees the idle caches is running.
231+ expiring : AtomicBool ,
211232}
212233
213234impl SharedMetadataCaches {
@@ -231,10 +252,60 @@ impl SharedMetadataCaches {
231252 Arc :: new ( TaskCache :: new ( stage) )
232253 }
233254
255+ /// Frees the retained caches that have gone unused for `timeout` at `now`, and returns when
256+ /// the next of the remaining ones would, or `None` when none is retained.
257+ fn expire_idle ( & self , now : Instant , timeout : Duration ) -> Option < Instant > {
258+ let mut registry = self . registry . lock ( ) . unwrap ( ) ;
259+ let mut expired = Vec :: new ( ) ;
260+ let mut next: Option < Instant > = None ;
261+ registry
262+ . retained
263+ . retain ( |( _, stage) | match stage. expires_at ( timeout) {
264+ Some ( at) if at <= now => {
265+ expired. push ( Arc :: clone ( stage) ) ;
266+ false
267+ }
268+ at => {
269+ // A stage with a running task is checked again a whole timeout later.
270+ let at = at. unwrap_or ( now + timeout) ;
271+ next = Some ( next. map_or ( at, |next| next. min ( at) ) ) ;
272+ true
273+ }
274+ } ) ;
275+ drop ( expired) ;
276+ registry. alive . retain ( |_, weak| weak. strong_count ( ) > 0 ) ;
277+ next
278+ }
279+
280+ /// Starts a task on `runtime` that frees retained caches as they time out, unless one is
281+ /// running. It stops once no cache is retained, and the next request for a cache starts another.
282+ fn expire_idle_on ( & ' static self , runtime : & tokio:: runtime:: Handle , timeout : Duration ) {
283+ if self . expiring . swap ( true , Ordering :: AcqRel ) {
284+ return ;
285+ }
286+ runtime. spawn ( async move {
287+ loop {
288+ while let Some ( next) = self . expire_idle ( Instant :: now ( ) , timeout) {
289+ tokio:: time:: sleep_until ( next. into ( ) ) . await ;
290+ }
291+ self . expiring . store ( false , Ordering :: Release ) ;
292+ // A cache retained after the last check found this task still running, so carry
293+ // on unless no cache is retained or another task has started since.
294+ if self . registry . lock ( ) . unwrap ( ) . retained . is_empty ( )
295+ || self . expiring . swap ( true , Ordering :: AcqRel )
296+ {
297+ break ;
298+ }
299+ }
300+ } ) ;
301+ }
302+
234303 /// Forgets every stage, freeing the retained caches. Running tasks keep the caches they hold.
235304 fn clear ( & self ) {
236305 let dropped = std:: mem:: take ( & mut * self . registry . lock ( ) . unwrap ( ) ) ;
237306 drop ( dropped) ;
307+ // The runtime that ran the expiring task is shut down along with the caches.
308+ self . expiring . store ( false , Ordering :: Release ) ;
238309 }
239310
240311 #[ cfg( test) ]
@@ -364,6 +435,7 @@ pub(crate) fn shared_metadata_cache_for_task(
364435 stage_attempt : parse ( COMET_STAGE_ATTEMPT ) ?,
365436 } ;
366437 let cache: Arc < FileMetadataCache > = CACHES . cache_for ( key) ;
438+ CACHES . expire_idle_on ( & get_runtime ( ) , IDLE_CACHE_TIMEOUT ) ;
367439 Some ( cache)
368440}
369441
@@ -447,6 +519,67 @@ mod tests {
447519 assert ! ( same_stage( & running, & caches. cache_for( key( 0 , 0 ) ) ) ) ;
448520 }
449521
522+ #[ test]
523+ fn an_idle_cache_is_freed_after_the_timeout ( ) {
524+ let caches = SharedMetadataCaches :: default ( ) ;
525+ let running = caches. cache_for ( key ( 0 , 0 ) ) ;
526+ let idle = Arc :: downgrade ( & caches. cache_for ( key ( 1 , 0 ) ) . stage ) ;
527+ let idle_since = Instant :: now ( ) ;
528+ let timeout = Duration :: from_secs ( 10 ) ;
529+
530+ // Not yet timed out; the running stage is checked a timeout from now.
531+ let next = caches. expire_idle ( idle_since, timeout) . unwrap ( ) ;
532+ assert ! ( idle. upgrade( ) . is_some( ) ) ;
533+ assert ! ( next <= idle_since + timeout + Duration :: from_secs( 1 ) ) ;
534+
535+ let later = idle_since + timeout + Duration :: from_secs ( 1 ) ;
536+ assert ! ( caches. expire_idle( later, timeout) . is_some( ) ) ;
537+ assert ! ( idle. upgrade( ) . is_none( ) ) ;
538+ assert_eq ! ( caches. len( ) , 1 ) ;
539+ // A stage with a running task is never timed out.
540+ assert ! ( same_stage( & running, & caches. cache_for( key( 0 , 0 ) ) ) ) ;
541+
542+ drop ( running) ;
543+ let much_later = later + 2 * timeout;
544+ assert ! ( caches. expire_idle( much_later, timeout) . is_none( ) ) ;
545+ assert_eq ! ( caches. len( ) , 0 ) ;
546+ }
547+
548+ #[ test]
549+ fn a_task_of_the_stage_resets_its_timeout ( ) {
550+ let caches = SharedMetadataCaches :: default ( ) ;
551+ drop ( caches. cache_for ( key ( 1 , 0 ) ) ) ;
552+ let timeout = Duration :: from_secs ( 10 ) ;
553+ let start = Instant :: now ( ) ;
554+ std:: thread:: sleep ( Duration :: from_millis ( 20 ) ) ;
555+ // The next task of the stage runs and finishes, so the cache went idle after `start`.
556+ drop ( caches. cache_for ( key ( 1 , 0 ) ) ) ;
557+ assert ! ( caches
558+ . expire_idle( start + timeout, timeout)
559+ . is_some_and( |next| next > start + timeout) ) ;
560+ assert_eq ! ( caches. len( ) , 1 ) ;
561+ }
562+
563+ #[ test]
564+ fn the_expiring_task_frees_idle_caches_and_stops ( ) {
565+ static CACHES : LazyLock < SharedMetadataCaches > =
566+ LazyLock :: new ( SharedMetadataCaches :: default) ;
567+ let runtime = tokio:: runtime:: Builder :: new_current_thread ( )
568+ . enable_time ( )
569+ . build ( )
570+ . unwrap ( ) ;
571+ let idle = Arc :: downgrade ( & CACHES . cache_for ( key ( 1 , 0 ) ) . stage ) ;
572+ let timeout = Duration :: from_millis ( 50 ) ;
573+ CACHES . expire_idle_on ( runtime. handle ( ) , timeout) ;
574+ runtime. block_on ( async {
575+ while CACHES . expiring . load ( Ordering :: Acquire ) {
576+ tokio:: time:: sleep ( Duration :: from_millis ( 10 ) ) . await ;
577+ }
578+ } ) ;
579+ assert ! ( idle. upgrade( ) . is_none( ) ) ;
580+ assert_eq ! ( CACHES . len( ) , 0 ) ;
581+ }
582+
450583 #[ test]
451584 fn clearing_frees_the_retained_caches_but_not_a_running_one ( ) {
452585 let caches = SharedMetadataCaches :: default ( ) ;
0 commit comments