3737//! tasks own it through their `RuntimeEnv`. It additionally holds strong references to the last
3838//! [`MAX_IDLE_CACHES`] caches it handed out. Those matter when an executor runs one task at a
3939//! time: the next task of the stage starts after the previous one released its plan. Whatever the
40- //! tasks do, at most [`MAX_IDLE_CACHES`] caches outlive the tasks that used them.
40+ //! tasks do, at most [`MAX_IDLE_CACHES`] caches outlive the tasks that used them. A retained cache
41+ //! is freed when tasks of [`MAX_IDLE_CACHES`] other stages have asked for theirs since, or when
42+ //! the plugin shuts down and [`clear_shared_metadata_caches`] runs. Nothing frees it sooner: an
43+ //! executor is not told that a stage has finished, so one that goes idle after a query keeps the
44+ //! caches of its last stages until it shuts down.
4145//!
4246//! A stage's cache holds DataFusion's per-task limit for each of its tasks that is running, and one
4347//! task's limit once none is, so that sharing does not shrink the room each task had with a cache
@@ -65,7 +69,7 @@ use datafusion::execution::cache::default_cache::DefaultCache;
6569use datafusion:: execution:: cache:: { Cache , CacheEntryInfo } ;
6670use object_store:: path:: Path ;
6771use std:: collections:: { HashMap , VecDeque } ;
68- use std:: sync:: { Arc , Mutex , OnceLock , Weak } ;
72+ use std:: sync:: { Arc , LazyLock , Mutex , Weak } ;
6973use std:: time:: Duration ;
7074
7175/// How many caches of stages with no running task the registry holds on to.
@@ -227,12 +231,26 @@ impl SharedMetadataCaches {
227231 Arc :: new ( TaskCache :: new ( stage) )
228232 }
229233
234+ /// Forgets every stage, freeing the retained caches. Running tasks keep the caches they hold.
235+ fn clear ( & self ) {
236+ let dropped = std:: mem:: take ( & mut * self . registry . lock ( ) . unwrap ( ) ) ;
237+ drop ( dropped) ;
238+ }
239+
230240 #[ cfg( test) ]
231241 fn len ( & self ) -> usize {
232242 self . registry . lock ( ) . unwrap ( ) . alive . len ( )
233243 }
234244}
235245
246+ /// The registry of the executor's stages.
247+ static CACHES : LazyLock < SharedMetadataCaches > = LazyLock :: new ( SharedMetadataCaches :: default) ;
248+
249+ /// Frees the caches retained for stages with no running task, as the plugin shuts down.
250+ pub fn clear_shared_metadata_caches ( ) {
251+ CACHES . clear ( ) ;
252+ }
253+
236254/// A view of a [`FileMetadataCache`] that keys every entry by the object store as well as the
237255/// path within it.
238256///
@@ -336,8 +354,6 @@ pub(crate) fn scope_to_store(
336354pub ( crate ) fn shared_metadata_cache_for_task (
337355 spark_config : & HashMap < String , String > ,
338356) -> Option < Arc < FileMetadataCache > > {
339- static CACHES : OnceLock < SharedMetadataCaches > = OnceLock :: new ( ) ;
340-
341357 if !spark_config. get_bool ( COMET_PARQUET_SHARED_METADATA_CACHE_ENABLED ) {
342358 return None ;
343359 }
@@ -347,9 +363,7 @@ pub(crate) fn shared_metadata_cache_for_task(
347363 stage_id : parse ( COMET_STAGE_ID ) ?,
348364 stage_attempt : parse ( COMET_STAGE_ATTEMPT ) ?,
349365 } ;
350- let cache: Arc < FileMetadataCache > = CACHES
351- . get_or_init ( SharedMetadataCaches :: default)
352- . cache_for ( key) ;
366+ let cache: Arc < FileMetadataCache > = CACHES . cache_for ( key) ;
353367 Some ( cache)
354368}
355369
@@ -433,6 +447,21 @@ mod tests {
433447 assert ! ( same_stage( & running, & caches. cache_for( key( 0 , 0 ) ) ) ) ;
434448 }
435449
450+ #[ test]
451+ fn clearing_frees_the_retained_caches_but_not_a_running_one ( ) {
452+ let caches = SharedMetadataCaches :: default ( ) ;
453+ let running = caches. cache_for ( key ( 0 , 0 ) ) ;
454+ let idle = Arc :: downgrade ( & caches. cache_for ( key ( 1 , 0 ) ) . stage ) ;
455+ assert ! ( idle. upgrade( ) . is_some( ) ) ;
456+
457+ caches. clear ( ) ;
458+ assert ! ( idle. upgrade( ) . is_none( ) ) ;
459+ assert_eq ! ( caches. len( ) , 0 ) ;
460+ assert_eq ! ( running. cache_limit( ) , DEFAULT_METADATA_CACHE_LIMIT ) ;
461+ drop ( running) ;
462+ assert_eq ! ( caches. len( ) , 0 ) ;
463+ }
464+
436465 #[ test]
437466 fn a_stage_cache_holds_one_task_limit_per_running_task ( ) {
438467 let caches = SharedMetadataCaches :: default ( ) ;
0 commit comments