Skip to content

Commit d80bb21

Browse files
committed
fix: free retained shared metadata caches at plugin shutdown
Clear the registry in release_runtime, as clear_file_io_cache does for the Iceberg FileIO cache, and document when a retained cache is freed.
1 parent 40beedd commit d80bb21

4 files changed

Lines changed: 45 additions & 11 deletions

File tree

‎docs/source/contributor-guide/memory_management.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -513,8 +513,10 @@ diverge for several structural reasons:
513513
`spark.comet.parquet.metadataCache.sharedWithinStage.enabled`, the default, the tasks of a stage
514514
share one cache per executor, which holds up to 50 MiB for each of its running tasks. The two
515515
most recently used caches are kept after their tasks finish, for the next task of their stage,
516-
and shrink to 50 MiB each, so up to 100 MiB stays resident with no task running. With the
517-
setting off, each task has its own 50 MiB cache, freed with the task.
516+
and shrink to 50 MiB each, so up to 100 MiB stays resident with no task running. A kept cache is
517+
freed once tasks of two other stages run on the executor, or when the plugin shuts down. An
518+
executor is not told when a stage ends, so one that goes idle after a query holds its last two
519+
caches until then. With the setting off, each task has its own 50 MiB cache, freed with the task.
518520
- **Batches in flight across the FFI boundary.** Reservations stop at the operator that made them.
519521
Imported JVM batches are reserved only while a reserving operator holds them, and exported native
520522
batches have usually been released by the time the JVM receives them yet stay resident until the

‎native/core/src/execution/jni_api.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -450,7 +450,8 @@ pub fn get_runtime() -> Handle {
450450
.clone()
451451
}
452452

453-
/// Tears down the global Tokio runtime, if it has been initialized.
453+
/// Tears down the global Tokio runtime, if it has been initialized, and frees the process-wide
454+
/// caches that outlive tasks: the Iceberg `FileIO`s and the retained Parquet metadata caches.
454455
///
455456
/// The runtime is moved out of the global slot, so the next [`init_runtime`] or
456457
/// [`get_runtime`] call builds a new one. Shutting it down blocks the calling (JNI) thread
@@ -463,6 +464,7 @@ pub fn get_runtime() -> Handle {
463464
/// would deadlock/panic.
464465
pub fn release_runtime() {
465466
crate::execution::operators::clear_file_io_cache();
467+
crate::parquet::shared_metadata_cache::clear_shared_metadata_caches();
466468
let runtime = TOKIO_RUNTIME.lock().take();
467469
if let Some(runtime) = runtime {
468470
runtime.shutdown_timeout(Duration::from_secs(3));

‎native/core/src/parquet/shared_metadata_cache.rs‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,11 @@
3737
//! tasks own it through their `RuntimeEnv`. It additionally holds strong references to the last
3838
//! [`MAX_IDLE_CACHES`] caches it handed out. Those matter when an executor runs one task at a
3939
//! time: the next task of the stage starts after the previous one released its plan. Whatever the
40-
//! tasks do, at most [`MAX_IDLE_CACHES`] caches outlive the tasks that used them.
40+
//! tasks do, at most [`MAX_IDLE_CACHES`] caches outlive the tasks that used them. A retained cache
41+
//! is freed when tasks of [`MAX_IDLE_CACHES`] other stages have asked for theirs since, or when
42+
//! the plugin shuts down and [`clear_shared_metadata_caches`] runs. Nothing frees it sooner: an
43+
//! executor is not told that a stage has finished, so one that goes idle after a query keeps the
44+
//! caches of its last stages until it shuts down.
4145
//!
4246
//! A stage's cache holds DataFusion's per-task limit for each of its tasks that is running, and one
4347
//! task's limit once none is, so that sharing does not shrink the room each task had with a cache
@@ -65,7 +69,7 @@ use datafusion::execution::cache::default_cache::DefaultCache;
6569
use datafusion::execution::cache::{Cache, CacheEntryInfo};
6670
use object_store::path::Path;
6771
use std::collections::{HashMap, VecDeque};
68-
use std::sync::{Arc, Mutex, OnceLock, Weak};
72+
use std::sync::{Arc, LazyLock, Mutex, Weak};
6973
use std::time::Duration;
7074

7175
/// How many caches of stages with no running task the registry holds on to.
@@ -227,12 +231,26 @@ impl SharedMetadataCaches {
227231
Arc::new(TaskCache::new(stage))
228232
}
229233

234+
/// Forgets every stage, freeing the retained caches. Running tasks keep the caches they hold.
235+
fn clear(&self) {
236+
let dropped = std::mem::take(&mut *self.registry.lock().unwrap());
237+
drop(dropped);
238+
}
239+
230240
#[cfg(test)]
231241
fn len(&self) -> usize {
232242
self.registry.lock().unwrap().alive.len()
233243
}
234244
}
235245

246+
/// The registry of the executor's stages.
247+
static CACHES: LazyLock<SharedMetadataCaches> = LazyLock::new(SharedMetadataCaches::default);
248+
249+
/// Frees the caches retained for stages with no running task, as the plugin shuts down.
250+
pub fn clear_shared_metadata_caches() {
251+
CACHES.clear();
252+
}
253+
236254
/// A view of a [`FileMetadataCache`] that keys every entry by the object store as well as the
237255
/// path within it.
238256
///
@@ -336,8 +354,6 @@ pub(crate) fn scope_to_store(
336354
pub(crate) fn shared_metadata_cache_for_task(
337355
spark_config: &HashMap<String, String>,
338356
) -> Option<Arc<FileMetadataCache>> {
339-
static CACHES: OnceLock<SharedMetadataCaches> = OnceLock::new();
340-
341357
if !spark_config.get_bool(COMET_PARQUET_SHARED_METADATA_CACHE_ENABLED) {
342358
return None;
343359
}
@@ -347,9 +363,7 @@ pub(crate) fn shared_metadata_cache_for_task(
347363
stage_id: parse(COMET_STAGE_ID)?,
348364
stage_attempt: parse(COMET_STAGE_ATTEMPT)?,
349365
};
350-
let cache: Arc<FileMetadataCache> = CACHES
351-
.get_or_init(SharedMetadataCaches::default)
352-
.cache_for(key);
366+
let cache: Arc<FileMetadataCache> = CACHES.cache_for(key);
353367
Some(cache)
354368
}
355369

@@ -433,6 +447,21 @@ mod tests {
433447
assert!(same_stage(&running, &caches.cache_for(key(0, 0))));
434448
}
435449

450+
#[test]
451+
fn clearing_frees_the_retained_caches_but_not_a_running_one() {
452+
let caches = SharedMetadataCaches::default();
453+
let running = caches.cache_for(key(0, 0));
454+
let idle = Arc::downgrade(&caches.cache_for(key(1, 0)).stage);
455+
assert!(idle.upgrade().is_some());
456+
457+
caches.clear();
458+
assert!(idle.upgrade().is_none());
459+
assert_eq!(caches.len(), 0);
460+
assert_eq!(running.cache_limit(), DEFAULT_METADATA_CACHE_LIMIT);
461+
drop(running);
462+
assert_eq!(caches.len(), 0);
463+
}
464+
436465
#[test]
437466
fn a_stage_cache_holds_one_task_limit_per_running_task() {
438467
let caches = SharedMetadataCaches::default();

‎spark/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,8 @@ object CometConf extends ShimCometConf {
203203
"cache holds up to 50 MiB of decoded footers and page indexes for each of its " +
204204
"running tasks, outside Spark's memory accounting. Up to two caches are kept after " +
205205
"their tasks finish, for the next task of their stage, and each of those holds up " +
206-
"to 50 MiB. When disabled, each task keeps its own 50 MiB cache, which is freed " +
206+
"to 50 MiB until tasks of two other stages run on the executor or the executor " +
207+
"shuts down. When disabled, each task keeps its own 50 MiB cache, which is freed " +
207208
"when the task ends.")
208209
.booleanConf
209210
.createWithDefault(true)

0 commit comments

Comments
 (0)