Skip to content

Commit 40beedd

Browse files
committed
fix: size the shared metadata cache by the stage's running tasks
The executor's core count that reaches native code is an estimate on a standalone executor without spark.executor.cores, so sizing the cache by it could give a two-core executor the limit of the host's processors. Count each task's plan in while it holds its stage's cache and out when it drops it, and size the cache by the tasks running. A cache with no task running shrinks to one task's limit, so a retained cache holds no more than a task's own cache would. The handle ignores the limit a task's CacheManager sets.
1 parent d14249f commit 40beedd

5 files changed

Lines changed: 223 additions & 133 deletions

File tree

‎docs/source/contributor-guide/memory_management.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -511,10 +511,10 @@ diverge for several structural reasons:
511511
- **Native caches that outlive their task.** The native Parquet scan's cache of decoded footers and
512512
page indexes is not reserved from the pool. With
513513
`spark.comet.parquet.metadataCache.sharedWithinStage.enabled`, the default, the tasks of a stage
514-
share one cache per executor, which holds up to 50 MiB for each task the executor runs at once,
515-
and the two most recently used caches are kept after their tasks finish, for the next task of
516-
their stage. So up to twice that limit stays resident with no task running. With the setting off,
517-
each task has its own 50 MiB cache, freed with the task.
514+
share one cache per executor, which holds up to 50 MiB for each of its running tasks. The two
515+
most recently used caches are kept after their tasks finish, for the next task of their stage,
516+
and shrink to 50 MiB each, so up to 100 MiB stays resident with no task running. With the
517+
setting off, each task has its own 50 MiB cache, freed with the task.
518518
- **Batches in flight across the FFI boundary.** Reservations stop at the operator that made them.
519519
Imported JVM batches are reserved only while a reserving operator holds them, and exported native
520520
batches have usually been released by the time the JVM receives them yet stay resident until the

‎native/core/src/execution/jni_api.rs‎

Lines changed: 5 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ use arrow::array::{Array, RecordBatch};
4747
use arrow::datatypes::DataType as ArrowDataType;
4848
use datafusion::common::{DataFusionError, Result as DataFusionResult};
4949
use datafusion::config::SpillCompression;
50-
use datafusion::execution::cache::cache_manager::{CacheManagerConfig, FileMetadataCache};
50+
use datafusion::execution::cache::cache_manager::CacheManagerConfig;
5151
use datafusion::execution::memory_pool::MemoryPool;
5252
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
5353
use datafusion::logical_expr::ScalarUDF;
@@ -995,15 +995,6 @@ fn configure_skip_partial_aggregation(config: &mut SessionConfig, plan: &Operato
995995
}
996996
}
997997

998-
/// A task's cache manager config over the metadata cache it shares with its stage. The limit is
999-
/// set here too: `CacheManager::try_new` applies the config's `metadata_cache_limit` to the cache
1000-
/// it is given, so leaving the default would reset the stage's cache to one task's limit.
1001-
fn shared_cache_manager_config(cache: Arc<FileMetadataCache>, limit: usize) -> CacheManagerConfig {
1002-
CacheManagerConfig::default()
1003-
.with_file_metadata_cache(Some(cache))
1004-
.with_metadata_cache_limit(limit)
1005-
}
1006-
1007998
/// Configure DataFusion session context.
1008999
fn prepare_datafusion_session_context(
10091000
batch_size: usize,
@@ -1022,9 +1013,10 @@ fn prepare_datafusion_session_context(
10221013

10231014
// Let the tasks of one stage on this executor share the Parquet footer and page index cache
10241015
// instead of each fetching and parsing the footers of the files it reads.
1025-
let task_slots = spark_config.get_usize(SPARK_EXECUTOR_CORES, 1) / task_cpus.max(1);
1026-
if let Some((cache, limit)) = shared_metadata_cache_for_task(spark_config, task_slots) {
1027-
rt_config = rt_config.with_cache_manager(shared_cache_manager_config(cache, limit));
1016+
if let Some(cache) = shared_metadata_cache_for_task(spark_config) {
1017+
rt_config = rt_config.with_cache_manager(
1018+
CacheManagerConfig::default().with_file_metadata_cache(Some(cache)),
1019+
);
10281020
}
10291021

10301022
let mut session_config = SessionConfig::new()
@@ -4333,30 +4325,4 @@ mod tests {
43334325
);
43344326
release_plan(exec_context, &mut |_| Ok(())).unwrap();
43354327
}
4336-
4337-
/// Every task builds a `RuntimeEnv` over its stage's metadata cache, and building one applies
4338-
/// the config's `metadata_cache_limit` to that cache. The config has to carry the stage's
4339-
/// limit, or each new task would shrink the cache back to one task's.
4340-
#[test]
4341-
fn a_task_keeps_the_limit_of_its_stage_metadata_cache() {
4342-
use crate::parquet::shared_metadata_cache::shared_cache_limit;
4343-
use datafusion::execution::cache::cache_manager::DEFAULT_METADATA_CACHE_LIMIT;
4344-
use datafusion::execution::cache::default_cache::DefaultCache;
4345-
4346-
let limit = shared_cache_limit(8);
4347-
let cache: Arc<FileMetadataCache> = Arc::new(DefaultCache::new(limit));
4348-
RuntimeEnvBuilder::new()
4349-
.with_cache_manager(shared_cache_manager_config(Arc::clone(&cache), limit))
4350-
.build()
4351-
.unwrap();
4352-
assert_eq!(cache.cache_limit(), limit);
4353-
4354-
RuntimeEnvBuilder::new()
4355-
.with_cache_manager(
4356-
CacheManagerConfig::default().with_file_metadata_cache(Some(Arc::clone(&cache))),
4357-
)
4358-
.build()
4359-
.unwrap();
4360-
assert_eq!(cache.cache_limit(), DEFAULT_METADATA_CACHE_LIMIT);
4361-
}
43624328
}

‎native/core/src/parquet/parquet_exec.rs‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2166,9 +2166,7 @@ mod tests {
21662166
#[tokio::test]
21672167
async fn equal_paths_in_two_stores_do_not_share_footers() {
21682168
use crate::parquet::shared_metadata_cache::{SharedMetadataCaches, StageKey};
2169-
use datafusion::execution::cache::cache_manager::{
2170-
CacheManagerConfig, DEFAULT_METADATA_CACHE_LIMIT,
2171-
};
2169+
use datafusion::execution::cache::cache_manager::{CacheManagerConfig, FileMetadataCache};
21722170
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
21732171

21742172
let location = object_store::path::Path::from("table/part-00000.parquet");
@@ -2197,14 +2195,11 @@ mod tests {
21972195
let runtime_env = match &task_env {
21982196
Some(env) => Arc::clone(env),
21992197
None => {
2200-
let stage_cache = registry.cache_for(
2201-
StageKey {
2202-
app_id: "app-1".to_string(),
2203-
stage_id: 0,
2204-
stage_attempt: 0,
2205-
},
2206-
DEFAULT_METADATA_CACHE_LIMIT,
2207-
);
2198+
let stage_cache: Arc<FileMetadataCache> = registry.cache_for(StageKey {
2199+
app_id: "app-1".to_string(),
2200+
stage_id: 0,
2201+
stage_attempt: 0,
2202+
});
22082203
RuntimeEnvBuilder::new()
22092204
.with_cache_manager(
22102205
CacheManagerConfig::default()

0 commit comments

Comments
 (0)