Skip to content

Commit c96de0b

Browse files
AgentEndernx-cloud[bot]
AgentEnder
and
nx-cloud[bot]
authored
fix(core): runtime inputs shouldn't be cached at task_hasher layer and filesets should be in the hash_plans layer (#34971)
This pull request refactors the caching strategy in the task hashing logic to improve cache isolation and correctness. The main change is to eliminate shared, long-lived caches in favor of creating fresh caches for each invocation, preventing stale data from persisting across CLI commands. Additionally, new caches for project and workspace file set hashes are introduced, and cache handling is streamlined for better maintainability. **Cache management improvements:** * Removed the `runtime_cache` field from the `TaskHasher` struct and now create fresh `DashMap` caches for runtime, project file set, and workspace file set hashes within each invocation of the main hash function. This ensures no stale cache data persists across CLI commands. [[1]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bL153) [[2]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bL184) [[3]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bL196-R208) * Updated the `hash_runtime` function and its usage to accept a reference to a `DashMap` instead of an `Arc<DashMap>`, simplifying ownership and concurrency concerns. [[1]](diffhunk://#diff-77cbfff0572545027ac8ad107859d5135e6f78ee7c32babc56b2c9bbf89cfaccL5-R11) [[2]](diffhunk://#diff-77cbfff0572545027ac8ad107859d5135e6f78ee7c32babc56b2c9bbf89cfaccL54-R61) **File set hash caching:** * Introduced the `CachedFileSetHash` struct to store both the hash value and the list of matched files for project and workspace file set hashing, enabling more efficient input collection and cache lookups. * Added per-invocation caches for project and workspace file set hashes, with logic to check the cache before computing a new hash and to insert results after computation. [[1]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bL196-R208) [[2]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bR353-R385) [[3]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bR412-R438) [[4]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bR554-R556) **Function signature and argument updates:** * Updated function signatures and argument passing to propagate the new cache references throughout the hashing logic, including `HashInstructionArgs` and related functions. [[1]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bR264-R266) [[2]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bR341-R343) [[3]](diffhunk://#diff-d81ca7875513ef544822c24671104ce52cd09409a41a796768aba507d87b3c0bR554-R556) Fixes #30170 --------- Co-authored-by: nx-cloud[bot] <71083854+nx-cloud[bot]@users.noreply.github.com>
1 parent d659486 commit c96de0b

2 files changed

Lines changed: 64 additions & 30 deletions

File tree

‎packages/nx/src/native/tasks/hashers/hash_runtime.rs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,13 @@ use crate::native::hasher::hash;
22
use crate::native::utils::command::create_shell_command;
33
use dashmap::DashMap;
44
use std::collections::HashMap;
5-
use std::sync::Arc;
65
use tracing::trace;
76

87
pub fn hash_runtime(
98
workspace_root: &str,
109
command: &str,
1110
env: &HashMap<String, String>,
12-
cache: Arc<DashMap<String, String>>,
11+
cache: &DashMap<String, String>,
1312
) -> anyhow::Result<String> {
1413
let cache_key = runtime_cache_key(command, env);
1514

@@ -51,16 +50,15 @@ mod tests {
5150
use super::*;
5251
use dashmap::DashMap;
5352
use std::collections::HashMap;
54-
use std::sync::Arc;
5553

5654
#[test]
5755
fn test_hash_runtime() {
5856
let workspace_root = if cfg!(windows) { "C:\\" } else { "/tmp" };
5957
let command = "echo runtime";
6058
let env: HashMap<String, String> = HashMap::new();
61-
let cache = Arc::new(DashMap::new());
59+
let cache = DashMap::new();
6260

63-
let result = hash_runtime(workspace_root, command, &env, Arc::clone(&cache)).unwrap();
61+
let result = hash_runtime(workspace_root, command, &env, &cache).unwrap();
6462
assert_eq!(result, "10571312846059850300");
6563
}
6664

‎packages/nx/src/native/tasks/task_hasher.rs‎

Lines changed: 61 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,14 @@ pub struct HasherOptions {
139139
pub selectively_hash_ts_config: bool,
140140
}
141141

142+
/// Cached result for project/workspace file hashing.
143+
/// Stores both the hash value and the matched file paths (for input collection).
144+
#[derive(Clone)]
145+
struct CachedFileSetHash {
146+
hash: String,
147+
files: Vec<String>,
148+
}
149+
142150
#[napi]
143151
pub struct TaskHasher {
144152
workspace_root: String,
@@ -150,7 +158,6 @@ pub struct TaskHasher {
150158
root_tsconfig_path: Option<String>,
151159
options: Option<HasherOptions>,
152160
external_cache: Arc<DashMap<String, String>>,
153-
runtime_cache: Arc<DashMap<String, String>>,
154161
}
155162
#[napi]
156163
impl TaskHasher {
@@ -181,7 +188,6 @@ impl TaskHasher {
181188
root_tsconfig_path,
182189
options,
183190
external_cache: Arc::new(DashMap::new()),
184-
runtime_cache: Arc::new(DashMap::new()),
185191
}
186192
}
187193

@@ -193,10 +199,13 @@ impl TaskHasher {
193199
cwd: String,
194200
collect_task_inputs: Option<bool>,
195201
) -> anyhow::Result<NapiDashMap<String, HashDetails>> {
196-
// Create a fresh task output cache for this invocation
202+
// Create fresh caches for this invocation.
197203
// This ensures no stale caches across multiple CLI commands when the daemon holds
198-
// the TaskHasher instance
204+
// the TaskHasher instance.
199205
let task_output_cache = DashMap::new();
206+
let runtime_cache: DashMap<String, String> = DashMap::new();
207+
let project_file_set_cache: DashMap<String, CachedFileSetHash> = DashMap::new();
208+
let workspace_file_set_cache: DashMap<String, CachedFileSetHash> = DashMap::new();
200209
let should_collect_inputs = collect_task_inputs.unwrap_or(false);
201210

202211
let function_start = std::time::Instant::now();
@@ -252,6 +261,9 @@ impl TaskHasher {
252261
sorted_externals: &sorted_externals,
253262
selectively_hash_tsconfig,
254263
task_output_cache: &task_output_cache,
264+
runtime_cache: &runtime_cache,
265+
project_file_set_cache: &project_file_set_cache,
266+
workspace_file_set_cache: &workspace_file_set_cache,
255267
cwd: cwd_path,
256268
collect_inputs: should_collect_inputs,
257269
},
@@ -326,6 +338,9 @@ impl TaskHasher {
326338
sorted_externals,
327339
selectively_hash_tsconfig,
328340
task_output_cache,
341+
runtime_cache,
342+
project_file_set_cache,
343+
workspace_file_set_cache,
329344
cwd,
330345
collect_inputs,
331346
}: HashInstructionArgs,
@@ -335,29 +350,36 @@ impl TaskHasher {
335350
let empty = HashInputsBuilder::default();
336351
let (hash, inputs) = match instruction {
337352
HashInstruction::WorkspaceFileSet(workspace_file_set) => {
338-
let result = hash_workspace_files_with_inputs(
339-
workspace_file_set,
340-
&self.all_workspace_files,
341-
)?;
353+
let cache_key = instruction.to_string();
354+
// Check cache first; clone and drop the Ref before any insert
355+
let cached_entry = if let Some(entry) = workspace_file_set_cache.get(&cache_key) {
356+
entry.clone()
357+
} else {
358+
let result = hash_workspace_files_with_inputs(
359+
workspace_file_set,
360+
&self.all_workspace_files,
361+
)?;
362+
let entry = CachedFileSetHash {
363+
hash: result.hash,
364+
files: result.files,
365+
};
366+
workspace_file_set_cache.insert(cache_key, entry.clone());
367+
entry
368+
};
342369
trace!(parent: &span, "hash_workspace_files: {:?}", now.elapsed());
343370
let inputs = if collect_inputs {
344371
HashInputsBuilder {
345-
files: result.files.into_iter().collect(),
372+
files: cached_entry.files.into_iter().collect(),
346373
..Default::default()
347374
}
348375
} else {
349-
drop(result.files);
350376
empty
351377
};
352-
(result.hash, inputs)
378+
(cached_entry.hash, inputs)
353379
}
354380
HashInstruction::Runtime(runtime) => {
355-
let hashed_runtime = hash_runtime(
356-
&self.workspace_root,
357-
runtime,
358-
js_env,
359-
Arc::clone(&self.runtime_cache),
360-
)?;
381+
let hashed_runtime =
382+
hash_runtime(&self.workspace_root, runtime, js_env, runtime_cache)?;
361383
trace!(parent: &span, "hash_runtime: {:?}", now.elapsed());
362384
let inputs = if collect_inputs {
363385
instruction.into()
@@ -383,22 +405,33 @@ impl TaskHasher {
383405
(hashed_cwd, empty)
384406
}
385407
HashInstruction::ProjectFileSet(project_name, file_sets) => {
386-
let result = hash_project_files_with_inputs(
387-
project_name,
388-
file_sets,
389-
&self.project_file_map,
390-
)?;
408+
let cache_key = instruction.to_string();
409+
// Check cache first; clone and drop the Ref before any insert
410+
let cached_entry = if let Some(entry) = project_file_set_cache.get(&cache_key) {
411+
entry.clone()
412+
} else {
413+
let result = hash_project_files_with_inputs(
414+
project_name,
415+
file_sets,
416+
&self.project_file_map,
417+
)?;
418+
let entry = CachedFileSetHash {
419+
hash: result.hash,
420+
files: result.files,
421+
};
422+
project_file_set_cache.insert(cache_key, entry.clone());
423+
entry
424+
};
391425
trace!(parent: &span, "hash_project_files: {:?}", now.elapsed());
392426
let inputs = if collect_inputs {
393427
HashInputsBuilder {
394-
files: result.files.into_iter().collect(),
428+
files: cached_entry.files.into_iter().collect(),
395429
..Default::default()
396430
}
397431
} else {
398-
drop(result.files);
399432
empty
400433
};
401-
(result.hash, inputs)
434+
(cached_entry.hash, inputs)
402435
}
403436
HashInstruction::ProjectConfiguration(project_name) => {
404437
let hashed_project_config =
@@ -514,6 +547,9 @@ struct HashInstructionArgs<'a> {
514547
sorted_externals: &'a [&'a String],
515548
selectively_hash_tsconfig: bool,
516549
task_output_cache: &'a DashMap<String, CachedTaskOutput>,
550+
runtime_cache: &'a DashMap<String, String>,
551+
project_file_set_cache: &'a DashMap<String, CachedFileSetHash>,
552+
workspace_file_set_cache: &'a DashMap<String, CachedFileSetHash>,
517553
cwd: &'a std::path::Path,
518554
collect_inputs: bool,
519555
}

0 commit comments

Comments
 (0)