Skip to content

Commit 1979349

Browse files
Reuse compiled workspace exclusion patterns (#20988)
Workspace discovery normalized, escaped, and recompiled every exclusion glob for every candidate member, multiplying work in large workspaces with long exclusion lists. Centralize exclusion matching, initialize it only when a candidate needs checking, and reuse patterns already compiled while parsing `pyproject.toml`. Patterns that require normalization or absolute matching retain their existing behavior, including parent-directory exclusions and workspace roots containing glob characters. Optimized microbenchmark with 512 members and nonmatching exclusions: | Exclusions | Before | After | Improvement | | ---------: | -------: | -----: | ----------: | | 64 | 168 ms | 35 ms | 4.8× | | 128 | 311 ms | 126 ms | 2.5× | | 256 | 934 ms | 267 ms | 3.5× | | 512 | 2,497 ms | 683 ms | 3.7× | --------- Co-authored-by: Zanie Blue <[email protected]>
1 parent b60da7a commit 1979349

2 files changed

Lines changed: 183 additions & 20 deletions

File tree

‎crates/uv-bench/benches/workspace_discovery.rs‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ const EXCLUDE_NEWER: &str = "2024-08-08";
2121

2222
/// Mirroring the airflow workspace size at time of writing.
2323
const MEMBER_COUNT: usize = 127;
24+
const EXCLUDE_COUNT: usize = 128;
2425
const OPTIONAL_DEPENDENCY_GROUP_COUNT: usize = 122;
2526
const DEPENDENCY_GROUP_COUNT: usize = 64;
2627
const UNUSED_ROOT_TABLE_COUNT: usize = 128;
@@ -36,7 +37,7 @@ fn provider_requirement(member_index: usize) -> String {
3637

3738
/// Create a synthetic workspace with a root and many members, returning the directories to run
3839
/// discovery from.
39-
fn create_workspace(root: &Path) -> Vec<PathBuf> {
40+
fn create_workspace(root: &Path, exclude_count: usize) -> Vec<PathBuf> {
4041
let mut discovery_roots = Vec::with_capacity(MEMBER_COUNT + 1);
4142
discovery_roots.push(root.to_path_buf());
4243

@@ -53,13 +54,13 @@ fn create_workspace(root: &Path) -> Vec<PathBuf> {
5354
discovery_roots.push(member_root);
5455
}
5556

56-
fs_err::write(root.join("pyproject.toml"), root_pyproject())
57+
fs_err::write(root.join("pyproject.toml"), root_pyproject(exclude_count))
5758
.expect("Failed to write workspace root pyproject.toml");
5859

5960
discovery_roots
6061
}
6162

62-
fn root_pyproject() -> String {
63+
fn root_pyproject(exclude_count: usize) -> String {
6364
let dependencies: Vec<String> = (0..MEMBER_COUNT).map(provider_requirement).collect();
6465

6566
let optional_dependencies: toml::Table = (0..OPTIONAL_DEPENDENCY_GROUP_COUNT)
@@ -104,6 +105,16 @@ fn root_pyproject() -> String {
104105
})
105106
.collect();
106107

108+
let mut workspace = toml::toml! {
109+
members = ["packages/*"]
110+
};
111+
if exclude_count > 0 {
112+
let excludes: Vec<String> = (0..exclude_count)
113+
.map(|exclude_index| format!("packages/excluded-{exclude_index:03}"))
114+
.collect();
115+
workspace.insert("exclude".to_string(), toml::Value::from(excludes));
116+
}
117+
107118
// Generate some unrelated work for the toml parser, mimicking real tool configuration.
108119
let generated: toml::Table = (0..UNUSED_ROOT_TABLE_COUNT)
109120
.map(|table_index| {
@@ -168,9 +179,7 @@ fn root_pyproject() -> String {
168179
[tool.uv]
169180
package = false
170181
sources = (sources)
171-
172-
[tool.uv.workspace]
173-
members = ["packages/*"]
182+
workspace = (workspace)
174183

175184
[tool.linter]
176185
generated = (generated)
@@ -258,16 +267,28 @@ fn member_pyproject(member_index: usize) -> String {
258267
}
259268

260269
fn discover_workspace_from_all_members(c: &mut Criterion<WallTime>) {
270+
discover_workspace(c, "discover_workspace_from_all_members", 0);
271+
}
272+
273+
fn discover_workspace_from_all_members_with_excludes(c: &mut Criterion<WallTime>) {
274+
discover_workspace(
275+
c,
276+
"discover_workspace_from_all_members_with_excludes",
277+
EXCLUDE_COUNT,
278+
);
279+
}
280+
281+
fn discover_workspace(c: &mut Criterion<WallTime>, name: &str, exclude_count: usize) {
261282
let dir = tempfile::tempdir().expect("Failed to create temporary directory");
262-
let discovery_roots = create_workspace(dir.path());
283+
let discovery_roots = create_workspace(dir.path(), exclude_count);
263284
let cache = Cache::from_path(dir.path().join(".uv-cache"));
264285
let options = DiscoveryOptions::default();
265286
let runtime = tokio::runtime::Builder::new_current_thread()
266287
.enable_all()
267288
.build()
268289
.expect("Failed to create Tokio runtime");
269290

270-
c.bench_function("discover_workspace_from_all_members", |b| {
291+
c.bench_function(name, |b| {
271292
b.iter(|| {
272293
let workspace_cache = WorkspaceCache::default();
273294
for root in &discovery_roots {
@@ -287,7 +308,7 @@ fn discover_workspace_from_all_members(c: &mut Criterion<WallTime>) {
287308

288309
fn run_python_version_synthetic_workspace(c: &mut Criterion<WallTime>) {
289310
let workspace_dir = tempfile::tempdir().expect("Failed to create temporary directory");
290-
create_workspace(workspace_dir.path());
311+
create_workspace(workspace_dir.path(), 0);
291312

292313
let cache_dir = workspace_dir.path().join(".uv-cache");
293314
let runtime = tokio::runtime::Builder::new_current_thread()
@@ -365,6 +386,7 @@ fn run_cli(
365386
criterion_group!(
366387
workspace_discovery,
367388
discover_workspace_from_all_members,
389+
discover_workspace_from_all_members_with_excludes,
368390
run_python_version_synthetic_workspace
369391
);
370392
criterion_main!(workspace_discovery);

‎crates/uv-workspace/src/workspace.rs‎

Lines changed: 152 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
//! Resolve the current [`ProjectWorkspace`] or [`Workspace`].
22
3+
use std::borrow::Cow;
34
use std::collections::{BTreeMap, BTreeSet};
45
use std::error::Error;
56
use std::fmt;
@@ -8,7 +9,7 @@ use std::hash::BuildHasherDefault;
89
use std::path::{Path, PathBuf};
910
use std::sync::Arc;
1011

11-
use glob::{GlobError, PatternError, glob};
12+
use glob::{GlobError, Pattern, PatternError, glob};
1213
use itertools::Itertools;
1314
use rustc_hash::{FxHashSet, FxHasher};
1415
use tracing::{debug, trace, warn};
@@ -1163,6 +1164,9 @@ impl Workspace {
11631164
);
11641165
}
11651166

1167+
// Prepare exclusions only after finding a member that is not explicitly ignored.
1168+
let mut exclusions = None;
1169+
11661170
// Add all other workspace members.
11671171
for member_glob in workspace_definition.clone().members.unwrap_or_default() {
11681172
// Normalize the member glob to remove leading `./` and other relative path components
@@ -1210,7 +1214,14 @@ impl Workspace {
12101214
}
12111215

12121216
// If the member is excluded, ignore it.
1213-
if is_excluded_from_workspace(&member_root, workspace_root, workspace_definition)? {
1217+
if exclusions
1218+
.get_or_insert_with(|| {
1219+
WorkspaceExclusions::new(workspace_root, workspace_definition)
1220+
})
1221+
.as_ref()
1222+
.map_err(WorkspaceError::clone)?
1223+
.matches(&member_root)
1224+
{
12141225
debug!(
12151226
"Ignoring workspace member: `{}`",
12161227
member_root.simplified_display()
@@ -1951,21 +1962,77 @@ fn is_excluded_from_workspace(
19511962
workspace_root: &Path,
19521963
workspace: &ToolUvWorkspace,
19531964
) -> Result<bool, WorkspaceError> {
1954-
for exclude_glob in workspace.exclude.iter().flatten() {
1955-
// Normalize the exclude glob to remove leading `./` and other relative path components
1965+
Ok(WorkspaceExclusions::new(workspace_root, workspace)?.matches(project_path))
1966+
}
1967+
1968+
/// Compiled workspace exclusion patterns.
1969+
#[derive(Debug)]
1970+
struct WorkspaceExclusions<'workspace> {
1971+
workspace_root: &'workspace Path,
1972+
patterns: Vec<WorkspaceExclusion<'workspace>>,
1973+
}
1974+
1975+
/// A workspace exclusion that can reuse its parsed pattern or requires normalization.
1976+
#[derive(Debug)]
1977+
enum WorkspaceExclusion<'workspace> {
1978+
Relative(&'workspace Pattern),
1979+
Absolute(Pattern),
1980+
}
1981+
1982+
impl<'workspace> WorkspaceExclusions<'workspace> {
1983+
/// Compile the normalized workspace exclusion patterns.
1984+
fn new(
1985+
workspace_root: &'workspace Path,
1986+
workspace: &'workspace ToolUvWorkspace,
1987+
) -> Result<Self, WorkspaceError> {
1988+
let patterns = workspace
1989+
.exclude
1990+
.iter()
1991+
.flatten()
1992+
.map(|exclude_glob| Self::compile_pattern(workspace_root, exclude_glob))
1993+
.collect::<Result<_, _>>()?;
1994+
1995+
Ok(Self {
1996+
workspace_root,
1997+
patterns,
1998+
})
1999+
}
2000+
2001+
/// Return whether any workspace exclusion matches the project path.
2002+
fn matches(&self, project_path: &Path) -> bool {
2003+
let relative_path = project_path
2004+
.simplified()
2005+
.strip_prefix(self.workspace_root.simplified())
2006+
.ok();
2007+
2008+
self.patterns.iter().any(|pattern| match pattern {
2009+
WorkspaceExclusion::Relative(pattern) => {
2010+
relative_path.is_some_and(|relative_path| pattern.matches_path(relative_path))
2011+
}
2012+
WorkspaceExclusion::Absolute(pattern) => pattern.matches_path(project_path),
2013+
})
2014+
}
2015+
2016+
/// Reuse an already compiled relative pattern or compile its normalized absolute equivalent.
2017+
fn compile_pattern(
2018+
workspace_root: &Path,
2019+
exclude_glob: &'workspace Pattern,
2020+
) -> Result<WorkspaceExclusion<'workspace>, WorkspaceError> {
2021+
// Normalize the exclude glob to remove leading `./` and other relative path components.
19562022
let normalized_glob = normalize_path(Path::new(exclude_glob.as_str()));
1957-
let absolute_glob = PathBuf::from(glob::Pattern::escape(
2023+
if matches!(&normalized_glob, Cow::Borrowed(_)) && normalized_glob.is_relative() {
2024+
return Ok(WorkspaceExclusion::Relative(exclude_glob));
2025+
}
2026+
2027+
let absolute_glob = PathBuf::from(Pattern::escape(
19582028
workspace_root.simplified().to_string_lossy().as_ref(),
19592029
))
19602030
.join(normalized_glob.as_ref());
19612031
let absolute_glob = absolute_glob.to_string_lossy();
1962-
let exclude_pattern = glob::Pattern::new(&absolute_glob)
1963-
.map_err(|err| WorkspaceErrorKind::Pattern(absolute_glob.to_string(), err))?;
1964-
if exclude_pattern.matches_path(project_path) {
1965-
return Ok(true);
1966-
}
2032+
Pattern::new(&absolute_glob)
2033+
.map(WorkspaceExclusion::Absolute)
2034+
.map_err(|err| WorkspaceErrorKind::Pattern(absolute_glob.to_string(), err).into())
19672035
}
1968-
Ok(false)
19692036
}
19702037

19712038
/// Check if we're in the `tool.uv.workspace.members` of a workspace.
@@ -3303,6 +3370,80 @@ mod tests {
33033370
Ok(())
33043371
}
33053372

3373+
#[tokio::test]
3374+
async fn exclude_package_with_normalized_glob_and_escaped_root() -> Result<()> {
3375+
let temp_dir = tempfile::TempDir::new()?;
3376+
let temp_dir_root = ChildPath::new(temp_dir.path());
3377+
let root = temp_dir_root.child("workspace[glob]?");
3378+
3379+
root.child("pyproject.toml").write_str(
3380+
r#"
3381+
[project]
3382+
name = "albatross"
3383+
version = "0.1.0"
3384+
requires-python = ">=3.12"
3385+
3386+
[tool.uv.workspace]
3387+
members = ["./packages/*", "../external-*"]
3388+
exclude = [
3389+
"packages/excluded-borrowed-*",
3390+
"./ignored/../packages/excluded",
3391+
"./packages/./excluded-glob-*",
3392+
"../external-excluded",
3393+
]
3394+
"#,
3395+
)?;
3396+
3397+
for member in [
3398+
"included",
3399+
"excluded",
3400+
"excluded-glob-one",
3401+
"excluded-borrowed-one",
3402+
] {
3403+
root.child("packages")
3404+
.child(member)
3405+
.child("pyproject.toml")
3406+
.write_str(&format!(
3407+
r#"
3408+
[project]
3409+
name = "{member}"
3410+
version = "0.1.0"
3411+
requires-python = ">=3.12"
3412+
"#,
3413+
))?;
3414+
}
3415+
3416+
for member in ["external-included", "external-excluded"] {
3417+
temp_dir_root
3418+
.child(member)
3419+
.child("pyproject.toml")
3420+
.write_str(&format!(
3421+
r#"
3422+
[project]
3423+
name = "{member}"
3424+
version = "0.1.0"
3425+
requires-python = ">=3.12"
3426+
"#,
3427+
))?;
3428+
}
3429+
3430+
let (project, _) = temporary_test(root.as_ref())
3431+
.await
3432+
.map_err(|(error, _)| error)?;
3433+
assert_json_snapshot!(
3434+
project.workspace().packages().keys().collect::<Vec<_>>(),
3435+
@r#"
3436+
[
3437+
"albatross",
3438+
"external-included",
3439+
"included"
3440+
]
3441+
"#
3442+
);
3443+
3444+
Ok(())
3445+
}
3446+
33063447
#[test]
33073448
fn read_dependency_groups() {
33083449
let toml = r#"

0 commit comments

Comments
 (0)