Skip to content

Commit fab2c0c

Browse files
authored
Improve basic space accounting when reporting cache clean space savings (#21261)
## Summary While looking at Charlie's new preview space accounting PR which utilises special APIs to count extents I noticed we have some areas where small changes could improve accounting accuracy. This PR uses block counts instead of file sizes to determine file space savings and additionally uses hardlink counts to avoid double-counting space when hardlinks are in use. Notably the PR changes a small detail about our output format, it now reports the number of bytes recovered (even if it's zero) whenever any files or directories were deleted. Previously we had some quirky logic where we'd only do this if we had "logical" or "physical" counts, regardless of which one we'd then print. I think the intent is to report 0 when things were deleted but nothing got saved so I picked that as a simpler condition. ## Test Plan Adjustments to existing tests and some new tests. Unit tests avoid some of the potential uncertainty when doing accounting.
1 parent 64c73b1 commit fab2c0c

10 files changed

Lines changed: 234 additions & 91 deletions

File tree

‎crates/uv-cache/src/lib.rs‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pub use crate::by_timestamp::CachedByTimestamp;
1818
#[cfg(feature = "clap")]
1919
pub use crate::cli::CacheArgs;
2020
use crate::removal::Remover;
21-
pub use crate::removal::{Removal, RemovalMode};
21+
pub use crate::removal::{Removal, RemovalAccounting};
2222
pub use crate::wheel::WheelCache;
2323
use crate::wheel::WheelCacheKind;
2424
pub use archive::ArchiveId;
@@ -175,7 +175,7 @@ pub struct Cache {
175175
/// uv process.
176176
lock_file: Option<Arc<LockedFile>>,
177177
/// The storage accounting used when removing cache entries.
178-
removal_mode: RemovalMode,
178+
removal_accounting: RemovalAccounting,
179179
}
180180

181181
impl Cache {
@@ -186,7 +186,7 @@ impl Cache {
186186
refresh: Refresh::None(Timestamp::now()),
187187
temp_dir: None,
188188
lock_file: None,
189-
removal_mode: RemovalMode::Logical,
189+
removal_accounting: RemovalAccounting::Coarse,
190190
}
191191
}
192192

@@ -198,7 +198,7 @@ impl Cache {
198198
refresh: Refresh::None(Timestamp::now()),
199199
temp_dir: Some(Arc::new(temp_dir)),
200200
lock_file: None,
201-
removal_mode: RemovalMode::Logical,
201+
removal_accounting: RemovalAccounting::Coarse,
202202
})
203203
}
204204

@@ -210,22 +210,24 @@ impl Cache {
210210

211211
/// Set the storage accounting used when removing cache entries.
212212
///
213-
/// Falls back to logical accounting when physical accounting is unsupported.
213+
/// Falls back to [`RemovalAccounting::Coarse`] when fine-grained accounting is unsupported.
214214
#[must_use]
215-
pub fn with_removal_mode(self, removal_mode: RemovalMode) -> Self {
216-
let removal_mode = match removal_mode {
217-
RemovalMode::Physical if !uv_fs::supports_physical_space() => RemovalMode::Logical,
218-
removal_mode => removal_mode,
215+
pub fn with_removal_accounting(self, removal_accounting: RemovalAccounting) -> Self {
216+
let removal_accounting = match removal_accounting {
217+
RemovalAccounting::Fine if !uv_fs::supports_fine_grained_accounting() => {
218+
RemovalAccounting::Coarse
219+
}
220+
removal_accounting => removal_accounting,
219221
};
220222
Self {
221-
removal_mode,
223+
removal_accounting,
222224
..self
223225
}
224226
}
225227

226-
/// Create an empty removal summary using the cache's configured accounting mode.
228+
/// Create an empty removal summary using the cache's configured accounting.
227229
pub fn removal(&self) -> Removal {
228-
Removal::new(self.removal_mode)
230+
Removal::new(self.removal_accounting)
229231
}
230232

231233
/// Acquire a lock that allows removing entries from the cache.
@@ -235,7 +237,7 @@ impl Cache {
235237
refresh,
236238
temp_dir,
237239
lock_file,
238-
removal_mode,
240+
removal_accounting,
239241
} = self;
240242

241243
// Release the existing lock, avoid deadlocks from a cloned cache.
@@ -258,7 +260,7 @@ impl Cache {
258260
refresh,
259261
temp_dir,
260262
lock_file: Some(Arc::new(lock_file)),
261-
removal_mode,
263+
removal_accounting,
262264
})
263265
}
264266

@@ -271,7 +273,7 @@ impl Cache {
271273
refresh,
272274
temp_dir,
273275
lock_file,
274-
removal_mode,
276+
removal_accounting,
275277
} = self;
276278

277279
match LockedFile::acquire_no_wait(
@@ -284,14 +286,14 @@ impl Cache {
284286
refresh,
285287
temp_dir,
286288
lock_file: Some(Arc::new(lock_file)),
287-
removal_mode,
289+
removal_accounting,
288290
}),
289291
None => Err(Self {
290292
root,
291293
refresh,
292294
temp_dir,
293295
lock_file,
294-
removal_mode,
296+
removal_accounting,
295297
}),
296298
}
297299
}
@@ -551,7 +553,7 @@ impl Cache {
551553
pub fn clear(self, reporter: Box<dyn CleanReporter>) -> Result<Removal, io::Error> {
552554
// Remove everything but `.lock`, Windows does not allow removal of a locked file
553555
let mut removal = Remover::new(reporter)
554-
.with_removal_mode(self.removal_mode)
556+
.with_removal_accounting(self.removal_accounting)
555557
.rm_rf(&self.root, true)?;
556558
let Self {
557559
root, lock_file, ..
@@ -745,7 +747,7 @@ impl Cache {
745747
/// Remove a cache path using the cache's configured storage accounting.
746748
pub fn remove_path(&self, path: impl AsRef<Path>) -> io::Result<Removal> {
747749
Remover::default()
748-
.with_removal_mode(self.removal_mode)
750+
.with_removal_accounting(self.removal_accounting)
749751
.rm_rf(path, false)
750752
}
751753

‎crates/uv-cache/src/removal.rs‎

Lines changed: 56 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,32 @@
22
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
33
//! Source: <https://github.com/rust-lang/cargo/blob/e1ebce1035f9b53bb46a55bd4b0ecf51e24c6458/src/cargo/ops/cargo_clean.rs#L324>
44
5+
use std::fs::Metadata;
56
use std::io;
7+
#[cfg(unix)]
8+
use std::os::unix::fs::MetadataExt;
69
use std::path::Path;
710

811
use tracing::debug;
912
use uv_fs::PhysicalSpaceError;
1013

1114
use crate::CleanReporter;
1215

13-
/// The storage accounting used when removing cache entries.
16+
/// How to estimate reclaimed storage when removing cache entries.
1417
#[derive(Debug, Clone, Copy, Default)]
15-
pub enum RemovalMode {
16-
/// Report the logical size of the removed files.
18+
pub enum RemovalAccounting {
19+
/// Estimate reclaimed storage from ordinary filesystem metadata.
1720
#[default]
18-
Logical,
19-
/// Report the exclusively owned physical storage reclaimed by the removed files.
20-
Physical,
21+
Coarse,
22+
/// Inspect filesystem allocation and sharing where supported.
23+
Fine,
2124
}
2225

2326
/// A builder for a [`Remover`] that can remove files and directories.
2427
#[derive(Default)]
2528
pub(crate) struct Remover {
2629
reporter: Option<Box<dyn CleanReporter>>,
27-
removal_mode: RemovalMode,
30+
removal_accounting: RemovalAccounting,
2831
}
2932

3033
impl Remover {
@@ -37,8 +40,8 @@ impl Remover {
3740
}
3841

3942
/// Set the storage accounting used before each file is removed.
40-
pub(crate) fn with_removal_mode(mut self, removal_mode: RemovalMode) -> Self {
41-
self.removal_mode = removal_mode;
43+
pub(crate) fn with_removal_accounting(mut self, removal_accounting: RemovalAccounting) -> Self {
44+
self.removal_accounting = removal_accounting;
4245
self
4346
}
4447

@@ -49,65 +52,78 @@ impl Remover {
4952
path: impl AsRef<Path>,
5053
skip_locked_file: bool,
5154
) -> io::Result<Removal> {
52-
let mut removal = Removal::new(self.removal_mode);
55+
let mut removal = Removal::new(self.removal_accounting);
5356
removal.rm_rf(path.as_ref(), self.reporter.as_deref(), skip_locked_file)?;
5457
Ok(removal)
5558
}
5659
}
5760

61+
/// Estimate the storage reclaimed by removing a non-directory entry.
62+
#[cfg(unix)]
63+
fn file_size(metadata: &Metadata) -> u64 {
64+
if metadata.nlink() == 1 {
65+
metadata.blocks().saturating_mul(512)
66+
} else {
67+
0
68+
}
69+
}
70+
71+
/// Estimate the storage reclaimed by removing a non-directory entry.
72+
#[cfg(not(unix))]
73+
fn file_size(metadata: &Metadata) -> u64 {
74+
metadata.len()
75+
}
76+
5877
/// A removal operation with statistics on the number of files and directories removed.
5978
#[derive(Debug, Default)]
6079
pub struct Removal {
6180
/// The number of files removed.
6281
pub num_files: u64,
6382
/// The number of directories removed.
6483
pub num_dirs: u64,
65-
/// The logical number of bytes removed.
66-
///
67-
/// Note: this will both over-count bytes removed for hard-linked files, and under-count
68-
/// bytes in general since it's a measure of the exact byte size (as opposed to the block size).
69-
pub logical_bytes: u64,
70-
/// The exclusively owned physical file data reclaimed by the removal, when available.
71-
pub physical_bytes: Option<u64>,
72-
/// Whether any removed entries could not be measured, making the physical count a lower bound.
73-
pub physical_bytes_incomplete: bool,
84+
/// The coarse estimate of the number of bytes occupied by the removed files.
85+
pub coarse_bytes: u64,
86+
/// The fine-grained estimate of reclaimed physical file data, when available.
87+
pub fine_bytes: Option<u64>,
88+
/// Whether any removed entries could not be measured, making the fine-grained count a lower bound.
89+
pub fine_bytes_incomplete: bool,
7490
}
7591

7692
impl Removal {
7793
/// Create an empty removal summary with the requested storage accounting.
78-
pub(crate) fn new(removal_mode: RemovalMode) -> Self {
94+
pub(crate) fn new(removal_accounting: RemovalAccounting) -> Self {
7995
Self {
80-
physical_bytes: match removal_mode {
81-
RemovalMode::Logical => None,
82-
RemovalMode::Physical => Some(0),
96+
fine_bytes: match removal_accounting {
97+
RemovalAccounting::Coarse => None,
98+
RemovalAccounting::Fine => Some(0),
8399
},
84100
..Self::default()
85101
}
86102
}
87103

88104
/// Account for a file while its current sharing state can still be inspected.
89-
fn add_file(&mut self, path: &Path, metadata: &std::fs::Metadata) {
90-
self.logical_bytes += metadata.len();
105+
fn add_file(&mut self, path: &Path, metadata: &Metadata) {
106+
self.coarse_bytes += file_size(metadata);
91107

92-
if let Some(physical_bytes) = self.physical_bytes {
108+
if let Some(fine_bytes) = self.fine_bytes {
93109
match uv_fs::physical_space(path, metadata) {
94-
Ok(physical) => {
95-
self.physical_bytes = Some(physical_bytes.saturating_add(physical));
110+
Ok(bytes) => {
111+
self.fine_bytes = Some(fine_bytes.saturating_add(bytes));
96112
}
97113
Err(PhysicalSpaceError::UnsupportedFilesystem) => {
98114
debug!(
99-
"Physical space accounting is unsupported for {}; falling back to logical space",
115+
"Fine-grained space accounting is unsupported for {}; falling back to coarse accounting",
100116
path.display()
101117
);
102-
self.physical_bytes = None;
103-
self.physical_bytes_incomplete = false;
118+
self.fine_bytes = None;
119+
self.fine_bytes_incomplete = false;
104120
}
105121
Err(PhysicalSpaceError::UnmeasurableFile(error)) => {
106122
debug!(
107123
"Failed to measure physical space for {}: {error}",
108124
path.display()
109125
);
110-
self.physical_bytes_incomplete = true;
126+
self.fine_bytes_incomplete = true;
111127
}
112128
}
113129
}
@@ -218,8 +234,8 @@ impl Removal {
218234
// Remove the file.
219235
if let Ok(metadata) = entry.metadata() {
220236
self.add_file(entry.path(), &metadata);
221-
} else if self.physical_bytes.is_some() {
222-
self.physical_bytes_incomplete = true;
237+
} else if self.fine_bytes.is_some() {
238+
self.fine_bytes_incomplete = true;
223239
}
224240
remove_file(entry.path())?;
225241
}
@@ -237,13 +253,13 @@ impl std::ops::AddAssign for Removal {
237253
fn add_assign(&mut self, other: Self) {
238254
self.num_files += other.num_files;
239255
self.num_dirs += other.num_dirs;
240-
self.logical_bytes += other.logical_bytes;
241-
self.physical_bytes = self
242-
.physical_bytes
243-
.zip(other.physical_bytes)
256+
self.coarse_bytes += other.coarse_bytes;
257+
self.fine_bytes = self
258+
.fine_bytes
259+
.zip(other.fine_bytes)
244260
.map(|(left, right)| left.saturating_add(right));
245-
self.physical_bytes_incomplete = self.physical_bytes.is_some()
246-
&& (self.physical_bytes_incomplete || other.physical_bytes_incomplete);
261+
self.fine_bytes_incomplete = self.fine_bytes.is_some()
262+
&& (self.fine_bytes_incomplete || other.fine_bytes_incomplete);
247263
}
248264
}
249265

0 commit comments

Comments
 (0)