Skip to content

Commit bf1d7e8

Browse files
Encapsulate wheel extraction directory ownership (#21419)
## Summary Stacked on #21372. We now create wheel extraction directories through `WheelExtractor`. Both streaming and seekable extraction consume it and return an `ExtractedWheel` that owns the directory and file metadata through `RECORD` validation and cache persistence. This removes the separate `TempDir` transfers from callers and keeps the seekable worker's directory alive until extraction finishes. Implements the ownership refactor suggested in #21372 (comment).
1 parent 6dacee0 commit bf1d7e8

2 files changed

Lines changed: 111 additions & 90 deletions

File tree

‎crates/uv-distribution/src/distribution_database.rs‎

Lines changed: 47 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ use uv_types::{BuildContext, BuildStack};
4040

4141
use crate::archive::Archive;
4242
use crate::error::PythonVersion;
43-
use crate::extracted_wheel::{ExtractedWheel, HashedWheel};
43+
use crate::extracted_wheel::{ExtractedWheel, HashedWheel, WheelExtractor};
4444
use crate::hash::http_hash_algorithms;
4545
use crate::metadata::{ArchiveMetadata, Metadata};
4646
use crate::source::SourceDistributionBuilder;
@@ -716,27 +716,24 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
716716
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);
717717

718718
// Download and unzip the wheel to a temporary directory.
719-
let temp_dir = tempfile::tempdir_in(self.build_context.cache().root())
720-
.map_err(Error::CacheWrite)?;
719+
let extractor = WheelExtractor::new(
720+
self.build_context.cache().root(),
721+
self.content_addressed_cache,
722+
)
723+
.map_err(Error::CacheWrite)?;
721724

722-
let (temp_dir, mut extracted) = match progress {
725+
let mut extracted = match progress {
723726
Some((reporter, progress)) => {
724727
let mut reader = ProgressReader::new(&mut hasher, progress, &**reporter);
725-
ExtractedWheel::extract_streaming(
726-
&mut reader,
727-
temp_dir,
728-
self.content_addressed_cache,
729-
)
730-
.await
731-
.map_err(|err| Error::Extract(filename.to_string(), err))?
728+
extractor
729+
.extract_streaming(&mut reader)
730+
.await
731+
.map_err(|err| Error::Extract(filename.to_string(), err))?
732732
}
733-
None => ExtractedWheel::extract_streaming(
734-
&mut hasher,
735-
temp_dir,
736-
self.content_addressed_cache,
737-
)
738-
.await
739-
.map_err(|err| Error::Extract(filename.to_string(), err))?,
733+
None => extractor
734+
.extract_streaming(&mut hasher)
735+
.await
736+
.map_err(|err| Error::Extract(filename.to_string(), err))?,
740737
};
741738
// Exhaust the reader to compute the hashes.
742739
hasher.finish().await.map_err(Error::HashExhaustion)?;
@@ -753,11 +750,11 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
753750

754751
// Before we make the wheel accessible by persisting it, ensure that the RECORD is
755752
// valid.
756-
extracted.validate_and_heal_record(temp_dir.path(), dist)?;
753+
extracted.validate_and_heal_record(dist)?;
757754

758755
// Persist the temporary directory to the directory store.
759756
let id = self
760-
.persist_extracted_wheel(temp_dir, wheel_entry.path(), extracted.into_hashed())
757+
.persist_extracted_wheel(extracted, wheel_entry.path())
761758
.await?;
762759

763760
if let Some((reporter, progress)) = progress {
@@ -936,29 +933,28 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
936933
let actual_size = hasher.bytes_read();
937934

938935
// Unzip the wheel to a temporary directory.
939-
let temp_dir = tempfile::tempdir_in(self.build_context.cache().root())
940-
.map_err(Error::CacheWrite)?;
936+
let extractor =
937+
WheelExtractor::new(self.build_context.cache().root(), content_addressed_cache)
938+
.map_err(Error::CacheWrite)?;
941939
let mut file = writer.into_inner();
942940
file.seek(io::SeekFrom::Start(0))
943941
.await
944942
.map_err(Error::CacheWrite)?;
945943

946-
let target = temp_dir.path().to_owned();
947944
let file = file.into_std().await;
948-
let mut extracted = tokio::task::spawn_blocking(move || {
949-
ExtractedWheel::extract_seekable(file, &target, content_addressed_cache)
950-
})
951-
.await?
952-
.map_err(|err| Error::Extract(filename.to_string(), err))?;
945+
let mut extracted =
946+
tokio::task::spawn_blocking(move || extractor.extract_seekable(file))
947+
.await?
948+
.map_err(|err| Error::Extract(filename.to_string(), err))?;
953949
let hashes = hashers.into_iter().map(HashDigest::from).collect();
954950

955951
// Before we make the wheel accessible by persisting it, ensure that the RECORD is
956952
// valid.
957-
extracted.validate_and_heal_record(temp_dir.path(), dist)?;
953+
extracted.validate_and_heal_record(dist)?;
958954

959955
// Persist the temporary directory to the directory store.
960956
let id = self
961-
.persist_extracted_wheel(temp_dir, wheel_entry.path(), extracted.into_hashed())
957+
.persist_extracted_wheel(extracted, wheel_entry.path())
962958
.await?;
963959

964960
if let Some((reporter, progress)) = progress {
@@ -1128,22 +1124,22 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
11281124
let file = fs_err::tokio::File::open(path)
11291125
.await
11301126
.map_err(Error::CacheRead)?;
1131-
let temp_dir = tempfile::tempdir_in(self.build_context.cache().root())
1132-
.map_err(Error::CacheWrite)?;
1127+
let extractor = WheelExtractor::new(
1128+
self.build_context.cache().root(),
1129+
self.content_addressed_cache,
1130+
)
1131+
.map_err(Error::CacheWrite)?;
11331132

11341133
// Create a hasher for each hash algorithm.
11351134
let algorithms = hashes.algorithms();
11361135
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
11371136
let mut hasher = uv_extract::hash::HashReader::new(file, &mut hashers);
11381137

11391138
// Unzip the wheel to a temporary directory.
1140-
let (temp_dir, mut extracted) = ExtractedWheel::extract_streaming(
1141-
&mut hasher,
1142-
temp_dir,
1143-
self.content_addressed_cache,
1144-
)
1145-
.await
1146-
.map_err(|err| Error::Extract(filename.to_string(), err))?;
1139+
let mut extracted = extractor
1140+
.extract_streaming(&mut hasher)
1141+
.await
1142+
.map_err(|err| Error::Extract(filename.to_string(), err))?;
11471143

11481144
// Exhaust the reader to compute the hash.
11491145
hasher.finish().await.map_err(Error::HashExhaustion)?;
@@ -1152,11 +1148,11 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
11521148

11531149
// Before we make the wheel accessible by persisting it, ensure that the RECORD is
11541150
// valid.
1155-
extracted.validate_and_heal_record(temp_dir.path(), dist)?;
1151+
extracted.validate_and_heal_record(dist)?;
11561152

11571153
// Persist the temporary directory to the directory store.
11581154
let id = self
1159-
.persist_extracted_wheel(temp_dir, wheel_entry.path(), extracted.into_hashed())
1155+
.persist_extracted_wheel(extracted, wheel_entry.path())
11601156
.await?;
11611157

11621158
// Create an archive.
@@ -1193,31 +1189,26 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
11931189
) -> Result<ArchiveId, Error> {
11941190
let content_addressed_cache = self.content_addressed_cache;
11951191

1196-
let (temp_dir, mut extracted) = tokio::task::spawn_blocking({
1192+
let mut extracted = tokio::task::spawn_blocking({
11971193
let path = path.to_owned();
11981194
let root = self.build_context.cache().root().to_path_buf();
11991195
move || -> Result<_, Error> {
12001196
// Unzip the wheel into a temporary directory.
1201-
let temp_dir = tempfile::tempdir_in(root).map_err(Error::CacheWrite)?;
1197+
let extractor = WheelExtractor::new(&root, content_addressed_cache)
1198+
.map_err(Error::CacheWrite)?;
12021199
let reader = fs_err::File::open(&path).map_err(Error::CacheWrite)?;
1203-
let extracted = ExtractedWheel::extract_seekable(
1204-
reader,
1205-
temp_dir.path(),
1206-
content_addressed_cache,
1207-
)
1208-
.map_err(|err| Error::Extract(path.to_string_lossy().into_owned(), err))?;
1209-
Ok((temp_dir, extracted))
1200+
extractor
1201+
.extract_seekable(reader)
1202+
.map_err(|err| Error::Extract(path.to_string_lossy().into_owned(), err))
12101203
}
12111204
})
12121205
.await??;
12131206

12141207
// Before we make the wheel accessible by persisting it, ensure that the RECORD is valid.
1215-
extracted.validate_and_heal_record(temp_dir.path(), dist)?;
1208+
extracted.validate_and_heal_record(dist)?;
12161209

12171210
// Persist the temporary directory to the directory store.
1218-
let id = self
1219-
.persist_extracted_wheel(temp_dir, target, extracted.into_hashed())
1220-
.await?;
1211+
let id = self.persist_extracted_wheel(extracted, target).await?;
12211212

12221213
Ok(id)
12231214
}
@@ -1228,10 +1219,10 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
12281219
/// persistence retains the existing behavior of assigning a unique archive ID.
12291220
async fn persist_extracted_wheel(
12301221
&self,
1231-
temp_dir: tempfile::TempDir,
1222+
extracted: ExtractedWheel,
12321223
target: &Path,
1233-
hashed_wheel: Option<HashedWheel>,
12341224
) -> Result<ArchiveId, Error> {
1225+
let (temp_dir, hashed_wheel) = extracted.into_parts();
12351226
let cache = self.build_context.cache();
12361227
let (temp_dir, id) = if let Some(HashedWheel { files, tree }) = hashed_wheel {
12371228
let digest = DirectoryDigest::from(tree.hash());

‎crates/uv-distribution/src/extracted_wheel.rs‎

Lines changed: 64 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
use std::fmt::Display;
2+
use std::io;
23
use std::path::Path;
34

45
use either::Either;
6+
use tempfile::TempDir;
57
use tokio::io::AsyncRead;
68

79
use uv_extract::dirhash::{DirhashTree, HashedFile, UnhashedFile, dirhash_path};
@@ -16,68 +18,96 @@ pub(crate) struct HashedWheel {
1618
pub(crate) tree: DirhashTree,
1719
}
1820

21+
/// A temporary directory and configuration for extracting a wheel.
22+
pub(crate) struct WheelExtractor {
23+
temp_dir: TempDir,
24+
content_addressed: bool,
25+
}
26+
27+
/// An extracted wheel that owns its temporary directory until persistence.
28+
pub(crate) struct ExtractedWheel {
29+
temp_dir: TempDir,
30+
files: ExtractedFiles,
31+
}
32+
1933
/// Files extracted from a wheel, with or without content-addressing metadata.
20-
pub(crate) enum ExtractedWheel {
34+
enum ExtractedFiles {
2135
Unhashed(Vec<UnhashedFile>),
2236
Hashed(HashedWheel),
2337
}
2438

25-
impl ExtractedWheel {
39+
impl WheelExtractor {
40+
/// Create a temporary directory under the cache root for extracting a wheel.
41+
pub(crate) fn new(root: &Path, content_addressed: bool) -> io::Result<Self> {
42+
Ok(Self {
43+
temp_dir: tempfile::tempdir_in(root)?,
44+
content_addressed,
45+
})
46+
}
47+
2648
/// Extract a wheel from a streaming reader, optionally retaining its per-file digests.
2749
///
2850
/// See [`uv_extract::stream::unzip`] for buffering, cleanup, and download hash requirements.
2951
pub(crate) async fn extract_streaming<R>(
52+
self,
3053
reader: R,
31-
temp_dir: tempfile::TempDir,
32-
content_addressed: bool,
33-
) -> Result<(tempfile::TempDir, Self), uv_extract::Error>
54+
) -> Result<ExtractedWheel, uv_extract::Error>
3455
where
3556
R: AsyncRead + Unpin,
3657
{
37-
if content_addressed {
58+
if self.content_addressed {
3859
let (temp_dir, files, tree) =
39-
uv_extract::stream::unzip_and_hash(reader, temp_dir).await?;
40-
Ok((temp_dir, Self::Hashed(HashedWheel { files, tree })))
60+
uv_extract::stream::unzip_and_hash(reader, self.temp_dir).await?;
61+
Ok(ExtractedWheel {
62+
temp_dir,
63+
files: ExtractedFiles::Hashed(HashedWheel { files, tree }),
64+
})
4165
} else {
42-
let (temp_dir, files) = uv_extract::stream::unzip(reader, temp_dir).await?;
43-
Ok((temp_dir, Self::Unhashed(files)))
66+
let (temp_dir, files) = uv_extract::stream::unzip(reader, self.temp_dir).await?;
67+
Ok(ExtractedWheel {
68+
temp_dir,
69+
files: ExtractedFiles::Unhashed(files),
70+
})
4471
}
4572
}
4673

4774
/// Extract a wheel from a seekable file, optionally retaining its per-file digests.
4875
pub(crate) fn extract_seekable(
76+
self,
4977
reader: fs_err::File,
50-
target: &Path,
51-
content_addressed: bool,
52-
) -> Result<Self, uv_extract::Error> {
53-
if content_addressed {
54-
let (files, tree) = uv_extract::unzip_and_hash(reader, target)?;
55-
Ok(Self::Hashed(HashedWheel { files, tree }))
78+
) -> Result<ExtractedWheel, uv_extract::Error> {
79+
let files = if self.content_addressed {
80+
let (files, tree) = uv_extract::unzip_and_hash(reader, self.temp_dir.path())?;
81+
ExtractedFiles::Hashed(HashedWheel { files, tree })
5682
} else {
57-
let files = uv_extract::unzip(reader, target)?;
58-
Ok(Self::Unhashed(files))
59-
}
83+
let files = uv_extract::unzip(reader, self.temp_dir.path())?;
84+
ExtractedFiles::Unhashed(files)
85+
};
86+
Ok(ExtractedWheel {
87+
temp_dir: self.temp_dir,
88+
files,
89+
})
6090
}
91+
}
6192

62-
/// Return the hashed wheel if content hashing was enabled.
63-
pub(crate) fn into_hashed(self) -> Option<HashedWheel> {
64-
match self {
65-
Self::Unhashed(_) => None,
66-
Self::Hashed(wheel) => Some(wheel),
67-
}
93+
impl ExtractedWheel {
94+
/// Return the temporary directory and optional content hashes for persistence.
95+
pub(crate) fn into_parts(self) -> (TempDir, Option<HashedWheel>) {
96+
let hashed_wheel = match self.files {
97+
ExtractedFiles::Unhashed(_) => None,
98+
ExtractedFiles::Hashed(wheel) => Some(wheel),
99+
};
100+
(self.temp_dir, hashed_wheel)
68101
}
69102

70103
/// Heal the wheel's `RECORD` and keep its hash tree consistent with the repaired contents.
71-
pub(crate) fn validate_and_heal_record(
72-
&mut self,
73-
root: &Path,
74-
dist: impl Display,
75-
) -> Result<(), Error> {
76-
let files = match self {
77-
Self::Unhashed(files) => {
104+
pub(crate) fn validate_and_heal_record(&mut self, dist: impl Display) -> Result<(), Error> {
105+
let root = self.temp_dir.path();
106+
let files = match &self.files {
107+
ExtractedFiles::Unhashed(files) => {
78108
Either::Left(files.iter().map(|file| (file.path(), file.size())))
79109
}
80-
Self::Hashed(wheel) => {
110+
ExtractedFiles::Hashed(wheel) => {
81111
Either::Right(wheel.files.iter().map(|file| (file.path(), file.size())))
82112
}
83113
};
@@ -86,7 +116,7 @@ impl ExtractedWheel {
86116
else {
87117
return Ok(());
88118
};
89-
let Self::Hashed(hashed_wheel) = self else {
119+
let ExtractedFiles::Hashed(hashed_wheel) = &mut self.files else {
90120
return Ok(());
91121
};
92122

0 commit comments

Comments
 (0)