Skip to content

Commit d3cd632

Browse files
Always compute SHA256 for remote distributions (#19662)
## Summary Prior to this change, when we downloaded an HTTP wheel or source distribution without an active hash policy, we cached its extracted contents without recording a digest. If a later operation required the SHA-256 recorded in `uv.lock`, we couldn't verify the cached distribution without downloading it again. (Note that we don't store the archive itself in the cache; only the unpacked contents.) This PR always computes SHA-256 while downloading remote wheels and source distributions so that there's no need to re-download a distribution if you encounter it through a different path. It could make `uv pip install` a bit slower in some cases, but it should be negligible given that we hash as we stream (and this is already our default for `uv lock`-based operations). Closes #16786.
1 parent 48548c4 commit d3cd632

6 files changed

Lines changed: 128 additions & 60 deletions

File tree

‎crates/uv-distribution/src/distribution_database.rs‎

Lines changed: 22 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@ use uv_git::{GIT_LFS, GitError};
2828
use uv_install_wheel::validate_and_heal_record;
2929
use uv_platform_tags::Tags;
3030
use uv_pypi_types::{HashDigest, HashDigests, PyProjectToml};
31+
use uv_python::PythonVariant;
3132
use uv_redacted::DisplaySafeUrl;
3233
use uv_types::{BuildContext, BuildStack};
3334
use uv_warnings::warn_user_once;
3435

3536
use crate::archive::Archive;
36-
use uv_python::PythonVariant;
37-
3837
use crate::error::PythonVersion;
38+
use crate::hash::http_hash_algorithms;
3939
use crate::metadata::{ArchiveMetadata, Metadata};
4040
use crate::source::SourceDistributionBuilder;
4141
use crate::{Error, LocalWheel, Reporter, RequiresDist};
@@ -731,7 +731,7 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
731731
.into_async_read();
732732

733733
// Create a hasher for each hash algorithm.
734-
let algorithms = hashes.algorithms();
734+
let algorithms = http_hash_algorithms(hashes);
735735
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
736736
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);
737737

@@ -768,10 +768,8 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
768768
}
769769
},
770770
};
771-
// If necessary, exhaust the reader to compute the hash.
772-
if !hashes.is_none() {
773-
hasher.finish().await.map_err(Error::HashExhaustion)?;
774-
}
771+
// Exhaust the reader to compute the hashes.
772+
hasher.finish().await.map_err(Error::HashExhaustion)?;
775773

776774
// Before we make the wheel accessible by persisting it, ensure that the RECORD is
777775
// valid.
@@ -891,8 +889,6 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
891889
// Create an entry for the HTTP cache.
892890
let http_entry = wheel_entry.with_file(format!("{}.http", filename.cache_key()));
893891

894-
let query_url = &url.clone();
895-
896892
let download = |response: reqwest::Response| {
897893
async {
898894
let size = size.or_else(|| content_length(&response));
@@ -906,6 +902,9 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
906902
.bytes_stream()
907903
.map_err(|err| self.handle_response_errors(err))
908904
.into_async_read();
905+
let algorithms = http_hash_algorithms(hashes);
906+
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
907+
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);
909908

910909
// Download the wheel to a temporary file.
911910
let temp_file = tempfile::tempfile_in(self.build_context.cache().root())
@@ -917,18 +916,16 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
917916

918917
match progress {
919918
Some((reporter, progress)) => {
920-
// Wrap the reader in a progress reporter. This will report 100% progress
921-
// after the download is complete, even if we still have to unzip and hash
922-
// part of the file.
923-
let mut reader =
924-
ProgressReader::new(reader.compat(), progress, &**reporter);
919+
// Wrap the reader in a progress reporter. This will report 100% progress once
920+
// the download is complete, before the wheel is unzipped.
921+
let mut reader = ProgressReader::new(&mut hasher, progress, &**reporter);
925922

926923
tokio::io::copy(&mut reader, &mut writer)
927924
.await
928925
.map_err(Error::CacheWrite)?;
929926
}
930927
None => {
931-
tokio::io::copy(&mut reader.compat(), &mut writer)
928+
tokio::io::copy(&mut hasher, &mut writer)
932929
.await
933930
.map_err(Error::CacheWrite)?;
934931
}
@@ -942,47 +939,17 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
942939
.await
943940
.map_err(Error::CacheWrite)?;
944941

945-
// If no hashes are required, extract the wheel without hashing.
946-
let (files, hashes) = if hashes.is_none() {
947-
let target = temp_dir.path().to_owned();
948-
let files = match extension {
949-
WheelExtension::Whl => {
950-
let file = file.into_std().await;
951-
tokio::task::spawn_blocking(move || uv_extract::unzip(file, &target))
952-
.await?
953-
}
954-
WheelExtension::WhlZst => {
955-
uv_extract::stream::untar_zst(file, &target).await
956-
}
942+
let target = temp_dir.path().to_owned();
943+
let files = match extension {
944+
WheelExtension::Whl => {
945+
let file = file.into_std().await;
946+
tokio::task::spawn_blocking(move || uv_extract::unzip(file, &target))
947+
.await?
957948
}
958-
.map_err(|err| Error::Extract(filename.to_string(), err))?;
959-
960-
(files, HashDigests::empty())
961-
} else {
962-
// Create a hasher for each hash algorithm.
963-
let algorithms = hashes.algorithms();
964-
let mut hashers = algorithms.into_iter().map(Hasher::from).collect::<Vec<_>>();
965-
let mut hasher = uv_extract::hash::HashReader::new(file, &mut hashers);
966-
967-
let files = match extension {
968-
WheelExtension::Whl => {
969-
uv_extract::stream::unzip(query_url, &mut hasher, temp_dir.path())
970-
.await
971-
.map_err(|err| Error::Extract(filename.to_string(), err))?
972-
}
973-
WheelExtension::WhlZst => {
974-
uv_extract::stream::untar_zst(&mut hasher, temp_dir.path())
975-
.await
976-
.map_err(|err| Error::Extract(filename.to_string(), err))?
977-
}
978-
};
979-
980-
// If necessary, exhaust the reader to compute the hash.
981-
hasher.finish().await.map_err(Error::HashExhaustion)?;
982-
let hashes = hashers.into_iter().map(HashDigest::from).collect();
983-
984-
(files, hashes)
985-
};
949+
WheelExtension::WhlZst => uv_extract::stream::untar_zst(file, &target).await,
950+
}
951+
.map_err(|err| Error::Extract(filename.to_string(), err))?;
952+
let hashes = hashers.into_iter().map(HashDigest::from).collect();
986953

987954
// Before we make the wheel accessible by persisting it, ensure that the RECORD is
988955
// valid.

‎crates/uv-distribution/src/hash.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
use uv_distribution_types::HashPolicy;
2+
use uv_pypi_types::HashAlgorithm;
3+
4+
/// Return the algorithms to compute for an HTTP distribution.
5+
pub(crate) fn http_hash_algorithms(hashes: HashPolicy<'_>) -> Vec<HashAlgorithm> {
6+
let mut algorithms = hashes.algorithms();
7+
algorithms.push(HashAlgorithm::Sha256);
8+
algorithms.sort();
9+
algorithms.dedup();
10+
algorithms
11+
}

‎crates/uv-distribution/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ mod archive;
1414
mod distribution_database;
1515
mod download;
1616
mod error;
17+
mod hash;
1718
mod index;
1819
mod metadata;
1920
mod reporter;

‎crates/uv-distribution/src/source/mod.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ use uv_workspace::pyproject::ToolUvSources;
5050

5151
use crate::distribution_database::ManagedClient;
5252
use crate::error::Error;
53+
use crate::hash::http_hash_algorithms;
5354
use crate::metadata::{ArchiveMetadata, GitWorkspaceMember, Metadata};
5455
use crate::source::built_wheel_metadata::{BuiltWheelFile, BuiltWheelMetadata};
5556
use crate::source::revision::Revision;
@@ -980,7 +981,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
980981
// Download the source distribution.
981982
debug!("Downloading source distribution: {source}");
982983
let entry = cache_shard.shard(revision.id()).entry(SOURCE);
983-
let algorithms = hashes.algorithms();
984+
let algorithms = http_hash_algorithms(hashes);
984985
let hashes = self
985986
.download_archive(query_url, response, source, ext, entry.path(), &algorithms)
986987
.await?;
@@ -2718,7 +2719,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
27182719
async {
27192720
// Take the union of the requested and existing hash algorithms.
27202721
let algorithms = {
2721-
let mut algorithms = hashes.algorithms();
2722+
let mut algorithms = http_hash_algorithms(hashes);
27222723
for digest in revision.hashes() {
27232724
algorithms.push(digest.algorithm());
27242725
}

‎crates/uv/tests/it/pip_sync.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3913,6 +3913,7 @@ fn require_hashes_wrong_algorithm() -> Result<()> {
39133913
sha512:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f
39143914
39153915
Computed:
3916+
sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f
39163917
sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2
39173918
"
39183919
);

‎crates/uv/tests/it/sync.rs‎

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,95 @@ fn sync() -> Result<()> {
4949
Ok(())
5050
}
5151

52+
/// Ensure that `uv sync` reuses remote wheels cached by `uv pip install`.
53+
#[test]
54+
fn sync_reuses_pip_install_wheel_cache() -> Result<()> {
55+
let context = uv_test::test_context!("3.12");
56+
57+
let pyproject_toml = context.temp_dir.child("pyproject.toml");
58+
pyproject_toml.write_str(
59+
r#"
60+
[project]
61+
name = "project"
62+
version = "0.1.0"
63+
requires-python = ">=3.12"
64+
dependencies = ["iniconfig==2.0.0"]
65+
"#,
66+
)?;
67+
68+
// Create a lockfile with hashes, but populate the wheel cache via `uv pip install`.
69+
context.lock().assert().success();
70+
context
71+
.pip_install()
72+
.arg("iniconfig==2.0.0")
73+
.assert()
74+
.success();
75+
fs_err::remove_dir_all(&context.venv)?;
76+
77+
uv_snapshot!(context.filters(), context.sync().arg("--offline"), @"
78+
success: true
79+
exit_code: 0
80+
----- stdout -----
81+
82+
----- stderr -----
83+
Using CPython 3.12.[X] interpreter at: [PYTHON-3.12]
84+
Creating virtual environment at: .venv
85+
Resolved 2 packages in [TIME]
86+
Installed 1 package in [TIME]
87+
+ iniconfig==2.0.0
88+
");
89+
90+
Ok(())
91+
}
92+
93+
/// Ensure that `uv sync` reuses remote source distributions cached by `uv pip install`.
94+
#[test]
95+
fn sync_reuses_pip_install_sdist_cache() -> Result<()> {
96+
let context = uv_test::test_context!("3.12");
97+
98+
let pyproject_toml = context.temp_dir.child("pyproject.toml");
99+
pyproject_toml.write_str(
100+
r#"
101+
[project]
102+
name = "project"
103+
version = "0.1.0"
104+
requires-python = ">=3.12"
105+
dependencies = ["iniconfig==2.0.0"]
106+
"#,
107+
)?;
108+
109+
// Create a lockfile with hashes, but populate the source distribution cache via `uv pip
110+
// install`.
111+
context.lock().assert().success();
112+
context
113+
.pip_install()
114+
.arg("iniconfig==2.0.0")
115+
.arg("--no-binary")
116+
.arg("iniconfig")
117+
.assert()
118+
.success();
119+
fs_err::remove_dir_all(&context.venv)?;
120+
121+
uv_snapshot!(context.filters(), context
122+
.sync()
123+
.arg("--offline")
124+
.arg("--no-binary-package")
125+
.arg("iniconfig"), @"
126+
success: true
127+
exit_code: 0
128+
----- stdout -----
129+
130+
----- stderr -----
131+
Using CPython 3.12.[X] interpreter at: [PYTHON-3.12]
132+
Creating virtual environment at: .venv
133+
Resolved 2 packages in [TIME]
134+
Installed 1 package in [TIME]
135+
+ iniconfig==2.0.0
136+
");
137+
138+
Ok(())
139+
}
140+
52141
#[test]
53142
fn locked() -> Result<()> {
54143
let context = uv_test::test_context!("3.12");
@@ -9932,7 +10021,6 @@ fn sync_all_extras() -> Result<()> {
993210021
993310022
----- stderr -----
993410023
Resolved 8 packages in [TIME]
9935-
Prepared 1 package in [TIME]
993610024
Uninstalled 2 packages in [TIME]
993710025
Installed 1 package in [TIME]
993810026
+ packaging==24.0
@@ -10231,7 +10319,6 @@ fn sync_all_groups() -> Result<()> {
1023110319
1023210320
----- stderr -----
1023310321
Resolved 8 packages in [TIME]
10234-
Prepared 1 package in [TIME]
1023510322
Uninstalled 2 packages in [TIME]
1023610323
Installed 1 package in [TIME]
1023710324
+ packaging==24.0
@@ -10617,7 +10704,7 @@ fn sync_stale_egg_info() -> Result<()> {
1061710704
1061810705
----- stderr -----
1061910706
Resolved 4 packages in [TIME]
10620-
Prepared 3 packages in [TIME]
10707+
Prepared 2 packages in [TIME]
1062110708
Installed 3 packages in [TIME]
1062210709
+ member==0.1.dev5+gfea1041 (from git+https://github.com/astral-sh/uv-stale-egg-info-test.git@fea10416b9c479ac88fb217e14e40249b63bfbee#subdirectory=member)
1062310710
+ root==0.1.dev5+gfea1041 (from git+https://github.com/astral-sh/uv-stale-egg-info-test.git@fea10416b9c479ac88fb217e14e40249b63bfbee)

0 commit comments

Comments
 (0)