Skip to content

Commit a01ed3e

Browse files
🐛 fix(seed): verify downloaded wheel digests (#3251)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent d721ff1 commit a01ed3e

5 files changed

Lines changed: 171 additions & 4 deletions

File tree

‎docs/changelog/3251.bugfix.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Verify a downloaded seed wheel's sha256 against PyPI before seeding it into a virtual environment, skipped when a custom
2+
pip index is configured.

‎src/virtualenv/seed/wheels/acquire.py‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,31 @@
88
from operator import eq, lt
99
from pathlib import Path
1010
from subprocess import PIPE, CalledProcessError, Popen
11-
from typing import TYPE_CHECKING
11+
from typing import TYPE_CHECKING, Final
1212

1313
from .bundle import from_bundle
14-
from .periodic_update import add_wheel_to_update_log
14+
from .periodic_update import add_wheel_to_update_log, verify_wheel_digest
1515
from .util import Version, Wheel, discover_wheels
1616

1717
if TYPE_CHECKING:
1818
from virtualenv.app_data.base import AppData
1919

2020
LOGGER = logging.getLogger(__name__)
2121

22+
# A private index can legitimately serve a rebuilt wheel under the same distribution, version, and
23+
# filename PyPI also uses - an internal mirror re-signing or re-packaging a release for policy reasons,
24+
# for example. Verifying that file against PyPI's own digest would then reject a wheel that came from
25+
# exactly the source it was configured to come from, not a compromised one. This does not attempt to
26+
# find every way pip can be pointed elsewhere (a pip.conf file has the same effect and is invisible
27+
# here), only the common, explicit one, so treat this as a conservative signal rather than proof either
28+
# way: it exists to avoid false rejections, not to guarantee verification runs whenever it safely could.
29+
_CUSTOM_INDEX_ENV_VARS: Final[tuple[str, ...]] = ("PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_INDEX")
30+
31+
32+
def _uses_default_index(env: dict[str, str]) -> bool:
33+
return not any(env.get(var) for var in _CUSTOM_INDEX_ENV_VARS)
34+
35+
2236
# PEP 503 normalized distribution name. Anything outside this character set on the way to ``pip download`` means
2337
# somebody is smuggling pip options or extras, so reject it before we build the command line.
2438
_DISTRIBUTION_RE = re.compile(
@@ -108,6 +122,9 @@ def download_wheel( # ruff:ignore[too-many-arguments]
108122
109123
:raises ValueError: if ``distribution`` or ``version_spec`` fail the strict allow-list check.
110124
:raises CalledProcessError: if ``pip download`` exits with a non-zero status.
125+
:raises RuntimeError: if no downloaded wheel can be identified from ``pip``'s output or the search directory, or if
126+
the caller has not configured a custom index and PyPI has a published digest for the downloaded filename that
127+
does not match, see :func:`virtualenv.seed.wheels.periodic_update.verify_wheel_digest`.
111128
112129
"""
113130
_check_distribution(distribution)
@@ -130,6 +147,7 @@ def download_wheel( # ruff:ignore[too-many-arguments]
130147
str(to_folder),
131148
to_download,
132149
]
150+
verify_against_pypi = _uses_default_index(env)
133151
# pip has no interface in python - must be a new sub-process
134152
env = pip_wheel_env_run(search_dirs, app_data, env)
135153
process = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True, encoding="utf-8")
@@ -138,8 +156,13 @@ def download_wheel( # ruff:ignore[too-many-arguments]
138156
kwargs = {"output": out, "stderr": err}
139157
raise CalledProcessError(process.returncode, cmd, **kwargs)
140158
result = _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out)
141-
LOGGER.debug("downloaded wheel %s", result.name) # ty: ignore[unresolved-attribute]
142-
return result # ty: ignore[invalid-return-type]
159+
if result is None:
160+
msg = f"could not find downloaded wheel for {to_download}"
161+
raise RuntimeError(msg)
162+
LOGGER.debug("downloaded wheel %s", result.name)
163+
if verify_against_pypi:
164+
verify_wheel_digest(result)
165+
return result
143166

144167

145168
def _find_downloaded_wheel(

‎src/virtualenv/seed/wheels/periodic_update.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from __future__ import annotations
44

5+
import hashlib
56
import json
67
import logging
78
import os
@@ -360,6 +361,42 @@ def release_date_for_wheel_path(dest: Path) -> datetime | None:
360361
return None
361362

362363

364+
def verify_wheel_digest(wheel: Wheel) -> None:
365+
"""Verify a downloaded wheel's sha256 against the digest PyPI's JSON API reports for that release.
366+
367+
``pip download`` trusts whatever index it is configured to use, and nothing else in this module checks the bytes it
368+
hands back. A missing PyPI record only means this check cannot run - the wheel may come from a private index PyPI
369+
has never heard of - so that case is logged and let through. An actual mismatch means the file on disk is not the
370+
release PyPI published under that filename, which is what a compromised index, a stale mirror, or a MITM'd download
371+
would produce, so that case is fatal: the wheel must never be cached or seeded into a venv.
372+
373+
:raises RuntimeError: if PyPI's record for this exact filename exists and the digest does not match.
374+
375+
"""
376+
entry = _pypi_release_entry_for_wheel(wheel)
377+
if entry is None:
378+
LOGGER.debug("could not verify %s against PyPI: no matching release record", wheel.name)
379+
return
380+
digests = entry.get("digests")
381+
expected = digests.get("sha256") if isinstance(digests, dict) else None
382+
if not isinstance(expected, str):
383+
LOGGER.debug("could not verify %s against PyPI: no sha256 digest published", wheel.name)
384+
return
385+
actual = hashlib.sha256(wheel.path.read_bytes()).hexdigest()
386+
if actual != expected:
387+
msg = f"downloaded wheel {wheel.name} has sha256 {actual}, but PyPI reports {expected} for this release"
388+
raise RuntimeError(msg)
389+
390+
391+
def _pypi_release_entry_for_wheel(wheel: Wheel) -> dict[str, object] | None:
392+
content = _pypi_get_distribution_info_cached(wheel.distribution)
393+
releases = content.get("releases") if isinstance(content, dict) else None
394+
entries = releases.get(wheel.version) if isinstance(releases, dict) else None
395+
if not isinstance(entries, list):
396+
return None
397+
return next((entry for entry in entries if isinstance(entry, dict) and entry.get("filename") == wheel.name), None)
398+
399+
363400
#: Opt-in escape hatch to restore the pre-2026 behavior of falling back to an unverified HTTPS context when the
364401
#: verified request fails. Off by default: a failed TLS handshake on the PyPI metadata lookup now aborts the update
365402
#: instead of silently downgrading, because the response drives which wheel version virtualenv thinks is up to date.
@@ -470,4 +507,5 @@ def _run_manual_upgrade(app_data: AppData, distribution: str, for_py_version: st
470507
"periodic_update",
471508
"release_date_for_wheel_path",
472509
"trigger_update",
510+
"verify_wheel_digest",
473511
]

‎tests/unit/seed/wheels/test_acquire.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ def test_download_wheel_bad_output(mocker, for_py_version, session_app_data) ->
3838
distribution = "setuptools"
3939
p_open = mocker.MagicMock()
4040
mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open)
41+
mocker.patch("virtualenv.seed.wheels.acquire.verify_wheel_digest")
4142
p_open.communicate.return_value = "", ""
4243
p_open.returncode = 0
4344

@@ -58,6 +59,19 @@ def test_download_wheel_bad_output(mocker, for_py_version, session_app_data) ->
5859
assert result.path == embed.path
5960

6061

62+
def test_download_wheel_not_found_raises(mocker, for_py_version, session_app_data) -> None:
63+
"""download_wheel must raise a clear error rather than crash when no wheel can be identified at all."""
64+
p_open = mocker.MagicMock()
65+
mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open)
66+
p_open.communicate.return_value = "", ""
67+
p_open.returncode = 0
68+
as_path = mocker.MagicMock()
69+
as_path.iterdir.return_value = []
70+
71+
with pytest.raises(RuntimeError, match="could not find downloaded wheel"):
72+
download_wheel("setuptools", None, for_py_version, [], session_app_data, as_path, os.environ)
73+
74+
6175
def test_download_fails(mocker, for_py_version, session_app_data) -> None:
6276
p_open = mocker.MagicMock()
6377
mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open)
@@ -94,13 +108,51 @@ def test_download_wheel_python_io_encoding(mocker, for_py_version, session_app_d
94108
mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", ""
95109
mock_popen.return_value.returncode = 0
96110
mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl"))
111+
mocker.patch("virtualenv.seed.wheels.acquire.verify_wheel_digest")
97112

98113
download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", os.environ.copy())
99114

100115
env = mock_popen.call_args[1]["env"]
101116
assert env["PYTHONIOENCODING"] == "utf-8"
102117

103118

119+
def test_download_wheel_verifies_digest(mocker, for_py_version, session_app_data) -> None:
120+
"""download_wheel must reject a wheel whose bytes do not match PyPI's published digest."""
121+
mock_popen = mocker.patch("virtualenv.seed.wheels.acquire.Popen")
122+
mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", ""
123+
mock_popen.return_value.returncode = 0
124+
mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl"))
125+
verify = mocker.patch(
126+
"virtualenv.seed.wheels.acquire.verify_wheel_digest",
127+
side_effect=RuntimeError("digest mismatch"),
128+
)
129+
130+
with pytest.raises(RuntimeError, match="digest mismatch"):
131+
download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", os.environ.copy())
132+
133+
assert verify.call_count == 1
134+
135+
136+
@pytest.mark.parametrize("env_var", ["PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_INDEX"])
137+
def test_download_wheel_skips_digest_check_for_custom_index(mocker, for_py_version, session_app_data, env_var) -> None:
138+
"""A configured custom index may legitimately serve a same-named wheel PyPI never built."""
139+
mock_popen = mocker.patch("virtualenv.seed.wheels.acquire.Popen")
140+
mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", ""
141+
mock_popen.return_value.returncode = 0
142+
mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl"))
143+
verify = mocker.patch(
144+
"virtualenv.seed.wheels.acquire.verify_wheel_digest",
145+
side_effect=RuntimeError("would have rejected a legitimate wheel"),
146+
)
147+
env = os.environ.copy()
148+
env[env_var] = "https://example.com/simple"
149+
150+
result = download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", env)
151+
152+
assert result.name == "a-b-c.whl"
153+
assert verify.call_count == 0
154+
155+
104156
@pytest.fixture
105157
def downloaded_wheel(mocker):
106158
wheel = Wheel.from_path(Path("setuptools-0.0.0-py2.py3-none-any.whl"))

‎tests/unit/seed/wheels/test_periodic_update.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from __future__ import annotations
22

3+
import hashlib
34
import json
45
import os
56
import subprocess
@@ -29,6 +30,7 @@
2930
periodic_update,
3031
release_date_for_wheel_path,
3132
trigger_update,
33+
verify_wheel_digest,
3234
)
3335
from virtualenv.util.subprocess import CREATE_NO_WINDOW
3436

@@ -545,6 +547,56 @@ def test_get_release_fails(mocker, caplog) -> None:
545547
assert repr(exc) in caplog.text
546548

547549

550+
def _pypi_release_response(mocker, filename, digests, version="20.1"):
551+
entry = {"filename": filename, "digests": digests}
552+
body = json.dumps({"releases": {version: [entry]}})
553+
return mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", return_value=StringIO(body))
554+
555+
556+
def test_verify_wheel_digest_matches(tmp_path, mocker) -> None:
557+
wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl")
558+
wheel.path.write_bytes(b"content")
559+
sha256 = hashlib.sha256(b"content").hexdigest()
560+
_pypi_release_response(mocker, wheel.name, {"sha256": sha256})
561+
562+
verify_wheel_digest(wheel) # must not raise
563+
564+
565+
def test_verify_wheel_digest_mismatch(tmp_path, mocker) -> None:
566+
wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl")
567+
wheel.path.write_bytes(b"content")
568+
_pypi_release_response(mocker, wheel.name, {"sha256": "0" * 64})
569+
570+
with pytest.raises(RuntimeError, match=r"has sha256 .* but PyPI reports"):
571+
verify_wheel_digest(wheel)
572+
573+
574+
@pytest.mark.parametrize(
575+
("filename", "digests", "version"),
576+
[
577+
pytest.param("setuptools-99.0-py3-none-any.whl", {"sha256": "0" * 64}, "20.1", id="filename_not_in_release"),
578+
pytest.param(
579+
"pip-20.1-py3-none-any.whl", {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "20.1", id="no_sha256_digest"
580+
),
581+
pytest.param("pip-20.1-py3-none-any.whl", {"sha256": "0" * 64}, "19.0", id="version_not_in_releases"),
582+
],
583+
)
584+
def test_verify_wheel_digest_skips_when_unverifiable(tmp_path, mocker, filename, digests, version) -> None:
585+
wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl")
586+
wheel.path.write_bytes(b"content")
587+
_pypi_release_response(mocker, filename, digests, version)
588+
589+
verify_wheel_digest(wheel) # must not raise: nothing to compare against
590+
591+
592+
def test_verify_wheel_digest_skips_on_pypi_lookup_failure(tmp_path, mocker) -> None:
593+
wheel = Wheel(tmp_path / "pip-20.1-py3-none-any.whl")
594+
wheel.path.write_bytes(b"content")
595+
mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=URLError("offline"))
596+
597+
verify_wheel_digest(wheel) # must not raise: a private mirror PyPI never heard of is legitimate
598+
599+
548600
def mock_download(mocker, pip_version_remote):
549601
def download():
550602
index = 0

0 commit comments

Comments
 (0)