Skip to content

Commit 7babec6

Browse files
authored
🐛 fix(seed): verify wheels before caching (#3430)
1 parent 2dc8bd8 commit 7babec6

5 files changed

Lines changed: 285 additions & 252 deletions

File tree

‎docs/changelog/3430.bugfix.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Verify downloads in a private staging directory before adding them to the reusable wheel cache. Select the downloaded
2+
archive from that directory instead of pip's output. The pip seeder now uses the same download verification path and
3+
installs pinned versions from a temporary directory containing only the selected wheels, without another index lookup.
4+
Seeded packages retain version pins in pip freeze output.

‎src/virtualenv/seed/embed/pip_invoke.py‎

Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,20 @@
22

33
import logging
44
from contextlib import contextmanager
5+
from pathlib import Path
6+
from shutil import copyfile
57
from subprocess import Popen
6-
from typing import TYPE_CHECKING
8+
from tempfile import TemporaryDirectory
9+
from typing import TYPE_CHECKING, Final
710

811
from virtualenv.seed.embed.base_embed import BaseEmbed
9-
from virtualenv.seed.wheels import Version, get_wheel, pip_wheel_env_run
12+
from virtualenv.seed.wheels import get_wheel, pip_wheel_env_run
1013
from virtualenv.seed.wheels.acquire import PIP_RUN
14+
from virtualenv.seed.wheels.util import Version
1115
from virtualenv.util.subprocess import LogCmd
1216

1317
if TYPE_CHECKING:
1418
from collections.abc import Generator
15-
from pathlib import Path
1619

1720
from virtualenv.config.cli.parser import VirtualEnvOptions
1821
from virtualenv.create.creator import Creator
@@ -44,7 +47,7 @@ def _execute(cmd: list[str], env: dict[str, str]) -> Popen[bytes]:
4447

4548
@contextmanager
4649
def get_pip_install_cmd(self, exe: Path, for_py_version: str) -> Generator[list[str], None, None]:
47-
cmd = [
50+
cmd: Final = [
4851
str(exe),
4952
"-c",
5053
PIP_RUN,
@@ -54,29 +57,28 @@ def get_pip_install_cmd(self, exe: Path, for_py_version: str) -> Generator[list[
5457
":all:",
5558
"--disable-pip-version-check",
5659
"--ignore-installed",
60+
"--no-index",
5761
]
58-
if not self.download:
59-
cmd.append("--no-index")
60-
folders = set()
61-
for dist, version in self.distribution_to_versions().items():
62-
wheel = get_wheel(
63-
distribution=dist,
64-
version=version,
65-
for_py_version=for_py_version,
66-
search_dirs=self.extra_search_dir,
67-
download=False,
68-
app_data=self.app_data,
69-
do_periodic_update=self.periodic_update,
70-
env=self.env,
71-
)
72-
if wheel is None:
73-
msg = f"could not get wheel for distribution {dist}"
74-
raise RuntimeError(msg)
75-
folders.add(str(wheel.path.parent))
76-
cmd.append(Version.as_pip_req(dist, wheel.version))
77-
for folder in sorted(folders):
78-
cmd.extend(["--find-links", str(folder)])
79-
yield cmd
62+
# Only the selected archives may participate in pip's filename-based selection.
63+
with TemporaryDirectory(prefix="virtualenv-seed-") as folder:
64+
cmd.extend(["--find-links", folder])
65+
for dist, version in self.distribution_to_versions().items():
66+
wheel: Final = get_wheel(
67+
distribution=dist,
68+
version=version,
69+
for_py_version=for_py_version,
70+
search_dirs=self.extra_search_dir,
71+
download=self.download,
72+
app_data=self.app_data,
73+
do_periodic_update=self.periodic_update,
74+
env=self.env,
75+
)
76+
if wheel is None:
77+
msg: Final = f"could not get wheel for distribution {dist}"
78+
raise RuntimeError(msg)
79+
copyfile(wheel.path, Path(folder) / wheel.name)
80+
cmd.append(Version.as_pip_req(dist, wheel.version))
81+
yield cmd
8082

8183

8284
__all__ = [

‎src/virtualenv/seed/wheels/acquire.py‎

Lines changed: 56 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77
import sys
88
from pathlib import Path
99
from subprocess import PIPE, CalledProcessError, Popen
10+
from tempfile import TemporaryDirectory
1011
from typing import TYPE_CHECKING, Final
1112

1213
from packaging.specifiers import InvalidSpecifier, Specifier
14+
from packaging.utils import canonicalize_name
1315

1416
from .bundle import from_bundle
1517
from .periodic_update import UnverifiedWheelError, add_wheel_to_update_log, verify_wheel_digest
@@ -106,30 +108,18 @@ def download_wheel( # ruff:ignore[too-many-arguments]
106108
to_folder: Path,
107109
env: dict[str, str],
108110
) -> Wheel:
109-
"""Invoke ``pip download`` in a subprocess to fetch a seed wheel.
110-
111-
:param distribution: PEP 503 normalized project name; rejected if it contains anything other than
112-
``[A-Za-z0-9._-]``.
113-
:param version_spec: optional version specifier of the form ``==<ver>`` or ``<<ver>`` as emitted by
114-
:func:`Version.as_version_spec`, or ``None``/empty for the latest compatible release.
115-
:param for_py_version: major.minor Python version to pass through to ``pip --python-version``.
116-
:param search_dirs: additional directories to treat as a local wheel index when bootstrapping pip.
117-
:param app_data: application data store used to locate the embedded pip wheel.
118-
:param to_folder: directory the downloaded wheel is written into.
119-
:param env: environment mapping passed through to the subprocess.
120-
121-
:returns: the downloaded :class:`Wheel`.
122-
123-
:raises ValueError: if ``distribution`` or ``version_spec`` fail the strict allow-list check.
124-
:raises CalledProcessError: if ``pip download`` exits with a non-zero status.
125-
:raises RuntimeError: if no downloaded wheel can be identified from ``pip``'s output or the search directory, or if
126-
the caller has not configured a custom index and PyPI's sha256 for the downloaded filename cannot be obtained or
127-
does not match, see :func:`virtualenv.seed.wheels.periodic_update.verify_wheel_digest`.
111+
"""Download one wheel into a private staging directory and move it into ``to_folder`` once it checks out.
112+
113+
:raises ValueError: for an invalid distribution name or version specifier.
114+
:raises CalledProcessError: when ``pip download`` fails.
115+
:raises UnverifiedWheelError: when PyPI cannot supply the digest to verify against.
116+
:raises RuntimeError: when the staged file is not the requested wheel, its digest differs from PyPI's, or it does
117+
not support ``for_py_version``; nothing reaches ``to_folder`` in these cases.
128118
129119
"""
130120
_check_distribution(distribution)
131121
_check_version_spec(version_spec)
132-
to_download = f"{distribution}{version_spec or ''}"
122+
to_download: Final = f"{distribution}{version_spec or ''}"
133123
if to_download.lower().endswith((
134124
".zip",
135125
".whl",
@@ -147,52 +137,57 @@ def download_wheel( # ruff:ignore[too-many-arguments]
147137
msg = f"refusing to download wheel with archive-like requirement: {to_download!r}"
148138
raise ValueError(msg)
149139
LOGGER.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder)
150-
cmd = [
151-
sys.executable,
152-
"-c",
153-
PIP_RUN,
154-
"download",
155-
"--progress-bar",
156-
"off",
157-
"--disable-pip-version-check",
158-
"--only-binary=:all:",
159-
"--no-deps",
160-
"--python-version",
161-
for_py_version,
162-
"-d",
163-
str(to_folder),
164-
to_download,
165-
]
166-
verify_against_pypi = _uses_default_index(env)
167-
# pip has no interface in python - must be a new sub-process
140+
verify_against_pypi: Final = _uses_default_index(env)
168141
env = pip_wheel_env_run(search_dirs, app_data, env)
169-
process = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True, encoding="utf-8")
170-
out, err = process.communicate()
171-
if process.returncode != 0:
172-
kwargs = {"output": out, "stderr": err}
173-
raise CalledProcessError(process.returncode, cmd, **kwargs)
174-
result = _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out)
175-
if result is None:
176-
msg = f"could not find downloaded wheel for {to_download}"
142+
to_folder.mkdir(parents=True, exist_ok=True)
143+
with TemporaryDirectory(dir=to_folder, prefix="download-") as staging:
144+
cmd: Final = [
145+
sys.executable,
146+
"-c",
147+
PIP_RUN,
148+
"download",
149+
"--progress-bar",
150+
"off",
151+
"--disable-pip-version-check",
152+
"--only-binary=:all:",
153+
"--no-deps",
154+
"--python-version",
155+
for_py_version,
156+
"-d",
157+
staging,
158+
to_download,
159+
]
160+
process: Final = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True, encoding="utf-8")
161+
out, err = process.communicate()
162+
if process.returncode != 0:
163+
raise CalledProcessError(process.returncode, cmd, output=out, stderr=err)
164+
result: Final = _downloaded_wheel(Path(staging), distribution, version_spec)
165+
if verify_against_pypi:
166+
verify_wheel_digest(result)
167+
if not result.support_py(for_py_version):
168+
msg: Final = f"downloaded wheel {result.name} does not support Python {for_py_version}"
169+
raise RuntimeError(msg)
170+
destination: Final = to_folder / result.name
171+
result.path.replace(destination)
172+
return Wheel(destination)
173+
174+
175+
def _downloaded_wheel(folder: Path, distribution: str, version_spec: str | None) -> Wheel:
176+
files: Final = list(folder.iterdir())
177+
if len(files) != 1 or files[0].is_symlink() or not files[0].is_file():
178+
msg: Final = f"could not find downloaded wheel for {distribution}: expected one archive"
179+
raise RuntimeError(msg)
180+
result: Final = Wheel.from_path(files[0])
181+
if (
182+
result is None
183+
or canonicalize_name(result.distribution) != canonicalize_name(distribution)
184+
or (version_spec and not Specifier(version_spec).contains(result.version))
185+
):
186+
msg: Final = f"unexpected downloaded wheel {files[0].name} for {distribution}{version_spec or ''}"
177187
raise RuntimeError(msg)
178-
LOGGER.debug("downloaded wheel %s", result.name)
179-
if verify_against_pypi:
180-
verify_wheel_digest(result)
181188
return result
182189

183190

184-
def _find_downloaded_wheel(
185-
distribution: str, version_spec: str | None, for_py_version: str, to_folder: Path, out: str
186-
) -> Wheel | None:
187-
for line in out.splitlines():
188-
stripped_line = line.lstrip()
189-
for marker in ("Saved ", "File was already downloaded "):
190-
if stripped_line.startswith(marker):
191-
return Wheel(Path(stripped_line[len(marker) :]).absolute())
192-
# if for some reason the output does not match fallback to the latest version with that spec
193-
return find_compatible_in_house(distribution, version_spec, for_py_version, to_folder)
194-
195-
196191
def find_compatible_in_house(
197192
distribution: str, version_spec: str | None, for_py_version: str, in_folder: Path
198193
) -> Wheel | None:
Lines changed: 52 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,14 @@
11
from __future__ import annotations
22

3-
import itertools
43
import os
54
import subprocess
65
import sys
7-
from shutil import copy2
86
from typing import TYPE_CHECKING, Final
97

108
import pytest
119

1210
from virtualenv.run import cli_run
13-
from virtualenv.seed.embed.pip_invoke import PipInvoke
14-
from virtualenv.seed.wheels.bundle import load_embed_wheel
15-
from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT
11+
from virtualenv.seed.wheels.embed import get_embed_wheel
1612

1713
if TYPE_CHECKING:
1814
from pathlib import Path
@@ -46,66 +42,63 @@ def test_pip_invoke_uses_selected_wheel(
4642
assert os.path.normcase(str(session.creator.purelib)) in os.path.normcase(result.stdout)
4743

4844

49-
@pytest.mark.slow
50-
@pytest.mark.parametrize("no", ["pip", "setuptools", ""], ids=["no-pip", "no-setuptools", "all"])
51-
def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no) -> None:
52-
extra_search_dir = tmp_path / "extra"
53-
extra_search_dir.mkdir()
54-
for_py_version = f"{sys.version_info.major}.{sys.version_info.minor}"
55-
new = BUNDLE_SUPPORT[for_py_version]
56-
for wheel_filename in BUNDLE_SUPPORT[for_py_version].values():
57-
copy2(str(BUNDLE_FOLDER / wheel_filename), str(extra_search_dir))
58-
59-
def _load_embed_wheel(app_data, distribution, _for_py_version, version):
60-
return load_embed_wheel(app_data, distribution, old_ver, version)
61-
62-
old_ver = "3.9"
63-
old = BUNDLE_SUPPORT[old_ver]
64-
mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", side_effect=_load_embed_wheel)
65-
66-
def _execute(cmd, env):
67-
expected = set()
68-
for distribution, with_version in versions.items():
69-
if distribution == no:
70-
continue
71-
if with_version == "embed" or old[distribution] == new[distribution]:
72-
expected.add(BUNDLE_FOLDER)
73-
else:
74-
expected.add(extra_search_dir)
75-
expected_list = list(
76-
itertools.chain.from_iterable(["--find-links", str(e)] for e in sorted(expected, key=str)),
77-
)
78-
found = cmd[-len(expected_list) :] if expected_list else []
79-
assert "--no-index" not in cmd
80-
cmd.append("--no-index")
81-
assert found == expected_list
82-
return original(cmd, env)
83-
84-
original = PipInvoke._execute # ruff:ignore[private-member-access]
85-
run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute)
86-
versions = {"pip": "embed", "setuptools": "bundle"}
87-
88-
create_cmd = [
45+
@pytest.mark.parametrize(
46+
"excluded",
47+
[pytest.param("pip", id="no-pip"), pytest.param("setuptools", id="no-setuptools"), pytest.param("", id="all")],
48+
)
49+
def test_base_bootstrap_via_pip_invoke(tmp_path: Path, current_fastest: str, excluded: str) -> None:
50+
command: Final = [
8951
"--seeder",
9052
"pip",
9153
str(tmp_path / "env"),
9254
"--download",
9355
"--creator",
9456
current_fastest,
95-
"--extra-search-dir",
96-
str(extra_search_dir),
97-
"--app-data",
98-
str(tmp_path / "app-data"),
57+
"--pip",
58+
"embed",
59+
"--setuptools",
60+
"embed",
61+
"--no-periodic-update",
9962
]
100-
for dist, version in versions.items():
101-
create_cmd.extend([f"--{dist}", version])
102-
if no:
103-
create_cmd.append(f"--no-{no}")
104-
result = cli_run(create_cmd)
105-
coverage_env()
63+
if excluded:
64+
command.append(f"--no-{excluded}")
65+
session: Final = cli_run(command)
66+
assert {path.name for path in session.creator.purelib.iterdir()} & {"pip", "setuptools"} == {
67+
"pip",
68+
"setuptools",
69+
} - {excluded}
10670

107-
assert result
108-
assert run.call_count == 1
10971

110-
installed = {path.name for path in result.creator.purelib.iterdir()}
111-
assert installed & {"pip", "setuptools"} == {"pip", "setuptools"} - {no}
72+
def test_pip_invoke_reports_unavailable_wheel(tmp_path: Path) -> None:
73+
with pytest.raises(RuntimeError, match="could not get wheel for distribution pip"):
74+
cli_run([
75+
str(tmp_path / "env"),
76+
"--seeder",
77+
"pip",
78+
"--pip",
79+
"0.0.0",
80+
"--no-download",
81+
"--no-periodic-update",
82+
])
83+
84+
85+
@pytest.mark.usefixtures("_no_coverage")
86+
def test_pip_invoke_freeze_uses_version_pin(tmp_path: Path) -> None:
87+
session: Final = cli_run([
88+
str(tmp_path / "env"),
89+
"--seeder",
90+
"pip",
91+
"--pip",
92+
"embed",
93+
"--no-setuptools",
94+
"--no-periodic-update",
95+
])
96+
result: Final = subprocess.run(
97+
[str(session.creator.exe), "-I", "-m", "pip", "freeze", "--all"],
98+
capture_output=True,
99+
text=True,
100+
encoding="utf-8",
101+
check=True,
102+
)
103+
version: Final = get_embed_wheel("pip", f"{sys.version_info.major}.{sys.version_info.minor}").version
104+
assert f"pip=={version}" in result.stdout.splitlines()

0 commit comments

Comments
 (0)