Skip to content

Commit 70a0102

Browse files
authored
🐛 fix(seed): preserve shared cache when reseeding with pip (#3423)
1 parent 554d9f9 commit 70a0102

5 files changed

Lines changed: 266 additions & 18 deletions

File tree

‎docs/changelog/3423.bugfix.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Detach seed-package and launcher symlinks before running the pip seeder. Reseeding an environment created with
2+
``--symlink-app-data`` no longer rewrites its shared wheel image. Unrelated launcher symlinks remain in place.
3+
4+
Preserve case in console entry-point names and use the last value for duplicate entries when regenerating launchers.
5+
6+
Remove linked metadata from previous seed versions when reseeding with pip so package listings report the installed
7+
version.

‎src/virtualenv/seed/embed/pip_invoke.py‎

Lines changed: 77 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
from __future__ import annotations
22

33
import logging
4+
import os
45
from contextlib import contextmanager
56
from pathlib import Path
67
from shutil import copyfile
78
from subprocess import Popen
89
from tempfile import TemporaryDirectory
910
from typing import TYPE_CHECKING, Final
11+
from zipfile import ZipFile
1012

1113
from virtualenv.seed.embed.base_embed import BaseEmbed
14+
from virtualenv.seed.embed.via_app_data.pip_install.base import EntryPointConfig
1215
from virtualenv.seed.wheels import get_wheel, pip_wheel_env_run
1316
from virtualenv.seed.wheels.acquire import PIP_RUN
1417
from virtualenv.seed.wheels.util import Version
@@ -20,33 +23,22 @@
2023
from virtualenv.config.cli.parser import VirtualEnvOptions
2124
from virtualenv.create.creator import Creator
2225

23-
LOGGER = logging.getLogger(__name__)
24-
2526

2627
class PipInvoke(BaseEmbed):
2728
def __init__(self, options: VirtualEnvOptions) -> None:
2829
super().__init__(options)
30+
self._wheels: list[Path] = []
2931

3032
def run(self, creator: Creator) -> None:
3133
if not self.enabled:
3234
return
33-
for_py_version = creator.interpreter.version_release_str
34-
with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd:
35-
env = pip_wheel_env_run(self.extra_search_dir, self.app_data, self.env)
36-
self._execute(cmd, env)
37-
38-
@staticmethod
39-
def _execute(cmd: list[str], env: dict[str, str]) -> Popen[bytes]:
40-
LOGGER.debug("pip seed by running: %s", LogCmd(cmd, env))
41-
process = Popen(cmd, env=env)
42-
process.communicate()
43-
if process.returncode != 0:
44-
msg = f"failed seed with code {process.returncode}"
45-
raise RuntimeError(msg)
46-
return process
35+
with self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) as cmd:
36+
self._detach_links(creator)
37+
self._execute(cmd, pip_wheel_env_run(self.extra_search_dir, self.app_data, self.env))
4738

4839
@contextmanager
4940
def get_pip_install_cmd(self, exe: Path, for_py_version: str) -> Generator[list[str], None, None]:
41+
self._wheels.clear()
5042
cmd: Final = [
5143
str(exe),
5244
"-c",
@@ -76,10 +68,79 @@ def get_pip_install_cmd(self, exe: Path, for_py_version: str) -> Generator[list[
7668
if wheel is None:
7769
msg: Final = f"could not get wheel for distribution {dist}"
7870
raise RuntimeError(msg)
71+
self._wheels.append(wheel.path)
7972
copyfile(wheel.path, Path(folder) / wheel.name)
8073
cmd.append(Version.as_pip_req(dist, wheel.version))
8174
yield cmd
8275

76+
def _detach_links(self, creator: Creator) -> None:
77+
for wheel in self._wheels:
78+
with ZipFile(wheel) as archive:
79+
names: Final = archive.namelist()
80+
for root in {creator.purelib, creator.platlib}:
81+
for name in {entry.partition("/")[0] for entry in names}:
82+
if Path(name).name != name or name in {"", ".", ".."}:
83+
msg: Final = f"invalid seed wheel entry {name!r}"
84+
raise RuntimeError(msg)
85+
self._detach_tree(root / name)
86+
for name in names:
87+
if name.endswith(".dist-info/entry_points.txt"):
88+
self._detach_entry_points(creator, archive.read(name).decode("utf-8"))
89+
90+
self._detach_metadata(creator)
91+
92+
def _detach_metadata(self, creator: Creator) -> None:
93+
for root in {creator.purelib, creator.platlib}:
94+
for distribution in self.distribution_to_versions():
95+
for entry in root.glob(f"{distribution.replace('-', '_')}-*"):
96+
if entry.suffix in {".dist-info", ".virtualenv"} and entry.is_symlink():
97+
entry.unlink()
98+
99+
@staticmethod
100+
def _detach_tree(path: Path) -> None:
101+
if path.is_symlink():
102+
path.unlink()
103+
elif path.is_dir():
104+
for root, dirs, files in os.walk(path, followlinks=False):
105+
for name in [*dirs, *files]:
106+
if (entry := Path(root) / name).is_symlink():
107+
entry.unlink()
108+
109+
def _detach_entry_points(self, creator: Creator, content: str) -> None:
110+
entries: Final = EntryPointConfig(interpolation=None, strict=False)
111+
entries.read_string(content)
112+
for section in ("console_scripts", "gui_scripts"):
113+
if entries.has_section(section):
114+
for script in entries[section]:
115+
self._detach_script(creator, script)
116+
117+
@staticmethod
118+
def _detach_script(creator: Creator, name: str) -> None:
119+
if Path(name).name != name or name in {"", ".", ".."}:
120+
msg: Final = f"invalid seed wheel script {name!r}"
121+
raise RuntimeError(msg)
122+
scripts: Final = {name}
123+
if name == "pip" or (name.startswith("pip") and name[3:4].isdigit()):
124+
scripts.update({
125+
"pip",
126+
f"pip{creator.interpreter.version_info.major}",
127+
f"pip{creator.interpreter.version_info.major}.{creator.interpreter.version_info.minor}",
128+
})
129+
for script in scripts:
130+
for suffix in (".exe", "-script.py", "-script.pyw") if os.name == "nt" else ("",):
131+
if (path := creator.script_dir / f"{script}{suffix}").is_symlink():
132+
path.unlink()
133+
134+
@staticmethod
135+
def _execute(cmd: list[str], env: dict[str, str]) -> Popen[bytes]:
136+
logging.getLogger(__name__).debug("pip seed by running: %s", LogCmd(cmd, env))
137+
process: Final = Popen(cmd, env=env)
138+
process.communicate()
139+
if process.returncode != 0:
140+
msg: Final = f"failed seed with code {process.returncode}"
141+
raise RuntimeError(msg)
142+
return process
143+
83144

84145
__all__ = [
85146
"PipInvoke",

‎src/virtualenv/seed/embed/via_app_data/pip_install/base.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ def _console_scripts(self) -> dict[str, str] | None:
156156
self._console_entry_points = {}
157157
entry_points = self._dist_info / "entry_points.txt" # ty: ignore[unsupported-operator]
158158
if entry_points.exists():
159-
parser = ConfigParser()
159+
parser = EntryPointConfig(interpolation=None, strict=False)
160160
with entry_points.open(encoding="utf-8") as file_handler:
161161
parser.read_file(file_handler)
162162
if "console_scripts" in parser.sections():
@@ -233,6 +233,12 @@ def has_image(self) -> bool:
233233
return self._image_dir.exists() and any(self._image_dir.iterdir())
234234

235235

236+
class EntryPointConfig(ConfigParser):
237+
@staticmethod
238+
def optionxform(optionstr: str) -> str:
239+
return optionstr
240+
241+
236242
class ScriptMakerCustom(ScriptMaker):
237243
def __init__(self, target_dir: Path, version_info: tuple[int, ...], executable: Path, name: str) -> None:
238244
super().__init__(None, str(target_dir))
@@ -255,5 +261,6 @@ def _write_script(
255261

256262

257263
__all__ = [
264+
"EntryPointConfig",
258265
"PipInstall",
259266
]

‎tests/unit/seed/embed/test_bootstrap_link_via_app_data.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from virtualenv.info import fs_supports_symlink
1717
from virtualenv.run import cli_run
1818
from virtualenv.seed.embed.via_app_data.pip_install.base import _safe_extract_zip
19+
from virtualenv.seed.embed.via_app_data.pip_install.copy import CopyPipInstall
1920
from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT
2021
from virtualenv.util.path import safe_delete
2122

@@ -413,6 +414,26 @@ def test_safe_extract_zip_rejects_absolute_entry(tmp_path: Path, entry: str) ->
413414
_safe_extract_zip(zip_ref, target)
414415

415416

417+
def test_seed_entry_points_preserve_case_and_last_duplicate(tmp_path: Path) -> None:
418+
session = cli_run([str(tmp_path / "env"), "--no-seed"])
419+
image = tmp_path / "image"
420+
metadata = image / "example-1.dist-info"
421+
metadata.mkdir(parents=True)
422+
(metadata / "entry_points.txt").write_text(
423+
"[console_scripts]\nMyTool = old:main\nMyTool = example:main\n", encoding="utf-8"
424+
)
425+
CopyPipInstall(tmp_path / "example-1-py3-none-any.whl", session.creator, image).install(
426+
session.interpreter.version_info
427+
)
428+
script = session.creator.script("MyTool")
429+
if sys.platform == "win32":
430+
with zipfile.ZipFile(script) as archive:
431+
source = archive.read("__main__.py")
432+
else:
433+
source = script.read_bytes()
434+
assert b"from example import main" in source
435+
436+
416437
@pytest.mark.skipif(not fs_supports_symlink(), reason="requires symlinks")
417438
def test_symlink_seed_compiles_with_empty_local_module(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
418439
(tmp_path / "compileall.py").touch()

‎tests/unit/seed/embed/test_pip_invoke.py‎

Lines changed: 153 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,145 @@
33
import os
44
import subprocess
55
import sys
6+
from shutil import copytree
67
from typing import TYPE_CHECKING, Final
8+
from zipfile import ZipFile
79

810
import pytest
911

12+
from virtualenv.info import fs_supports_symlink
1013
from virtualenv.run import cli_run
11-
from virtualenv.seed.wheels.embed import get_embed_wheel
14+
from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT, get_embed_wheel
15+
from virtualenv.util.path import safe_delete
1216

1317
if TYPE_CHECKING:
1418
from pathlib import Path
1519

20+
from pytest_mock import MockerFixture
21+
22+
from virtualenv.run.session import Session
23+
24+
25+
@pytest.mark.skipif(not fs_supports_symlink(), reason="symlinks are unavailable")
26+
def test_pip_reseed_keeps_shared_image(linked_seed: Session) -> None:
27+
package: Final = linked_seed.creator.purelib / "pip"
28+
image: Final = package.resolve().parent
29+
before: Final = {
30+
str(path.relative_to(image)): (path.read_bytes(), path.stat().st_mode)
31+
for path in image.rglob("*")
32+
if path.is_file()
33+
}
34+
cli_run([str(linked_seed.creator.dest), "--seeder", "pip", "--pip", "embed", "--no-periodic-update"])
35+
assert {
36+
str(path.relative_to(image)): (path.read_bytes(), path.stat().st_mode)
37+
for path in image.rglob("*")
38+
if path.is_file()
39+
} == before
40+
41+
42+
@pytest.mark.skipif(not fs_supports_symlink(), reason="symlinks are unavailable")
43+
def test_pip_reseed_installs_local_package(linked_seed: Session) -> None:
44+
cli_run([str(linked_seed.creator.dest), "--seeder", "pip", "--pip", "embed", "--no-periodic-update"])
45+
result: Final = subprocess.run(
46+
[str(linked_seed.creator.exe), "-I", "-m", "pip", "--version"],
47+
capture_output=True,
48+
text=True,
49+
encoding="utf-8",
50+
check=True,
51+
)
52+
assert not (linked_seed.creator.purelib / "pip").is_symlink()
53+
assert os.path.normcase(str(linked_seed.creator.purelib)) in os.path.normcase(result.stdout)
54+
55+
56+
@pytest.mark.skipif(not fs_supports_symlink(), reason="symlinks are unavailable")
57+
def test_pip_reseed_keeps_unrelated_launcher(linked_seed: Session) -> None:
58+
unrelated: Final = linked_seed.creator.script_dir / "unrelated-tool"
59+
unrelated.symlink_to(linked_seed.creator.dest / "missing-tool")
60+
cli_run([str(linked_seed.creator.dest), "--seeder", "pip", "--pip", "embed", "--no-periodic-update"])
61+
assert (unrelated.is_symlink(), unrelated.exists()) == (True, False)
62+
63+
64+
@pytest.mark.skipif(not fs_supports_symlink(), reason="symlinks are unavailable")
65+
@pytest.mark.parametrize("name", [pytest.param("__main__.py", id="file"), pytest.param("_internal", id="directory")])
66+
def test_pip_reseed_replaces_nested_link(linked_seed: Session, name: str) -> None:
67+
package: Final = linked_seed.creator.purelib / "pip"
68+
source: Final = package.resolve()
69+
package.unlink()
70+
copytree(source, package)
71+
for path in package.rglob("*"):
72+
if path.is_file():
73+
path.chmod(0o600)
74+
entry: Final = package / name
75+
directory: Final = entry.is_dir()
76+
if directory:
77+
safe_delete(entry)
78+
else:
79+
entry.unlink()
80+
entry.symlink_to(source / name, target_is_directory=directory)
81+
cli_run([str(linked_seed.creator.dest), "--seeder", "pip", "--pip", "embed", "--no-periodic-update"])
82+
assert (entry.exists(), entry.is_symlink()) == (True, False)
83+
84+
85+
@pytest.mark.skipif(not fs_supports_symlink(), reason="symlinks are unavailable")
86+
def test_pip_reseed_replaces_launcher_link(linked_seed: Session) -> None:
87+
launcher: Final = linked_seed.creator.script_dir / ("pip.exe" if os.name == "nt" else "pip")
88+
launcher.unlink()
89+
missing: Final = linked_seed.creator.dest / "missing-launcher"
90+
launcher.symlink_to(missing)
91+
cli_run([str(linked_seed.creator.dest), "--seeder", "pip", "--pip", "embed", "--no-periodic-update"])
92+
assert (launcher.is_file(), launcher.is_symlink(), missing.exists()) == (True, False, False)
93+
94+
95+
@pytest.mark.parametrize(
96+
"invalid", [pytest.param("entry", id="package-name"), pytest.param("script", id="script-name")]
97+
)
98+
def test_pip_reseed_rejects_invalid_wheel_names(tmp_path: Path, invalid: str) -> None:
99+
with ZipFile(tmp_path / "pip-99.0-py3-none-any.whl", "w") as archive:
100+
archive.writestr("pip-99.0.dist-info/METADATA", "Name: pip\nVersion: 99.0\n")
101+
if invalid == "entry":
102+
archive.writestr("./", "")
103+
else:
104+
archive.writestr("pip-99.0.dist-info/entry_points.txt", "[console_scripts]\n. = pip:main\n")
105+
with pytest.raises(RuntimeError, match=f"invalid seed wheel {invalid}"):
106+
cli_run([
107+
str(tmp_path / "env"),
108+
"--seeder",
109+
"pip",
110+
"--pip",
111+
"99.0",
112+
"--extra-search-dir",
113+
str(tmp_path),
114+
"--no-download",
115+
"--no-periodic-update",
116+
])
117+
118+
119+
def test_pip_reseed_reports_process_failure(tmp_path: Path, mocker: MockerFixture) -> None:
120+
process: Final = mocker.patch("virtualenv.seed.embed.pip_invoke.Popen", autospec=True)
121+
process.return_value.configure_mock(returncode=7)
122+
with pytest.raises(RuntimeError, match="failed seed with code 7"):
123+
cli_run([str(tmp_path / "env"), "--seeder", "pip", "--pip", "embed", "--no-periodic-update"])
124+
125+
126+
def test_pip_reseed_reports_missing_wheel(tmp_path: Path) -> None:
127+
with pytest.raises(RuntimeError, match="could not get wheel for distribution pip"):
128+
cli_run([str(tmp_path / "env"), "--seeder", "pip", "--pip", "99.1234", "--no-download", "--no-periodic-update"])
129+
130+
131+
@pytest.fixture
132+
def linked_seed(tmp_path: Path, current_fastest: str) -> Session:
133+
return cli_run([
134+
str(tmp_path / "env"),
135+
"--creator",
136+
current_fastest,
137+
"--pip",
138+
"embed",
139+
"--no-periodic-update",
140+
"--symlink-app-data",
141+
"--app-data",
142+
str(tmp_path / "cache"),
143+
])
144+
16145

17146
@pytest.mark.parametrize("safe_path", [pytest.param("", id="default"), pytest.param("1", id="safe-path")])
18147
def test_pip_invoke_uses_selected_wheel(
@@ -102,3 +231,26 @@ def test_pip_invoke_freeze_uses_version_pin(tmp_path: Path) -> None:
102231
)
103232
version: Final = get_embed_wheel("pip", f"{sys.version_info.major}.{sys.version_info.minor}").version
104233
assert f"pip=={version}" in result.stdout.splitlines()
234+
235+
236+
@pytest.mark.skipif(not fs_supports_symlink(), reason="symlinks are unavailable")
237+
@pytest.mark.skipif(sys.version_info < (3, 10), reason="requires two compatible bundled pip versions")
238+
def test_pip_reseed_removes_previous_version_metadata(tmp_path: Path) -> None:
239+
old: Final = BUNDLE_SUPPORT["3.9"]["pip"].split("-")[1]
240+
session: Final = cli_run([
241+
str(tmp_path / "env"),
242+
"--pip",
243+
old,
244+
"--extra-search-dir",
245+
str(BUNDLE_FOLDER),
246+
"--symlink-app-data",
247+
"--no-periodic-update",
248+
"--app-data",
249+
str(tmp_path / "cache"),
250+
])
251+
cli_run([str(session.creator.dest), "--seeder", "pip", "--pip", "embed", "--no-periodic-update"])
252+
expected: Final = BUNDLE_SUPPORT[f"{sys.version_info.major}.{sys.version_info.minor}"]["pip"].split("-")[1]
253+
assert sorted(entry.name for entry in session.creator.purelib.glob("pip-*.dist-info")) == [
254+
f"pip-{expected}.dist-info",
255+
]
256+
assert not list(session.creator.purelib.glob(f"pip-{old}.virtualenv"))

0 commit comments

Comments
 (0)