Skip to content

Commit b19d105

Browse files
Environment: skip other Pythons' requirements files, install a project's extras, repair a conftest that cannot load
Two live runs ended "unsupported" because the sandbox could not even start pytest: - pallets/jinja#2244: the baseline at 3.1.6 installed requirements/tests.txt and then requirements/tests37.txt (the pin set of Python 3.7: trio 0.22.2, pytest 7.4.4). On Python 3.13 that trio cannot be imported, and the conftest of main imports trio at the top, so every run on the fix side died with "TypeError: ('parser', <class 'module'>)". testdeps now skips a requirements file whose name carries another interpreter's version. - theskumar/python-dotenv#697: tests/conftest.py imports click, which only the cli extra declares, and the issue says "With python-dotenv[cli] installed". The project's own name is never installed from the index, so nothing installed the extra. build_baseline and run_test_on_side take project_extras (read from the thread by project_extras_from_text), and the helper also installs the extra that a console script of the project asks for when it cannot start ("Run pip install "python-dotenv[cli]""). A new `repair` sub-command of the sandbox helper collects the test file and, when pytest dies loading the conftest or a plugin, installs the requirement that the project itself declares for the missing module, or upgrades a package that crashes on import (a stale pin). It runs at the end of the baseline and, with the extras, after the fix side's checkout. Offline tests for each, and Docker tests that run the helper against real pip.
1 parent d0d8726 commit b19d105

5 files changed

Lines changed: 1312 additions & 14 deletions

File tree

‎src/fixedyet/envbuild.py‎

Lines changed: 159 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
_PATCH_STAGING = "/tmp/ir_patch.diff" # noqa: S108
4141
_PREFIX_LINES = "awk '{ print \"| \" $0 }'"
4242
"""Prefixes every line, and always ends the last one with a newline (sed would not)."""
43+
_HELPER_LOG_ONLY = "grep '^| '"
44+
"""The sandbox helper prints machine markers and ``| `` log lines; a side run keeps the latter."""
4345

4446
_REPO_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$")
4547
_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_./+@-]*$")
@@ -52,6 +54,14 @@
5254
_RUN_START_RE = re.compile(r"^__RUN_START__=\d+$", re.MULTILINE)
5355
_EXTRA_OK_RE = re.compile(r"^__EXTRA_OK__=(\S+)$", re.MULTILINE)
5456
_EXTRA_FAILED_RE = re.compile(r"^__EXTRA_FAILED__=(\S+)$", re.MULTILINE)
57+
_PROJECT_EXTRA_RE = {
58+
name: re.compile(rf"^__PROJECT_EXTRA_{name}__=(\S+)$", re.MULTILINE)
59+
for name in ("OK", "FAILED", "UNKNOWN", "HINT")
60+
}
61+
_REPAIR_RE = {
62+
name: re.compile(rf"^__REPAIR_{name}__=(.+)$", re.MULTILINE)
63+
for name in ("OK", "FAILED", "UNRESOLVED")
64+
}
5565
_TESTDEPS_RE = {
5666
name: re.compile(rf"^__TESTDEPS_{name}__=(.+)$", re.MULTILINE)
5767
for name in ("OK", "PARTIAL", "FAILED", "MISSING", "IGNORED", "CRASHED")
@@ -63,6 +73,8 @@
6373
NO_SUMMARY_EXIT_CODE = 70 # recorded instead of pytest's exit 1 when no summary line was printed
6474

6575
TESTDEPS_PATH = "/tmp/ir_testdeps.py" # noqa: S108 - inside the sandbox, not on the host
76+
PROBE_TEST_PATH = "tests/test_fixedyet_probe.py"
77+
"""The stub the baseline's repair step collects: it lives in the directory the real test will."""
6678

6779

6880
class EnvBuildError(RuntimeError):
@@ -152,6 +164,44 @@ def extra_packages_from_steps(
152164
return found[:MAX_EXTRA_PACKAGES]
153165

154166

167+
# `python-dotenv[cli]`, `"jinja2[i18n]"`, `requests[socks,security]`: the project's own name
168+
# followed by extras. A bracket without a letter in it (`dotenv[0]`) is an index, not an extra.
169+
_PROJECT_EXTRAS_RE = re.compile(
170+
r"(?<![A-Za-z0-9._-])(?P<name>[A-Za-z0-9][A-Za-z0-9._-]*)"
171+
r"\[(?P<extras>(?=[^\]]*[A-Za-z])[A-Za-z0-9._-]+(?:\s*,\s*[A-Za-z0-9._-]+)*)\]"
172+
)
173+
_EXTRA_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
174+
MAX_PROJECT_EXTRAS = 5
175+
176+
177+
def project_extras_from_text(texts: Iterable[str], names: Iterable[str]) -> list[str]:
178+
"""The optional extras of the project itself that ``texts`` ask for, in order of appearance.
179+
180+
``pip install "python-dotenv[cli]"`` and "with `python-dotenv[cli]` installed" both name the
181+
extra ``cli`` of the project. ``names`` are the project's names (repository, distribution);
182+
an extra of any other package is ignored here (`extra_packages_from_steps` handles those).
183+
The sandbox installs only the extras the project declares, so a wrong guess costs nothing.
184+
"""
185+
wanted = {_dist_name(name) for name in names if name}
186+
found: list[str] = []
187+
for text in texts:
188+
for match in _PROJECT_EXTRAS_RE.finditer(text):
189+
if _dist_name(match["name"]) not in wanted:
190+
continue
191+
for extra in re.split(r"\s*,\s*", match["extras"]):
192+
if extra not in found:
193+
found.append(extra)
194+
return found[:MAX_PROJECT_EXTRAS]
195+
196+
197+
def _extras_arg(extras: Sequence[str]) -> str:
198+
"""The comma list the sandbox helper takes, quoted; the empty string for no extras."""
199+
for extra in extras:
200+
if not _EXTRA_NAME_RE.match(extra):
201+
raise EnvBuildError(f"invalid extra {extra!r}")
202+
return shlex.quote(",".join(extras)) if extras else ""
203+
204+
155205
def _extras_lines(extras: Sequence[str]) -> list[str]:
156206
"""Install each extra on its own, so one bad name cannot hide the others. Never fatal."""
157207
lines: list[str] = []
@@ -172,6 +222,50 @@ def installed_extras(result: ExecResult) -> tuple[list[str], list[str]]:
172222
return _EXTRA_OK_RE.findall(result.stdout), _EXTRA_FAILED_RE.findall(result.stdout)
173223

174224

225+
def installed_project_extras(result: ExecResult) -> tuple[list[str], list[str]]:
226+
"""``(installed, failed)`` extras of the project itself, as the baseline script reports them."""
227+
return (
228+
_PROJECT_EXTRA_RE["OK"].findall(result.stdout),
229+
_PROJECT_EXTRA_RE["FAILED"].findall(result.stdout),
230+
)
231+
232+
233+
def hinted_project_extras(result: ExecResult) -> list[str]:
234+
"""Extras that no one asked for but the project's own console script asked for."""
235+
return _PROJECT_EXTRA_RE["HINT"].findall(result.stdout)
236+
237+
238+
@dataclass
239+
class RepairReport:
240+
"""What the repair step did so that pytest can load the project's conftest (`testdeps.py`)."""
241+
242+
installed: list[str] = field(default_factory=list)
243+
failed: list[str] = field(default_factory=list)
244+
unresolved: list[str] = field(default_factory=list)
245+
246+
def __bool__(self) -> bool:
247+
return bool(self.installed or self.failed or self.unresolved)
248+
249+
def describe(self) -> str:
250+
parts = []
251+
if self.installed:
252+
parts.append(f"installed {', '.join(self.installed)}")
253+
if self.failed:
254+
parts.append(f"could not install {', '.join(self.failed)}")
255+
if self.unresolved:
256+
parts.append(f"pytest still cannot collect a test here ({self.unresolved[-1]})")
257+
return "; ".join(parts)
258+
259+
260+
def repairs_made(result: ExecResult) -> RepairReport:
261+
"""The `RepairReport` read from the baseline script's markers (empty when none apply)."""
262+
return RepairReport(
263+
installed=[v.strip() for v in _REPAIR_RE["OK"].findall(result.stdout)],
264+
failed=[v.strip() for v in _REPAIR_RE["FAILED"].findall(result.stdout)],
265+
unresolved=[v.strip() for v in _REPAIR_RE["UNRESOLVED"].findall(result.stdout)],
266+
)
267+
268+
175269
@dataclass
176270
class DepsReport:
177271
"""What the test-dependency step of the baseline script says it did (`testdeps.py`)."""
@@ -225,19 +319,27 @@ def helper_source() -> str:
225319
return Path(testdeps.__file__).read_text(encoding="utf-8")
226320

227321

228-
def baseline_script(repo: str, ref: str, extras: Sequence[str] = ()) -> str:
322+
def baseline_script(
323+
repo: str,
324+
ref: str,
325+
extras: Sequence[str] = (),
326+
project_extras: Sequence[str] = (),
327+
) -> str:
229328
"""Shell script that clones ``repo``, checks out ``ref`` and installs it with pytest.
230329
231330
After the editable install it runs `fixedyet.testdeps` (shipped to `TESTDEPS_PATH` by
232-
`build_baseline`): the project's PEP 735 test dependency groups and test requirements files.
233-
``extras`` are further requirements (see `extra_packages_from_steps`) installed last; a
234-
failure to install one of them, or a test dependency, is reported on stdout and does not fail
235-
the build.
331+
`build_baseline`): the project's PEP 735 test dependency groups and test requirements files,
332+
then ``project_extras`` (optional extras of the project itself that the issue names; the
333+
helper adds the ones the project's console script asks for), then ``extras`` (further
334+
requirements, see `extra_packages_from_steps`), and last a repair step that makes sure pytest
335+
can load the project's conftest. A failure of any of these is reported on stdout and does not
336+
fail the build.
236337
"""
237338
url = shlex.quote(f"https://github.com/{_check_repo(repo)}")
238339
quoted_ref = shlex.quote(_check_ref(ref))
239340
repo_dir = shlex.quote(REPO_DIR)
240341
extra_lines = _extras_lines(extras)
342+
project_extras_arg = _extras_arg(project_extras)
241343
if extra_lines:
242344
extra_lines = ["echo __STEP__=extras", *extra_lines]
243345
return "\n".join(
@@ -256,7 +358,13 @@ def baseline_script(repo: str, ref: str, extras: Sequence[str] = ()) -> str:
256358
"pip install pytest || exit 13",
257359
"echo __STEP__=testdeps",
258360
f"python {TESTDEPS_PATH} {repo_dir} || echo __TESTDEPS_CRASHED__=exit",
361+
"echo __STEP__=project_extras",
362+
f"python {TESTDEPS_PATH} extras {repo_dir} {project_extras_arg} "
363+
"|| echo __PROJECT_EXTRA_CRASHED__=exit",
259364
*extra_lines,
365+
"echo __STEP__=repair",
366+
f"python {TESTDEPS_PATH} repair {repo_dir} {PROBE_TEST_PATH} "
367+
"|| echo __REPAIR_CRASHED__=exit",
260368
"echo __STEP__=sanity",
261369
'python -c "import sys; print(sys.version)" || exit 14',
262370
"pip freeze | head -n 30",
@@ -285,17 +393,21 @@ async def build_baseline(
285393
progress: ProgressCallback | None = None,
286394
timeout_s: int = BUILD_TIMEOUT_S,
287395
extra_packages: Sequence[str] = (),
396+
project_extras: Sequence[str] = (),
288397
) -> tuple[str, ExecResult]:
289398
"""Build and checkpoint the environment for ``repo`` at ``ref``.
290399
291400
Clones github.com/<repo> into `REPO_DIR`, checks out ``ref``, installs it editable with
292401
test extras when available, installs pytest, the project's test dependency groups and
293-
requirements files (`installed_test_deps` reads back what that did) and ``extra_packages``
294-
(`installed_extras` reads back which of them worked), runs a sanity step, and saves the
295-
filesystem.
402+
requirements files (`installed_test_deps` reads back what that did), ``project_extras`` (the
403+
project's own optional extras, plus those its console script asks for;
404+
`installed_project_extras` reads back which worked) and ``extra_packages``
405+
(`installed_extras` reads back which of them worked), repairs what stops pytest from loading
406+
the project's conftest (`repairs_made`), runs a sanity step, and saves the filesystem.
296407
Returns ``(checkpoint_image, setup_result)``. Raises `EnvBuildError` on any failure.
297408
"""
298-
script = baseline_script(repo, ref, extra_packages) # validates the inputs before any call
409+
# validates the inputs before any call
410+
script = baseline_script(repo, ref, extra_packages, project_extras)
299411
started = time.monotonic()
300412
py = python_version or "default"
301413
await _emit(progress, f"Building baseline for {repo} at {ref} (Python {py})")
@@ -335,6 +447,16 @@ async def build_baseline(
335447
note = f"installed {', '.join(installed) or 'none'}"
336448
note += f"; could not install {', '.join(failed)}" if failed else ""
337449
await _emit(progress, f"Extra packages named in the issue: {note}")
450+
extras_ok, extras_failed = installed_project_extras(result)
451+
if extras_ok or extras_failed:
452+
note = f"installed {', '.join(extras_ok) or 'none'}"
453+
note += f"; could not install {', '.join(extras_failed)}" if extras_failed else ""
454+
hinted = hinted_project_extras(result)
455+
note += f" (the project's own script asked for {', '.join(hinted)})" if hinted else ""
456+
await _emit(progress, f"Optional extras of {repo}: {note}")
457+
repaired = repairs_made(result)
458+
if repaired:
459+
await _emit(progress, f"Test environment repair: {repaired.describe()}")
338460
await _emit(
339461
progress,
340462
f"Baseline ready in {time.monotonic() - started:.0f}s, checkpoint {_short(result.image)}",
@@ -422,13 +544,22 @@ def side_script(
422544
patch: bool,
423545
runs: int,
424546
per_run_timeout_s: int,
547+
project_extras: Sequence[str] = (),
425548
) -> str:
426549
"""Shell script: optional checkout and patch, reinstall, write the test, run pytest N times.
427550
551+
Another ref has its own test requirements and its own conftest, so after a checkout the
552+
sandbox helper (`TESTDEPS_PATH`, shipped by `run_test_on_side`) installs ``project_extras``
553+
again and, with the test file in place, repairs whatever stops pytest from loading the
554+
conftest of that ref (a module the baseline never needed, a package pinned at a version this
555+
interpreter cannot import). Only the helper's ``| `` log lines are echoed, never its markers.
556+
428557
Each pytest run writes to its own log; only the last bytes of it are echoed, each line
429558
prefixed with ``| `` so test output can never forge a ``__RC__=`` marker line.
430559
"""
431560
path = shlex.quote(_check_test_path(test_path))
561+
repo_dir = shlex.quote(REPO_DIR)
562+
extras_arg = _extras_arg(project_extras)
432563
lines = [
433564
"set -u",
434565
"export GIT_TERMINAL_PROMPT=0 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_INPUT=1",
@@ -451,9 +582,19 @@ def side_script(
451582
"|| { echo __SETUP_FAILED__=reinstall; "
452583
f"tail -n 40 /tmp/ir_reinstall.log | {_PREFIX_LINES}; exit 22; }}"
453584
)
454-
lines += [
585+
if checkout:
586+
lines.append(
587+
f"python {TESTDEPS_PATH} extras {repo_dir} {extras_arg} 2>&1 | {_HELPER_LOG_ONLY}"
588+
)
589+
lines.append(
455590
f'mkdir -p "$(dirname {path})" && cp {_TEST_FILE_STAGING} {path} '
456-
"|| { echo __SETUP_FAILED__=write_test; exit 24; }",
591+
"|| { echo __SETUP_FAILED__=write_test; exit 24; }"
592+
)
593+
if checkout:
594+
lines.append(
595+
f"python {TESTDEPS_PATH} repair {repo_dir} {path} 2>&1 | {_HELPER_LOG_ONLY}"
596+
)
597+
lines += [
457598
f"for i in $(seq 1 {runs}); do",
458599
' echo "__RUN_START__=$i"',
459600
f" timeout -k 5 {per_run_timeout_s} python -m pytest -x -q -p no:cacheprovider {path}"
@@ -575,6 +716,7 @@ async def run_test_on_side(
575716
ref: str | None = None,
576717
timeout_s: int = TEST_TIMEOUT_S,
577718
network: bool = True,
719+
project_extras: Sequence[str] = (),
578720
) -> SideRun:
579721
"""Run the reproduction test ``runs`` times on one side, in ONE sandbox run from ``image``.
580722
@@ -583,6 +725,9 @@ async def run_test_on_side(
583725
`SideRun.ref` (default: the checkout, ``gold_patch``, or ``HEAD``). Exit codes map 0 to PASS,
584726
1 to FAIL, anything else to ERROR. A setup failure or a sandbox timeout yields ERROR outcomes
585727
rather than an exception; only a broken sandbox (`SandboxError`) propagates.
728+
729+
After a ``checkout`` the sandbox helper also installs ``project_extras`` again and repairs a
730+
conftest that cannot load (see `side_script`), which is why it travels with the test file.
586731
"""
587732
if runs < 1:
588733
raise EnvBuildError("runs must be at least 1")
@@ -594,8 +739,11 @@ async def run_test_on_side(
594739
patch=patch is not None,
595740
runs=runs,
596741
per_run_timeout_s=per_run,
742+
project_extras=project_extras,
597743
)
598744
files = {_TEST_FILE_STAGING: test_code}
745+
if checkout:
746+
files[TESTDEPS_PATH] = helper_source()
599747
if patch is not None:
600748
files[_PATCH_STAGING] = patch if patch.endswith("\n") else patch + "\n"
601749
result = await sb.run(

0 commit comments

Comments
 (0)