Skip to content

Commit 6a7c475

Browse files
CrazyBaranclaude
andcommitted
test(ci): pin the extension version guard's failure behavior with tests
Copilot review round 5 on #4351: the guard script (.github/scripts/check_extension_version_bump.py) is the primary regression prevention in this series, but its failure behavior was only ever exercised manually against throwaway commits - a change to its diff or parsing logic could have silently disabled it while CI stayed green. The existing contract test only checks the working tree's manifest/catalog sync, never the base/head diff logic. Add tests/contract/test_extension_version_guard_script.py: each case builds a real throwaway git repository (one cataloged and one uncataloged extension at base) and runs the script against base/head SHAs exactly as the extension-version-guard.yml workflow does, asserting exit codes and diagnostics for: - valid bump (content + manifest + catalog) -> pass - unbumped content change -> fail ("did not increase") - plain downgrade and PEP 440 prerelease downgrade -> fail - manifest bump without catalog sync -> fail ("must move together") - uncataloged extension change -> exempt, pass - new cataloged extension with no base version -> pass - unparseable version -> fails closed Refs #4345 Assisted-by: Claude Code (model: claude-fable-5, autonomous) Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 16c5642 commit 6a7c475

1 file changed

Lines changed: 181 additions & 0 deletions

File tree

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
"""Tests for the extension version-bump CI guard script (#4345).
2+
3+
The guard (`.github/scripts/check_extension_version_bump.py`) is the
4+
primary regression prevention for bundled-extension version staleness, so
5+
its failure behavior must be pinned by tests: each scenario builds a real
6+
throwaway git repository and invokes the script against base/head SHAs,
7+
exactly as the `extension-version-guard.yml` workflow does. Without this,
8+
a change to the script's diff or parsing logic could silently disable the
9+
guard while CI stays green.
10+
"""
11+
12+
from __future__ import annotations
13+
14+
import json
15+
import subprocess
16+
import sys
17+
from pathlib import Path
18+
19+
import pytest
20+
21+
REPO_ROOT = Path(__file__).parents[2]
22+
SCRIPT = REPO_ROOT / ".github" / "scripts" / "check_extension_version_bump.py"
23+
24+
25+
def _git(repo: Path, *args: str) -> str:
26+
result = subprocess.run(
27+
["git", "-C", str(repo), *args],
28+
check=True,
29+
capture_output=True,
30+
text=True,
31+
)
32+
return result.stdout.strip()
33+
34+
35+
def _write_extension(repo: Path, ext_id: str, version: str, script_line: str) -> None:
36+
ext_dir = repo / "extensions" / ext_id
37+
ext_dir.mkdir(parents=True, exist_ok=True)
38+
(ext_dir / "extension.yml").write_text(
39+
'schema_version: "1.0"\n'
40+
"\n"
41+
"extension:\n"
42+
f" id: {ext_id}\n"
43+
f' version: "{version}"\n',
44+
encoding="utf-8",
45+
)
46+
(ext_dir / "script.sh").write_text(f"{script_line}\n", encoding="utf-8")
47+
48+
49+
def _write_catalog(repo: Path, versions: dict[str, str]) -> None:
50+
(repo / "extensions").mkdir(exist_ok=True)
51+
payload = {
52+
"schema_version": "1.0",
53+
"extensions": {
54+
ext_id: {"id": ext_id, "version": version, "bundled": True}
55+
for ext_id, version in versions.items()
56+
},
57+
}
58+
(repo / "extensions" / "catalog.json").write_text(
59+
json.dumps(payload, indent=2), encoding="utf-8"
60+
)
61+
62+
63+
def _commit_all(repo: Path, message: str) -> str:
64+
_git(repo, "add", "-A")
65+
_git(repo, "commit", "-q", "-m", message)
66+
return _git(repo, "rev-parse", "HEAD")
67+
68+
69+
@pytest.fixture
70+
def guard_repo(tmp_path: Path) -> tuple[Path, str]:
71+
"""A git repo with one cataloged and one uncataloged extension at base."""
72+
repo = tmp_path / "repo"
73+
repo.mkdir()
74+
_git(repo, "init", "-q")
75+
_git(repo, "config", "user.email", "[email protected]")
76+
_git(repo, "config", "user.name", "Guard Tests")
77+
_git(repo, "config", "commit.gpgsign", "false")
78+
79+
_write_extension(repo, "demo", "1.0.0", "echo base")
80+
_write_extension(repo, "scratch", "1.0.0", "echo base") # not in catalog
81+
_write_catalog(repo, {"demo": "1.0.0"})
82+
base_sha = _commit_all(repo, "base")
83+
return repo, base_sha
84+
85+
86+
def _run_guard(repo: Path, base: str, head: str = "HEAD") -> subprocess.CompletedProcess:
87+
return subprocess.run(
88+
[sys.executable, str(SCRIPT), base, head],
89+
cwd=repo,
90+
capture_output=True,
91+
text=True,
92+
)
93+
94+
95+
def test_valid_bump_passes(guard_repo):
96+
repo, base = guard_repo
97+
_write_extension(repo, "demo", "1.1.0", "echo changed")
98+
_write_catalog(repo, {"demo": "1.1.0"})
99+
_commit_all(repo, "content change with bump")
100+
101+
result = _run_guard(repo, base)
102+
assert result.returncode == 0, result.stdout + result.stderr
103+
assert "all invariants hold" in result.stdout
104+
105+
106+
def test_unbumped_content_change_fails(guard_repo):
107+
repo, base = guard_repo
108+
_write_extension(repo, "demo", "1.0.0", "echo changed")
109+
_commit_all(repo, "content change without bump")
110+
111+
result = _run_guard(repo, base)
112+
assert result.returncode == 1, result.stdout + result.stderr
113+
assert "did not increase" in result.stdout
114+
assert "extensions/demo/extension.yml" in result.stdout
115+
116+
117+
def test_downgrade_fails(guard_repo):
118+
repo, base = guard_repo
119+
_write_extension(repo, "demo", "0.9.0", "echo changed")
120+
_write_catalog(repo, {"demo": "0.9.0"})
121+
_commit_all(repo, "downgrade")
122+
123+
result = _run_guard(repo, base)
124+
assert result.returncode == 1, result.stdout + result.stderr
125+
assert "did not increase" in result.stdout
126+
127+
128+
def test_prerelease_downgrade_fails(guard_repo):
129+
"""PEP 440 semantics: 1.0.0rc1 is lower than 1.0.0, and it must not slip
130+
through as a plain string inequality."""
131+
repo, base = guard_repo
132+
_write_extension(repo, "demo", "1.0.0rc1", "echo changed")
133+
_write_catalog(repo, {"demo": "1.0.0rc1"})
134+
_commit_all(repo, "prerelease downgrade")
135+
136+
result = _run_guard(repo, base)
137+
assert result.returncode == 1, result.stdout + result.stderr
138+
assert "did not increase" in result.stdout
139+
140+
141+
def test_manifest_bump_without_catalog_sync_fails(guard_repo):
142+
repo, base = guard_repo
143+
_write_extension(repo, "demo", "1.1.0", "echo changed")
144+
_commit_all(repo, "bump without catalog sync")
145+
146+
result = _run_guard(repo, base)
147+
assert result.returncode == 1, result.stdout + result.stderr
148+
assert "must move together" in result.stdout
149+
assert "catalog.json" in result.stdout
150+
151+
152+
def test_uncataloged_extension_change_is_exempt(guard_repo):
153+
repo, base = guard_repo
154+
_write_extension(repo, "scratch", "1.0.0", "echo changed")
155+
_commit_all(repo, "uncataloged change without bump")
156+
157+
result = _run_guard(repo, base)
158+
assert result.returncode == 0, result.stdout + result.stderr
159+
assert "all invariants hold" in result.stdout
160+
161+
162+
def test_new_cataloged_extension_passes_without_base_version(guard_repo):
163+
repo, base = guard_repo
164+
_write_extension(repo, "fresh", "0.1.0", "echo new")
165+
_write_catalog(repo, {"demo": "1.0.0", "fresh": "0.1.0"})
166+
_commit_all(repo, "add new extension")
167+
168+
result = _run_guard(repo, base)
169+
assert result.returncode == 0, result.stdout + result.stderr
170+
assert "all invariants hold" in result.stdout
171+
172+
173+
def test_unparseable_version_fails_closed(guard_repo):
174+
repo, base = guard_repo
175+
_write_extension(repo, "demo", "not-a-version", "echo changed")
176+
_write_catalog(repo, {"demo": "not-a-version"})
177+
_commit_all(repo, "unparseable version")
178+
179+
result = _run_guard(repo, base)
180+
assert result.returncode == 1, result.stdout + result.stderr
181+
assert "could not compare versions" in result.stdout

0 commit comments

Comments
 (0)