|
| 1 | +"""The release workflows hold credentials, so their shape is tested. |
| 2 | +
|
| 3 | +A workflow is the one place in this repository that can publish to something other |
| 4 | +people read. Nothing here runs it -- that needs GitHub -- but the properties that keep |
| 5 | +it safe are properties of the file, and a file can be checked: the credential is minted |
| 6 | +by one job and no other, nothing installs an unverified binary, and every action is |
| 7 | +pinned to a commit rather than to a tag somebody can move. |
| 8 | +""" |
| 9 | + |
| 10 | +import re |
| 11 | +from pathlib import Path |
| 12 | +from typing import Any |
| 13 | + |
| 14 | +import pytest |
| 15 | +import yaml |
| 16 | + |
| 17 | +WORKFLOWS = Path(__file__).resolve().parents[1] / ".github" / "workflows" |
| 18 | + |
| 19 | + |
| 20 | +def _load(name: str) -> dict[Any, Any]: |
| 21 | + return yaml.safe_load((WORKFLOWS / name).read_text(encoding="utf-8")) |
| 22 | + |
| 23 | + |
| 24 | +def _triggers(workflow: dict[Any, Any]) -> dict[str, Any]: |
| 25 | + # YAML 1.1 reads a bare `on` as the boolean True, which is how PyYAML sees it. |
| 26 | + return workflow.get("on") or workflow[True] |
| 27 | + |
| 28 | + |
| 29 | +@pytest.mark.parametrize("path", sorted(WORKFLOWS.glob("*.yml")), ids=lambda p: p.name) |
| 30 | +def test_every_action_is_pinned_to_a_commit(path: Path): |
| 31 | + """A tag can be re-pointed at different code by whoever controls the repository.""" |
| 32 | + text = path.read_text(encoding="utf-8") |
| 33 | + for use in re.findall(r"^\s*-?\s*uses:\s*(\S+)", text, re.MULTILINE): |
| 34 | + if use.startswith("./"): |
| 35 | + continue |
| 36 | + assert re.search(r"@[0-9a-f]{40}$", use), f"{path.name}: {use} is not pinned to a SHA" |
| 37 | + |
| 38 | + |
| 39 | +class TestRegistryWorkflow: |
| 40 | + """Publishing to the MCP registry, by OIDC, from the repository's own identity.""" |
| 41 | + |
| 42 | + @pytest.fixture(scope="class") |
| 43 | + @classmethod |
| 44 | + def registry(cls) -> dict[Any, Any]: |
| 45 | + return _load("registry.yml") |
| 46 | + |
| 47 | + def test_it_can_be_run_by_hand_and_called_by_the_release(self, registry: dict[Any, Any]): |
| 48 | + """By hand because the first release needs listing without re-tagging it.""" |
| 49 | + triggers = _triggers(registry) |
| 50 | + assert set(triggers) == {"workflow_dispatch", "workflow_call"} |
| 51 | + assert "tag" in triggers["workflow_dispatch"]["inputs"] |
| 52 | + assert "tag" in triggers["workflow_call"]["inputs"] |
| 53 | + |
| 54 | + def test_nothing_is_granted_workflow_wide(self, registry: dict[Any, Any]): |
| 55 | + assert registry["permissions"] == {"contents": "read"} |
| 56 | + |
| 57 | + def test_the_credential_is_minted_by_one_job_only(self, registry: dict[Any, Any]): |
| 58 | + minting = [ |
| 59 | + name |
| 60 | + for name, job in registry["jobs"].items() |
| 61 | + if (job.get("permissions") or {}).get("id-token") == "write" |
| 62 | + ] |
| 63 | + assert minting == ["publish"] |
| 64 | + |
| 65 | + def test_the_tool_is_pinned_and_checksummed(self, registry: dict[Any, Any]): |
| 66 | + """The binary runs with the OIDC credential in reach, so it is not `latest`.""" |
| 67 | + (step,) = [ |
| 68 | + s |
| 69 | + for s in registry["jobs"]["publish"]["steps"] |
| 70 | + if "mcp-publisher" in s.get("name", "") and "Install" in s["name"] |
| 71 | + ] |
| 72 | + assert re.fullmatch(r"v\d+\.\d+\.\d+", step["env"]["VERSION"]) |
| 73 | + assert re.fullmatch(r"[0-9a-f]{64}", step["env"]["SHA256"]) |
| 74 | + assert "releases/download/${VERSION}/" in step["run"] |
| 75 | + assert "/releases/latest/" not in step["run"] |
| 76 | + assert "sha256sum --check" in step["run"] |
| 77 | + assert step["run"].index("sha256sum --check") < step["run"].index("tar ") |
| 78 | + |
| 79 | + def test_it_checks_before_it_publishes(self, registry: dict[Any, Any]): |
| 80 | + script = "\n".join(step.get("run", "") for step in registry["jobs"]["publish"]["steps"]) |
| 81 | + order = [ |
| 82 | + script.index(command) |
| 83 | + for command in ( |
| 84 | + "./mcp-publisher validate", |
| 85 | + "./mcp-publisher login github-oidc", |
| 86 | + "./mcp-publisher publish", |
| 87 | + ) |
| 88 | + ] |
| 89 | + assert order == sorted(order) |
| 90 | + |
| 91 | + def test_it_never_relies_on_a_dry_run_flag(self, registry: dict[Any, Any]): |
| 92 | + """`publish` has no such flag, so one would be ignored and the publish would happen.""" |
| 93 | + script = "\n".join(step.get("run", "") for step in registry["jobs"]["publish"]["steps"]) |
| 94 | + assert "--dry-run" not in script |
| 95 | + |
| 96 | + def test_it_waits_for_the_package_it_points_at(self, registry: dict[Any, Any]): |
| 97 | + """The registry checks the PyPI description, so the version has to be there.""" |
| 98 | + script = "\n".join(step.get("run", "") for step in registry["jobs"]["publish"]["steps"]) |
| 99 | + assert "pypi.org/pypi/scenet/" in script |
| 100 | + |
| 101 | + |
| 102 | +class TestReleaseWorkflowCallsIt: |
| 103 | + @pytest.fixture(scope="class") |
| 104 | + @classmethod |
| 105 | + def release(cls) -> dict[Any, Any]: |
| 106 | + return _load("release.yml") |
| 107 | + |
| 108 | + def test_it_is_opt_in(self, release: dict[Any, Any]): |
| 109 | + """OIDC login for an org namespace is unproven until it has worked once.""" |
| 110 | + assert "vars.MCP_REGISTRY_PUBLISH == 'true'" in release["jobs"]["registry"]["if"] |
| 111 | + |
| 112 | + def test_it_runs_after_everything_else_and_blocks_nothing(self, release: dict[Any, Any]): |
| 113 | + job = release["jobs"]["registry"] |
| 114 | + assert set(job["needs"]) == {"publish", "github-release"} |
| 115 | + for name, other in release["jobs"].items(): |
| 116 | + assert "registry" not in (other.get("needs") or []), name |
| 117 | + |
| 118 | + def test_it_calls_the_reusable_workflow_with_the_tag(self, release: dict[Any, Any]): |
| 119 | + job = release["jobs"]["registry"] |
| 120 | + assert job["uses"] == "./.github/workflows/registry.yml" |
| 121 | + assert job["with"]["tag"] == "${{ github.ref_name }}" |
| 122 | + |
| 123 | + def test_the_credential_is_granted_to_that_job_only(self, release: dict[Any, Any]): |
| 124 | + assert release["permissions"] == {"contents": "read"} |
| 125 | + job = release["jobs"]["registry"] |
| 126 | + assert job["permissions"]["id-token"] == "write" |
0 commit comments