Skip to content

Commit 95935ce

Browse files
noreplyclaude
andcommitted
Fork CI: run every fork script test, checked against a list
The workflows job runs each .github/scripts/fork-*.test.{sh,py,cjs} and checks them against .github/fork-ci-tests.txt. A present test that is not listed fails. A listed test that is missing fails once it has been in the tested commit's history; before that it is a note in the job summary, so a test whose pull request has not merged yet does not fail the run. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 9510d2f commit 95935ce

4 files changed

Lines changed: 148 additions & 2 deletions

File tree

‎.github/fork-ci-tests.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Fork script tests the workflow job runs (.github/scripts/fork-ci-tests.cjs).
2+
# Every fork-*.test.{sh,py,cjs} must be listed here. A listed file that is not
3+
# present yet is a note; one that was present and then deleted fails.
4+
.github/scripts/fork-ci-tests.test.cjs
5+
.github/scripts/fork-follow.test.sh
6+
.github/scripts/fork-release-workflow.test.py
7+
.github/scripts/fork-release.test.sh
8+
.github/scripts/fork-schema-check.test.cjs
9+
.github/scripts/fork-test-shards.test.cjs
10+
.github/scripts/fork-tests-drift.test.cjs

‎.github/scripts/fork-ci-tests.cjs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// Fork only: runs every `.github/scripts/fork-*.test.{sh,py,cjs}` and checks
2+
// them against `.github/fork-ci-tests.txt`. A present test missing from the list
3+
// fails; a listed test that is missing fails once it has been in the tested
4+
// commit's history (deleted while still listed) and is a note before that.
5+
const childProcess = require("node:child_process");
6+
const fs = require("node:fs");
7+
const path = require("node:path");
8+
9+
const ROOT = path.resolve(__dirname, "../..");
10+
const LIST = ".github/fork-ci-tests.txt";
11+
const TEST_FILE = /^fork-.+\.test\.(sh|py|cjs)$/;
12+
const RUNNER = { sh: ["bash"], py: ["python3"], cjs: ["node", "--test"] };
13+
14+
// `ever(file)` says whether the file is in the tested commit's history.
15+
function plan(listed, present, ever) {
16+
const unlisted = present.filter((file) => !listed.includes(file));
17+
const missing = listed.filter((file) => !present.includes(file));
18+
const deleted = missing.filter((file) => ever(file));
19+
const pending = missing.filter((file) => !deleted.includes(file));
20+
return { run: present, unlisted, deleted, pending };
21+
}
22+
23+
function readList(text) {
24+
return text
25+
.split("\n")
26+
.map((line) => line.replace(/#.*/, "").trim())
27+
.filter(Boolean);
28+
}
29+
30+
function git(args) {
31+
return childProcess.execFileSync("git", args, { cwd: ROOT, encoding: "utf8" }).trim();
32+
}
33+
34+
// A shallow checkout first fetches the full history of HEAD, trees only.
35+
function everInHistory(file) {
36+
if (git(["rev-parse", "--is-shallow-repository"]) === "true")
37+
git([
38+
"fetch",
39+
"--quiet",
40+
"--no-tags",
41+
"--filter=blob:none",
42+
"--unshallow",
43+
"origin",
44+
git(["rev-parse", "HEAD"]),
45+
]);
46+
return git(["log", "-1", "--format=%h", "HEAD", "--", file]) !== "";
47+
}
48+
49+
function main() {
50+
const listed = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8"));
51+
const present = fs
52+
.readdirSync(path.join(ROOT, ".github/scripts"))
53+
.filter((name) => TEST_FILE.test(name))
54+
.map((name) => `.github/scripts/${name}`)
55+
.toSorted();
56+
const result = plan(listed, present, everInHistory);
57+
const rows = [];
58+
let failed = result.unlisted.length + result.deleted.length;
59+
for (const file of result.run) {
60+
const [command, ...args] = RUNNER[path.basename(file).match(TEST_FILE)[1]];
61+
const started = Date.now();
62+
console.log(`::group::${file}`);
63+
const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" });
64+
console.log("::endgroup::");
65+
const seconds = ((Date.now() - started) / 1000).toFixed(1);
66+
if (run.status !== 0) {
67+
failed += 1;
68+
console.log(`::error file=${file}::${file} failed (exit ${run.status ?? run.signal})`);
69+
}
70+
rows.push(`| \`${file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${seconds} s |`);
71+
}
72+
for (const file of result.unlisted)
73+
console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`);
74+
for (const file of result.deleted)
75+
console.log(`::error::${file} is listed in ${LIST} but was deleted; restore it or unlist it`);
76+
for (const file of result.pending)
77+
rows.push(`| \`${file}\` | not present: its PR has not merged | |`);
78+
for (const file of result.unlisted) rows.push(`| \`${file}\` | **not in ${LIST}** | |`);
79+
for (const file of result.deleted) rows.push(`| \`${file}\` | **deleted while listed** | |`);
80+
81+
const summary = process.env.GITHUB_STEP_SUMMARY;
82+
if (summary)
83+
fs.appendFileSync(
84+
summary,
85+
[
86+
"### Fork script tests",
87+
"",
88+
"| Test | Result | Time |",
89+
"| --- | --- | --- |",
90+
...rows,
91+
"",
92+
].join("\n") + "\n",
93+
);
94+
if (failed) process.exitCode = 1;
95+
}
96+
97+
module.exports = { plan, readList };
98+
99+
if (require.main === module) main();
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
const assert = require("node:assert/strict");
2+
const test = require("node:test");
3+
4+
const { plan, readList } = require("./fork-ci-tests.cjs");
5+
6+
const never = () => false;
7+
8+
test("runs every present test, listed or not", () => {
9+
assert.deepEqual(plan(["a"], ["a", "b"], never).run, ["a", "b"]);
10+
});
11+
12+
test("a present test missing from the list fails", () => {
13+
assert.deepEqual(plan(["a"], ["a", "b"], never).unlisted, ["b"]);
14+
});
15+
16+
test("a listed test never in history is pending, not a failure", () => {
17+
const result = plan(["a", "later"], ["a"], never);
18+
assert.deepEqual(result.pending, ["later"]);
19+
assert.deepEqual(result.deleted, []);
20+
});
21+
22+
test("a listed test deleted from history fails", () => {
23+
const result = plan(["a", "gone"], ["a"], (file) => file === "gone");
24+
assert.deepEqual(result.deleted, ["gone"]);
25+
assert.deepEqual(result.pending, []);
26+
});
27+
28+
test("the list ignores comments and blank lines", () => {
29+
assert.deepEqual(readList("# head\n\na.sh # from #2\n b.py\n"), ["a.sh", "b.py"]);
30+
});

‎.github/workflows/fork-tests.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,15 @@ jobs:
143143
tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" actionlint
144144
"$RUNNER_TEMP/actionlint" -color .github/workflows/fork-*.yml
145145
146-
- name: Test the fork CI scripts
147-
run: node --test .github/scripts/fork-tests-drift.test.cjs .github/scripts/fork-test-shards.test.cjs
146+
# Every fork script test, checked against .github/fork-ci-tests.txt: an
147+
# unlisted test or a listed one deleted from history fails, a listed one
148+
# whose pull request has not merged yet is a note in the summary.
149+
- name: Fork script tests
150+
run: |
151+
if compgen -G '.github/scripts/fork-*.test.py' >/dev/null && ! python3 -c 'import yaml' 2>/dev/null; then
152+
sudo apt-get update -qq && sudo apt-get install -y -qq python3-yaml
153+
fi
154+
node .github/scripts/fork-ci-tests.cjs
148155
149156
- name: Compare upstream's test jobs with the recorded copy
150157
run: node .github/scripts/fork-tests-drift.cjs

0 commit comments

Comments
 (0)