Skip to content

Commit 6d6e0b5

Browse files
noreplyclaude
andcommitted
fork-tests: fix round 1 (secret scan range, every test run, one commit per run, drift)
Secret scan (.github/scripts/fork-secret-scan.sh, tests: fork-secret-scan.test.sh) - Pull requests scan base.sha..head.sha on a full blob-less clone instead of the newest N commits by date. After a merge of main, the PR's oldest commits fell outside --max-count and were never scanned; a squash merge never brings them to main, yet they stay public under refs/pull/N/head. - Merge commits are diffed with --remerge-diff. Plain `git log -p` shows merges with no diff, so a key added only in a merge commit was never read. - Push: before..after; a force push scans what the new tip has and the old one had not. A new branch, or an old tip that cannot be fetched, scans only the pushed commit and says so in a warning. - The Secret scan job runs the test against the pinned gitleaks before scanning. Every test file under .github/scripts runs somewhere (fork-ci-tests.cjs) - Finds *.test.* in subdirectories too. Runs .sh, .py, .cjs, .mjs and .js; a listed test with another extension fails instead of being skipped. - Upstream's tests are listed as "<test> run by <file>"; the file must name the test and cannot be a disabled upstream workflow. thread-transfer-report.test.cjs was run nowhere (upstream runs it from a workflow_run of its disabled CI); Test 1 runs it now with upstream's other release-check tests. One commit per release run - `changes` resolves a workflow_dispatch/workflow_call ref to a SHA once; every job checks that out, so a branch that moves mid-run cannot split a run over two commits. workflows, lint and the server shards now wait for `changes`. Upstream drift (fork-tests-drift.cjs) - Also digests the top-level env, defaults, concurrency and permissions. - --record refuses (exit 1) while a tracked job is missing instead of saving null, and an old null entry is reported instead of crashing. - --record prints the upstream changes it accepts; the commit and PR must say what was ported for each. Re-recorded .github/fork-tests-upstream.json. It accepts: - ci.yml top-level permissions (contents: read): fork-tests.yml has the same. - ci.yml top-level concurrency (group per PR/sha, cancel on pull_request): fork-tests.yml has the equivalent, with the event name in the group. - release.yml top-level concurrency and permissions: they apply to upstream's release run, not its test jobs. fork-release.yml has its own. No job or step changed upstream. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent f956059 commit 6d6e0b5

9 files changed

Lines changed: 662 additions & 99 deletions

‎.github/fork-ci-tests.txt‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,26 @@
1-
# Fork script tests the workflow job runs (.github/scripts/fork-ci-tests.cjs).
2-
# Every fork-*.test.{sh,py,cjs} must be listed here. A listed file that is not
3-
# present yet is a note; one that was present and then deleted fails.
1+
# Every test file under .github/scripts (*.test.*, subdirectories included) and
2+
# where it runs (.github/scripts/fork-ci-tests.cjs checks this list).
3+
# <test> run by the workflows job, by extension (sh, py, cjs, mjs, js)
4+
# <test> run by <file> run by <file>, which must name the test's path (not one
5+
# of upstream's workflows: they are disabled in the fork)
6+
# A present test that is not listed fails. A listed file that is not present yet
7+
# is a note; one that was present and then deleted fails.
48
.github/scripts/fork-ci-tests.test.cjs
59
.github/scripts/fork-follow.test.sh
610
.github/scripts/fork-release-workflow.test.py
711
.github/scripts/fork-release.test.sh
812
.github/scripts/fork-schema-check.test.cjs
13+
.github/scripts/fork-secret-scan.test.sh
914
.github/scripts/fork-test-shards.test.cjs
1015
.github/scripts/fork-tests-drift.test.cjs
16+
17+
# Upstream's tests, run where fork-tests.yml follows upstream's ci.yml.
18+
.github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml
19+
.github/scripts/mobile-preview-comment.test.cjs run by .github/workflows/fork-tests.yml
20+
.github/scripts/stage-preview-bundle.test.py run by .github/workflows/fork-tests.yml
21+
.github/scripts/test-delta.test.cjs run by .github/workflows/fork-tests.yml
22+
.github/scripts/vercel-deployment-output.test.cjs run by .github/workflows/fork-tests.yml
23+
# Upstream runs this one from thread-transfer-report.yml (workflow_run of the disabled CI).
24+
.github/scripts/thread-transfer-report.test.cjs run by .github/workflows/fork-tests.yml
25+
# Release Smoke runs scripts/release-smoke.ts, which runs this one.
26+
.github/scripts/relay-state-output.test.cjs run by scripts/release-smoke.ts

‎.github/fork-tests-upstream.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
{
22
".github/workflows/ci.yml": {
3+
"(top-level)": {
4+
"header": "e3b0c44298fc1c14",
5+
"steps": [
6+
{
7+
"name": "permissions",
8+
"digest": "893e619df25e8e99"
9+
},
10+
{
11+
"name": "concurrency",
12+
"digest": "8782324e66667532"
13+
}
14+
]
15+
},
316
"changes": {
417
"header": "8ff211d87838fee8",
518
"steps": [
@@ -340,6 +353,19 @@
340353
}
341354
},
342355
".github/workflows/release.yml": {
356+
"(top-level)": {
357+
"header": "e3b0c44298fc1c14",
358+
"steps": [
359+
{
360+
"name": "concurrency",
361+
"digest": "7675d281fd1ce2db"
362+
},
363+
{
364+
"name": "permissions",
365+
"digest": "2fa142cb2564a193"
366+
}
367+
]
368+
},
343369
"quality": {
344370
"header": "64aee6ef731d6767",
345371
"steps": [

‎.github/scripts/fork-ci-tests.cjs‎

Lines changed: 91 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,86 @@
1-
// Fork only: runs every `.github/scripts/fork-*.test.{sh,py,cjs}` and checks
2-
// them against `.github/fork-ci-tests.txt`. A present test missing from the list
3-
// fails; a listed test that is missing fails once it has been in the tested
4-
// commit's history (deleted while still listed) and is a note before that.
1+
// Fork only: every test file under `.github/scripts` (`*.test.*`, subdirectories
2+
// included) must be run somewhere, checked against `.github/fork-ci-tests.txt`.
3+
// A plain entry is run here, by its extension (RUNNER). An entry
4+
// `<test> run by <file>` is run by that file, which must name the test's path; it
5+
// cannot be one of upstream's workflows, which are disabled in the fork. A
6+
// present test missing from the list fails, and so does a test with no runner
7+
// for its extension. A listed test that is missing fails once it has been in the
8+
// tested commit's history (deleted while still listed) and is a note before that.
59
const childProcess = require("node:child_process");
610
const fs = require("node:fs");
711
const path = require("node:path");
812

913
const ROOT = path.resolve(__dirname, "../..");
1014
const LIST = ".github/fork-ci-tests.txt";
11-
const TEST_FILE = /^fork-.+\.test\.(sh|py|cjs)$/;
12-
const RUNNER = { sh: ["bash"], py: ["python3"], cjs: ["node", "--test"] };
15+
const SCRIPTS = ".github/scripts";
16+
const TEST_FILE = /\.test\.([^./]+)$/;
17+
const RUNNER = {
18+
sh: ["bash"],
19+
py: ["python3"],
20+
cjs: ["node", "--test"],
21+
mjs: ["node", "--test"],
22+
js: ["node", "--test"],
23+
};
24+
const UPSTREAM_WORKFLOW = /^\.github\/workflows\/(?!fork-)[^/]+\.ya?ml$/;
1325

14-
// `ever(file)` says whether the file is in the tested commit's history.
15-
function plan(listed, present, ever) {
26+
// `entries`: [{ file, runBy }]; `present`: test files in the tree;
27+
// `ever(file)` says whether the file is in the tested commit's history;
28+
// `read(file)` returns a runner file's text, or null when it does not exist.
29+
function plan(entries, present, ever, read = () => null) {
30+
const listed = entries.map((entry) => entry.file);
1631
const unlisted = present.filter((file) => !listed.includes(file));
1732
const missing = listed.filter((file) => !present.includes(file));
1833
const deleted = missing.filter((file) => ever(file));
1934
const pending = missing.filter((file) => !deleted.includes(file));
20-
return { run: present, unlisted, deleted, pending };
35+
const here = entries.filter((entry) => !entry.runBy && present.includes(entry.file));
36+
const noRunner = here
37+
.filter((entry) => !RUNNER[entry.file.match(TEST_FILE)?.[1]])
38+
.map((entry) => entry.file);
39+
const elsewhere = [];
40+
const notRun = [];
41+
for (const { file, runBy } of entries) {
42+
if (!runBy || !present.includes(file)) continue;
43+
let why = null;
44+
if (UPSTREAM_WORKFLOW.test(runBy)) why = `${runBy} is an upstream workflow, disabled in the fork`;
45+
else {
46+
const text = read(runBy);
47+
if (text === null) why = `${runBy} does not exist`;
48+
else if (!text.includes(file)) why = `${runBy} does not name ${file}`;
49+
}
50+
if (why) notRun.push({ file, why });
51+
else elsewhere.push({ file, runBy });
52+
}
53+
return {
54+
run: here.map((entry) => entry.file).filter((file) => !noRunner.includes(file)),
55+
unlisted,
56+
deleted,
57+
pending,
58+
noRunner,
59+
elsewhere,
60+
notRun,
61+
};
2162
}
2263

2364
function readList(text) {
2465
return text
2566
.split("\n")
2667
.map((line) => line.replace(/#.*/, "").trim())
27-
.filter(Boolean);
68+
.filter(Boolean)
69+
.map((line) => {
70+
const match = line.match(/^(\S+)(?:\s+run by\s+(\S+))?$/);
71+
if (!match) throw new Error(`${LIST}: cannot read "${line}" (expected "<test>" or "<test> run by <file>")`);
72+
return { file: match[1], runBy: match[2] ?? null };
73+
});
74+
}
75+
76+
function findTests(dir, root = ROOT) {
77+
const out = [];
78+
for (const entry of fs.readdirSync(path.join(root, dir), { withFileTypes: true })) {
79+
const rel = `${dir}/${entry.name}`;
80+
if (entry.isDirectory()) out.push(...findTests(rel, root));
81+
else if (TEST_FILE.test(entry.name)) out.push(rel);
82+
}
83+
return out.toSorted();
2884
}
2985

3086
function git(args) {
@@ -46,18 +102,23 @@ function everInHistory(file) {
46102
return git(["log", "-1", "--format=%h", "HEAD", "--", file]) !== "";
47103
}
48104

105+
function readFile(file) {
106+
try {
107+
return fs.readFileSync(path.join(ROOT, file), "utf8");
108+
} catch (error) {
109+
if (error.code === "ENOENT") return null;
110+
throw error;
111+
}
112+
}
113+
49114
function main() {
50-
const listed = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8"));
51-
const present = fs
52-
.readdirSync(path.join(ROOT, ".github/scripts"))
53-
.filter((name) => TEST_FILE.test(name))
54-
.map((name) => `.github/scripts/${name}`)
55-
.toSorted();
56-
const result = plan(listed, present, everInHistory);
115+
const entries = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8"));
116+
const result = plan(entries, findTests(SCRIPTS), everInHistory, readFile);
57117
const rows = [];
58-
let failed = result.unlisted.length + result.deleted.length;
118+
let failed =
119+
result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length;
59120
for (const file of result.run) {
60-
const [command, ...args] = RUNNER[path.basename(file).match(TEST_FILE)[1]];
121+
const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]];
61122
const started = Date.now();
62123
console.log(`::group::${file}`);
63124
const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" });
@@ -73,10 +134,20 @@ function main() {
73134
console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`);
74135
for (const file of result.deleted)
75136
console.log(`::error::${file} is listed in ${LIST} but was deleted; restore it or unlist it`);
137+
for (const file of result.noRunner)
138+
console.log(
139+
`::error file=${file}::${file}: no runner for this extension (${Object.keys(RUNNER).join(", ")}); ` +
140+
`rename it or list it as "run by <file>"`,
141+
);
142+
for (const { file, why } of result.notRun)
143+
console.log(`::error file=${file}::${file} is not run: ${why}`);
144+
for (const { file, runBy } of result.elsewhere) rows.push(`| \`${file}\` | run by \`${runBy}\` | |`);
76145
for (const file of result.pending)
77146
rows.push(`| \`${file}\` | not present: its PR has not merged | |`);
78147
for (const file of result.unlisted) rows.push(`| \`${file}\` | **not in ${LIST}** | |`);
79148
for (const file of result.deleted) rows.push(`| \`${file}\` | **deleted while listed** | |`);
149+
for (const file of result.noRunner) rows.push(`| \`${file}\` | **no runner for its extension** | |`);
150+
for (const { file, why } of result.notRun) rows.push(`| \`${file}\` | **not run: ${why}** | |`);
80151

81152
const summary = process.env.GITHUB_STEP_SUMMARY;
82153
if (summary)
@@ -94,6 +165,6 @@ function main() {
94165
if (failed) process.exitCode = 1;
95166
}
96167

97-
module.exports = { plan, readList };
168+
module.exports = { plan, readList, findTests };
98169

99170
if (require.main === module) main();
Lines changed: 70 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,90 @@
11
const assert = require("node:assert/strict");
2+
const fs = require("node:fs");
3+
const os = require("node:os");
4+
const path = require("node:path");
25
const test = require("node:test");
36

4-
const { plan, readList } = require("./fork-ci-tests.cjs");
7+
const { plan, readList, findTests } = require("./fork-ci-tests.cjs");
58

69
const never = () => false;
10+
const here = (...files) => files.map((file) => ({ file, runBy: null }));
711

8-
test("runs every present test, listed or not", () => {
9-
assert.deepEqual(plan(["a"], ["a", "b"], never).run, ["a", "b"]);
12+
test("runs every listed present test with a runner", () => {
13+
assert.deepEqual(plan(here("a.test.sh", "b.test.mjs"), ["a.test.sh", "b.test.mjs"], never).run, [
14+
"a.test.sh",
15+
"b.test.mjs",
16+
]);
1017
});
1118

1219
test("a present test missing from the list fails", () => {
13-
assert.deepEqual(plan(["a"], ["a", "b"], never).unlisted, ["b"]);
20+
assert.deepEqual(plan(here("a.test.sh"), ["a.test.sh", "b.test.js"], never).unlisted, [
21+
"b.test.js",
22+
]);
23+
});
24+
25+
test("a listed test with no runner for its extension fails and is not run", () => {
26+
const result = plan(here("a.test.ts"), ["a.test.ts"], never);
27+
assert.deepEqual(result.noRunner, ["a.test.ts"]);
28+
assert.deepEqual(result.run, []);
1429
});
1530

1631
test("a listed test never in history is pending, not a failure", () => {
17-
const result = plan(["a", "later"], ["a"], never);
18-
assert.deepEqual(result.pending, ["later"]);
32+
const result = plan(here("a.test.sh", "later.test.sh"), ["a.test.sh"], never);
33+
assert.deepEqual(result.pending, ["later.test.sh"]);
1934
assert.deepEqual(result.deleted, []);
2035
});
2136

2237
test("a listed test deleted from history fails", () => {
23-
const result = plan(["a", "gone"], ["a"], (file) => file === "gone");
24-
assert.deepEqual(result.deleted, ["gone"]);
38+
const result = plan(here("a.test.sh", "gone.test.sh"), ["a.test.sh"], (f) => f === "gone.test.sh");
39+
assert.deepEqual(result.deleted, ["gone.test.sh"]);
2540
assert.deepEqual(result.pending, []);
2641
});
2742

28-
test("the list ignores comments and blank lines", () => {
29-
assert.deepEqual(readList("# head\n\na.sh # from #2\n b.py\n"), ["a.sh", "b.py"]);
43+
test("a test run elsewhere must be named by its runner, which is not an upstream workflow", () => {
44+
const files = {
45+
".github/workflows/fork-tests.yml": "run: node --test x/named.test.cjs",
46+
".github/workflows/ci.yml": "run: node --test x/upstream.test.cjs",
47+
};
48+
const entries = [
49+
{ file: "x/named.test.cjs", runBy: ".github/workflows/fork-tests.yml" },
50+
{ file: "x/unnamed.test.cjs", runBy: ".github/workflows/fork-tests.yml" },
51+
{ file: "x/upstream.test.cjs", runBy: ".github/workflows/ci.yml" },
52+
{ file: "x/nowhere.test.cjs", runBy: "scripts/missing.ts" },
53+
];
54+
const result = plan(
55+
entries,
56+
entries.map((e) => e.file),
57+
never,
58+
(file) => files[file] ?? null,
59+
);
60+
assert.deepEqual(result.run, []);
61+
assert.deepEqual(result.elsewhere, [
62+
{ file: "x/named.test.cjs", runBy: ".github/workflows/fork-tests.yml" },
63+
]);
64+
assert.deepEqual(
65+
result.notRun.map((n) => n.file),
66+
["x/unnamed.test.cjs", "x/upstream.test.cjs", "x/nowhere.test.cjs"],
67+
);
68+
assert.match(result.notRun[1].why, /upstream workflow/);
69+
});
70+
71+
test("the list ignores comments and blank lines and reads `run by`", () => {
72+
assert.deepEqual(readList("# head\n\na.test.sh # from #2\n b.test.py run by c.yml\n"), [
73+
{ file: "a.test.sh", runBy: null },
74+
{ file: "b.test.py", runBy: "c.yml" },
75+
]);
76+
assert.throws(() => readList("a.test.sh run c.yml\n"), /cannot read/);
77+
});
78+
79+
test("finds every *.test.* file, subdirectories included", () => {
80+
const root = fs.mkdtempSync(path.join(os.tmpdir(), "fork-ci-tests-"));
81+
try {
82+
for (const file of ["s/a.test.cjs", "s/b.cjs", "s/sub/c.test.mjs", "s/sub/deep/d.test.ts"]) {
83+
fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
84+
fs.writeFileSync(path.join(root, file), "");
85+
}
86+
assert.deepEqual(findTests("s", root), ["s/a.test.cjs", "s/sub/c.test.mjs", "s/sub/deep/d.test.ts"]);
87+
} finally {
88+
fs.rmSync(root, { recursive: true, force: true });
89+
}
3090
});

0 commit comments

Comments
 (0)