Skip to content

Commit f927a1c

Browse files
ci: flag pull requests that grow the test suite, and browser tests that need no browser
The suite grew 41% in the three weeks after #22557 pruned it, mostly through agent-written tests. PR Size now comments when a pull request adds 20 or more test cases net, or a new browser test file, counting declarations in the diff so the report never moves with runner load. The test rules gain one line: browser tests only for behavior that needs a browser, logic in a unit test beside it. The Macroscope test-signal check reads that section; its production model missed a logic-only browser test before this rule and catches it after, with every guard case still clean. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 836b86a commit f927a1c

7 files changed

Lines changed: 193 additions & 2 deletions

File tree

‎.github/scripts/test-delta.cjs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Counts the test cases a pull request adds and removes, from a zero-context diff, so PR Size can
2+
// say when a change grows the suite. Counting declarations keeps the number deterministic, unlike
3+
// CI timings, which move with runner load.
4+
const MARKER = "<!-- test-delta -->";
5+
const NET_CASES_TO_REPORT = 20;
6+
const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/;
7+
const TEST_CASE = /^\s*(?:it|test)(?:\.\w+)*\s*\(/;
8+
9+
function kindOf(path) {
10+
if (path.includes(".browser.test.")) return "browser";
11+
if (path.startsWith("apps/server/") || path.includes(".integration.")) return "server";
12+
return "unit";
13+
}
14+
15+
/** @param {string} diff output of `git diff -U0 base...head` */
16+
function countTestDelta(diff) {
17+
const kinds = { browser: 0, server: 0, unit: 0 };
18+
const newBrowserFiles = [];
19+
let path = null;
20+
for (const line of diff.split("\n")) {
21+
if (line.startsWith("diff --git ")) {
22+
path = null;
23+
continue;
24+
}
25+
if (line.startsWith("--- ")) {
26+
const source = line.slice(4).replace(/^a\//, "");
27+
path = TEST_FILE.test(source) ? source : null;
28+
continue;
29+
}
30+
if (line.startsWith("+++ ")) {
31+
const target = line.slice(4).replace(/^b\//, "");
32+
// A deleted file's target is /dev/null; keep its old path to count the removed cases.
33+
if (target !== "/dev/null") path = TEST_FILE.test(target) ? target : null;
34+
continue;
35+
}
36+
if (line.startsWith("new file mode") || !path) continue;
37+
const sign = line[0] === "+" ? 1 : line[0] === "-" ? -1 : 0;
38+
if (sign !== 0 && TEST_CASE.test(line.slice(1))) kinds[kindOf(path)] += sign;
39+
}
40+
for (const match of diff.matchAll(/^diff --git a\/\S+ b\/(\S+)\nnew file mode/gm)) {
41+
if (match[1].includes(".browser.test.")) newBrowserFiles.push(match[1]);
42+
}
43+
return { kinds, net: kinds.browser + kinds.server + kinds.unit, newBrowserFiles };
44+
}
45+
46+
/** Returns the comment body, or null when the change does not grow the suite enough to mention. */
47+
function render(delta) {
48+
if (delta.net < NET_CASES_TO_REPORT && delta.newBrowserFiles.length === 0) return null;
49+
const signed = (n) => (n > 0 ? `+${n}` : String(n));
50+
const lines = [
51+
MARKER,
52+
`This PR changes the test suite by **${signed(delta.net)} test cases** ` +
53+
`(browser ${signed(delta.kinds.browser)}, unit ${signed(delta.kinds.unit)}, server ${signed(delta.kinds.server)}).`,
54+
];
55+
if (delta.newBrowserFiles.length > 0) {
56+
lines.push(
57+
"",
58+
`New browser test files: ${delta.newBrowserFiles.map((file) => `\`${file}\``).join(", ")}.`,
59+
"Each browser test file costs CI seconds of module loading before its first test runs; a unit test costs milliseconds.",
60+
);
61+
}
62+
lines.push(
63+
"",
64+
"Keep tests that would fail for a real reason, and test logic below the UI where it can be. See [Testing](https://github.com/pingdotgg/t3code/blob/main/docs/internals/testing.md).",
65+
);
66+
return lines.join("\n");
67+
}
68+
69+
module.exports = { MARKER, countTestDelta, render };
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
const assert = require("node:assert/strict");
2+
const test = require("node:test");
3+
4+
const { countTestDelta, render } = require("./test-delta.cjs");
5+
6+
const diff = `diff --git a/apps/web/src/Panel.browser.test.tsx b/apps/web/src/Panel.browser.test.tsx
7+
new file mode 100644
8+
--- /dev/null
9+
+++ b/apps/web/src/Panel.browser.test.tsx
10+
@@ -0,0 +1,4 @@
11+
+it("opens", async () => {});
12+
+ it.each([1, 2])("renders %i", async () => {});
13+
+// it("is a comment, still counted only when it starts the line")
14+
diff --git a/apps/server/src/thing.test.ts b/apps/server/src/thing.test.ts
15+
--- a/apps/server/src/thing.test.ts
16+
+++ b/apps/server/src/thing.test.ts
17+
@@ -3,2 +3,1 @@
18+
-it.effect("old", () => {});
19+
-test("older", () => {});
20+
+ it.effect("new", () => {});
21+
diff --git a/packages/shared/src/gone.test.ts b/packages/shared/src/gone.test.ts
22+
deleted file mode 100644
23+
--- a/packages/shared/src/gone.test.ts
24+
+++ /dev/null
25+
@@ -1,1 +0,0 @@
26+
-it("goes", () => {});
27+
diff --git a/apps/web/src/Panel.tsx b/apps/web/src/Panel.tsx
28+
--- a/apps/web/src/Panel.tsx
29+
+++ b/apps/web/src/Panel.tsx
30+
@@ -1 +1 @@
31+
+it("is not a test file", () => {});
32+
`;
33+
34+
test("counts added and removed cases by kind, including deleted files", () => {
35+
assert.deepEqual(countTestDelta(diff), {
36+
kinds: { browser: 2, server: -1, unit: -1 },
37+
net: 0,
38+
newBrowserFiles: ["apps/web/src/Panel.browser.test.tsx"],
39+
});
40+
});
41+
42+
test("reports only growth worth a reviewer's attention", () => {
43+
const quiet = { kinds: { browser: 0, server: 3, unit: 5 }, net: 8, newBrowserFiles: [] };
44+
assert.equal(render(quiet), null);
45+
const growth = { kinds: { browser: 4, server: 10, unit: 12 }, net: 26, newBrowserFiles: [] };
46+
assert.match(render(growth), /\*\*\+26 test cases\*\* \(browser \+4, unit \+12, server \+10\)/);
47+
assert.match(
48+
render(countTestDelta(diff)),
49+
/New browser test files: `apps\/web\/src\/Panel\.browser\.test\.tsx`/,
50+
);
51+
});

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ jobs:
326326
run: python3 -B .github/scripts/stage-preview-bundle.test.py
327327

328328
- name: Test release checks
329-
run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs
329+
run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs .github/scripts/test-delta.test.cjs
330330

331331
- name: Finish installing browser secret helper build libraries
332332
timeout-minutes: 5

‎.github/workflows/pr-size.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,3 +293,55 @@ jobs:
293293
core.info(
294294
`PR #${issueNumber}: ${nonTestChangedLines} non-test lines, ${testChangedLines} test lines, ${changedLines} effective lines -> ${nextLabelName} (${classification})`,
295295
);
296+
# Reads the script from the base checkout above; the PR's own commits are only diffed.
297+
- name: Report test suite growth
298+
uses: actions/github-script@v8
299+
with:
300+
script: |
301+
const { execFileSync } = require("node:child_process");
302+
const { MARKER, countTestDelta, render } = require(
303+
`${process.env.GITHUB_WORKSPACE}/.github/scripts/test-delta.cjs`,
304+
);
305+
306+
const issueNumber = context.payload.pull_request.number;
307+
const diff = execFileSync(
308+
"git",
309+
["diff", "-U0", `${context.payload.pull_request.base.sha}...refs/remotes/pr-size/${issueNumber}`],
310+
{ encoding: "utf8", maxBuffer: 256 * 1024 * 1024 },
311+
);
312+
const delta = countTestDelta(diff);
313+
const body = render(delta);
314+
core.info(`PR #${issueNumber}: test cases ${JSON.stringify(delta.kinds)}, net ${delta.net}`);
315+
316+
const comments = await github.paginate(github.rest.issues.listComments, {
317+
owner: context.repo.owner,
318+
repo: context.repo.repo,
319+
issue_number: issueNumber,
320+
per_page: 100,
321+
});
322+
const existing = comments.find(
323+
(comment) => comment.user?.type === "Bot" && comment.body?.startsWith(MARKER),
324+
);
325+
if (body && existing) {
326+
if (existing.body !== body) {
327+
await github.rest.issues.updateComment({
328+
owner: context.repo.owner,
329+
repo: context.repo.repo,
330+
comment_id: existing.id,
331+
body,
332+
});
333+
}
334+
} else if (body) {
335+
await github.rest.issues.createComment({
336+
owner: context.repo.owner,
337+
repo: context.repo.repo,
338+
issue_number: issueNumber,
339+
body,
340+
});
341+
} else if (existing) {
342+
await github.rest.issues.deleteComment({
343+
owner: context.repo.owner,
344+
repo: context.repo.repo,
345+
comment_id: existing.id,
346+
});
347+
}

‎AGENTS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ An empty database is a bad test. Seed your worktree's `.t3` with a copy of real
105105
- Test behavior that can break, such as orchestration and event-sourcing, provider protocol translation, auth and permission guards, persistence and migrations, external-format parsing, race fixes, and the logic a bug fix changed. Leave out tests for what types already guarantee, trivial logic, copy text, smoke runs, mock-call mirrors, callback wiring, and props rendered to static markup.
106106
- Extend before you add. When an existing test or table covers the behavior, add a row or an assertion there. Write a new `it` only when no case fits.
107107
- Test at one layer: the lowest one that fails on `main`. Do not repeat the case in a wrapper test, at each call site, or in web and mobile copies of the same logic.
108+
- Write a browser test only for behavior that needs a real browser: layout and scrolling, focus and pointer input, real workers, or a component wired to its hooks. Logic a component calls gets a unit test beside it (`*.logic.test.ts`). A browser test file spends seconds in CI loading its module graph before its first test runs; a unit test takes milliseconds.
108109
- [Testing](docs/internals/testing.md) carries these same test rules, plus review guidance, for review bots. Change both together.
109110
- **Do not run repo-wide checks.** No `vp check`, no `vp run -r test`, no `vp run -r typecheck` unless I ask. CI owns the full suite.
110111
- Backend behavior changes ship with focused tests for that behavior.

‎docs/internals/testing.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ Rules for writing tests, and for asking for them in review. AGENTS.md carries th
77
- Test behavior that can break, such as orchestration and event-sourcing, provider protocol translation, auth and permission guards, persistence and migrations, external-format parsing, race fixes, and the logic a bug fix changed. Leave out tests for what types already guarantee, trivial logic, copy text, smoke runs, mock-call mirrors, callback wiring, and props rendered to static markup.
88
- Extend before you add. When an existing test or table covers the behavior, add a row or an assertion there. Write a new `it` only when no case fits.
99
- Test at one layer: the lowest one that fails on `main`. Do not repeat the case in a wrapper test, at each call site, or in web and mobile copies of the same logic.
10-
- In review, ask for a missing test only when it would catch a real regression. Do not ask for one assertion per constant, label, or `switch` case. Flag a new test that breaks the rules above, and name the fix: fold it into the existing case or table, delete it, or keep one layer.
10+
- Write a browser test only for behavior that needs a real browser: layout and scrolling, focus and pointer input, real workers, or a component wired to its hooks. Logic a component calls gets a unit test beside it (`*.logic.test.ts`). A browser test file spends seconds in CI loading its module graph before its first test runs; a unit test takes milliseconds.
11+
- In review, ask for a missing test only when it would catch a real regression. Do not ask for one assertion per constant, label, or `switch` case. Flag a new test that breaks the rules above, and name the fix: fold it into the existing case or table, delete it, keep one layer, or move a browser test whose assertions would pass without rendering into a unit test.
1112
- Backend behavior changes ship with focused tests for that behavior.
1213
- Proof for a bug fix depends on the bug. Logic, data, state, and protocol bugs get a failing test first (the fewest that fail on `main` for that reason, in an existing case of the module's test file where one fits), then the fix. Purely visual bugs (spacing, colour, alignment, a layout shift) get before/after captures as their proof and no test: do not add UI tests for nitpicks, and do not ask for one in review when a capture already shows it. A server-only fix with no visible surface ships the test plus a short runtime transcript.
1314

‎evals/test-signal/cases.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,23 @@ export const cases: ReadonlyArray<Case> = [
133133
'\n it("returns a boolean", () => {\n expect(typeof isMarkdownFileLinkLabel("a.ts", "/repo/a.ts")).toBe("boolean");\n });\n',
134134
targets: [{ path: "packages/shared/src/markdownLinks.test.ts", title: "returns a boolean" }],
135135
},
136+
{
137+
id: "logic-in-browser-test",
138+
kind: "bad",
139+
source: "Synthetic: a browser test that never renders, for a pure function in a .logic.ts file",
140+
type: "synthetic",
141+
rev: base,
142+
path: "apps/web/src/components/chat/OpenInPicker.browser.test.tsx",
143+
after: ' expect(await spoken()).toBe("Opened in VS Code");\n });\n});\n',
144+
insert:
145+
'\ndescribe("open in picker visibility", () => {\n it("shows the picker for the primary environment or a remote that can open links", async () => {\n const { shouldShowOpenInPicker } = await import("./OpenInPicker.logic");\n const show = (overrides: Partial<Parameters<typeof shouldShowOpenInPicker>[0]>) =>\n shouldShowOpenInPicker({\n activeProjectName: "app",\n activeThreadEnvironmentId: selected,\n primaryEnvironmentId: primary,\n remoteOpenMode: "local-exec",\n ...overrides,\n });\n expect(show({ activeProjectName: undefined })).toBe(false);\n expect(show({ activeThreadEnvironmentId: primary })).toBe(true);\n expect(show({})).toBe(false);\n expect(show({ remoteOpenMode: "remote-links" })).toBe(true);\n });\n});\n',
146+
targets: [
147+
{
148+
path: "apps/web/src/components/chat/OpenInPicker.browser.test.tsx",
149+
title: "shows the picker for the primary environment or a remote that can open links",
150+
},
151+
],
152+
},
136153
{
137154
id: "deadlock-race-fix",
138155
kind: "guard",

0 commit comments

Comments
 (0)