Skip to content

Commit 02f13ce

Browse files
authored
refactor(coverage): drop the test-only fallow seam and isolate the suite (#3486)
* refactor(coverage): drop the test-only fallow seam and isolate the suite `run()` grew a `fallowBin` parameter so a test could swap in a fallow that reported a chosen finding. docs/agents/testing.md forbids that shape, and the seam was not needed: real fallow produces every report those cases asked for when the fixture is built to the shape it scores. It is gone, and the blocking exit, `unmeasured_touched` and the `unscoped` listing are each proven by a real run whose source really scores what the case claims. The suite also inherited GITHUB_ACTIONS and GITHUB_STEP_SUMMARY from the job running it, so every fixture verdict was appended to the real step summary alongside the real one. Both are now cleared per case and restored after, and the stream capture restores the descriptor it replaced instead of leaving an own `write` shadowing the one the prototype still provides. Part of #3414. * docs(coverage): say the CRAP gate has no waiver label The changed-line gate honours the coverage-waiver PR label; this one has no equivalent, and a reader who assumes otherwise loses an hour to a label that does nothing.
1 parent d4c4fec commit 02f13ce

6 files changed

Lines changed: 103 additions & 178 deletions

File tree

‎.fallowrc.json‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,6 @@
3434
// subprocess (test/integration/daemon-replace-exit-flush.test.ts), so
3535
// dependency analysis cannot follow the runCmdSync string path to it.
3636
"test/integration/support/exit-after-flush.ts",
37-
// Same reason: scripts/coverage-crap/run.test.ts passes this to run() as the fallow binary,
38-
// and it is invoked as a `node <path>` subprocess rather than imported.
39-
"scripts/coverage-crap/fallow-stand-in.fixture.mjs",
4037
"src/cli/update-check-entry.ts",
4138
"examples/sdk/client-session.ts",
4239
"examples/sdk/metro-runtime.ts",

‎CONTRIBUTING.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,12 @@ fails when it cannot score them. It prints `no trustworthy verdict` and names th
252252
or when a touched function postdates the map and would be scored from complexity alone. Every one of
253253
those is a stale coverage run, not a debt you owe: re-run `pnpm test:coverage` on the commit and
254254
re-check. What does count as debt is editing a function that already scores 30 or more — the score
255-
does not have to be new. Moving a file without editing its lines touches none of its functions.
255+
does not have to be new. Moving a file without editing its lines touches none of its functions, and
256+
neither does deleting lines from one: removing code retires debt instead of adding it, which is also
257+
why deleting a whole function passes. A deletion that leaves an untested complex function standing is
258+
not charged, and that function keeps showing in the `--all` backlog. Unlike the changed-line gate,
259+
this one has no `coverage-waiver` label: the only way past a score is to test or simplify the
260+
function, or to exempt the file in `fallow-crap.json`.
256261

257262
Two things are listed rather than blocked, so a green run does not mean they were checked. A touched
258263
function in a file coverage never instrumented — one `coverage.exclude` drops, or one a stale local

‎scripts/coverage-crap/fallow-stand-in.fixture.mjs‎

Lines changed: 0 additions & 47 deletions
This file was deleted.

‎scripts/coverage-crap/run.test.ts‎

Lines changed: 81 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -98,45 +98,57 @@ function commit(message: string): void {
9898

9999
// The entrypoint writes reports and reads the job summary, so each case needs the stream capture as
100100
// well as the exit code: `code: 0` with a "no trustworthy verdict" body is the failure mode here.
101+
//
102+
// Restoring means putting back the descriptor, not a function of the same name. `process.stdout`
103+
// inherits `write` from its prototype, so a saved-and-reassigned property would leave an own
104+
// property shadowing the prototype's, and `bind` would have handed back a bound copy no later
105+
// `delete` could undo.
106+
type CapturedStream = Readonly<{
107+
target: NodeJS.WriteStream;
108+
key: 'write';
109+
descriptor: PropertyDescriptor | undefined;
110+
}>;
111+
101112
function capture(fn: () => number): { code: number; out: string } {
102113
const chunks: string[] = [];
103-
const original = process.stdout.write.bind(process.stdout);
104-
const originalErr = process.stderr.write.bind(process.stderr);
114+
const restore: CapturedStream[] = [];
105115
const collect = (chunk: string | Uint8Array): boolean => {
106116
chunks.push(String(chunk));
107117
return true;
108118
};
109-
process.stdout.write = collect as typeof process.stdout.write;
110-
process.stderr.write = collect as typeof process.stderr.write;
119+
for (const target of [process.stdout, process.stderr]) {
120+
restore.push({
121+
target,
122+
key: 'write',
123+
descriptor: Object.getOwnPropertyDescriptor(target, 'write'),
124+
});
125+
Object.defineProperty(target, 'write', {
126+
value: collect,
127+
writable: true,
128+
configurable: true,
129+
});
130+
}
111131
try {
112132
return { code: fn(), out: chunks.join('') };
113133
} finally {
114-
process.stdout.write = original;
115-
process.stderr.write = originalErr;
134+
for (const { target, key, descriptor } of restore) {
135+
if (descriptor) Object.defineProperty(target, key, descriptor);
136+
else delete (target as unknown as Record<string, unknown>)[key];
137+
}
116138
}
117139
}
118140

119-
function check(args: readonly string[], fallowBin?: string): { code: number; out: string } {
120-
return capture(() => run(args, repo, fallowBin));
121-
}
122-
123-
/**
124-
* A fallow that reports what a case says rather than what it measures, for the two claims the real
125-
* binary cannot make on command: that the blocking exit follows a reported finding, and that a
126-
* no-verdict branch wins over a finding that asks to be ignored. Also the only way to see which
127-
* flags this gate passes.
128-
*/
129-
const STANDIN = path.join(import.meta.dirname, 'fallow-stand-in.fixture.mjs');
141+
const HEADS = ['main', 'feature'];
130142

131-
/** Tells the stand-in which finding to report, in fallow's own report shape. */
132-
function standInReports(finding: Record<string, unknown>): void {
133-
write('coverage/fallow-stand-in.json', JSON.stringify([finding]));
134-
}
143+
// The real CI job sets these, and `run()` honours both: it appends its report to the step summary
144+
// file and emits annotations. A fixture repo is not a CI job, so leaving them inherited wrote fake
145+
// debt tables into the real job's summary while this suite ran. `checkOnActions` opts one case back
146+
// in, the only case where the annotation format is what is under test.
147+
const ACTIONS_ENV_KEYS = ['GITHUB_ACTIONS', 'GITHUB_STEP_SUMMARY'] as const;
148+
let savedActionsEnv: Record<string, string | undefined> = {};
135149

136-
function standInFlags(): { diffFile: string | null; coverage: string | null } {
137-
return JSON.parse(
138-
fs.readFileSync(path.join(repo, 'coverage/fallow-stand-in-called-with.json'), 'utf8'),
139-
) as { diffFile: string | null; coverage: string | null };
150+
function check(args: readonly string[]): { code: number; out: string } {
151+
return capture(() => run(args, repo));
140152
}
141153

142154
/** Annotations are a CI-surface output, so the case that reads them has to stand in for Actions. */
@@ -145,13 +157,17 @@ function checkOnActions(args: readonly string[]): { code: number; out: string }
145157
try {
146158
return check(args);
147159
} finally {
148-
delete process.env.GITHUB_ACTIONS;
160+
if (savedActionsEnv.GITHUB_ACTIONS === undefined) delete process.env.GITHUB_ACTIONS;
161+
else process.env.GITHUB_ACTIONS = savedActionsEnv.GITHUB_ACTIONS;
149162
}
150163
}
151164

152-
const HEADS = ['main', 'feature'];
153-
154165
beforeEach(() => {
166+
savedActionsEnv = {};
167+
for (const key of ACTIONS_ENV_KEYS) {
168+
savedActionsEnv[key] = process.env[key];
169+
delete process.env[key];
170+
}
155171
repo = fs.mkdtempSync(path.join(os.tmpdir(), 'coverage-crap-'));
156172
git('init', '-q');
157173
git('config', 'user.email', '[email protected]');
@@ -187,6 +203,10 @@ beforeEach(() => {
187203

188204
afterEach(() => {
189205
fs.rmSync(repo, { recursive: true, force: true });
206+
for (const [key, value] of Object.entries(savedActionsEnv)) {
207+
if (value === undefined) delete process.env[key];
208+
else process.env[key] = value;
209+
}
190210
});
191211

192212
test('a change that touches no function at or above the threshold passes', () => {
@@ -220,6 +240,10 @@ test('editing a function that already scores too high blocks, even though the sc
220240
});
221241

222242
test("the score of an untouched file is not this change's verdict", () => {
243+
// Half of this is the exit code: the map holds `src/debt.ts` at CRAP 42 and the run is still green,
244+
// which only happens if fallow filtered its findings by this diff rather than scoring the repo.
245+
// The other half is that diff itself, since a flag that silently went missing would score
246+
// everything and a diff built from the wrong range would look just as clean.
223247
writeCoverage({ 'src/debt.ts': {}, 'src/clean.ts': { hits: 4 } });
224248
write('src/clean.ts', `${TRIVIAL}export const also_trivial = 2;\n`);
225249
commit('clean edit, debt left alone');
@@ -228,6 +252,9 @@ test("the score of an untouched file is not this change's verdict", () => {
228252

229253
assert.equal(code, 0);
230254
assert.doesNotMatch(out, /src\/debt\.ts:1/);
255+
const scoped = fs.readFileSync(path.join(repo, 'coverage/fallow-crap.diff'), 'utf8');
256+
assert.match(scoped, /^\+\+\+ b\/src\/clean\.ts$/m);
257+
assert.doesNotMatch(scoped, /^\+\+\+ b\/src\/debt\.ts$/m);
231258
});
232259

233260
test('a pure rename does not touch the functions that moved with the file', () => {
@@ -327,94 +354,52 @@ test('an lcov that lists no files is a gap too, not a trivially-complete enumera
327354
assert.match(out, /no usable lcov report lists the files the coverage run measured/);
328355
});
329356

330-
test("the blocking exit follows the reported finding, not this suite's guess about scoring", () => {
331-
// The real-fallow cases above prove fallow reports debt; this one proves the plumbing: a finding
332-
// fallow returns for a touched function is what turns the exit code, and it reaches fallow scoped
333-
// to this diff. Losing `--diff-file` would widen every PR verdict to the whole repo, and a green
334-
// run would not notice.
335-
writeCoverage({ 'src/debt.ts': {}, 'src/clean.ts': { hits: 4 } });
336-
write('src/clean.ts', `${TRIVIAL}export const also_trivial = 2;\n`);
337-
commit('clean edit');
338-
standInReports({ path: 'src/debt.ts', name: 'score', line: 1, crap: 42 });
339-
340-
const { code, out } = check(['--base', HEADS[0]!], STANDIN);
341-
342-
assert.equal(code, 1);
343-
assert.match(out, /1 touched function\(s\) at or above 30/);
344-
assert.match(out, /\| `src\/debt\.ts:1` score \| 42\.0 \| 6 \| 0% \|/);
345-
assert.equal(standInFlags().diffFile, 'coverage/fallow-crap.diff');
346-
});
347-
348-
test('the same finding in report-only scope stays green and asks for no diff', () => {
349-
writeCoverage({ 'src/debt.ts': {}, 'src/clean.ts': { hits: 4 } });
350-
write('src/clean.ts', `${TRIVIAL}export const also_trivial = 2;\n`);
351-
commit('clean edit');
352-
standInReports({ path: 'src/debt.ts', name: 'score', line: 1, crap: 42 });
353-
354-
const { code, out } = check(['--all'], STANDIN);
355-
356-
assert.equal(code, 0);
357-
assert.match(out, /Coverage-based CRAP \(report-only\): 1 function\(s\) at or above 30/);
358-
assert.equal(standInFlags().diffFile, null);
359-
});
360-
361-
test('a finding does not outvote the missing enumeration it would have been scored from', () => {
362-
// Both are true at once: there is debt to show and no record of what was measured. Showing the
363-
// count would let a report that cannot answer its own question read as an answered one.
364-
writeCoverage({ 'src/debt.ts': {}, 'src/clean.ts': { hits: 4 } });
365-
write('src/clean.ts', `${TRIVIAL}export const also_trivial = 2;\n`);
366-
commit('clean edit');
357+
test('a debt finding and a missing enumeration together still say the report cannot answer', () => {
358+
// Both are true at once: there is debt to show, and no record of what was measured. Showing the
359+
// count would let a report that cannot answer its own question read as an answered one, so the
360+
// gap wins the headline either way.
361+
writeCoverage({ 'src/debt.ts': { span: [1, 19] }, 'src/clean.ts': { hits: 4 } });
362+
write(
363+
'src/debt.ts',
364+
DEBT.replace(' return total;', ' if (a === b) {\n total += 2;\n }\n return total;'),
365+
);
366+
commit('edit the debt, then lose lcov');
367367
fs.rmSync(path.join(repo, 'coverage/lcov.info'));
368-
standInReports({ path: 'src/debt.ts', name: 'score', line: 1, crap: 42 });
369368

370-
const { code, out } = check(['--base', HEADS[0]!], STANDIN);
369+
const { code, out } = check(['--base', HEADS[0]!]);
371370

372371
assert.equal(code, 1);
373-
assert.match(out, /no trustworthy verdict/i);
374372
assert.match(out, /no usable lcov report lists the files the coverage run measured/);
375373
assert.doesNotMatch(out, /1 touched function\(s\) at or above 30/);
376374
});
377375

378-
test('an estimated score for a touched function the map predates yields no verdict', () => {
379-
// The file is in the map, so only this function postdates it: fallow scored it from complexity
380-
// alone, which is a guess about a function this PR added and no re-run can leave unguessed.
376+
test('a touched function the map predates is estimated, and an estimate is no verdict', () => {
377+
// `src/debt.ts` IS in the map, which records only `score`, so appending a second complex function
378+
// leaves the map older than the code and fallow scores the new one from complexity alone. Real
379+
// fallow produces exactly that report here, which is what the gate refuses to score from.
381380
writeCoverage({ 'src/debt.ts': {}, 'src/clean.ts': { hits: 4 } });
382-
write('src/clean.ts', `${TRIVIAL}export const also_trivial = 2;\n`);
383-
commit('clean edit');
384-
standInReports({
385-
path: 'src/clean.ts',
386-
name: 'also_trivial',
387-
line: 2,
388-
crap: 31.6,
389-
coverage_source: 'estimated',
390-
});
381+
write('src/debt.ts', `${DEBT}${DEBT.replace('score', 'added')}`);
382+
commit('append a complex function to a file the map records');
391383

392-
const { code, out } = check(['--base', HEADS[0]!], STANDIN);
384+
const { code, out } = check(['--base', HEADS[0]!]);
393385

394386
assert.equal(code, 1);
395387
assert.match(out, /no trustworthy verdict/i);
396-
assert.match(out, /estimated from complexity alone\*\*: `src\/clean\.ts:2` also_trivial/);
388+
assert.match(out, /estimated from complexity alone\*\*: `src\/debt\.ts:15` added/);
397389
});
398390

399-
test('an estimated score in a file coverage never instruments is listed, not blocked on', () => {
400-
// `scripts/**` is the shape of a normal tooling PR: no re-run would measure it, so blocking would
401-
// refuse the PR for a file coverage was never asked to instrument. Silent would hide the 31.6.
391+
test('an estimate in a file coverage never instruments is listed, not blocked on', () => {
392+
// `scripts/**` is outside both coverage roots, so no re-run would measure it: blocking on the
393+
// estimate would refuse every tooling PR, and staying silent would hide the 42.
402394
writeCoverage({ 'src/debt.ts': {}, 'src/clean.ts': { hits: 4 } });
403-
write('scripts/tool.ts', 'export const flag = 1;\n');
404-
commit('add a tooling file');
405-
standInReports({
406-
path: 'scripts/tool.ts',
407-
name: 'flag',
408-
line: 1,
409-
crap: 31.6,
410-
coverage_source: 'estimated',
411-
});
395+
write('scripts/tool.ts', DEBT);
396+
commit('add a tooling file coverage does not instrument');
412397

413-
const { code, out } = check(['--base', HEADS[0]!], STANDIN);
398+
const { code, out } = check(['--base', HEADS[0]!]);
414399

415400
assert.equal(code, 0);
416401
assert.match(out, /no touched function\(s\) at or above 30/);
417-
assert.match(out, /Not measured, and not blocking.*`scripts\/tool\.ts:1` flag \(CRAP 31\.6\)/s);
402+
assert.match(out, /Not measured, and not blocking.*`scripts\/tool\.ts:1` score \(CRAP 42\.0\)/s);
418403
});
419404

420405
test('a missing coverage map says so with the command to run', () => {

0 commit comments

Comments
 (0)