Skip to content

Commit 8203d7f

Browse files
committed
feat(coverage): decide the touched-function CRAP verdict
`check:coverage-crap --base <ref>` was report-only: a PR that pushed a function over the threshold printed a warning annotation and went green. It now returns 1 for that PR, and also when the reports cannot be trusted to answer at all — a missing, malformed, or partial coverage map, a touched file the map dropped or never recorded, or a touched function the map predates, which fallow scores from complexity alone. `--all` stays the advisory whole-repo backlog that `main` reports. Attribution reuses the rename-aware diff, so moving a file owes nothing for the functions that travelled with it, while a rename that also edits a body still owes those lines. The report lists the file-wide `fallow-crap.json` exemptions it honoured, read from the resolved config rather than a copy that can drift. Completeness is checked per path, not by counting entries: two coverage runs mixed into one directory produce equal counts over different files, and a count would trust that. An lcov that parses to zero files counts as missing rather than as an empty enumeration, because every check reads that list and an empty one silences them all at once. An estimate in a file coverage never instruments — a script, a worker entry — is listed without blocking, since it is not debt this change owes yet is still not a measurement, and no re-run will make it one. CI keeps its report-only step: flipping it is the next change, so this one can be reviewed as the verdict alone. Part of #3426
1 parent 18d22c7 commit 8203d7f

3 files changed

Lines changed: 926 additions & 73 deletions

File tree

‎scripts/coverage-crap/model.test.ts‎

Lines changed: 315 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,22 @@ import assert from 'node:assert/strict';
22
import fs from 'node:fs';
33
import { test } from 'node:test';
44
import {
5+
coverageMapPaths,
6+
decideCrapVerdict,
57
parseFallowJsonReport,
8+
parseIstanbulCoverageMap,
9+
readCrapExemptions,
610
readCrapFindings,
11+
readConfiguredExemptions,
712
renderCrapAnnotations,
813
renderCrapReport,
914
sanitizeIstanbulCoverage,
15+
usableEnumeration,
16+
type CrapFinding,
17+
type CrapProblem,
18+
type CrapScope,
19+
type CrapVerdict,
20+
type EstimatedFinding,
1021
} from './model.ts';
1122

1223
const valid = { s: { 0: 3, 1: 0 }, f: { 0: 1 }, b: { 0: [2, 0] } };
@@ -71,13 +82,33 @@ test('a report without a findings array is an error, not an empty verdict', () =
7182
assert.throws(() => readCrapFindings({ error: true }, 30), /no findings array/);
7283
});
7384

85+
/** A verdict that owes nothing: what `decideCrapVerdict` returns when the reports answer cleanly. */
86+
const clean = (scope: CrapScope, findings: readonly CrapFinding[] = []): CrapVerdict => ({
87+
blocking: false,
88+
problems: [],
89+
unscoped: [],
90+
offenders: scope === 'touched' ? findings : [],
91+
});
92+
93+
/** A verdict that blocks or lists problems, for the renderer cases that build one by hand. */
94+
const verdictWith = (
95+
problems: readonly CrapProblem[],
96+
unscoped: readonly EstimatedFinding[] = [],
97+
) => ({
98+
blocking: true,
99+
problems,
100+
unscoped,
101+
offenders: [],
102+
});
103+
74104
test('the report names touched functions and unscored files', () => {
75105
const findings = readCrapFindings({ findings: [finding({})] }, 30);
76106
const report = renderCrapReport({
77107
findings,
78108
dropped: ['src/odd.ts'],
79109
threshold: 30,
80110
scope: 'touched',
111+
verdict: { blocking: true, problems: [], unscoped: [], offenders: findings },
81112
});
82113
assert.match(report, /1 touched function\(s\) at or above 30/);
83114
assert.match(report, /\| `src\/a\.ts:10` run \| 42\.0 \| 6 \| 20% \|/);
@@ -88,17 +119,299 @@ test('the report names touched functions and unscored files', () => {
88119
});
89120

90121
test('an empty report says none and lists no table', () => {
91-
const report = renderCrapReport({ findings: [], dropped: [], threshold: 30, scope: 'touched' });
122+
const report = renderCrapReport({
123+
findings: [],
124+
dropped: [],
125+
threshold: 30,
126+
scope: 'touched',
127+
verdict: clean('touched'),
128+
});
92129
assert.match(report, /: no touched function/);
93130
assert.doesNotMatch(report, /\| Function/);
131+
// Each section owns its own emptiness guard, so a clean run must not announce empty ones: an
132+
// "Unscored: 0 file(s)" or "exemptions (0)" line reads as a finding to someone scanning the log.
133+
assert.doesNotMatch(report, /Unscored:/);
134+
assert.doesNotMatch(report, /exemptions/i);
94135
});
95136

96137
test('a whole-repository report does not claim the functions were touched', () => {
97138
const findings = readCrapFindings({ findings: [finding({})] }, 30);
98-
const report = renderCrapReport({ findings, dropped: [], threshold: 30, scope: 'all' });
139+
const report = renderCrapReport({
140+
findings,
141+
dropped: [],
142+
threshold: 30,
143+
scope: 'all',
144+
verdict: clean('all'),
145+
});
99146
assert.match(report, /: 1 function\(s\) at or above 30/);
100147
});
101148

149+
test('a touched report whose reports cannot answer names the reason and stays ahead of the table', () => {
150+
const report = renderCrapReport({
151+
findings: [],
152+
dropped: [],
153+
threshold: 30,
154+
scope: 'touched',
155+
verdict: verdictWith([
156+
{ kind: 'unscored_touched', files: [{ path: 'src/new.ts', addedLines: 4 }] },
157+
{ kind: 'corrupt_touched', files: ['src/odd.ts'] },
158+
]),
159+
});
160+
assert.match(report, /no trustworthy verdict/);
161+
assert.match(report, /no record in the coverage map\*\*: `src\/new\.ts` \(4 changed line\(s\)\)/);
162+
assert.match(report, /impossible counts\*\*: `src\/odd\.ts`/);
163+
// Every problem carries the command that clears it, since the reader has no other way to act.
164+
assert.match(report, /Run `pnpm test:coverage` again/);
165+
});
166+
167+
test('an unanswerable touched report never reads as the empty verdict it was computed from', () => {
168+
const report = renderCrapReport({
169+
findings: [],
170+
dropped: [],
171+
threshold: 30,
172+
scope: 'touched',
173+
verdict: verdictWith([{ kind: 'enumeration_missing', detail: 'no coverage/lcov.info' }]),
174+
});
175+
assert.doesNotMatch(report, /no touched function\(s\)/);
176+
assert.match(
177+
report,
178+
/which files the coverage run enumerated is missing or unreadable\*\* \(no coverage\/lcov\.info\)/,
179+
);
180+
});
181+
182+
test('the same gap in the whole-repo report is a note, not a verdict', () => {
183+
const report = renderCrapReport({
184+
findings: [],
185+
dropped: [],
186+
threshold: 30,
187+
scope: 'all',
188+
verdict: {
189+
...verdictWith([{ kind: 'map_partial', mapFiles: 9, enumeratedFiles: 12 }]),
190+
blocking: false,
191+
},
192+
});
193+
assert.match(report, /report-only\): no function\(s\) at or above 30/);
194+
assert.match(report, /Not gating, but the report is incomplete/);
195+
assert.match(report, /\(9 recorded, 12 enumerated\)/);
196+
});
197+
198+
test('the exemptions are read from the resolved config and stated with their file-wide scope', () => {
199+
const exemptions = readCrapExemptions({
200+
health: {
201+
thresholdOverrides: [
202+
{ files: ['src/worker.ts'], maxCrap: 1e12, reason: 'Worker thread.' },
203+
{ files: ['src/cheaper.ts'], maxCrap: 12, reason: 'A cap, not an exemption.' },
204+
{ files: ['src/no-crap.ts'], reason: 'Raises nothing.' },
205+
],
206+
},
207+
});
208+
assert.deepEqual(exemptions, [
209+
{ files: ['src/worker.ts'], maxCrap: 1e12, reason: 'Worker thread.' },
210+
]);
211+
const report = renderCrapReport({
212+
findings: [],
213+
dropped: [],
214+
threshold: 30,
215+
scope: 'all',
216+
verdict: clean('all'),
217+
exemptions: { available: true, exemptions },
218+
});
219+
assert.match(report, /exemptions in `fallow-crap\.json` \(1\).*every function in its files/s);
220+
assert.match(report, /- `src\/worker\.ts` — `maxCrap: 1e\+12`: Worker thread\./);
221+
});
222+
223+
test('the exemption listing distinguishes none, unknown, and unreadable', () => {
224+
// None and unknown are different claims, and the third must not be a crash: fallow can exit 0 and
225+
// print a warning with no report, and which files are exempt is reporting, not gating. Throwing
226+
// there would turn a clean advisory `--all` run into exit 1.
227+
assert.deepEqual(readCrapExemptions({ health: {} }), []);
228+
assert.deepEqual(readConfiguredExemptions('{"health":{"thresholdOverrides":[]}}'), {
229+
available: true,
230+
exemptions: [],
231+
});
232+
assert.deepEqual(readConfiguredExemptions('Error: no config'), {
233+
available: false,
234+
detail: 'fallow printed no JSON report:',
235+
});
236+
});
237+
238+
test('exemptions that could not be read are reported as unknown rather than as none', () => {
239+
const report = renderCrapReport({
240+
findings: [],
241+
dropped: [],
242+
threshold: 30,
243+
scope: 'all',
244+
verdict: clean('all'),
245+
exemptions: { available: false, detail: 'fallow config exited with code 1' },
246+
});
247+
assert.match(report, /could not be read from the config \(fallow config exited with code 1\)/);
248+
assert.doesNotMatch(report, /exemptions in `fallow-crap\.json` \(0\)/);
249+
});
250+
251+
/**
252+
* A verdict over one touched file in one measured file, with whatever a case cares about overridden.
253+
* The defaults are the shape of a clean answer, so each case states only its difference.
254+
*/
255+
const decide = (input: Partial<Parameters<typeof decideCrapVerdict>[0]> = {}) =>
256+
decideCrapVerdict({
257+
scope: 'touched',
258+
findings: [],
259+
estimated: [],
260+
touched: [{ path: 'src/a.ts', addedLines: 1 }],
261+
dropped: new Set(),
262+
mapPaths: new Set(['src/a.ts']),
263+
enumeratedPaths: new Set(['src/a.ts']),
264+
ioProblems: [],
265+
...input,
266+
});
267+
268+
const kinds = (verdict: CrapVerdict) => verdict.problems.map((problem) => problem.kind);
269+
270+
test('the verdict blocks a touched function at or above the threshold', () => {
271+
const findings = readCrapFindings({ findings: [finding({})] }, 30);
272+
const verdict = decide({ findings });
273+
assert.equal(verdict.blocking, true);
274+
assert.deepEqual(verdict.problems, []);
275+
assert.deepEqual(verdict.offenders, findings);
276+
});
277+
278+
test('the whole-repo verdict reports the backlog without blocking', () => {
279+
const verdict = decide({
280+
scope: 'all',
281+
findings: readCrapFindings({ findings: [finding({})] }, 30),
282+
});
283+
assert.equal(verdict.blocking, false);
284+
assert.deepEqual(verdict.offenders, []);
285+
});
286+
287+
test('a touched file the sanitizer dropped is named once, as corrupt', () => {
288+
assert.deepEqual(
289+
kinds(
290+
decide({
291+
touched: [{ path: 'src/odd.ts', addedLines: 1 }],
292+
dropped: new Set(['src/odd.ts']),
293+
}),
294+
),
295+
['corrupt_touched'],
296+
);
297+
});
298+
299+
test('an unmeasured touched function is a problem only in a file the map records', () => {
300+
const estimated = [{ path: 'src/a.ts', name: 'added', line: 40, crap: 42 }];
301+
// The map has the file but not the function: it predates the code and the score is a guess.
302+
assert.deepEqual(kinds(decide({ estimated })), ['unmeasured_touched']);
303+
// A file entirely outside the map is already reported as unscored, not also as unmeasured — and
304+
// an empty map is partial on its own, so both names appear and neither is a duplicate.
305+
assert.deepEqual(kinds(decide({ estimated, mapPaths: new Set() })), [
306+
'map_partial',
307+
'unscored_touched',
308+
]);
309+
});
310+
311+
test('an estimate in a file coverage never enumerated is listed, not blocked on', () => {
312+
// `scripts/**` and other out-of-scope files are the shape of a normal tooling PR: coverage does not
313+
// instrument them at all, so re-running cannot measure them and the estimate is the only score
314+
// available. Blocking would refuse every such PR; staying silent would hide a number ≥ 30.
315+
const verdict = decide({
316+
estimated: [{ path: 'scripts/tool.ts', name: 'parse', line: 12, crap: 31.6 }],
317+
touched: [{ path: 'scripts/tool.ts', addedLines: 3 }],
318+
});
319+
assert.deepEqual(verdict.problems, []);
320+
assert.equal(verdict.blocking, false);
321+
assert.deepEqual(verdict.unscoped, [
322+
{ path: 'scripts/tool.ts', name: 'parse', line: 12, crap: 31.6 },
323+
]);
324+
const report = renderCrapReport({
325+
findings: [],
326+
dropped: [],
327+
threshold: 30,
328+
scope: 'touched',
329+
verdict,
330+
});
331+
assert.match(report, /no touched function\(s\) at or above 30/);
332+
assert.match(
333+
report,
334+
/Not measured, and not blocking.*`scripts\/tool\.ts:12` parse \(CRAP 31\.6\)/s,
335+
);
336+
});
337+
338+
test('an estimate in a file the map predates stays a blocking problem, not a listing', () => {
339+
const verdict = decide({ estimated: [{ path: 'src/a.ts', name: 'added', line: 40, crap: 42 }] });
340+
assert.deepEqual(kinds(verdict), ['unmeasured_touched']);
341+
assert.deepEqual(verdict.unscoped, []);
342+
assert.equal(verdict.blocking, true);
343+
});
344+
345+
test('an estimate is never called out-of-scope without an enumeration to check it against', () => {
346+
// Without lcov the run cannot tell "coverage does not instrument this" from "the run should have
347+
// measured this and did not" — the latter is the gap it already reports. Listing the file as
348+
// out-of-scope beside `no trustworthy verdict` would answer a question this run cannot ask.
349+
const verdict = decide({
350+
estimated: [{ path: 'scripts/tool.ts', name: 'parse', line: 12, crap: 31.6 }],
351+
touched: [{ path: 'scripts/tool.ts', addedLines: 3 }],
352+
mapPaths: new Set(),
353+
enumeratedPaths: new Set(),
354+
ioProblems: [{ kind: 'enumeration_missing', detail: 'no coverage/lcov.info' }],
355+
});
356+
assert.deepEqual(verdict.unscoped, []);
357+
assert.equal(verdict.blocking, true);
358+
const report = renderCrapReport({
359+
findings: [],
360+
dropped: [],
361+
threshold: 30,
362+
scope: 'touched',
363+
verdict,
364+
});
365+
assert.doesNotMatch(report, /does not instrument/);
366+
});
367+
368+
test('a map that records every enumerated file is trusted, and none is a partial map', () => {
369+
const both = ['src/a.ts', 'src/b.ts'];
370+
assert.deepEqual(kinds(decide({ mapPaths: new Set(both), enumeratedPaths: new Set(both) })), []);
371+
// One enumerated file the map does not record is enough to call the map incomplete.
372+
assert.deepEqual(
373+
kinds(decide({ mapPaths: new Set(['src/a.ts']), enumeratedPaths: new Set(both) })),
374+
['map_partial'],
375+
);
376+
});
377+
378+
test('a same-sized map holding the wrong paths is still a partial map', () => {
379+
// Two coverage runs mixed into one directory can produce equal counts over different files, and a
380+
// count comparison would call that complete. The check is per path, so one unrecorded file is
381+
// enough — and it names the touched file rather than only the discrepancy.
382+
const verdict = decide({
383+
mapPaths: new Set(['src/other.ts', 'src/b.ts']),
384+
enumeratedPaths: new Set(['src/a.ts', 'src/b.ts']),
385+
});
386+
assert.deepEqual(kinds(verdict), ['map_partial', 'unscored_touched']);
387+
assert.equal(verdict.blocking, true);
388+
});
389+
390+
test('an enumeration of zero files cannot serve as the record of what ran', () => {
391+
// A truncated or malformed lcov parses to nothing rather than failing, and every completeness
392+
// check reads this list: an empty one would silence them all and leave a clean-looking verdict.
393+
assert.equal(usableEnumeration([]), null);
394+
assert.deepEqual([...usableEnumeration(['src/a.ts'])!], ['src/a.ts']);
395+
});
396+
397+
test('a coverage map is only readable as files to counter objects', () => {
398+
assert.deepEqual(parseIstanbulCoverageMap('{"src/a.ts":{"s":{"0":0}}}'), {
399+
'src/a.ts': { s: { 0: 0 } },
400+
});
401+
assert.throws(() => parseIstanbulCoverageMap('not json'), /not valid JSON/);
402+
assert.throws(() => parseIstanbulCoverageMap('[]'), /an array, not a file -> counters object/);
403+
assert.throws(() => parseIstanbulCoverageMap('{"src/a.ts":[]}'), /for src\/a\.ts is object/);
404+
assert.throws(() => parseIstanbulCoverageMap('{"src/a.ts":{"s":[]}}'), /has s as object/);
405+
});
406+
407+
test("coverage keys from another checkout do not match this root's files", () => {
408+
const paths = coverageMapPaths(
409+
{ '/work/other/src/a.ts': {}, 'src/relative.ts': {} },
410+
'/work/this',
411+
);
412+
assert.deepEqual([...paths], ['../other/src/a.ts', 'src/relative.ts']);
413+
});
414+
102415
test('annotations put each finding on its file and line', () => {
103416
const findings = readCrapFindings({ findings: [finding({})] }, 30);
104417
assert.equal(

0 commit comments

Comments
 (0)