Skip to content

Commit 46845a9

Browse files
committed
feat(coverage): decide the touched-function CRAP verdict
`check:coverage-crap --base <ref>` scores the functions this change touched against the Istanbul map `pnpm test:coverage` already writes, and says when the report cannot answer that question at all rather than reporting no findings. This commit is the model and the run: the verdict, its reasons, and the renderer. CI keeps reporting only until the layer above flips it. The model owns every reason a run is untrustworthy, including a missing lcov enumeration, so a caller cannot decide that fact by passing an empty set. One relativizer turns a coverage map key, an lcov path, and a diff path into the same key; three that disagreed meant a map from another checkout matched nothing.
1 parent f9a0012 commit 46845a9

5 files changed

Lines changed: 914 additions & 85 deletions

File tree

‎scripts/coverage-changed/model.ts‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
// lcov was dropped by an exclude glob — reported (non-gating) so exclusions
1111
// cannot silently absorb new logic.
1212

13+
import { repoRelativePath } from '../lib/repo-path.ts';
14+
1315
// Single source of truth for the gate. Changed-line coverage below this
1416
// percentage fails the Coverage CI job (unless waived).
1517
export const CHANGED_LINE_COVERAGE_THRESHOLD = 70;
@@ -33,16 +35,11 @@ export type ChangedFileDiff = {
3335
readonly deleted: boolean;
3436
};
3537

36-
// Normalize an lcov/diff path to a repo-relative posix path. `rootDir` strips an
37-
// absolute-path prefix some coverage providers emit; `./` and `b/` prefixes are
38-
// dropped so lcov and diff paths join on the same key.
38+
// Normalize an lcov/diff path to a repo-relative posix path, then drop the `a/`-style prefixes git
39+
// writes. Relativization itself is shared with the coverage-map keys (scripts/lib/repo-path.ts): a
40+
// key from another checkout must relativize identically here and there, or the two never join.
3941
export function normalizePath(raw: string, rootDir?: string): string {
40-
let p = raw.trim().replaceAll('\\', '/');
41-
if (rootDir) {
42-
const root = rootDir.replaceAll('\\', '/').replace(/\/$/, '');
43-
if (p === root) return '';
44-
if (p.startsWith(`${root}/`)) p = p.slice(root.length + 1);
45-
}
42+
let p = repoRelativePath(raw, rootDir);
4643
if (p.startsWith('b/')) p = p.slice(2);
4744
if (p.startsWith('./')) p = p.slice(2);
4845
return p;

‎scripts/coverage-crap/model.test.ts‎

Lines changed: 292 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,20 @@ import assert from 'node:assert/strict';
22
import fs from 'node:fs';
33
import { test } from 'node:test';
44
import {
5+
coverageMapPaths,
6+
decideCrapVerdict,
57
parseFallowJsonReport,
8+
parseIstanbulCoverageMap,
69
readCrapFindings,
10+
readCrapExemptions,
711
renderCrapAnnotations,
812
renderCrapReport,
913
sanitizeIstanbulCoverage,
14+
type CrapFinding,
15+
type CrapProblem,
16+
type CrapScope,
17+
type CrapVerdict,
18+
type EstimatedFinding,
1019
} from './model.ts';
1120

1221
const valid = { s: { 0: 3, 1: 0 }, f: { 0: 1 }, b: { 0: [2, 0] } };
@@ -71,13 +80,42 @@ test('a report without a findings array is an error, not an empty verdict', () =
7180
assert.throws(() => readCrapFindings({ error: true }, 30), /no findings array/);
7281
});
7382

83+
/** A verdict that owes nothing: what `decideCrapVerdict` returns when the reports answer cleanly. */
84+
const clean = (scope: CrapScope, findings: readonly CrapFinding[] = []): CrapVerdict => ({
85+
blocking: false,
86+
problems: [],
87+
unscoped: [],
88+
offenders: scope === 'touched' ? findings : [],
89+
});
90+
91+
/** A report in the shape a case does not care about, so each case states only its difference. */
92+
const render = (input: Partial<Parameters<typeof renderCrapReport>[0]> = {}) =>
93+
renderCrapReport({
94+
findings: [],
95+
dropped: [],
96+
threshold: 30,
97+
scope: 'touched',
98+
verdict: clean('touched'),
99+
...input,
100+
});
101+
102+
/** A verdict that blocks or lists problems, for the renderer cases that build one by hand. */
103+
const verdictWith = (
104+
problems: readonly CrapProblem[],
105+
unscoped: readonly EstimatedFinding[] = [],
106+
) => ({
107+
blocking: true,
108+
problems,
109+
unscoped,
110+
offenders: [],
111+
});
112+
74113
test('the report names touched functions and unscored files', () => {
75114
const findings = readCrapFindings({ findings: [finding({})] }, 30);
76-
const report = renderCrapReport({
115+
const report = render({
77116
findings,
78117
dropped: ['src/odd.ts'],
79-
threshold: 30,
80-
scope: 'touched',
118+
verdict: { blocking: true, problems: [], unscoped: [], offenders: findings },
81119
});
82120
assert.match(report, /1 touched function\(s\) at or above 30/);
83121
assert.match(report, /\| `src\/a\.ts:10` run \| 42\.0 \| 6 \| 20% \|/);
@@ -88,17 +126,266 @@ test('the report names touched functions and unscored files', () => {
88126
});
89127

90128
test('an empty report says none and lists no table', () => {
91-
const report = renderCrapReport({ findings: [], dropped: [], threshold: 30, scope: 'touched' });
129+
const report = render();
92130
assert.match(report, /: no touched function/);
93131
assert.doesNotMatch(report, /\| Function/);
132+
// Each section owns its own emptiness guard, so a clean run must not announce empty ones: an
133+
// "Unscored: 0 file(s)" or "exemptions (0)" line reads as a finding to someone scanning the log.
134+
assert.doesNotMatch(report, /Unscored:/);
135+
assert.doesNotMatch(report, /exemptions/i);
94136
});
95137

96138
test('a whole-repository report does not claim the functions were touched', () => {
97139
const findings = readCrapFindings({ findings: [finding({})] }, 30);
98-
const report = renderCrapReport({ findings, dropped: [], threshold: 30, scope: 'all' });
140+
const report = render({
141+
findings,
142+
scope: 'all',
143+
verdict: clean('all'),
144+
});
99145
assert.match(report, /: 1 function\(s\) at or above 30/);
100146
});
101147

148+
test('a touched report whose reports cannot answer names the reason and stays ahead of the table', () => {
149+
const report = render({
150+
verdict: verdictWith([
151+
{ kind: 'unscored_touched', files: [{ path: 'src/new.ts', addedLines: 4 }] },
152+
{ kind: 'corrupt_touched', files: ['src/odd.ts'] },
153+
]),
154+
});
155+
assert.match(report, /no trustworthy verdict/);
156+
assert.match(report, /no record in the coverage map\*\*: `src\/new\.ts` \(4 changed line\(s\)\)/);
157+
assert.match(report, /impossible counts\*\*: `src\/odd\.ts`/);
158+
// Every problem carries the command that clears it, since the reader has no other way to act.
159+
assert.match(report, /Run `pnpm test:coverage` again/);
160+
});
161+
162+
test('an unanswerable touched report never reads as the empty verdict it was computed from', () => {
163+
const report = render({
164+
verdict: verdictWith([{ kind: 'enumeration_missing' }]),
165+
});
166+
assert.doesNotMatch(report, /no touched function\(s\)/);
167+
assert.match(report, /\*\*no usable lcov report lists the files the coverage run measured/);
168+
});
169+
170+
test('the same gap in the whole-repo report is a note, not a verdict', () => {
171+
const report = render({
172+
scope: 'all',
173+
verdict: {
174+
...verdictWith([{ kind: 'map_partial', mapFiles: 9, enumeratedFiles: 12 }]),
175+
blocking: false,
176+
},
177+
});
178+
assert.match(report, /report-only\): no function\(s\) at or above 30/);
179+
assert.match(report, /Not gating, but the report is incomplete/);
180+
assert.match(report, /\(9 recorded, 12 enumerated\)/);
181+
});
182+
183+
// Only an override that raises `maxCrap` above the default exempts a file: the others are caps, and
184+
// reading a cap as an exemption would hide debt the report is supposed to show.
185+
const CONFIGURED = JSON.stringify({
186+
health: {
187+
thresholdOverrides: [
188+
{ files: ['src/worker.ts'], maxCrap: 1e12, reason: 'Worker thread.' },
189+
{ files: ['src/cheaper.ts'], maxCrap: 12, reason: 'A cap, not an exemption.' },
190+
{ files: ['src/no-crap.ts'], reason: 'Raises nothing.' },
191+
],
192+
},
193+
});
194+
195+
test('the exemptions are read from the resolved config and stated with their file-wide scope', () => {
196+
const exemptions = readCrapExemptions(CONFIGURED);
197+
assert.deepEqual(exemptions.exemptions, [
198+
{ files: ['src/worker.ts'], maxCrap: 1e12, reason: 'Worker thread.' },
199+
]);
200+
const report = render({
201+
scope: 'all',
202+
verdict: clean('all'),
203+
exemptions,
204+
});
205+
assert.match(report, /exemptions in `fallow-crap\.json` \(1\).*every function in its files/s);
206+
assert.match(report, /- `src\/worker\.ts` — `maxCrap: 1e\+12`: Worker thread\./);
207+
});
208+
209+
test('the exemption listing distinguishes none, empty, and unreadable', () => {
210+
// None and unknown are different claims, and an unreadable one must not be a crash: fallow can
211+
// exit 0 and print a warning with no report, and which files are exempt is reporting, not gating.
212+
// Throwing there would turn a clean advisory `--all` run into exit 1.
213+
assert.deepEqual(readCrapExemptions('{"health":{}}'), { available: true, exemptions: [] });
214+
assert.deepEqual(readCrapExemptions('{"health":{"thresholdOverrides":[]}}'), {
215+
available: true,
216+
exemptions: [],
217+
});
218+
assert.deepEqual(readCrapExemptions('Error: no config'), {
219+
available: false,
220+
detail: 'fallow printed no JSON report:',
221+
});
222+
});
223+
224+
test('exemptions that could not be read are reported as unknown rather than as none', () => {
225+
const report = render({
226+
scope: 'all',
227+
verdict: clean('all'),
228+
exemptions: { available: false, detail: 'fallow config exited with code 1' },
229+
});
230+
assert.match(report, /could not be read from the config \(fallow config exited with code 1\)/);
231+
assert.doesNotMatch(report, /exemptions in `fallow-crap\.json` \(0\)/);
232+
});
233+
234+
/**
235+
* A verdict over one touched file in one measured file, with whatever a case cares about overridden.
236+
* The defaults are the shape of a clean answer, so each case states only its difference.
237+
*/
238+
const decide = (input: Partial<Parameters<typeof decideCrapVerdict>[0]> = {}) =>
239+
decideCrapVerdict({
240+
scope: 'touched',
241+
findings: [],
242+
estimated: [],
243+
touched: [{ path: 'src/a.ts', addedLines: 1 }],
244+
dropped: new Set(),
245+
mapPaths: new Set(['src/a.ts']),
246+
enumeratedPaths: new Set(['src/a.ts']),
247+
...input,
248+
});
249+
250+
const kinds = (verdict: CrapVerdict) => verdict.problems.map((problem) => problem.kind);
251+
252+
test('the verdict blocks a touched function at or above the threshold', () => {
253+
const findings = readCrapFindings({ findings: [finding({})] }, 30);
254+
const verdict = decide({ findings });
255+
assert.equal(verdict.blocking, true);
256+
assert.deepEqual(verdict.problems, []);
257+
assert.deepEqual(verdict.offenders, findings);
258+
});
259+
260+
test('the whole-repo verdict reports the backlog without blocking', () => {
261+
const verdict = decide({
262+
scope: 'all',
263+
findings: readCrapFindings({ findings: [finding({})] }, 30),
264+
});
265+
assert.equal(verdict.blocking, false);
266+
assert.deepEqual(verdict.offenders, []);
267+
});
268+
269+
test('a touched file the sanitizer dropped is named once, as corrupt', () => {
270+
assert.deepEqual(
271+
kinds(
272+
decide({
273+
touched: [{ path: 'src/odd.ts', addedLines: 1 }],
274+
dropped: new Set(['src/odd.ts']),
275+
}),
276+
),
277+
['corrupt_touched'],
278+
);
279+
});
280+
281+
test('an unmeasured touched function is a problem only in a file the map records', () => {
282+
const estimated = [{ path: 'src/a.ts', name: 'added', line: 40, crap: 42 }];
283+
// The map has the file but not the function: it predates the code and the score is a guess.
284+
assert.deepEqual(kinds(decide({ estimated })), ['unmeasured_touched']);
285+
// A file entirely outside the map is already reported as unscored, not also as unmeasured — and
286+
// an empty map is partial on its own, so both names appear and neither is a duplicate.
287+
assert.deepEqual(kinds(decide({ estimated, mapPaths: new Set() })), [
288+
'map_partial',
289+
'unscored_touched',
290+
]);
291+
});
292+
293+
test('an estimate in a file coverage never enumerated is listed, not blocked on', () => {
294+
// `scripts/**` and other out-of-scope files are the shape of a normal tooling PR: coverage does not
295+
// instrument them at all, so re-running cannot measure them and the estimate is the only score
296+
// available. Blocking would refuse every such PR; staying silent would hide a number ≥ 30.
297+
const verdict = decide({
298+
estimated: [{ path: 'scripts/tool.ts', name: 'parse', line: 12, crap: 31.6 }],
299+
touched: [{ path: 'scripts/tool.ts', addedLines: 3 }],
300+
});
301+
assert.deepEqual(verdict.problems, []);
302+
assert.equal(verdict.blocking, false);
303+
assert.deepEqual(verdict.unscoped, [
304+
{ path: 'scripts/tool.ts', name: 'parse', line: 12, crap: 31.6 },
305+
]);
306+
const report = render({
307+
verdict,
308+
});
309+
assert.match(report, /no touched function\(s\) at or above 30/);
310+
assert.match(
311+
report,
312+
/Not measured, and not blocking.*`scripts\/tool\.ts:12` parse \(CRAP 31\.6\)/s,
313+
);
314+
});
315+
316+
test('an estimate in a file the map predates stays a blocking problem, not a listing', () => {
317+
const verdict = decide({ estimated: [{ path: 'src/a.ts', name: 'added', line: 40, crap: 42 }] });
318+
assert.deepEqual(kinds(verdict), ['unmeasured_touched']);
319+
assert.deepEqual(verdict.unscoped, []);
320+
assert.equal(verdict.blocking, true);
321+
});
322+
323+
test('an estimate is never called out-of-scope without an enumeration to check it against', () => {
324+
// Without lcov the run cannot tell "coverage does not instrument this" from "the run should have
325+
// measured this and did not" — the latter is the gap it already reports. Listing the file as
326+
// out-of-scope beside `no trustworthy verdict` would answer a question this run cannot ask.
327+
const verdict = decide({
328+
estimated: [{ path: 'scripts/tool.ts', name: 'parse', line: 12, crap: 31.6 }],
329+
touched: [{ path: 'scripts/tool.ts', addedLines: 3 }],
330+
mapPaths: new Set(),
331+
enumeratedPaths: null,
332+
});
333+
assert.deepEqual(verdict.unscoped, []);
334+
assert.equal(verdict.blocking, true);
335+
const report = render({
336+
verdict,
337+
});
338+
assert.doesNotMatch(report, /does not instrument/);
339+
});
340+
341+
test('a map that records every enumerated file is trusted, and none is a partial map', () => {
342+
const both = ['src/a.ts', 'src/b.ts'];
343+
assert.deepEqual(kinds(decide({ mapPaths: new Set(both), enumeratedPaths: new Set(both) })), []);
344+
// One enumerated file the map does not record is enough to call the map incomplete.
345+
assert.deepEqual(
346+
kinds(decide({ mapPaths: new Set(['src/a.ts']), enumeratedPaths: new Set(both) })),
347+
['map_partial'],
348+
);
349+
});
350+
351+
test('a same-sized map holding the wrong paths is still a partial map', () => {
352+
// Two coverage runs mixed into one directory can produce equal counts over different files, and a
353+
// count comparison would call that complete. The check is per path, so one unrecorded file is
354+
// enough — and it names the touched file rather than only the discrepancy.
355+
const verdict = decide({
356+
mapPaths: new Set(['src/other.ts', 'src/b.ts']),
357+
enumeratedPaths: new Set(['src/a.ts', 'src/b.ts']),
358+
});
359+
assert.deepEqual(kinds(verdict), ['map_partial', 'unscored_touched']);
360+
assert.equal(verdict.blocking, true);
361+
});
362+
363+
test('a run that recorded no files has no record of what it ran, and no verdict', () => {
364+
// A truncated or malformed lcov parses to nothing rather than failing, and every completeness
365+
// check reads this list: an empty one would silence them all and leave a clean-looking verdict.
366+
assert.deepEqual(kinds(decide({ enumeratedPaths: null })), ['enumeration_missing']);
367+
// The whole-repo backlog is scored from the map alone, so an absent lcov says nothing about it.
368+
assert.deepEqual(kinds(decide({ scope: 'all', enumeratedPaths: null })), []);
369+
});
370+
371+
test('a coverage map is only readable as files to counter objects', () => {
372+
assert.deepEqual(parseIstanbulCoverageMap('{"src/a.ts":{"s":{"0":0}}}'), {
373+
'src/a.ts': { s: { 0: 0 } },
374+
});
375+
assert.throws(() => parseIstanbulCoverageMap('not json'), /not valid JSON/);
376+
assert.throws(() => parseIstanbulCoverageMap('[]'), /an array, not a file -> counters object/);
377+
assert.throws(() => parseIstanbulCoverageMap('{"src/a.ts":[]}'), /for src\/a\.ts is object/);
378+
assert.throws(() => parseIstanbulCoverageMap('{"src/a.ts":{"s":[]}}'), /has s as object/);
379+
});
380+
381+
test("coverage keys from another checkout do not match this root's files", () => {
382+
const paths = coverageMapPaths(
383+
{ '/work/other/src/a.ts': {}, 'src/relative.ts': {} },
384+
'/work/this',
385+
);
386+
assert.deepEqual([...paths], ['../other/src/a.ts', 'src/relative.ts']);
387+
});
388+
102389
test('annotations put each finding on its file and line', () => {
103390
const findings = readCrapFindings({ findings: [finding({})] }, 30);
104391
assert.equal(

0 commit comments

Comments
 (0)