Skip to content

Commit e843a9a

Browse files
committed
fix(mutation): score and publish a short shard set before failing on the count
The expected-count check ran inside readShardedReports, before anything was summarized, so on the weekly's real `--expect-shards 10` one dead shard threw away the nine that had reported — the earlier reorder only moved the zero-mutants check. The merge now returns the shard count, and both verdicts run after emit() with the same exit code and `score` stage. Regression uses the weekly argument shape (`--expect-shards 10`, one shard present) and asserts the reporting kernel's row reaches stdout while the run still fails.
1 parent 91f6f4a commit e843a9a

2 files changed

Lines changed: 60 additions & 25 deletions

File tree

‎scripts/mutation/envelope.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,35 @@ test('an incomplete shard set fails instead of scoring the missing module as zer
200200
fs.rmSync(path.join(repoRoot, '.tmp/mutation/partial-shards'), { recursive: true, force: true });
201201
});
202202

203+
// The weekly job's real shape: `--expect-shards 10` over a directory missing
204+
// shards. The count check used to run inside the merge, before anything was
205+
// scored, so one dead shard discarded the nine that had reported.
206+
test('a short shard set publishes the shards that did report, then fails', () => {
207+
const shards = path.join(repoRoot, '.tmp/mutation/short-shards/shard-kernel-errors');
208+
fs.mkdirSync(shards, { recursive: true });
209+
fs.writeFileSync(
210+
path.join(shards, 'mutation.json'),
211+
JSON.stringify({
212+
files: { 'packages/kernel/src/errors.ts': { mutants: [{ status: 'Killed' }] } },
213+
}),
214+
);
215+
const result = runMutation([
216+
'--report-dir',
217+
'.tmp/mutation/short-shards',
218+
'--expect-shards',
219+
'10',
220+
'--modules',
221+
'kernel-errors',
222+
]);
223+
assert.notEqual(result.exitCode, 0, 'a short shard set must fail the aggregate');
224+
assert.match(result.stderr, /1 report\(s\), expected 10/);
225+
assert.match(result.stdout, /\| `kernel-errors` — [^|]+\| 100% \|/);
226+
const envelope = readEnvelope();
227+
assert.equal(envelope.result, 'fail');
228+
assert.equal(envelope.data.stage, 'score');
229+
fs.rmSync(path.join(repoRoot, '.tmp/mutation/short-shards'), { recursive: true, force: true });
230+
});
231+
203232
// Argument parsing and the provenance read used to sit outside the envelope
204233
// boundary, so the lane could exit without declaring itself at all.
205234
test('a malformed invocation still writes an envelope', () => {

‎scripts/mutation/run.ts‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,12 @@ function readReport(file: string): StrykerReport {
337337
return JSON.parse(fs.readFileSync(absolute, 'utf8')) as StrykerReport;
338338
}
339339

340-
function readShardedReports(dir: string, expected: number | undefined): StrykerReport {
340+
/**
341+
* Merges whatever shard reports are present and says how many there were. The
342+
* expected-count verdict is the caller's, so a short set is still scored and
343+
* published before it fails.
344+
*/
345+
function readShardedReports(dir: string): { report: StrykerReport; count: number } {
341346
const root = path.isAbsolute(dir) ? dir : path.join(repoRoot, dir);
342347
// Shard artifacts also carry the lane envelope and the derived test scope, so
343348
// the report is selected by name rather than by "every .json here".
@@ -346,17 +351,8 @@ function readShardedReports(dir: string, expected: number | undefined): StrykerR
346351
.map((file) => path.join(root, file))
347352
.sort();
348353
if (files.length === 0) throw new Error(`No Stryker JSON reports found under ${dir}`);
349-
// Sub-sharded modules make "every module has mutants" too weak on its own: the
350-
// surviving slices would still cover the module, so the expected shard count is
351-
// asserted as well.
352-
if (expected !== undefined && files.length !== expected) {
353-
throw new Error(
354-
`Incomplete shard set from ${dir}: ${files.length} report(s), expected ${expected}. ` +
355-
'A shard job failed or its artifact is absent; the aggregate is not a sweep.',
356-
);
357-
}
358354
process.stdout.write(`mutation: merging ${files.length} shard report(s) from ${dir}\n`);
359-
return mergeReports(files.map(readReport));
355+
return { report: mergeReports(files.map(readReport)), count: files.length };
360356
}
361357

362358
async function produceReport(
@@ -369,15 +365,22 @@ async function produceReport(
369365
}
370366

371367
/**
372-
* A merged shard set must cover every requested module. `summarizeReport` scores
373-
* a module with no mutants as 0, so a matrix shard that died would otherwise be
374-
* aggregated into a `complete`/`pass` envelope reporting a 0% kernel as if the
375-
* sweep had happened.
368+
* A merged shard set is only a sweep if every expected shard reported and every
369+
* requested module has mutants — sub-sharded modules make the second check too
370+
* weak alone, since a surviving slice still covers its module. `summarizeReport`
371+
* scores an absent module as 0, so without this a dead shard would be published
372+
* as a `complete`/`pass` envelope claiming the sweep happened.
376373
*
377-
* Runs after the table is published: the kernels that did complete are the part
378-
* of the run still worth reading on a failed-shard day.
374+
* Both verdicts run after the table is published: the kernels that did complete
375+
* are the part of the run still worth reading on a failed-shard day.
379376
*/
380-
function assertShardsCoverModules(state: LaneState, dir: string): void {
377+
function assertCompleteSweep(state: LaneState, dir: string, shards: number, args: Args): void {
378+
if (args.expectShards !== undefined && shards !== args.expectShards) {
379+
throw new Error(
380+
`Incomplete shard set from ${dir}: ${shards} report(s), expected ${args.expectShards}. ` +
381+
'A shard job failed or its artifact is absent; the aggregate is not a sweep.',
382+
);
383+
}
381384
const missing = state.scores.filter((score) => score.total === 0).map((score) => score.module);
382385
if (missing.length === 0) return;
383386
throw new Error(
@@ -386,15 +389,18 @@ function assertShardsCoverModules(state: LaneState, dir: string): void {
386389
);
387390
}
388391

389-
/** Reads shard reports, an existing report, or runs Stryker — and scores them. */
390-
async function scoreModules(args: Args, state: LaneState): Promise<void> {
392+
/**
393+
* Reads shard reports, an existing report, or runs Stryker — and scores them.
394+
* Returns the shard count when the source was a shard directory.
395+
*/
396+
async function scoreModules(args: Args, state: LaneState): Promise<number | undefined> {
391397
state.stage = args.reportDir || args.report ? 'report' : 'stryker';
392-
const report = args.reportDir
393-
? readShardedReports(args.reportDir, args.expectShards)
394-
: await produceReport(state.modules, args.report, args.shard);
398+
const shards = args.reportDir ? readShardedReports(args.reportDir) : undefined;
399+
const report = shards?.report ?? (await produceReport(state.modules, args.report, args.shard));
395400

396401
state.stage = 'score';
397402
state.scores = summarizeReport(report, state.modules);
403+
return shards?.count;
398404
}
399405

400406
async function sweep(args: Args, state: LaneState): Promise<number> {
@@ -410,14 +416,14 @@ async function sweep(args: Args, state: LaneState): Promise<number> {
410416
return 0;
411417
}
412418

413-
await scoreModules(args, state);
419+
const shards = await scoreModules(args, state);
414420

415421
const title = args.affected
416422
? 'Mutation score — affected decision kernels'
417423
: 'Mutation score — decision kernels';
418424
emit(renderReport(state.scores, state.provenance, { title }), args.summary);
419425

420-
if (args.reportDir) assertShardsCoverModules(state, args.reportDir);
426+
if (args.reportDir) assertCompleteSweep(state, args.reportDir, shards ?? 0, args);
421427
state.stage = 'complete';
422428
return 0;
423429
}

0 commit comments

Comments
 (0)