Skip to content

Commit 0ea6905

Browse files
zoomote[bot]roomote
andauthored
[Improve] Make mutation warnings easier to review (#1619)
* fix(ci): deduplicate mutation annotations * test(ci): cover mutation result classification --------- Co-authored-by: Roomote <[email protected]>
1 parent 7cd8549 commit 0ea6905

2 files changed

Lines changed: 109 additions & 16 deletions

File tree

‎scripts/stryker-diff.mjs‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -460,26 +460,36 @@ export function formatAdvisoryCommand(advisory) {
460460

461461
export function formatAnnotations(blockingMutants, packageRoot, state = { total: 0, perFile: new Map() }) {
462462
const annotations = []
463+
const mutantsByLocation = new Map()
463464

464-
for (const mutant of blockingMutants.sort((left, right) => {
465+
for (const mutant of [...blockingMutants].sort((left, right) => {
465466
const pathOrder = left.filePath.localeCompare(right.filePath)
466467
return pathOrder || left.location.start.line - right.location.start.line
467468
})) {
468469
const repositoryPath = path.posix.join(packageRoot, mutant.filePath.replaceAll("\\", "/"))
469470
const key = `${repositoryPath}:${mutant.location.start.line}`
471+
const group = mutantsByLocation.get(key) ?? { repositoryPath, mutants: [] }
472+
group.mutants.push(mutant)
473+
mutantsByLocation.set(key, group)
474+
}
475+
476+
for (const [key, { repositoryPath, mutants }] of mutantsByLocation) {
470477
const fileCount = state.perFile.get(repositoryPath) ?? 0
471-
if (annotations.some((annotation) => annotation.key === key) || fileCount >= 7 || state.total >= 20) continue
478+
if (fileCount >= 7 || state.total >= 20) continue
472479

480+
const mutant = mutants[0]
473481
const replacement = String(mutant.replacement ?? "")
474482
.replace(/\s+/g, " ")
475483
.trim()
476484
.slice(0, 160)
485+
const location = `${repositoryPath}:${mutant.location.start.line}`
486+
const detail = `${mutant.status} ${mutant.mutatorName} mutant${replacement ? ` (replacement: ${replacement})` : ""}`
477487
annotations.push({
478488
key,
479489
file: repositoryPath,
480490
line: mutant.location.start.line,
481491
message:
482-
`${mutant.status} ${mutant.mutatorName} mutant${replacement ? ` (replacement: ${replacement})` : ""}. ` +
492+
`${location}: ${mutants.length === 1 ? detail : `${mutants.length} mutation test gaps; example: ${detail}`}. ` +
483493
"See the job summary for the complete list and resolution guidance.",
484494
})
485495
state.perFile.set(repositoryPath, fileCount + 1)
@@ -652,16 +662,18 @@ export function evaluateReport(report, packageEntry) {
652662
"The result is inconclusive; consider fixing flaky or slow tests, or reducing the changed scope.",
653663
)
654664
}
655-
if (counts.blocking.length > 0) {
656-
advisories.push(
657-
`${packageEntry.id} has ${counts.survived} surviving and ${counts.noCoverage} uncovered changed-code mutants. ` +
658-
"Consider adding or strengthening focused tests.",
659-
)
660-
}
661665
return { ...counts, advisories }
662666
}
663667

664-
export function runManifest(repoRoot, manifest, reportRoot) {
668+
export function runManifest(
669+
repoRoot,
670+
manifest,
671+
reportRoot,
672+
{
673+
runMutation = runStryker,
674+
readMutationReport = (reportPath) => JSON.parse(fs.readFileSync(reportPath, "utf8")),
675+
} = {},
676+
) {
665677
const rows = []
666678
const advisories = [...(manifest.advisories ?? [])]
667679
const annotationState = { total: 0, perFile: new Map() }
@@ -677,7 +689,7 @@ export function runManifest(repoRoot, manifest, reportRoot) {
677689
if (packageEntry.discoverRelatedTests) {
678690
packageEntry.testFiles = discoverRelatedTestFiles(repoRoot, packageEntry, reportDirectory)
679691
}
680-
const preflightOutput = stripAnsi(runStryker(repoRoot, packageEntry, reportRoot, true))
692+
const preflightOutput = stripAnsi(runMutation(repoRoot, packageEntry, reportRoot, true))
681693
const mutantMatch = /Instrumented \d+ source file\(s\) with (\d+) mutant\(s\)/.exec(preflightOutput)
682694
if (!mutantMatch) throw new Error(`${packageEntry.id} preflight did not report a mutant count`)
683695
const generatedMutants = Number(mutantMatch[1])
@@ -707,9 +719,9 @@ export function runManifest(repoRoot, manifest, reportRoot) {
707719
continue
708720
}
709721

710-
runStryker(repoRoot, packageEntry, reportRoot, false)
722+
runMutation(repoRoot, packageEntry, reportRoot, false)
711723
const jsonReportPath = path.join(reportRoot, packageEntry.id, "mutation.json")
712-
const report = JSON.parse(fs.readFileSync(jsonReportPath, "utf8"))
724+
const report = readMutationReport(jsonReportPath)
713725
packageEntry.testFiles = testsFromMutationReport(report, packageEntry.testFiles)
714726
counts = evaluateReport(report, packageEntry)
715727
for (const annotation of formatAnnotations(
@@ -729,7 +741,7 @@ export function runManifest(repoRoot, manifest, reportRoot) {
729741
reportPath,
730742
changedLines: packageEntry.changedExecutableLines,
731743
...counts,
732-
result: counts.advisories.length > 0 ? "Advisory findings" : "Passed",
744+
result: counts.blocking.length > 0 || counts.advisories.length > 0 ? "Advisory findings" : "Passed",
733745
})
734746
} catch (error) {
735747
advisories.push(error.message)

‎scripts/stryker-diff.test.mjs‎

Lines changed: 83 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -529,6 +529,43 @@ describe("failure output", () => {
529529
for (const mutant of manyMutants) assert.ok(grouped.includes(mutant.mutatorName))
530530
})
531531

532+
it("emits one distinguishable annotation per source location", () => {
533+
const mutants = [
534+
blocking[2],
535+
blocking[1],
536+
{
537+
filePath: "utils/other.ts",
538+
status: "NoCoverage",
539+
mutatorName: "ConditionalExpression",
540+
replacement: "true",
541+
location: { start: { line: 9 } },
542+
},
543+
blocking[0],
544+
]
545+
const originalOrder = [...mutants]
546+
const annotations = formatAnnotations(mutants, "src")
547+
548+
assert.equal(annotations.length, 2)
549+
assert.deepEqual(mutants, originalOrder)
550+
assert.match(
551+
annotations[0].message,
552+
/^src\/core\/value\.ts:4: 2 mutation test gaps; example: NoCoverage StringLiteral mutant \(replacement: "left \| right"\)/,
553+
)
554+
assert.match(
555+
annotations[1].message,
556+
/^src\/utils\/other\.ts:9: 2 mutation test gaps; example: Survived BooleanLiteral mutant \(replacement: false\)/,
557+
)
558+
})
559+
560+
it("prefixes singleton annotations with their source location", () => {
561+
const [annotation] = formatAnnotations([blocking[2]], "src")
562+
563+
assert.equal(
564+
annotation.message,
565+
"src/utils/other.ts:9: Survived BooleanLiteral mutant (replacement: false). See the job summary for the complete list and resolution guidance.",
566+
)
567+
})
568+
532569
it("shares annotation limits across packages", () => {
533570
const state = { total: 0, perFile: new Map() }
534571
const first = formatAnnotations(
@@ -605,6 +642,50 @@ describe("failure output", () => {
605642
}
606643
})
607644

645+
it("classifies successful package rows from blocking mutants", () => {
646+
const repo = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-success-"))
647+
const packageEntry = {
648+
id: "core",
649+
root: "packages/core",
650+
vitestConfig: "vitest.unit.config.ts",
651+
selectors: ["src/value.ts:1-1"],
652+
changedExecutableLines: 1,
653+
}
654+
const execute = (report) =>
655+
runManifest(repo, { packages: [{ ...packageEntry }] }, path.join(repo, "reports"), {
656+
runMutation: (_repoRoot, _entry, _reportRoot, dryRunOnly) =>
657+
dryRunOnly ? "Instrumented 1 source file(s) with 1 mutant(s)" : "",
658+
readMutationReport: () => report,
659+
})[0]
660+
661+
try {
662+
const advisoryRow = execute({
663+
files: {
664+
"src/value.ts": {
665+
mutants: [
666+
{
667+
status: "Survived",
668+
mutatorName: "BooleanLiteral",
669+
replacement: "false",
670+
location: { start: { line: 1 } },
671+
},
672+
],
673+
},
674+
},
675+
})
676+
assert.equal(advisoryRow.result, "Advisory findings")
677+
assert.deepEqual(advisoryRow.advisories, [])
678+
679+
const passedRow = execute({
680+
files: { "src/value.ts": { mutants: [{ status: "Killed", location: { start: { line: 1 } } }] } },
681+
})
682+
assert.equal(passedRow.result, "Passed")
683+
assert.deepEqual(passedRow.advisories, [])
684+
} finally {
685+
fs.rmSync(repo, { recursive: true, force: true })
686+
}
687+
})
688+
608689
it("does not fail when the GitHub job summary cannot be written", () => {
609690
const previousSummary = process.env.GITHUB_STEP_SUMMARY
610691
const summaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "stryker-summary-"))
@@ -681,7 +762,7 @@ describe("failure output", () => {
681762
describe("report evaluation", () => {
682763
const packageEntry = { id: "core", root: "packages/core" }
683764

684-
it("reports surviving and uncovered changed-code mutants as advisory", () => {
765+
it("reports surviving and uncovered mutants through detailed annotations without a redundant aggregate", () => {
685766
const report = {
686767
files: {
687768
"src/value.ts": {
@@ -704,7 +785,7 @@ describe("report evaluation", () => {
704785
}
705786

706787
const result = evaluateReport(report, packageEntry)
707-
assert.match(result.advisories.join("\n"), /1 surviving and 1 uncovered/)
788+
assert.deepEqual(result.advisories, [])
708789
assert.equal(formatAnnotations(mutantCounts(report).blocking, packageEntry.root).length, 2)
709790
})
710791

0 commit comments

Comments
 (0)