Skip to content

Commit 1833535

Browse files
committed
fix(ci): pass the stacked map as a file and keep a failed reinstall visible
The open-pull-request map crossed jobs as a job output and then as one command-line argument. Linux caps a single argument or environment string at 128 KiB (MAX_ARG_STRLEN), so a map past that cap makes the gate step unstartable instead of merely large, and the parse fallback never runs. The map job now writes a validated file and publishes it as an artifact; the gate reads --stacked-map-file. A missing artifact degrades to the event base, which charges the unit every unmerged ancestor rather than silently measuring less. A stacked parent is now matched by head SHA and base branch in the same predicate. Two open pull requests can share one head commit while targeting different branches; stopping at the first SHA match rejected the parent for the wrong reason and charged the unit the whole chain. A failed aligned dependency reinstall now writes the job summary with the stacked base before rethrowing, so an author can see which base the gate chose.
1 parent 780eec4 commit 1833535

3 files changed

Lines changed: 214 additions & 30 deletions

File tree

‎.github/workflows/mutation-testing.yml‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ jobs:
2626
permissions:
2727
contents: read
2828
pull-requests: read
29-
outputs:
30-
stacked_map: ${{ steps.read_map.outputs.stacked_map }}
29+
# The map is published as an artifact for the gate job, which is the only artifact this
30+
# job writes; it never sees pull-request code because this job checks out nothing.
31+
actions: write
3132
steps:
3233
- name: Read the open pull request heads
3334
id: read_map
@@ -42,24 +43,36 @@ jobs:
4243
# The base ref travels with every entry: the gate only measures a unit against a parent
4344
# that targets the branch the unit itself targets.
4445
STACKED_MAP="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls?state=open&per_page=100" --jq '[.[] | {number: .number, headSha: .head.sha, draft: (.draft == true), baseRef: .base.ref}]')" || STACKED_MAP="$(printf '[]')"
45-
# Publish the whole value as one record from a validated string, and let a failed
46-
# write abort the job: a truncated or skipped output would leave the gate measuring
47-
# against the event base while still reporting success.
46+
# Publish the map as a file in an artifact rather than as a job output that the gate
47+
# step then receives as an argument: Linux caps one argument or environment string at
48+
# 128 KiB (MAX_ARG_STRLEN), and a map past that cap makes the gate step unstartable
49+
# instead of merely large. A failed write aborts this job: a truncated or missing map
50+
# would leave the gate measuring against the event base while still reporting success.
4851
case "$STACKED_MAP" in
4952
"["*) ;;
5053
*) echo "::error title=Stacked unit base::The stacked map is not a JSON array" >&2; exit 1 ;;
5154
esac
52-
printf 'stacked_map<<EOF\n%s\nEOF\n' "$STACKED_MAP" >> "$GITHUB_OUTPUT"
55+
printf '%s\n' "$STACKED_MAP" > stacked-map.json
56+
57+
- name: Publish the stacked map
58+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
59+
with:
60+
name: mutation-stacked-map
61+
path: stacked-map.json
62+
if-no-files-found: error
63+
retention-days: 1
5364

5465
mutation-diff:
5566
name: mutation-diff
5667
needs: stacked_map
5768
if: github.event_name == 'merge_group' || github.event.pull_request.draft == false
5869
runs-on: ubuntu-latest
5970
timeout-minutes: 30
60-
env:
61-
# Only the filtered map crosses into this job; the token stays in the job that read it.
62-
STACKED_MAP: ${{ needs.stacked_map.outputs.stacked_map }}
71+
# Only the filtered map crosses into this job, and it crosses as a file: the token stays in the
72+
# job that read it. actions:write covers reading that artifact and publishing the reports below.
73+
permissions:
74+
contents: read
75+
actions: write
6376
steps:
6477
- name: Record merge-queue enforcement
6578
if: github.event_name == 'merge_group'
@@ -87,6 +100,16 @@ jobs:
87100
if: github.event_name == 'pull_request'
88101
run: pnpm test:mutation-ci
89102

103+
- name: Read the stacked map
104+
if: github.event_name == 'pull_request'
105+
# A missing artifact degrades to the event base, which charges this unit every unmerged
106+
# ancestor - stricter than the intended measurement, never looser - and the gate logs it.
107+
continue-on-error: true
108+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
109+
with:
110+
name: mutation-stacked-map
111+
path: .mutation
112+
90113
- name: Enforce executable-line scope and run advisory mutation testing
91114
if: github.event_name == 'pull_request'
92115
env:
@@ -98,7 +121,7 @@ jobs:
98121
# parent is another open pull request's head, this PR is a stacked unit, so the gate
99122
# measures only this unit's delta instead of every unmerged ancestor.
100123
PR_HEAD_SHA="$(git rev-parse "$HEAD_SHA^2" 2>/dev/null || git rev-parse "$HEAD_SHA")"
101-
node scripts/stryker-diff.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA" --pr-head "$PR_HEAD_SHA" --pr-base-ref "$PR_BASE_REF" --stacked-map "$STACKED_MAP"
124+
node scripts/stryker-diff.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA" --pr-head "$PR_HEAD_SHA" --pr-base-ref "$PR_BASE_REF" --stacked-map-file .mutation/stacked-map.json
102125
103126
- name: Upload mutation reports
104127
id: mutation_report

‎scripts/stryker-diff.mjs‎

Lines changed: 64 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -313,15 +313,22 @@ export function resolveStackedUnitBase(repoRoot, eventBaseSha, prHeadSha, openPu
313313
// request, not here - no branch ruleset requires the mutation-diff check upstream, so requiring a
314314
// green parent would turn an advisory measurement into a hard dependency between two units, and a
315315
// unit would be charged its parent's unrelated findings on top of its own.
316-
const parent = openPullRequests.find((pr) => pr.draft !== true && String(pr.headSha).toLowerCase() === parentSha)
316+
// The base-ref test belongs inside the find. Two open pull requests can share one head SHA while
317+
// targeting different branches (the same commit pushed to two branches), and stopping at the first
318+
// SHA match would fall back to the event base even when a later entry targets the branch this unit
319+
// lands on - charging the unit every unmerged ancestor of a branch it is not merging into.
320+
// A parent whose base is a feature branch is measured against that feature branch by its own run, so
321+
// measuring this unit against its head hides the parent's feature-branch delta from the changed-line
322+
// cap while merging this unit lands that delta on main. Without either base ref the comparison cannot
323+
// be made, and the unit keeps the event base.
324+
const parent = openPullRequests.find(
325+
(pr) =>
326+
pr.draft !== true &&
327+
String(pr.headSha).toLowerCase() === parentSha &&
328+
Boolean(childBaseRef) &&
329+
(typeof pr.baseRef === "string" ? pr.baseRef : "") === childBaseRef,
330+
)
317331
if (!parent) return { baseSha: eventBaseSha, stackedOn: null }
318-
// The gate measures a unit against its parent head only when the parent is a pull request against
319-
// the very branch this unit targets. A parent whose base is a feature branch would be measured
320-
// against that feature branch by its own run, so measuring this unit against its head hides the
321-
// parent's feature-branch delta from the changed-line cap while merging this unit lands that delta
322-
// on main. Without either base ref the comparison cannot be made, and the unit keeps the event base.
323-
const parentBaseRef = typeof parent.baseRef === "string" ? parent.baseRef : ""
324-
if (!childBaseRef || parentBaseRef !== childBaseRef) return { baseSha: eventBaseSha, stackedOn: null }
325332
return { baseSha: parentSha, stackedOn: parent.number }
326333
}
327334

@@ -794,6 +801,21 @@ export function appendSummary(rows, advisories, manifest) {
794801
}
795802
}
796803

804+
/**
805+
* A failed dependency reinstall aborts the gate before any manifest exists, so without this the only
806+
* trace of the run is a step log: the author cannot see which base the gate chose, which is the fact
807+
* that explains a stacked unit's numbers. The caller rethrows - this only makes the failure visible.
808+
*/
809+
export function summarizeReinstallFailure(stackedOn, driftedManifests, error) {
810+
appendSummary(
811+
[],
812+
[
813+
`Dependency reinstall failed for the aligned tree (${driftedManifests.join(", ")}): ${error.message} The gate measured nothing; fix the lockfile drift and re-run.`,
814+
],
815+
{ stackedOn },
816+
)
817+
}
818+
797819
export function evaluateReport(report, packageEntry) {
798820
const counts = mutantCounts(report)
799821
const advisories = []
@@ -924,6 +946,28 @@ function argument(name) {
924946
// emits one array per page, so the map can be a stream of arrays rather than one document; accept
925947
// both shapes. An unparsable map must not change the gate's base, so it degrades to the event base
926948
// instead of throwing.
949+
/**
950+
* The stacked map arrives either as an inline argument (local runs) or as a file written by the
951+
* stacked_map job. The file is the CI path on purpose: Linux caps one argument or environment string
952+
* (MAX_ARG_STRLEN, 128 KiB), so a map past that cap would make the gate step unstartable instead of
953+
* merely large. A map that cannot be read falls back to the event base, which charges this unit every
954+
* unmerged ancestor - stricter than the intended measurement, never looser - and says so in the log.
955+
*/
956+
export function readStackedMap(repoRoot, stackedMapArg, stackedMapFile) {
957+
if (!stackedMapFile) return stackedMapArg ?? ""
958+
const mapPath = path.resolve(repoRoot, stackedMapFile)
959+
try {
960+
return fs.readFileSync(mapPath, "utf8")
961+
} catch (error) {
962+
const reason =
963+
error instanceof Error && error.code === "ENOENT" ? "the artifact was not published" : error.message
964+
console.warn(
965+
`::warning title=Stacked unit base::Could not read the stacked map from ${stackedMapFile} (${reason}); measuring against the event base.`,
966+
)
967+
return ""
968+
}
969+
}
970+
927971
export function parseStackedMap(value) {
928972
if (!value) return []
929973
// gh api --paginate emits one JSON array per page, so the input is a stream of arrays rather
@@ -983,15 +1027,17 @@ function main() {
9831027
const command = process.argv[2]
9841028
if (command !== "ci")
9851029
throw new Error(
986-
"Usage: node scripts/stryker-diff.mjs ci --base <sha> --head <sha> [--pr-head <sha>] [--reports <path>] [--stacked-map <json>] [--pr-base-ref <ref>]",
1030+
"Usage: node scripts/stryker-diff.mjs ci --base <sha> --head <sha> [--pr-head <sha>] [--reports <path>] [--stacked-map <json> | --stacked-map-file <path>] [--pr-base-ref <ref>]",
9871031
)
9881032

9891033
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
9901034
const baseSha = argument("--base")
9911035
const headSha = argument("--head")
9921036
if (!baseSha || !headSha) throw new Error("--base and --head are required")
9931037

994-
const openPullRequests = parseStackedMap(argument("--stacked-map"))
1038+
const openPullRequests = parseStackedMap(
1039+
readStackedMap(repoRoot, argument("--stacked-map"), argument("--stacked-map-file")),
1040+
)
9951041
// --pr-head is the pull request head itself. The workflow runs on GitHub's merge commit, whose
9961042
// first parent is the base tip, so the merge commit cannot identify the stacked unit; its second
9971043
// parent can. A plain pull request keeps the merge commit as the diff head; a stacked unit
@@ -1010,7 +1056,14 @@ function main() {
10101056
`Aligned the working tree to the diff head ${invocation.diffHead.slice(0, 12)} so selection, source reads, and mutation run on the same tree.`,
10111057
)
10121058
const driftedManifests = dependencyManifestDrift(repoRoot, headSha, invocation.diffHead)
1013-
if (driftedManifests.length > 0) reinstallAlignedDependencies(repoRoot, driftedManifests)
1059+
if (driftedManifests.length > 0) {
1060+
try {
1061+
reinstallAlignedDependencies(repoRoot, driftedManifests)
1062+
} catch (error) {
1063+
summarizeReinstallFailure(invocation.stackedOn, driftedManifests, error)
1064+
throw error
1065+
}
1066+
}
10141067
}
10151068

10161069
const reportRoot = path.resolve(repoRoot, argument("--reports") ?? "reports/mutation")

‎scripts/stryker-diff.test.mjs‎

Lines changed: 117 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ import {
2626
parseVitestTestFiles,
2727
preferDirectTestFiles,
2828
parseStackedMap,
29+
readStackedMap,
30+
summarizeReinstallFailure,
2931
dependencyManifestDrift,
3032
alignExecutionTree,
3133
resolveCiInvocation,
@@ -67,7 +69,12 @@ describe("mutation testing workflow", () => {
6769
assert.ok(
6870
workflow.includes('PR_HEAD_SHA="$(git rev-parse "$HEAD_SHA^2" 2>/dev/null || git rev-parse "$HEAD_SHA")"'),
6971
)
70-
assert.ok(workflow.includes('--stacked-map "$STACKED_MAP"'))
72+
// The map crosses jobs as a file, never as an argument or an environment string: Linux caps one
73+
// argument/env string at 128 KiB (MAX_ARG_STRLEN), and a map past that cap makes the step
74+
// unstartable rather than merely large.
75+
assert.ok(workflow.includes("--stacked-map-file .mutation/stacked-map.json"))
76+
assert.ok(!workflow.includes('--stacked-map "$STACKED_MAP"'))
77+
assert.ok(workflow.includes("name: mutation-stacked-map"))
7178
assert.ok(workflow.includes('--pr-head "$PR_HEAD_SHA"'))
7279
// The open pull request map must cover every page, otherwise a parent unit beyond the first
7380
// page is missing and the gate charges the whole unmerged chain again.
@@ -97,20 +104,27 @@ describe("mutation testing workflow", () => {
97104
// in a job that never checks out or runs pull-request code; only the filtered map crosses over.
98105
const gateJob = workflow.slice(workflow.indexOf(" mutation-diff:"))
99106
assert.ok(!gateJob.includes("GH_TOKEN"))
100-
assert.ok(workflow.includes("STACKED_MAP: ${{ needs.stacked_map.outputs.stacked_map }}"))
107+
// The map arrives from the artifact the map job publishes, into the job that runs the gate.
108+
assert.ok(gateJob.includes("uses: actions/download-artifact@"))
109+
assert.ok(gateJob.includes("name: mutation-stacked-map"))
110+
assert.ok(!workflow.includes("needs.stacked_map.outputs.stacked_map"))
101111
const mapJob = workflow.slice(workflow.indexOf(" stacked_map:"), workflow.indexOf(" mutation-diff:"))
102112
assert.ok(mapJob.includes("GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}"))
103113
assert.ok(!mapJob.includes("actions/checkout"))
104114
assert.ok(!mapJob.includes("setup-node-pnpm"))
105115
assert.ok(!mapJob.includes("pnpm test:mutation-ci"))
106116
assert.ok(workflow.includes("needs: stacked_map"))
107-
// The job output is only real if the step it reads from carries that id; without it the map is
108-
// always an empty string and the gate silently falls back to the event base.
109-
assert.ok(mapJob.includes("id: read_map"))
110-
// Publish as one record from a validated value, and do not continue past a failed write.
111-
assert.ok(mapJob.includes("printf 'stacked_map<<EOF"))
112-
assert.ok(!mapJob.includes("Could not publish the stacked map"))
113-
assert.equal(workflow.match(/continue-on-error: true/g)?.length, 1)
117+
// The map is validated before it is written, and the write is a plain file in the job that
118+
// uploads it: no step output, no job output, and no argument for the gate to receive.
119+
assert.ok(mapJob.includes(`printf '%s\\n' "$STACKED_MAP" > stacked-map.json`))
120+
assert.ok(mapJob.includes("path: stacked-map.json"))
121+
assert.ok(mapJob.includes("if-no-files-found: error"))
122+
assert.ok(!mapJob.includes("GITHUB_OUTPUT"))
123+
// Both jobs need actions:write - one to publish the map, one to read it and publish the reports.
124+
assert.equal(workflow.match(/actions: write/g)?.length, 2)
125+
// A missing map artifact degrades to the event base inside the script, so the download step must
126+
// not fail the job; the report upload keeps its own tolerance.
127+
assert.equal(workflow.match(/continue-on-error: true/g)?.length, 2)
114128
assert.equal(workflow.match(/Could not write the job summary/g)?.length, 2)
115129
const script = fs.readFileSync(path.join(repositoryRoot, "scripts/stryker-diff.mjs"), "utf8")
116130
assert.ok(script.includes("appendSummary([], manifest.advisories, manifest)"))
@@ -457,6 +471,100 @@ describe("stacked unit base resolution", () => {
457471
}
458472
})
459473

474+
it("matches a stacked parent by base branch when another pull request shares its head", () => {
475+
// Two open pull requests can point at the same commit while targeting different branches. Stopping
476+
// at the first head-SHA match rejects that parent for the wrong reason and falls back to the event
477+
// base, charging this unit every unmerged ancestor of a branch it is not merging into.
478+
const { repository, eventBaseSha, parentSha, childSha } = createSyntheticStack()
479+
480+
try {
481+
const map = [
482+
{ number: 9, headSha: parentSha, baseRef: "feature/other" },
483+
{ number: 1, headSha: parentSha, baseRef: "main" },
484+
]
485+
486+
const unitOnMain = resolveStackedUnitBase(repository, eventBaseSha, childSha, map, "main")
487+
assert.equal(unitOnMain.baseSha, parentSha)
488+
assert.equal(unitOnMain.stackedOn, 1)
489+
490+
// The same map read by a unit that targets the feature branch picks the other pull request,
491+
// which is the only parent whose own gate measured against this unit's base.
492+
const unitOnFeature = resolveStackedUnitBase(repository, eventBaseSha, childSha, map, "feature/other")
493+
assert.equal(unitOnFeature.baseSha, parentSha)
494+
assert.equal(unitOnFeature.stackedOn, 9)
495+
496+
// A map whose entries match no base still degrades instead of guessing.
497+
const noMatch = resolveStackedUnitBase(repository, eventBaseSha, childSha, map, "release/none")
498+
assert.equal(noMatch.baseSha, eventBaseSha)
499+
assert.equal(noMatch.stackedOn, null)
500+
} finally {
501+
fs.rmSync(repository, { recursive: true, force: true })
502+
}
503+
})
504+
505+
it("reads the stacked map from the artifact file the workflow publishes", () => {
506+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "stacked-map-"))
507+
try {
508+
fs.writeFileSync(
509+
path.join(directory, "stacked-map.json"),
510+
'[{ "number": 1, "headSha": "' + "a".repeat(40) + '", "draft": false, "baseRef": "main" }]',
511+
)
512+
const fromFile = parseStackedMap(readStackedMap(directory, "[]", "stacked-map.json"))
513+
assert.deepEqual(fromFile, [{ number: 1, headSha: "a".repeat(40), draft: false, baseRef: "main" }])
514+
515+
// The inline form stays available for local runs, where no artifact exists.
516+
const inline = parseStackedMap(
517+
readStackedMap(
518+
directory,
519+
'[{ "number": 2, "headSha": "' + "b".repeat(40) + '", "draft": false, "baseRef": "main" }]',
520+
undefined,
521+
),
522+
)
523+
assert.deepEqual(inline, [{ number: 2, headSha: "b".repeat(40), draft: false, baseRef: "main" }])
524+
525+
// A missing artifact must not stop the gate. Falling back to the event base charges this unit
526+
// every unmerged ancestor - stricter than the intended measurement, never looser - and the
527+
// warning records which case ran.
528+
const warnings = []
529+
const warn = console.warn
530+
console.warn = (message) => warnings.push(message)
531+
try {
532+
assert.equal(readStackedMap(directory, '[{ "number": 3 }]', "absent.json"), "")
533+
} finally {
534+
console.warn = warn
535+
}
536+
assert.equal(warnings.length, 1)
537+
assert.ok(warnings[0].includes("the artifact was not published"))
538+
assert.deepEqual(parseStackedMap(""), [])
539+
} finally {
540+
fs.rmSync(directory, { recursive: true, force: true })
541+
}
542+
})
543+
544+
it("reports the stacked base in the job summary when the aligned reinstall fails", () => {
545+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "reinstall-summary-"))
546+
const summaryPath = path.join(directory, "step-summary.md")
547+
const previous = process.env.GITHUB_STEP_SUMMARY
548+
process.env.GITHUB_STEP_SUMMARY = summaryPath
549+
const warnings = []
550+
const warn = console.warn
551+
console.warn = (message) => warnings.push(message)
552+
try {
553+
summarizeReinstallFailure(7, ["src/package.json", "pnpm-lock.yaml"], new Error("pnpm install exited 1"))
554+
const summary = fs.readFileSync(summaryPath, "utf8")
555+
// The whole point of the summary here: which base this unit was measured against.
556+
assert.ok(summary.includes("Stacked unit: measured against the head of parent PR #7"))
557+
assert.ok(summary.includes("src/package.json, pnpm-lock.yaml"))
558+
assert.ok(summary.includes("pnpm install exited 1"))
559+
assert.equal(warnings.length, 1)
560+
} finally {
561+
console.warn = warn
562+
if (previous === undefined) delete process.env.GITHUB_STEP_SUMMARY
563+
else process.env.GITHUB_STEP_SUMMARY = previous
564+
fs.rmSync(directory, { recursive: true, force: true })
565+
}
566+
})
567+
460568
it("degrades to the event base for an unparsable stacked map", () => {
461569
assert.deepEqual(parseStackedMap("not json"), [])
462570
assert.deepEqual(parseStackedMap('{"number": 1}'), [])

0 commit comments

Comments
 (0)