Skip to content

Commit 056d362

Browse files
test: ratchet mutation score over enumerated decision kernels
Adds a Stryker (vitest runner) mutation lane scoped to the decision kernels, a per-module baseline with tool/config provenance, and a ratchet that only lets scores rise. Non-gating until two consecutive stable weekly sweeps. Refs #1415 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 4c02b6a commit 056d362

23 files changed

Lines changed: 3063 additions & 36 deletions

‎.fallowrc.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
},
6767
{
6868
"comment": "Tool config default exports, loaded by the tool rather than imported.",
69-
"file": "{tsdown.config.ts,website/rspress.config.ts,test/skillgym/skillgym.config.ts,test/skillgym/suites/*.ts}",
69+
"file": "{tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts,test/skillgym/skillgym.config.ts,test/skillgym/suites/*.ts}",
7070
"exports": [
7171
"default"
7272
]
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Mutation Affected
2+
3+
# PR-side half of the decision-kernel mutation lane (issue #1415).
4+
#
5+
# Graduation, not a flag day: this job runs on every PR that touches a kernel
6+
# module, but `scripts/mutation/run.ts` only exits non-zero once the committed
7+
# baseline reports `gating: true` — earned by two consecutive stable weekly
8+
# sweeps (mutation-weekly.yml). Until then it is a report with the same numbers,
9+
# so the gate's first failing day is not also its first running day.
10+
#
11+
# Scope is the AFFECTED modules only (changed file in the kernel registry -> its
12+
# mutants). The weekly run stays the full sweep.
13+
14+
on:
15+
pull_request:
16+
paths:
17+
- "src/kernel/errors.ts"
18+
- "src/daemon/ref-frame.ts"
19+
- "src/commands/interaction/runtime/settle.ts"
20+
- "src/utils/scroll-edge-state.ts"
21+
- "src/selectors/**"
22+
- "scripts/mutation/**"
23+
- "stryker.config.json"
24+
- "mutation-baselines/**"
25+
- ".github/workflows/mutation-affected.yml"
26+
27+
permissions:
28+
contents: read
29+
30+
concurrency:
31+
group: ci-${{ github.workflow }}-${{ github.ref }}
32+
cancel-in-progress: true
33+
34+
jobs:
35+
mutation-affected:
36+
name: Affected decision-kernel mutants
37+
runs-on: ubuntu-latest
38+
timeout-minutes: 45
39+
steps:
40+
- name: Checkout
41+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
42+
with:
43+
fetch-depth: 0
44+
45+
- name: Setup toolchain
46+
uses: ./.github/actions/setup-node-pnpm
47+
48+
- name: Ratchet self-test
49+
run: pnpm mutation:test
50+
51+
- name: Run affected mutants
52+
run: pnpm mutation:affected --base "origin/${{ github.event.pull_request.base.ref }}"
53+
54+
- name: Upload mutation report
55+
if: always()
56+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
57+
with:
58+
name: mutation-affected
59+
path: |
60+
.tmp/mutation/mutation.json
61+
.tmp/mutation/mutation.html
62+
if-no-files-found: ignore
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: Mutation Weekly
2+
3+
# Weekly mutation sweep over the enumerated decision kernels (issue #1415).
4+
# Mutation score is the mechanical answer to "is this test load-bearing or
5+
# decorative"; a full-suite sweep is unaffordable, so the scope is the kernel
6+
# registry in scripts/mutation/modules.ts and nothing else.
7+
#
8+
# Sharded one job per module: the whole sweep is ~2,150 mutants, and the selector
9+
# module alone is ~1,280 of them, so a single job would sit near the 30-minute
10+
# acceptance budget on an ubuntu runner. The shards' JSON reports are merged into
11+
# one verdict by the ratchet job (`--report-dir`), so the ratchet still sees a
12+
# full sweep.
13+
#
14+
# The lane reports; it does not commit. The proposed baseline rides in the
15+
# artifact and applying it is a reviewed `pnpm mutation:baseline` commit — a score
16+
# can never lower itself. Gating (after two consecutive stable weekly runs) is
17+
# enforced on PRs by mutation-affected.yml.
18+
19+
on:
20+
schedule:
21+
# Sundays 05:00 UTC — after the nightly lanes, before the working week.
22+
- cron: "0 5 * * 0"
23+
workflow_dispatch:
24+
25+
permissions:
26+
contents: read
27+
28+
concurrency:
29+
group: ci-${{ github.workflow }}-${{ github.ref }}
30+
cancel-in-progress: true
31+
32+
jobs:
33+
shard:
34+
name: Mutants (${{ matrix.module }})
35+
runs-on: ubuntu-latest
36+
# Budget is 30 minutes per shard; the headroom covers checkout/install so a
37+
# slow runner still reports instead of being killed mid-sweep.
38+
timeout-minutes: 45
39+
strategy:
40+
fail-fast: false
41+
matrix:
42+
# Kept in step with KERNEL_MODULES by the shard-coverage assertion in
43+
# scripts/mutation/workflow.test.ts.
44+
module:
45+
- kernel-errors
46+
- daemon-ref-frame
47+
- interaction-settle
48+
- scroll-edge-state
49+
- selectors
50+
steps:
51+
- name: Checkout
52+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
53+
54+
- name: Setup toolchain
55+
uses: ./.github/actions/setup-node-pnpm
56+
57+
- name: Run mutants for ${{ matrix.module }}
58+
run: pnpm mutation:run --modules ${{ matrix.module }}
59+
60+
- name: Upload shard report
61+
if: always()
62+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
63+
with:
64+
name: mutation-shard-${{ matrix.module }}
65+
path: .tmp/mutation/mutation.json
66+
if-no-files-found: warn
67+
68+
ratchet:
69+
name: Mutation ratchet
70+
runs-on: ubuntu-latest
71+
needs: shard
72+
if: always()
73+
timeout-minutes: 15
74+
steps:
75+
- name: Checkout
76+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
77+
78+
- name: Setup toolchain
79+
uses: ./.github/actions/setup-node-pnpm
80+
81+
- name: Ratchet self-test
82+
run: pnpm mutation:test
83+
84+
- name: Download shard reports
85+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
86+
with:
87+
pattern: mutation-shard-*
88+
path: .tmp/mutation/shards
89+
90+
# run.ts writes the markdown verdict to $GITHUB_STEP_SUMMARY when the
91+
# runner exports it, so the summary and the artifact carry the same numbers.
92+
- name: Ratchet the merged sweep and propose the next baseline
93+
run: |
94+
pnpm mutation:check --report-dir .tmp/mutation/shards --update
95+
cp mutation-baselines/decision-kernels.json .tmp/mutation/proposed-baseline.json
96+
git checkout -- mutation-baselines/decision-kernels.json
97+
98+
- name: Upload mutation report
99+
if: always()
100+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
101+
with:
102+
name: mutation-decision-kernels
103+
path: |
104+
.tmp/mutation/shards
105+
.tmp/mutation/proposed-baseline.json
106+
if-no-files-found: warn

‎docs/agents/testing.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ The mapping it encodes, for when you need to run a gate directly or reason about
2020
| Help benchmark cases (`scripts/help-conformance-*.mjs`) | `pnpm exec vitest run scripts/__tests__` (deterministic gates); model-backed: `pnpm bench:help-conformance` (paid LLM calls, local only) |
2121
| SkillGym prompts/assertions | `pnpm test:skillgym:case <case-id>` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) — agentic routing + local-help-consumption proof only; command-planning knowledge checks belong in the help bench |
2222
| Anything in `src/`, `test/`, `skills/` | `pnpm format` |
23+
| A decision kernel or its tests (`src/kernel/errors.ts`, `src/daemon/ref-frame.ts`, `src/commands/interaction/runtime/settle.ts`, `src/utils/scroll-edge-state.ts`, `src/selectors/`) | `pnpm mutation:affected --base origin/main` (minutes; GitHub runs it per PR — see the mutation ratchet section) |
2324

2425
Two traps worth naming:
2526

@@ -119,6 +120,40 @@ The plan documents the rule and changed path behind every selected check.
119120
Model and catalog live under `scripts/check-affected/`; the derivation is guarded
120121
by `pnpm check:affected:test` (the `Affected-check Selector` CI job).
121122

123+
## Mutation ratchet over decision kernels
124+
125+
Mutation score is the mechanical answer to "is this test load-bearing or decorative". A full-suite
126+
sweep is unaffordable, so the scope is an enumerated list of pure decision kernels — modules where a
127+
surviving mutant means a silently wrong agent-facing decision. The registry
128+
(`scripts/mutation/modules.ts`) is the single source of truth: `stryker.config.json`'s `mutate` globs
129+
are asserted against it, and PR-affected selection maps changed files through it. Modules that spawn
130+
subprocesses or wait real time stay out by construction.
131+
132+
```sh
133+
pnpm mutation:test # ratchet self-test (fast, no Stryker)
134+
pnpm mutation:run --modules selectors # one module locally (~7 min for selectors)
135+
pnpm mutation:check # ratchet an existing .tmp/mutation/mutation.json
136+
pnpm mutation:baseline # full sweep, then record it (reviewed commit)
137+
```
138+
139+
- **Weekly full sweep** (`.github/workflows/mutation-weekly.yml`) shards one job per module so no job
140+
approaches the 30-minute budget, then merges the shard reports (`--report-dir`) for one verdict,
141+
reported as a job summary plus an artifact. It never commits: the proposed baseline rides in the
142+
artifact, and applying it is a reviewed `pnpm mutation:baseline` commit, so a score cannot lower
143+
itself.
144+
- **PR lane** (`.github/workflows/mutation-affected.yml`) mutates only the modules the diff touches.
145+
- **Ratchet**: scores may only rise. `mutation-baselines/decision-kernels.json` records the
146+
high-water score per module plus the Stryker version and config content hash that produced it, so a
147+
score change caused by a tool/config change is reported as provenance drift, never as a
148+
test-strength regression.
149+
- **Graduation, not a flag day**: gating is off until two consecutive comparable weekly sweeps pass
150+
(`stableRuns`/`requiredStableRuns` in the baseline); the PR job runs the whole time and only starts
151+
failing once the committed baseline says `gating: true`. A regression or provenance drift resets the
152+
counter.
153+
- **Test scope** is derived from Vitest's module graph (`vitest related` over the mutated files), the
154+
same delegation `pnpm check:affected` uses; see `scripts/mutation/test-scope.ts` for the three
155+
groups it drops and why dropping them cannot hide a surviving mutant.
156+
122157
## Live web smoke
123158

124159
The live web platform smoke runs the public built CLI against a local fixture page through the managed web backend:
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"schemaVersion": 1,
3+
"requiredStableRuns": 2,
4+
"stableRuns": 0,
5+
"gating": false,
6+
"modules": {
7+
"kernel-errors": {
8+
"score": 55.19,
9+
"killed": 101,
10+
"survived": 82,
11+
"total": 183,
12+
"strykerVersion": "9.6.1",
13+
"configHash": "sha256:806d9f2e657f",
14+
"updatedAt": "2026-07-27T14:29:40.750Z"
15+
},
16+
"daemon-ref-frame": {
17+
"score": 100,
18+
"killed": 55,
19+
"survived": 0,
20+
"total": 55,
21+
"strykerVersion": "9.6.1",
22+
"configHash": "sha256:806d9f2e657f",
23+
"updatedAt": "2026-07-27T14:29:40.750Z"
24+
},
25+
"interaction-settle": {
26+
"score": 68.95,
27+
"killed": 151,
28+
"survived": 68,
29+
"total": 219,
30+
"strykerVersion": "9.6.1",
31+
"configHash": "sha256:806d9f2e657f",
32+
"updatedAt": "2026-07-27T14:29:40.750Z"
33+
},
34+
"scroll-edge-state": {
35+
"score": 28.75,
36+
"killed": 92,
37+
"survived": 228,
38+
"total": 320,
39+
"strykerVersion": "9.6.1",
40+
"configHash": "sha256:806d9f2e657f",
41+
"updatedAt": "2026-07-27T14:29:40.750Z"
42+
},
43+
"selectors": {
44+
"score": 70.56,
45+
"killed": 901,
46+
"survived": 376,
47+
"total": 1277,
48+
"strykerVersion": "9.6.1",
49+
"configHash": "sha256:806d9f2e657f",
50+
"updatedAt": "2026-07-27T14:29:40.750Z"
51+
}
52+
}
53+
}

‎package.json‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@
9999
"build:macos-helper": "swift build -c release --package-path apple/macos-helper",
100100
"build:all": "pnpm build:node && pnpm build:xcuitest",
101101
"ad": "node bin/agent-device.mjs",
102-
"bench:help-conformance": "node scripts/help-conformance-bench.mjs",
102+
"bench:help-conformance": "node scripts/mutation scripts/help-conformance-bench.mjs",
103103
"maestro:conformance": "node --experimental-strip-types --test scripts/maestro-conformance/verify.test.ts scripts/maestro-conformance/differential/run.test.ts scripts/maestro-conformance/differential/invariants.test.ts",
104104
"maestro:conformance:regenerate": "node scripts/maestro-conformance/regenerate.mjs",
105105
"maestro:conformance:differential": "node --experimental-strip-types scripts/maestro-conformance/differential/run.ts",
@@ -108,9 +108,14 @@
108108
"perf": "node --experimental-strip-types scripts/perf/run.ts",
109109
"perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios",
110110
"perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android",
111+
"mutation:run": "node --experimental-strip-types scripts/mutation/run.ts",
112+
"mutation:baseline": "node --experimental-strip-types scripts/mutation/run.ts --update",
113+
"mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run",
114+
"mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected",
115+
"mutation:test": "node --experimental-strip-types --test scripts/mutation/*.test.ts",
111116
"lint": "oxlint . --deny-warnings",
112-
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
113-
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
117+
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/mutation scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
118+
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/mutation scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
114119
"fallow": "fallow audit --base origin/main",
115120
"fallow:all": "fallow --summary",
116121
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",
@@ -228,6 +233,8 @@
228233
},
229234
"devDependencies": {
230235
"@chenglou/freerange": "^0.0.1",
236+
"@stryker-mutator/core": "9.6.1",
237+
"@stryker-mutator/vitest-runner": "9.6.1",
231238
"@types/node": "^22.19.21",
232239
"@vitest/coverage-v8": "4.1.8",
233240
"fallow": "^2.95.0",

0 commit comments

Comments
 (0)