Skip to content

Commit 8c1dbe7

Browse files
test: derive kernel test ownership and land the scheduled-lane health monitor
Ownership of a kernel's tests is now computed from the static import graph (scripts/mutation/ownership.ts) instead of a hand-listed set, so a test that reaches a kernel indirectly -- src/__tests__/daemon-error.test.ts through src/daemon.ts -- selects that kernel on a PR. The PR lane triggers on every src test and shards the derived modules, keeping wall clock at one module. The lane envelope (#1430) is now written on every exit path with the stage it reached, so a crash before any mutant runs is distinguishable from a lane that never ran. Adds the derived cadence monitor (scripts/lane-health, daily workflow): scheduled lanes are enumerated from .github/workflows/ and reported dark, failing, or never-run against their own cron cadence.
1 parent a5ad8ad commit 8c1dbe7

15 files changed

Lines changed: 991 additions & 183 deletions

File tree

‎.fallowrc.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,13 @@
8888
"readTestScope"
8989
]
9090
},
91+
{
92+
"comment": "Mutation-lane completeness gate: ownedTestFiles enumerates the whole derived ownership map, which only the assertion in ownership.test.ts consumes — a production consumer would defeat the point.",
93+
"file": "scripts/mutation/ownership.ts",
94+
"exports": [
95+
"ownedTestFiles"
96+
]
97+
},
9198
{
9299
"comment": "Help-benchmark conformance seams: helpTopicIds feeds the topic-coverage gate and PRIVATE_AX_RECOVERY_SAMPLE feeds the skillgym suite; both consumers are test-tree files outside --production analysis.",
93100
"file": "{src/cli/parser/cli-help.ts,scripts/help-conformance-sample-outputs.mjs}",

‎.github/workflows/lane-health.yml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Lane Health
2+
3+
# Watches the watchers (issue #1430): every `schedule:`-triggered workflow is
4+
# checked for freshness and for two consecutive failures. Nightly and weekly lanes
5+
# can go dark for weeks — a disabled schedule, a 60-day inactivity suspension, a
6+
# rename — while PR CI stays green, so no other job can notice.
7+
#
8+
# The lane list is derived from .github/workflows/, not hand-maintained, so a lane
9+
# added in the same PR as its first run is already watched.
10+
11+
on:
12+
schedule:
13+
# Daily 06:00 UTC — after the nightlies and the weekly sweep have reported.
14+
- cron: "0 6 * * *"
15+
workflow_dispatch:
16+
17+
permissions:
18+
contents: read
19+
actions: read
20+
issues: write
21+
22+
concurrency:
23+
group: ci-${{ github.workflow }}
24+
cancel-in-progress: true
25+
26+
jobs:
27+
lane-health:
28+
name: Scheduled lane freshness
29+
runs-on: ubuntu-latest
30+
timeout-minutes: 10
31+
steps:
32+
- name: Checkout
33+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
34+
35+
- name: Setup toolchain
36+
uses: ./.github/actions/setup-node-pnpm
37+
38+
- name: Self-test
39+
run: pnpm lane-health:test
40+
41+
# Reports to the job summary and pings one tracking issue; it never fails
42+
# the build, because a second red lane nobody watches is not an alert.
43+
- name: Check scheduled lane freshness
44+
env:
45+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
46+
run: pnpm lane-health --alert

‎.github/workflows/mutation-affected.yml‎

Lines changed: 87 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,26 @@ name: Mutation Affected
88
# sweeps (mutation-weekly.yml). Until then it is a report with the same numbers,
99
# so the gate's first failing day is not also its first running day.
1010
#
11-
# Scope is the AFFECTED modules only (changed file in the kernel registry -> its
12-
# mutants). The weekly run stays the full sweep.
11+
# Scope is the AFFECTED modules only, and affectedness is DERIVED from the import
12+
# graph (scripts/mutation/ownership.ts): a kernel source, or any test that reaches
13+
# one. Reaching a kernel is a superset of killing its mutants, so the `select` job
14+
# frequently returns several modules — sharded like the weekly sweep so the PR's
15+
# wall clock is one module, not their sum. The weekly run stays the full sweep.
1316

1417
on:
1518
pull_request:
1619
paths:
17-
# Kernel sources and the tests whose strength they score, mirroring
18-
# `KERNEL_MODULES` (scripts/mutation/workflow.test.ts asserts the parity).
20+
# Kernel sources, every src test (ownership is derived, so any test may own
21+
# a kernel — `select` decides, not this filter), and the lane's own tooling.
22+
# scripts/mutation/workflow.test.ts asserts this covers the registry.
1923
- "src/kernel/errors.ts"
20-
- "src/kernel/__tests__/errors.test.ts"
2124
- "src/daemon/ref-frame.ts"
22-
- "src/daemon/__tests__/ref-frame.test.ts"
23-
- "src/daemon/handlers/__tests__/snapshot-scoped-refs.test.ts"
2425
- "src/commands/interaction/runtime/settle.ts"
25-
- "src/commands/interaction/runtime/settle.test.ts"
26-
- "src/daemon/handlers/__tests__/interaction-settle.test.ts"
2726
- "src/utils/scroll-edge-state.ts"
28-
- "src/commands/interaction/runtime/gestures.test.ts"
29-
- "src/core/__tests__/dispatch-scroll.test.ts"
3027
- "src/selectors/**"
28+
- "src/**/*.test.ts"
3129
- "scripts/mutation/**"
30+
- "scripts/lib/**"
3231
- "stryker.config.json"
3332
- "mutation-baselines/**"
3433
- ".github/workflows/mutation-affected.yml"
@@ -41,10 +40,12 @@ concurrency:
4140
cancel-in-progress: true
4241

4342
jobs:
44-
mutation-affected:
45-
name: Affected decision-kernel mutants
43+
select:
44+
name: Select affected kernels
4645
runs-on: ubuntu-latest
47-
timeout-minutes: 45
46+
timeout-minutes: 10
47+
outputs:
48+
modules: ${{ steps.select.outputs.modules }}
4849
steps:
4950
- name: Checkout
5051
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -57,16 +58,84 @@ jobs:
5758
- name: Ratchet self-test
5859
run: pnpm mutation:test
5960

60-
- name: Run affected mutants
61-
run: pnpm mutation:affected --base "origin/${{ github.event.pull_request.base.ref }}"
61+
- id: select
62+
name: Derive affected modules
63+
run: |
64+
modules=$(pnpm --silent mutation:affected --list-affected \
65+
--base "origin/${{ github.event.pull_request.base.ref }}" | tail -n1)
66+
echo "modules=$modules" >> "$GITHUB_OUTPUT"
67+
echo "Affected decision kernels: $modules" >> "$GITHUB_STEP_SUMMARY"
6268
63-
- name: Upload mutation report
69+
mutants:
70+
name: Mutants (${{ matrix.module }})
71+
needs: select
72+
if: needs.select.outputs.modules != '[]'
73+
runs-on: ubuntu-latest
74+
timeout-minutes: 45
75+
strategy:
76+
fail-fast: false
77+
matrix:
78+
module: ${{ fromJSON(needs.select.outputs.modules) }}
79+
steps:
80+
- name: Checkout
81+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
82+
83+
- name: Setup toolchain
84+
uses: ./.github/actions/setup-node-pnpm
85+
86+
- name: Run mutants for ${{ matrix.module }}
87+
run: pnpm mutation:run --modules ${{ matrix.module }}
88+
89+
- name: Upload shard report
6490
if: always()
6591
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
6692
with:
67-
name: mutation-affected
93+
name: mutation-affected-shard-${{ matrix.module }}
6894
path: |
6995
.tmp/mutation/mutation.json
7096
.tmp/mutation/mutation.html
7197
.tmp/mutation/lane-envelope.json
98+
if-no-files-found: warn
99+
100+
ratchet:
101+
name: Affected decision-kernel mutants
102+
needs: [select, mutants]
103+
if: always() && needs.select.result == 'success'
104+
runs-on: ubuntu-latest
105+
timeout-minutes: 15
106+
steps:
107+
- name: Checkout
108+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
109+
with:
110+
fetch-depth: 0
111+
112+
- name: Setup toolchain
113+
uses: ./.github/actions/setup-node-pnpm
114+
115+
- name: Download shard reports
116+
if: needs.select.outputs.modules != '[]'
117+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
118+
with:
119+
pattern: mutation-affected-shard-*
120+
path: .tmp/mutation/shards
121+
122+
# No shards means no affected kernel: run.ts reports "nothing to mutate"
123+
# and still writes the envelope, so the lane is never silently absent.
124+
- name: Ratchet the affected modules
125+
run: |
126+
if [ -d .tmp/mutation/shards ]; then
127+
pnpm mutation:check --report-dir .tmp/mutation/shards --affected \
128+
--base "origin/${{ github.event.pull_request.base.ref }}"
129+
else
130+
pnpm mutation:affected --base "origin/${{ github.event.pull_request.base.ref }}"
131+
fi
132+
133+
- name: Upload mutation report
134+
if: always()
135+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
136+
with:
137+
name: mutation-affected
138+
path: |
139+
.tmp/mutation/shards
140+
.tmp/mutation/lane-envelope.json
72141
if-no-files-found: ignore

‎docs/agents/testing.md‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ pnpm mutation:baseline # full sweep, then record it (reviewed c
141141
reported as a job summary plus an artifact. It never commits: the proposed baseline rides in the
142142
artifact, and applying it is a reviewed `pnpm mutation:baseline` commit, so a score cannot lower
143143
itself.
144-
- **PR lane** (`.github/workflows/mutation-affected.yml`) mutates only the modules the diff touches.
144+
- **PR lane** (`.github/workflows/mutation-affected.yml`) derives the affected modules
145+
(`--list-affected`), shards one job per module, and merges them into one verdict.
145146
- **Ratchet**: scores may only rise. `mutation-baselines/decision-kernels.json` records the
146147
high-water score per module plus the Stryker version and config content hash that produced it, so a
147148
score change caused by a tool/config change is reported as provenance drift, never as a
@@ -153,15 +154,35 @@ pnpm mutation:baseline # full sweep, then record it (reviewed c
153154
- **Test scope** is derived from Vitest's module graph (`vitest related` over the mutated files), the
154155
same delegation `pnpm check:affected` uses; see `scripts/mutation/test-scope.ts` for the three
155156
groups it drops and why dropping them cannot hide a surviving mutant.
156-
- **Kernel tests own their module too**: a diff that touches `src/kernel/__tests__/errors.test.ts` or
157-
`src/commands/interaction/runtime/settle.test.ts` selects that kernel's mutants, because weakening a
158-
test is exactly the change whose score must be re-measured. The registry's `tests` field lists them
159-
and `scripts/mutation/modules.test.ts` asserts each listed file actually reaches the kernel through
160-
the import graph.
157+
- **Test ownership is derived, never listed** (`scripts/mutation/ownership.ts`): a test owns every
158+
kernel its imports reach, so `src/__tests__/daemon-error.test.ts` selects `kernel-errors` through
159+
`src/daemon.ts` without naming it. A listed set of test files would silently omit exactly those
160+
indirect tests and rot as tests are added — weakening one would skip the ratchet. Reaching a kernel
161+
is a superset of killing its mutants, so the PR lane over-selects on purpose and shards the
162+
selected modules; a false positive costs runner minutes, a false negative costs the gate. Non-kernel
163+
*sources* are not owned: they can only move a score through those tests, and the weekly sweep
164+
re-measures the whole surface.
161165
- **Lane envelope** (`scripts/lib/lane-envelope.ts`, issue #1430): every run writes
162166
`.tmp/mutation/lane-envelope.json` — schema version, commit, Stryker version, config hash, seed
163-
(`null`; the input is enumerated, not randomized), duration, result, per-module scores — and both
164-
workflows upload it, so lane freshness and tool drift are readable without parsing logs.
167+
(`null`; the input is enumerated, not randomized), duration, result, stage, per-module scores — and
168+
both workflows upload it, so lane freshness and tool drift are readable without parsing logs. It is
169+
written on every exit path, including a crash before any mutant runs: an absent envelope would be
170+
indistinguishable from a lane that never ran.
171+
172+
## Scheduled lane health (#1430)
173+
174+
Nightly and weekly lanes can stop running — a disabled schedule, a rename, GitHub's 60-day inactivity
175+
suspension — while PR CI stays green, so nothing fails and nobody notices. `pnpm lane-health`
176+
(`.github/workflows/lane-health.yml`, daily) derives the lane list from `.github/workflows/` rather
177+
than a hand-maintained list, reads each lane's scheduled runs through `gh`, and reports:
178+
179+
- `dark` — two consecutive cadences missed (the cadence comes from the lane's own `cron`);
180+
- `failing` — two consecutive scheduled runs concluded non-success (cancellations excluded);
181+
- `never-run` — declared longer than two cadences ago and still never fired.
182+
183+
One miss is a runner hiccup; two is the lane not doing its job. Unhealthy lanes ping a single tracking
184+
issue (`--alert`) instead of opening one per run, and the job never fails — a second red lane nobody
185+
watches is not an alert. `pnpm lane-health:test` covers the model.
165186

166187
## Live web smoke
167188

‎package.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,11 @@
113113
"mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run",
114114
"mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected",
115115
"mutation:test": "node --experimental-strip-types --test scripts/mutation/*.test.ts",
116+
"lane-health": "node --experimental-strip-types scripts/lane-health/run.ts",
117+
"lane-health:test": "node --experimental-strip-types --test scripts/lane-health/*.test.ts",
116118
"lint": "oxlint . --deny-warnings",
117-
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/mutation scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
118-
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/mutation scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
119+
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/mutation scripts/lane-health scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
120+
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/mutation scripts/lane-health scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
119121
"fallow": "fallow audit --base origin/main",
120122
"fallow:all": "fallow --summary",
121123
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",

0 commit comments

Comments
 (0)