Skip to content

Commit 058343a

Browse files
Merge origin/main into the parser fuzz lane, adopt #1430's shared envelope
main landed scripts/lib/lane-envelope.ts, which #1430 requires this lane to use: scripts/fuzz/envelope.ts is now only the mapping onto it, with the lane's own facts under `data` and a `stage` field carrying the crash case the shared two-value `result` cannot express. Conflicts were in package.json (format globs: keep scripts/fuzz alongside the new scripts/mutation + scripts/lib) and docs/agents/testing.md (both lanes' sections).
2 parents 5ec4233 + 8cce0ef commit 058343a

64 files changed

Lines changed: 5587 additions & 110 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.fallowrc.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
},
6767
{
6868
"comment": "Tool config default exports, loaded by the tool rather than imported.",
69-
"file": "{tsdown.config.ts,website/rspress.config.ts,test/skillgym/skillgym.config.ts,test/skillgym/suites/*.ts}",
69+
"file": "{tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts,test/skillgym/skillgym.config.ts,test/skillgym/suites/*.ts}",
7070
"exports": [
7171
"default"
7272
]
@@ -81,6 +81,20 @@
8181
"GatedKeysAreResolverKeys"
8282
]
8383
},
84+
{
85+
"comment": "Mutation-lane seam: readTestScope is consumed by vitest.mutation.config.ts, a tool config outside --production analysis.",
86+
"file": "scripts/mutation/test-scope.ts",
87+
"exports": [
88+
"readTestScope"
89+
]
90+
},
91+
{
92+
"comment": "Mutation-lane completeness gate: ownedTestFiles enumerates the whole derived ownership map, which only the assertion in ownership.test.ts consumes — a production consumer would defeat the point.",
93+
"file": "scripts/mutation/ownership.ts",
94+
"exports": [
95+
"ownedTestFiles"
96+
]
97+
},
8498
{
8599
"comment": "Help-benchmark conformance seams: helpTopicIds feeds the topic-coverage gate and PRIVATE_AX_RECOVERY_SAMPLE feeds the skillgym suite; both consumers are test-tree files outside --production analysis.",
86100
"file": "{src/cli/parser/cli-help.ts,scripts/help-conformance-sample-outputs.mjs}",

‎.github/workflows/ci.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,26 @@ jobs:
216216
- name: Check for production-unused exports
217217
run: pnpm check:production-exports
218218

219+
replay-compat-provenance:
220+
# The frozen replay-compat corpus (#1417) claims each entry was published by
221+
# a released tag. Only a full-history checkout can re-derive that claim, so
222+
# this job exists separately from the shallow-clone-safe unit lane.
223+
name: Replay-Compat Provenance
224+
runs-on: ubuntu-latest
225+
timeout-minutes: 10
226+
steps:
227+
- name: Checkout
228+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
229+
with:
230+
fetch-depth: 0
231+
fetch-tags: true
232+
233+
- name: Setup toolchain
234+
uses: ./.github/actions/setup-node-pnpm
235+
236+
- name: Verify corpus entries against their released blobs
237+
run: pnpm check:replay-compat
238+
219239
coverage:
220240
# Runs the full unit + provider-integration suites under coverage with
221241
# thresholds, so a separate unit-tests job would rerun the same tests.
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
name: Mutation Affected
2+
3+
# PR-side half of the decision-kernel mutation lane (issue #1415).
4+
#
5+
# Graduation, not a flag day: this job runs on every PR that touches a kernel
6+
# module, but `scripts/mutation/run.ts` only exits non-zero once the committed
7+
# baseline reports `gating: true` — earned by two consecutive stable weekly
8+
# sweeps (mutation-weekly.yml). Until then it is a report with the same numbers,
9+
# so the gate's first failing day is not also its first running day.
10+
#
11+
# Cost control: before graduation an affected run is a report nobody acts on, so
12+
# `select` returns an empty matrix and no mutants run — except when the diff
13+
# touches the lane's own tooling, the one case where a pre-graduation run buys
14+
# something (the gate has to be proven before it can bite).
15+
#
16+
# Scope is the AFFECTED modules only, and affectedness is DERIVED from the import
17+
# graph (scripts/mutation/ownership.ts): a kernel source, or any test that reaches
18+
# one. Reaching a kernel is a superset of killing its mutants, so the `select` job
19+
# frequently returns several modules — sharded like the weekly sweep so the PR's
20+
# wall clock is one module, not their sum. The weekly run stays the full sweep.
21+
22+
on:
23+
pull_request:
24+
paths:
25+
# Kernel sources, every src test (ownership is derived, so any test may own
26+
# a kernel — `select` decides, not this filter), and the lane's own tooling.
27+
# scripts/mutation/workflow.test.ts asserts this covers the registry.
28+
- "src/kernel/errors.ts"
29+
- "src/daemon/ref-frame.ts"
30+
- "src/commands/interaction/runtime/settle.ts"
31+
- "src/utils/scroll-edge-state.ts"
32+
- "src/selectors/**"
33+
- "src/**/*.test.ts"
34+
- "scripts/mutation/**"
35+
- "scripts/lib/**"
36+
- "stryker.config.json"
37+
- "mutation-baselines/**"
38+
- ".github/workflows/mutation-affected.yml"
39+
40+
permissions:
41+
contents: read
42+
43+
concurrency:
44+
group: ci-${{ github.workflow }}-${{ github.ref }}
45+
cancel-in-progress: true
46+
47+
jobs:
48+
select:
49+
name: Select affected kernels
50+
runs-on: ubuntu-latest
51+
timeout-minutes: 10
52+
outputs:
53+
modules: ${{ steps.select.outputs.modules }}
54+
steps:
55+
- name: Checkout
56+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
57+
with:
58+
fetch-depth: 0
59+
60+
- name: Setup toolchain
61+
uses: ./.github/actions/setup-node-pnpm
62+
63+
- name: Ratchet self-test
64+
run: pnpm mutation:test
65+
66+
- id: select
67+
name: Derive the affected shard matrix
68+
run: |
69+
modules=$(pnpm --silent mutation:affected --list-affected \
70+
--base "origin/${{ github.event.pull_request.base.ref }}" | tail -n1)
71+
echo "modules=$modules" >> "$GITHUB_OUTPUT"
72+
echo "Affected mutation shards: $modules" >> "$GITHUB_STEP_SUMMARY"
73+
74+
# The self-test and the derivation run before any mutant, so a failure here
75+
# would leave the lane with no envelope at all (#1430).
76+
- name: Record a failed lane envelope
77+
if: failure()
78+
run: |
79+
pnpm mutation:run --affected --fail-envelope \
80+
"affected selection failed before any mutant ran (run ${{ github.run_id }})" || true
81+
82+
- name: Upload selection envelope
83+
if: failure()
84+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
85+
with:
86+
name: mutation-affected-select
87+
path: .tmp/mutation/lane-envelope.json
88+
if-no-files-found: warn
89+
90+
mutants:
91+
name: Mutants (${{ matrix.name }})
92+
needs: select
93+
if: needs.select.outputs.modules != '[]'
94+
runs-on: ubuntu-latest
95+
timeout-minutes: 30
96+
strategy:
97+
fail-fast: false
98+
matrix:
99+
include: ${{ fromJSON(needs.select.outputs.modules) }}
100+
steps:
101+
- name: Checkout
102+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
103+
104+
- name: Setup toolchain
105+
uses: ./.github/actions/setup-node-pnpm
106+
107+
- name: Run mutants for ${{ matrix.name }}
108+
run: |
109+
pnpm mutation:run --modules ${{ matrix.module }} \
110+
${{ matrix.shard && format('--shard {0}', matrix.shard) || '' }}
111+
112+
- name: Record a failed shard envelope
113+
if: failure()
114+
run: |
115+
pnpm mutation:run --affected --modules ${{ matrix.module }} --fail-envelope \
116+
"shard ${{ matrix.name }} failed before producing a report (run ${{ github.run_id }})" || true
117+
118+
- name: Upload shard report
119+
if: always()
120+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
121+
with:
122+
name: mutation-affected-shard-${{ matrix.name }}
123+
path: |
124+
.tmp/mutation/mutation.json
125+
.tmp/mutation/mutation.html
126+
.tmp/mutation/lane-envelope.json
127+
if-no-files-found: warn
128+
129+
ratchet:
130+
name: Affected decision-kernel mutants
131+
needs: [select, mutants]
132+
if: always() && needs.select.result == 'success'
133+
runs-on: ubuntu-latest
134+
timeout-minutes: 15
135+
steps:
136+
- name: Checkout
137+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
138+
with:
139+
fetch-depth: 0
140+
141+
- name: Setup toolchain
142+
uses: ./.github/actions/setup-node-pnpm
143+
144+
- name: Download shard reports
145+
if: needs.select.outputs.modules != '[]'
146+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
147+
with:
148+
pattern: mutation-affected-shard-*
149+
path: .tmp/mutation/shards
150+
151+
# No shards means no affected kernel: run.ts reports "nothing to mutate"
152+
# and still writes the envelope, so the lane is never silently absent.
153+
- name: Ratchet the affected modules
154+
run: |
155+
if [ -d .tmp/mutation/shards ]; then
156+
expected=$(echo '${{ needs.select.outputs.modules }}' | jq length)
157+
pnpm mutation:check --report-dir .tmp/mutation/shards --affected \
158+
--expect-shards "$expected" \
159+
--base "origin/${{ github.event.pull_request.base.ref }}"
160+
else
161+
pnpm mutation:affected --base "origin/${{ github.event.pull_request.base.ref }}"
162+
fi
163+
164+
- name: Record a failed lane envelope
165+
if: failure()
166+
run: |
167+
pnpm mutation:run --affected --fail-envelope \
168+
"affected ratchet failed before producing a verdict (run ${{ github.run_id }})" || true
169+
170+
- name: Upload mutation report
171+
if: always()
172+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
173+
with:
174+
name: mutation-affected
175+
path: |
176+
.tmp/mutation/shards
177+
.tmp/mutation/lane-envelope.json
178+
if-no-files-found: ignore
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
name: Mutation Weekly
2+
3+
# Weekly mutation sweep over the enumerated decision kernels (issue #1415).
4+
# Mutation score is the mechanical answer to "is this test load-bearing or
5+
# decorative"; a full-suite sweep is unaffordable, so the scope is the kernel
6+
# registry in scripts/mutation/modules.ts and nothing else.
7+
#
8+
# Sharded one job per module: the whole sweep is ~2,150 mutants, and the selector
9+
# module alone is ~1,280 of them, so a single job would sit near the 30-minute
10+
# acceptance budget on an ubuntu runner. The shards' JSON reports are merged into
11+
# one verdict by the ratchet job (`--report-dir`), so the ratchet still sees a
12+
# full sweep.
13+
#
14+
# The lane reports; it does not commit. The proposed baseline rides in the
15+
# artifact and applying it is a reviewed `pnpm mutation:baseline` commit — a score
16+
# can never lower itself. Gating (after two consecutive stable weekly runs) is
17+
# enforced on PRs by mutation-affected.yml.
18+
19+
on:
20+
schedule:
21+
# Sundays 05:00 UTC — after the nightly lanes, before the working week.
22+
- cron: "0 5 * * 0"
23+
workflow_dispatch:
24+
25+
permissions:
26+
contents: read
27+
28+
concurrency:
29+
group: ci-${{ github.workflow }}-${{ github.ref }}
30+
cancel-in-progress: true
31+
32+
jobs:
33+
shard:
34+
name: Mutants (${{ matrix.name }})
35+
runs-on: ubuntu-latest
36+
# The acceptance budget is 30 minutes of wall clock for the lane, and shards
37+
# run in parallel, so the budget is per shard. The observed rate on a 2-core
38+
# runner is ~3s/mutant, which is why selectors (~1,280 mutants) is sliced.
39+
timeout-minutes: 30
40+
strategy:
41+
fail-fast: false
42+
matrix:
43+
# Kept in step with KERNEL_MODULES by the shard-coverage assertion in
44+
# scripts/mutation/workflow.test.ts, which also pins --expect-shards to
45+
# the number of entries here.
46+
include:
47+
- { name: kernel-errors, module: kernel-errors }
48+
- { name: daemon-ref-frame, module: daemon-ref-frame }
49+
- { name: interaction-settle, module: interaction-settle }
50+
- { name: scroll-edge-state, module: scroll-edge-state }
51+
- { name: selectors-1, module: selectors, shard: 1/4 }
52+
- { name: selectors-2, module: selectors, shard: 2/4 }
53+
- { name: selectors-3, module: selectors, shard: 3/4 }
54+
- { name: selectors-4, module: selectors, shard: 4/4 }
55+
steps:
56+
- name: Checkout
57+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
58+
59+
- name: Setup toolchain
60+
uses: ./.github/actions/setup-node-pnpm
61+
62+
- name: Run mutants for ${{ matrix.name }}
63+
run: |
64+
pnpm mutation:run --modules ${{ matrix.module }} \
65+
${{ matrix.shard && format('--shard {0}', matrix.shard) || '' }}
66+
67+
# A shard can die before Stryker writes anything (install, config, crash);
68+
# without this the artifact is absent and "shard failed" is indistinguishable
69+
# from "lane never ran". --fail-envelope leaves a real verdict untouched.
70+
- name: Record a failed shard envelope
71+
if: failure()
72+
run: |
73+
pnpm mutation:run --modules ${{ matrix.module }} --fail-envelope \
74+
"shard ${{ matrix.name }} failed before producing a report (run ${{ github.run_id }})" || true
75+
76+
- name: Upload shard report
77+
if: always()
78+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
79+
with:
80+
name: mutation-shard-${{ matrix.name }}
81+
path: |
82+
.tmp/mutation/mutation.json
83+
.tmp/mutation/lane-envelope.json
84+
if-no-files-found: warn
85+
86+
ratchet:
87+
name: Mutation ratchet
88+
runs-on: ubuntu-latest
89+
needs: shard
90+
if: always()
91+
timeout-minutes: 15
92+
steps:
93+
- name: Checkout
94+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
95+
96+
- name: Setup toolchain
97+
uses: ./.github/actions/setup-node-pnpm
98+
99+
- name: Ratchet self-test
100+
run: pnpm mutation:test
101+
102+
- name: Download shard reports
103+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
104+
with:
105+
pattern: mutation-shard-*
106+
path: .tmp/mutation/shards
107+
108+
# run.ts writes the markdown verdict to $GITHUB_STEP_SUMMARY when the
109+
# runner exports it, so the summary and the artifact carry the same numbers.
110+
- name: Ratchet the merged sweep and propose the next baseline
111+
run: |
112+
pnpm mutation:check --report-dir .tmp/mutation/shards --expect-shards 8 --update
113+
cp mutation-baselines/decision-kernels.json .tmp/mutation/proposed-baseline.json
114+
git checkout -- mutation-baselines/decision-kernels.json
115+
116+
# The self-test and the artifact download both run before the ratchet, so a
117+
# failure there would otherwise leave the aggregate lane with no envelope.
118+
- name: Record a failed lane envelope
119+
if: failure()
120+
run: |
121+
pnpm mutation:run --fail-envelope \
122+
"weekly ratchet job failed before producing a verdict (run ${{ github.run_id }})" || true
123+
124+
# Freshness/drift telemetry (#1430): the envelope states commit, Stryker
125+
# version, config hash, duration and result, so a lane going dark or a tool
126+
# bump is visible without reading these logs.
127+
- name: Lane envelope
128+
if: always()
129+
run: |
130+
if [ ! -f .tmp/mutation/lane-envelope.json ]; then
131+
echo 'No lane envelope was written — the job died before it could run node.' \
132+
>> "$GITHUB_STEP_SUMMARY"
133+
exit 0
134+
fi
135+
{
136+
echo '<details><summary>Lane envelope (schema #1430)</summary>'
137+
echo
138+
echo '```json'
139+
cat .tmp/mutation/lane-envelope.json
140+
echo '```'
141+
echo '</details>'
142+
} >> "$GITHUB_STEP_SUMMARY"
143+
144+
- name: Upload mutation report
145+
if: always()
146+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
147+
with:
148+
name: mutation-decision-kernels
149+
path: |
150+
.tmp/mutation/shards
151+
.tmp/mutation/lane-envelope.json
152+
.tmp/mutation/proposed-baseline.json
153+
if-no-files-found: warn

0 commit comments

Comments
 (0)