Skip to content

Commit a19412b

Browse files
thymikeedevin-ai-integration[bot]
thymikee
and
devin-ai-integration[bot]
committed
test: own kernel tests in the mutation registry and ship the #1430 lane envelope
- restore bench:help-conformance, broken by a formatting-path edit - kernel test files select their module on PRs (registry `tests` + workflow paths), asserted to reach the kernel through the import graph - every mutation run writes the standard scheduled-lane artifact envelope - move src/utils/__tests__/errors.test.ts beside its source per the mirror rule Refs #1415, #1430 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 0146dd4 commit a19412b

11 files changed

Lines changed: 334 additions & 8 deletions

File tree

‎.github/workflows/mutation-affected.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,19 @@ name: Mutation Affected
1414
on:
1515
pull_request:
1616
paths:
17+
# Kernel sources and the tests whose strength they score, mirroring
18+
# `KERNEL_MODULES` (scripts/mutation/workflow.test.ts asserts the parity).
1719
- "src/kernel/errors.ts"
20+
- "src/kernel/__tests__/errors.test.ts"
1821
- "src/daemon/ref-frame.ts"
22+
- "src/daemon/__tests__/ref-frame.test.ts"
23+
- "src/daemon/handlers/__tests__/snapshot-scoped-refs.test.ts"
1924
- "src/commands/interaction/runtime/settle.ts"
25+
- "src/commands/interaction/runtime/settle.test.ts"
26+
- "src/daemon/handlers/__tests__/interaction-settle.test.ts"
2027
- "src/utils/scroll-edge-state.ts"
28+
- "src/commands/interaction/runtime/gestures.test.ts"
29+
- "src/core/__tests__/dispatch-scroll.test.ts"
2130
- "src/selectors/**"
2231
- "scripts/mutation/**"
2332
- "stryker.config.json"
@@ -59,4 +68,5 @@ jobs:
5968
path: |
6069
.tmp/mutation/mutation.json
6170
.tmp/mutation/mutation.html
71+
.tmp/mutation/lane-envelope.json
6272
if-no-files-found: ignore

‎.github/workflows/mutation-weekly.yml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,9 @@ jobs:
6262
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
6363
with:
6464
name: mutation-shard-${{ matrix.module }}
65-
path: .tmp/mutation/mutation.json
65+
path: |
66+
.tmp/mutation/mutation.json
67+
.tmp/mutation/lane-envelope.json
6668
if-no-files-found: warn
6769

6870
ratchet:
@@ -95,12 +97,28 @@ jobs:
9597
cp mutation-baselines/decision-kernels.json .tmp/mutation/proposed-baseline.json
9698
git checkout -- mutation-baselines/decision-kernels.json
9799
100+
# Freshness/drift telemetry (#1430): the envelope states commit, Stryker
101+
# version, config hash, duration and result, so a lane going dark or a tool
102+
# bump is visible without reading these logs.
103+
- name: Lane envelope
104+
if: always()
105+
run: |
106+
{
107+
echo '<details><summary>Lane envelope (schema #1430)</summary>'
108+
echo
109+
echo '```json'
110+
cat .tmp/mutation/lane-envelope.json
111+
echo '```'
112+
echo '</details>'
113+
} >> "$GITHUB_STEP_SUMMARY"
114+
98115
- name: Upload mutation report
99116
if: always()
100117
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
101118
with:
102119
name: mutation-decision-kernels
103120
path: |
104121
.tmp/mutation/shards
122+
.tmp/mutation/lane-envelope.json
105123
.tmp/mutation/proposed-baseline.json
106124
if-no-files-found: warn

‎docs/agents/testing.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,15 @@ pnpm mutation:baseline # full sweep, then record it (reviewed c
153153
- **Test scope** is derived from Vitest's module graph (`vitest related` over the mutated files), the
154154
same delegation `pnpm check:affected` uses; see `scripts/mutation/test-scope.ts` for the three
155155
groups it drops and why dropping them cannot hide a surviving mutant.
156+
- **Kernel tests own their module too**: a diff that touches `src/kernel/__tests__/errors.test.ts` or
157+
`src/commands/interaction/runtime/settle.test.ts` selects that kernel's mutants, because weakening a
158+
test is exactly the change whose score must be re-measured. The registry's `tests` field lists them
159+
and `scripts/mutation/modules.test.ts` asserts each listed file actually reaches the kernel through
160+
the import graph.
161+
- **Lane envelope** (`scripts/lib/lane-envelope.ts`, issue #1430): every run writes
162+
`.tmp/mutation/lane-envelope.json` — schema version, commit, Stryker version, config hash, seed
163+
(`null`; the input is enumerated, not randomized), duration, result, per-module scores — and both
164+
workflows upload it, so lane freshness and tool drift are readable without parsing logs.
156165

157166
## Live web smoke
158167

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@
9999
"build:macos-helper": "swift build -c release --package-path apple/macos-helper",
100100
"build:all": "pnpm build:node && pnpm build:xcuitest",
101101
"ad": "node bin/agent-device.mjs",
102-
"bench:help-conformance": "node scripts/mutation scripts/help-conformance-bench.mjs",
102+
"bench:help-conformance": "node scripts/help-conformance-bench.mjs",
103103
"maestro:conformance": "node --experimental-strip-types --test scripts/maestro-conformance/verify.test.ts scripts/maestro-conformance/differential/run.test.ts scripts/maestro-conformance/differential/invariants.test.ts",
104104
"maestro:conformance:regenerate": "node scripts/maestro-conformance/regenerate.mjs",
105105
"maestro:conformance:differential": "node --experimental-strip-types scripts/maestro-conformance/differential/run.ts",

‎scripts/lib/lane-envelope.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Standard artifact envelope for scheduled lanes (issue #1430).
2+
//
3+
// A scheduled lane can go dark or drift for weeks while PR CI stays green, so
4+
// every artifact it uploads must say — without a human reading logs — which
5+
// commit produced it, which tool/config version measured it, how long it took,
6+
// and whether it passed. Freshness monitoring reads `finishedAt`/`result`; drift
7+
// analysis reads `tool`/`configHash`.
8+
//
9+
// Lanes that adopt the envelope later should import this module rather than
10+
// re-deriving the field names.
11+
12+
export const LANE_ENVELOPE_SCHEMA_VERSION = 1;
13+
14+
export type LaneResult = 'pass' | 'fail';
15+
16+
export type LaneEnvelope<T> = {
17+
schemaVersion: number;
18+
/** Stable lane id — the metric key freshness/health jobs group by. */
19+
lane: string;
20+
commit: string;
21+
ref: string | undefined;
22+
runUrl: string | undefined;
23+
/** Tool versions, or content hashes where a version is not enough. */
24+
tool: Record<string, string>;
25+
configHash: string;
26+
/** Only lanes with randomized input record a seed; `null` states "not applicable". */
27+
seed: string | null;
28+
startedAt: string;
29+
finishedAt: string;
30+
durationMs: number;
31+
result: LaneResult;
32+
/** Lane-specific payload; kept separate so the envelope shape stays stable. */
33+
data: T;
34+
};
35+
36+
export type LaneEnvelopeInput<T> = {
37+
lane: string;
38+
commit: string;
39+
tool: Record<string, string>;
40+
configHash: string;
41+
startedAtMs: number;
42+
result: LaneResult;
43+
data: T;
44+
seed?: string | null;
45+
now?: number;
46+
};
47+
48+
export function laneEnvelope<T>(input: LaneEnvelopeInput<T>): LaneEnvelope<T> {
49+
const finished = input.now ?? Date.now();
50+
const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID, GITHUB_REF_NAME } = process.env;
51+
return {
52+
schemaVersion: LANE_ENVELOPE_SCHEMA_VERSION,
53+
lane: input.lane,
54+
commit: input.commit,
55+
ref: GITHUB_REF_NAME,
56+
runUrl:
57+
GITHUB_SERVER_URL && GITHUB_REPOSITORY && GITHUB_RUN_ID
58+
? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`
59+
: undefined,
60+
tool: input.tool,
61+
configHash: input.configHash,
62+
seed: input.seed ?? null,
63+
startedAt: new Date(input.startedAtMs).toISOString(),
64+
finishedAt: new Date(finished).toISOString(),
65+
durationMs: Math.max(0, finished - input.startedAtMs),
66+
result: input.result,
67+
data: input.data,
68+
};
69+
}

‎scripts/mutation/envelope.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// The envelope is the only thing a downloaded scheduled-lane artifact can be
2+
// interpreted from months later (#1430), so its required fields are asserted
3+
// rather than assumed: a lane that stops emitting one of them makes freshness
4+
// and tool-drift monitoring silently useless.
5+
6+
import assert from 'node:assert/strict';
7+
import fs from 'node:fs';
8+
import path from 'node:path';
9+
import { test } from 'node:test';
10+
import { laneEnvelope, LANE_ENVELOPE_SCHEMA_VERSION } from '../lib/lane-envelope.ts';
11+
12+
const repoRoot = path.resolve(import.meta.dirname, '../..');
13+
14+
function workflow(name: string): string {
15+
return fs.readFileSync(path.join(repoRoot, '.github/workflows', name), 'utf8');
16+
}
17+
18+
test('the envelope carries schema, commit, tool/config provenance, duration and result', () => {
19+
const envelope = laneEnvelope({
20+
lane: 'mutation-decision-kernels',
21+
commit: 'a'.repeat(40),
22+
tool: { stryker: '9.6.1' },
23+
configHash: 'sha256:abcdef123456',
24+
startedAtMs: 1_000,
25+
now: 61_000,
26+
result: 'pass',
27+
data: { scope: 'full-sweep' },
28+
});
29+
assert.equal(envelope.schemaVersion, LANE_ENVELOPE_SCHEMA_VERSION);
30+
assert.equal(envelope.lane, 'mutation-decision-kernels');
31+
assert.equal(envelope.commit, 'a'.repeat(40));
32+
assert.deepEqual(envelope.tool, { stryker: '9.6.1' });
33+
assert.equal(envelope.configHash, 'sha256:abcdef123456');
34+
// Mutation input is enumerated, not randomized: `null` is an explicit
35+
// "not applicable", not a forgotten field.
36+
assert.equal(envelope.seed, null);
37+
assert.equal(envelope.durationMs, 60_000);
38+
assert.equal(envelope.finishedAt, '1970-01-01T00:01:01.000Z');
39+
assert.equal(envelope.result, 'pass');
40+
assert.deepEqual(envelope.data, { scope: 'full-sweep' });
41+
});
42+
43+
test('a failed ratchet is recorded as a failed lane run', () => {
44+
const envelope = laneEnvelope({
45+
lane: 'mutation-decision-kernels',
46+
commit: 'b'.repeat(40),
47+
tool: { stryker: '9.6.1' },
48+
configHash: 'sha256:abcdef123456',
49+
startedAtMs: 0,
50+
now: 0,
51+
result: 'fail',
52+
data: {},
53+
});
54+
assert.equal(envelope.result, 'fail');
55+
assert.equal(envelope.durationMs, 0);
56+
});
57+
58+
test('both mutation lanes publish the envelope', () => {
59+
for (const name of ['mutation-weekly.yml', 'mutation-affected.yml']) {
60+
assert.match(
61+
workflow(name),
62+
/\.tmp\/mutation\/lane-envelope\.json/,
63+
`${name} does not upload the lane envelope`,
64+
);
65+
}
66+
assert.match(workflow('mutation-weekly.yml'), /Lane envelope/);
67+
});

‎scripts/mutation/modules.test.ts‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../
1515

1616
test('every enumerated kernel path exists', () => {
1717
for (const module of KERNEL_MODULES) {
18-
for (const owned of module.owns) {
18+
for (const owned of [...module.owns, ...module.tests]) {
1919
assert.ok(
2020
fs.existsSync(path.join(repoRoot, owned)),
2121
`${module.id} owns a path that no longer exists: ${owned}`,
@@ -24,13 +24,81 @@ test('every enumerated kernel path exists', () => {
2424
}
2525
});
2626

27+
/** Repository-relative modules a file imports, following relative specifiers only. */
28+
function imports(file: string): string[] {
29+
const text = fs.readFileSync(path.join(repoRoot, file), 'utf8');
30+
return [...text.matchAll(/from '(?<spec>\.[^']+)'/g)]
31+
.map((match) =>
32+
path.posix.normalize(path.posix.join(path.posix.dirname(file), match.groups!.spec)),
33+
)
34+
.filter((resolved) => fs.existsSync(path.join(repoRoot, resolved)));
35+
}
36+
37+
/** Whether `file` reaches `target` through the import graph. */
38+
function reaches(file: string, target: string): boolean {
39+
const seen = new Set<string>();
40+
const queue = [file];
41+
while (queue.length > 0) {
42+
const current = queue.shift()!;
43+
if (current === target) return true;
44+
if (seen.has(current)) continue;
45+
seen.add(current);
46+
queue.push(...imports(current));
47+
}
48+
return false;
49+
}
50+
51+
test('each owned test file exercises the module it is listed under', () => {
52+
for (const module of KERNEL_MODULES) {
53+
const sources = module.mutate.filter((glob) => !glob.startsWith('!') && !glob.includes('*'));
54+
if (sources.length === 0) continue; // directory module: ownership is the prefix
55+
for (const testFile of module.tests) {
56+
assert.ok(
57+
sources.some((source) => reaches(testFile, source)),
58+
`${module.id} lists ${testFile}, which never reaches ${sources.join(', ')}`,
59+
);
60+
}
61+
}
62+
});
63+
64+
test('a kernel source with a mirrored test file lists that test', () => {
65+
for (const module of KERNEL_MODULES) {
66+
for (const source of module.mutate.filter(
67+
(glob) => !glob.startsWith('!') && !glob.includes('*'),
68+
)) {
69+
const dir = path.dirname(source);
70+
const name = path.basename(source, '.ts');
71+
const mirrors = [`${dir}/${name}.test.ts`, `${dir}/__tests__/${name}.test.ts`].filter(
72+
(candidate) => fs.existsSync(path.join(repoRoot, candidate)),
73+
);
74+
for (const mirror of mirrors) {
75+
assert.equal(
76+
moduleForFile(mirror),
77+
module.id,
78+
`${mirror} mirrors ${source} but does not select ${module.id}`,
79+
);
80+
}
81+
}
82+
}
83+
});
84+
2785
test('changed files map onto the module that owns them', () => {
2886
assert.equal(moduleForFile('src/kernel/errors.ts'), 'kernel-errors');
2987
assert.equal(moduleForFile('./src/daemon/ref-frame.ts'), 'daemon-ref-frame');
3088
assert.equal(moduleForFile('src/selectors/parse.ts'), 'selectors');
3189
// Tests of a kernel are owned too: a weakened test is exactly the change
3290
// whose mutation score must be re-measured.
3391
assert.equal(moduleForFile('src/selectors/__tests__/resolve.test.ts'), 'selectors');
92+
assert.equal(moduleForFile('src/kernel/__tests__/errors.test.ts'), 'kernel-errors');
93+
assert.equal(moduleForFile('src/daemon/__tests__/ref-frame.test.ts'), 'daemon-ref-frame');
94+
assert.equal(
95+
moduleForFile('src/commands/interaction/runtime/settle.test.ts'),
96+
'interaction-settle',
97+
);
98+
assert.equal(
99+
moduleForFile('src/commands/interaction/runtime/gestures.test.ts'),
100+
'scroll-edge-state',
101+
);
34102
assert.equal(moduleForFile('src/kernel/rect.ts'), undefined);
35103
assert.equal(moduleForFile('README.md'), undefined);
36104
});

‎scripts/mutation/modules.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ export type KernelModule = {
2525
readonly mutate: readonly string[];
2626
/** Paths this module owns; a trailing `/` marks a directory prefix. */
2727
readonly owns: readonly string[];
28+
/**
29+
* Test files whose strength this module's score measures. Editing one of
30+
* these is exactly the change the PR lane must re-measure, so they own the
31+
* module too. `modules.test.ts` asserts each one exists and actually exercises
32+
* the module's sources — no stale or aspirational entries.
33+
*/
34+
readonly tests: readonly string[];
2835
};
2936

3037
export const KERNEL_MODULES: readonly KernelModule[] = [
@@ -33,30 +40,47 @@ export const KERNEL_MODULES: readonly KernelModule[] = [
3340
label: 'Error retriability + hints',
3441
mutate: ['src/kernel/errors.ts'],
3542
owns: ['src/kernel/errors.ts'],
43+
tests: ['src/kernel/__tests__/errors.test.ts'],
3644
},
3745
{
3846
id: 'daemon-ref-frame',
3947
label: 'Ref-frame admission matrix (ADR 0014)',
4048
mutate: ['src/daemon/ref-frame.ts'],
4149
owns: ['src/daemon/ref-frame.ts'],
50+
tests: [
51+
'src/daemon/__tests__/ref-frame.test.ts',
52+
'src/daemon/handlers/__tests__/snapshot-scoped-refs.test.ts',
53+
],
4254
},
4355
{
4456
id: 'interaction-settle',
4557
label: 'Interaction settle decisions',
4658
mutate: ['src/commands/interaction/runtime/settle.ts'],
4759
owns: ['src/commands/interaction/runtime/settle.ts'],
60+
tests: [
61+
'src/commands/interaction/runtime/settle.test.ts',
62+
'src/daemon/handlers/__tests__/interaction-settle.test.ts',
63+
],
4864
},
4965
{
5066
id: 'scroll-edge-state',
5167
label: 'Scroll edge-state detection',
5268
mutate: ['src/utils/scroll-edge-state.ts'],
5369
owns: ['src/utils/scroll-edge-state.ts'],
70+
// No mirrored test file: edge-state decisions are only reached through the
71+
// gesture/dispatch callers, which is also why this kernel scores lowest.
72+
tests: [
73+
'src/commands/interaction/runtime/gestures.test.ts',
74+
'src/core/__tests__/dispatch-scroll.test.ts',
75+
],
5476
},
5577
{
5678
id: 'selectors',
5779
label: 'Selector parsing + matching',
5880
mutate: ['src/selectors/**/*.ts', '!src/selectors/**/*.test.ts', '!src/selectors/__tests__/**'],
81+
// Selector tests live under the owned directory, so the prefix covers them.
5982
owns: ['src/selectors/'],
83+
tests: ['src/selectors/__tests__/'],
6084
},
6185
];
6286

@@ -92,7 +116,7 @@ export function normalizePath(filePath: string): string {
92116
export function moduleForFile(filePath: string): ModuleId | undefined {
93117
const normalized = normalizePath(filePath);
94118
for (const module of KERNEL_MODULES) {
95-
for (const owned of module.owns) {
119+
for (const owned of [...module.owns, ...module.tests]) {
96120
const match = owned.endsWith('/') ? normalized.startsWith(owned) : normalized === owned;
97121
if (match) return module.id;
98122
}

0 commit comments

Comments
 (0)