Skip to content

Commit ed97ad5

Browse files
test: mark recovery quizzes structurally and derive retriability from the enumeration
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 258d778 commit ed97ad5

6 files changed

Lines changed: 411 additions & 274 deletions

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@
114114
"mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected",
115115
"mutation:test": "node --experimental-strip-types --test scripts/mutation/*.test.ts",
116116
"lint": "oxlint . --deny-warnings",
117-
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-error-recovery-coverage.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
118-
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-error-recovery-coverage.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
117+
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-error-recovery-coverage.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-sample-producers.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
118+
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-error-recovery-coverage.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-sample-producers.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
119119
"fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts",
120120
"fallow": "fallow audit --base origin/main",
121121
"fallow:all": "fallow --summary",

‎scripts/__tests__/help-conformance-error-recovery-coverage.test.ts‎

Lines changed: 109 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,26 @@
1-
import fs from 'node:fs';
21
import assert from 'node:assert/strict';
3-
import { join } from 'node:path';
42
import { test } from 'vitest';
53
import { CASES } from '../help-conformance-cases.mjs';
64
import type { CapturedSample } from '../help-conformance-sample-outputs.mjs';
75
import * as samples from '../help-conformance-sample-outputs.mjs';
6+
import { SAMPLE_PRODUCERS } from './help-conformance-sample-producers.ts';
87
import {
98
defaultHintForCode,
109
KNOWN_APP_ERROR_CODES,
1110
retriableForErrorCode,
1211
} from '../../src/kernel/errors.ts';
1312

1413
// "What enumerates N" for the error surface. The benchmark's recovery quizzes
15-
// (a captured `Error (CODE)` output plus "what runs next?") are keyed to the
14+
// (a pinned rendered error plus "what command runs next?") are keyed to the
1615
// error registry itself, so the two error-side policies cannot drift from the
1716
// cases silently:
18-
// - a code `retriableForErrorCode` classifies is one an agent is told to
19-
// retry, so the bench must show whether a model actually recovers from it;
20-
// - a code with no specific `defaultHintForCode` entry falls back to generic
17+
// - a code retriableForErrorCode classifies is one an agent is told to retry,
18+
// so the bench must show whether a model actually recovers from it;
19+
// - a code with no specific defaultHintForCode entry falls back to generic
2120
// "retry with --debug" guidance, which is a decision, not a default.
2221
// Uncovered codes need a named waiver below, and a waiver fails once it goes
2322
// stale — the same shape as the help-topic gate.
2423

25-
const PINNING_TEST_FILE = 'help-conformance-sample-outputs.test.ts';
26-
2724
/** Recovery-quiz waivers, keyed by error code: why the code is unbenchmarked. */
2825
const WAIVED_RECOVERY_QUIZ_CODES: Record<string, string> = {};
2926

@@ -37,90 +34,150 @@ const WAIVED_GENERIC_HINT_CODES: Record<string, string> = {
3734
};
3835

3936
// Human error rendering (src/utils/output.ts printHumanError) opens with
40-
// `Error (CODE): message`, so a quoted error sample names its own code.
41-
const QUOTED_ERROR_CODE = /(?:^|\n)Error \(([A-Z_]+)\):/g;
37+
// `Error (CODE): message`, so any rendered error text names its own code.
38+
const RENDERED_ERROR_CODE = /(?:^|\n)Error \(([A-Z_]+)\):/g;
4239

43-
const CAPTURED_SAMPLES: [string, CapturedSample][] = Object.entries(samples).filter(
44-
(entry): entry is [string, CapturedSample] => typeof entry[1] === 'object' && entry[1] !== null,
45-
);
40+
type RecoveryCase = {
41+
id: string;
42+
task: string;
43+
expectations?: string[];
44+
matchers?: { id: string; pattern: RegExp }[];
45+
forbidden?: { id: string; pattern: RegExp }[];
46+
recovery: { code: string; sample: CapturedSample };
47+
};
4648

47-
function quotedErrorCodes(text: string): string[] {
48-
return [...text.matchAll(QUOTED_ERROR_CODE)].map((match) => match[1]!);
49+
function renderedErrorCodes(text: string): string[] {
50+
return [...`\n${text}`.matchAll(RENDERED_ERROR_CODE)].map((match) => match[1]!);
4951
}
5052

51-
/** Error codes an agent is quizzed on recovering from, derived from the cases. */
52-
function quizzedErrorCodes(): Set<string> {
53-
return new Set(CASES.flatMap((testCase) => quotedErrorCodes(testCase.task)));
54-
}
53+
/** The cases that declare themselves recovery quizzes (see the cases module). */
54+
const RECOVERY_CASES: RecoveryCase[] = CASES.filter(
55+
(testCase: { recovery?: unknown }): testCase is RecoveryCase => testCase.recovery !== undefined,
56+
);
57+
58+
const QUIZZED_CODES = new Set(RECOVERY_CASES.map((testCase) => testCase.recovery.code));
5559

5660
test('every code with a retriability verdict has a recovery quiz or an explicit waiver', () => {
57-
const quizzed = quizzedErrorCodes();
5861
const classified = KNOWN_APP_ERROR_CODES.filter(
5962
(code) => retriableForErrorCode(code) !== undefined,
6063
);
6164
assert.ok(classified.length > 0, 'retriableForErrorCode classifies at least one code');
6265
const uncovered = classified.filter(
63-
(code) => !quizzed.has(code) && !(code in WAIVED_RECOVERY_QUIZ_CODES),
66+
(code) => !QUIZZED_CODES.has(code) && !(code in WAIVED_RECOVERY_QUIZ_CODES),
6467
);
6568
assert.deepEqual(
6669
uncovered,
6770
[],
68-
'a code retriableForErrorCode classifies needs a recovery-quiz case in scripts/help-conformance-cases.mjs (quoting a pinned sample) or a waiver above',
71+
'a code retriableForErrorCode classifies needs a recovery-quiz case in scripts/help-conformance-cases.mjs (declaring recovery: { code, sample }) or a waiver above',
6972
);
7073
});
7174

75+
test('retriability cannot be claimed for a code outside the enumeration', () => {
76+
// Otherwise a new retriable code could be classified in production and still
77+
// escape the coverage gate above by never entering KNOWN_APP_ERROR_CODES.
78+
for (const unenumerated of ['RUNNER_BUSY', 'DEVICE_IN_USE_SOON', 'device_in_use', '']) {
79+
assert.equal(
80+
retriableForErrorCode(unenumerated),
81+
undefined,
82+
`"${unenumerated}" is not in KNOWN_APP_ERROR_CODES, so it must not carry a retriability verdict — add it to the enumeration (and quiz it) instead`,
83+
);
84+
}
85+
});
86+
7287
test('recovery-quiz waivers only name real, classified, unquizzed codes', () => {
7388
const known = new Set<string>(KNOWN_APP_ERROR_CODES);
74-
const quizzed = quizzedErrorCodes();
7589
for (const [code, reason] of Object.entries(WAIVED_RECOVERY_QUIZ_CODES)) {
7690
assert.ok(known.has(code), `waived code "${code}" no longer exists — remove the waiver`);
7791
assert.ok(
7892
retriableForErrorCode(code) !== undefined,
7993
`waived code "${code}" is no longer classified by retriableForErrorCode — remove the waiver`,
8094
);
8195
assert.ok(
82-
!quizzed.has(code),
96+
!QUIZZED_CODES.has(code),
8397
`waived code "${code}" now has a recovery quiz — remove the waiver`,
8498
);
8599
assert.ok(reason.trim().length > 0, `waiver for "${code}" needs a reason`);
86100
}
87101
});
88102

89-
test('every quiz-quoted error code is a real error code', () => {
103+
test('every recovery case quizzes a real code through the pinned sample that renders it', () => {
90104
const known = new Set<string>(KNOWN_APP_ERROR_CODES);
91-
for (const testCase of CASES) {
92-
for (const code of quotedErrorCodes(testCase.task)) {
93-
assert.ok(
94-
known.has(code),
95-
`case "${testCase.id}" quotes error code "${code}", which is not in KNOWN_APP_ERROR_CODES — a renamed or invented code`,
96-
);
97-
}
105+
const exported = new Map<CapturedSample, string>(
106+
Object.entries(samples)
107+
.filter((entry): entry is [string, CapturedSample] => typeof entry[1] === 'object')
108+
.map(([name, sample]) => [sample, name]),
109+
);
110+
for (const testCase of RECOVERY_CASES) {
111+
const { code, sample } = testCase.recovery;
112+
assert.ok(
113+
known.has(code),
114+
`case "${testCase.id}" claims recovery from "${code}", which is not in KNOWN_APP_ERROR_CODES — a renamed or invented code`,
115+
);
116+
assert.ok(
117+
exported.has(sample),
118+
`case "${testCase.id}" must quote a sample exported from scripts/help-conformance-sample-outputs.mjs, never an inline literal`,
119+
);
120+
assert.deepEqual(
121+
renderedErrorCodes(sample.output),
122+
[code],
123+
`case "${testCase.id}" declares recovery from "${code}" but its sample renders a different error`,
124+
);
125+
assert.equal(
126+
testCase.task.split(samples.sampleText(sample)).length,
127+
2,
128+
`case "${testCase.id}" must quote ${exported.get(sample)} verbatim, exactly once`,
129+
);
98130
}
99131
});
100132

101-
test('every quiz-quoted error output comes from a pinned sample, never hand-transcribed', () => {
102-
for (const testCase of CASES) {
103-
for (const code of quotedErrorCodes(testCase.task)) {
104-
const source = CAPTURED_SAMPLES.find(
105-
([, sample]) =>
106-
quotedErrorCodes(`\n${sample.output}`).includes(code) &&
107-
testCase.task.includes(samples.sampleText(sample)),
108-
);
109-
assert.ok(
110-
source,
111-
`case "${testCase.id}" quotes ${code} output that is not a verbatim sample from scripts/help-conformance-sample-outputs.mjs — quoted output must be pinned to its real producer, never hand-transcribed`,
112-
);
113-
}
133+
test('a recovery case scores the recovery command, not just plan validity', () => {
134+
for (const testCase of RECOVERY_CASES) {
135+
assert.ok(
136+
testCase.expectations?.includes('validPlanCommands'),
137+
`case "${testCase.id}" must expect validPlanCommands`,
138+
);
139+
// Without these, "recovery" would be satisfied by any parseable plan — the
140+
// point of an error quiz is which next command the error's hint implies,
141+
// and which retry the error rules out.
142+
assert.ok(
143+
(testCase.matchers ?? []).length > 0,
144+
`recovery case "${testCase.id}" needs at least one matcher asserting the recovery command`,
145+
);
146+
assert.ok(
147+
(testCase.forbidden ?? []).length > 0,
148+
`recovery case "${testCase.id}" needs at least one forbidden pattern for the failure mode this error rules out`,
149+
);
114150
}
115151
});
116152

117-
test('every captured sample is exercised by the sample-outputs pinning test', () => {
118-
const pinningTest = fs.readFileSync(join(import.meta.dirname, PINNING_TEST_FILE), 'utf8');
119-
for (const [name] of CAPTURED_SAMPLES) {
120-
assert.match(
121-
pinningTest,
122-
new RegExp(`\\b${name}\\b`),
123-
`sample ${name} is not referenced by ${PINNING_TEST_FILE} — rebuild it through the real producer instead of trusting the transcription`,
153+
test('no case quotes rendered error text outside a declared pinned sample', () => {
154+
for (const testCase of CASES as {
155+
id: string;
156+
task: string;
157+
recovery?: RecoveryCase['recovery'];
158+
}[]) {
159+
const pinned = testCase.recovery ? samples.sampleText(testCase.recovery.sample) : undefined;
160+
const outsidePinned = pinned ? testCase.task.split(pinned).join('\n') : testCase.task;
161+
assert.deepEqual(
162+
renderedErrorCodes(outsidePinned),
163+
[],
164+
`case "${testCase.id}" quotes rendered error text that is not part of its declared pinned sample — quoted output must come from scripts/help-conformance-sample-outputs.mjs, never hand-transcribed`,
165+
);
166+
assert.doesNotMatch(
167+
outsidePinned,
168+
/(?:^|\n)Hint: /,
169+
`case "${testCase.id}" quotes a rendered error hint outside its declared pinned sample`,
170+
);
171+
}
172+
});
173+
174+
test('every captured sample is rebuilt through a real producer', () => {
175+
const pinned = new Set(SAMPLE_PRODUCERS.map((producer) => producer.sample));
176+
for (const [name, sample] of Object.entries(samples)) {
177+
if (typeof sample !== 'object') continue;
178+
assert.ok(
179+
pinned.has(sample),
180+
`sample ${name} has no entry in help-conformance-sample-producers.ts — captured output must be rebuilt through the real producer, never trusted as a transcription`,
124181
);
125182
}
126183
});

0 commit comments

Comments
 (0)