1- import fs from 'node:fs' ;
21import assert from 'node:assert/strict' ;
3- import { join } from 'node:path' ;
42import { test } from 'vitest' ;
53import { CASES } from '../help-conformance-cases.mjs' ;
64import type { CapturedSample } from '../help-conformance-sample-outputs.mjs' ;
75import * as samples from '../help-conformance-sample-outputs.mjs' ;
6+ import { SAMPLE_PRODUCERS } from './help-conformance-sample-producers.ts' ;
87import {
98 defaultHintForCode ,
109 KNOWN_APP_ERROR_CODES ,
1110 retriableForErrorCode ,
1211} from '../../src/kernel/errors.ts' ;
1312
1413// "What enumerates N" for the error surface. The benchmark's recovery quizzes
15- // (a captured `Error (CODE)` output plus "what runs next?") are keyed to the
14+ // (a pinned rendered error plus "what command runs next?") are keyed to the
1615// error registry itself, so the two error-side policies cannot drift from the
1716// cases silently:
18- // - a code ` retriableForErrorCode` classifies is one an agent is told to
19- // retry, so the bench must show whether a model actually recovers from it;
20- // - a code with no specific ` defaultHintForCode` entry falls back to generic
17+ // - a code retriableForErrorCode classifies is one an agent is told to retry,
18+ // so the bench must show whether a model actually recovers from it;
19+ // - a code with no specific defaultHintForCode entry falls back to generic
2120// "retry with --debug" guidance, which is a decision, not a default.
2221// Uncovered codes need a named waiver below, and a waiver fails once it goes
2322// stale — the same shape as the help-topic gate.
2423
25- const PINNING_TEST_FILE = 'help-conformance-sample-outputs.test.ts' ;
26-
2724/** Recovery-quiz waivers, keyed by error code: why the code is unbenchmarked. */
2825const WAIVED_RECOVERY_QUIZ_CODES : Record < string , string > = { } ;
2926
@@ -37,90 +34,150 @@ const WAIVED_GENERIC_HINT_CODES: Record<string, string> = {
3734} ;
3835
3936// Human error rendering (src/utils/output.ts printHumanError) opens with
40- // `Error (CODE): message`, so a quoted error sample names its own code.
41- const QUOTED_ERROR_CODE = / (?: ^ | \n ) E r r o r \( ( [ A - Z _ ] + ) \) : / g;
37+ // `Error (CODE): message`, so any rendered error text names its own code.
38+ const RENDERED_ERROR_CODE = / (?: ^ | \n ) E r r o r \( ( [ A - Z _ ] + ) \) : / g;
4239
43- const CAPTURED_SAMPLES : [ string , CapturedSample ] [ ] = Object . entries ( samples ) . filter (
44- ( entry ) : entry is [ string , CapturedSample ] => typeof entry [ 1 ] === 'object' && entry [ 1 ] !== null ,
45- ) ;
40+ type RecoveryCase = {
41+ id : string ;
42+ task : string ;
43+ expectations ?: string [ ] ;
44+ matchers ?: { id : string ; pattern : RegExp } [ ] ;
45+ forbidden ?: { id : string ; pattern : RegExp } [ ] ;
46+ recovery : { code : string ; sample : CapturedSample } ;
47+ } ;
4648
47- function quotedErrorCodes ( text : string ) : string [ ] {
48- return [ ...text . matchAll ( QUOTED_ERROR_CODE ) ] . map ( ( match ) => match [ 1 ] ! ) ;
49+ function renderedErrorCodes ( text : string ) : string [ ] {
50+ return [ ...`\n ${ text } ` . matchAll ( RENDERED_ERROR_CODE ) ] . map ( ( match ) => match [ 1 ] ! ) ;
4951}
5052
51- /** Error codes an agent is quizzed on recovering from, derived from the cases. */
52- function quizzedErrorCodes ( ) : Set < string > {
53- return new Set ( CASES . flatMap ( ( testCase ) => quotedErrorCodes ( testCase . task ) ) ) ;
54- }
53+ /** The cases that declare themselves recovery quizzes (see the cases module). */
54+ const RECOVERY_CASES : RecoveryCase [ ] = CASES . filter (
55+ ( testCase : { recovery ?: unknown } ) : testCase is RecoveryCase => testCase . recovery !== undefined ,
56+ ) ;
57+
58+ const QUIZZED_CODES = new Set ( RECOVERY_CASES . map ( ( testCase ) => testCase . recovery . code ) ) ;
5559
5660test ( 'every code with a retriability verdict has a recovery quiz or an explicit waiver' , ( ) => {
57- const quizzed = quizzedErrorCodes ( ) ;
5861 const classified = KNOWN_APP_ERROR_CODES . filter (
5962 ( code ) => retriableForErrorCode ( code ) !== undefined ,
6063 ) ;
6164 assert . ok ( classified . length > 0 , 'retriableForErrorCode classifies at least one code' ) ;
6265 const uncovered = classified . filter (
63- ( code ) => ! quizzed . has ( code ) && ! ( code in WAIVED_RECOVERY_QUIZ_CODES ) ,
66+ ( code ) => ! QUIZZED_CODES . has ( code ) && ! ( code in WAIVED_RECOVERY_QUIZ_CODES ) ,
6467 ) ;
6568 assert . deepEqual (
6669 uncovered ,
6770 [ ] ,
68- 'a code retriableForErrorCode classifies needs a recovery-quiz case in scripts/help-conformance-cases.mjs (quoting a pinned sample) or a waiver above' ,
71+ 'a code retriableForErrorCode classifies needs a recovery-quiz case in scripts/help-conformance-cases.mjs (declaring recovery: { code, sample } ) or a waiver above' ,
6972 ) ;
7073} ) ;
7174
75+ test ( 'retriability cannot be claimed for a code outside the enumeration' , ( ) => {
76+ // Otherwise a new retriable code could be classified in production and still
77+ // escape the coverage gate above by never entering KNOWN_APP_ERROR_CODES.
78+ for ( const unenumerated of [ 'RUNNER_BUSY' , 'DEVICE_IN_USE_SOON' , 'device_in_use' , '' ] ) {
79+ assert . equal (
80+ retriableForErrorCode ( unenumerated ) ,
81+ undefined ,
82+ `"${ unenumerated } " is not in KNOWN_APP_ERROR_CODES, so it must not carry a retriability verdict — add it to the enumeration (and quiz it) instead` ,
83+ ) ;
84+ }
85+ } ) ;
86+
7287test ( 'recovery-quiz waivers only name real, classified, unquizzed codes' , ( ) => {
7388 const known = new Set < string > ( KNOWN_APP_ERROR_CODES ) ;
74- const quizzed = quizzedErrorCodes ( ) ;
7589 for ( const [ code , reason ] of Object . entries ( WAIVED_RECOVERY_QUIZ_CODES ) ) {
7690 assert . ok ( known . has ( code ) , `waived code "${ code } " no longer exists — remove the waiver` ) ;
7791 assert . ok (
7892 retriableForErrorCode ( code ) !== undefined ,
7993 `waived code "${ code } " is no longer classified by retriableForErrorCode — remove the waiver` ,
8094 ) ;
8195 assert . ok (
82- ! quizzed . has ( code ) ,
96+ ! QUIZZED_CODES . has ( code ) ,
8397 `waived code "${ code } " now has a recovery quiz — remove the waiver` ,
8498 ) ;
8599 assert . ok ( reason . trim ( ) . length > 0 , `waiver for "${ code } " needs a reason` ) ;
86100 }
87101} ) ;
88102
89- test ( 'every quiz-quoted error code is a real error code' , ( ) => {
103+ test ( 'every recovery case quizzes a real code through the pinned sample that renders it ' , ( ) => {
90104 const known = new Set < string > ( KNOWN_APP_ERROR_CODES ) ;
91- for ( const testCase of CASES ) {
92- for ( const code of quotedErrorCodes ( testCase . task ) ) {
93- assert . ok (
94- known . has ( code ) ,
95- `case "${ testCase . id } " quotes error code "${ code } ", which is not in KNOWN_APP_ERROR_CODES — a renamed or invented code` ,
96- ) ;
97- }
105+ const exported = new Map < CapturedSample , string > (
106+ Object . entries ( samples )
107+ . filter ( ( entry ) : entry is [ string , CapturedSample ] => typeof entry [ 1 ] === 'object' )
108+ . map ( ( [ name , sample ] ) => [ sample , name ] ) ,
109+ ) ;
110+ for ( const testCase of RECOVERY_CASES ) {
111+ const { code, sample } = testCase . recovery ;
112+ assert . ok (
113+ known . has ( code ) ,
114+ `case "${ testCase . id } " claims recovery from "${ code } ", which is not in KNOWN_APP_ERROR_CODES — a renamed or invented code` ,
115+ ) ;
116+ assert . ok (
117+ exported . has ( sample ) ,
118+ `case "${ testCase . id } " must quote a sample exported from scripts/help-conformance-sample-outputs.mjs, never an inline literal` ,
119+ ) ;
120+ assert . deepEqual (
121+ renderedErrorCodes ( sample . output ) ,
122+ [ code ] ,
123+ `case "${ testCase . id } " declares recovery from "${ code } " but its sample renders a different error` ,
124+ ) ;
125+ assert . equal (
126+ testCase . task . split ( samples . sampleText ( sample ) ) . length ,
127+ 2 ,
128+ `case "${ testCase . id } " must quote ${ exported . get ( sample ) } verbatim, exactly once` ,
129+ ) ;
98130 }
99131} ) ;
100132
101- test ( 'every quiz-quoted error output comes from a pinned sample, never hand-transcribed' , ( ) => {
102- for ( const testCase of CASES ) {
103- for ( const code of quotedErrorCodes ( testCase . task ) ) {
104- const source = CAPTURED_SAMPLES . find (
105- ( [ , sample ] ) =>
106- quotedErrorCodes ( `\n${ sample . output } ` ) . includes ( code ) &&
107- testCase . task . includes ( samples . sampleText ( sample ) ) ,
108- ) ;
109- assert . ok (
110- source ,
111- `case "${ testCase . id } " quotes ${ code } output that is not a verbatim sample from scripts/help-conformance-sample-outputs.mjs — quoted output must be pinned to its real producer, never hand-transcribed` ,
112- ) ;
113- }
133+ test ( 'a recovery case scores the recovery command, not just plan validity' , ( ) => {
134+ for ( const testCase of RECOVERY_CASES ) {
135+ assert . ok (
136+ testCase . expectations ?. includes ( 'validPlanCommands' ) ,
137+ `case "${ testCase . id } " must expect validPlanCommands` ,
138+ ) ;
139+ // Without these, "recovery" would be satisfied by any parseable plan — the
140+ // point of an error quiz is which next command the error's hint implies,
141+ // and which retry the error rules out.
142+ assert . ok (
143+ ( testCase . matchers ?? [ ] ) . length > 0 ,
144+ `recovery case "${ testCase . id } " needs at least one matcher asserting the recovery command` ,
145+ ) ;
146+ assert . ok (
147+ ( testCase . forbidden ?? [ ] ) . length > 0 ,
148+ `recovery case "${ testCase . id } " needs at least one forbidden pattern for the failure mode this error rules out` ,
149+ ) ;
114150 }
115151} ) ;
116152
117- test ( 'every captured sample is exercised by the sample-outputs pinning test' , ( ) => {
118- const pinningTest = fs . readFileSync ( join ( import . meta. dirname , PINNING_TEST_FILE ) , 'utf8' ) ;
119- for ( const [ name ] of CAPTURED_SAMPLES ) {
120- assert . match (
121- pinningTest ,
122- new RegExp ( `\\b${ name } \\b` ) ,
123- `sample ${ name } is not referenced by ${ PINNING_TEST_FILE } — rebuild it through the real producer instead of trusting the transcription` ,
153+ test ( 'no case quotes rendered error text outside a declared pinned sample' , ( ) => {
154+ for ( const testCase of CASES as {
155+ id : string ;
156+ task : string ;
157+ recovery ?: RecoveryCase [ 'recovery' ] ;
158+ } [ ] ) {
159+ const pinned = testCase . recovery ? samples . sampleText ( testCase . recovery . sample ) : undefined ;
160+ const outsidePinned = pinned ? testCase . task . split ( pinned ) . join ( '\n' ) : testCase . task ;
161+ assert . deepEqual (
162+ renderedErrorCodes ( outsidePinned ) ,
163+ [ ] ,
164+ `case "${ testCase . id } " quotes rendered error text that is not part of its declared pinned sample — quoted output must come from scripts/help-conformance-sample-outputs.mjs, never hand-transcribed` ,
165+ ) ;
166+ assert . doesNotMatch (
167+ outsidePinned ,
168+ / (?: ^ | \n ) H i n t : / ,
169+ `case "${ testCase . id } " quotes a rendered error hint outside its declared pinned sample` ,
170+ ) ;
171+ }
172+ } ) ;
173+
174+ test ( 'every captured sample is rebuilt through a real producer' , ( ) => {
175+ const pinned = new Set ( SAMPLE_PRODUCERS . map ( ( producer ) => producer . sample ) ) ;
176+ for ( const [ name , sample ] of Object . entries ( samples ) ) {
177+ if ( typeof sample !== 'object' ) continue ;
178+ assert . ok (
179+ pinned . has ( sample ) ,
180+ `sample ${ name } has no entry in help-conformance-sample-producers.ts — captured output must be rebuilt through the real producer, never trusted as a transcription` ,
124181 ) ;
125182 }
126183} ) ;
0 commit comments