@@ -4,7 +4,11 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
44import { basename , join } from 'node:path' ;
55import { fileURLToPath } from 'node:url' ;
66import { promisify } from 'node:util' ;
7- import { opensAndCloses , usesValidationPrep } from './help-conformance-expectations.mjs' ;
7+ import {
8+ assertCaseDefinitions ,
9+ countChecks ,
10+ scoreExpectations ,
11+ } from './help-conformance-case-checks.mjs' ;
812import { validatePlanCommands } from './help-conformance-plan-validator.mjs' ;
913import { detectRunnerError , extractCommands } from './help-conformance-runner-output.mjs' ;
1014import { summarizeResults } from './help-conformance-summary.mjs' ;
@@ -374,7 +378,7 @@ function withDefault(value, fallback) {
374378async function main ( ) {
375379 const args = parseArgs ( process . argv . slice ( 2 ) ) ;
376380 assertRuntimeConfiguration ( ) ;
377- assertCaseDefinitions ( ) ;
381+ assertCaseDefinitions ( CASES ) ;
378382 const outDir = resolveOutDir ( args ) ;
379383 await mkdir ( outDir , { recursive : true } ) ;
380384 const selectedCases = selectCases ( args . cases ) ;
@@ -708,118 +712,11 @@ async function runCodex(model, prompt, outDir) {
708712 return lastMessage . trim ( ) . length > 0 ? lastMessage : stdout ;
709713}
710714
711- const EXPECTATION_SCORERS = {
712- validPlanCommands : ( { commands, commandValidation } ) =>
713- commands . length > 0 && commandValidation . every ( ( { issues } ) => issues . length === 0 ) ,
714- fullPrefix : ( { commands } ) =>
715- commands . length > 0 &&
716- commands . every (
717- ( command ) => ! / ^ ( o p e n | s n a p s h o t | p r e s s | f i l l | c l i c k | l o n g p r e s s | w a i t | c l o s e ) \b / . test ( command ) ,
718- ) ,
719- usesSnapshotI : ( { commands } ) => commands . some ( ( command ) => / \b s n a p s h o t \b .* \s - i \b / . test ( command ) ) ,
720- usesSettleOnMutations : ( { commands } ) => allMutationsUseSettle ( commands ) ,
721- noWaitStable : ( { joined } ) => ! joined . includes ( 'wait stable' ) ,
722- verifiesNamedExpectation : ( { joined } ) => / \b ( w a i t | i s | g e t | f i n d ) \b / . test ( joined ) ,
723- usesDogfoodEvidence : ( { joined } ) =>
724- / (?: \b s c r e e n s h o t \b | \b r e c o r d \b | \b l o g s \b | \b n e t w o r k \b | \b p e r f \b | \b t r a c e \b | d o g f o o d - o u t p u t ) / i. test (
725- joined ,
726- ) ,
727- usesValidationPrep,
728- opensAndCloses,
729- } ;
730-
731- /**
732- * Every check a case declares, normalized to a uniform `{ id, test }` shape:
733- * - `expectations`: named lookups into EXPECTATION_SCORERS (the original 4
734- * help-layout cases).
735- * - `matchers`: the check passes when the pattern matches the planned
736- * commands (ported skillgym quiz cases' `outputs`).
737- * - `forbidden`: the check passes when the pattern does NOT match (ported
738- * skillgym quiz cases' `forbiddenOutputs`).
739- */
740- function resolveChecks ( testCase ) {
741- const named = ( testCase . expectations ?? [ ] ) . map ( ( id ) => ( {
742- id,
743- test : ( context ) => scoreExpectation ( id , context ) ,
744- } ) ) ;
745- const matched = ( testCase . matchers ?? [ ] ) . map ( ( { id, pattern } ) => ( {
746- id,
747- test : ( context ) => pattern . test ( context . joined ) ,
748- } ) ) ;
749- const forbidden = ( testCase . forbidden ?? [ ] ) . map ( ( { id, pattern } ) => ( {
750- id,
751- test : ( context ) => ! pattern . test ( context . joined ) ,
752- } ) ) ;
753- return [ ...named , ...matched , ...forbidden ] ;
754- }
755-
756715function assertRuntimeConfiguration ( ) {
757716 parsePositiveInteger ( String ( CONCURRENCY ) , 'HELP_BENCH_CONCURRENCY' ) ;
758717 parsePositiveInteger ( String ( RUN_TIMEOUT_MS ) , 'HELP_BENCH_TIMEOUT_MS' ) ;
759718}
760719
761- function assertCaseDefinitions ( ) {
762- assertUniqueIds (
763- CASES . map ( ( { id } ) => id ) ,
764- 'benchmark case' ,
765- ) ;
766- for ( const testCase of CASES ) assertCaseDefinition ( testCase ) ;
767- }
768-
769- function assertCaseDefinition ( testCase ) {
770- const unknownExpectation = ( testCase . expectations ?? [ ] ) . find (
771- ( expectation ) => ! ( expectation in EXPECTATION_SCORERS ) ,
772- ) ;
773- if ( unknownExpectation ) {
774- throw new Error ( `Unknown expectation "${ unknownExpectation } " in case "${ testCase . id } ".` ) ;
775- }
776- assertUniqueIds (
777- resolveChecks ( testCase ) . map ( ( { id } ) => id ) ,
778- `check in case "${ testCase . id } "` ,
779- ) ;
780- }
781-
782- function assertUniqueIds ( ids , label ) {
783- const seen = new Set ( ) ;
784- const duplicates = new Set ( ) ;
785- for ( const id of ids ) {
786- if ( seen . has ( id ) ) duplicates . add ( id ) ;
787- seen . add ( id ) ;
788- }
789- if ( duplicates . size > 0 ) {
790- throw new Error ( `Duplicate ${ label } id(s): ${ [ ...duplicates ] . join ( ', ' ) } ` ) ;
791- }
792- }
793-
794- function countChecks ( testCase ) {
795- return resolveChecks ( testCase ) . length ;
796- }
797-
798- function scoreExpectations ( testCase , commands , raw , commandValidation ) {
799- const context = {
800- commands,
801- joined : commands . join ( '\n' ) . toLowerCase ( ) ,
802- raw,
803- commandValidation,
804- } ;
805- return Object . fromEntries ( resolveChecks ( testCase ) . map ( ( { id, test } ) => [ id , test ( context ) ] ) ) ;
806- }
807-
808- function scoreExpectation ( expectation , context ) {
809- const scorer = EXPECTATION_SCORERS [ expectation ] ;
810- if ( ! scorer ) throw new Error ( `Unknown expectation: ${ expectation } ` ) ;
811- return scorer ( context ) ;
812- }
813-
814- function allMutationsUseSettle ( commands ) {
815- const mutating = commands . filter ( isMutationCommand ) ;
816- return mutating . length > 0 && mutating . every ( ( command ) => command . includes ( '--settle' ) ) ;
817- }
818-
819- function isMutationCommand ( command ) {
820- return / ^ a g e n t - d e v i c e \s + (?: p r e s s | c l i c k | f i l l | l o n g p r e s s ) \b / . test ( command ) ;
821- }
822-
823720function countPassingChecks ( checks ) {
824721 return Object . values ( checks ) . filter ( Boolean ) . length ;
825722}
0 commit comments