Skip to content

Commit 250e30a

Browse files
authored
test(bench): falsification fixtures for oracles + typed runner outcomes (callstack#1893)
* test(bench): falsification fixtures for oracles + typed runner outcomes Two deterministic PR-time quality gates for the help-conformance bench (the repo's single non-gating small-model planning oracle): - Every EXPECTATION_SCORERS entry in help-conformance-case-checks.mjs now has a falsification fixture (a minimal passing witness plus at least one known-bad counterexample, and a metamorphic variant where useful) in the new help-conformance-expectation-fixtures.ts, run through the real validatePlanCommands/scoreExpectations pipeline. help-conformance-expectation-falsification.test.ts is the "what enumerates N" completeness gate: a new named expectation with no fixture fails it. Counterexamples cover swallowed lifecycle command prefixes, unsupported flags/selectors, pseudo refs, shell operators, and invalid positional ordering. - help-conformance-runner-output.mjs now returns a discriminated RunnerOutcome ({kind:'success',commands}|{kind:'runner-error', message,reason}) instead of a raw-string success inference. Only a 'success' outcome ever reaches validatePlanCommands/scoreExpectations in runCase, so a runner-error result can no longer also carry model-validation checks, and an all-runner-error aggregate now reports passRate: null (rendered as "N/A") instead of "0/0 (0%)". Fixes callstack#1481 * refactor(bench): dedupe RunnerOutcome construction, drop leftover narrowing Thermo-nuclear pass over 4b2df0a's diff: - help-conformance-bench.mjs's runOutcome() catch block was hand-building the exact {kind:'runner-error', raw, message, reason} shape that runner-output.mjs's private runnerError() helper already constructs for its own two error paths. Export it as runnerErrorOutcome so the discriminated union has exactly one constructor for its error variant, reused by both error sources instead of duplicated. - runCase's two return branches repeated the same {runner, caseId, trial, outputPath} fields; pulled into a shared `base` object. - Reverted bench.test.ts's rateLimitedOutcome block: it had an explicit `: RunnerOutcome` annotation and an if/throw narrowing guard, added only to give fallow's dead-code checker a "real consumer" of the type before the actual fix (adding the .d.mts to .fallowrc.json's ignorePatterns, matching the existing sample-outputs.d.mts precedent) was found. That workaround is now unnecessary scaffolding — replaced with the same flat assert.deepEqual style the surrounding assertions already use.
1 parent d072819 commit 250e30a

10 files changed

Lines changed: 488 additions & 55 deletions

‎.fallowrc.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@
4242
"scripts/maestro-conformance/corpus/**",
4343
"apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan",
4444
"scripts/write-xcuitest-cache-metadata.mjs",
45-
"scripts/help-conformance-sample-outputs.d.mts"
45+
"scripts/help-conformance-sample-outputs.d.mts",
46+
"scripts/help-conformance-runner-output.d.mts"
4647
],
4748
"ignoreDependencies": [
4849
"@theme",

‎scripts/__tests__/help-conformance-bench.test.ts‎

Lines changed: 72 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { scoreExpectations } from '../help-conformance-case-checks.mjs';
99
import { validateAgentDeviceCommand } from '../help-conformance-command-validator.ts';
1010
import { opensAndCloses, usesValidationPrep } from '../help-conformance-expectations.mjs';
1111
import { validatePlanCommands } from '../help-conformance-plan-validator.mjs';
12-
import { detectRunnerError, extractCommands } from '../help-conformance-runner-output.mjs';
12+
import { classifyRunnerOutput, extractCommands } from '../help-conformance-runner-output.mjs';
1313
import { summarizeResults } from '../help-conformance-summary.mjs';
1414

1515
const execFileAsync = promisify(execFile);
@@ -324,35 +324,52 @@ test('validation prep accepts intervening checks and the Android build path in o
324324
);
325325
});
326326

327-
test('runner output distinguishes model commands from infrastructure errors', () => {
327+
test('runner output classifies model commands as success and infrastructure noise as runner-error', () => {
328328
const successEnvelope = JSON.stringify({
329329
is_error: false,
330330
result: JSON.stringify({ commands: ['agent-device snapshot -i'] }),
331331
});
332332
assert.deepEqual(extractCommands(successEnvelope), ['agent-device snapshot -i']);
333-
assert.equal(detectRunnerError(successEnvelope), undefined);
333+
assert.deepEqual(classifyRunnerOutput(successEnvelope), {
334+
kind: 'success',
335+
raw: successEnvelope,
336+
commands: ['agent-device snapshot -i'],
337+
});
334338

335339
const claudeError = JSON.stringify({
336340
is_error: true,
337341
result: 'API Error: Unable to connect to API',
338342
});
339-
assert.equal(detectRunnerError(claudeError), 'API Error: Unable to connect to API');
343+
assert.deepEqual(classifyRunnerOutput(claudeError), {
344+
kind: 'runner-error',
345+
raw: claudeError,
346+
message: 'API Error: Unable to connect to API',
347+
reason: 'error-envelope',
348+
});
340349
assert.deepEqual(extractCommands(claudeError), []);
341350

342-
assert.equal(
343-
detectRunnerError(JSON.stringify({ type: 'error', message: 'rate limit exceeded' })),
344-
'rate limit exceeded',
345-
);
346-
assert.equal(
347-
detectRunnerError(
348-
JSON.stringify({
349-
status: 'failed',
350-
commands: ['agent-device snapshot -i'],
351-
}),
352-
),
353-
undefined,
354-
);
355-
assert.equal(detectRunnerError(''), 'Runner returned empty output.');
351+
const rateLimited = JSON.stringify({ type: 'error', message: 'rate limit exceeded' });
352+
assert.deepEqual(classifyRunnerOutput(rateLimited), {
353+
kind: 'runner-error',
354+
raw: rateLimited,
355+
message: 'rate limit exceeded',
356+
reason: 'error-envelope',
357+
});
358+
359+
// status: 'failed' alone does not shadow a real commands payload.
360+
const statusFailedWithCommands = JSON.stringify({
361+
status: 'failed',
362+
commands: ['agent-device snapshot -i'],
363+
});
364+
assert.equal(classifyRunnerOutput(statusFailedWithCommands).kind, 'success');
365+
366+
const empty = classifyRunnerOutput('');
367+
assert.deepEqual(empty, {
368+
kind: 'runner-error',
369+
raw: '',
370+
message: 'Runner returned empty output.',
371+
reason: 'empty-output',
372+
});
356373
});
357374

358375
test('plan validator rejects shell projection and non-permitted executables', async () => {
@@ -601,20 +618,16 @@ test('aggregate summary exposes stability and failure taxonomy per runner x case
601618
checks: { validPlanCommands: true, usesSettle: true },
602619
commandValidation: [],
603620
},
621+
// A runner-error result (see runCase in help-conformance-bench.mjs)
622+
// never carries checks/commandValidation alongside runnerError: an
623+
// infrastructure failure must not also read as a model validation
624+
// failure in the aggregate taxonomy.
604625
{
605626
runner: 'claude:haiku',
606627
caseId: 'metamorphic',
607628
passed: false,
608-
checks: { validPlanCommands: false, usesSettle: true },
609-
commandValidation: [
610-
{
611-
issues: [
612-
{ kind: 'pseudo-ref', error: 'bad ref' },
613-
{ kind: 'shell-projection', error: 'bad shell' },
614-
],
615-
},
616-
],
617629
runnerError: 'failed',
630+
runnerErrorReason: 'process-failure',
618631
},
619632
{
620633
runner: 'claude:haiku',
@@ -642,3 +655,35 @@ test('aggregate summary exposes stability and failure taxonomy per runner x case
642655
},
643656
]);
644657
});
658+
659+
test('a group with only runner-error trials reports passRate: null, not 0/0', () => {
660+
const summary = summarizeResults([
661+
{
662+
runner: 'codex:gpt',
663+
caseId: 'metamorphic',
664+
passed: false,
665+
runnerError: 'timed out',
666+
runnerErrorReason: 'process-failure',
667+
},
668+
{
669+
runner: 'codex:gpt',
670+
caseId: 'metamorphic',
671+
passed: false,
672+
runnerError: 'Runner returned empty output.',
673+
runnerErrorReason: 'empty-output',
674+
},
675+
]);
676+
assert.deepEqual(summary, [
677+
{
678+
runner: 'codex:gpt',
679+
caseId: 'metamorphic',
680+
trials: 2,
681+
evaluatedTrials: 0,
682+
passed: 0,
683+
failedChecks: {},
684+
validationIssues: {},
685+
runnerErrors: 2,
686+
passRate: null,
687+
},
688+
]);
689+
});
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import { KNOWN_EXPECTATIONS, scoreExpectations } from '../help-conformance-case-checks.mjs';
4+
import { validatePlanCommands } from '../help-conformance-plan-validator.mjs';
5+
import {
6+
EXPECTATION_FIXTURES,
7+
type ExpectationWitness,
8+
} from './help-conformance-expectation-fixtures.ts';
9+
10+
// "What enumerates N" for the expectation surface (see the analogous gates
11+
// for help topics and error-recovery quizzes): KNOWN_EXPECTATIONS is the
12+
// scorer registry itself (help-conformance-case-checks.mjs
13+
// EXPECTATION_SCORERS), so a new named expectation with no falsification
14+
// fixture fails this gate instead of silently shipping unfalsifiable. An
15+
// oracle nobody has proven can fail is not an oracle — it is decoration that
16+
// reports every plan as passing whether or not its regex or predicate still
17+
// matches anything.
18+
19+
test('every named expectation has a falsification fixture', () => {
20+
const missing = KNOWN_EXPECTATIONS.filter((id) => !(id in EXPECTATION_FIXTURES));
21+
assert.deepEqual(
22+
missing,
23+
[],
24+
`add a scripts/__tests__/help-conformance-expectation-fixtures.ts entry for: ${missing.join(', ')}`,
25+
);
26+
});
27+
28+
test('the fixture registry names only real, current expectations', () => {
29+
const known = new Set<string>(KNOWN_EXPECTATIONS);
30+
const stale = Object.keys(EXPECTATION_FIXTURES).filter((id) => !known.has(id));
31+
assert.deepEqual(
32+
stale,
33+
[],
34+
`fixture(s) for a retired/renamed expectation: ${stale.join(', ')} — remove them`,
35+
);
36+
});
37+
38+
test('every fixture set declares a passing witness and at least one counterexample', () => {
39+
for (const [id, fixture] of Object.entries(EXPECTATION_FIXTURES)) {
40+
assert.ok(fixture.passing, `expectation "${id}" needs a passing witness`);
41+
assert.ok(
42+
fixture.counterexamples.length > 0,
43+
`expectation "${id}" needs at least one known-bad counterexample`,
44+
);
45+
for (const counter of fixture.counterexamples) {
46+
assert.ok(
47+
counter.id.trim().length > 0,
48+
`expectation "${id}" has a counterexample with no id`,
49+
);
50+
}
51+
}
52+
});
53+
54+
/** Runs a witness through the real production scorer pipeline, not a reimplementation. */
55+
async function score(id: string, witness: ExpectationWitness): Promise<boolean> {
56+
const commandValidation = await validatePlanCommands(witness.commands, {
57+
allowedExternalCommands: witness.allowedExternalCommands,
58+
});
59+
const raw = witness.commands.join('\n');
60+
const checks = scoreExpectations(
61+
{ expectations: [id] },
62+
witness.commands,
63+
raw,
64+
commandValidation,
65+
);
66+
return checks[id] === true;
67+
}
68+
69+
// One vitest test per witness/counterexample (rather than one loop inside a
70+
// single test) so a failure names the exact fixture that broke, and so the
71+
// many real validatePlanCommands subprocess calls this drives don't compound
72+
// into one test's timeout budget.
73+
for (const [id, fixture] of Object.entries(EXPECTATION_FIXTURES)) {
74+
test(`${id}: passing witness scores true`, async () => {
75+
assert.equal(
76+
await score(id, fixture.passing),
77+
true,
78+
`expectation "${id}" passing witness must score true: ${JSON.stringify(fixture.passing.commands)}`,
79+
);
80+
});
81+
82+
for (const counter of fixture.counterexamples) {
83+
test(`${id}: counterexample "${counter.id}" scores false`, async () => {
84+
assert.equal(
85+
await score(id, counter),
86+
false,
87+
`expectation "${id}" counterexample "${counter.id}" must score false: ${JSON.stringify(counter.commands)}`,
88+
);
89+
});
90+
}
91+
92+
if (fixture.metamorphic) {
93+
const metamorphic = fixture.metamorphic;
94+
test(`${id}: metamorphic variant differs from the witness and still scores true`, async () => {
95+
assert.notDeepEqual(
96+
metamorphic.commands,
97+
fixture.passing.commands,
98+
`expectation "${id}" metamorphic variant must actually differ from the passing witness`,
99+
);
100+
assert.equal(
101+
await score(id, metamorphic),
102+
true,
103+
`expectation "${id}" metamorphic variant must still score true: ${JSON.stringify(metamorphic.commands)}`,
104+
);
105+
});
106+
}
107+
}

0 commit comments

Comments
 (0)