Skip to content

Commit 7a91891

Browse files
committed
fix: close help benchmark validation bypasses
1 parent 983249c commit 7a91891

10 files changed

Lines changed: 365 additions & 209 deletions

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,8 @@
109109
"perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios",
110110
"perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android",
111111
"lint": "oxlint . --deny-warnings",
112-
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
113-
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
112+
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
113+
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
114114
"fallow": "fallow audit --base origin/main",
115115
"fallow:all": "fallow --summary",
116116
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",

‎scripts/__tests__/help-conformance-bench.test.ts‎

Lines changed: 70 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os';
55
import { join } from 'node:path';
66
import { promisify } from 'node:util';
77
import { test } from 'vitest';
8+
import { scoreExpectations } from '../help-conformance-case-checks.mjs';
89
import { validateAgentDeviceCommand } from '../help-conformance-command-validator.ts';
910
import { opensAndCloses, usesValidationPrep } from '../help-conformance-expectations.mjs';
1011
import { validatePlanCommands } from '../help-conformance-plan-validator.mjs';
@@ -330,24 +331,86 @@ test('runner output distinguishes model commands from infrastructure errors', ()
330331
detectRunnerError(JSON.stringify({ type: 'error', message: 'rate limit exceeded' })),
331332
'rate limit exceeded',
332333
);
334+
assert.equal(
335+
detectRunnerError(
336+
JSON.stringify({
337+
status: 'failed',
338+
commands: ['agent-device snapshot -i'],
339+
}),
340+
),
341+
undefined,
342+
);
333343
assert.equal(detectRunnerError(''), 'Runner returned empty output.');
334344
});
335345

336346
test('plan validator rejects shell projection and non-permitted executables', async () => {
337-
const [redirected, piped, shellCommand] = await validatePlanCommands([
338-
'agent-device snapshot -i > evidence.json',
339-
'agent-device snapshot -i | tee evidence.txt',
340-
'echo done',
341-
]);
342-
assert.equal(redirected.issues[0]?.kind, 'shell-projection');
343-
assert.equal(piped.issues[0]?.kind, 'shell-projection');
347+
const [redirected, piped, newline, carriageReturn, escapedNewline, shellCommand] =
348+
await validatePlanCommands([
349+
'agent-device snapshot -i > evidence.json',
350+
'agent-device snapshot -i | tee evidence.txt',
351+
'agent-device press @e1 --settle\ncat',
352+
'agent-device press @e1 --settle\rcat',
353+
'agent-device press @e1 --settle\\\ncat',
354+
'echo done',
355+
]);
356+
for (const result of [redirected, piped, newline, carriageReturn, escapedNewline]) {
357+
assert.equal(result.issues[0]?.kind, 'shell-projection');
358+
}
344359
assert.equal(shellCommand.issues[0]?.kind, 'executable-policy');
345360

361+
const [quotedNewline] = await validatePlanCommands([
362+
'agent-device fill label=Biography "line one\nline two" --settle',
363+
]);
364+
assert.equal(quotedNewline.issues.length, 0);
365+
assert.deepEqual(quotedNewline.tokens, [
366+
'agent-device',
367+
'fill',
368+
'label=Biography',
369+
'line one\nline two',
370+
'--settle',
371+
]);
372+
346373
const [placeholder] = await validatePlanCommands(['agent-device press @<search-ref> --settle']);
347374
assert.ok(placeholder.issues.some(({ kind }) => kind === 'pseudo-ref'));
348375
assert.ok(placeholder.issues.some(({ kind }) => kind === 'shell-projection'));
349376
});
350377

378+
test('case matchers score parsed tokens so shell quoting does not change results', async () => {
379+
const commands = [
380+
'agent-device open "com.example.shop"',
381+
`agent-device fill 'label=Search' "react native" --settle`,
382+
`agent-device open 'settings'`,
383+
];
384+
const commandValidation = await validatePlanCommands(commands);
385+
const checks = scoreExpectations(
386+
{
387+
matchers: [
388+
{
389+
id: 'opensKnownDogfoodApp',
390+
pattern: /\bagent-device\s+open\s+com\.example\.shop\b/i,
391+
},
392+
{
393+
id: 'fillsExpectedSearch',
394+
pattern:
395+
/\bagent-device\s+fill\b[^\n]*(?:"react native"|'react native')[^\n]*--settle\b/i,
396+
},
397+
{
398+
id: 'opensSettings',
399+
pattern: /\bagent-device\s+open\s+(?:settings|com\.apple\.Preferences)\b/i,
400+
},
401+
],
402+
},
403+
commands,
404+
'',
405+
commandValidation,
406+
);
407+
assert.deepEqual(checks, {
408+
opensKnownDogfoodApp: true,
409+
fillsExpectedSearch: true,
410+
opensSettings: true,
411+
});
412+
});
413+
351414
test('plan validator applies narrow grammar to permitted external commands', async () => {
352415
const results = await validatePlanCommands(
353416
[

‎scripts/help-conformance-bench.mjs‎

Lines changed: 6 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
44
import { basename, join } from 'node:path';
55
import { fileURLToPath } from 'node:url';
66
import { promisify } from 'node:util';
7-
import { opensAndCloses, usesValidationPrep } from './help-conformance-expectations.mjs';
7+
import {
8+
assertCaseDefinitions,
9+
countChecks,
10+
scoreExpectations,
11+
} from './help-conformance-case-checks.mjs';
812
import { validatePlanCommands } from './help-conformance-plan-validator.mjs';
913
import { detectRunnerError, extractCommands } from './help-conformance-runner-output.mjs';
1014
import { summarizeResults } from './help-conformance-summary.mjs';
@@ -374,7 +378,7 @@ function withDefault(value, fallback) {
374378
async function main() {
375379
const args = parseArgs(process.argv.slice(2));
376380
assertRuntimeConfiguration();
377-
assertCaseDefinitions();
381+
assertCaseDefinitions(CASES);
378382
const outDir = resolveOutDir(args);
379383
await mkdir(outDir, { recursive: true });
380384
const selectedCases = selectCases(args.cases);
@@ -708,118 +712,11 @@ async function runCodex(model, prompt, outDir) {
708712
return lastMessage.trim().length > 0 ? lastMessage : stdout;
709713
}
710714

711-
const EXPECTATION_SCORERS = {
712-
validPlanCommands: ({ commands, commandValidation }) =>
713-
commands.length > 0 && commandValidation.every(({ issues }) => issues.length === 0),
714-
fullPrefix: ({ commands }) =>
715-
commands.length > 0 &&
716-
commands.every(
717-
(command) => !/^(open|snapshot|press|fill|click|longpress|wait|close)\b/.test(command),
718-
),
719-
usesSnapshotI: ({ commands }) => commands.some((command) => /\bsnapshot\b.*\s-i\b/.test(command)),
720-
usesSettleOnMutations: ({ commands }) => allMutationsUseSettle(commands),
721-
noWaitStable: ({ joined }) => !joined.includes('wait stable'),
722-
verifiesNamedExpectation: ({ joined }) => /\b(wait|is|get|find)\b/.test(joined),
723-
usesDogfoodEvidence: ({ joined }) =>
724-
/(?:\bscreenshot\b|\brecord\b|\blogs\b|\bnetwork\b|\bperf\b|\btrace\b|dogfood-output)/i.test(
725-
joined,
726-
),
727-
usesValidationPrep,
728-
opensAndCloses,
729-
};
730-
731-
/**
732-
* Every check a case declares, normalized to a uniform `{ id, test }` shape:
733-
* - `expectations`: named lookups into EXPECTATION_SCORERS (the original 4
734-
* help-layout cases).
735-
* - `matchers`: the check passes when the pattern matches the planned
736-
* commands (ported skillgym quiz cases' `outputs`).
737-
* - `forbidden`: the check passes when the pattern does NOT match (ported
738-
* skillgym quiz cases' `forbiddenOutputs`).
739-
*/
740-
function resolveChecks(testCase) {
741-
const named = (testCase.expectations ?? []).map((id) => ({
742-
id,
743-
test: (context) => scoreExpectation(id, context),
744-
}));
745-
const matched = (testCase.matchers ?? []).map(({ id, pattern }) => ({
746-
id,
747-
test: (context) => pattern.test(context.joined),
748-
}));
749-
const forbidden = (testCase.forbidden ?? []).map(({ id, pattern }) => ({
750-
id,
751-
test: (context) => !pattern.test(context.joined),
752-
}));
753-
return [...named, ...matched, ...forbidden];
754-
}
755-
756715
function assertRuntimeConfiguration() {
757716
parsePositiveInteger(String(CONCURRENCY), 'HELP_BENCH_CONCURRENCY');
758717
parsePositiveInteger(String(RUN_TIMEOUT_MS), 'HELP_BENCH_TIMEOUT_MS');
759718
}
760719

761-
function assertCaseDefinitions() {
762-
assertUniqueIds(
763-
CASES.map(({ id }) => id),
764-
'benchmark case',
765-
);
766-
for (const testCase of CASES) assertCaseDefinition(testCase);
767-
}
768-
769-
function assertCaseDefinition(testCase) {
770-
const unknownExpectation = (testCase.expectations ?? []).find(
771-
(expectation) => !(expectation in EXPECTATION_SCORERS),
772-
);
773-
if (unknownExpectation) {
774-
throw new Error(`Unknown expectation "${unknownExpectation}" in case "${testCase.id}".`);
775-
}
776-
assertUniqueIds(
777-
resolveChecks(testCase).map(({ id }) => id),
778-
`check in case "${testCase.id}"`,
779-
);
780-
}
781-
782-
function assertUniqueIds(ids, label) {
783-
const seen = new Set();
784-
const duplicates = new Set();
785-
for (const id of ids) {
786-
if (seen.has(id)) duplicates.add(id);
787-
seen.add(id);
788-
}
789-
if (duplicates.size > 0) {
790-
throw new Error(`Duplicate ${label} id(s): ${[...duplicates].join(', ')}`);
791-
}
792-
}
793-
794-
function countChecks(testCase) {
795-
return resolveChecks(testCase).length;
796-
}
797-
798-
function scoreExpectations(testCase, commands, raw, commandValidation) {
799-
const context = {
800-
commands,
801-
joined: commands.join('\n').toLowerCase(),
802-
raw,
803-
commandValidation,
804-
};
805-
return Object.fromEntries(resolveChecks(testCase).map(({ id, test }) => [id, test(context)]));
806-
}
807-
808-
function scoreExpectation(expectation, context) {
809-
const scorer = EXPECTATION_SCORERS[expectation];
810-
if (!scorer) throw new Error(`Unknown expectation: ${expectation}`);
811-
return scorer(context);
812-
}
813-
814-
function allMutationsUseSettle(commands) {
815-
const mutating = commands.filter(isMutationCommand);
816-
return mutating.length > 0 && mutating.every((command) => command.includes('--settle'));
817-
}
818-
819-
function isMutationCommand(command) {
820-
return /^agent-device\s+(?:press|click|fill|longpress)\b/.test(command);
821-
}
822-
823720
function countPassingChecks(checks) {
824721
return Object.values(checks).filter(Boolean).length;
825722
}
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import { opensAndCloses, usesValidationPrep } from './help-conformance-expectations.mjs';
2+
3+
const EXPECTATION_SCORERS = {
4+
validPlanCommands: ({ commands, commandValidation }) =>
5+
commands.length > 0 && commandValidation.every(({ issues }) => issues.length === 0),
6+
fullPrefix: ({ commands }) =>
7+
commands.length > 0 &&
8+
commands.every(
9+
(command) => !/^(open|snapshot|press|fill|click|longpress|wait|close)\b/.test(command),
10+
),
11+
usesSnapshotI: ({ commands }) => commands.some((command) => /\bsnapshot\b.*\s-i\b/.test(command)),
12+
usesSettleOnMutations: ({ commands }) => allMutationsUseSettle(commands),
13+
noWaitStable: ({ joined }) => !joined.includes('wait stable'),
14+
verifiesNamedExpectation: ({ joined }) => /\b(wait|is|get|find)\b/.test(joined),
15+
usesDogfoodEvidence: ({ joined }) =>
16+
/(?:\bscreenshot\b|\brecord\b|\blogs\b|\bnetwork\b|\bperf\b|\btrace\b|dogfood-output)/i.test(
17+
joined,
18+
),
19+
usesValidationPrep,
20+
opensAndCloses,
21+
};
22+
23+
export function assertCaseDefinitions(cases) {
24+
assertUniqueIds(
25+
cases.map(({ id }) => id),
26+
'benchmark case',
27+
);
28+
for (const testCase of cases) assertCaseDefinition(testCase);
29+
}
30+
31+
export function countChecks(testCase) {
32+
return resolveChecks(testCase).length;
33+
}
34+
35+
export function scoreExpectations(testCase, commands, raw, commandValidation) {
36+
const context = {
37+
commands,
38+
joined: canonicalPlan(commandValidation).toLowerCase(),
39+
raw,
40+
commandValidation,
41+
};
42+
return Object.fromEntries(resolveChecks(testCase).map(({ id, test }) => [id, test(context)]));
43+
}
44+
45+
function resolveChecks(testCase) {
46+
const named = (testCase.expectations ?? []).map((id) => ({
47+
id,
48+
test: (context) => scoreExpectation(id, context),
49+
}));
50+
const matched = (testCase.matchers ?? []).map(({ id, pattern }) => ({
51+
id,
52+
test: (context) => pattern.test(context.joined),
53+
}));
54+
const forbidden = (testCase.forbidden ?? []).map(({ id, pattern }) => ({
55+
id,
56+
test: (context) => !pattern.test(context.joined),
57+
}));
58+
return [...named, ...matched, ...forbidden];
59+
}
60+
61+
function canonicalPlan(commandValidation) {
62+
return commandValidation.map(({ tokens }) => tokens.map(canonicalToken).join(' ')).join('\n');
63+
}
64+
65+
function canonicalToken(token) {
66+
return /\s/.test(token) ? JSON.stringify(token) : token;
67+
}
68+
69+
function assertCaseDefinition(testCase) {
70+
const unknownExpectation = (testCase.expectations ?? []).find(
71+
(expectation) => !(expectation in EXPECTATION_SCORERS),
72+
);
73+
if (unknownExpectation) {
74+
throw new Error(`Unknown expectation "${unknownExpectation}" in case "${testCase.id}".`);
75+
}
76+
assertUniqueIds(
77+
resolveChecks(testCase).map(({ id }) => id),
78+
`check in case "${testCase.id}"`,
79+
);
80+
}
81+
82+
function assertUniqueIds(ids, label) {
83+
const seen = new Set();
84+
const duplicates = new Set();
85+
for (const id of ids) {
86+
if (seen.has(id)) duplicates.add(id);
87+
seen.add(id);
88+
}
89+
if (duplicates.size > 0) {
90+
throw new Error(`Duplicate ${label} id(s): ${[...duplicates].join(', ')}`);
91+
}
92+
}
93+
94+
function scoreExpectation(expectation, context) {
95+
const scorer = EXPECTATION_SCORERS[expectation];
96+
if (!scorer) throw new Error(`Unknown expectation: ${expectation}`);
97+
return scorer(context);
98+
}
99+
100+
function allMutationsUseSettle(commands) {
101+
const mutating = commands.filter(isMutationCommand);
102+
return mutating.length > 0 && mutating.every((command) => command.includes('--settle'));
103+
}
104+
105+
function isMutationCommand(command) {
106+
return /^agent-device\s+(?:press|click|fill|longpress)\b/.test(command);
107+
}

0 commit comments

Comments
 (0)