Repository navigation
Expand file tree
/
Copy pathhelp-conformance-case-checks.mjs
More file actions
125 lines (111 loc) · 4.31 KB
/
Copy pathhelp-conformance-case-checks.mjs
File metadata and controls
125 lines (111 loc) · 4.31 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import {
findExistsSource,
opensAndCloses,
usesValidationPrep,
} from './help-conformance-expectations.mjs';
const VERIFICATION = new RegExp(String.raw`\b(?:wait|is|get)\b|${findExistsSource()}`);
const EXPECTATION_SCORERS = {
validPlanCommands: ({ commands, commandValidation }) =>
commands.length > 0 && commandValidation.every(({ issues }) => issues.length === 0),
fullPrefix: ({ commands }) =>
commands.length > 0 &&
commands.every(
(command) => !/^(open|snapshot|press|fill|click|longpress|wait|close)\b/.test(command),
),
usesSnapshotI: ({ commands }) => commands.some((command) => /\bsnapshot\b.*\s-i\b/.test(command)),
usesSettleOnMutations: ({ commands }) => allMutationsUseSettle(commands),
// help workflow teaches chaining confident consecutive steps with an
// unquoted &&. This reads the raw (pre-split) command lines, not `joined`
// (canonicalPlan flattens a chain into separate lines once the plan
// validator splits and validates each segment), so it is the only place
// that can tell whether the model actually chained.
usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)),
noWaitStable: ({ joined }) => !joined.includes('wait stable'),
verifiesNamedExpectation: ({ joined }) => VERIFICATION.test(joined),
usesDogfoodEvidence: ({ joined }) =>
/(?:\bscreenshot\b|\brecord\b|\blogs\b|\bnetwork\b|\bperf\b|\btrace\b|dogfood-output)/i.test(
joined,
),
usesValidationPrep,
opensAndCloses,
};
// The scorer registry itself, so a falsification-fixture completeness gate
// (scripts/__tests__/help-conformance-expectation-falsification.test.ts) can
// enumerate every named expectation instead of hand-maintaining a second list
// that silently drifts from EXPECTATION_SCORERS.
export const KNOWN_EXPECTATIONS = Object.keys(EXPECTATION_SCORERS);
export function assertCaseDefinitions(cases) {
assertUniqueIds(
cases.map(({ id }) => id),
'benchmark case',
);
for (const testCase of cases) assertCaseDefinition(testCase);
}
export function countChecks(testCase) {
return resolveChecks(testCase).length;
}
export function scoreExpectations(testCase, commands, raw, commandValidation) {
const context = {
commands,
joined: canonicalPlan(commandValidation).toLowerCase(),
raw,
commandValidation,
};
return Object.fromEntries(resolveChecks(testCase).map(({ id, test }) => [id, test(context)]));
}
function resolveChecks(testCase) {
const named = (testCase.expectations ?? []).map((id) => ({
id,
test: (context) => scoreExpectation(id, context),
}));
const matched = (testCase.matchers ?? []).map(({ id, pattern }) => ({
id,
test: (context) => pattern.test(context.joined),
}));
const forbidden = (testCase.forbidden ?? []).map(({ id, pattern }) => ({
id,
test: (context) => !pattern.test(context.joined),
}));
return [...named, ...matched, ...forbidden];
}
function canonicalPlan(commandValidation) {
return commandValidation.map(({ tokens }) => tokens.map(canonicalToken).join(' ')).join('\n');
}
function canonicalToken(token) {
return /\s/.test(token) ? JSON.stringify(token) : token;
}
function assertCaseDefinition(testCase) {
const unknownExpectation = (testCase.expectations ?? []).find(
(expectation) => !(expectation in EXPECTATION_SCORERS),
);
if (unknownExpectation) {
throw new Error(`Unknown expectation "${unknownExpectation}" in case "${testCase.id}".`);
}
assertUniqueIds(
resolveChecks(testCase).map(({ id }) => id),
`check in case "${testCase.id}"`,
);
}
function assertUniqueIds(ids, label) {
const seen = new Set();
const duplicates = new Set();
for (const id of ids) {
if (seen.has(id)) duplicates.add(id);
seen.add(id);
}
if (duplicates.size > 0) {
throw new Error(`Duplicate ${label} id(s): ${[...duplicates].join(', ')}`);
}
}
function scoreExpectation(expectation, context) {
const scorer = EXPECTATION_SCORERS[expectation];
if (!scorer) throw new Error(`Unknown expectation: ${expectation}`);
return scorer(context);
}
function allMutationsUseSettle(commands) {
const mutating = commands.filter(isMutationCommand);
return mutating.length > 0 && mutating.every((command) => command.includes('--settle'));
}
function isMutationCommand(command) {
return /^agent-device\s+(?:press|click|fill|longpress)\b/.test(command);
}