#!/usr/bin/env node

import { execFile } from 'node:child_process';

import { mkdir, readFile, writeFile } from 'node:fs/promises';

import { basename, join } from 'node:path';

import { fileURLToPath } from 'node:url';

import { promisify } from 'node:util';

import {

assertCaseDefinitions,

countChecks,

scoreExpectations,

} from './help-conformance-case-checks.mjs';

import { CASES } from './help-conformance-cases.mjs';

import { validatePlanCommands } from './help-conformance-plan-validator.mjs';

import { classifyRunnerOutput, runnerErrorOutcome } from './help-conformance-runner-output.mjs';

import { summarizeResults } from './help-conformance-summary.mjs';

const execFileAsync = promisify(execFile);

const ROOT = fileURLToPath(new URL('..', import.meta.url));

const OUT_DIR = process.env.HELP_BENCH_OUT ?? join(ROOT, '.tmp', 'help-conformance-bench');

const RUN_TIMEOUT_MS = Number(process.env.HELP_BENCH_TIMEOUT_MS ?? 90_000);

// Runner x case pairs run concurrently, capped low: these are paid LLM calls

// and the CLI help subprocess calls behind loadDocs share this same machine.

const CONCURRENCY = Number(process.env.HELP_BENCH_CONCURRENCY ?? 4);

const DEFAULT_RUNNERS = ['codex:gpt-5.6-luna', 'claude:claude-haiku-4-5'];

const USAGE = `Usage: node scripts/help-conformance-bench.mjs [options]

Feeds a help slice + task into one non-agentic LLM call per runner x case and

scores the returned command plan with production parser-backed syntax checks.

Options:

--runner <kind:model> Add one runner (repeatable). Default: ${DEFAULT_RUNNERS.join(', ')}

--runners <a,b> Comma-separated runner list

--case <id> Add one case id (repeatable). Default: all cases

--cases <a,b> Comma-separated case id list

--repeat <n> Run every runner x case pair n times (default: 1)

--out <dir> Output directory (default: .tmp/help-conformance-bench)

--override-doc <topicId>=<path>

Grade a DRAFT doc: load this topic's text from the file

instead of the live CLI help. Repeatable; the last

occurrence per topic wins. Override text goes through the

same post-processing as the live source (e.g. the

--help:first30 doc id is still capped to its first 30

lines), so an A/B grade compares like with like.

--dry-run Build prompts and write the report without any LLM calls

--help Show this usage text

Environment:

HELP_BENCH_CONCURRENCY Concurrent runner x case calls (default: 4)

HELP_BENCH_TIMEOUT_MS Per-call timeout (default: 90000)

HELP_BENCH_OUT Default output directory`;

const OPTION_SPECS = {

'--runner': { target: 'runners', mode: 'append' },

'--runners': { target: 'runners', mode: 'csv' },

'--case': { target: 'cases', mode: 'append' },

'--cases': { target: 'cases', mode: 'csv' },

'--repeat': { target: 'repeat', mode: 'positiveInteger' },

'--out': { target: 'outDir', mode: 'value' },

'--override-doc': { target: 'overrideDocs', mode: 'keyvalue' },

};

const OPTION_APPLIERS = {

append: (args, target, value) => {

args[target] = [...(args[target] ?? []), value];

},

csv: (args, target, value) => {

args[target] = [...(args[target] ?? []), ...value.split(',').filter(Boolean)];

},

value: (args, target, value) => {

args[target] = value;

},

positiveInteger: (args, target, value) => {

args[target] = parsePositiveInteger(value, `--${target}`);

},

keyvalue: (args, target, value) => {

const separatorIndex = value.indexOf('=');

if (separatorIndex <= 0) {

throw new Error(`--override-doc expects <topicId>=<path>, got: ${value}`);

}

const topicId = value.slice(0, separatorIndex);

const path = value.slice(separatorIndex + 1);

const map = args[target] ?? new Map();

map.set(topicId, path);

args[target] = map;

},

};

function parseArgs(argv) {

const args = { runners: undefined, cases: undefined, dryRun: false };

readArgs(args, argv, 0);

applyDefaultArgs(args);

return args;

}

function readArgs(args, argv, index) {

if (index >= argv.length) return;

readArgs(args, argv, readArg(args, argv, index));

}

function readArg(args, argv, index) {

const arg = argv[index];

if (arg === '--help' || arg === '-h') {

console.log(USAGE);

process.exit(0);

}

if (arg === '--dry-run') return applyDryRun(args, index);

applyOption(args, optionSpec(arg), argv[index + 1]);

return index + 2;

}

function applyDryRun(args, index) {

args.dryRun = true;

return index + 1;

}

function optionSpec(arg) {

const spec = OPTION_SPECS[arg];

if (!spec) throw new Error(`Unknown argument: ${arg}. Run with --help for usage.`);

return spec;

}

function applyOption(args, spec, value) {

assertOptionValue(spec, value);

OPTION_APPLIERS[spec.mode](args, spec.target, value);

}

function assertOptionValue(spec, value) {

if (value === undefined) throw new Error(`Missing value for ${spec.target}`);

}

function applyDefaultArgs(args) {

args.runners = withDefault(args.runners, DEFAULT_RUNNERS);

args.cases = withDefault(

args.cases,

CASES.map((testCase) => testCase.id),

);

args.overrideDocs = withDefault(args.overrideDocs, new Map());

args.repeat = withDefault(args.repeat, 1);

}

function withDefault(value, fallback) {

return value ?? fallback;

}

async function main() {

const args = parseArgs(process.argv.slice(2));

assertRuntimeConfiguration();

assertCaseDefinitions(CASES);

const outDir = resolveOutDir(args);

await mkdir(outDir, { recursive: true });

const selectedCases = selectCases(args.cases);

const docIds = requiredDocIds(selectedCases);

assertOverrideDocIds(args.overrideDocs, docIds);

const docs = await loadDocs(docIds, args.overrideDocs);

const results = await runBenchmarkMatrix(

args.runners,

selectedCases,

docs,

outDir,

args.dryRun,

args.repeat,

);

const timestamp = Date.now();

const reportPath = join(outDir, `report-${timestamp}.json`);

await writeFile(reportPath, `${JSON.stringify(results, null, 2)}\n`);

console.log(`Wrote ${reportPath}`);

if (!args.dryRun && args.repeat > 1) {

const summary = summarizeResults(results);

const summaryPath = join(outDir, `summary-${timestamp}.json`);

await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);

printSummary(summary);

console.log(`Wrote ${summaryPath}`);

}

updateExitCode(results, args.dryRun);

}

function resolveOutDir(args) {

return args.outDir ?? OUT_DIR;

}

function selectCases(caseIds) {

const knownCaseIds = new Set(CASES.map((testCase) => testCase.id));

const unknown = [...new Set(caseIds.filter((caseId) => !knownCaseIds.has(caseId)))];

if (unknown.length > 0) throw new Error(`Unknown benchmark case id(s): ${unknown.join(', ')}`);

const selectedCases = CASES.filter((testCase) => caseIds.includes(testCase.id));

assertCasesSelected(selectedCases);

return selectedCases;

}

function assertCasesSelected(selectedCases) {

if (selectedCases.length === 0) throw new Error('No benchmark cases selected.');

}

function requiredDocIds(selectedCases) {

return [...new Set(selectedCases.flatMap((testCase) => testCase.docs))];

}

// A typo'd or stale --override-doc topic id must not silently grade the real

// doc while the caller believes the draft was measured: fail fast instead.

function assertOverrideDocIds(overrideDocs, docIds) {

const unknown = [...overrideDocs.keys()].filter((topicId) => !docIds.includes(topicId));

if (unknown.length === 0) return;

throw new Error(

`--override-doc topic id(s) not used by the selected cases: ${unknown.join(', ')}. Valid doc ids: ${docIds.join(', ')}.`,

);

}

function updateExitCode(results, dryRun) {

if (!dryRun && results.some((result) => !result.passed)) process.exitCode = 1;

}

async function runBenchmarkMatrix(runners, selectedCases, docs, outDir, dryRun, repeat) {

const entries = runners.flatMap((runner) =>

selectedCases.flatMap((testCase) =>

Array.from({ length: repeat }, (_, index) => ({

runner,

testCase,

trial: index + 1,

})),

),

);

// Concurrency-capped, but results print in the original runner x case

// matrix order (not completion order) once every entry has settled, so

// output stays as readable as the old sequential loop.

const results = await mapWithConcurrency(entries, CONCURRENCY, ({ runner, testCase, trial }) =>

runBenchmarkEntry(runner, testCase, docs, outDir, dryRun, trial, repeat),

);

for (const result of results) printResult(result, dryRun, repeat);

return results;

}

async function mapWithConcurrency(items, limit, worker) {

const results = new Array(items.length);

let nextIndex = 0;

async function runNext() {

for (;;) {

const current = nextIndex;

nextIndex += 1;

if (current >= items.length) return;

results[current] = await worker(items[current], current);

}

}

const workerCount = Math.max(1, Math.min(limit, items.length));

await Promise.all(Array.from({ length: workerCount }, runNext));

return results;

}

async function runBenchmarkEntry(runner, testCase, docs, outDir, dryRun, trial, repeat) {

const prompt = buildPrompt(testCase, docs);

return dryRun

? { runner, caseId: testCase.id, trial, prompt }

: runCase(runner, testCase, prompt, outDir, trial, repeat);

}

function printResult(result, dryRun, repeat) {

if (dryRun) return;

const trial = repeat > 1 ? ` trial=${result.trial}/${repeat}` : '';

console.log(

`${resultStatus(result)} ${result.runner} ${result.caseId}${trial} ${resultDetail(result)}`,

);

}

function resultStatus(result) {

if (result.runnerError) return 'ERROR';

return result.passed ? 'PASS' : 'FAIL';

}

function resultDetail(result) {

return result.runnerError

? `${result.runnerErrorReason}: ${result.runnerError}`

: `${result.score}/${result.total}`;

}

function printSummary(summary) {

console.log('Aggregate stability:');

for (const group of summary) {

const rate =

group.passRate === null

? 'N/A'

: `${group.passed}/${group.evaluatedTrials} (${Math.round(group.passRate * 100)}%)`;

const details = summaryDetails(group);

console.log(` ${group.runner} ${group.caseId}: ${rate}${details ? `; ${details}` : ''}`);

}

}

function summaryDetails(group) {

return [

countDetails('failed', group.failedChecks),

countDetails('validation', group.validationIssues),

group.runnerErrors > 0 ? `runner errors=${group.runnerErrors}` : '',

]

.filter(Boolean)

.join('; ');

}

function countDetails(label, counts) {

const entries = Object.entries(counts).map(([id, count]) => `${id}=${count}`);

return entries.length > 0 ? `${label} ${entries.join(', ')}` : '';

}

async function loadDocs(docIds, overrideDocs) {

return Object.fromEntries(

await Promise.all(docIds.map((docId) => loadDocEntry(docId, overrideDocs))),

);

}

async function loadDocEntry(docId, overrideDocs) {

return [docId, await loadDoc(docId, overrideDocs)];

}

async function loadDoc(docId, overrideDocs) {

// An override swaps only WHERE the text comes from; the per-doc

// post-processing below (e.g. the --help:first30 30-line cap) applies to

// both sources so an A/B grade compares like with like. Without this, a

// draft longer than 30 lines would be graded on content the live path

// always truncates away.

return postProcessDoc(docId, await loadDocSource(docId, overrideDocs));

}

async function loadDocSource(docId, overrideDocs) {

const overridePath = overrideDocs?.get(docId);

if (overridePath) return readOverrideDoc(docId, overridePath);

return docId === '--help:first30' ? cliHelp(['--help']) : cliHelp(['help', docId]);

}

async function readOverrideDoc(docId, overridePath) {

try {

return await readFile(overridePath, 'utf8');

} catch (error) {

throw new Error(

`--override-doc file for "${docId}" is not readable: ${overridePath} (${error?.code ?? errorMessage(error)})`,

);

}

}

function postProcessDoc(docId, text) {

const trimmed = text.trim();

return docId === '--help:first30' ? firstLines(trimmed, 30) : trimmed;

}

async function cliHelp(args) {

const { stdout } = await execFileAsync('node', [join(ROOT, 'bin', 'agent-device.mjs'), ...args], {

cwd: ROOT,

maxBuffer: 1024 * 1024 * 10,

});

return stdout.trim();

}

function firstLines(text, count) {

return text.split('\n').slice(0, count).join('\n');

}

function buildPrompt(testCase, docs) {

const helpText = testCase.docs.map((docId) => `### ${docId}\n${docs[docId]}`).join('\n\n');

return [

'Do not run shell commands.',

'You are evaluating agent-device CLI help. Return only JSON with keys commands and rationale.',

'commands must be an array of command lines you would run.',

`Task: ${testCase.task}`,

'',

helpText,

].join('\n');

}

// Only a 'success' RunnerOutcome ever reaches validatePlanCommands/

// scoreExpectations: a runner-error result carries runnerError/

// runnerErrorReason and nothing else, so infrastructure noise (an empty

// payload, a codex timeout, a Claude is_error envelope) can never masquerade

// as a model's failed command plan in the report or the aggregate summary.

async function runCase(runner, testCase, prompt, outDir, trial, repeat) {

const outcome = await runOutcome(runner, prompt, outDir);

const trialSuffix = repeat > 1 ? `-trial-${trial}` : '';

const outputPath = join(outDir, `${safeName(runner)}-${testCase.id}${trialSuffix}.txt`);

await writeFile(outputPath, outcome.raw);

const base = { runner, caseId: testCase.id, trial, outputPath };

if (outcome.kind === 'runner-error') {

return {

...base,

passed: false,

runnerError: outcome.message,

runnerErrorReason: outcome.reason,

};

}

const commandValidation = await validatePlanCommands(outcome.commands, {

allowedExternalCommands: testCase.allowedExternalCommands,

});

const checks = scoreExpectations(testCase, outcome.commands, outcome.raw, commandValidation);

const score = countPassingChecks(checks);

const total = countChecks(testCase);

return {

...base,

commands: outcome.commands,

commandValidation,

checks,

score,

total,

passed: score === total,

};

}

async function runOutcome(runner, prompt, outDir) {

const [kind, model] = runner.split(':');

try {

const raw = await runModel(kind, model, prompt, outDir);

return classifyRunnerOutput(raw);

} catch (error) {

return runnerErrorOutcome(errorOutput(error), errorMessage(error), 'process-failure');

}

}

function errorOutput(error) {

const payload = Object(error);

return [payload.stdout, payload.stderr].filter(Boolean).join('\n');

}

function errorMessage(error) {

return error instanceof Error ? error.message : String(error);

}

async function runModel(kind, model, prompt, outDir) {

return kind === 'claude' ? runClaude(model, prompt) : runCodex(model, prompt, outDir);

}

async function runClaude(model, prompt) {

const { stdout } = await execFileWithInput(

'claude',

[

'-p',

'--model',

model,

'--tools',

'',

'--permission-mode',

'dontAsk',

'--no-session-persistence',

'--output-format',

'json',

],

prompt,

{ cwd: ROOT, maxBuffer: 1024 * 1024 * 20, timeout: RUN_TIMEOUT_MS },

);

return stdout;

}

function execFileWithInput(file, args, input, options) {

return new Promise((resolve, reject) => {

const child = execFile(file, args, options, (error, stdout, stderr) => {

if (error) {

reject(Object.assign(error, { stdout, stderr }));

return;

}

resolve({ stdout, stderr });

});

child.stdin?.end(input);

});

}

async function runCodex(model, prompt, outDir) {

const outFile = join(

outDir,

`codex-${model}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.json`,

);

const pending = execFileAsync(

'codex',

[

'exec',

'--ignore-rules',

'--ignore-user-config',

'--ephemeral',

'--sandbox',

'read-only',

'-m',

model,

'-C',

ROOT,

'-o',

outFile,

prompt,

],

{ cwd: ROOT, maxBuffer: 1024 * 1024 * 20, timeout: RUN_TIMEOUT_MS },

);

// codex exec reads from stdin until EOF when it isn't a TTY. execFile never

// closes the child's stdin pipe on its own, so without this the process

// blocks on "Reading additional input from stdin..." until RUN_TIMEOUT_MS

// kills it and every codex case reports empty/error output.

pending.child?.stdin?.end();

const { stdout } = await pending;

let lastMessage = '';

try {

lastMessage = await readFile(outFile, 'utf8');

} catch {

// stdout still carries the transcript when -o fails.

}

// `-o` writes the same final JSON message that codex also prints to

// stdout when it isn't attached to a TTY. Concatenating both produces two

// back-to-back JSON objects, which breaks every downstream JSON.parse

// candidate and silently zeroes out extractCommands(). Prefer the clean

// -o payload and only fall back to stdout if it's missing/empty.

return lastMessage.trim().length > 0 ? lastMessage : stdout;

}

function assertRuntimeConfiguration() {

parsePositiveInteger(String(CONCURRENCY), 'HELP_BENCH_CONCURRENCY');

parsePositiveInteger(String(RUN_TIMEOUT_MS), 'HELP_BENCH_TIMEOUT_MS');

}

function countPassingChecks(checks) {

return Object.values(checks).filter(Boolean).length;

}

function safeName(name) {

return basename(name).replaceAll(/[^a-z0-9_.-]+/gi, '-');

}

function parsePositiveInteger(value, label) {

const parsed = Number(value);

if (!Number.isSafeInteger(parsed) || parsed < 1) {

throw new Error(`${label} expects a positive integer, got: ${value}`);

}

return parsed;

}

// Expected failures (bad flags, unreadable override files, unknown topic ids)

// print as one clean line instead of an unhandled stack trace.

try {

await main();

} catch (error) {

console.error(`Error: ${errorMessage(error)}`);

process.exitCode = 1;

}