Skip to content

Commit 7acf07a

Browse files
committed
feat: support custom replay test reporters
1 parent e3907c5 commit 7acf07a

9 files changed

Lines changed: 264 additions & 7 deletions

File tree

‎src/__tests__/cli-network.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -709,3 +709,42 @@ test('test command supports explicit reporter lists', async () => {
709709
await fs.rm(tmpDir, { recursive: true, force: true });
710710
}
711711
});
712+
713+
test('test command loads custom reporter modules with JSON options', async () => {
714+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-custom-reporter-test-'));
715+
const reporterPath = path.join(tmpDir, 'custom-reporter.mjs');
716+
const outputPath = path.join(tmpDir, 'custom-report.json');
717+
718+
try {
719+
await fs.writeFile(
720+
reporterPath,
721+
[
722+
"import fs from 'node:fs';",
723+
'export default function createReporter(options) {',
724+
' return {',
725+
" name: 'custom-json',",
726+
' onSuiteEnd(suite) {',
727+
' fs.writeFileSync(options.output, JSON.stringify({ total: suite.total, failed: suite.failed }), "utf8");',
728+
' },',
729+
' getExitCode() { return 0; },',
730+
' };',
731+
'}',
732+
].join('\n'),
733+
'utf8',
734+
);
735+
736+
const reporterSpec = `${reporterPath}:${JSON.stringify({ output: outputPath })}`;
737+
const result = await runCliCapture(['test', './suite', '--reporter', reporterSpec], async () =>
738+
makeReplaySuiteResponse(),
739+
);
740+
741+
assert.equal(result.code, null);
742+
assert.doesNotMatch(result.stdout, /Test summary:/);
743+
assert.deepEqual(JSON.parse(await fs.readFile(outputPath, 'utf8')), {
744+
total: 3,
745+
failed: 1,
746+
});
747+
} finally {
748+
await fs.rm(tmpDir, { recursive: true, force: true });
749+
}
750+
});

‎src/cli-test-reporters/custom.ts‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import os from 'node:os';
2+
import path from 'node:path';
3+
import { pathToFileURL } from 'node:url';
4+
import { AppError } from '../utils/errors.ts';
5+
import type {
6+
ReplayTestReporter,
7+
ReplayTestReporterFactory,
8+
ReplayTestReporterLoadContext,
9+
} from './types.ts';
10+
11+
type CustomReporterModule = {
12+
default?: unknown;
13+
createReporter?: unknown;
14+
reporter?: unknown;
15+
};
16+
17+
type CustomReporterSpec = {
18+
modulePath: string;
19+
options: unknown;
20+
};
21+
22+
export function isCustomReplayTestReporterSpec(spec: string): boolean {
23+
const modulePath = readCustomReporterModulePath(spec);
24+
return (
25+
modulePath.startsWith('.') ||
26+
modulePath.startsWith('/') ||
27+
modulePath.startsWith('~') ||
28+
modulePath.startsWith('file:')
29+
);
30+
}
31+
32+
export async function createCustomReplayTestReporter(spec: string): Promise<ReplayTestReporter> {
33+
const customSpec = splitCustomReplayTestReporterSpec(spec);
34+
const modulePath = resolveCustomReporterModulePath(customSpec.modulePath);
35+
const module = await importCustomReporterModule(modulePath);
36+
const exported = module.createReporter ?? module.default ?? module.reporter;
37+
if (!exported) {
38+
throw new AppError(
39+
'INVALID_ARGS',
40+
`Custom test reporter ${customSpec.modulePath} must export default, createReporter, or reporter.`,
41+
);
42+
}
43+
44+
const reporter =
45+
typeof exported === 'function'
46+
? await (exported as ReplayTestReporterFactory)(customSpec.options, {
47+
spec,
48+
modulePath,
49+
} satisfies ReplayTestReporterLoadContext)
50+
: exported;
51+
52+
return validateCustomReplayTestReporter(reporter, customSpec.modulePath);
53+
}
54+
55+
function splitCustomReplayTestReporterSpec(spec: string): CustomReporterSpec {
56+
const optionsSeparator = spec.indexOf(':{');
57+
if (optionsSeparator < 0) return { modulePath: spec.trim(), options: undefined };
58+
const modulePath = readCustomReporterModulePath(spec);
59+
const rawOptions = spec.slice(optionsSeparator + 1);
60+
if (!modulePath) {
61+
throw new AppError('INVALID_ARGS', 'Custom test reporter path cannot be empty.');
62+
}
63+
try {
64+
return { modulePath, options: JSON.parse(rawOptions) };
65+
} catch (error) {
66+
const message = error instanceof Error ? error.message : String(error);
67+
throw new AppError(
68+
'INVALID_ARGS',
69+
`Invalid JSON options for custom test reporter ${modulePath}: ${message}`,
70+
);
71+
}
72+
}
73+
74+
function readCustomReporterModulePath(spec: string): string {
75+
const optionsSeparator = spec.indexOf(':{');
76+
return (optionsSeparator < 0 ? spec : spec.slice(0, optionsSeparator)).trim();
77+
}
78+
79+
function resolveCustomReporterModulePath(modulePath: string): string {
80+
if (modulePath.startsWith('file:')) return modulePath;
81+
const expandedPath = modulePath.startsWith('~/')
82+
? path.join(os.homedir(), modulePath.slice(2))
83+
: modulePath;
84+
return path.resolve(process.cwd(), expandedPath);
85+
}
86+
87+
async function importCustomReporterModule(modulePath: string): Promise<CustomReporterModule> {
88+
try {
89+
const href = modulePath.startsWith('file:') ? modulePath : pathToFileURL(modulePath).href;
90+
return (await import(href)) as CustomReporterModule;
91+
} catch (error) {
92+
const message = error instanceof Error ? error.message : String(error);
93+
throw new AppError(
94+
'INVALID_ARGS',
95+
`Failed to load custom test reporter ${modulePath}: ${message}`,
96+
);
97+
}
98+
}
99+
100+
function validateCustomReplayTestReporter(
101+
reporter: unknown,
102+
modulePath: string,
103+
): ReplayTestReporter {
104+
if (!reporter || typeof reporter !== 'object') {
105+
throw new AppError(
106+
'INVALID_ARGS',
107+
`Custom test reporter ${modulePath} must export a reporter object or factory.`,
108+
);
109+
}
110+
const candidate = reporter as Partial<ReplayTestReporter>;
111+
if (typeof candidate.name !== 'string' || candidate.name.trim().length === 0) {
112+
throw new AppError('INVALID_ARGS', `Custom test reporter ${modulePath} must define name.`);
113+
}
114+
if (candidate.onProgress !== undefined && typeof candidate.onProgress !== 'function') {
115+
throw new AppError(
116+
'INVALID_ARGS',
117+
`Custom test reporter ${modulePath} onProgress must be a function.`,
118+
);
119+
}
120+
if (candidate.onSuiteEnd !== undefined && typeof candidate.onSuiteEnd !== 'function') {
121+
throw new AppError(
122+
'INVALID_ARGS',
123+
`Custom test reporter ${modulePath} onSuiteEnd must be a function.`,
124+
);
125+
}
126+
if (candidate.getExitCode !== undefined && typeof candidate.getExitCode !== 'function') {
127+
throw new AppError(
128+
'INVALID_ARGS',
129+
`Custom test reporter ${modulePath} getExitCode must be a function.`,
130+
);
131+
}
132+
return candidate as ReplayTestReporter;
133+
}

‎src/cli-test-reporters/registry.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
import type { ReplaySuiteResult } from '../daemon/types.ts';
22
import { AppError } from '../utils/errors.ts';
3+
import { createCustomReplayTestReporter, isCustomReplayTestReporterSpec } from './custom.ts';
34
import { createDefaultReplayTestReporter } from './default.ts';
45
import { getReplayTestExitCode } from './format.ts';
56
import { createJunitReplayTestReporter } from './junit.ts';
67
import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts';
78

8-
export function resolveReplayTestReporters(options: {
9+
export async function resolveReplayTestReporters(options: {
910
reporters?: string[];
1011
reportJunit?: string;
1112
json?: boolean;
12-
}): ReplayTestReporter[] {
13+
}): Promise<ReplayTestReporter[]> {
1314
const specs =
1415
options.reporters && options.reporters.length > 0
1516
? [...options.reporters]
@@ -21,7 +22,7 @@ export function resolveReplayTestReporters(options: {
2122
specs.push(`junit:${options.reportJunit}`);
2223
}
2324

24-
return specs.map(resolveReplayTestReporter);
25+
return await Promise.all(specs.map(resolveReplayTestReporter));
2526
}
2627

2728
export async function runReplayTestReporters(
@@ -45,7 +46,11 @@ export function getReplayTestReporterExitCode(
4546
return getReplayTestExitCode(suite);
4647
}
4748

48-
function resolveReplayTestReporter(spec: string): ReplayTestReporter {
49+
async function resolveReplayTestReporter(spec: string): Promise<ReplayTestReporter> {
50+
if (isCustomReplayTestReporterSpec(spec)) {
51+
return await createCustomReplayTestReporter(spec);
52+
}
53+
4954
const { name, value } = splitReplayTestReporterSpec(spec);
5055
if (name === 'default') {
5156
if (value) {
@@ -65,7 +70,7 @@ function resolveReplayTestReporter(spec: string): ReplayTestReporter {
6570

6671
throw new AppError(
6772
'INVALID_ARGS',
68-
`Unknown test reporter "${name}". Built-in reporters: default, junit:<path>.`,
73+
`Unknown test reporter "${name}". Built-in reporters: default, junit:<path>. Custom reporters must be file paths.`,
6974
);
7075
}
7176

‎src/cli-test-reporters/types.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,19 @@ export type ReplayTestReporterContext = {
55
debug?: boolean;
66
};
77

8+
export type ReplayTestReporterLoadContext = {
9+
spec: string;
10+
modulePath: string;
11+
};
12+
813
export type ReplayTestReporter = {
914
name: string;
1015
onProgress?(event: RequestProgressEvent, context: ReplayTestReporterContext): void;
1116
onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Promise<void> | void;
1217
getExitCode?(suite: ReplaySuiteResult): number | undefined;
1318
};
19+
20+
export type ReplayTestReporterFactory = (
21+
options: unknown,
22+
context: ReplayTestReporterLoadContext,
23+
) => ReplayTestReporter | Promise<ReplayTestReporter>;

‎src/cli-test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export async function renderReplayTestResponse(options: {
1414
reportJunit?: string;
1515
}): Promise<number> {
1616
const { suite, json, debug, reporter, reportJunit } = options;
17-
const reporters = resolveReplayTestReporters({ reporters: reporter, reportJunit, json });
17+
const reporters = await resolveReplayTestReporters({ reporters: reporter, reportJunit, json });
1818
await runReplayTestReporters(reporters, suite, { debug });
1919
if (json) {
2020
printJson({ success: true, data: suite });

‎src/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@ export type {
2020

2121
export type { AppErrorCode, NormalizedError } from './utils/errors.ts';
2222

23+
export type {
24+
ReplayTestReporter,
25+
ReplayTestReporterContext,
26+
ReplayTestReporterFactory,
27+
ReplayTestReporterLoadContext,
28+
} from './cli-test-reporters/types.ts';
29+
2330
export type { CommandResult } from './core/command-descriptor/command-result.ts';
2431
export type { BootCommandResult, ShutdownCommandResult } from './contracts/device.ts';
2532
export type { ViewportCommandResult } from './contracts/viewport.ts';

‎src/utils/__tests__/args.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2071,6 +2071,10 @@ test('command usage describes test suite flags', () => {
20712071
assert.match(help, /--retries <n>/);
20722072
assert.match(help, /--record-video/);
20732073
assert.match(help, /--artifacts-dir <path>/);
2074+
assert.match(help, /--reporter <name\[:options\]>/);
2075+
assert.match(help, /custom reporters are file paths/);
2076+
assert.match(help, /--report-junit <path>/);
2077+
assert.match(help, /compatibility alias for --reporter junit:<path>/);
20742078
assert.doesNotMatch(help, /test --verbose prints per-test step timings without debug logs/);
20752079
});
20762080

‎src/utils/cli-flags.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -950,7 +950,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [
950950
multiple: true,
951951
usageLabel: '--reporter <name[:options]>',
952952
usageDescription:
953-
'Test: add a replay suite reporter; built-ins are default and junit:<path> (repeatable)',
953+
'Test: add a replay suite reporter; built-ins are default and junit:<path>; custom reporters are file paths with optional :<json> options (repeatable)',
954954
},
955955
{
956956
key: 'reportJunit',

‎website/docs/docs/replay-e2e.md‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ agent-device test ./workflows
8787
agent-device test "./workflows/**/*.ad" --platform android
8888
agent-device test ./workflows --timeout 60000 --retries 1
8989
agent-device test ./workflows --artifacts-dir ./tmp/agent-device-artifacts
90+
agent-device test ./workflows --reporter default --reporter junit:./tmp/junit.xml
9091
```
9192

9293
- `test` discovers `.ad` files from files, directories, or globs and runs them serially.
@@ -97,8 +98,66 @@ agent-device test ./workflows --artifacts-dir ./tmp/agent-device-artifacts
9798
- `replay-timing.ndjson` records attempt, cleanup, and per-step start/stop events with durations. Upload it from CI even for passing runs when comparing local and CI performance.
9899
- Timeouts are cooperative: the runner marks the attempt failed at the timeout boundary, then gives the underlying replay a short grace period to stop before session cleanup.
99100
- The default text reporter streams one-line `pass`, `fail`, or `skip` progress on stderr as each suite entry finishes or retries. Each line includes current/total suite position and elapsed seconds such as `pass 3/6 ... duration=12.34s`, then the final summary prints failed tests and passed-on-retry flaky tests; use `--verbose` to print every final result.
101+
- `--reporter` is repeatable. Built-ins are `default` for the console summary and `junit:<path>` for JUnit XML. Passing any explicit reporter list replaces the implicit default reporter, so include `--reporter default` when you also want terminal output. `--report-junit <path>` remains a compatibility alias for `--reporter junit:<path>`.
100102
- When `--fail-fast` and retries are both set, the current test still consumes its retries before the suite stops.
101103

104+
### Custom test reporters
105+
106+
Custom reporters are CLI-only presentation adapters. The daemon still returns the structured replay suite result; reporters run in the local CLI process after the suite finishes.
107+
108+
```bash
109+
agent-device test ./workflows --reporter ./scripts/replay-reporter.mjs
110+
agent-device test ./workflows --reporter './scripts/replay-reporter.mjs:{"output":"./tmp/report.txt"}'
111+
```
112+
113+
Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive parsed JSON options and load context:
114+
115+
```js
116+
// scripts/replay-reporter.mjs
117+
import fs from 'node:fs';
118+
119+
export default function createReporter(options, context) {
120+
return {
121+
name: 'summary-file',
122+
onSuiteEnd(suite) {
123+
fs.writeFileSync(
124+
options.output,
125+
JSON.stringify(
126+
{
127+
total: suite.total,
128+
passed: suite.passed,
129+
failed: suite.failed,
130+
modulePath: context.modulePath,
131+
},
132+
null,
133+
2,
134+
),
135+
);
136+
},
137+
getExitCode(suite) {
138+
return suite.failed > 0 ? 1 : 0;
139+
},
140+
};
141+
}
142+
```
143+
144+
TypeScript reporters can import the public types from `agent-device`:
145+
146+
```ts
147+
import type { ReplayTestReporterFactory } from 'agent-device';
148+
149+
const createReporter: ReplayTestReporterFactory = (options) => ({
150+
name: 'typed-reporter',
151+
onSuiteEnd(suite) {
152+
// Write artifacts, annotations, or summaries from suite.
153+
},
154+
});
155+
156+
export default createReporter;
157+
```
158+
159+
The supported hook today is final-result reporting through `onSuiteEnd`. `getExitCode` can override whether the finished suite exits successfully; when no reporter supplies one, failed tests exit with `1`. The `onProgress` hook is part of the reporter interface for live reporters, but the CLI currently invokes reporters after the suite result is available.
160+
102161
## Parametrise `.ad` scripts
103162

104163
Substitute `${VAR}` tokens in `.ad` scripts using values from the CLI, shell env, script-local `env` directives, or built-ins.

0 commit comments

Comments
 (0)