Skip to content

Commit 4932e0f

Browse files
committed
refactor(replay-test): make an attempt failure's warnings channel always present
An empty array and an absent key said the same thing two ways, so every consumer needed both guards. The failed outcome now always carries the warnings array, like the passed outcome already did, and the result builder narrows the attempt once instead of per field. This also removes the clone the shared candidate-view formatter had grown between the CLI and MCP error renderers — the MCP error text gained the same warnings lines the CLI stderr prints.
1 parent c1c3d30 commit 4932e0f

10 files changed

Lines changed: 50 additions & 37 deletions

File tree

‎packages/replay-test/src/internal/__tests__/session-test-artifacts.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ test('materializeReplayTestAttemptArtifacts writes failure manifest and copies l
106106
details: { reason: 'timeout', artifactPaths: [screenshotPath] },
107107
},
108108
artifactPaths: [screenshotPath],
109+
warnings: [],
109110
infrastructure: false,
110111
},
111112
filePath: replayPath,
@@ -189,6 +190,7 @@ test('materialization preserves replay sources and diagnostics named after attem
189190
status: 'failed',
190191
error: { code: 'COMMAND_FAILED', message: 'original failure' },
191192
artifactPaths,
193+
warnings: [],
192194
infrastructure: false,
193195
},
194196
filePath: replayPath,
@@ -267,6 +269,7 @@ test('materialization copies a log listed in both the outcome and error only onc
267269
status: 'failed',
268270
error: { code: 'COMMAND_FAILED', message: 'failed', logPath },
269271
artifactPaths: [logPath, logPath],
272+
warnings: [],
270273
infrastructure: false,
271274
},
272275
filePath: replayPath,
@@ -300,6 +303,7 @@ test.each(['result.txt', 'failure.txt', 'RESULT.TXT'])(
300303
status: 'failed',
301304
error: { code: 'COMMAND_FAILED', message: 'original failure' },
302305
artifactPaths: [diagnosticPath],
306+
warnings: [],
303307
infrastructure: false,
304308
},
305309
filePath: replayPath,

‎packages/replay-test/src/internal/__tests__/session-test-attempt.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const FAILED_WITHOUT_WARNINGS: ReplayTestAttemptOutcome = {
2222
status: 'failed',
2323
error: { code: 'COMMAND_FAILED', message: 'tap failed' },
2424
artifactPaths: [],
25+
warnings: [],
2526
infrastructure: false,
2627
};
2728

‎packages/replay-test/src/internal/__tests__/session-test-runtime.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se
117117
status: 'failed',
118118
error: { code: 'COMMAND_FAILED', message: 'request canceled' },
119119
artifactPaths: [],
120+
warnings: [],
120121
infrastructure: false,
121122
});
122123
await replaySettled;
@@ -142,6 +143,7 @@ test('runReplayTestAttempt keeps a passing replay passed when finalization fails
142143
status: 'failed',
143144
error: { code: 'COMMAND_FAILED', message: 'failed to stop recording' },
144145
artifactPaths: [],
146+
warnings: [],
145147
infrastructure: false,
146148
}),
147149
cleanupSession,
@@ -167,6 +169,7 @@ test('runReplayTestAttempt marks a failed cleanup as infrastructure so the sched
167169
status: 'failed',
168170
error: { code: 'COMMAND_FAILED', message: 'open "System Settings" failed' },
169171
artifactPaths: [],
172+
warnings: [],
170173
infrastructure: false,
171174
}),
172175
cleanupSession,

‎packages/replay-test/src/internal/session-test-attempt.ts‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -388,16 +388,21 @@ function buildReplayTestFailedResult(
388388
attempts: outcome.attempts,
389389
artifactsDir: context.testArtifactsDir,
390390
error,
391-
...(attemptOutcome?.status === 'failed' && (attemptOutcome.warnings?.length ?? 0) > 0
392-
? { warnings: [...attemptOutcome.warnings!] }
393-
: {}),
394-
...(attemptOutcome?.status === 'failed' && attemptOutcome.infrastructure
395-
? { infrastructure: true as const }
396-
: {}),
391+
...replayTestFailedAttemptFields(attemptOutcome),
392+
...replayTestShardResultMetadata(shard),
393+
};
394+
}
395+
396+
function replayTestFailedAttemptFields(
397+
attemptOutcome: ReplayTestCaseOutcome['finalOutcome'],
398+
): Pick<ReplaySuiteTestFailed, 'warnings' | 'infrastructure' | 'snapshotDiagnostics'> {
399+
const failed = attemptOutcome?.status === 'failed' ? attemptOutcome : undefined;
400+
return {
401+
...(failed && failed.warnings.length > 0 ? { warnings: [...failed.warnings] } : {}),
402+
...(failed?.infrastructure ? { infrastructure: true as const } : {}),
397403
...(attemptOutcome?.snapshotDiagnostics
398404
? { snapshotDiagnostics: attemptOutcome.snapshotDiagnostics }
399405
: {}),
400-
...replayTestShardResultMetadata(shard),
401406
};
402407
}
403408

‎packages/replay-test/src/internal/session-test-types.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ export type ReplayTestAttemptFailed = {
145145
error: ReplayTestAttemptError;
146146
artifactPaths: readonly string[];
147147
/** Warnings accumulated before the failing step (skipped `optional` steps, capture degradations). */
148-
warnings?: readonly string[];
148+
warnings: readonly string[];
149149
snapshotDiagnostics?: SnapshotDiagnosticsSummary;
150150
/**
151151
* The host's verdict that this failure is environmental (device/runner/boot) rather than a
@@ -276,13 +276,15 @@ export type ReplayTestExecutionDependencies = Omit<
276276
export function replayTestAttemptFailure(params: {
277277
error: ReplayTestAttemptError;
278278
artifactPaths?: readonly string[];
279+
warnings?: readonly string[];
279280
infrastructure?: boolean;
280281
snapshotDiagnostics?: SnapshotDiagnosticsSummary;
281282
}): ReplayTestAttemptFailed {
282283
return {
283284
status: 'failed',
284285
error: params.error,
285286
artifactPaths: params.artifactPaths ?? [],
287+
warnings: params.warnings ?? [],
286288
infrastructure: params.infrastructure ?? false,
287289
...(params.snapshotDiagnostics ? { snapshotDiagnostics: params.snapshotDiagnostics } : {}),
288290
};

‎src/commands/output/error.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,14 @@ export function printHumanError(
5454
}
5555
}
5656

57-
function readResponseWarnings(details: Record<string, unknown> | undefined): string[] {
57+
export function readResponseWarnings(details: Record<string, unknown> | undefined): string[] {
5858
const warnings = details?.warnings;
5959
return Array.isArray(warnings)
6060
? warnings.filter((warning): warning is string => typeof warning === 'string')
6161
: [];
6262
}
6363

64-
function formatErrorCandidateViews(views: ErrorCandidateView[]): string[] {
64+
export function formatErrorCandidateViews(views: ErrorCandidateView[]): string[] {
6565
return views.flatMap((view) => {
6666
if (view.kind === 'element-match') {
6767
const remaining = view.matches - view.candidates.length;

‎src/daemon/replay/internal/__tests__/session-test-outcome.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,18 @@ test('failed attempt outcome carries warnings from the error details (#2560)', (
2525
});
2626
});
2727

28-
test('failed attempt outcome omits absent or empty warnings', () => {
28+
test('failed attempt outcome reads an empty warnings array when absent or non-string', () => {
2929
const withoutWarnings = toReplayTestAttemptOutcome({
3030
ok: false,
3131
error: { code: 'COMMAND_FAILED', message: 'step failed' },
3232
});
33-
expect('warnings' in withoutWarnings).toBe(false);
33+
expect(withoutWarnings.status === 'failed' && withoutWarnings.warnings).toEqual([]);
3434

3535
const emptyWarnings = toReplayTestAttemptOutcome({
3636
ok: false,
3737
error: { code: 'COMMAND_FAILED', message: 'step failed', details: { warnings: [7] } },
3838
});
39-
expect('warnings' in emptyWarnings).toBe(false);
39+
expect(emptyWarnings.status === 'failed' && emptyWarnings.warnings).toEqual([]);
4040
});
4141

4242
test('passed attempt outcome keeps reading warnings from response data', () => {

‎src/daemon/replay/internal/session-test-outcome.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,11 @@ import type { ReplayTestAttemptFailed, ReplayTestAttemptOutcome } from '@agent-d
1414
*/
1515
export function toReplayTestAttemptOutcome(response: DaemonResponse): ReplayTestAttemptOutcome {
1616
if (!response.ok) {
17-
const warnings = readStringArray(response.error.details?.warnings);
1817
return {
1918
status: 'failed',
2019
error: response.error,
2120
artifactPaths: readArtifactPaths(response.error.details?.artifactPaths),
22-
...(warnings.length > 0 ? { warnings } : {}),
21+
warnings: readStringArray(response.error.details?.warnings),
2322
infrastructure: isReplayInfrastructureFailure(response),
2423
...snapshotDiagnostics(response.error.details?.snapshotDiagnostics),
2524
};

‎src/mcp/__tests__/tool-error.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,22 @@ test('formatToolErrorText omits the candidates block for non-ambiguous errors',
3636
const text = formatToolErrorText(normalizeToolError(err));
3737

3838
assert.equal(text.includes('Candidates:'), false);
39+
assert.equal(text.includes('Warning:'), false);
40+
});
41+
42+
// #2560: a failed `replay` carries the run's accumulated warnings at error level;
43+
// the MCP reader must see them too, not only --json consumers.
44+
test('formatToolErrorText renders run-level warnings carried in error details', () => {
45+
const err = new AppError('REPLAY_DIVERGENCE', 'Replay failed at step 2 (tapOn "Save")', {
46+
warnings: ['Optional Maestro assertVisible skipped at line 1: no match'],
47+
});
48+
49+
const text = formatToolErrorText(normalizeToolError(err));
50+
51+
assert.match(
52+
text,
53+
/^Error \(REPLAY_DIVERGENCE\)[\s\S]*\nWarning: Optional Maestro assertVisible skipped at line 1: no match/,
54+
);
3955
});
4056

4157
test('formatToolErrorText renders a structured cause', () => {

‎src/mcp/tool-error.ts‎

Lines changed: 5 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import {
22
normalizeError,
33
readErrorCandidateViews,
4-
type ErrorCandidateView,
54
type NormalizedError,
65
} from '@agent-device/kernel/errors';
76
import { formatReplayDivergenceReport } from '@agent-device/ad-replay/divergence';
7+
import { formatErrorCandidateViews, readResponseWarnings } from '../commands/output/error.ts';
8+
import { collapseWarningText } from '../commands/output-common.ts';
89

910
export function normalizeToolError(error: unknown): NormalizedError {
1011
return normalizeError(error);
@@ -17,30 +18,12 @@ export function formatToolErrorText(normalized: NormalizedError): string {
1718
lines.push(`Cause: ${code}${normalized.cause.message}`);
1819
}
1920
if (normalized.hint) lines.push(`Hint: ${normalized.hint}`);
21+
for (const warning of readResponseWarnings(normalized.details)) {
22+
lines.push(`Warning: ${collapseWarningText(warning)}`);
23+
}
2024
lines.push(...formatErrorCandidateViews(readErrorCandidateViews(normalized.details)));
2125
if (normalized.supportedOn) lines.push(`Supported on: ${normalized.supportedOn}`);
2226
const divergence = formatReplayDivergenceReport(normalized.details);
2327
if (divergence) lines.push(divergence);
2428
return lines.join('\n');
2529
}
26-
27-
function formatErrorCandidateViews(views: ErrorCandidateView[]): string[] {
28-
return views.flatMap((view) => {
29-
if (view.kind === 'element-match') {
30-
const remaining = view.matches - view.candidates.length;
31-
return [
32-
'Candidates:',
33-
...view.candidates.map(
34-
(candidate) => ` ${pinCandidateLine(candidate, view.refsGeneration)}`,
35-
),
36-
...(remaining > 0 ? [` +${remaining} more`] : []),
37-
];
38-
}
39-
return ['Devices:', ...view.devices.map((device) => ` ${device.id} ${device.name}`)];
40-
});
41-
}
42-
43-
function pinCandidateLine(candidate: string, generation: number | undefined): string {
44-
if (generation === undefined) return candidate;
45-
return candidate.replace(/^@(e\d+)(?=\s|$)/, `@$1~s${generation}`);
46-
}

0 commit comments

Comments
 (0)