Skip to content

Commit b2b116f

Browse files
authored
fix(replay): resolve Maestro failures and resize progress (#1450)
* fix(replay): secure Maestro failure diagnostics * refactor(replay): keep Maestro env API compatible * fix(replay): preserve diagnostic contracts * test(replay): verify resize cleanup rows * fix(replay): distinguish terminal resize reflow
1 parent 0557fee commit b2b116f

19 files changed

Lines changed: 586 additions & 148 deletions

‎docs/adr/0012-interactive-replay.md‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -481,10 +481,22 @@ number of suggestions available at default/full) so a caller knows whether a re-
481481
has material; default and full carry at most **20** screen refs and **5** suggestions ranked per
482482
decision 1's total order. These counts are absolute, including error payloads. Individual
483483
labels, ids, selectors, source paths, mismatch values, cause messages, and hints are UTF-8 truncated to
484-
**256 bytes**; an action summary has no positional array, and fill text, expanded variables, and arbitrary
485-
nested cause details are never serialized. All rendered strings and any overflow artifact pass through the
486-
central diagnostics redactor before truncation. The report sets truncation/redaction markers for every
487-
omission.
484+
**256 bytes**; an action summary has no positional array, and arbitrary nested cause details are never
485+
serialized. Maestro failure provenance renders resolved diagnostic identifiers, including targets and
486+
`runFlow` paths, so the report names what the runtime actually attempted instead of emitting an unresolved
487+
`${VAR}` or synthetic `<var:VAR>` token. Text-entry payloads remain semantic secrets: `inputText` progress,
488+
failure messages, suggestions, and overflow artifacts never serialize the entered text. Injected replay
489+
values are not registered as global sensitive literals: a short ordinary value such as `2` or `on` would
490+
otherwise corrupt unrelated timestamps, paths, and typed error fields throughout the request log.
491+
Text-entry values are registered at the actual dispatch boundary before platform work, independently of
492+
the user-facing failure projection. Users must not place secrets in selectors, links, filenames, or other
493+
diagnostic identifiers that are expected to appear in failure output.
494+
495+
Native `.ad` replay retains its categorical `<var:NAME>` replacement in human-readable divergence
496+
messages, hints, and bounded diagnostic fields. That existing fail-closed policy is intentionally
497+
separate from Maestro compatibility output; semantically masked positionals and daemon-owned
498+
machine-readable fields and paths are never substring-rewritten. The report sets truncation/redaction
499+
markers for every omission.
488500

489501
When the bounded form would omit material, the daemon writes the same redacted, bounded-per-field detail
490502
to a session-scoped divergence artifact and returns its path plus `overflow: { omittedBytes, artifactPath

‎src/compat/maestro/__tests__/daemon-runtime-port.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,53 @@ import path from 'node:path';
44
import { expect, test, vi } from 'vitest';
55
import type { DaemonInvokeFn, DaemonRequest } from '../../../daemon/types.ts';
66
import { PNG } from '../../../utils/png.ts';
7+
import {
8+
emitDiagnostic,
9+
flushDiagnosticsToSessionFile,
10+
withDiagnosticsScope,
11+
} from '../../../utils/diagnostics.ts';
712
import { createDaemonMaestroRuntimePort } from '../daemon-runtime-port.ts';
813
import { MAESTRO_OBSERVATION_POLL_MS } from '../daemon-runtime-port-observation.ts';
914
import { parseMaestroProgram } from '../program-ir-parser.ts';
1015
import { executeMaestroProgram } from './runtime-port-fixtures.ts';
1116
import { makeBaseRequest, makeDependencies, makeSnapshot } from './daemon-runtime-port-fixtures.ts';
1217

18+
test('registers Maestro inputText as sensitive before nested platform work', async () => {
19+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-input-diagnostics-'));
20+
const logPath = path.join(root, 'request.ndjson');
21+
const text = 'opaque-maestro-input';
22+
const port = createDaemonMaestroRuntimePort({
23+
baseReq: makeBaseRequest({ flags: { platform: 'android', replayBackend: 'maestro' } }),
24+
invoke: async (request) => {
25+
if (request.command === 'type') {
26+
emitDiagnostic({
27+
phase: 'platform_echo',
28+
data: { message: `Backend echoed ${request.positionals?.[0]}` },
29+
});
30+
}
31+
return request.command === 'snapshot'
32+
? { ok: true, data: { nodes: [], createdAt: 0 } }
33+
: { ok: true, data: {} };
34+
},
35+
dependencies: makeDependencies(),
36+
platform: 'android',
37+
});
38+
39+
await withDiagnosticsScope({ command: 'replay', logPath }, async () => {
40+
await port.execute({
41+
command: { kind: 'inputText', source: { line: 2 }, text },
42+
generation: 0,
43+
env: {},
44+
invalidateObservation() {},
45+
});
46+
flushDiagnosticsToSessionFile({ force: true });
47+
});
48+
49+
const diagnostics = fs.readFileSync(logPath, 'utf8');
50+
expect(diagnostics).not.toContain(text);
51+
expect(diagnostics).toContain('Backend echoed [REDACTED]');
52+
});
53+
1354
test('delegates lifecycle and coordinate gestures through public daemon commands', async () => {
1455
const requests: DaemonRequest[] = [];
1556
const invoke: DaemonInvokeFn = async (request) => {

‎src/compat/maestro/__tests__/engine-context.test.ts‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import { expect, test, vi } from 'vitest';
2+
import { maestroTestFailure } from '../compatibility-errors.ts';
23
import { createMaestroExecutionContext } from '../engine-context.ts';
34
import type { MaestroRuntimePort } from '../engine-types.ts';
45
import { parseMaestroProgram } from '../program-ir-parser.ts';
56
import { executeMaestroProgram } from './runtime-port-fixtures.ts';
67

7-
test('resolves transitive scoped variables to their final value', () => {
8+
test('resolves transitive scoped variables', () => {
89
const context = createMaestroExecutionContext();
910
const leave = context.enter({
1011
TARGET: '${NEXT}',
@@ -13,22 +14,35 @@ test('resolves transitive scoped variables to their final value', () => {
1314
});
1415

1516
expect(context.resolve('${TARGET}')).toBe('Done');
16-
expect(context.expandedVariables).toEqual({ TARGET: 'Done' });
1717
leave();
1818
});
1919

20-
test('retains expanded values after nested scopes unwind', () => {
20+
test('keeps nested scopes valid until they unwind', () => {
2121
const context = createMaestroExecutionContext();
2222
const rootLeave = context.enter({ SECRET: 'nested-scope-secret' });
2323
const nestedLeave = context.enter({ TARGET: '${SECRET}' });
2424

2525
expect(context.resolve('${TARGET}')).toBe('nested-scope-secret');
2626
nestedLeave();
2727
rootLeave();
28+
});
2829

29-
expect(context.expandedVariables).toEqual({
30-
TARGET: 'nested-scope-secret',
31-
});
30+
test('renders resolved target variables in optional-step warnings', async () => {
31+
const target = 'Missing checkout button';
32+
const program = parseMaestroProgram(
33+
['---', '- tapOn:', ' text: ${TARGET}', ' optional: true'].join('\n'),
34+
{ sourcePath: '/flows/optional.yaml' },
35+
);
36+
const port: MaestroRuntimePort = {
37+
execute: vi.fn(async () => {
38+
throw maestroTestFailure(`Missing ${target}`);
39+
}),
40+
observe: vi.fn(async ({ generation }) => ({ generation, matched: true })),
41+
};
42+
43+
const result = await executeMaestroProgram(program, port, { env: { TARGET: target } });
44+
45+
expect(result.warnings).toEqual([expect.stringContaining(target)]);
3246
});
3347

3448
test('rejects cyclic references instead of recursing indefinitely', () => {

‎src/compat/maestro/daemon-runtime-port.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { AppError, asAppError } from '../../kernel/errors.ts';
22
import type { Rect } from '../../kernel/snapshot.ts';
3-
import { emitDiagnostic } from '../../utils/diagnostics.ts';
3+
import { emitDiagnostic, registerDiagnosticSensitiveValue } from '../../utils/diagnostics.ts';
44
import { stripUndefined } from '../../utils/parsing.ts';
55
import { executeRunScriptFile } from './run-script-execution.ts';
66
import {
@@ -96,6 +96,7 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper
9696
text: string,
9797
context: MaestroRuntimeOperationContext,
9898
): Promise<void> => {
99+
registerDiagnosticSensitiveValue(text);
99100
await invokeMutation({ kind: 'typeText', text }, context);
100101
const stable = await waitForTypedSnapshotStability({
101102
timeoutMs: MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS,

‎src/compat/maestro/engine-context.ts‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ export function createMaestroExecutionContext(
1111
// Flow config and runFlow env values are stack-scoped; script output variables persist.
1212
let persistentValues = stringifyValues(defaults);
1313
const scopes: Record<string, string>[] = [];
14-
const expandedValues = new Map<string, string>();
1514
let cachedValues: Readonly<Record<string, string>> | undefined;
1615
let generation = 0;
1716
let observation: MaestroObservation | undefined;
@@ -26,9 +25,6 @@ export function createMaestroExecutionContext(
2625
get observation(): MaestroObservation | undefined {
2726
return observation?.generation === generation ? observation : undefined;
2827
},
29-
get expandedVariables(): Readonly<Record<string, string>> {
30-
return Object.fromEntries(expandedValues);
31-
},
3228
enter(scopedValues: Record<string, string | number | boolean> = {}): () => void {
3329
const resolved = resolveScopedValues(scopedValues);
3430
scopes.push(resolved);
@@ -62,10 +58,10 @@ export function createMaestroExecutionContext(
6258
observation = undefined;
6359
},
6460
resolve(value: string): string {
65-
return resolveValue(value, currentValues(), recordExpandedValue);
61+
return resolveValue(value, currentValues());
6662
},
6763
resolveDeferred(value: string): string {
68-
return resolveValue(value, currentValues(), undefined, new Set(), false);
64+
return resolveValue(value, currentValues(), new Set(), false);
6965
},
7066
};
7167

@@ -92,17 +88,12 @@ export function createMaestroExecutionContext(
9288
...resolved,
9389
...overrides,
9490
},
95-
undefined,
9691
new Set(),
9792
false,
9893
);
9994
}
10095
return resolved;
10196
}
102-
103-
function recordExpandedValue(name: string, value: string): void {
104-
expandedValues.set(name, value);
105-
}
10697
}
10798

10899
function stringifyValues(
@@ -114,7 +105,6 @@ function stringifyValues(
114105
function resolveValue(
115106
value: string,
116107
values: Readonly<Record<string, string>>,
117-
onExpanded?: (name: string, value: string) => void,
118108
resolving = new Set<string>(),
119109
failOnUnresolved = true,
120110
): string {
@@ -130,11 +120,9 @@ function resolveValue(
130120
const resolved = resolveValue(
131121
values[key]!,
132122
values,
133-
onExpanded,
134123
new Set([...resolving, key]),
135124
failOnUnresolved,
136125
);
137-
onExpanded?.(key, resolved);
138126
return resolved;
139127
});
140128
if (failOnUnresolved) assertNoUnsupportedInterpolation(resolved);

‎src/compat/maestro/engine-types.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,6 @@ export type MaestroEngineObserver = {
137137
runtimeMetrics?: MaestroRuntimeMetrics;
138138
error: unknown;
139139
artifactPaths: readonly string[];
140-
expandedVariables: Readonly<Record<string, string>>;
141140
},
142141
): void;
143142
};

‎src/compat/maestro/replay-plan-execution.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ async function executeObservedStep(
9696
...runtimeMetricsDelta(metricsBefore, state.port.readMetrics?.()),
9797
error: failure.error,
9898
artifactPaths: [...state.artifacts],
99-
expandedVariables: state.context.expandedVariables,
10099
}),
101100
);
102101
throw failure;

‎src/compat/maestro/replay-plan-step-execution.ts‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -75,18 +75,21 @@ async function executeStep(
7575
}
7676

7777
async function executeOptionalCommand(
78-
command: MaestroRuntimeCommand,
78+
rawCommand: MaestroRuntimeCommand,
7979
appId: string | undefined,
8080
state: MaestroReplayPlanExecutionState,
8181
): Promise<MaestroRuntimeResult | undefined> {
82+
const command = resolveCommand(rawCommand, state.context);
8283
try {
83-
return await executeCommand(command, appId, state);
84+
return await executeResolvedCommand(command, appId, state);
8485
} catch (error) {
8586
checkpointMaestroCancellation(state.options.signal);
86-
if (!isOptionalCommand(command) || !isMaestroTestFailure(error)) throw error;
87-
state.warnings.push(formatOptionalWarning(command, error));
88-
state.skipped += 1;
89-
return undefined;
87+
if (isOptionalCommand(command) && isMaestroTestFailure(error)) {
88+
state.warnings.push(formatOptionalWarning(command, error));
89+
state.skipped += 1;
90+
return undefined;
91+
}
92+
throw commandFailure(error, command);
9093
}
9194
}
9295

@@ -100,12 +103,11 @@ function isOptionalCommand(command: MaestroRuntimeCommand): boolean {
100103
return 'optional' in command && command.optional === true;
101104
}
102105

103-
async function executeCommand(
104-
rawCommand: MaestroRuntimeCommand,
106+
async function executeResolvedCommand(
107+
command: MaestroRuntimeCommand,
105108
appId: string | undefined,
106109
state: MaestroReplayPlanExecutionState,
107110
): Promise<MaestroRuntimeResult | undefined> {
108-
const command = resolveCommand(rawCommand, state.context);
109111
switch (command.kind) {
110112
case 'assertVisible':
111113
await requireObservation(
@@ -303,6 +305,15 @@ export function asMaestroReplayPlanStepFailure(
303305
};
304306
}
305307

308+
function commandFailure(error: unknown, command: MaestroRuntimeCommand): PlanStepFailure {
309+
return {
310+
kind: 'maestroPlanStepFailure',
311+
error: withSource(error, command),
312+
source: command.source,
313+
command,
314+
};
315+
}
316+
306317
function isPlanStepFailure(value: unknown): value is PlanStepFailure {
307318
return Boolean(
308319
value &&

‎src/compat/maestro/support-matrix.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ export const MAESTRO_COMPAT_LIMITATIONS = [
99
'Runtime: iOS and Android only; launchApp.clearState supports Android and iOS simulators, launch arguments are Apple-only, and standalone device utility/state commands are unsupported.',
1010
'Expressions: when.true supports boolean literals and maestro.platform comparisons; repeat.while, evalScript, and broader JavaScript expressions are unsupported.',
1111
'Environment: flow env is the default, AD_VAR_* overrides it, and CLI -e KEY=VALUE wins over both.',
12+
'Failure diagnostics: resolved targets and runFlow paths are rendered, while inputText payloads remain hidden; do not place secrets in diagnostic identifiers.',
1213
'Trust: runScript executes trusted scripts, may make http.post network requests, and is not a security sandbox; output keys cannot contain a dot.',
1314
'Errors and tracking: unsupported commands and fields fail with source context when available; open a focused issue only when implementation work is planned.',
1415
] as const;
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import fs from 'node:fs';
2+
import os from 'node:os';
3+
import path from 'node:path';
4+
import { expect, test } from 'vitest';
5+
import { makeSessionStore } from '../../__tests__/test-utils/index.ts';
6+
import { LeaseRegistry } from '../lease-registry.ts';
7+
import { createRequestHandler } from '../request-router.ts';
8+
9+
function createHarness() {
10+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-router-replay-env-'));
11+
return {
12+
root,
13+
handler: createRequestHandler({
14+
logPath: path.join(root, 'daemon.log'),
15+
stateDir: root,
16+
token: 'test-token',
17+
sessionStore: makeSessionStore('agent-device-router-replay-env-store-'),
18+
leaseRegistry: new LeaseRegistry(),
19+
trackDownloadableArtifact: () => 'artifact-id',
20+
}),
21+
};
22+
}
23+
24+
test('malformed replay env returns a normalized INVALID_ARGS response', async () => {
25+
const { root, handler } = createHarness();
26+
const flowPath = path.join(root, 'flow.ad');
27+
fs.writeFileSync(flowPath, 'wait 1\n');
28+
29+
await expect(
30+
handler({
31+
token: 'test-token',
32+
session: 'default',
33+
command: 'replay',
34+
positionals: [flowPath],
35+
flags: { replayEnv: ['NOEQUAL'] },
36+
meta: { requestId: 'req-invalid-replay-env' },
37+
}),
38+
).resolves.toMatchObject({
39+
ok: false,
40+
error: {
41+
code: 'INVALID_ARGS',
42+
message: expect.stringContaining('expected KEY=VALUE'),
43+
},
44+
});
45+
});
46+
47+
test('ordinary replay env values do not globally corrupt request diagnostics', async () => {
48+
const { root, handler } = createHarness();
49+
const missingPath = path.join(root, '2-missing.ad');
50+
51+
const response = await handler({
52+
token: 'test-token',
53+
session: 'default',
54+
command: 'replay',
55+
positionals: [missingPath],
56+
flags: { replayEnv: ['RETRIES=2', 'USER=demo'] },
57+
meta: { requestId: 'req-ordinary-replay-env' },
58+
});
59+
60+
expect(response.ok).toBe(false);
61+
if (response.ok) return;
62+
expect(response.error.logPath).toBeTruthy();
63+
const diagnostics = fs.readFileSync(response.error.logPath!, 'utf8');
64+
expect(diagnostics).toContain(missingPath);
65+
expect(diagnostics).not.toContain('[REDACTED]-missing.ad');
66+
for (const line of diagnostics.trim().split('\n')) {
67+
const event = JSON.parse(line) as { ts: string };
68+
expect(event.ts).not.toContain('[REDACTED]');
69+
}
70+
});

0 commit comments

Comments
 (0)