Skip to content

Commit 77d74a6

Browse files
committed
perf(web): measure snapshot geometry in batched backend calls, decided per entry
Snapshot geometry asked the backend for one ref's box at a time. Every backend call is a fresh process spawn, so the capture of a page cost a spawn per ref: 87 s of spawns for the 408 refs of one local page, and `is visible label="Submit"` — a selector assertion that requests rects — took 49.6 s. The batch the backend already offers runs each `get box` inside one process and answers with one entry per command, so a group of refs is one round trip. `stdin` reaches the spawn because `batch` reads its command list from there, which `ExecOptions` already supports: `runManagedAgentBrowser` forwards it and `runAgentBrowserCommand` passes it through. The per-ref decision moves onto a typed field. A ref the backend could not measure comes back as a failed entry and gets no rect, and nothing else changes; `isIgnorableBoxError`, which decided the same question from the failure's message text, is deleted. `--bail` is deliberately not requested: measured against a batch holding one unresolvable ref, it truncates the output at that entry, so every ref after it would silently lose its geometry. A capture whose refs come back entirely unmeasurable refuses with a typed `COMMAND_FAILED` naming the counts. The backend answers a request for a ref its current tree does not carry, which is what a stale ref looks like: verified live, an element hidden with `display: none` still measures against the refs of the snapshot taken before it was hidden, while every ref requested after an unrelated DOM change fails. Returning an unaddressable-shaped tree would surface later as a target with no usable bounds. A partial answer is not a failure.
1 parent a1d1eb8 commit 77d74a6

5 files changed

Lines changed: 373 additions & 51 deletions

File tree

‎packages/platform-web/src/__tests__/agent-browser-snapshot-refless.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,14 @@ describe('normalizeAgentBrowserSnapshot ref-less content', () => {
282282
});
283283

284284
test('attaches geometry only to the nodes the backend can measure', async () => {
285-
const result = await normalizeAgentBrowserSnapshot(FORM_AFTER_SUBMIT, async (ref) =>
286-
ref === 'e2' ? { x: 1, y: 2, width: 10, height: 20 } : undefined,
285+
const result = await normalizeAgentBrowserSnapshot(
286+
FORM_AFTER_SUBMIT,
287+
async (refs) =>
288+
new Map(
289+
refs.flatMap((ref) =>
290+
ref === 'e2' ? [[ref, { x: 1, y: 2, width: 10, height: 20 }] as const] : [],
291+
),
292+
),
287293
);
288294

289295
expect(result.nodes.find((node) => node.ref === 'e2')?.rect).toEqual({

‎packages/platform-web/src/agent-browser-provider.test.ts‎

Lines changed: 255 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ vi.mock('./agent-browser-lifecycle.ts', async (importOriginal) => {
2121

2222
import { createAgentBrowserWebProvider } from './agent-browser-provider.ts';
2323
import type { WebSnapshotResult } from './provider.ts';
24-
import { withCommandExecutorOverride, type ExecResult } from '@agent-device/host-kit/command';
24+
import {
25+
withCommandExecutorOverride,
26+
type ExecOptions,
27+
type ExecResult,
28+
} from '@agent-device/host-kit/command';
2529
import { type OwnedProcessRecordStore } from '@agent-device/host-kit/process';
2630
import { AppError } from '@agent-device/kernel/errors';
2731
import { buildSelectorChainForNode, resolveRecordedTarget } from '@agent-device/selectors';
@@ -30,6 +34,8 @@ import { attachRefs } from '@agent-device/kernel/snapshot';
3034
type AgentBrowserCall = {
3135
cmd: string;
3236
args: readonly string[];
37+
/** What the provider wrote to the backend's stdin, for the command that reads its work from it. */
38+
stdin?: string;
3339
};
3440

3541
const mockProviderStartupCleanup = vi.mocked(providerStartupCleanupMock);
@@ -284,18 +290,223 @@ test('agent-browser provider fetches snapshot rects only when requested', async
284290
calls.map((call) => call.args.slice(0, 3)),
285291
[
286292
['snapshot', '--json', '--session'],
287-
['get', 'box', '@e1'],
288-
['get', 'box', '@e2'],
289-
['get', 'box', '@e3'],
290-
['get', 'box', '@e4'],
293+
['batch', '--json', '--session'],
291294
],
292295
);
296+
// One round trip for every ref of the capture: each backend call is a process spawn, and a
297+
// page can hold hundreds of refs.
298+
assert.deepEqual(JSON.parse(String(calls[1]?.stdin)), [
299+
['get', 'box', '@e1'],
300+
['get', 'box', '@e2'],
301+
['get', 'box', '@e3'],
302+
['get', 'box', '@e4'],
303+
]);
293304
assertNormalizedSnapshot(snapshot);
294305
assertSnapshotRects(snapshot);
295306
assertRoleSelectorResolves(snapshot);
296307
});
297308
});
298309

310+
test('agent-browser provider keeps the geometry a batch did measure', async () => {
311+
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
312+
const calls: AgentBrowserCall[] = [];
313+
const snapshot = await withCommandExecutorOverride(
314+
async (cmd, args, options) => {
315+
const cliArgs = recordAgentBrowserCall(calls, cmd, args, options.stdin);
316+
if (cliArgs[0] !== 'batch') return snapshotPayload();
317+
const commands = JSON.parse(String(stdinText(options.stdin))) as string[][];
318+
return jsonResult(
319+
commands.map((command, index) =>
320+
index === 0
321+
? { command, success: false, error: 'Unknown ref: e1', result: null }
322+
: { command, success: true, error: null, result: boxRect(command[2]) },
323+
),
324+
1,
325+
);
326+
},
327+
async () => await provider.snapshot({ includeRects: true }),
328+
);
329+
330+
// A ref the backend could not measure is an ordinary answer: the rest of the tree keeps its
331+
// geometry and the entry's own `success` is what decided that, never the error wording.
332+
assert.deepEqual(
333+
snapshot.nodes.map((node) => node.rect),
334+
[
335+
undefined,
336+
{ x: 11, y: 12, width: 100, height: 20 },
337+
{ x: 31, y: 32, width: 100, height: 20 },
338+
{ x: 31, y: 32, width: 100, height: 20 },
339+
],
340+
);
341+
});
342+
});
343+
344+
test('agent-browser provider refuses snapshot geometry when the batch resolves no refs', async () => {
345+
// An element the backend can see measures as a box even when hidden, so a capture whose every ref
346+
// went unmeasurable answered to refs this snapshot does not publish. Returning no geometry at all
347+
// would hand the caller a tree of targets with no usable bounds.
348+
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
349+
await assert.rejects(
350+
() =>
351+
withCommandExecutorOverride(
352+
async (_cmd, args, options) => {
353+
const cliArgs = agentBrowserCliArgs(args);
354+
if (cliArgs[0] !== 'batch') return snapshotPayload();
355+
const commands = JSON.parse(String(stdinText(options.stdin))) as string[][];
356+
return jsonResult(
357+
commands.map((command) => ({
358+
command,
359+
success: false,
360+
error: 'Unknown ref',
361+
result: null,
362+
})),
363+
1,
364+
);
365+
},
366+
async () => await provider.snapshot({ includeRects: true }),
367+
),
368+
(error: unknown) =>
369+
error instanceof AppError &&
370+
error.code === 'COMMAND_FAILED' &&
371+
/resolved none of the snapshot refs/.test(error.message),
372+
);
373+
});
374+
});
375+
376+
test('agent-browser provider refuses geometry when the batch answers with too few entries', async () => {
377+
// Entries pair back to refs by position, so a short answer cannot be trusted: pairing four refs
378+
// against the entries that exist would hand each node some other node's box.
379+
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
380+
await assert.rejects(
381+
() =>
382+
withCommandExecutorOverride(
383+
async (_cmd, args, options) => {
384+
const cliArgs = agentBrowserCliArgs(args);
385+
if (cliArgs[0] !== 'batch') return snapshotPayload();
386+
const commands = JSON.parse(String(stdinText(options.stdin))) as string[][];
387+
return jsonResult(
388+
[{ command: commands[0], success: true, error: null, result: boxRect('@e1') }],
389+
1,
390+
);
391+
},
392+
async () => await provider.snapshot({ includeRects: true }),
393+
),
394+
(error: unknown) =>
395+
error instanceof AppError &&
396+
error.code === 'COMMAND_FAILED' &&
397+
/unusable batch result/.test(error.message),
398+
);
399+
});
400+
});
401+
402+
test('agent-browser provider refuses geometry when the batch itself faults', async () => {
403+
// A whole-batch fault comes back as one envelope rather than one entry per command, so nothing
404+
// can be paired back to a ref; the refusal is the typed one the envelope carries.
405+
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
406+
await assert.rejects(
407+
() =>
408+
withCommandExecutorOverride(
409+
async (_cmd, args) =>
410+
agentBrowserCliArgs(args)[0] === 'batch'
411+
? jsonResult({ success: false, error: 'Invalid JSON input' }, 1)
412+
: snapshotPayload(),
413+
async () => await provider.snapshot({ includeRects: true }),
414+
),
415+
(error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED',
416+
);
417+
});
418+
});
419+
420+
test('agent-browser provider splits snapshot geometry into bounded batches', async () => {
421+
// The backend reads a batch from stdin as one JSON array, so the refs are fetched in bounded
422+
// groups rather than in one unbounded payload; the bound is what keeps a large page from turning
423+
// into a single unreadable request.
424+
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
425+
const calls: AgentBrowserCall[] = [];
426+
const snapshot = await withCommandExecutorOverride(
427+
async (cmd, args, options) => {
428+
const cliArgs = recordAgentBrowserCall(calls, cmd, args, options.stdin);
429+
if (cliArgs[0] !== 'batch') return manyRefSnapshotPayload(250);
430+
const commands = JSON.parse(String(stdinText(options.stdin))) as string[][];
431+
return jsonResult(
432+
commands.map((command) => ({
433+
command,
434+
success: true,
435+
error: null,
436+
result: { x: 1, y: 2, width: 10, height: 20 },
437+
})),
438+
1,
439+
);
440+
},
441+
async () => await provider.snapshot({ includeRects: true }),
442+
);
443+
444+
const groups = calls
445+
.filter((call) => call.args[0] === 'batch')
446+
.map((call) => JSON.parse(String(call.stdin)) as string[][]);
447+
assert.deepEqual(
448+
groups.map((group) => group.length),
449+
[200, 50],
450+
);
451+
assert.equal(
452+
snapshot.nodes.filter((node) => node.rect !== undefined).length,
453+
250,
454+
'every ref of both groups keeps the box its entry reported',
455+
);
456+
});
457+
});
458+
459+
test('agent-browser provider refuses geometry when the batch answers with an object', async () => {
460+
// Entries pair back to refs by position. An answer that is not a list of entries cannot be
461+
// paired at all, so the geometry is refused rather than guessed at.
462+
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
463+
await assert.rejects(
464+
() =>
465+
withCommandExecutorOverride(
466+
async (_cmd, args) =>
467+
agentBrowserCliArgs(args)[0] === 'batch'
468+
? jsonResult({ success: true, data: { measured: 4 } })
469+
: snapshotPayload(),
470+
async () => await provider.snapshot({ includeRects: true }),
471+
),
472+
(error: unknown) =>
473+
error instanceof AppError &&
474+
error.code === 'COMMAND_FAILED' &&
475+
/unusable batch result/.test(error.message),
476+
);
477+
});
478+
});
479+
480+
test('agent-browser provider refuses geometry when a batch entry is not an entry', async () => {
481+
// Right number of answers, one of them not a result object: the positions no longer describe
482+
// the refs they measured.
483+
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
484+
await assert.rejects(
485+
() =>
486+
withCommandExecutorOverride(
487+
async (_cmd, args, options) => {
488+
const cliArgs = agentBrowserCliArgs(args);
489+
if (cliArgs[0] !== 'batch') return snapshotPayload();
490+
const commands = JSON.parse(String(stdinText(options.stdin))) as string[][];
491+
return jsonResult(
492+
commands.map((command, index) =>
493+
index === 0
494+
? 'get box @e1'
495+
: { command, success: true, error: null, result: boxRect(command[2]) },
496+
),
497+
1,
498+
);
499+
},
500+
async () => await provider.snapshot({ includeRects: true }),
501+
),
502+
(error: unknown) =>
503+
error instanceof AppError &&
504+
error.code === 'COMMAND_FAILED' &&
505+
/unusable batch result/.test(error.message),
506+
);
507+
});
508+
});
509+
299510
test('agent-browser provider dumps session network requests', async () => {
300511
await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => {
301512
const calls: AgentBrowserCall[] = [];
@@ -601,25 +812,44 @@ function agentBrowserCliArgs(args: readonly string[]): string[] {
601812
return args.slice(1);
602813
}
603814

815+
/** The backend's stdin as the string the provider wrote; the tests only ever write JSON text. */
816+
function stdinText(stdin: string | Buffer | undefined): string | undefined {
817+
return typeof stdin === 'string' ? stdin : undefined;
818+
}
819+
604820
function recordAgentBrowserCall(
605821
calls: AgentBrowserCall[],
606822
cmd: string,
607823
args: readonly string[],
824+
stdin?: string | Buffer,
608825
): string[] {
609826
const cliArgs = agentBrowserCliArgs(args);
610-
calls.push({ cmd, args: cliArgs });
827+
const written = stdinText(stdin);
828+
calls.push({ cmd, args: cliArgs, ...(written === undefined ? {} : { stdin: written }) });
611829
return cliArgs;
612830
}
613831

832+
/**
833+
* The backend's `batch` answers with one entry per command, and a ref it cannot measure comes back
834+
* as a failed entry rather than a failed batch: e3 of this fixture is the element with no box.
835+
*/
614836
function snapshotExecutor(calls: AgentBrowserCall[]) {
615-
return async (cmd: string, args: readonly string[], options: { allowFailure?: boolean }) => {
616-
const cliArgs = recordAgentBrowserCall(calls, cmd, args);
837+
return async (cmd: string, args: readonly string[], options: ExecOptions) => {
838+
const cliArgs = recordAgentBrowserCall(calls, cmd, args, options.stdin);
617839
if (cliArgs[0] === 'snapshot') return snapshotPayload();
618-
if (cliArgs.slice(0, 3).join(' ') === 'get box @e3') {
840+
if (cliArgs[0] === 'batch') {
841+
const commands = JSON.parse(String(stdinText(options.stdin))) as string[][];
619842
assert.equal(options.allowFailure, true);
620-
return jsonResult({ success: false, error: 'No box for element' }, 1);
843+
assert.ok(!cliArgs.includes('--bail'), 'a bailed batch drops the refs after a failed one');
844+
return jsonResult(
845+
commands.map((command) =>
846+
command.slice(0, 3).join(' ') === 'get box @e3'
847+
? { command, success: false, error: 'No box for element', result: null }
848+
: { command, success: true, error: null, result: boxRect(command[2]) },
849+
),
850+
1,
851+
);
621852
}
622-
if (cliArgs[0] === 'get' && cliArgs[1] === 'box') return boxPayload(cliArgs[2]);
623853
return jsonResult({ success: true, data: {} });
624854
};
625855
}
@@ -684,6 +914,17 @@ function expectedNode(
684914
};
685915
}
686916

917+
/** A capture of `count` addressable nodes, to exercise a page bigger than one batch group. */
918+
function manyRefSnapshotPayload(count: number): ExecResult {
919+
const refs: Record<string, { role: string; name: string }> = {};
920+
const lines: string[] = [];
921+
for (let index = 1; index <= count; index += 1) {
922+
refs[`e${index}`] = { role: 'button', name: `Item ${index}` };
923+
lines.push(`- button "Item ${index}" [ref=e${index}]`);
924+
}
925+
return jsonResult({ success: true, data: { refs, snapshot: lines.join('\n') } });
926+
}
927+
687928
function snapshotPayload(): ExecResult {
688929
return jsonResult({
689930
success: true,
@@ -705,12 +946,10 @@ function snapshotPayload(): ExecResult {
705946
});
706947
}
707948

708-
function boxPayload(ref: string | undefined): ExecResult {
949+
/** The box the backend reports for a ref, as a batch entry carries it inline. */
950+
function boxRect(ref: string | undefined): { x: number; y: number; width: number; height: number } {
709951
const offset = ref === '@e1' ? 0 : ref === '@e2' ? 10 : 30;
710-
return jsonResult({
711-
success: true,
712-
data: { x: offset + 1, y: offset + 2, width: 100, height: 20 },
713-
});
952+
return { x: offset + 1, y: offset + 2, width: 100, height: 20 };
714953
}
715954

716955
function jsonResult(value: unknown, exitCode = 0): ExecResult {

0 commit comments

Comments
 (0)