Skip to content

Commit 69c8c25

Browse files
committed
feat(recording): build a contact sheet from a finished recording
1 parent 814b2cc commit 69c8c25

9 files changed

Lines changed: 713 additions & 13 deletions

File tree

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { describe, expect, test } from 'vitest';
2+
import {
3+
collectedRecordingPath,
4+
nativeRecordingPath,
5+
recordingContactSheetPath,
6+
} from './artifact-paths.ts';
7+
8+
describe('recording artifact sibling paths', () => {
9+
test('keeps the recorder file and the collected copy beside the export', () => {
10+
expect(nativeRecordingPath('/tmp/recording.mp4')).toBe('/tmp/recording.native.mp4');
11+
expect(collectedRecordingPath('/tmp/recording.mp4')).toBe('/tmp/recording.collected.mp4');
12+
});
13+
14+
test('names the contact sheet after the report it is, not the container it came from', () => {
15+
expect(recordingContactSheetPath('/tmp/recording.mp4')).toBe(
16+
'/tmp/recording.contact-sheet.png',
17+
);
18+
expect(recordingContactSheetPath('/tmp/rec')).toBe('/tmp/rec.contact-sheet.png');
19+
expect(recordingContactSheetPath('/tmp/rec.TAKE-2.MP4')).toBe(
20+
'/tmp/rec.TAKE-2.contact-sheet.png',
21+
);
22+
});
23+
});

‎packages/capture-kit/src/recording/artifact-paths.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@ export function collectedRecordingPath(exportPath: string): string {
1919
return siblingRecordingPath(exportPath, 'collected');
2020
}
2121

22+
/**
23+
* The report a caller asks a finished export to carry: one PNG holding the frames that changed, so
24+
* an agent can read a recording without playing it. It is derived, so unlike the paths above it is
25+
* never a recorder's and is always safe to rebuild from the export.
26+
*/
27+
export function recordingContactSheetPath(exportPath: string): string {
28+
const extension = path.extname(exportPath);
29+
const base = extension === '' ? exportPath : exportPath.slice(0, -extension.length);
30+
return `${base}.contact-sheet.png`;
31+
}
32+
2233
function siblingRecordingPath(exportPath: string, role: 'native' | 'collected'): string {
2334
const extension = path.extname(exportPath);
2435
const base = extension === '' ? exportPath : exportPath.slice(0, -extension.length);

‎packages/capture-kit/src/recording/contact-sheet-selection.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { computePngChangedPixelRatio, type PngRgbImage } from '../png-changed-pi
1313
* unchanged chrome — measured 0.030. Any value in that gap separates the two, and this one sits
1414
* nearer the noise so a quiet change costs an extra cell instead of going unshown.
1515
*/
16-
const CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD = 0.02;
16+
export const CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD = 0.02;
1717

1818
/** Cells one sheet prints. The grid thins evenly across time to hold this, keeping both ends. */
1919
export const MAX_CONTACT_SHEET_CELLS = 24;
Lines changed: 335 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,335 @@
1+
import { beforeEach, describe, expect, test, vi } from 'vitest';
2+
import fs from 'node:fs';
3+
import path from 'node:path';
4+
import {
5+
CONTACT_SHEET_CONTAINER_REASON,
6+
CONTACT_SHEET_DURATION_REASON,
7+
CONTACT_SHEET_EXTRACTION_REASON,
8+
CONTACT_SHEET_OUTPUT_COLLISION_REASON,
9+
CONTACT_SHEET_PIXEL_BUDGET_REASON,
10+
CONTACT_SHEET_UNSUPPORTED_HOST_REASON,
11+
} from '@agent-device/contracts/recording-contact-sheet';
12+
import { AppError } from '@agent-device/kernel/errors';
13+
import { BLACK, RED, WHITE, paintPng, solidPng } from '../png-pixels.fixtures.ts';
14+
import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts';
15+
import { decodePng } from '../png.ts';
16+
import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts';
17+
import { writeDecodedFrames, type StubFrame } from './contact-sheet.fixtures.ts';
18+
import { buildRecordingContactSheet, recordingContactSheetPath } from './contact-sheet.ts';
19+
20+
vi.mock(import('@agent-device/host-kit/command'), async (importOriginal) => ({
21+
...(await importOriginal()),
22+
runCmd: vi.fn(),
23+
}));
24+
25+
vi.mock(import('./swift-cache.ts'), async (importOriginal) => ({
26+
...(await importOriginal<typeof import('./swift-cache.ts')>()),
27+
compileSwiftSourceFile: vi.fn(async () => '/cached/bin/recording-frames'),
28+
}));
29+
30+
import { runCmd } from '@agent-device/host-kit/command';
31+
import { likelyPlayableWebmContainer } from '../__tests__/test-utils/video-fixtures.ts';
32+
33+
const mockRunCmd = vi.mocked(runCmd);
34+
const directory = mkdtempForTestSync('agent-device-contact-sheet-pipeline-');
35+
36+
/** A recording the container sniff and the timeline reader both recognise, at a chosen length. */
37+
function recording(name: string, durationMs: number): string {
38+
const filePath = path.join(directory, `${name}.mp4`);
39+
fs.writeFileSync(
40+
filePath,
41+
Buffer.concat([
42+
mp4Atom('ftyp', Buffer.alloc(24)),
43+
mp4Atom('mdat', Buffer.alloc(16)),
44+
mp4Atom(
45+
'moov',
46+
mp4Atom(
47+
'mvhd',
48+
mp4MovieHeader({ version: 0, timescale: 1_000, duration: unknownable(durationMs) }),
49+
),
50+
),
51+
]),
52+
);
53+
return filePath;
54+
}
55+
56+
function unknownable(durationMs: number): number {
57+
return durationMs < 0 ? 0xffffffff : durationMs;
58+
}
59+
60+
function frame(timeMs: number, changed: boolean): StubFrame {
61+
return {
62+
png: changed
63+
? paintPng(solidPng(8, 8, BLACK), { x: 0, y: 0, width: 8, height: 8 }, RED)
64+
: solidPng(8, 8, BLACK),
65+
actualTimeMs: timeMs,
66+
};
67+
}
68+
69+
/**
70+
* Answers the grid the planner will ask for. `presentMs` names the times the decoder answers;
71+
* anything the grid asked for and is left out is reported skipped, as a real decoder declines.
72+
*/
73+
function answerForGrid(
74+
input: Readonly<{
75+
timesMs: readonly number[];
76+
presentMs?: readonly number[];
77+
changedMs?: readonly number[];
78+
}>,
79+
): void {
80+
const present = new Set(input.presentMs ?? input.timesMs);
81+
const changed = new Set(input.changedMs ?? []);
82+
const byRequestedTime = new Map<number, StubFrame>();
83+
for (const timeMs of input.timesMs) {
84+
if (present.has(timeMs)) byRequestedTime.set(timeMs, frame(timeMs, changed.has(timeMs)));
85+
}
86+
mockRunCmd.mockImplementation(async (_cmd, args) =>
87+
writeDecodedFrames({ args: args as string[], framesByRequestedTimeMs: byRequestedTime }),
88+
);
89+
}
90+
91+
/**
92+
* Every case runs on a macOS host unless it names another, because frame decoding is Apple tooling
93+
* and the unit lane also runs on Linux.
94+
*/
95+
type BuildInput = Partial<
96+
Omit<Parameters<typeof buildRecordingContactSheet>[0], 'videoPath' | 'hostPlatform'>
97+
> & { hostPlatform?: NodeJS.Platform };
98+
99+
function build(videoPath: string, input: BuildInput = {}) {
100+
return buildRecordingContactSheet({
101+
...input,
102+
videoPath,
103+
maxPixels: input.maxPixels ?? 20_000_000,
104+
changedPixelThreshold: input.changedPixelThreshold ?? 0.1,
105+
hostPlatform: input.hostPlatform ?? 'darwin',
106+
});
107+
}
108+
109+
async function reasonOf(action: () => Promise<unknown>): Promise<unknown> {
110+
try {
111+
await action();
112+
} catch (error) {
113+
return error instanceof AppError ? error.details?.reason : error;
114+
}
115+
return 'no error thrown';
116+
}
117+
118+
describe('buildRecordingContactSheet', () => {
119+
beforeEach(() => {
120+
mockRunCmd.mockReset();
121+
});
122+
123+
test('writes the sheet beside the recording and says what it covered', async () => {
124+
const video = recording('cover', 1_000);
125+
answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [500] });
126+
127+
const sheet = await build(video);
128+
129+
expect(sheet.path).toBe(recordingContactSheetPath(video));
130+
expect(fs.existsSync(sheet.path)).toBe(true);
131+
expect(sheet.videoPath).toBe(video);
132+
expect(sheet.durationMs).toBe(1_000);
133+
expect(sheet.sampledFrameCount).toBe(5);
134+
expect(sheet.decodedFrameCount).toBe(5);
135+
expect(sheet.skippedSampleCount).toBe(0);
136+
// 750 returns to the opening screen, which is a change against the frame kept before it.
137+
expect(sheet.cells.map((cell) => cell.timeMs)).toEqual([0, 500, 750, 1000]);
138+
expect(sheet.warning).toBeUndefined();
139+
140+
const decoded = decodePng(fs.readFileSync(sheet.path), 'sheet');
141+
expect([decoded.width, decoded.height]).toEqual([sheet.width, sheet.height]);
142+
expect(
143+
[...fs.readdirSync(directory)].some(
144+
(name) => name.endsWith('.writing') || name.endsWith('.tmp'),
145+
),
146+
).toBe(false);
147+
});
148+
149+
test('honours an explicit output path', async () => {
150+
const video = recording('explicit', 500);
151+
const outputPath = path.join(directory, 'named', 'sheet.png');
152+
answerForGrid({ timesMs: [0, 250, 500], changedMs: [250] });
153+
154+
const sheet = await build(video, { outputPath });
155+
156+
expect(sheet.path).toBe(outputPath);
157+
expect(fs.existsSync(outputPath)).toBe(true);
158+
});
159+
160+
test('keeps one cell when the recorded screen never changed', async () => {
161+
const video = recording('static', 500);
162+
answerForGrid({ timesMs: [0, 250, 500] });
163+
164+
const sheet = await build(video);
165+
166+
expect(sheet.cells).toHaveLength(2);
167+
expect(sheet.cells.at(-1)).toMatchObject({ timeMs: 500, changedPixelRatio: 0 });
168+
});
169+
170+
test('discloses the sample times the decoder declined', async () => {
171+
const video = recording('skipped', 1_000);
172+
answerForGrid({
173+
timesMs: [0, 250, 500, 750, 1000],
174+
presentMs: [0, 750, 1000],
175+
changedMs: [750],
176+
});
177+
178+
const sheet = await build(video);
179+
180+
expect(sheet.sampledFrameCount).toBe(5);
181+
expect(sheet.decodedFrameCount).toBe(3);
182+
expect(sheet.skippedSampleCount).toBe(2);
183+
expect(sheet.warning).toMatch(/2 sample times returned no frame/);
184+
});
185+
186+
test('refuses a WebM recording rather than decoding nothing and calling it a sheet', async () => {
187+
const webm = path.join(directory, 'web.webm');
188+
fs.writeFileSync(webm, likelyPlayableWebmContainer());
189+
190+
const error = await build(webm).catch((error: unknown) => error);
191+
192+
expect(error).toBeInstanceOf(AppError);
193+
expect((error as AppError).details?.reason).toBe(CONTACT_SHEET_CONTAINER_REASON);
194+
expect((error as AppError).message).toMatch(/WebM/);
195+
});
196+
197+
test('refuses a file that is not a readable MP4', async () => {
198+
const notAVideo = path.join(directory, 'broken.mp4');
199+
fs.writeFileSync(notAVideo, 'not a video at all');
200+
201+
expect(await reasonOf(() => build(notAVideo))).toBe(CONTACT_SHEET_CONTAINER_REASON);
202+
});
203+
204+
test('refuses a clip whose timeline the container cannot name', async () => {
205+
expect(await reasonOf(() => build(recording('no-duration', -1)))).toBe(
206+
CONTACT_SHEET_DURATION_REASON,
207+
);
208+
});
209+
210+
test('surfaces a failed extraction and writes no sheet', async () => {
211+
const video = recording('extraction-fails', 1_000);
212+
mockRunCmd.mockImplementation(async (_cmd, args) =>
213+
writeDecodedFrames({
214+
args: args as string[],
215+
framesByRequestedTimeMs: new Map(),
216+
failWithExitCode: 1,
217+
}),
218+
);
219+
220+
expect(await reasonOf(() => build(video))).toBe(CONTACT_SHEET_EXTRACTION_REASON);
221+
expect(fs.existsSync(recordingContactSheetPath(video))).toBe(false);
222+
});
223+
224+
test('refuses to try on a host with no frame decoder', async () => {
225+
const video = recording('unsupported-host', 1_000);
226+
227+
expect(await reasonOf(() => build(video, { hostPlatform: 'linux' }))).toBe(
228+
CONTACT_SHEET_UNSUPPORTED_HOST_REASON,
229+
);
230+
expect(mockRunCmd).not.toHaveBeenCalled();
231+
});
232+
233+
test('draws the sheet on a decoded frame whatever the source pixels were', async () => {
234+
const video = recording('pixels', 250);
235+
const byRequestedTime = new Map<number, StubFrame>([
236+
[0, { png: solidPng(8, 8, WHITE), actualTimeMs: 0 }],
237+
]);
238+
mockRunCmd.mockImplementation(async (_cmd, args) =>
239+
writeDecodedFrames({ args: args as string[], framesByRequestedTimeMs: byRequestedTime }),
240+
);
241+
242+
const sheet = await build(video);
243+
const decoded = decodePng(fs.readFileSync(sheet.path), 'sheet');
244+
245+
let white = 0;
246+
for (let offset = 0; offset < decoded.data.length; offset += 4) {
247+
if (decoded.data[offset] === 255 && decoded.data[offset + 1] === 255) white += 1;
248+
}
249+
expect(white).toBeGreaterThan(0);
250+
});
251+
252+
test('refuses to write the sheet where the recording it describes is', async () => {
253+
const video = recording('collision', 1_000);
254+
255+
expect(await reasonOf(() => build(video, { outputPath: video }))).toBe(
256+
CONTACT_SHEET_OUTPUT_COLLISION_REASON,
257+
);
258+
// Refused before the decoder ran, so a mistyped --out cannot cost a ruined recording.
259+
expect(mockRunCmd).not.toHaveBeenCalled();
260+
expect(fs.readFileSync(video).length).toBeGreaterThan(0);
261+
});
262+
263+
test('refuses an output path that reaches the recording through a second name', async () => {
264+
const video = recording('aliased', 1_000);
265+
const symlink = path.join(path.dirname(video), 'sheet-of-my-recording.mp4');
266+
const hardLink = path.join(path.dirname(video), 'recording-copy.mp4');
267+
fs.symlinkSync(video, symlink);
268+
fs.linkSync(video, hardLink);
269+
270+
for (const outputPath of [symlink, hardLink]) {
271+
expect(await reasonOf(() => build(video, { outputPath }))).toBe(
272+
CONTACT_SHEET_OUTPUT_COLLISION_REASON,
273+
);
274+
}
275+
// Neither alias was opened for writing, so the recording is still whole behind both names.
276+
expect(mockRunCmd).not.toHaveBeenCalled();
277+
expect(fs.readFileSync(video).length).toBe(fs.statSync(hardLink).size);
278+
});
279+
280+
test('refuses a decoded frame too large for the sheet it was meant to fill', async () => {
281+
const video = recording('oversized', 250);
282+
mockRunCmd.mockImplementation(async (_cmd, args) =>
283+
writeDecodedFrames({
284+
args: args as string[],
285+
framesByRequestedTimeMs: new Map([
286+
[0, { png: solidPng(16, 16, BLACK), actualTimeMs: 0 }],
287+
[250, { png: solidPng(16, 16, BLACK), actualTimeMs: 250 }],
288+
]),
289+
}),
290+
);
291+
292+
expect(await reasonOf(() => build(video, { maxPixels: 100 }))).toBe(
293+
CONTACT_SHEET_EXTRACTION_REASON,
294+
);
295+
});
296+
297+
test('refuses a wider frame than the cells the decoder was asked to draw', async () => {
298+
const video = recording('wide', 250);
299+
mockRunCmd.mockImplementation(async (_cmd, args) =>
300+
writeDecodedFrames({
301+
args: args as string[],
302+
framesByRequestedTimeMs: new Map([
303+
[0, { png: solidPng(400, 40, BLACK), actualTimeMs: 0 }],
304+
[250, { png: solidPng(400, 40, BLACK), actualTimeMs: 250 }],
305+
]),
306+
}),
307+
);
308+
309+
expect(await reasonOf(() => build(video))).toBe(CONTACT_SHEET_EXTRACTION_REASON);
310+
});
311+
312+
test('refuses a sheet the pixel budget cannot hold and leaves nothing at the output', async () => {
313+
const video = recording('budget', 1_000);
314+
const out = path.join(directory, 'budget-sheet.png');
315+
answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [250, 500, 750, 1000] });
316+
317+
expect(await reasonOf(() => build(video, { outputPath: out, maxPixels: 200 }))).toBe(
318+
CONTACT_SHEET_PIXEL_BUDGET_REASON,
319+
);
320+
expect(fs.existsSync(out)).toBe(false);
321+
expect(fs.readdirSync(directory).filter((name) => name.endsWith('.writing'))).toEqual([]);
322+
});
323+
324+
test('replaces a sheet that is already at the output path', async () => {
325+
const video = recording('replace', 1_000);
326+
const out = path.join(directory, 'replace-sheet.png');
327+
fs.writeFileSync(out, 'a sheet from an earlier run');
328+
answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [500] });
329+
330+
const sheet = await build(video, { outputPath: out });
331+
332+
expect(sheet.path).toBe(out);
333+
expect(fs.readFileSync(out).subarray(1, 4).toString()).toBe('PNG');
334+
});
335+
});

0 commit comments

Comments
 (0)