|
| 1 | +import { beforeEach, describe, expect, test, vi } from 'vitest'; |
| 2 | +import fs from 'node:fs'; |
| 3 | +import path from 'node:path'; |
| 4 | +import { |
| 5 | + CONTACT_SHEET_CONTAINER_REASON, |
| 6 | + CONTACT_SHEET_DURATION_REASON, |
| 7 | + CONTACT_SHEET_EXTRACTION_REASON, |
| 8 | + CONTACT_SHEET_OUTPUT_COLLISION_REASON, |
| 9 | + CONTACT_SHEET_PIXEL_BUDGET_REASON, |
| 10 | + CONTACT_SHEET_UNSUPPORTED_HOST_REASON, |
| 11 | +} from '@agent-device/contracts/recording-contact-sheet'; |
| 12 | +import { AppError } from '@agent-device/kernel/errors'; |
| 13 | +import { BLACK, RED, WHITE, paintPng, solidPng } from '../png-pixels.fixtures.ts'; |
| 14 | +import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; |
| 15 | +import { decodePng } from '../png.ts'; |
| 16 | +import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts'; |
| 17 | +import { writeDecodedFrames, type StubFrame } from './contact-sheet.fixtures.ts'; |
| 18 | +import { buildRecordingContactSheet, recordingContactSheetPath } from './contact-sheet.ts'; |
| 19 | + |
| 20 | +vi.mock(import('@agent-device/host-kit/command'), async (importOriginal) => ({ |
| 21 | + ...(await importOriginal()), |
| 22 | + runCmd: vi.fn(), |
| 23 | +})); |
| 24 | + |
| 25 | +vi.mock(import('./swift-cache.ts'), async (importOriginal) => ({ |
| 26 | + ...(await importOriginal<typeof import('./swift-cache.ts')>()), |
| 27 | + compileSwiftSourceFile: vi.fn(async () => '/cached/bin/recording-frames'), |
| 28 | +})); |
| 29 | + |
| 30 | +import { runCmd } from '@agent-device/host-kit/command'; |
| 31 | +import { likelyPlayableWebmContainer } from '../__tests__/test-utils/video-fixtures.ts'; |
| 32 | + |
| 33 | +const mockRunCmd = vi.mocked(runCmd); |
| 34 | +const directory = mkdtempForTestSync('agent-device-contact-sheet-pipeline-'); |
| 35 | + |
| 36 | +/** A recording the container sniff and the timeline reader both recognise, at a chosen length. */ |
| 37 | +function recording(name: string, durationMs: number): string { |
| 38 | + const filePath = path.join(directory, `${name}.mp4`); |
| 39 | + fs.writeFileSync( |
| 40 | + filePath, |
| 41 | + Buffer.concat([ |
| 42 | + mp4Atom('ftyp', Buffer.alloc(24)), |
| 43 | + mp4Atom('mdat', Buffer.alloc(16)), |
| 44 | + mp4Atom( |
| 45 | + 'moov', |
| 46 | + mp4Atom( |
| 47 | + 'mvhd', |
| 48 | + mp4MovieHeader({ version: 0, timescale: 1_000, duration: unknownable(durationMs) }), |
| 49 | + ), |
| 50 | + ), |
| 51 | + ]), |
| 52 | + ); |
| 53 | + return filePath; |
| 54 | +} |
| 55 | + |
| 56 | +function unknownable(durationMs: number): number { |
| 57 | + return durationMs < 0 ? 0xffffffff : durationMs; |
| 58 | +} |
| 59 | + |
| 60 | +function frame(timeMs: number, changed: boolean): StubFrame { |
| 61 | + return { |
| 62 | + png: changed |
| 63 | + ? paintPng(solidPng(8, 8, BLACK), { x: 0, y: 0, width: 8, height: 8 }, RED) |
| 64 | + : solidPng(8, 8, BLACK), |
| 65 | + actualTimeMs: timeMs, |
| 66 | + }; |
| 67 | +} |
| 68 | + |
| 69 | +/** |
| 70 | + * Answers the grid the planner will ask for. `presentMs` names the times the decoder answers; |
| 71 | + * anything the grid asked for and is left out is reported skipped, as a real decoder declines. |
| 72 | + */ |
| 73 | +function answerForGrid( |
| 74 | + input: Readonly<{ |
| 75 | + timesMs: readonly number[]; |
| 76 | + presentMs?: readonly number[]; |
| 77 | + changedMs?: readonly number[]; |
| 78 | + }>, |
| 79 | +): void { |
| 80 | + const present = new Set(input.presentMs ?? input.timesMs); |
| 81 | + const changed = new Set(input.changedMs ?? []); |
| 82 | + const byRequestedTime = new Map<number, StubFrame>(); |
| 83 | + for (const timeMs of input.timesMs) { |
| 84 | + if (present.has(timeMs)) byRequestedTime.set(timeMs, frame(timeMs, changed.has(timeMs))); |
| 85 | + } |
| 86 | + mockRunCmd.mockImplementation(async (_cmd, args) => |
| 87 | + writeDecodedFrames({ args: args as string[], framesByRequestedTimeMs: byRequestedTime }), |
| 88 | + ); |
| 89 | +} |
| 90 | + |
| 91 | +/** |
| 92 | + * Every case runs on a macOS host unless it names another, because frame decoding is Apple tooling |
| 93 | + * and the unit lane also runs on Linux. |
| 94 | + */ |
| 95 | +type BuildInput = Partial< |
| 96 | + Omit<Parameters<typeof buildRecordingContactSheet>[0], 'videoPath' | 'hostPlatform'> |
| 97 | +> & { hostPlatform?: NodeJS.Platform }; |
| 98 | + |
| 99 | +function build(videoPath: string, input: BuildInput = {}) { |
| 100 | + return buildRecordingContactSheet({ |
| 101 | + ...input, |
| 102 | + videoPath, |
| 103 | + maxPixels: input.maxPixels ?? 20_000_000, |
| 104 | + changedPixelThreshold: input.changedPixelThreshold ?? 0.1, |
| 105 | + hostPlatform: input.hostPlatform ?? 'darwin', |
| 106 | + }); |
| 107 | +} |
| 108 | + |
| 109 | +async function reasonOf(action: () => Promise<unknown>): Promise<unknown> { |
| 110 | + try { |
| 111 | + await action(); |
| 112 | + } catch (error) { |
| 113 | + return error instanceof AppError ? error.details?.reason : error; |
| 114 | + } |
| 115 | + return 'no error thrown'; |
| 116 | +} |
| 117 | + |
| 118 | +describe('buildRecordingContactSheet', () => { |
| 119 | + beforeEach(() => { |
| 120 | + mockRunCmd.mockReset(); |
| 121 | + }); |
| 122 | + |
| 123 | + test('writes the sheet beside the recording and says what it covered', async () => { |
| 124 | + const video = recording('cover', 1_000); |
| 125 | + answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [500] }); |
| 126 | + |
| 127 | + const sheet = await build(video); |
| 128 | + |
| 129 | + expect(sheet.path).toBe(recordingContactSheetPath(video)); |
| 130 | + expect(fs.existsSync(sheet.path)).toBe(true); |
| 131 | + expect(sheet.videoPath).toBe(video); |
| 132 | + expect(sheet.durationMs).toBe(1_000); |
| 133 | + expect(sheet.sampledFrameCount).toBe(5); |
| 134 | + expect(sheet.decodedFrameCount).toBe(5); |
| 135 | + expect(sheet.skippedSampleCount).toBe(0); |
| 136 | + // 750 returns to the opening screen, which is a change against the frame kept before it. |
| 137 | + expect(sheet.cells.map((cell) => cell.timeMs)).toEqual([0, 500, 750, 1000]); |
| 138 | + expect(sheet.warning).toBeUndefined(); |
| 139 | + |
| 140 | + const decoded = decodePng(fs.readFileSync(sheet.path), 'sheet'); |
| 141 | + expect([decoded.width, decoded.height]).toEqual([sheet.width, sheet.height]); |
| 142 | + expect( |
| 143 | + [...fs.readdirSync(directory)].some( |
| 144 | + (name) => name.endsWith('.writing') || name.endsWith('.tmp'), |
| 145 | + ), |
| 146 | + ).toBe(false); |
| 147 | + }); |
| 148 | + |
| 149 | + test('honours an explicit output path', async () => { |
| 150 | + const video = recording('explicit', 500); |
| 151 | + const outputPath = path.join(directory, 'named', 'sheet.png'); |
| 152 | + answerForGrid({ timesMs: [0, 250, 500], changedMs: [250] }); |
| 153 | + |
| 154 | + const sheet = await build(video, { outputPath }); |
| 155 | + |
| 156 | + expect(sheet.path).toBe(outputPath); |
| 157 | + expect(fs.existsSync(outputPath)).toBe(true); |
| 158 | + }); |
| 159 | + |
| 160 | + test('keeps one cell when the recorded screen never changed', async () => { |
| 161 | + const video = recording('static', 500); |
| 162 | + answerForGrid({ timesMs: [0, 250, 500] }); |
| 163 | + |
| 164 | + const sheet = await build(video); |
| 165 | + |
| 166 | + expect(sheet.cells).toHaveLength(2); |
| 167 | + expect(sheet.cells.at(-1)).toMatchObject({ timeMs: 500, changedPixelRatio: 0 }); |
| 168 | + }); |
| 169 | + |
| 170 | + test('discloses the sample times the decoder declined', async () => { |
| 171 | + const video = recording('skipped', 1_000); |
| 172 | + answerForGrid({ |
| 173 | + timesMs: [0, 250, 500, 750, 1000], |
| 174 | + presentMs: [0, 750, 1000], |
| 175 | + changedMs: [750], |
| 176 | + }); |
| 177 | + |
| 178 | + const sheet = await build(video); |
| 179 | + |
| 180 | + expect(sheet.sampledFrameCount).toBe(5); |
| 181 | + expect(sheet.decodedFrameCount).toBe(3); |
| 182 | + expect(sheet.skippedSampleCount).toBe(2); |
| 183 | + expect(sheet.warning).toMatch(/2 sample times returned no frame/); |
| 184 | + }); |
| 185 | + |
| 186 | + test('refuses a WebM recording rather than decoding nothing and calling it a sheet', async () => { |
| 187 | + const webm = path.join(directory, 'web.webm'); |
| 188 | + fs.writeFileSync(webm, likelyPlayableWebmContainer()); |
| 189 | + |
| 190 | + const error = await build(webm).catch((error: unknown) => error); |
| 191 | + |
| 192 | + expect(error).toBeInstanceOf(AppError); |
| 193 | + expect((error as AppError).details?.reason).toBe(CONTACT_SHEET_CONTAINER_REASON); |
| 194 | + expect((error as AppError).message).toMatch(/WebM/); |
| 195 | + }); |
| 196 | + |
| 197 | + test('refuses a file that is not a readable MP4', async () => { |
| 198 | + const notAVideo = path.join(directory, 'broken.mp4'); |
| 199 | + fs.writeFileSync(notAVideo, 'not a video at all'); |
| 200 | + |
| 201 | + expect(await reasonOf(() => build(notAVideo))).toBe(CONTACT_SHEET_CONTAINER_REASON); |
| 202 | + }); |
| 203 | + |
| 204 | + test('refuses a clip whose timeline the container cannot name', async () => { |
| 205 | + expect(await reasonOf(() => build(recording('no-duration', -1)))).toBe( |
| 206 | + CONTACT_SHEET_DURATION_REASON, |
| 207 | + ); |
| 208 | + }); |
| 209 | + |
| 210 | + test('surfaces a failed extraction and writes no sheet', async () => { |
| 211 | + const video = recording('extraction-fails', 1_000); |
| 212 | + mockRunCmd.mockImplementation(async (_cmd, args) => |
| 213 | + writeDecodedFrames({ |
| 214 | + args: args as string[], |
| 215 | + framesByRequestedTimeMs: new Map(), |
| 216 | + failWithExitCode: 1, |
| 217 | + }), |
| 218 | + ); |
| 219 | + |
| 220 | + expect(await reasonOf(() => build(video))).toBe(CONTACT_SHEET_EXTRACTION_REASON); |
| 221 | + expect(fs.existsSync(recordingContactSheetPath(video))).toBe(false); |
| 222 | + }); |
| 223 | + |
| 224 | + test('refuses to try on a host with no frame decoder', async () => { |
| 225 | + const video = recording('unsupported-host', 1_000); |
| 226 | + |
| 227 | + expect(await reasonOf(() => build(video, { hostPlatform: 'linux' }))).toBe( |
| 228 | + CONTACT_SHEET_UNSUPPORTED_HOST_REASON, |
| 229 | + ); |
| 230 | + expect(mockRunCmd).not.toHaveBeenCalled(); |
| 231 | + }); |
| 232 | + |
| 233 | + test('draws the sheet on a decoded frame whatever the source pixels were', async () => { |
| 234 | + const video = recording('pixels', 250); |
| 235 | + const byRequestedTime = new Map<number, StubFrame>([ |
| 236 | + [0, { png: solidPng(8, 8, WHITE), actualTimeMs: 0 }], |
| 237 | + ]); |
| 238 | + mockRunCmd.mockImplementation(async (_cmd, args) => |
| 239 | + writeDecodedFrames({ args: args as string[], framesByRequestedTimeMs: byRequestedTime }), |
| 240 | + ); |
| 241 | + |
| 242 | + const sheet = await build(video); |
| 243 | + const decoded = decodePng(fs.readFileSync(sheet.path), 'sheet'); |
| 244 | + |
| 245 | + let white = 0; |
| 246 | + for (let offset = 0; offset < decoded.data.length; offset += 4) { |
| 247 | + if (decoded.data[offset] === 255 && decoded.data[offset + 1] === 255) white += 1; |
| 248 | + } |
| 249 | + expect(white).toBeGreaterThan(0); |
| 250 | + }); |
| 251 | + |
| 252 | + test('refuses to write the sheet where the recording it describes is', async () => { |
| 253 | + const video = recording('collision', 1_000); |
| 254 | + |
| 255 | + expect(await reasonOf(() => build(video, { outputPath: video }))).toBe( |
| 256 | + CONTACT_SHEET_OUTPUT_COLLISION_REASON, |
| 257 | + ); |
| 258 | + // Refused before the decoder ran, so a mistyped --out cannot cost a ruined recording. |
| 259 | + expect(mockRunCmd).not.toHaveBeenCalled(); |
| 260 | + expect(fs.readFileSync(video).length).toBeGreaterThan(0); |
| 261 | + }); |
| 262 | + |
| 263 | + test('refuses an output path that reaches the recording through a second name', async () => { |
| 264 | + const video = recording('aliased', 1_000); |
| 265 | + const symlink = path.join(path.dirname(video), 'sheet-of-my-recording.mp4'); |
| 266 | + const hardLink = path.join(path.dirname(video), 'recording-copy.mp4'); |
| 267 | + fs.symlinkSync(video, symlink); |
| 268 | + fs.linkSync(video, hardLink); |
| 269 | + |
| 270 | + for (const outputPath of [symlink, hardLink]) { |
| 271 | + expect(await reasonOf(() => build(video, { outputPath }))).toBe( |
| 272 | + CONTACT_SHEET_OUTPUT_COLLISION_REASON, |
| 273 | + ); |
| 274 | + } |
| 275 | + // Neither alias was opened for writing, so the recording is still whole behind both names. |
| 276 | + expect(mockRunCmd).not.toHaveBeenCalled(); |
| 277 | + expect(fs.readFileSync(video).length).toBe(fs.statSync(hardLink).size); |
| 278 | + }); |
| 279 | + |
| 280 | + test('refuses a decoded frame too large for the sheet it was meant to fill', async () => { |
| 281 | + const video = recording('oversized', 250); |
| 282 | + mockRunCmd.mockImplementation(async (_cmd, args) => |
| 283 | + writeDecodedFrames({ |
| 284 | + args: args as string[], |
| 285 | + framesByRequestedTimeMs: new Map([ |
| 286 | + [0, { png: solidPng(16, 16, BLACK), actualTimeMs: 0 }], |
| 287 | + [250, { png: solidPng(16, 16, BLACK), actualTimeMs: 250 }], |
| 288 | + ]), |
| 289 | + }), |
| 290 | + ); |
| 291 | + |
| 292 | + expect(await reasonOf(() => build(video, { maxPixels: 100 }))).toBe( |
| 293 | + CONTACT_SHEET_EXTRACTION_REASON, |
| 294 | + ); |
| 295 | + }); |
| 296 | + |
| 297 | + test('refuses a wider frame than the cells the decoder was asked to draw', async () => { |
| 298 | + const video = recording('wide', 250); |
| 299 | + mockRunCmd.mockImplementation(async (_cmd, args) => |
| 300 | + writeDecodedFrames({ |
| 301 | + args: args as string[], |
| 302 | + framesByRequestedTimeMs: new Map([ |
| 303 | + [0, { png: solidPng(400, 40, BLACK), actualTimeMs: 0 }], |
| 304 | + [250, { png: solidPng(400, 40, BLACK), actualTimeMs: 250 }], |
| 305 | + ]), |
| 306 | + }), |
| 307 | + ); |
| 308 | + |
| 309 | + expect(await reasonOf(() => build(video))).toBe(CONTACT_SHEET_EXTRACTION_REASON); |
| 310 | + }); |
| 311 | + |
| 312 | + test('refuses a sheet the pixel budget cannot hold and leaves nothing at the output', async () => { |
| 313 | + const video = recording('budget', 1_000); |
| 314 | + const out = path.join(directory, 'budget-sheet.png'); |
| 315 | + answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [250, 500, 750, 1000] }); |
| 316 | + |
| 317 | + expect(await reasonOf(() => build(video, { outputPath: out, maxPixels: 200 }))).toBe( |
| 318 | + CONTACT_SHEET_PIXEL_BUDGET_REASON, |
| 319 | + ); |
| 320 | + expect(fs.existsSync(out)).toBe(false); |
| 321 | + expect(fs.readdirSync(directory).filter((name) => name.endsWith('.writing'))).toEqual([]); |
| 322 | + }); |
| 323 | + |
| 324 | + test('replaces a sheet that is already at the output path', async () => { |
| 325 | + const video = recording('replace', 1_000); |
| 326 | + const out = path.join(directory, 'replace-sheet.png'); |
| 327 | + fs.writeFileSync(out, 'a sheet from an earlier run'); |
| 328 | + answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [500] }); |
| 329 | + |
| 330 | + const sheet = await build(video, { outputPath: out }); |
| 331 | + |
| 332 | + expect(sheet.path).toBe(out); |
| 333 | + expect(fs.readFileSync(out).subarray(1, 4).toString()).toBe('PNG'); |
| 334 | + }); |
| 335 | +}); |
0 commit comments