Skip to content

Commit 2f940b2

Browse files
thymikeethymikee
committed
feat(recording): choose the frames a contact sheet shows
1 parent 442d367 commit 2f940b2

8 files changed

Lines changed: 456 additions & 8 deletions

‎packages/capture-kit/src/png-changed-pixel-ratio.test.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { describe, expect, test } from 'vitest';
2+
import { computePngChangedPixelRatio } from './png-changed-pixel-ratio.ts';
3+
import { BLACK, paintPng, solidPng, WHITE } from './png-pixels.fixtures.ts';
4+
5+
function pixels(png: { width: number; height: number; data: Buffer }) {
6+
return { width: png.width, height: png.height, data: png.data };
7+
}
8+
9+
describe('computePngChangedPixelRatio', () => {
10+
test('reports nothing moved for identical frames', () => {
11+
expect(
12+
computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 4, BLACK))),
13+
).toEqual({ status: 'compared', changedPixelRatio: 0 });
14+
});
15+
16+
test('counts a pixel whose blue channel alone moved', () => {
17+
const blue = paintPng(
18+
solidPng(4, 4, BLACK),
19+
{ x: 0, y: 0, width: 1, height: 1 },
20+
[0, 0, 9, 255],
21+
);
22+
expect(computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(blue))).toEqual({
23+
status: 'compared',
24+
changedPixelRatio: 1 / 16,
25+
});
26+
});
27+
28+
test('ignores alpha, which never reaches the recorded screen', () => {
29+
const transparent = solidPng(4, 4, BLACK);
30+
transparent.data[3] = 0;
31+
expect(computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(transparent))).toEqual(
32+
{ status: 'compared', changedPixelRatio: 0 },
33+
);
34+
});
35+
36+
test('counts every pixel when the whole frame moved', () => {
37+
expect(
38+
computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 4, WHITE))),
39+
).toEqual({ status: 'compared', changedPixelRatio: 1 });
40+
});
41+
42+
test('refuses a ratio across frames of different shapes', () => {
43+
expect(
44+
computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 8, BLACK))),
45+
).toEqual({ status: 'dimension_mismatch' });
46+
});
47+
});

‎packages/capture-kit/src/png-changed-pixel-ratio.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import type { PngRgbImage } from './png-rgb-difference.ts';
2+
3+
export type { PngRgbImage };
4+
5+
export type PngChangedPixelRatioResult =
6+
| { readonly status: 'compared'; readonly changedPixelRatio: number }
7+
| { readonly status: 'dimension_mismatch' };
8+
9+
/**
10+
* The share of pixels whose color moved at all between two decoded PNGs.
11+
*
12+
* A pixel counts as changed when any of its RGB channels differs, ignoring alpha, so a
13+
* translucent-layer fade that leaves the composite untouched is not reported. This is a
14+
* coverage metric, not a magnitude one: `computePngRgbDifference` answers how far the colors
15+
* moved, and this answers how much of the frame moved. A recording that only shifts a 1 px
16+
* progress line scores a ratio near zero either way.
17+
*
18+
* Dimensions must match. Callers comparing frames of different sizes are asking a different
19+
* question, and returning a made-up ratio would hide it.
20+
*/
21+
export function computePngChangedPixelRatio(
22+
first: PngRgbImage,
23+
second: PngRgbImage,
24+
): PngChangedPixelRatioResult {
25+
if (first.width !== second.width || first.height !== second.height) {
26+
return { status: 'dimension_mismatch' };
27+
}
28+
29+
const totalPixels = first.width * first.height;
30+
if (totalPixels === 0) return { status: 'compared', changedPixelRatio: 0 };
31+
if (first.data.length !== second.data.length) {
32+
return { status: 'dimension_mismatch' };
33+
}
34+
35+
let changedPixels = 0;
36+
for (let offset = 0; offset + 3 < first.data.length; offset += 4) {
37+
if (
38+
first.data[offset] !== second.data[offset] ||
39+
first.data[offset + 1] !== second.data[offset + 1] ||
40+
first.data[offset + 2] !== second.data[offset + 2]
41+
) {
42+
changedPixels += 1;
43+
}
44+
}
45+
46+
return { status: 'compared', changedPixelRatio: changedPixels / totalPixels };
47+
}

‎packages/capture-kit/src/png-pixels.fixtures.ts‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,43 @@
11
import { PNG } from './png.ts';
22

33
/**
4-
* Decoded-frame builders for tests that reason about pixels: a solid fill is enough to name what a
5-
* test claims a frame contains without shipping a PNG fixture through the repo.
4+
* Decoded-frame builders for tests that reason about pixels: a solid fill and one painted rectangle
5+
* name exactly what a test claims changed between two frames.
66
*/
77

88
export type Rgba = readonly [number, number, number, number];
9+
export type Rectangle = Readonly<{ x: number; y: number; width: number; height: number }>;
910

1011
export const BLACK: Rgba = [0, 0, 0, 255];
12+
export const WHITE: Rgba = [255, 255, 255, 255];
13+
export const RED: Rgba = [255, 0, 0, 255];
1114

1215
export function solidPng(width: number, height: number, color: Rgba = BLACK): PNG {
13-
const png = new PNG({ width, height });
14-
for (let offset = 0; offset < png.data.length; offset += 4) {
15-
png.data[offset] = color[0];
16-
png.data[offset + 1] = color[1];
17-
png.data[offset + 2] = color[2];
18-
png.data[offset + 3] = color[3];
16+
return fillPng(new PNG({ width, height }), () => true, color);
17+
}
18+
19+
/** Paints one rectangle of `color` onto a copy of `source`, leaving the source untouched. */
20+
export function paintPng(source: PNG, rectangle: Rectangle, color: Rgba): PNG {
21+
const copy = solidPng(source.width, source.height);
22+
source.data.copy(copy.data);
23+
const inside = (column: number, row: number) =>
24+
column >= rectangle.x &&
25+
column < rectangle.x + rectangle.width &&
26+
row >= rectangle.y &&
27+
row < rectangle.y + rectangle.height;
28+
return fillPng(copy, inside, color);
29+
}
30+
31+
function fillPng(png: PNG, paint: (column: number, row: number) => boolean, color: Rgba): PNG {
32+
for (let row = 0; row < png.height; row += 1) {
33+
for (let column = 0; column < png.width; column += 1) {
34+
if (!paint(column, row)) continue;
35+
const offset = (row * png.width + column) * 4;
36+
png.data[offset] = color[0];
37+
png.data[offset + 1] = color[1];
38+
png.data[offset + 2] = color[2];
39+
png.data[offset + 3] = color[3];
40+
}
1941
}
2042
return png;
2143
}

‎packages/capture-kit/src/recording/contact-sheet-plan.test.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import { describe, expect, test } from 'vitest';
2+
import { CONTACT_SHEET_DURATION_REASON } from '@agent-device/contracts/recording-contact-sheet';
3+
import { AppError } from '@agent-device/kernel/errors';
4+
import {
5+
CONTACT_SHEET_SAMPLE_INTERVAL_MS,
6+
MAX_CONTACT_SHEET_SAMPLED_FRAMES,
7+
planContactSheetSampleTimes,
8+
} from './contact-sheet-plan.ts';
9+
10+
const VIDEO = '/tmp/recording.mp4';
11+
12+
function reasonOf(action: () => unknown): unknown {
13+
try {
14+
action();
15+
} catch (error) {
16+
return error instanceof AppError ? error.details?.reason : `not an AppError: ${String(error)}`;
17+
}
18+
return 'no error thrown';
19+
}
20+
21+
describe('planContactSheetSampleTimes', () => {
22+
test('names both endpoints of a clip no longer than one sample step', () => {
23+
expect(planContactSheetSampleTimes(0, VIDEO)).toEqual({ durationMs: 0, timesMs: [0] });
24+
// One step is not one frame: a clip can change between its first and last presentation sample,
25+
// and the sheet promises its ending whatever the clip's length.
26+
expect(planContactSheetSampleTimes(CONTACT_SHEET_SAMPLE_INTERVAL_MS, VIDEO)).toEqual({
27+
durationMs: CONTACT_SHEET_SAMPLE_INTERVAL_MS,
28+
timesMs: [0, CONTACT_SHEET_SAMPLE_INTERVAL_MS],
29+
});
30+
expect(planContactSheetSampleTimes(100, VIDEO)).toEqual({ durationMs: 100, timesMs: [0, 100] });
31+
});
32+
33+
test('samples every step while the clip is shorter than the cap', () => {
34+
expect(planContactSheetSampleTimes(1_000, VIDEO).timesMs).toEqual([0, 250, 500, 750, 1000]);
35+
});
36+
37+
test('stretches the same grid over a long clip instead of adding samples', () => {
38+
const hour = planContactSheetSampleTimes(3_600_000, VIDEO);
39+
40+
expect(hour.timesMs).toHaveLength(MAX_CONTACT_SHEET_SAMPLED_FRAMES);
41+
expect(hour.timesMs[0]).toBe(0);
42+
expect(hour.timesMs.at(-1)).toBe(3_600_000);
43+
expect(new Set(hour.timesMs).size).toBe(MAX_CONTACT_SHEET_SAMPLED_FRAMES);
44+
hour.timesMs.forEach((time, index) => {
45+
if (index > 0) expect(time).toBeGreaterThan(hour.timesMs[index - 1]!);
46+
});
47+
});
48+
49+
test('keeps both endpoints, which is what a coverage claim rests on', () => {
50+
for (const durationMs of [5_000, 41_000, 900_000, 7_200_000]) {
51+
const plan = planContactSheetSampleTimes(durationMs, VIDEO);
52+
expect(plan.durationMs).toBe(durationMs);
53+
expect(plan.timesMs[0]).toBe(0);
54+
expect(plan.timesMs.at(-1)).toBe(durationMs);
55+
}
56+
});
57+
58+
test('refuses to plan over a timeline the container cannot name', () => {
59+
expect(reasonOf(() => planContactSheetSampleTimes(undefined, VIDEO))).toBe(
60+
CONTACT_SHEET_DURATION_REASON,
61+
);
62+
expect(reasonOf(() => planContactSheetSampleTimes(Number.NaN, VIDEO))).toBe(
63+
CONTACT_SHEET_DURATION_REASON,
64+
);
65+
expect(reasonOf(() => planContactSheetSampleTimes(-1, VIDEO))).toBe(
66+
CONTACT_SHEET_DURATION_REASON,
67+
);
68+
});
69+
});

‎packages/capture-kit/src/recording/contact-sheet-plan.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { CONTACT_SHEET_DURATION_REASON } from '@agent-device/contracts/recording-contact-sheet';
2+
import { AppError } from '@agent-device/kernel/errors';
3+
4+
/** Spacing between requested sample times, in milliseconds. */
5+
export const CONTACT_SHEET_SAMPLE_INTERVAL_MS = 250;
6+
/** Sample times one sheet asks the decoder for, however long the clip runs. */
7+
export const MAX_CONTACT_SHEET_SAMPLED_FRAMES = 48;
8+
9+
/**
10+
* The times to sample from a clip of `durationMs`, spread evenly across the whole timeline.
11+
*
12+
* Coverage is the point: the grid always names the start and the end, and it never grows with the
13+
* recording. A 5s take samples 21 times; a 2h take samples the same 48 times across its length, so
14+
* a long recording costs the same decode budget as a short one and still shows both endpoints.
15+
*
16+
* A grid is a promise with a hole in it. A flash that opens and closes entirely between two sample
17+
* times is not in the returned frames and no threshold can recover it; that is why the sheet
18+
* reports how many times it sampled rather than claiming to have reviewed the footage.
19+
*
20+
* A clip whose timeline cannot be read is refused rather than guessed at. Sampling an unknown
21+
* length would mean either walking the whole file or drawing a grid over a duration the container
22+
* never claimed, and a sheet built that way could not say what it covered.
23+
*/
24+
export type ContactSheetSamplePlan = Readonly<{
25+
/** Timeline the grid was planned over, in milliseconds. */
26+
durationMs: number;
27+
timesMs: readonly number[];
28+
}>;
29+
30+
export function planContactSheetSampleTimes(
31+
durationMs: number | undefined,
32+
videoPath: string,
33+
): ContactSheetSamplePlan {
34+
if (durationMs === undefined || !Number.isFinite(durationMs) || durationMs < 0) {
35+
throw new AppError(
36+
'COMMAND_FAILED',
37+
`Cannot plan a contact sheet: the video timeline of ${videoPath} could not be read`,
38+
{
39+
reason: CONTACT_SHEET_DURATION_REASON,
40+
videoPath,
41+
hint: 'Retry once the recording finished finalizing; a clip whose container is still being written has no readable duration.',
42+
},
43+
);
44+
}
45+
if (durationMs <= CONTACT_SHEET_SAMPLE_INTERVAL_MS) {
46+
// A clip shorter than one sampling interval still ends somewhere, and the sheet always promises
47+
// its last frame, so the closing sample is asked for even when it is the only other one.
48+
return { durationMs, timesMs: durationMs > 0 ? [0, Math.round(durationMs)] : [0] };
49+
}
50+
51+
const count = Math.min(
52+
MAX_CONTACT_SHEET_SAMPLED_FRAMES,
53+
Math.floor(durationMs / CONTACT_SHEET_SAMPLE_INTERVAL_MS) + 1,
54+
);
55+
const step = durationMs / (count - 1);
56+
return {
57+
durationMs,
58+
timesMs: Array.from({ length: count }, (_, index) => Math.round(index * step)),
59+
};
60+
}

‎packages/capture-kit/src/recording/contact-sheet-selection.test.ts‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import { describe, expect, test } from 'vitest';
2+
import { paintPng, RED, solidPng } from '../png-pixels.fixtures.ts';
3+
import {
4+
MAX_CONTACT_SHEET_CELLS,
5+
selectContactSheetCells,
6+
type ContactSheetSample,
7+
} from './contact-sheet-selection.ts';
8+
9+
const FRAME_SIZE = 10;
10+
11+
function sample(timeMs: number, changedPixels: number, offset = 0): ContactSheetSample {
12+
return {
13+
timeMs,
14+
image: paintPng(
15+
solidPng(FRAME_SIZE, FRAME_SIZE),
16+
{ x: offset, y: 0, width: changedPixels, height: 1 },
17+
RED,
18+
),
19+
};
20+
}
21+
22+
function distinctSample(index: number): ContactSheetSample {
23+
// Frames of a different shape always count as fully changed, which keeps this fixture honest
24+
// without asking it to invent pixels a video decoder would have had to produce.
25+
return { timeMs: index * 100, image: solidPng(FRAME_SIZE + index, FRAME_SIZE) };
26+
}
27+
28+
describe('selectContactSheetCells', () => {
29+
test('returns nothing for no samples', () => {
30+
expect(selectContactSheetCells([])).toEqual({ cells: [], keptCellCount: 0, thinned: false });
31+
});
32+
33+
test('keeps the only frame it was given', () => {
34+
const selection = selectContactSheetCells([sample(0, 0)]);
35+
36+
expect(selection.cells).toHaveLength(1);
37+
expect(selection.cells[0]).toMatchObject({ timeMs: 0, changedPixelRatio: 1 });
38+
expect(selection.thinned).toBe(false);
39+
});
40+
41+
test('ends with the final frame even when nothing visibly moved', () => {
42+
const selection = selectContactSheetCells([sample(0, 0), sample(250, 0), sample(500, 0)], 0.04);
43+
44+
expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 500]);
45+
expect(selection.cells.at(-1)?.changedPixelRatio).toBe(0);
46+
});
47+
48+
test('measures against the last kept cell so small changes add up', () => {
49+
const selection = selectContactSheetCells(
50+
[
51+
sample(0, 0),
52+
// 1% of the frame: too small on its own, and it does not become the baseline.
53+
sample(250, 1),
54+
// 10% of the frame, but only 9% against the previous sample.
55+
sample(500, FRAME_SIZE),
56+
],
57+
0.1,
58+
);
59+
60+
expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 500]);
61+
expect(selection.cells[1]?.changedPixelRatio).toBeCloseTo(0.1, 10);
62+
});
63+
64+
test('keeps a frame whose shape changed', () => {
65+
const selection = selectContactSheetCells(
66+
[sample(0, 0), { timeMs: 250, image: solidPng(20, FRAME_SIZE) }],
67+
0.5,
68+
);
69+
70+
expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 250]);
71+
expect(selection.cells[1]?.changedPixelRatio).toBe(1);
72+
});
73+
74+
test('thins evenly across the kept sequence and keeps both ends', () => {
75+
const samples = Array.from({ length: 30 }, (_, index) => distinctSample(index));
76+
77+
const selection = selectContactSheetCells(samples, 0.04, 4);
78+
79+
expect(selection.keptCellCount).toBe(30);
80+
expect(selection.thinned).toBe(true);
81+
expect(selection.cells).toHaveLength(4);
82+
expect(selection.cells[0]?.timeMs).toBe(0);
83+
expect(selection.cells.at(-1)?.timeMs).toBe(2_900);
84+
const times = selection.cells.map((cell) => cell.timeMs);
85+
expect(new Set(times).size).toBe(times.length);
86+
});
87+
88+
test('prints every kept cell while the count stays under the sheet cap', () => {
89+
const samples = Array.from({ length: MAX_CONTACT_SHEET_CELLS }, (_, index) =>
90+
distinctSample(index),
91+
);
92+
93+
const selection = selectContactSheetCells(samples);
94+
95+
expect(selection.thinned).toBe(false);
96+
expect(selection.cells).toHaveLength(MAX_CONTACT_SHEET_CELLS);
97+
});
98+
});

0 commit comments

Comments
 (0)