Skip to content

Commit 7b6ff78

Browse files
thymikeethymikee
committed
feat(capture-kit): paint the contact-sheet grid onto one PNG
1 parent 200296b commit 7b6ff78

8 files changed

Lines changed: 461 additions & 69 deletions

‎packages/capture-kit/src/png-pixels.fixtures.ts‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
import { PNG } from './png.ts';
2+
import { setPngPixel, type PngGlyphColor } from './screenshot-overlay-draw.ts';
23

34
/**
45
* Decoded-frame builders for tests that reason about pixels: a solid fill and one painted rectangle
56
* name exactly what a test claims changed between two frames.
67
*/
78

8-
export type Rgba = readonly [number, number, number, number];
9+
export type Rgba = PngGlyphColor;
910
export type Rectangle = Readonly<{ x: number; y: number; width: number; height: number }>;
1011

1112
export const BLACK: Rgba = [0, 0, 0, 255];
@@ -31,12 +32,7 @@ export function paintPng(source: PNG, rectangle: Rectangle, color: Rgba): PNG {
3132
function fillPng(png: PNG, paint: (column: number, row: number) => boolean, color: Rgba): PNG {
3233
for (let row = 0; row < png.height; row += 1) {
3334
for (let column = 0; column < png.width; column += 1) {
34-
if (!paint(column, row)) continue;
35-
const offset = (row * png.width + column) * 4;
36-
png.data[offset] = color[0];
37-
png.data[offset + 1] = color[1];
38-
png.data[offset + 2] = color[2];
39-
png.data[offset + 3] = color[3];
35+
if (paint(column, row)) setPngPixel(png, column, row, color);
4036
}
4137
}
4238
return png;

‎packages/capture-kit/src/png-resize.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export async function resizePngFile(
3939
await fs.writeFile(filePath, await encodePngAsync(resizePngBox(source, width, height)));
4040
}
4141

42-
function resizePngBox(source: PNG, width: number, height: number): PNG {
42+
export function resizePngBox(source: PNG, width: number, height: number): PNG {
4343
const output = new PNG({ width, height });
4444
for (let y = 0; y < height; y += 1) {
4545
const sourceTop = (y * source.height) / height;

‎packages/capture-kit/src/recording/contact-sheet-plan.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { AppError } from '@agent-device/kernel/errors';
55
export const CONTACT_SHEET_SAMPLE_INTERVAL_MS = 250;
66
/** Sample times one sheet asks the decoder for, however long the clip runs. */
77
export const MAX_CONTACT_SHEET_SAMPLED_FRAMES = 48;
8+
/** Width the decoder is asked to return frames at, which is also the widest cell the sheet draws. */
9+
export const CONTACT_SHEET_FRAME_WIDTH = 360;
810

911
/**
1012
* The times to sample from a clip of `durationMs`, spread evenly across the whole timeline.

‎packages/capture-kit/src/recording/contact-sheet-render.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import { describe, expect, test } from 'vitest';
2+
import { AppError } from '@agent-device/kernel/errors';
3+
import { BLACK, solidPng, WHITE } from '../png-pixels.fixtures.ts';
4+
import { decodePng } from '../png.ts';
5+
import { CONTACT_SHEET_FRAME_WIDTH } from './contact-sheet-plan.ts';
6+
import {
7+
MIN_CONTACT_SHEET_CELL_WIDTH,
8+
formatContactSheetTimestamp,
9+
renderContactSheet,
10+
} from './contact-sheet-render.ts';
11+
import type { ContactSheetCell } from './contact-sheet-selection.ts';
12+
13+
function cell(timeMs: number, width = 20, height = 10): ContactSheetCell {
14+
return { timeMs, changedPixelRatio: 1, image: solidPng(width, height, BLACK) };
15+
}
16+
17+
function countPixels(png: ReturnType<typeof decodePng>, color: readonly number[]): number {
18+
let count = 0;
19+
for (let offset = 0; offset < png.data.length; offset += 4) {
20+
if (
21+
png.data[offset] === color[0] &&
22+
png.data[offset + 1] === color[1] &&
23+
png.data[offset + 2] === color[2]
24+
) {
25+
count += 1;
26+
}
27+
}
28+
return count;
29+
}
30+
31+
describe('formatContactSheetTimestamp', () => {
32+
test('prints elapsed time the glyph table can spell', () => {
33+
expect(formatContactSheetTimestamp(0)).toBe('00:00:00.000');
34+
expect(formatContactSheetTimestamp(1_234)).toBe('00:00:01.234');
35+
expect(formatContactSheetTimestamp(3_723_456)).toBe('01:02:03.456');
36+
expect(formatContactSheetTimestamp(3_600_000)).toBe('01:00:00.000');
37+
});
38+
});
39+
40+
describe('renderContactSheet', () => {
41+
test('encodes one grid whose bytes carry the reported size', () => {
42+
const sheet = renderContactSheet({
43+
cells: [cell(0), cell(250)],
44+
maxPixels: 20_000_000,
45+
});
46+
47+
const decoded = decodePng(sheet.bytes, 'contact sheet');
48+
expect([decoded.width, decoded.height]).toEqual([sheet.width, sheet.height]);
49+
// Two cells in one row of four columns: padding, two cells, and the gap between them.
50+
expect(sheet.width).toBe(CONTACT_SHEET_FRAME_WIDTH * 2 + 8 * 3);
51+
});
52+
53+
test('burns a timestamp label above every cell', () => {
54+
const sheet = renderContactSheet({
55+
cells: [cell(0), cell(250), cell(500)],
56+
maxPixels: 20_000_000,
57+
});
58+
59+
const decoded = decodePng(sheet.bytes, 'contact sheet');
60+
expect(countPixels(decoded, WHITE)).toBeGreaterThan(0);
61+
});
62+
63+
test('shrinks cells to stay inside the caller’s pixel budget', () => {
64+
const sheet = renderContactSheet({
65+
cells: Array.from({ length: 12 }, (_, index) => cell(index * 250, 80, 40)),
66+
maxPixels: 250_000,
67+
});
68+
69+
expect(sheet.width * sheet.height).toBeLessThanOrEqual(250_000);
70+
expect(sheet.cellWidth).toBeLessThan(CONTACT_SHEET_FRAME_WIDTH);
71+
expect(sheet.cellWidth).toBeGreaterThanOrEqual(MIN_CONTACT_SHEET_CELL_WIDTH);
72+
});
73+
74+
test('refuses rather than printing a sheet nobody can read', () => {
75+
expect(() =>
76+
renderContactSheet({ cells: [cell(0, 40, 400), cell(250, 40, 400)], maxPixels: 4_000 }),
77+
).toThrow(/maxImagePixels/);
78+
});
79+
80+
test('refuses a sheet with no cells instead of writing an empty page', () => {
81+
try {
82+
renderContactSheet({ cells: [], maxPixels: 20_000_000 });
83+
throw new Error('expected renderContactSheet to refuse');
84+
} catch (error) {
85+
expect(error).toBeInstanceOf(AppError);
86+
}
87+
});
88+
});

‎packages/capture-kit/src/recording/contact-sheet-render.ts‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
import { CONTACT_SHEET_PIXEL_BUDGET_REASON } from '@agent-device/contracts/recording-contact-sheet';
2+
import { AppError } from '@agent-device/kernel/errors';
3+
import { encodePngPixels } from '../png-encode.ts';
4+
import { PNG } from '../png.ts';
5+
import { resizePngBox } from '../png-resize.ts';
6+
import {
7+
drawPngGlyphText,
8+
measurePngGlyphTextHeight,
9+
setPngPixel,
10+
type PngGlyphColor,
11+
} from '../screenshot-overlay-draw.ts';
12+
import { CONTACT_SHEET_FRAME_WIDTH } from './contact-sheet-plan.ts';
13+
import type { ContactSheetCell } from './contact-sheet-selection.ts';
14+
15+
/** Cells per row. Four keeps a portrait phone frame readable without a canvas wider than a screen. */
16+
const CONTACT_SHEET_COLUMNS = 4;
17+
/** Narrowest a cell may shrink to before the sheet refuses rather than print unreadable frames. */
18+
export const MIN_CONTACT_SHEET_CELL_WIDTH = 160;
19+
/**
20+
* Cell widths tried, widest first. A cell never grows past the width frames were decoded at, and it
21+
* shrinks in readable steps before the sheet refuses.
22+
*/
23+
const LAYOUT_CELL_WIDTHS = [
24+
CONTACT_SHEET_FRAME_WIDTH,
25+
320,
26+
280,
27+
240,
28+
200,
29+
MIN_CONTACT_SHEET_CELL_WIDTH,
30+
] as const;
31+
32+
const SHEET_PADDING = 8;
33+
const CELL_GAP = 8;
34+
const LABEL_INSET = 4;
35+
const LABEL_SCALE = 2;
36+
const LABEL_COLOR = [255, 255, 255, 255] as const;
37+
const SHEET_BACKGROUND = [17, 24, 39, 255] as const;
38+
const LABEL_HEIGHT = measurePngGlyphTextHeight(LABEL_SCALE) + LABEL_INSET * 2;
39+
40+
export type ContactSheetRenderInput = Readonly<{
41+
cells: readonly ContactSheetCell[];
42+
maxPixels: number;
43+
}>;
44+
45+
export type ContactSheetRenderResult = Readonly<{
46+
bytes: Buffer;
47+
width: number;
48+
height: number;
49+
cellWidth: number;
50+
}>;
51+
52+
/** Lays the kept cells out in one row-major grid and encodes it as PNG. */
53+
export function renderContactSheet(input: ContactSheetRenderInput): ContactSheetRenderResult {
54+
const { cells } = input;
55+
if (cells.length === 0) {
56+
throw new AppError('COMMAND_FAILED', 'Contact sheet has no cells to render');
57+
}
58+
59+
const aspect = cells[0]!.image.height / Math.max(1, cells[0]!.image.width);
60+
const layout = fitLayout(aspect, cells.length, input);
61+
const canvas = createCanvas(layout.width, layout.height, SHEET_BACKGROUND);
62+
63+
cells.forEach((cell, index) => {
64+
const column = index % layout.columns;
65+
const row = Math.floor(index / layout.columns);
66+
drawCell(
67+
canvas,
68+
SHEET_PADDING + column * (layout.cellWidth + CELL_GAP),
69+
SHEET_PADDING + row * (LABEL_HEIGHT + layout.cellHeight + CELL_GAP),
70+
cell,
71+
layout.cellWidth,
72+
layout.cellHeight,
73+
);
74+
});
75+
76+
return {
77+
bytes: encodePngPixels(canvas.data, layout.width, layout.height, 4),
78+
width: layout.width,
79+
height: layout.height,
80+
cellWidth: layout.cellWidth,
81+
};
82+
}
83+
84+
function drawCell(
85+
canvas: PNG,
86+
x: number,
87+
y: number,
88+
cell: ContactSheetCell,
89+
cellWidth: number,
90+
cellHeight: number,
91+
): void {
92+
drawPngGlyphText(canvas, {
93+
x: x + LABEL_INSET,
94+
y: y + LABEL_INSET,
95+
text: formatContactSheetTimestamp(cell.timeMs),
96+
color: LABEL_COLOR,
97+
scale: LABEL_SCALE,
98+
});
99+
blit(canvas, resizePngBox(toPng(cell.image), cellWidth, cellHeight), x, y + LABEL_HEIGHT);
100+
}
101+
102+
type ContactSheetLayout = Readonly<{
103+
width: number;
104+
height: number;
105+
columns: number;
106+
cellWidth: number;
107+
cellHeight: number;
108+
}>;
109+
110+
/**
111+
* Sizes the grid so it fits the caller's pixel budget.
112+
*
113+
* A cell shrinks before a sheet refuses: an over-budget clip is ordinary (a tall iPad recording
114+
* with many changes), and re-laying the same cells out at a smaller cell is always available.
115+
*/
116+
function fitLayout(
117+
aspect: number,
118+
cellCount: number,
119+
input: ContactSheetRenderInput,
120+
): ContactSheetLayout {
121+
const fitted = fitToBudget(aspect, cellCount, input.maxPixels);
122+
if (fitted) return fitted;
123+
const smallest = layoutAt(aspect, cellCount, MIN_CONTACT_SHEET_CELL_WIDTH);
124+
throw new AppError(
125+
'COMMAND_FAILED',
126+
`Contact sheet would be ${smallest.width * smallest.height} pixels, above the configured maxImagePixels limit of ${input.maxPixels}`,
127+
{
128+
reason: CONTACT_SHEET_PIXEL_BUDGET_REASON,
129+
cellCount: input.cells.length,
130+
maxPixels: input.maxPixels,
131+
hint: 'Shorten the recording or raise the command policy maxImagePixels limit.',
132+
},
133+
);
134+
}
135+
136+
/**
137+
* Walks the cell-width ladder until the grid fits, because the label strip does not shrink with the
138+
* cell and a single proportional guess can land over budget. The ladder is finite and strictly
139+
* descending, so the narrowest readable cell is always the last thing tried.
140+
*/
141+
function fitToBudget(
142+
aspect: number,
143+
cellCount: number,
144+
budget: number,
145+
): ContactSheetLayout | undefined {
146+
for (const cellWidth of LAYOUT_CELL_WIDTHS) {
147+
const layout = layoutAt(aspect, cellCount, cellWidth);
148+
if (layout.width * layout.height <= budget) return layout;
149+
}
150+
return undefined;
151+
}
152+
153+
function layoutAt(aspect: number, cellCount: number, cellWidth: number): ContactSheetLayout {
154+
const columns = Math.min(CONTACT_SHEET_COLUMNS, cellCount);
155+
const rows = Math.ceil(cellCount / columns);
156+
const cellHeight = Math.max(1, Math.round(cellWidth * aspect));
157+
return {
158+
columns,
159+
cellWidth,
160+
cellHeight,
161+
width: SHEET_PADDING * 2 + columns * cellWidth + (columns - 1) * CELL_GAP,
162+
height: SHEET_PADDING * 2 + rows * (LABEL_HEIGHT + cellHeight) + (rows - 1) * CELL_GAP,
163+
};
164+
}
165+
166+
function createCanvas(width: number, height: number, color: PngGlyphColor): PNG {
167+
const canvas = new PNG({ width, height });
168+
for (let row = 0; row < height; row += 1) {
169+
for (let column = 0; column < width; column += 1) {
170+
setPngPixel(canvas, column, row, color);
171+
}
172+
}
173+
return canvas;
174+
}
175+
176+
function blit(canvas: PNG, source: PNG, x: number, y: number): void {
177+
for (let row = 0; row < source.height; row += 1) {
178+
for (let column = 0; column < source.width; column += 1) {
179+
const sourceOffset = (row * source.width + column) * 4;
180+
setPngPixel(canvas, x + column, y + row, [
181+
source.data[sourceOffset]!,
182+
source.data[sourceOffset + 1]!,
183+
source.data[sourceOffset + 2]!,
184+
source.data[sourceOffset + 3]!,
185+
]);
186+
}
187+
}
188+
}
189+
190+
function toPng(image: ContactSheetCell['image']): PNG {
191+
const png = new PNG({ width: image.width, height: image.height });
192+
Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).copy(png.data);
193+
return png;
194+
}
195+
196+
/** Elapsed time from the clip start as `HH:MM:SS.mmm`, the shape the glyph table spells. */
197+
export function formatContactSheetTimestamp(timeMs: number): string {
198+
const wholeSeconds = Math.floor(timeMs / 1000);
199+
const hours = Math.floor(wholeSeconds / 3600);
200+
const minutes = Math.floor((wholeSeconds % 3600) / 60);
201+
const seconds = wholeSeconds % 60;
202+
const milliseconds = Math.floor(timeMs % 1000);
203+
const clock = [hours, minutes, seconds].map((part) => String(part).padStart(2, '0')).join(':');
204+
return `${clock}.${String(milliseconds).padStart(3, '0')}`;
205+
}

0 commit comments

Comments
 (0)