Skip to content

Commit 599eaad

Browse files
vanceingallsclaude
andauthored
feat(producer): a capture segment extracts the frames it shows and deletes them (#7241)
* feat(producer): a capture segment extracts the frames it shows and deletes them when it closes Co-Authored-By: Claude Opus 5.5 <[email protected]> * fix(producer): the windowed video hook reads its own frames outside a prepared window Co-Authored-By: Claude Opus 5.5 <[email protected]> * refactor(producer): trim the comments the segment video frames hook added Co-Authored-By: Claude Opus 5.5 <[email protected]> --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 35cbca9 commit 599eaad

5 files changed

Lines changed: 894 additions & 13 deletions

File tree

‎packages/producer/src/services/render/segmentManifest.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,33 @@ describe("validateCompletedSegments", () => {
190190
expect((await validateCompletedSegments(manifest, "h", async () => 3)).size).toBe(0);
191191
});
192192

193+
it("a resume whose segment files were removed after a join captures them again rather than failing", async () => {
194+
dir = mkdtempSync(join(tmpdir(), "hf-seg-"));
195+
const joined = [0, 1].map((index) => join(dir, `segment_0000${index}.mp4`));
196+
const manifest: SegmentManifest = {
197+
version: 1,
198+
planHash: "h",
199+
totalFrames: 6,
200+
segmentFrames: 3,
201+
completed: joined.map((path, index) => ({
202+
index,
203+
startFrame: index * 3,
204+
endFrame: index * 3 + 3,
205+
path,
206+
bytes: 2,
207+
completedAt: "",
208+
})),
209+
};
210+
// The manifest still lists them, but the join deleted the files.
211+
let probed = 0;
212+
const probe = async () => {
213+
probed += 1;
214+
return 3;
215+
};
216+
expect((await validateCompletedSegments(manifest, "h", probe)).size).toBe(0);
217+
expect(probed).toBe(0);
218+
});
219+
193220
it("skips nothing when the plan hash differs", async () => {
194221
// The entry is otherwise perfectly reusable — present, right size, right
195222
// frame count — so only the hash check can reject it. With an empty
Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
import { mkdirSync, mkdtempSync, existsSync, readdirSync, rmSync, writeFileSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { afterAll, describe, expect, it } from "bun:test";
5+
import type { EngineConfig, ExtractedFrames, VideoElement } from "@hyperframes/engine";
6+
import { type extractDeferredFrames, shownDeferredFrameIndexes } from "./deferredVideoFrames.js";
7+
import { createSegmentVideoFrames } from "./segmentVideoFrames.js";
8+
9+
const root = mkdtempSync(join(tmpdir(), "hf-segment-video-frames-"));
10+
afterAll(() => rmSync(root, { recursive: true, force: true }));
11+
12+
const metadata = {
13+
durationSeconds: 4,
14+
videoStreamDurationSeconds: 4,
15+
videoStreamStartSeconds: 0,
16+
width: 64,
17+
height: 36,
18+
fps: 30,
19+
videoCodec: "h264",
20+
hasAudio: false,
21+
isVFR: false,
22+
hasAlpha: false,
23+
colorSpace: null,
24+
};
25+
26+
const base = (videoId: string, totalFrames: number): ExtractedFrames => ({
27+
videoId,
28+
srcPath: `/source/${videoId}.mp4`,
29+
outputDir: "",
30+
framePattern: "frame_%05d.jpg",
31+
fps: 30,
32+
totalFrames,
33+
metadata,
34+
framePaths: new Map(),
35+
});
36+
37+
const deferredVideo = (videoId: string): ExtractedFrames => ({
38+
...base(videoId, 75),
39+
deferredRange: { startTime: 1.5, durationSeconds: 2.5, format: "jpg" },
40+
});
41+
42+
const wholeVideo = (videoId: string): ExtractedFrames => ({
43+
...base(videoId, 75),
44+
outputDir: "/full-extraction/whole",
45+
framePaths: new Map(
46+
Array.from({ length: 75 }, (_, i) => [i, `/full-extraction/whole/frame_${i + 1}.jpg`]),
47+
),
48+
});
49+
50+
const element = (id: string, start: number, end: number): VideoElement => ({
51+
id,
52+
src: `${id}.mp4`,
53+
start,
54+
end,
55+
mediaStart: 1.5,
56+
loop: false,
57+
hasAudio: false,
58+
});
59+
60+
const fps = { num: 30, den: 1 };
61+
const cfg = { ffmpegProcessTimeout: 1000 } as EngineConfig;
62+
const slice = (index: number, startFrame: number, endFrame: number) => ({
63+
index,
64+
startFrame,
65+
endFrame,
66+
});
67+
const span = (start: number, end: number) =>
68+
Array.from({ length: end - start }, (_, i) => start + i);
69+
70+
/** Stands in for the extractor: writes the frames the slice shows, as the real one names them. */
71+
function fakeExtractor() {
72+
const calls: Array<{ outputRoot: string; frames: number[]; sourceOf: string[] }> = [];
73+
const extractFrames = async (input: Parameters<typeof extractDeferredFrames>[0]) => {
74+
const frames = [...input.frames];
75+
const shown = shownDeferredFrameIndexes(input, frames, input.fps);
76+
calls.push({
77+
outputRoot: input.outputRoot,
78+
frames,
79+
sourceOf: input.extracted.filter((v) => v.deferredRange).map((v) => input.sourceOf(v)),
80+
});
81+
return input.extracted
82+
.filter((video) => video.deferredRange)
83+
.map((video) => {
84+
const outputDir = join(input.outputRoot, video.videoId);
85+
const framePaths = new Map<number, string>();
86+
for (const index of shown.get(video.videoId) ?? []) {
87+
mkdirSync(outputDir, { recursive: true });
88+
const path = join(outputDir, `frame_${String(index + 1).padStart(5, "0")}.jpg`);
89+
writeFileSync(path, "x");
90+
framePaths.set(index, path);
91+
}
92+
return { ...video, outputDir, framePaths, ownedByLookup: false };
93+
});
94+
};
95+
return { calls, extractFrames };
96+
}
97+
98+
function setup(extra: Partial<Parameters<typeof createSegmentVideoFrames>[0]> = {}) {
99+
const served: string[] = [];
100+
const dir = mkdtempSync(join(root, "work-"));
101+
const extractor = fakeExtractor();
102+
const segmentVideoFrames = createSegmentVideoFrames({
103+
videos: [element("clip", 0, 4), element("whole", 0, 4)],
104+
extracted: [deferredVideo("clip"), wholeVideo("whole")],
105+
fps,
106+
rootDir: dir,
107+
cfg,
108+
injectorOptions: {
109+
frameSrcResolver: (path) => {
110+
served.push(path);
111+
return `http://frames/${path}`;
112+
},
113+
},
114+
extractFrames: extractor.extractFrames,
115+
...extra,
116+
});
117+
const page = { evaluate: async () => [] } as never;
118+
return {
119+
dir,
120+
served,
121+
extractor,
122+
segmentVideoFrames,
123+
workerFrames: segmentVideoFrames.forWorker(),
124+
page,
125+
};
126+
}
127+
128+
describe("createSegmentVideoFrames", () => {
129+
it("extracts only the frames a slice shows, under that slice's own folder", async () => {
130+
const { dir, served, extractor, workerFrames, page } = setup();
131+
await workerFrames.prepare(slice(1, 30, 45));
132+
133+
const folder = join(dir, "segment-frames", "segment_00001");
134+
expect(extractor.calls).toHaveLength(1);
135+
expect(extractor.calls[0]!.outputRoot).toBe(folder);
136+
expect(extractor.calls[0]!.frames).toEqual(span(30, 45));
137+
expect(readdirSync(join(folder, "clip")).sort()).toEqual(
138+
span(30, 45).map((i) => `frame_${String(i + 1).padStart(5, "0")}.jpg`),
139+
);
140+
141+
await workerFrames.hook(page, 31 / 30);
142+
expect(served).toContain(join(folder, "clip", "frame_00032.jpg"));
143+
});
144+
145+
it("release removes the folder and a second release does nothing", async () => {
146+
const { dir, workerFrames } = setup();
147+
await workerFrames.prepare(slice(2, 60, 75));
148+
const folder = join(dir, "segment-frames", "segment_00002");
149+
expect(existsSync(folder)).toBe(true);
150+
151+
workerFrames.release();
152+
expect(existsSync(folder)).toBe(false);
153+
expect(() => workerFrames.release()).not.toThrow();
154+
});
155+
156+
it("a video that was fully extracted is served from its existing frames and never extracted again", async () => {
157+
const { served, extractor, workerFrames, page } = setup();
158+
await workerFrames.prepare(slice(0, 0, 15));
159+
160+
await workerFrames.hook(page, 3 / 30);
161+
expect(served).toContain("/full-extraction/whole/frame_4.jpg");
162+
// The extractor is handed the whole list and picks out the deferred ones; the full one has no source read.
163+
expect(extractor.calls[0]!.sourceOf).toEqual(["/source/clip.mp4"]);
164+
expect(existsSync(join(extractor.calls[0]!.outputRoot, "whole"))).toBe(false);
165+
});
166+
167+
it("a slice showing no deferred video extracts nothing", async () => {
168+
const dir = mkdtempSync(join(root, "work-"));
169+
const segmentVideoFrames = createSegmentVideoFrames({
170+
videos: [element("later", 10, 12)],
171+
extracted: [deferredVideo("later")],
172+
fps,
173+
rootDir: dir,
174+
cfg,
175+
injectorOptions: {},
176+
sourceOf: () => {
177+
throw new Error("a slice that shows no video must not read a source");
178+
},
179+
});
180+
const workerFrames = segmentVideoFrames.forWorker();
181+
182+
await workerFrames.prepare(slice(0, 0, 15));
183+
184+
const folder = join(dir, "segment-frames", "segment_00000");
185+
expect(readdirSync(folder).flatMap((entry) => readdirSync(join(folder, entry)))).toEqual([]);
186+
});
187+
188+
it("a prepare that fails mid-extraction leaves no folder behind", async () => {
189+
const { dir, workerFrames } = setup({
190+
extractFrames: async (input) => {
191+
mkdirSync(join(input.outputRoot, "clip"), { recursive: true });
192+
writeFileSync(join(input.outputRoot, "clip", "frame_00001.jpg"), "x");
193+
throw new Error("aborted");
194+
},
195+
});
196+
197+
await expect(workerFrames.prepare(slice(3, 90, 105))).rejects.toThrow("aborted");
198+
199+
expect(existsSync(join(dir, "segment-frames", "segment_00003"))).toBe(false);
200+
});
201+
describe("outside a prepared window", () => {
202+
const onDemandFolders = (dir: string) => {
203+
const root = join(dir, "segment-frames");
204+
return existsSync(root) ? readdirSync(root).filter((n) => n.startsWith("on-demand_")) : [];
205+
};
206+
207+
/** Records, at injection time, whether the folder the served frame lives in still existed. */
208+
function setupWatching() {
209+
const seen: Array<{ path: string; existed: boolean }> = [];
210+
const made = setup({
211+
injectorOptions: {
212+
frameSrcResolver: (path) => {
213+
seen.push({ path, existed: existsSync(path) });
214+
return `http://frames/${path}`;
215+
},
216+
},
217+
});
218+
return { ...made, seen };
219+
}
220+
221+
it("before any prepare, the hook extracts and injects the frame asked for, then removes its folder", async () => {
222+
const { dir, seen, extractor, workerFrames, page } = setupWatching();
223+
224+
await workerFrames.hook(page, 31 / 30);
225+
226+
expect(extractor.calls).toHaveLength(1);
227+
expect(extractor.calls[0]!.frames).toEqual([31]);
228+
expect(
229+
extractor.calls[0]!.outputRoot.startsWith(join(dir, "segment-frames", "on-demand_")),
230+
).toBe(true);
231+
const clip = seen.find((s) => s.path.includes("/clip/"));
232+
expect(clip).toEqual({ path: expect.stringContaining("frame_00032.jpg"), existed: true });
233+
// The fully extracted video is injected too, from its own frames.
234+
expect(seen.some((s) => s.path === "/full-extraction/whole/frame_32.jpg")).toBe(true);
235+
expect(onDemandFolders(dir)).toEqual([]);
236+
});
237+
238+
it("asks for the held frame too when the capture holds footage for a motion-blur sample", async () => {
239+
const { extractor, workerFrames, page } = setupWatching();
240+
241+
await workerFrames.hook(page, 32 / 30, 30 / 30);
242+
243+
expect(extractor.calls[0]!.frames).toEqual([32, 30]);
244+
});
245+
246+
it("after release, the hook extracts on demand again", async () => {
247+
const { dir, seen, extractor, workerFrames, page } = setupWatching();
248+
await workerFrames.prepare(slice(1, 30, 45));
249+
await workerFrames.hook(page, 31 / 30);
250+
expect(extractor.calls).toHaveLength(1);
251+
252+
workerFrames.release();
253+
await workerFrames.hook(page, 61 / 30);
254+
255+
expect(extractor.calls).toHaveLength(2);
256+
expect(extractor.calls[1]!.frames).toEqual([61]);
257+
expect(
258+
seen.some((s) => s.path.includes("on-demand_") && s.path.includes("frame_00062.jpg")),
259+
).toBe(true);
260+
expect(onDemandFolders(dir)).toEqual([]);
261+
});
262+
263+
it("fresh returns an independent hook that also extracts on demand", async () => {
264+
const { dir, seen, extractor, workerFrames, page } = setupWatching();
265+
await workerFrames.prepare(slice(1, 30, 45));
266+
267+
const fresh = workerFrames.hook.fresh!();
268+
expect(fresh).not.toBe(workerFrames.hook);
269+
// The worker's window is prepared, yet the fresh hook still reads its own frames.
270+
await fresh(page, 40 / 30);
271+
272+
expect(extractor.calls).toHaveLength(2);
273+
expect(extractor.calls[1]!.frames).toEqual([40]);
274+
expect(
275+
seen.some((s) => s.path.includes("on-demand_") && s.path.includes("frame_00041.jpg")),
276+
).toBe(true);
277+
expect(onDemandFolders(dir)).toEqual([]);
278+
expect(typeof fresh.fresh).toBe("function");
279+
});
280+
281+
it("an on-demand extraction that fails removes its folder and rethrows", async () => {
282+
const { dir, workerFrames, page } = setup({
283+
extractFrames: async (input) => {
284+
mkdirSync(join(input.outputRoot, "clip"), { recursive: true });
285+
writeFileSync(join(input.outputRoot, "clip", "frame_00001.jpg"), "x");
286+
throw new Error("aborted");
287+
},
288+
});
289+
290+
await expect(workerFrames.hook(page, 31 / 30)).rejects.toThrow("aborted");
291+
292+
expect(onDemandFolders(dir)).toEqual([]);
293+
});
294+
295+
it("gives two workers different folders", async () => {
296+
const { extractor, segmentVideoFrames, workerFrames, page } = setup();
297+
const other = segmentVideoFrames.forWorker();
298+
299+
await Promise.all([workerFrames.hook(page, 31 / 30), other.hook(page, 31 / 30)]);
300+
301+
expect(new Set(extractor.calls.map((call) => call.outputRoot)).size).toBe(2);
302+
});
303+
});
304+
});

0 commit comments

Comments
 (0)