Skip to content

Commit fc006f7

Browse files
fix(studio-server): a render that ran over 5 minutes can still be read after it finishes (heygen-com#7087)
* fix(studio-server): a render that ran over 5 minutes can still be read after it finishes * fix(studio-server): keep the finished-jobs helper's reasoning in its names
1 parent 4ceec01 commit fc006f7

5 files changed

Lines changed: 150 additions & 25 deletions

File tree

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// The adapter finishes a job in place, so its finish time is when a sweep first sees it finished.
2+
const firstSeenFinishedAt = new WeakMap<object, number>();
3+
4+
export function forgetJobsFinishedLongerThan<T extends object>(
5+
ttlMs: number,
6+
jobs: Map<string, T>,
7+
isFinished: (job: T) => boolean,
8+
now = Date.now(),
9+
): void {
10+
for (const [id, job] of jobs) {
11+
if (!isFinished(job)) continue;
12+
const finishedAt = firstSeenFinishedAt.get(job) ?? now;
13+
firstSeenFinishedAt.set(job, finishedAt);
14+
if (now - finishedAt > ttlMs) jobs.delete(id);
15+
}
16+
}
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import { afterEach, describe, expect, it, vi } from "vitest";
2+
import { Hono } from "hono";
3+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
4+
import { tmpdir } from "node:os";
5+
import { join } from "node:path";
6+
import { registerMediaRoutes } from "./media";
7+
import { registerRenderRoutes } from "./render";
8+
import type { MediaProcessingJobState, RenderJobState, StudioApiAdapter } from "../types";
9+
10+
const MINUTE = 60_000;
11+
const dirs: string[] = [];
12+
13+
afterEach(() => {
14+
vi.useRealTimers();
15+
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
16+
});
17+
18+
function adapterFor(project: string, extra: Partial<StudioApiAdapter>): StudioApiAdapter {
19+
return {
20+
listProjects: () => [],
21+
resolveProject: async (id: string) => ({ id, dir: project }),
22+
bundle: async () => null,
23+
lint: async () => ({ findings: [] }),
24+
runtimeUrl: "/api/runtime.js",
25+
rendersDir: () => join(project, "renders"),
26+
startRender: (opts) => ({
27+
id: opts.jobId,
28+
status: "rendering",
29+
progress: 0,
30+
outputPath: opts.outputPath,
31+
}),
32+
...extra,
33+
};
34+
}
35+
36+
/** The progress route's status for a job: 200 while the server still has it, 404 once it is forgotten. */
37+
async function progressStatus(app: Hono, path: string): Promise<number> {
38+
const response = await app.request(`http://localhost${path}`);
39+
await response.body?.cancel();
40+
return response.status;
41+
}
42+
43+
const project = () => {
44+
const dir = mkdtempSync(join(tmpdir(), "hf-finished-jobs-"));
45+
dirs.push(dir);
46+
return dir;
47+
};
48+
49+
describe("a finished job stays readable after it finishes, however long it ran", () => {
50+
it("keeps a render that ran 6 minutes for the sweep after it finished, then forgets it", async () => {
51+
vi.useFakeTimers();
52+
const app = new Hono();
53+
let job: RenderJobState | undefined;
54+
registerRenderRoutes(
55+
app,
56+
adapterFor(project(), {
57+
startRender: (opts) =>
58+
(job = { id: opts.jobId, status: "rendering", progress: 0, outputPath: opts.outputPath }),
59+
}),
60+
);
61+
const started = await app.request("http://localhost/projects/demo/render", {
62+
method: "POST",
63+
headers: { "content-type": "application/json" },
64+
body: JSON.stringify({ fps: 30, quality: "standard", format: "mp4" }),
65+
});
66+
const { jobId } = (await started.json()) as { jobId: string };
67+
const progress = `/render/${jobId}/progress`;
68+
69+
await vi.advanceTimersByTimeAsync(6 * MINUTE);
70+
Object.assign(job!, { status: "complete", progress: 100 });
71+
await vi.advanceTimersByTimeAsync(MINUTE + 1_000);
72+
expect(await progressStatus(app, progress)).toBe(200);
73+
74+
await vi.advanceTimersByTimeAsync(6 * MINUTE);
75+
expect(await progressStatus(app, progress)).toBe(404);
76+
});
77+
78+
it("keeps a background removal that ran 6 minutes for the TTL after it finished", async () => {
79+
vi.useFakeTimers({ toFake: ["Date"] });
80+
const dir = project();
81+
mkdirSync(join(dir, "assets"), { recursive: true });
82+
writeFileSync(join(dir, "assets", "clip.mp4"), "video");
83+
let job: MediaProcessingJobState | undefined;
84+
const app = new Hono();
85+
registerMediaRoutes(
86+
app,
87+
adapterFor(dir, {
88+
startBackgroundRemoval: (opts) =>
89+
(job = {
90+
id: opts.jobId,
91+
status: "processing",
92+
progress: 0,
93+
inputAssetPath: opts.inputAssetPath,
94+
outputAssetPath: opts.outputAssetPath,
95+
outputPath: opts.outputPath,
96+
}),
97+
}),
98+
);
99+
const started = await app.request("http://localhost/projects/demo/media/remove-background", {
100+
method: "POST",
101+
headers: { "content-type": "application/json" },
102+
body: JSON.stringify({ inputPath: "assets/clip.mp4" }),
103+
});
104+
const { jobId } = (await started.json()) as { jobId: string };
105+
const progress = `/media-jobs/${jobId}/progress`;
106+
107+
vi.setSystemTime(Date.now() + 6 * MINUTE);
108+
Object.assign(job!, { status: "complete", progress: 100 });
109+
expect(await progressStatus(app, progress)).toBe(200);
110+
vi.setSystemTime(Date.now() + MINUTE);
111+
expect(await progressStatus(app, progress)).toBe(200);
112+
113+
vi.setSystemTime(Date.now() + 5 * MINUTE);
114+
expect(await progressStatus(app, progress)).toBe(404);
115+
});
116+
});

‎packages/studio-server/src/routes/media.ts‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
66
import { requestedProjectPath } from "../helpers/requestSubPath.js";
77
import { mkdirWithinProject, pinWithinProject, resolveWithinProject } from "../helpers/safePath.js";
88
import { probeMediaMetadata } from "../helpers/mediaMetadata.js";
9+
import { forgetJobsFinishedLongerThan } from "../helpers/finishedJobs.js";
910

1011
const VIDEO_EXTENSIONS = new Set([
1112
".mp4",
@@ -35,7 +36,6 @@ interface BackgroundRemovalBody {
3536
device?: string;
3637
}
3738

38-
type JobWithCreatedAt = MediaProcessingJobState & { createdAt: number };
3939
type ProbeMediaMetadata = typeof probeMediaMetadata;
4040

4141
function isVideoPath(path: string): boolean {
@@ -92,14 +92,14 @@ function defaultPlatePath(
9292
return uniqueAssetPath(projectDir, `assets/cutouts/${slugFileBase(inputPath)}-plate.webm`, taken);
9393
}
9494

95-
function outputsInFlight(mediaJobs: Map<string, JobWithCreatedAt>): Set<string> {
95+
function outputsInFlight(mediaJobs: Map<string, MediaProcessingJobState>): Set<string> {
9696
const running = [...mediaJobs.values()].filter((job) => job.status === "processing");
9797
return new Set(
9898
running.flatMap((job) => [job.outputAssetPath, job.backgroundOutputAssetPath ?? ""]),
9999
);
100100
}
101101

102-
function makeJobId(projectId: string, mediaJobs: Map<string, JobWithCreatedAt>): string {
102+
function makeJobId(projectId: string, mediaJobs: Map<string, MediaProcessingJobState>): string {
103103
const stamp = new Date()
104104
.toISOString()
105105
.replace(/[-:.TZ]/g, "")
@@ -126,17 +126,16 @@ export function registerMediaRoutes(
126126
adapter: StudioApiAdapter,
127127
options: { probeMediaMetadata?: ProbeMediaMetadata } = {},
128128
): void {
129-
const mediaJobs = new Map<string, JobWithCreatedAt>();
129+
const mediaJobs = new Map<string, MediaProcessingJobState>();
130130
const TTL_MS = 300_000;
131131
const readMediaMetadata = options.probeMediaMetadata ?? probeMediaMetadata;
132132

133133
function cleanupFinishedJobs(): void {
134-
const now = Date.now();
135-
for (const [id, job] of mediaJobs) {
136-
if ((job.status === "complete" || job.status === "failed") && now - job.createdAt > TTL_MS) {
137-
mediaJobs.delete(id);
138-
}
139-
}
134+
forgetJobsFinishedLongerThan(
135+
TTL_MS,
136+
mediaJobs,
137+
(job) => job.status === "complete" || job.status === "failed",
138+
);
140139
}
141140

142141
api.get("/projects/:id/media/metadata", async (c) => {
@@ -242,8 +241,7 @@ export function registerMediaRoutes(
242241
quality: normalizeQuality(body.quality),
243242
device: normalizeDevice(body.device),
244243
jobId,
245-
}) as JobWithCreatedAt;
246-
state.createdAt = Date.now();
244+
});
247245
mediaJobs.set(jobId, state);
248246

249247
return c.json({

‎packages/studio-server/src/routes/render.ts‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {
2323
import { projectDirMissing } from "../helpers/projectDirMissing.js";
2424
import { isVariablesPayload, VARIABLES_PAYLOAD_ERROR } from "../helpers/variablesPayload.js";
2525
import { parseEncoderOptions, type EncoderOptionsBody } from "../helpers/encoderOptions.js";
26+
import { forgetJobsFinishedLongerThan } from "../helpers/finishedJobs.js";
2627

2728
const VALID_RESOLUTIONS = new Set<string>(VALID_CANVAS_RESOLUTIONS);
2829
const RENDER_MIME: Record<string, string> = {
@@ -56,7 +57,7 @@ function contentDispositionHeader(disposition: "inline" | "attachment", filename
5657

5758
export function registerRenderRoutes(api: Hono, adapter: StudioApiAdapter): void {
5859
// Scoped job store — not shared across createStudioApi() calls
59-
const renderJobs = new Map<string, RenderJobState & { createdAt: number }>();
60+
const renderJobs = new Map<string, RenderJobState>();
6061

6162
const nextFreeRenderJobId = (baseId: string, ext: string, rendersDir: string): string => {
6263
let jobId = baseId;
@@ -66,7 +67,7 @@ export function registerRenderRoutes(api: Hono, adapter: StudioApiAdapter): void
6667
return jobId;
6768
};
6869

69-
// TTL cleanup for completed jobs (5 minutes)
70+
// A finished job stays readable for 5 minutes after it finishes
7071
const TTL_MS = 300_000;
7172
const CLEANUP_INTERVAL_MS = 60_000;
7273
let cleanupTimer: ReturnType<typeof setInterval> | null = null;
@@ -77,12 +78,7 @@ export function registerRenderRoutes(api: Hono, adapter: StudioApiAdapter): void
7778
!process.argv.includes("build");
7879

7980
const cleanupFinishedJobs = () => {
80-
const now = Date.now();
81-
for (const [key, job] of renderJobs) {
82-
if (job.status !== "rendering" && now - job.createdAt > TTL_MS) {
83-
renderJobs.delete(key);
84-
}
85-
}
81+
forgetJobsFinishedLongerThan(TTL_MS, renderJobs, (job) => job.status !== "rendering");
8682
if (renderJobs.size === 0 && cleanupTimer) {
8783
clearInterval(cleanupTimer);
8884
cleanupTimer = null;
@@ -213,8 +209,7 @@ export function registerRenderRoutes(api: Hono, adapter: StudioApiAdapter): void
213209
telemetryOptOut: body.telemetryOptOut === true,
214210
...(pageMotionBlur === undefined ? {} : { pageMotionBlur }),
215211
});
216-
(jobState as RenderJobState & { createdAt: number }).createdAt = Date.now();
217-
renderJobs.set(jobId, jobState as RenderJobState & { createdAt: number });
212+
renderJobs.set(jobId, jobState);
218213

219214
ensureCleanupTimer();
220215

@@ -410,8 +405,7 @@ export function registerRenderRoutes(api: Hono, adapter: StudioApiAdapter): void
410405
status: file.status,
411406
progress: 100,
412407
outputPath: file.path,
413-
createdAt: file.createdAt,
414-
} as RenderJobState & { createdAt: number });
408+
});
415409
}
416410
}
417411
return c.json({ renders: files.map(({ path: _path, ...render }) => render) });

‎packages/studio-server/src/routes/renderDeletion.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ describe("DELETE /render/:jobId", () => {
9595
}
9696
writeFileSync(join(rendersDir, "keep.mp4"), "other-render");
9797

98-
vi.advanceTimersByTime(360_000);
98+
// Seen finished by the first sweep (1 min), forgotten 5 min after that.
99+
vi.advanceTimersByTime(7 * 60_000);
99100
expect((await app.request(`/render/${jobId}/progress`)).status).toBe(404);
100101
expect(existsSync(join(rendersDir, `${jobId}.mp4`))).toBe(true);
101102

0 commit comments

Comments
 (0)