Skip to content

Commit 61e720b

Browse files
fix(studio-server): history sweeps stop holding the host's event loop on large projects (heygen-com#7491)
* fix(studio-server): project history lists files without holding the host's event loop * test(studio-server): the history walk test counts the hold up to the sweep's end * fix(studio-server): history rewrites video-state.json only when it changed * refactor(studio-server): names carry what the history walk comments said
1 parent c6d243a commit 61e720b

6 files changed

Lines changed: 151 additions & 26 deletions

File tree

‎packages/studio-server/src/helpers/projectSignature.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import {
1717
affectsProjectSignature,
1818
createProjectSignature,
1919
createProjectSignatureAsync,
20-
listProjectFiles,
20+
listProjectFilesAsync,
2121
} from "./projectSignature.js";
2222

2323
const temporaryProjects: string[] = [];
@@ -72,15 +72,15 @@ describe("affectsProjectSignature", () => {
7272
});
7373
});
7474

75-
describe("listProjectFiles", () => {
76-
it("leaves out the temp file of a save in flight", () => {
75+
describe("listProjectFilesAsync", () => {
76+
it("leaves out the temp file of a save in flight", async () => {
7777
const project = mkdtempSync(resolve(tmpdir(), "hf-signature-"));
7878
temporaryProjects.push(project);
7979
writeFileSync(resolve(project, "index.html"), "<h1>Hello</h1>");
8080
writeFileSync(resolve(project, "index.html.hf0a1b2c.tmp"), "<h1>Bye</h1>");
8181
writeFileSync(resolve(project, "foo.12345678.tmp"), "mine");
8282

83-
expect(listProjectFiles(project).map((file) => file.path)).toEqual([
83+
expect((await listProjectFilesAsync(project)).map((file) => file.path)).toEqual([
8484
"foo.12345678.tmp",
8585
"index.html",
8686
]);

‎packages/studio-server/src/helpers/projectSignature.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -242,13 +242,16 @@ async function collectProjectFilesAsync(
242242
return collected;
243243
}
244244

245+
export type ListedProjectFile = { path: string; size: number; mtimeMs: number; ctimeMs: number };
246+
245247
/** The files a project is made of (the signature's set: source plus Studio's two manifests), paths with `/`. */
246-
export function listProjectFiles(
247-
projectDir: string,
248-
): Array<{ path: string; size: number; mtimeMs: number; ctimeMs: number }> {
248+
export async function listProjectFilesAsync(projectDir: string): Promise<ListedProjectFile[]> {
249249
const normalizedProjectDir = resolve(projectDir);
250-
return collectProjectFiles(normalizedProjectDir).map((entry) => ({
251-
path: relative(normalizedProjectDir, entry.file).split(sep).join("/"),
250+
const everyFileStartsWith = normalizedProjectDir.endsWith(sep)
251+
? normalizedProjectDir
252+
: normalizedProjectDir + sep;
253+
return (await collectProjectFilesAsync(normalizedProjectDir)).map((entry) => ({
254+
path: entry.file.slice(everyFileStartsWith.length).split(sep).join("/"),
252255
size: entry.size,
253256
mtimeMs: entry.mtimeMs,
254257
ctimeMs: entry.ctimeMs,

‎packages/studio-server/src/helpers/renderOutputs.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
33
import { join } from "node:path";
44
import { expect, it } from "vitest";
55
import { registerRenderOutput, readRenderOutputs } from "./renderOutputs.js";
6-
import { listProjectFiles, affectsProjectSignature } from "./projectSignature.js";
6+
import { listProjectFilesAsync, affectsProjectSignature } from "./projectSignature.js";
77
import { openProjectHistory } from "../history/projectHistory.js";
88
import { beginArtifactCapture, recordRenderSuccess } from "../history/videoState.js";
99

@@ -66,7 +66,7 @@ it("registers concurrent custom outputs and directory renders without excluding
6666
writeFileSync(join(root, "hls", "chunk.ts"), "output");
6767
writeFileSync(join(root, "a.webm"), "output");
6868
writeFileSync(join(root, "b.webm"), "input");
69-
expect(listProjectFiles(root).map((file) => file.path)).toEqual(["b.webm"]);
69+
expect((await listProjectFilesAsync(root)).map((file) => file.path)).toEqual(["b.webm"]);
7070
await registerRenderOutput(root, join(root, "..", "external.mp4"));
7171
expect(readRenderOutputs(root)).toEqual(["a.webm", "hls"]);
7272
await expect(registerRenderOutput(root, root)).rejects.toThrow("project directory");
@@ -75,7 +75,7 @@ it("registers concurrent custom outputs and directory renders without excluding
7575
}
7676
});
7777

78-
it("lists project files when the render output registry is unreadable", () => {
78+
it("lists project files when the render output registry is unreadable", async () => {
7979
const project = mkdtempSync(join(tmpdir(), "hf-render-registry-"));
8080
try {
8181
writeFileSync(join(project, "index.html"), "x");
@@ -85,7 +85,7 @@ it("lists project files when the render output registry is unreadable", () => {
8585
mkdirSync(join(project, ".hyperframes"));
8686
writeFileSync(join(project, ".hyperframes", "render-outputs.json"), "{not json");
8787
expect(() => readRenderOutputs(project)).toThrow();
88-
expect(listProjectFiles(project).length).toBeGreaterThan(0);
88+
expect((await listProjectFilesAsync(project)).length).toBeGreaterThan(0);
8989
expect(affectsProjectSignature(project, join(project, "index.html"))).toBe(true);
9090
} finally {
9191
rmSync(project, { recursive: true, force: true });

‎packages/studio-server/src/history/projectHistory.ts‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ import { realFilePath } from "../helpers/safePath.js";
2626
import {
2727
STUDIO_SIGNATURE_MANIFEST_PATHS,
2828
affectsProjectSignature,
29-
listProjectFiles,
29+
listProjectFilesAsync,
30+
type ListedProjectFile,
3031
} from "../helpers/projectSignature.js";
3132
import { isSettling, openBlobStore, readBlob, SETTLE_MS, type BlobStore } from "./blobStore.js";
3233
import { isUncopied, uncopiedOnlyHashes } from "./uncopied.js";
@@ -263,9 +264,9 @@ function withoutHiddenPaths(log: HistoryLog): HistoryLog {
263264
return log;
264265
}
265266

266-
const historyFiles = (dir: string) =>
267-
listProjectFiles(dir).filter((file) => isHistoryPath(file.path));
268-
type ListedFile = ReturnType<typeof listProjectFiles>[number];
267+
const historyFiles = async (dir: string) =>
268+
(await listProjectFilesAsync(dir)).filter((file) => isHistoryPath(file.path));
269+
type ListedFile = ListedProjectFile;
269270
const COPY_LATER_ABOVE_BYTES = 1024 ** 2;
270271

271272
const sameWho = (a: HistoryWho, b: HistoryWho) => a.kind === b.kind && a.name === b.name;
@@ -366,13 +367,29 @@ function keepTargetsOfItsFiles(entry: HistoryEntry): HistoryEntry {
366367
return entry;
367368
}
368369

370+
/** The tracked files; any change owes the project's video-state.json a rewrite. */
371+
class TrackedFiles extends Map<string, Tracked> {
372+
constructor(private readonly changed: () => void) {
373+
super();
374+
}
375+
override set(path: string, file: Tracked): this {
376+
this.changed();
377+
return super.set(path, file);
378+
}
379+
override delete(path: string): boolean {
380+
this.changed();
381+
return super.delete(path);
382+
}
383+
}
384+
369385
class Engine {
370386
readonly dir: string;
371387
readonly foldsCase: boolean;
372388
readonly folder: FolderIdentity & { dev: number };
373389
readonly home: string;
374390
log: HistoryLog = { baseline: new Map(), entries: [], pins: new Set() };
375-
tracked = new Map<string, Tracked>();
391+
owesVideoState = true;
392+
tracked = new TrackedFiles(() => (this.owesVideoState = true));
376393
windows: Group[] = [];
377394
outside: Group | null = null;
378395
/** A coalescing claim, open until another key, its idle timer, an operation, a window, or another write. */
@@ -426,15 +443,19 @@ class Engine {
426443
try {
427444
return await task();
428445
} finally {
429-
if (this.whereFolder() === "here") {
446+
if (this.owesVideoState && this.whereFolder() === "here") {
447+
this.owesVideoState = false;
430448
await writeHistoryState(
431449
this.dir,
432450
this.projectId,
433451
this.home,
434452
this.log.entries,
435453
this.manifest(),
436454
manifestAt(this.log, this.log.entries.at(-1)?.id ?? START) ?? new Map(),
437-
).catch((error) => this.options.onError?.(error));
455+
).catch((error) => {
456+
this.owesVideoState = true;
457+
this.options.onError?.(error);
458+
});
438459
}
439460
}
440461
});
@@ -457,6 +478,7 @@ class Engine {
457478
}
458479

459480
persistLog(record?: LogRecord): void {
481+
this.owesVideoState = true;
460482
if (record) saveRecord(this.logFile, this.log, record);
461483
else writeLog(this.logFile, this.log);
462484
if (!existsSync(join(this.home, "project.json")))
@@ -491,7 +513,7 @@ class Engine {
491513
const left = this.readAdopting();
492514
const unadopted = (file: { path: string }) =>
493515
left.has(file.path) && !this.tracked.has(file.path);
494-
this.adoptMediaInBackground(historyFiles(this.dir).filter(unadopted), this.now());
516+
this.adoptMediaInBackground((await historyFiles(this.dir)).filter(unadopted), this.now());
495517
// What changed while the project was closed is one outside entry, or the closed window's.
496518
this.reopenClosedWindow();
497519
await this.settleAll();
@@ -554,7 +576,7 @@ class Engine {
554576
async takeInUnnamedLedger(): Promise<void> {
555577
const named = (path: string) =>
556578
this.log.entries.some((entry) => entry.files.some((file) => file.path === path));
557-
const ledger = historyFiles(this.dir).find((file) => file.path === MEDIA_LEDGER);
579+
const ledger = (await historyFiles(this.dir)).find((file) => file.path === MEDIA_LEDGER);
558580
if (ledger && !this.log.baseline.has(MEDIA_LEDGER) && !named(MEDIA_LEDGER)) {
559581
const copied = await this.storeIfPresent(MEDIA_LEDGER);
560582
if (typeof copied === "object") this.log.baseline.set(MEDIA_LEDGER, copied.hash);
@@ -566,7 +588,7 @@ class Engine {
566588

567589
async firstOpen(): Promise<void> {
568590
const sweptAt = this.now();
569-
const files = historyFiles(this.dir);
591+
const files = await historyFiles(this.dir);
570592
const copied = files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES);
571593
const hashes = await Promise.allSettled(
572594
copied.map((file) => inFirstCopySlot(() => this.storeIfPresent(file.path))),
@@ -690,7 +712,7 @@ class Engine {
690712
const sweptAt = this.now();
691713
const changedAt = (file: { mtimeMs: number; ctimeMs: number }) =>
692714
Math.min(sweptAt, Math.max(file.mtimeMs, file.ctimeMs));
693-
const seen = historyFiles(this.dir).filter((file) => !this.adopting.has(file.path));
715+
const seen = (await historyFiles(this.dir)).filter((file) => !this.adopting.has(file.path));
694716
const heard = new Map(this.overwritten);
695717
const present = new Set(seen.map((file) => file.path));
696718
const removed = [...this.tracked.keys()]
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// @vitest-environment node
2+
import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { afterEach, expect, it, vi } from "vitest";
6+
import { openProjectHistory } from "./projectHistory";
7+
8+
// A slow disk, as a large project on a scanned drive is: each blocking stat or listing holds the thread this long.
9+
const slowDisk = vi.hoisted(() => ({ ms: 0 }));
10+
vi.mock("node:fs", async (importOriginal) => {
11+
const real = await importOriginal<typeof import("node:fs")>();
12+
const pause = new Int32Array(new SharedArrayBuffer(4));
13+
const slow =
14+
<A extends unknown[], R>(call: (...args: A) => R) =>
15+
(...args: A): R => {
16+
if (slowDisk.ms) Atomics.wait(pause, 0, 0, slowDisk.ms);
17+
return call(...args);
18+
};
19+
const lstatSync = slow(real.lstatSync);
20+
const readdirSync = slow(real.readdirSync);
21+
return { ...real, lstatSync, readdirSync, default: { ...real, lstatSync, readdirSync } };
22+
});
23+
24+
const cleanup: Array<() => unknown> = [];
25+
afterEach(async () => {
26+
slowDisk.ms = 0;
27+
for (const step of cleanup.splice(0).reverse()) await step();
28+
});
29+
30+
function tempDir(prefix: string): string {
31+
const dir = mkdtempSync(join(tmpdir(), prefix));
32+
cleanup.push(() => rmSync(dir, { recursive: true, force: true }));
33+
return dir;
34+
}
35+
36+
/** The longest stretch the event loop went without turning while `work` ran. */
37+
async function longestHold(work: () => Promise<unknown>): Promise<number> {
38+
let last = performance.now();
39+
let longest = 0;
40+
let running = true;
41+
const turn = () => {
42+
const now = performance.now();
43+
longest = Math.max(longest, now - last);
44+
last = now;
45+
if (running) setImmediate(turn);
46+
};
47+
setImmediate(turn);
48+
await work();
49+
running = false;
50+
// The stretch since the last turn counts too: a walk that never yields ends before a first turn.
51+
return Math.max(longest, performance.now() - last);
52+
}
53+
54+
// fails without the fix: the sweep listed the project with readdirSync and lstatSync, holding the loop for the walk.
55+
it("a sweep of a project on a slow disk lets the host keep answering while it lists the files", async () => {
56+
const projectDir = tempDir("hf-history-walk-");
57+
for (let folder = 0; folder < 20; folder++) {
58+
mkdirSync(join(projectDir, `scenes-${folder}`));
59+
for (let file = 0; file < 20; file++)
60+
writeFileSync(join(projectDir, `scenes-${folder}`, `clip-${file}.html`), `<p>${file}</p>`);
61+
}
62+
// A minute on, the files are settled, as in a project opened later: the sweep stats them and hashes none.
63+
const history = await openProjectHistory({
64+
projectDir,
65+
historyRoot: tempDir("hf-history-root-"),
66+
now: () => Date.now() + 60_000,
67+
});
68+
cleanup.push(() => history.close());
69+
70+
slowDisk.ms = 1;
71+
const longest = await longestHold(() => history.flush());
72+
73+
// A blocking walk of these 420 entries holds the loop at least 420 ms.
74+
expect(longest, `held the loop ${longest.toFixed(1)} ms`).toBeLessThan(200);
75+
});
76+
77+
// fails without the fix: every queued operation rebuilt and rewrote video-state.json, one record per project file.
78+
it("a sweep that finds nothing leaves the video state file alone, and a change rewrites it", async () => {
79+
const projectDir = tempDir("hf-history-state-");
80+
writeFileSync(join(projectDir, "index.html"), "<h1>Hello</h1>");
81+
const history = await openProjectHistory({
82+
projectDir,
83+
historyRoot: tempDir("hf-history-root-"),
84+
now: () => Date.now() + 60_000,
85+
});
86+
cleanup.push(() => history.close());
87+
await history.flush();
88+
// Each write replaces the file, so a rewrite shows as a new inode.
89+
const written = () => statSync(join(projectDir, ".hyperframes", "video-state.json")).ino;
90+
const before = written();
91+
92+
await history.flush();
93+
expect(written()).toBe(before);
94+
95+
writeFileSync(join(projectDir, "index.html"), "<h1>Bye</h1>");
96+
await history.flush();
97+
expect(written()).not.toBe(before);
98+
});

‎packages/studio-server/src/history/videoState.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { createReadStream, existsSync, readFileSync, renameSync, statSync } from
33
import { isAbsolute, join, resolve } from "node:path";
44
import { replaceFileAtomically } from "@hyperframes/core/atomic-file";
55
import { pinProjectRecord } from "../helpers/safePath.js";
6-
import { listProjectFiles } from "../helpers/projectSignature.js";
6+
import { listProjectFilesAsync } from "../helpers/projectSignature.js";
77
import { readId } from "./historyId.js";
88
import { takeHistoryOwnership } from "./ownerLock.js";
99
import {
@@ -198,7 +198,9 @@ function dropArtifactsOfGoneRevisions(state: VideoState): void {
198198

199199
async function fileHashes(dir: string): Promise<Record<string, string>> {
200200
const files: Record<string, string> = {};
201-
for (const { path } of listProjectFiles(dir).sort((a, b) => a.path.localeCompare(b.path))) {
201+
for (const { path } of (await listProjectFilesAsync(dir)).sort((a, b) =>
202+
a.path.localeCompare(b.path),
203+
)) {
202204
const hash = createHash("sha256");
203205
for await (const chunk of createReadStream(join(dir, path))) hash.update(chunk);
204206
files[path] = hash.digest("hex");

0 commit comments

Comments
 (0)