Skip to content

Commit 2927fe6

Browse files
perf(studio-server): a large project's first history copy takes a fraction of the time (#7415)
* perf(studio-server): a large project's first history copy takes a fraction of the time * perf(studio-server): read a small file through one handle, with fewer comments * fix(studio-server): small files keep their exact permissions in history under a strict umask
1 parent 327e202 commit 2927fe6

5 files changed

Lines changed: 272 additions & 11 deletions

File tree

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// A project's first history open, killed outright mid-copy with one copy half written.
2+
import { writeFileSync } from "node:fs";
3+
import fsp from "node:fs/promises";
4+
import { syncBuiltinESMExports } from "node:module";
5+
6+
const [projectDir, historyRoot, dieAtWrite] = process.argv.slice(2);
7+
const realWriteFile = fsp.writeFile;
8+
let writes = 0;
9+
fsp.writeFile = ((...args: Parameters<typeof realWriteFile>) => {
10+
if (String(args[0]).includes("incoming-") && ++writes === Number(dieAtWrite)) {
11+
writeFileSync(String(args[0]), "half a co");
12+
process.kill(process.pid, "SIGKILL");
13+
}
14+
return realWriteFile(...args);
15+
}) as typeof realWriteFile;
16+
syncBuiltinESMExports();
17+
18+
const { openProjectHistory } = await import("../projectHistory.js");
19+
await openProjectHistory({ projectDir: projectDir!, historyRoot: historyRoot! });

‎packages/studio-server/src/history/blobStore.test.ts‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,17 @@ describe("the history store on Windows, while another process briefly holds a fi
107107
const store = await openBlobStore(join(root, "history", "blobs"));
108108
let refused = 0;
109109
vi.mocked(fsp.rename).mockImplementation(async (from, to) => {
110-
if (refused++ < 2) throw busy();
110+
if (refused < 2) {
111+
refused += 1;
112+
throw busy();
113+
}
111114
return realRename(from, to);
112115
});
113116

114117
const hash = await store.put(source);
115118

116119
expect((await store.read(hash)).toString("utf8")).toBe("<p>kept</p>");
117-
expect(refused).toBe(3);
120+
expect(refused).toBe(2);
118121
});
119122
});
120123

@@ -170,3 +173,20 @@ describe("a restore of a file the preview is serving", () => {
170173
expect(heard).toEqual([]);
171174
});
172175
});
176+
177+
describe("the history store's count of what it holds", () => {
178+
it("counts a file once when two copies of it are filed together", async () => {
179+
const root = mkdtempSync(join(tmpdir(), "hf-blob-count-"));
180+
try {
181+
const small = join(root, "small.txt");
182+
const large = join(root, "large.txt");
183+
writeFileSync(small, "same bytes");
184+
writeFileSync(large, "x".repeat(200 * 1024));
185+
const store = await openBlobStore(join(root, "history", "blobs"));
186+
await Promise.all([store.put(small), store.put(small), store.put(large), store.put(large)]);
187+
expect(store.bytes()).toBe("same bytes".length + 200 * 1024);
188+
} finally {
189+
rmSync(root, { recursive: true, force: true });
190+
}
191+
});
192+
});

‎packages/studio-server/src/history/blobStore.ts‎

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
import { createHash, randomUUID } from "node:crypto";
22
import { constants, createReadStream, createWriteStream } from "node:fs";
3-
import { copyFile, mkdir, open, readdir, readFile, rm, stat } from "node:fs/promises";
3+
import {
4+
chmod,
5+
copyFile,
6+
mkdir,
7+
open,
8+
readdir,
9+
readFile,
10+
rm,
11+
stat,
12+
writeFile,
13+
} from "node:fs/promises";
414
import { dirname, join } from "node:path";
515
import { pipeline } from "node:stream/promises";
616
import { atomicTempPath } from "@hyperframes/core/atomic-file";
@@ -13,7 +23,7 @@ import { stopServingAndClose } from "../helpers/servedFiles.js";
1323

1424
/** File contents stored once by sha256, text and binary alike. */
1525
export interface BlobStore {
16-
/** Copies the file in (a clone where the file system can) and returns the copy's hash. */
26+
/** Copies the file in (a clone where the file system can, for all but small files) and returns the copy's hash. */
1727
put(absPath: string): Promise<string>;
1828
reference(absPath: string): Promise<string>;
1929
has(hash: string): boolean;
@@ -29,6 +39,7 @@ export interface BlobStore {
2939
}
3040

3141
const BLOB_HASH = /^[0-9a-f]{64}$/;
42+
const READ_ONCE_AT_MOST_BYTES = 64 * 1024;
3243

3344
export function blobPath(dir: string, hash: string): string {
3445
// Checked where the path is joined, so no caller can read or write outside the store.
@@ -44,6 +55,16 @@ async function hashFile(path: string): Promise<string> {
4455
return hash.digest("hex");
4556
}
4657

58+
async function readIfSmall(path: string): Promise<{ bytes: Buffer; mode: number } | null> {
59+
const handle = await open(path, "r");
60+
try {
61+
const { size, mode } = await handle.stat();
62+
return size > READ_ONCE_AT_MOST_BYTES ? null : { bytes: await handle.readFile(), mode };
63+
} finally {
64+
await handle.close();
65+
}
66+
}
67+
4768
async function cloneOrStream(from: string, to: string): Promise<void> {
4869
try {
4970
await copyFile(from, to, constants.COPYFILE_FICLONE_FORCE);
@@ -144,8 +165,39 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
144165
holds.delete(hash);
145166
};
146167

168+
const file = (hash: string, size: number) => {
169+
if (!sizes.has(hash)) total += size;
170+
sizes.set(hash, size);
171+
};
172+
const withFolderMadeIfMissing = (folder: string, write: () => Promise<void>) =>
173+
write().catch(async (error: NodeJS.ErrnoException) => {
174+
if (error.code !== "ENOENT") throw error;
175+
await mkdir(folder, { recursive: true });
176+
await write();
177+
});
178+
// The bytes hashed are the bytes filed, so a write racing the read can never file bytes under the wrong hash.
179+
const putBytes = async (bytes: Buffer, mode: number) => {
180+
const hash = createHash("sha256").update(bytes).digest("hex");
181+
if (sizes.has(hash)) return hash;
182+
const temp = join(dir, `incoming-${randomUUID()}`);
183+
try {
184+
await withFolderMadeIfMissing(dir, () => writeFile(temp, bytes, { mode }));
185+
// A write applies the umask; the clone used for larger files keeps the mode exactly.
186+
await chmod(temp, mode & 0o7777);
187+
const to = blobPath(dir, hash);
188+
await withFolderMadeIfMissing(dirname(to), () => renameWithRetryAsync(temp, to));
189+
} catch (error) {
190+
await rm(temp, { force: true });
191+
throw error;
192+
}
193+
file(hash, bytes.length);
194+
return hash;
195+
};
196+
147197
return {
148198
async put(absPath) {
199+
const small = await readIfSmall(absPath);
200+
if (small) return putBytes(small.bytes, small.mode);
149201
// Hash the copy, not the source, so a write racing the copy can never file bytes under the wrong hash.
150202
const temp = join(dir, `incoming-${randomUUID()}`);
151203
// Recreated if removed while open, so a missing folder is never mistaken for a deleted project file.
@@ -162,8 +214,7 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
162214
await mkdir(dirname(blobPath(dir, hash)), { recursive: true });
163215
const size = (await stat(temp)).size;
164216
await renameWithRetryAsync(temp, blobPath(dir, hash));
165-
sizes.set(hash, size);
166-
total += size;
217+
file(hash, size);
167218
return hash;
168219
},
169220
reference: (absPath) => hashFile(absPath),
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// @vitest-environment node
2+
import { fork } from "node:child_process";
3+
import { createHash } from "node:crypto";
4+
import {
5+
chmodSync,
6+
existsSync,
7+
mkdirSync,
8+
mkdtempSync,
9+
readdirSync,
10+
readFileSync,
11+
rmSync,
12+
statSync,
13+
writeFileSync,
14+
} from "node:fs";
15+
import { tmpdir } from "node:os";
16+
import { join } from "node:path";
17+
import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest";
18+
19+
vi.mock("node:fs/promises", async (importOriginal) => {
20+
const actual = await importOriginal<typeof import("node:fs/promises")>();
21+
return { ...actual, writeFile: vi.fn(actual.writeFile) };
22+
});
23+
24+
import * as fsp from "node:fs/promises";
25+
import { blobPath } from "./blobStore.js";
26+
import { START } from "./historyLog.js";
27+
import { openProjectHistory } from "./projectHistory.js";
28+
29+
const { writeFile: realWriteFile } =
30+
await vi.importActual<typeof import("node:fs/promises")>("node:fs/promises");
31+
const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
32+
const temps: string[] = [];
33+
const tempDir = (prefix: string) => {
34+
const dir = mkdtempSync(join(tmpdir(), prefix));
35+
temps.push(dir);
36+
return dir;
37+
};
38+
const closes: (() => Promise<void>)[] = [];
39+
40+
afterEach(async () => {
41+
vi.mocked(fsp.writeFile).mockImplementation(realWriteFile);
42+
for (const close of closes.splice(0)) await close();
43+
for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true });
44+
});
45+
46+
function projectOf(count: number, tag: string): string {
47+
const dir = tempDir("hf-first-copy-");
48+
mkdirSync(join(dir, "parts"));
49+
writeFileSync(join(dir, "index.html"), `<html>${tag}</html>`);
50+
for (let i = 0; i < count; i++) writeFileSync(join(dir, "parts", `p${i}.js`), `// ${tag} ${i}\n`);
51+
return dir;
52+
}
53+
const homeIn = (historyRoot: string) => {
54+
const homes = readdirSync(historyRoot, { withFileTypes: true }).filter((e) => e.isDirectory());
55+
return join(historyRoot, homes.find((e) => existsSync(join(historyRoot, e.name, "blobs")))!.name);
56+
};
57+
async function open(projectDir: string, historyRoot: string) {
58+
const history = await openProjectHistory({ projectDir, historyRoot });
59+
closes.push(() => history.close());
60+
return history;
61+
}
62+
/** Every file of the project is in the baseline, and its blob holds exactly the file's bytes. */
63+
function expectAllKept(
64+
baseline: Record<string, string> | null,
65+
projectDir: string,
66+
blobs: string,
67+
paths: string[],
68+
) {
69+
expect(Object.keys(baseline ?? {}).sort()).toEqual([...paths].sort());
70+
for (const path of paths) {
71+
const bytes = readFileSync(join(projectDir, path));
72+
expect(baseline![path]).toBe(sha256(bytes));
73+
expect(readFileSync(blobPath(blobs, sha256(bytes))).equals(bytes)).toBe(true);
74+
}
75+
}
76+
const pathsOf = (count: number) => [
77+
"index.html",
78+
...Array.from({ length: count }, (_, i) => `parts/p${i}.js`),
79+
];
80+
81+
describe("a project's first history copy", () => {
82+
it("loses nothing when the process dies midway: the next open finishes it", async () => {
83+
const projectDir = projectOf(120, "dies");
84+
const historyRoot = tempDir("hf-first-copy-root-");
85+
const before = readFileSync(join(projectDir, "parts", "p7.js"));
86+
const died = await new Promise<NodeJS.Signals | number | null>((done) => {
87+
const fixture = join(import.meta.dirname, "__fixtures__", "dieMidFirstCopy.ts");
88+
fork(fixture, [projectDir, historyRoot, "40"], { execArgv: ["--import", "tsx"] }).once(
89+
"exit",
90+
(code, signal) => done(signal ?? code),
91+
);
92+
});
93+
expect(died).toBe("SIGKILL");
94+
const home = homeIn(historyRoot);
95+
expect(existsSync(join(home, "log.jsonl")), "the copy finished before the death").toBe(false);
96+
const leftovers = () =>
97+
readdirSync(join(home, "blobs")).filter((name) => name.startsWith("incoming-"));
98+
expect(leftovers().length, "the death left no half-written copy").toBeGreaterThan(0);
99+
100+
const history = await open(projectDir, historyRoot);
101+
102+
expectAllKept(history.peek(START), projectDir, join(home, "blobs"), pathsOf(120));
103+
expect(leftovers()).toEqual([]);
104+
for (const shard of readdirSync(join(home, "blobs")))
105+
for (const hash of readdirSync(join(home, "blobs", shard)))
106+
expect(sha256(readFileSync(join(home, "blobs", shard, hash)))).toBe(hash);
107+
expect(readFileSync(join(projectDir, "parts", "p7.js")).equals(before)).toBe(true);
108+
});
109+
110+
it("keeps the same contents and permissions, small files and large, under a strict umask", async () => {
111+
const umask = process.umask(0o077);
112+
onTestFinished(() => void process.umask(umask));
113+
const projectDir = projectOf(3, "kept");
114+
writeFileSync(join(projectDir, "large.js"), "x".repeat(200 * 1024));
115+
writeFileSync(join(projectDir, "same-a.js"), "same bytes");
116+
writeFileSync(join(projectDir, "same-b.js"), "same bytes");
117+
writeFileSync(join(projectDir, "run.sh"), "#!/bin/sh\n");
118+
chmodSync(join(projectDir, "run.sh"), 0o755);
119+
const historyRoot = tempDir("hf-first-copy-root-");
120+
121+
const history = await open(projectDir, historyRoot);
122+
123+
const blobs = join(homeIn(historyRoot), "blobs");
124+
const paths = [...pathsOf(3), "large.js", "same-a.js", "same-b.js", "run.sh"];
125+
expectAllKept(history.peek(START), projectDir, blobs, paths);
126+
const script = readFileSync(join(projectDir, "run.sh"));
127+
expect(statSync(blobPath(blobs, sha256(script))).mode & 0o111).toBe(
128+
statSync(join(projectDir, "run.sh")).mode & 0o111,
129+
);
130+
});
131+
132+
it("copies a few files at a time across every project opening together", async () => {
133+
let writing = 0;
134+
let most = 0;
135+
vi.mocked(fsp.writeFile).mockImplementation(async (...args) => {
136+
const counted = String(args[0]).includes("incoming-");
137+
if (counted) most = Math.max(most, ++writing);
138+
await realWriteFile(...args);
139+
if (counted) writing -= 1;
140+
});
141+
const historyRoot = tempDir("hf-first-copy-root-");
142+
143+
await Promise.all(["a", "b", "c"].map((tag) => open(projectOf(30, tag), historyRoot)));
144+
145+
expect(most).toBeGreaterThan(1);
146+
expect(most).toBeLessThanOrEqual(4);
147+
});
148+
});

‎packages/studio-server/src/history/projectHistory.ts‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,22 @@ const statKey = (file: { size: number; mtimeMs: number; ctimeMs: number }, swept
212212
? ""
213213
: `${file.size}:${file.mtimeMs}:${file.ctimeMs}`;
214214

215+
const FIRST_COPY_SLOTS_IN_PROCESS = 4;
216+
let firstCopiesRunning = 0;
217+
const firstCopiesWaiting: (() => void)[] = [];
218+
219+
async function inFirstCopySlot<T>(copy: () => Promise<T>): Promise<T> {
220+
if (firstCopiesRunning < FIRST_COPY_SLOTS_IN_PROCESS) firstCopiesRunning += 1;
221+
else await new Promise<void>((yours) => firstCopiesWaiting.push(yours));
222+
try {
223+
return await copy();
224+
} finally {
225+
const next = firstCopiesWaiting.shift();
226+
if (next) next();
227+
else firstCopiesRunning -= 1;
228+
}
229+
}
230+
215231
const MEDIA_LEDGER = ".media/manifest.jsonl";
216232
const KEPT_HIDDEN_PATHS = new Set<string>([...STUDIO_SIGNATURE_MANIFEST_PATHS, MEDIA_LEDGER]);
217233

@@ -548,11 +564,18 @@ class Engine {
548564
async firstOpen(): Promise<void> {
549565
const sweptAt = this.now();
550566
const files = historyFiles(this.dir);
551-
for (const file of files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES)) {
552-
const hash = await this.storeIfPresent(file.path);
553-
if (this.whereFolder() !== "here") throw this.replaced();
554-
if (hash !== null) this.tracked.set(file.path, { hash, stat: statKey(file, sweptAt) });
555-
}
567+
const copied = files.filter((file) => file.size <= COPY_LATER_ABOVE_BYTES);
568+
const hashes = await Promise.allSettled(
569+
copied.map((file) => inFirstCopySlot(() => this.storeIfPresent(file.path))),
570+
);
571+
// Nothing names these copies until the log below is written, so one look at the end covers the whole copy.
572+
if (this.whereFolder() !== "here") throw this.replaced();
573+
copied.forEach((file, i) => {
574+
const result = hashes[i]!;
575+
if (result.status === "rejected") throw result.reason;
576+
if (result.value !== null)
577+
this.tracked.set(file.path, { hash: result.value, stat: statKey(file, sweptAt) });
578+
});
556579
this.log.baseline = this.manifest();
557580
this.log.keepsLedger = true;
558581
this.persistLog();

0 commit comments

Comments
 (0)