Skip to content

Commit bee6a8e

Browse files
fix(studio-server): remove the history of a project that no longer exists (#4634)
* fix(studio-server): remove the history of a project that no longer exists * fix(studio-server): keep a project's history while its drive is disconnected * fix(studio-server): keep an older history whose drive may only be unmounted
1 parent 5383e48 commit bee6a8e

6 files changed

Lines changed: 507 additions & 3 deletions

File tree

‎packages/cli/src/utils/historyOwner.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ async function directOwner(projectDir: string, turn: Turn | null): Promise<Owner
155155
const history = await openProjectHistory({
156156
projectDir,
157157
historyRoot: historyDeps.historyRoot,
158+
pruneGoneProjectsBudgetMs: 1000,
158159
// A turn begun through a preview that has since stopped is still the agent's, until its idle limit.
159160
...(turn && {
160161
closedWindow: {

‎packages/studio-server/src/history/historyId.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ export const ID_PATH = join(".hyperframes", "history-id");
77
/** The only shape minted here; the id is project content and becomes a path, so nothing else is trusted. */
88
const ID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
99

10+
export const isHistoryId = (name: string) => ID_SHAPE.test(name);
11+
1012
export function readId(projectDir: string): string | null {
1113
try {
1214
const id = readFileSync(join(projectDir, ID_PATH), "utf-8").trim();
@@ -30,7 +32,9 @@ export function isRecordedFolder(historyDir: string, folder: FolderIdentity): bo
3032
return !isCopyOf0878Folder(was!.dir as string, basename(historyDir), folder);
3133
}
3234

33-
function readRecord(historyDir: string): { dir?: unknown; ino?: unknown; born?: number } | null {
35+
export function readRecord(
36+
historyDir: string,
37+
): { dir?: unknown; ino?: unknown; born?: number; dev?: unknown } | null {
3438
try {
3539
return JSON.parse(readFileSync(join(historyDir, "project.json"), "utf-8"));
3640
} catch {
@@ -79,8 +83,12 @@ export class HistoryIdError extends Error {
7983
}
8084
}
8185

82-
export function recordProject(historyDir: string, dir: string, folder: FolderIdentity): void {
83-
const record = { dir, ino: folder.ino, born: folder.birthtimeMs };
86+
export function recordProject(
87+
historyDir: string,
88+
dir: string,
89+
folder: FolderIdentity & { dev: number },
90+
): void {
91+
const record = { dir, ino: folder.ino, born: folder.birthtimeMs, dev: folder.dev };
8492
mkdirSync(historyDir, { recursive: true });
8593
replaceFileAtomically(join(historyDir, "project.json"), JSON.stringify(record), 0o644);
8694
}

‎packages/studio-server/src/history/projectHistory.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
} from "../helpers/fileVersion.js";
2727
import { affectsProjectSignature, listProjectFiles } from "../helpers/projectSignature.js";
2828
import { openBlobStore, type BlobStore } from "./blobStore.js";
29+
import { pruneGoneProjectHistoriesDaily } from "./pruneHistories.js";
2930
import {
3031
ID_PATH,
3132
isRecordedFolder,
@@ -77,6 +78,7 @@ export interface ProjectHistoryOptions {
7778
/** A CLI turn's window from an earlier open: writes since, within its idle limit, become the entry with its id. */
7879
closedWindow?: ClosedWindow;
7980
undoScope?: "own" | "everyone";
81+
pruneGoneProjectsBudgetMs?: number;
8082
}
8183

8284
interface ClaimOptions {
@@ -1085,6 +1087,10 @@ export async function openProjectHistory(options: ProjectHistoryOptions): Promis
10851087
const blobs = await openBlobStore(join(options.historyRoot, projectId, "blobs"));
10861088
const engine = new Engine(options, projectId, blobs);
10871089
await engine.queue(() => engine.start());
1090+
pruneGoneProjectHistoriesDaily(options.historyRoot, engine.now(), {
1091+
onError: options.onError,
1092+
budgetMs: options.pruneGoneProjectsBudgetMs,
1093+
});
10881094
const api = engine.api();
10891095
return {
10901096
...api,
Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
import {
2+
chmodSync,
3+
existsSync,
4+
mkdirSync,
5+
mkdtempSync,
6+
readFileSync,
7+
rmSync,
8+
writeFileSync,
9+
} from "node:fs";
10+
import { tmpdir } from "node:os";
11+
import { dirname, join } from "node:path";
12+
import { describe, expect, it, onTestFinished, vi } from "vitest";
13+
import { ID_PATH, readId } from "./historyId.js";
14+
import { takeHistoryOwnership } from "./ownerLock.js";
15+
import { openProjectHistory } from "./projectHistory.js";
16+
import { KEEP_GONE_PROJECT_HISTORY_MS, pruneGoneProjectHistories } from "./pruneHistories.js";
17+
18+
vi.mock("./ownerLock.js", async (importOriginal) => {
19+
const actual = await importOriginal<typeof import("./ownerLock.js")>();
20+
return { ...actual, takeHistoryOwnership: vi.fn(actual.takeHistoryOwnership) };
21+
});
22+
23+
function tempDir(prefix: string): string {
24+
const dir = mkdtempSync(join(tmpdir(), prefix));
25+
onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
26+
return dir;
27+
}
28+
29+
const DAY_MS = 24 * 60 * 60 * 1000;
30+
31+
async function projectWithHistory(
32+
historyRoot: string,
33+
{
34+
at = Date.now(),
35+
projectDir = tempDir("hf-prune-project-"),
36+
pruneGoneProjectsBudgetMs = undefined as number | undefined,
37+
} = {},
38+
) {
39+
mkdirSync(projectDir, { recursive: true });
40+
writeFileSync(join(projectDir, "index.html"), "<h1>A</h1>");
41+
const history = await openProjectHistory({
42+
projectDir,
43+
historyRoot,
44+
now: () => at,
45+
pruneGoneProjectsBudgetMs,
46+
});
47+
// Lets the open's own daily prune scan first, so it cannot race the test's.
48+
await new Promise((settle) => setImmediate(settle));
49+
return { projectDir, history, id: readId(projectDir)! };
50+
}
51+
52+
function editRecord(
53+
historyRoot: string,
54+
id: string,
55+
edit: (record: Record<string, unknown>) => Record<string, unknown>,
56+
) {
57+
const file = join(historyRoot, id, "project.json");
58+
writeFileSync(file, JSON.stringify(edit(JSON.parse(readFileSync(file, "utf-8")))));
59+
}
60+
61+
const prunedIds = async (...args: Parameters<typeof pruneGoneProjectHistories>) =>
62+
(await pruneGoneProjectHistories(...args)).map((pruned) => pruned.id);
63+
64+
describe("pruneGoneProjectHistories", () => {
65+
it("in a dry run reports what would go, with its folder and size, and removes nothing", async () => {
66+
const historyRoot = tempDir("hf-prune-root-");
67+
const gone = await projectWithHistory(historyRoot);
68+
await gone.history.close();
69+
rmSync(gone.projectDir, { recursive: true, force: true });
70+
71+
const [pruned, ...rest] = await pruneGoneProjectHistories(historyRoot, { dryRun: true });
72+
expect(rest).toEqual([]);
73+
expect(pruned).toMatchObject({ id: gone.id, projectDir: gone.projectDir });
74+
expect(pruned!.bytes).toBeGreaterThan(0);
75+
expect(existsSync(join(historyRoot, gone.id))).toBe(true);
76+
});
77+
78+
it("removes a deleted project's history once no process holds it, and keeps a live project's", async () => {
79+
const historyRoot = tempDir("hf-prune-root-");
80+
const gone = await projectWithHistory(historyRoot);
81+
const kept = await projectWithHistory(historyRoot);
82+
rmSync(gone.projectDir, { recursive: true, force: true });
83+
84+
expect(await prunedIds(historyRoot)).toEqual([]);
85+
await gone.history.close();
86+
expect(await prunedIds(historyRoot)).toEqual([gone.id]);
87+
88+
expect(existsSync(join(historyRoot, gone.id))).toBe(false);
89+
expect(existsSync(join(historyRoot, kept.id))).toBe(true);
90+
await kept.history.close();
91+
});
92+
93+
it("keeps a gone project's history outside the temp folder until it has gone unused for 14 days", async () => {
94+
const historyRoot = tempDir("hf-prune-root-");
95+
const gone = await projectWithHistory(historyRoot);
96+
await gone.history.close();
97+
rmSync(gone.projectDir, { recursive: true, force: true });
98+
const elsewhere = { tempDir: historyRoot };
99+
100+
expect(await prunedIds(historyRoot, elsewhere)).toEqual([]);
101+
expect(await prunedIds(historyRoot, { ...elsewhere, now: Date.now() + 13 * DAY_MS })).toEqual(
102+
[],
103+
);
104+
expect(await prunedIds(historyRoot, { ...elsewhere, now: Date.now() + 14 * DAY_MS })).toEqual([
105+
gone.id,
106+
]);
107+
});
108+
109+
it("prunes when a history opens a day after the last full prune", async () => {
110+
const historyRoot = tempDir("hf-prune-root-");
111+
const start = Date.now();
112+
const gone = await projectWithHistory(historyRoot, { at: start });
113+
await gone.history.close();
114+
rmSync(gone.projectDir, { recursive: true, force: true });
115+
116+
const cli = await projectWithHistory(historyRoot, {
117+
at: start + DAY_MS,
118+
pruneGoneProjectsBudgetMs: 0,
119+
});
120+
expect(existsSync(join(historyRoot, gone.id))).toBe(true);
121+
await cli.history.close();
122+
const next = await projectWithHistory(historyRoot, { at: start + DAY_MS });
123+
await vi.waitFor(() => expect(existsSync(join(historyRoot, gone.id))).toBe(false));
124+
await next.history.close();
125+
});
126+
127+
it("finishes an interrupted removal and leaves folders that are not histories alone", async () => {
128+
const historyRoot = tempDir("hf-prune-root-");
129+
const leftover = join(historyRoot, ".pruned-leftover");
130+
mkdirSync(join(leftover, "blobs"), { recursive: true });
131+
writeFileSync(join(leftover, "blobs", "b"), "bytes");
132+
const copy = join(historyRoot, "backup");
133+
mkdirSync(copy);
134+
writeFileSync(join(copy, "project.json"), JSON.stringify({ dir: join(historyRoot, "gone") }));
135+
136+
expect(await prunedIds(historyRoot)).toEqual([]);
137+
expect(existsSync(leftover)).toBe(false);
138+
expect(existsSync(copy)).toBe(true);
139+
});
140+
141+
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
142+
"lets go of a history it could not move away",
143+
async () => {
144+
const historyRoot = tempDir("hf-prune-root-");
145+
const gone = await projectWithHistory(historyRoot);
146+
await gone.history.close();
147+
rmSync(gone.projectDir, { recursive: true, force: true });
148+
const errors: unknown[] = [];
149+
150+
chmodSync(historyRoot, 0o500);
151+
try {
152+
expect(await prunedIds(historyRoot, { onError: (error) => errors.push(error) })).toEqual(
153+
[],
154+
);
155+
} finally {
156+
chmodSync(historyRoot, 0o700);
157+
}
158+
expect(errors).toHaveLength(1);
159+
expect(existsSync(join(historyRoot, gone.id, "owner.pid"))).toBe(false);
160+
},
161+
);
162+
163+
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
164+
"reports a history it cannot check and still prunes the rest",
165+
async () => {
166+
const historyRoot = tempDir("hf-prune-root-");
167+
const locked = tempDir("hf-prune-locked-");
168+
const hidden = await projectWithHistory(historyRoot, { projectDir: join(locked, "project") });
169+
await hidden.history.close();
170+
const gone = await projectWithHistory(historyRoot);
171+
await gone.history.close();
172+
rmSync(gone.projectDir, { recursive: true, force: true });
173+
const errors: unknown[] = [];
174+
175+
chmodSync(locked, 0o000);
176+
try {
177+
const later = Date.now() + KEEP_GONE_PROJECT_HISTORY_MS;
178+
const onError = (error: unknown) => errors.push(error);
179+
expect(await prunedIds(historyRoot, { now: later, onError })).toEqual([gone.id]);
180+
} finally {
181+
chmodSync(locked, 0o700);
182+
}
183+
expect(errors).toHaveLength(1);
184+
expect(existsSync(join(historyRoot, hidden.id))).toBe(true);
185+
},
186+
);
187+
188+
it("keeps a gone project's history while the disk that held it is not here", async () => {
189+
const historyRoot = tempDir("hf-prune-root-");
190+
const onDrive = await projectWithHistory(historyRoot, {
191+
projectDir: join(tempDir("hf-prune-drive-"), "project"),
192+
});
193+
const deleted = await projectWithHistory(historyRoot, {
194+
projectDir: join(tempDir("hf-prune-tree-"), "project"),
195+
});
196+
const mountPoint = await projectWithHistory(historyRoot);
197+
await Promise.all([onDrive, deleted, mountPoint].map(({ history }) => history.close()));
198+
for (const { id } of [onDrive, mountPoint])
199+
editRecord(historyRoot, id, (record) => ({ ...record, dev: Number(record.dev) + 1 }));
200+
rmSync(dirname(onDrive.projectDir), { recursive: true, force: true });
201+
// What an unmounted drive leaves at its mount point: the folder, without the project in it.
202+
rmSync(join(mountPoint.projectDir, ".hyperframes"), { recursive: true, force: true });
203+
rmSync(dirname(deleted.projectDir), { recursive: true, force: true });
204+
205+
const later = Date.now() + KEEP_GONE_PROJECT_HISTORY_MS;
206+
expect(await prunedIds(historyRoot, { tempDir: historyRoot, now: later })).toEqual([
207+
deleted.id,
208+
]);
209+
});
210+
211+
it("prunes a history recorded without its disk only when its folder is there without the project", async () => {
212+
const historyRoot = tempDir("hf-prune-root-");
213+
const emptied = await projectWithHistory(historyRoot);
214+
const folderGone = await projectWithHistory(historyRoot);
215+
const parentGone = await projectWithHistory(historyRoot, {
216+
projectDir: join(tempDir("hf-prune-parent-"), "project"),
217+
});
218+
const all = [emptied, folderGone, parentGone];
219+
await Promise.all(all.map(({ history }) => history.close()));
220+
for (const { id } of all) editRecord(historyRoot, id, ({ dev: _dev, ...record }) => record);
221+
rmSync(join(emptied.projectDir, ".hyperframes"), { recursive: true, force: true });
222+
// An unmounted disk leaves its mount point, so a missing folder under it may just be unplugged.
223+
rmSync(folderGone.projectDir, { recursive: true, force: true });
224+
rmSync(dirname(parentGone.projectDir), { recursive: true, force: true });
225+
226+
const later = Date.now() + KEEP_GONE_PROJECT_HISTORY_MS;
227+
expect(await prunedIds(historyRoot, { tempDir: historyRoot, now: later })).toEqual([
228+
emptied.id,
229+
]);
230+
});
231+
232+
it("prunes a history recorded without its disk under the temp folder once its project is gone", async () => {
233+
const historyRoot = tempDir("hf-prune-root-");
234+
const scratch = await projectWithHistory(historyRoot, {
235+
projectDir: join(tempDir("hf-prune-scratch-"), "project"),
236+
});
237+
await scratch.history.close();
238+
editRecord(historyRoot, scratch.id, ({ dev: _dev, ...record }) => record);
239+
rmSync(dirname(scratch.projectDir), { recursive: true, force: true });
240+
241+
expect(await prunedIds(historyRoot)).toEqual([scratch.id]);
242+
});
243+
244+
it("keeps a history whose project comes back while the prune takes its lock", async () => {
245+
const historyRoot = tempDir("hf-prune-root-");
246+
const back = await projectWithHistory(historyRoot);
247+
await back.history.close();
248+
const idFile = join(back.projectDir, ID_PATH);
249+
const id = readFileSync(idFile, "utf-8");
250+
rmSync(dirname(idFile), { recursive: true, force: true });
251+
const real = await vi.importActual<typeof import("./ownerLock.js")>("./ownerLock.js");
252+
vi.mocked(takeHistoryOwnership).mockImplementationOnce(async (home, waitMs) => {
253+
mkdirSync(dirname(idFile), { recursive: true });
254+
writeFileSync(idFile, id);
255+
return real.takeHistoryOwnership(home, waitMs);
256+
});
257+
258+
expect(await prunedIds(historyRoot)).toEqual([]);
259+
expect(existsSync(idFile)).toBe(true);
260+
expect(existsSync(join(historyRoot, back.id))).toBe(true);
261+
});
262+
263+
it("removes the history of a folder that now holds another project", async () => {
264+
const historyRoot = tempDir("hf-prune-root-");
265+
const replaced = await projectWithHistory(historyRoot);
266+
await replaced.history.close();
267+
writeFileSync(join(replaced.projectDir, ID_PATH), "00000000-0000-4000-8000-000000000000\n");
268+
269+
expect(await prunedIds(historyRoot)).toEqual([replaced.id]);
270+
});
271+
272+
it("keeps the history of a project whose id file cannot be read", async () => {
273+
const historyRoot = tempDir("hf-prune-root-");
274+
const unreadable = await projectWithHistory(historyRoot);
275+
await unreadable.history.close();
276+
mkdirSync(join(unreadable.projectDir, ".hyperframes"), { recursive: true });
277+
writeFileSync(join(unreadable.projectDir, ID_PATH), "not an id\n");
278+
279+
expect(await prunedIds(historyRoot)).toEqual([]);
280+
expect(existsSync(join(historyRoot, unreadable.id))).toBe(true);
281+
});
282+
});

0 commit comments

Comments
 (0)