|
| 1 | +// @vitest-environment node |
| 2 | +import { fork } from "node:child_process"; |
| 3 | +import { createHash } from "node:crypto"; |
| 4 | +import { |
| 5 | + chmodSync, |
| 6 | + existsSync, |
| 7 | + mkdirSync, |
| 8 | + mkdtempSync, |
| 9 | + readdirSync, |
| 10 | + readFileSync, |
| 11 | + rmSync, |
| 12 | + statSync, |
| 13 | + writeFileSync, |
| 14 | +} from "node:fs"; |
| 15 | +import { tmpdir } from "node:os"; |
| 16 | +import { join } from "node:path"; |
| 17 | +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; |
| 18 | + |
| 19 | +vi.mock("node:fs/promises", async (importOriginal) => { |
| 20 | + const actual = await importOriginal<typeof import("node:fs/promises")>(); |
| 21 | + return { ...actual, writeFile: vi.fn(actual.writeFile) }; |
| 22 | +}); |
| 23 | + |
| 24 | +import * as fsp from "node:fs/promises"; |
| 25 | +import { blobPath } from "./blobStore.js"; |
| 26 | +import { START } from "./historyLog.js"; |
| 27 | +import { openProjectHistory } from "./projectHistory.js"; |
| 28 | + |
| 29 | +const { writeFile: realWriteFile } = |
| 30 | + await vi.importActual<typeof import("node:fs/promises")>("node:fs/promises"); |
| 31 | +const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); |
| 32 | +const temps: string[] = []; |
| 33 | +const tempDir = (prefix: string) => { |
| 34 | + const dir = mkdtempSync(join(tmpdir(), prefix)); |
| 35 | + temps.push(dir); |
| 36 | + return dir; |
| 37 | +}; |
| 38 | +const closes: (() => Promise<void>)[] = []; |
| 39 | + |
| 40 | +afterEach(async () => { |
| 41 | + vi.mocked(fsp.writeFile).mockImplementation(realWriteFile); |
| 42 | + for (const close of closes.splice(0)) await close(); |
| 43 | + for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); |
| 44 | +}); |
| 45 | + |
| 46 | +function projectOf(count: number, tag: string): string { |
| 47 | + const dir = tempDir("hf-first-copy-"); |
| 48 | + mkdirSync(join(dir, "parts")); |
| 49 | + writeFileSync(join(dir, "index.html"), `<html>${tag}</html>`); |
| 50 | + for (let i = 0; i < count; i++) writeFileSync(join(dir, "parts", `p${i}.js`), `// ${tag} ${i}\n`); |
| 51 | + return dir; |
| 52 | +} |
| 53 | +const homeIn = (historyRoot: string) => { |
| 54 | + const homes = readdirSync(historyRoot, { withFileTypes: true }).filter((e) => e.isDirectory()); |
| 55 | + return join(historyRoot, homes.find((e) => existsSync(join(historyRoot, e.name, "blobs")))!.name); |
| 56 | +}; |
| 57 | +async function open(projectDir: string, historyRoot: string) { |
| 58 | + const history = await openProjectHistory({ projectDir, historyRoot }); |
| 59 | + closes.push(() => history.close()); |
| 60 | + return history; |
| 61 | +} |
| 62 | +/** Every file of the project is in the baseline, and its blob holds exactly the file's bytes. */ |
| 63 | +function expectAllKept( |
| 64 | + baseline: Record<string, string> | null, |
| 65 | + projectDir: string, |
| 66 | + blobs: string, |
| 67 | + paths: string[], |
| 68 | +) { |
| 69 | + expect(Object.keys(baseline ?? {}).sort()).toEqual([...paths].sort()); |
| 70 | + for (const path of paths) { |
| 71 | + const bytes = readFileSync(join(projectDir, path)); |
| 72 | + expect(baseline![path]).toBe(sha256(bytes)); |
| 73 | + expect(readFileSync(blobPath(blobs, sha256(bytes))).equals(bytes)).toBe(true); |
| 74 | + } |
| 75 | +} |
| 76 | +const pathsOf = (count: number) => [ |
| 77 | + "index.html", |
| 78 | + ...Array.from({ length: count }, (_, i) => `parts/p${i}.js`), |
| 79 | +]; |
| 80 | + |
| 81 | +describe("a project's first history copy", () => { |
| 82 | + it("loses nothing when the process dies midway: the next open finishes it", async () => { |
| 83 | + const projectDir = projectOf(120, "dies"); |
| 84 | + const historyRoot = tempDir("hf-first-copy-root-"); |
| 85 | + const before = readFileSync(join(projectDir, "parts", "p7.js")); |
| 86 | + const died = await new Promise<NodeJS.Signals | number | null>((done) => { |
| 87 | + const fixture = join(import.meta.dirname, "__fixtures__", "dieMidFirstCopy.ts"); |
| 88 | + fork(fixture, [projectDir, historyRoot, "40"], { execArgv: ["--import", "tsx"] }).once( |
| 89 | + "exit", |
| 90 | + (code, signal) => done(signal ?? code), |
| 91 | + ); |
| 92 | + }); |
| 93 | + expect(died).toBe("SIGKILL"); |
| 94 | + const home = homeIn(historyRoot); |
| 95 | + expect(existsSync(join(home, "log.jsonl")), "the copy finished before the death").toBe(false); |
| 96 | + const leftovers = () => |
| 97 | + readdirSync(join(home, "blobs")).filter((name) => name.startsWith("incoming-")); |
| 98 | + expect(leftovers().length, "the death left no half-written copy").toBeGreaterThan(0); |
| 99 | + |
| 100 | + const history = await open(projectDir, historyRoot); |
| 101 | + |
| 102 | + expectAllKept(history.peek(START), projectDir, join(home, "blobs"), pathsOf(120)); |
| 103 | + expect(leftovers()).toEqual([]); |
| 104 | + for (const shard of readdirSync(join(home, "blobs"))) |
| 105 | + for (const hash of readdirSync(join(home, "blobs", shard))) |
| 106 | + expect(sha256(readFileSync(join(home, "blobs", shard, hash)))).toBe(hash); |
| 107 | + expect(readFileSync(join(projectDir, "parts", "p7.js")).equals(before)).toBe(true); |
| 108 | + }); |
| 109 | + |
| 110 | + it("keeps the same contents and permissions, small files and large, under a strict umask", async () => { |
| 111 | + const umask = process.umask(0o077); |
| 112 | + onTestFinished(() => void process.umask(umask)); |
| 113 | + const projectDir = projectOf(3, "kept"); |
| 114 | + writeFileSync(join(projectDir, "large.js"), "x".repeat(200 * 1024)); |
| 115 | + writeFileSync(join(projectDir, "same-a.js"), "same bytes"); |
| 116 | + writeFileSync(join(projectDir, "same-b.js"), "same bytes"); |
| 117 | + writeFileSync(join(projectDir, "run.sh"), "#!/bin/sh\n"); |
| 118 | + chmodSync(join(projectDir, "run.sh"), 0o755); |
| 119 | + const historyRoot = tempDir("hf-first-copy-root-"); |
| 120 | + |
| 121 | + const history = await open(projectDir, historyRoot); |
| 122 | + |
| 123 | + const blobs = join(homeIn(historyRoot), "blobs"); |
| 124 | + const paths = [...pathsOf(3), "large.js", "same-a.js", "same-b.js", "run.sh"]; |
| 125 | + expectAllKept(history.peek(START), projectDir, blobs, paths); |
| 126 | + const script = readFileSync(join(projectDir, "run.sh")); |
| 127 | + expect(statSync(blobPath(blobs, sha256(script))).mode & 0o111).toBe( |
| 128 | + statSync(join(projectDir, "run.sh")).mode & 0o111, |
| 129 | + ); |
| 130 | + }); |
| 131 | + |
| 132 | + it("copies a few files at a time across every project opening together", async () => { |
| 133 | + let writing = 0; |
| 134 | + let most = 0; |
| 135 | + vi.mocked(fsp.writeFile).mockImplementation(async (...args) => { |
| 136 | + const counted = String(args[0]).includes("incoming-"); |
| 137 | + if (counted) most = Math.max(most, ++writing); |
| 138 | + await realWriteFile(...args); |
| 139 | + if (counted) writing -= 1; |
| 140 | + }); |
| 141 | + const historyRoot = tempDir("hf-first-copy-root-"); |
| 142 | + |
| 143 | + await Promise.all(["a", "b", "c"].map((tag) => open(projectOf(30, tag), historyRoot))); |
| 144 | + |
| 145 | + expect(most).toBeGreaterThan(1); |
| 146 | + expect(most).toBeLessThanOrEqual(4); |
| 147 | + }); |
| 148 | +}); |
0 commit comments