Skip to content

Commit d5339c9

Browse files
committed
fix(cli): serialize concurrent model downloads
Use an owned filesystem lock and unique temporary files so concurrent processes cannot race. Reclaim stale locks safely and cover same-destination deduplication with a regression test.
1 parent 75eedf5 commit d5339c9

2 files changed

Lines changed: 183 additions & 43 deletions

File tree

‎packages/cli/src/utils/download.test.ts‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import { EventEmitter } from "node:events";
2-
import { existsSync, mkdtempSync, rmSync } from "node:fs";
2+
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
3+
import type { ClientRequest, IncomingMessage } from "node:http";
4+
import { get as httpsGet } from "node:https";
35
import { tmpdir } from "node:os";
46
import { join } from "node:path";
5-
import { PassThrough } from "node:stream";
6-
import { get as httpsGet } from "node:https";
7-
import type { ClientRequest, IncomingMessage } from "node:http";
7+
import { PassThrough, Readable } from "node:stream";
88
import { afterEach, describe, expect, it, vi } from "vitest";
99
import { downloadFile } from "./download.js";
1010

@@ -60,7 +60,37 @@ describe("downloadFile", () => {
6060
await expect(
6161
downloadFile("https://example.test/model.onnx", dest, { timeoutMs: 10 }),
6262
).rejects.toThrow("Download timed out after 10ms");
63-
expect(existsSync(`${dest}.tmp`)).toBe(false);
6463
expect(existsSync(dest)).toBe(false);
64+
expect(existsSync(`${dest}.download.lock`)).toBe(false);
65+
});
66+
67+
it("serializes concurrent downloads to the same cache destination", async () => {
68+
const dir = mkdtempSync(join(tmpdir(), "hyperframes-download-lock-"));
69+
tempDirs.push(dir);
70+
const dest = join(dir, "model.bin");
71+
let requests = 0;
72+
73+
mockGet.mockImplementation(((_url: string, callback: (response: IncomingMessage) => void) => {
74+
requests += 1;
75+
const response = Readable.from([Buffer.from("complete-model")]) as Readable & {
76+
statusCode: number;
77+
headers: Record<string, string>;
78+
};
79+
response.statusCode = 200;
80+
response.headers = {};
81+
const request = new EventEmitter() as ClientRequest;
82+
request.setTimeout = vi.fn(() => request);
83+
queueMicrotask(() => callback(response as unknown as IncomingMessage));
84+
return request;
85+
}) as typeof httpsGet);
86+
87+
await Promise.all([
88+
downloadFile("https://example.test/model.bin", dest),
89+
downloadFile("https://example.test/model.bin", dest),
90+
]);
91+
92+
expect(requests).toBe(1);
93+
expect(readFileSync(dest, "utf8")).toBe("complete-model");
94+
expect(existsSync(`${dest}.download.lock`)).toBe(false);
6595
});
6696
});

‎packages/cli/src/utils/download.ts‎

Lines changed: 148 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,33 @@
1-
import { createWriteStream, renameSync, unlinkSync } from "node:fs";
1+
import { randomUUID } from "node:crypto";
2+
import {
3+
createWriteStream,
4+
existsSync,
5+
mkdirSync,
6+
readFileSync,
7+
renameSync,
8+
rmSync,
9+
statSync,
10+
unlinkSync,
11+
utimesSync,
12+
writeFileSync,
13+
} from "node:fs";
214
import { get as httpsGet } from "node:https";
315
import { pipeline } from "node:stream/promises";
416

517
const DEFAULT_DOWNLOAD_TIMEOUT_MS = 30_000;
18+
const LOCK_STALE_MS = 120_000;
19+
const LOCK_POLL_MS = 200;
20+
const LOCK_HEARTBEAT_MS = 15_000;
621

722
export interface DownloadOptions {
823
/** Abort after this many milliseconds without network activity. */
924
timeoutMs?: number;
1025
}
1126

27+
function isErrno(err: unknown, code: string): boolean {
28+
return (err as NodeJS.ErrnoException).code === code;
29+
}
30+
1231
function removePartialFile(path: string): void {
1332
try {
1433
unlinkSync(path);
@@ -17,54 +36,145 @@ function removePartialFile(path: string): void {
1736
}
1837
}
1938

39+
function sleep(ms: number): Promise<void> {
40+
return new Promise((resolve) => setTimeout(resolve, ms));
41+
}
42+
43+
function readLockOwner(lockDir: string): string | null {
44+
try {
45+
return readFileSync(`${lockDir}/owner`, "utf8");
46+
} catch (err) {
47+
if (!isErrno(err, "ENOENT")) throw err;
48+
return null;
49+
}
50+
}
51+
52+
function tryAcquireLock(lockDir: string, owner: string): boolean {
53+
try {
54+
mkdirSync(lockDir, { recursive: false });
55+
} catch (err) {
56+
if (!isErrno(err, "EEXIST")) throw err;
57+
return false;
58+
}
59+
60+
try {
61+
writeFileSync(`${lockDir}/owner`, owner);
62+
return true;
63+
} catch (err) {
64+
rmSync(lockDir, { recursive: true, force: true });
65+
throw err;
66+
}
67+
}
68+
69+
function reclaimStaleLock(lockDir: string): void {
70+
const staleDir = `${lockDir}.stale.${randomUUID()}`;
71+
try {
72+
if (Date.now() - statSync(lockDir).mtimeMs > LOCK_STALE_MS) {
73+
renameSync(lockDir, staleDir);
74+
rmSync(staleDir, { recursive: true, force: true });
75+
}
76+
} catch (err) {
77+
if (!isErrno(err, "ENOENT")) throw err;
78+
}
79+
}
80+
81+
async function withDownloadLock<T>(dest: string, fn: (waited: boolean) => Promise<T>): Promise<T> {
82+
const lockDir = `${dest}.download.lock`;
83+
const owner = `${process.pid}:${randomUUID()}`;
84+
let waited = false;
85+
86+
for (;;) {
87+
if (tryAcquireLock(lockDir, owner)) break;
88+
waited = true;
89+
try {
90+
if (Date.now() - statSync(lockDir).mtimeMs > LOCK_STALE_MS) {
91+
reclaimStaleLock(lockDir);
92+
continue;
93+
}
94+
} catch (err) {
95+
if (!isErrno(err, "ENOENT")) throw err;
96+
}
97+
await sleep(LOCK_POLL_MS);
98+
}
99+
100+
const heartbeat = setInterval(() => {
101+
try {
102+
if (readLockOwner(lockDir) !== owner) return;
103+
const now = new Date();
104+
utimesSync(lockDir, now, now);
105+
} catch {
106+
// Best effort. A stale lock is still recoverable if this process dies.
107+
}
108+
}, LOCK_HEARTBEAT_MS);
109+
heartbeat.unref();
110+
111+
try {
112+
return await fn(waited);
113+
} finally {
114+
clearInterval(heartbeat);
115+
// A stale owner can resume after its directory was reclaimed. Only the
116+
// current owner may release this path, otherwise it could delete the
117+
// successor's live lock and reintroduce concurrent writes.
118+
if (readLockOwner(lockDir) === owner) {
119+
rmSync(lockDir, { recursive: true, force: true });
120+
}
121+
}
122+
}
123+
20124
/**
21125
* Download a file from a URL, following redirects.
22-
* Uses atomic write (download to .tmp, rename on success) to prevent
23-
* corrupt partial files from persisting in the cache on interruption.
126+
* Uses an owned cross-process lock plus atomic temp-file rename so concurrent
127+
* model warmups cannot corrupt a shared cache entry.
24128
*/
25129
export function downloadFile(
26130
url: string,
27131
dest: string,
28132
options: DownloadOptions = {},
29133
): Promise<void> {
30-
const tmp = `${dest}.tmp`;
31-
const timeoutMs = options.timeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS;
32-
return new Promise((resolve, reject) => {
33-
const follow = (u: string) => {
34-
const request = httpsGet(u, (res) => {
35-
if (res.statusCode === 301 || res.statusCode === 302) {
36-
const location = res.headers.location;
37-
if (location) {
134+
return withDownloadLock(dest, async (waited) => {
135+
// Another process may have completed the same cache download while this
136+
// caller waited for the lock.
137+
if (waited && existsSync(dest)) return;
138+
139+
const tmp = `${dest}.${process.pid}.${randomUUID()}.tmp`;
140+
const timeoutMs = options.timeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS;
141+
await new Promise<void>((resolve, reject) => {
142+
const follow = (currentUrl: string) => {
143+
const request = httpsGet(currentUrl, (res) => {
144+
if (res.statusCode === 301 || res.statusCode === 302) {
145+
const location = res.headers.location;
146+
if (location) {
147+
res.resume();
148+
follow(location);
149+
return;
150+
}
151+
}
152+
if (res.statusCode !== 200) {
38153
res.resume();
39-
follow(location);
154+
removePartialFile(tmp);
155+
reject(new Error(`Download failed: HTTP ${res.statusCode}`));
40156
return;
41157
}
42-
}
43-
if (res.statusCode !== 200) {
44-
res.resume();
158+
const file = createWriteStream(tmp);
159+
pipeline(res, file)
160+
.then(() => {
161+
renameSync(tmp, dest);
162+
resolve();
163+
})
164+
.catch((err) => {
165+
removePartialFile(tmp);
166+
reject(err);
167+
});
168+
});
169+
request.setTimeout(timeoutMs, () => {
170+
request.destroy(new Error(`Download timed out after ${timeoutMs}ms`));
171+
});
172+
request.on("error", (err) => {
45173
removePartialFile(tmp);
46-
reject(new Error(`Download failed: HTTP ${res.statusCode}`));
47-
return;
48-
}
49-
const file = createWriteStream(tmp);
50-
pipeline(res, file)
51-
.then(() => {
52-
renameSync(tmp, dest);
53-
resolve();
54-
})
55-
.catch((err) => {
56-
removePartialFile(tmp);
57-
reject(err);
58-
});
59-
});
60-
request.setTimeout(timeoutMs, () => {
61-
request.destroy(new Error(`Download timed out after ${timeoutMs}ms`));
62-
});
63-
request.on("error", (err) => {
64-
removePartialFile(tmp);
65-
reject(err);
66-
});
67-
};
68-
follow(url);
174+
reject(err);
175+
});
176+
};
177+
follow(url);
178+
});
69179
});
70180
}

0 commit comments

Comments
 (0)