Skip to content

Commit d393541

Browse files
authored
fix(cloud): preserve existing output when downloads fail (#3840)
* fix(cloud): preserve existing output when downloads fail * test(cloud): pin same-directory download staging
1 parent 962c954 commit d393541

3 files changed

Lines changed: 153 additions & 95 deletions

File tree

‎packages/cli/src/cloud/download.test.ts‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,14 @@
1-
import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
1+
import {
2+
chmodSync,
3+
lstatSync,
4+
mkdtempSync,
5+
readFileSync,
6+
readdirSync,
7+
rmSync,
8+
statSync,
9+
symlinkSync,
10+
writeFileSync,
11+
} from "node:fs";
212
import { tmpdir } from "node:os";
313
import { join } from "node:path";
414
import { afterEach, beforeEach, describe, expect, it } from "vitest";
@@ -109,4 +119,88 @@ describe("cloud/download", () => {
109119
).rejects.toThrow();
110120
expect(() => statSync(dest)).toThrow();
111121
});
122+
123+
async function expectFailedReplacement(dest: string, fetchImpl: typeof fetch, error: RegExp) {
124+
await expect(downloadToFile("https://example/x", dest, { fetchImpl })).rejects.toThrow(error);
125+
expect(readFileSync(dest, "utf8")).toBe("previous render");
126+
expect(readdirSync(dir)).toEqual(["out.mp4"]);
127+
}
128+
129+
it("preserves an existing output when the response is truncated", async () => {
130+
const dest = join(dir, "out.mp4");
131+
writeFileSync(dest, "previous render");
132+
await expectFailedReplacement(
133+
dest,
134+
makeBytesFetch(new Uint8Array([1]), { "content-length": "2" }),
135+
/Truncated download/,
136+
);
137+
});
138+
139+
it("keeps the old output visible until successful replacement and preserves its mode", async () => {
140+
const dest = join(dir, "out.mp4");
141+
writeFileSync(dest, "previous render");
142+
chmodSync(dest, 0o640);
143+
await downloadToFile("https://example/x", dest, {
144+
fetchImpl: makeBytesFetch(new Uint8Array([42])),
145+
onProgress: () => {
146+
expect(readFileSync(dest, "utf8")).toBe("previous render");
147+
expect(readdirSync(dir).filter((name) => name.startsWith(".hf-download-"))).toHaveLength(1);
148+
},
149+
});
150+
expect(readFileSync(dest)).toEqual(Buffer.from([42]));
151+
if (process.platform !== "win32") expect(statSync(dest).mode & 0o777).toBe(0o640);
152+
expect(readdirSync(dir)).toEqual(["out.mp4"]);
153+
});
154+
155+
it("preserves the previous output when cancelled during progress", async () => {
156+
const dest = join(dir, "out.mp4");
157+
writeFileSync(dest, "previous render");
158+
const controller = new AbortController();
159+
await expect(
160+
downloadToFile("https://example/x", dest, {
161+
fetchImpl: makeBytesFetch(new Uint8Array(1024 * 1024)),
162+
signal: controller.signal,
163+
onProgress: () => controller.abort(new Error("user cancelled")),
164+
}),
165+
).rejects.toThrow(/user cancelled/);
166+
expect(readFileSync(dest, "utf8")).toBe("previous render");
167+
expect(readdirSync(dir)).toEqual(["out.mp4"]);
168+
});
169+
170+
it("preserves the previous output when the response stream errors", async () => {
171+
const dest = join(dir, "out.mp4");
172+
writeFileSync(dest, "previous render");
173+
let sent = false;
174+
const body = new ReadableStream<Uint8Array>({
175+
pull(controller) {
176+
if (sent) controller.error(new Error("network interrupted"));
177+
else {
178+
sent = true;
179+
controller.enqueue(new Uint8Array([42]));
180+
}
181+
},
182+
});
183+
await expectFailedReplacement(
184+
dest,
185+
(async () => new Response(body)) as typeof fetch,
186+
/network interrupted/,
187+
);
188+
});
189+
190+
it.skipIf(process.platform === "win32")(
191+
"preserves existing and dangling output symlinks",
192+
async () => {
193+
const target = join(dir, "target.mp4");
194+
const alias = join(dir, "alias.mp4");
195+
symlinkSync("target.mp4", alias);
196+
for (const value of [1, 2]) {
197+
await downloadToFile("https://example/x", alias, {
198+
fetchImpl: makeBytesFetch(new Uint8Array([value])),
199+
});
200+
expect(lstatSync(alias).isSymbolicLink()).toBe(true);
201+
expect(readFileSync(target)).toEqual(Buffer.from([value]));
202+
}
203+
expect(readdirSync(dir).sort()).toEqual(["alias.mp4", "target.mp4"]);
204+
},
205+
);
112206
});

‎packages/cli/src/cloud/download.ts‎

Lines changed: 57 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,23 @@
99
* Failure behavior is "all or nothing": on any error we (1) listen for
1010
* stream errors / aborts so awaits resolve promptly instead of hanging,
1111
* (2) verify the final byte count matches `content-length` when the
12-
* server supplied one, and (3) `unlinkSync` the partial output so a
13-
* subsequent retry doesn't pick up a corrupted file.
12+
* server supplied one, and (3) discard the private staged output. An
13+
* existing destination is replaced only after the download succeeds.
1414
*/
1515

16-
import { createWriteStream, mkdirSync, unlinkSync } from "node:fs";
17-
import { dirname } from "node:path";
16+
import {
17+
chmodSync,
18+
createWriteStream,
19+
lstatSync,
20+
mkdirSync,
21+
mkdtempSync,
22+
readlinkSync,
23+
renameSync,
24+
rmSync,
25+
statSync,
26+
} from "node:fs";
27+
import { dirname, join, resolve } from "node:path";
28+
import { pipeline } from "node:stream/promises";
1829

1930
export interface DownloadOptions {
2031
signal?: AbortSignal;
@@ -31,9 +42,11 @@ export interface DownloadResult {
3142

3243
/**
3344
* Stream `url` into `destPath`. Creates the parent directory if needed,
34-
* truncates any existing file at the destination, and deletes the
35-
* partial output on any error so the caller never observes a corrupt
36-
* file at the returned path.
45+
* replaces an existing file only after the complete response is written,
46+
* and removes staged bytes on failure while preserving the old output.
47+
* Atomic replacement requires a writable parent directory and creates a new
48+
* inode: mode is retained, owner/group are not, and hard links keep old bytes.
49+
* A read-only file can be replaced when its parent permits the rename.
3750
*/
3851
// fallow-ignore-next-line complexity
3952
export async function downloadToFile(
@@ -56,96 +69,48 @@ export async function downloadToFile(
5669
const total = totalHeader ? Number.parseInt(totalHeader, 10) : undefined;
5770
const totalOpt = total !== undefined && Number.isFinite(total) ? total : undefined;
5871

59-
const file = createWriteStream(destPath);
72+
const destination = resolveDownloadDestination(destPath);
73+
const previous = statSync(destination, { throwIfNoEntry: false });
74+
const stage = mkdtempSync(join(dirname(destination), ".hf-download-"));
75+
const stagedFile = join(stage, "download");
6076
let bytes = 0;
61-
let errored = false;
6277
try {
63-
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
64-
if (options.signal?.aborted) {
65-
throw options.signal.reason instanceof Error
66-
? options.signal.reason
67-
: new Error("Download aborted");
68-
}
69-
bytes += chunk.byteLength;
70-
options.onProgress?.(bytes, totalOpt);
71-
if (!file.write(chunk)) {
72-
await waitForDrain(file, options.signal);
73-
}
74-
}
75-
if (totalOpt !== undefined && bytes !== totalOpt) {
76-
throw new Error(
77-
`Truncated download: got ${bytes} bytes, expected ${totalOpt} (content-length). ` +
78-
`The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
79-
);
80-
}
81-
} catch (err) {
82-
errored = true;
83-
throw err;
78+
await pipeline(
79+
async function* () {
80+
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
81+
bytes += chunk.byteLength;
82+
options.onProgress?.(bytes, totalOpt);
83+
yield chunk;
84+
}
85+
if (totalOpt !== undefined && bytes !== totalOpt) {
86+
throw new Error(
87+
`Truncated download: got ${bytes} bytes, expected ${totalOpt} (content-length). ` +
88+
`The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
89+
);
90+
}
91+
},
92+
createWriteStream(stagedFile, { flags: "wx" }),
93+
{ signal: options.signal },
94+
);
95+
options.signal?.throwIfAborted();
96+
if (previous?.isFile()) chmodSync(stagedFile, previous.mode & 0o777);
97+
renameSync(stagedFile, destination);
98+
} catch (error) {
99+
const reason = options.signal?.reason;
100+
if (options.signal?.aborted && reason instanceof Error) throw reason;
101+
throw error;
84102
} finally {
85-
await closeFile(file);
86-
if (errored) {
87-
// Don't let a partial file pose as the final artifact. Best-
88-
// effort unlink — if it fails (already gone, permission), we
89-
// re-throw the original error.
90-
try {
91-
unlinkSync(destPath);
92-
} catch {
93-
/* swallow */
94-
}
95-
}
103+
rmSync(stage, { recursive: true, force: true });
96104
}
97105
return { path: destPath, bytes };
98106
}
99107

100-
/**
101-
* Resolve when the write stream emits `drain`, or reject on `error` /
102-
* `close` / signal abort — avoids the hang from awaiting a one-shot
103-
* `drain` event that never fires because the stream tore down first.
104-
*/
105-
function waitForDrain(file: NodeJS.WritableStream, signal?: AbortSignal): Promise<void> {
106-
return new Promise<void>((resolve, reject) => {
107-
const cleanup = () => {
108-
file.off("drain", onDrain);
109-
file.off("error", onError);
110-
file.off("close", onClose);
111-
signal?.removeEventListener("abort", onAbort);
112-
};
113-
const onDrain = (): void => {
114-
cleanup();
115-
resolve();
116-
};
117-
const onError = (err: Error): void => {
118-
cleanup();
119-
reject(err);
120-
};
121-
const onClose = (): void => {
122-
cleanup();
123-
reject(new Error("write stream closed before drain"));
124-
};
125-
const onAbort = (): void => {
126-
cleanup();
127-
const reason = signal?.reason;
128-
reject(reason instanceof Error ? reason : new Error("Download aborted"));
129-
};
130-
file.once("drain", onDrain);
131-
file.once("error", onError);
132-
file.once("close", onClose);
133-
signal?.addEventListener("abort", onAbort, { once: true });
134-
});
135-
}
136-
137-
function closeFile(file: NodeJS.WritableStream): Promise<void> {
138-
return new Promise<void>((resolve) => {
139-
// Best-effort cleanup: any underlying failure has already been
140-
// surfaced as the original throw from the for-await loop. We
141-
// listen for `error` so a failing close (bad fd, late ENOSPC on
142-
// flush) doesn't leak an unhandled 'error' onto the stream, and
143-
// resolve either way so the finally block proceeds to unlinkSync.
144-
const done = (): void => {
145-
file.off("error", done);
146-
resolve();
147-
};
148-
file.once("error", done);
149-
file.end(() => done());
150-
});
108+
/** Preserve the existing behavior of writing through a caller-selected symlink. */
109+
function resolveDownloadDestination(destPath: string): string {
110+
let destination = resolve(destPath);
111+
for (let hops = 0; hops < 40; hops++) {
112+
if (!lstatSync(destination, { throwIfNoEntry: false })?.isSymbolicLink()) return destination;
113+
destination = resolve(dirname(destination), readlinkSync(destination));
114+
}
115+
throw new Error(`Too many symbolic links in download destination: ${destPath}`);
151116
}

‎packages/cli/src/commands/cloud/render.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -852,8 +852,7 @@ async function streamVideo(
852852
destPath: string,
853853
asJson: boolean,
854854
): Promise<{ bytes: number }> {
855-
// `downloadToFile` already creates the parent directory and cleans
856-
// up the partial file on error — no pre-mkdir needed here.
855+
// `downloadToFile` creates the parent directory and preserves existing output on failure.
857856
if (!asJson) {
858857
console.log("");
859858
console.log(`${c.accent("◆")} Downloading to ${c.accent(destPath)}`);

0 commit comments

Comments
 (0)