Skip to content

Commit cea0bad

Browse files
fix(cli): resume a cloud download part when its connection drops (#5873)
A ranged part that loses its connection now re-requests only the bytes it is missing, up to 3 retries with backoff, then fails naming the byte range. A server that ignores Range still fails fast: there is nothing to resume from. The test server no longer closes idle keep-alive sockets, which closed a socket under a slow reader on Windows.
1 parent 8434945 commit cea0bad

2 files changed

Lines changed: 70 additions & 26 deletions

File tree

‎packages/cli/src/cloud/download.test.ts‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ interface RangeServer {
4040

4141
async function serveBytes(
4242
body: Buffer,
43-
options: { honourRange: boolean; failRangeStart?: number },
43+
// `drop` cuts the first `times` responses whose range covers byte `at`, right before it.
44+
options: { honourRange: boolean; drop?: { at: number; times: number } },
4445
): Promise<RangeServer> {
4546
const ranges: (string | undefined)[] = [];
4647
const server: Server = createServer((req, res) => {
@@ -60,12 +61,17 @@ async function serveBytes(
6061
"content-length": end - start + 1,
6162
"content-range": `bytes ${start}-${end}/${body.length}`,
6263
});
63-
if (start === options.failRangeStart) {
64-
res.write(body.subarray(start, start + 1024), () => res.destroy());
64+
const drop = options.drop;
65+
if (drop && drop.times > 0 && start <= drop.at && drop.at <= end) {
66+
drop.times--;
67+
res.write(body.subarray(start, drop.at), () => res.destroy());
6568
return;
6669
}
6770
res.end(body.subarray(start, end + 1));
6871
});
72+
// Windows completes a large loopback write at once, so the 5 s keep-alive idle clock
73+
// started before a slow reader finished and closed the socket under it.
74+
server.keepAliveTimeout = 0;
6975
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
7076
const { port } = server.address() as AddressInfo;
7177
return {
@@ -319,13 +325,26 @@ describe("cloud/download", () => {
319325
expect(server.ranges).toEqual([`bytes=0-${8 * MiB - 1}`]);
320326
});
321327

322-
it("keeps the previous output and leaves no staged file when one part fails", async () => {
328+
it("resumes a range whose connection drops mid-body from the bytes it already has", async () => {
323329
const body = patternBytes(50 * MiB);
324-
const server = await serve(body, { honourRange: true, failRangeStart: 8 * MiB });
330+
const at = 8 * MiB + 1024;
331+
const server = await serve(body, { honourRange: true, drop: { at, times: 2 } });
332+
const dest = join(dir, "out.mp4");
333+
const result = await downloadToFile(server.url, dest);
334+
expect(result.bytes).toBe(body.length);
335+
expect(readFileSync(dest).equals(body)).toBe(true);
336+
expect(server.ranges.filter((range) => range?.startsWith(`bytes=${at}-`))).toHaveLength(2);
337+
});
338+
339+
it("keeps the previous output and leaves no staged file when one part keeps dropping", async () => {
340+
const body = patternBytes(50 * MiB);
341+
const server = await serve(body, {
342+
honourRange: true,
343+
drop: { at: 8 * MiB + 1024, times: Infinity },
344+
});
325345
const dest = join(dir, "out.mp4");
326346
writeFileSync(dest, "previous render");
327-
await expect(downloadToFile(server.url, dest)).rejects.toThrow();
328-
expect(server.ranges.length).toBeGreaterThan(1);
347+
await expect(downloadToFile(server.url, dest)).rejects.toThrow(/dropped 4 times/);
329348
expect(readFileSync(dest, "utf8")).toBe("previous render");
330349
expect(readdirSync(dir)).toEqual(["out.mp4"]);
331350
});

‎packages/cli/src/cloud/download.ts‎

Lines changed: 44 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,15 @@
1616
import { chmodSync, mkdirSync, mkdtempSync, renameSync, rmSync, statSync } from "node:fs";
1717
import { open } from "node:fs/promises";
1818
import { dirname, join } from "node:path";
19+
import { setTimeout as sleep } from "node:timers/promises";
1920
import { resolveWritePath } from "@hyperframes/core/atomic-file";
2021

2122
// The first request asks for PART_BYTES; a larger file is split into up to
2223
// CONNECTIONS ranged GETs. A server that ignores Range answers 200 and is streamed.
2324
const PART_BYTES = 8 * 1024 * 1024;
2425
const CONNECTIONS = 8;
26+
// A ranged part whose connection drops resumes where it stopped, after 0.25, 0.5 and 1 s.
27+
const RETRIES = 3;
2528

2629
export interface DownloadOptions {
2730
signal?: AbortSignal;
@@ -79,39 +82,61 @@ export async function downloadToFile(
7982
let firstError: unknown;
8083
try {
8184
const file = await open(stagedFile, "wx");
82-
const writePart = async (res: Response, start: number, length: number | undefined) => {
83-
let written = 0;
85+
// Writes the body at `part.at` and advances it, so a dropped range resumes there.
86+
const pump = async (res: Response, part: { at: number }) => {
8487
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
85-
await file.write(chunk, 0, chunk.byteLength, start + written);
86-
written += chunk.byteLength;
88+
await file.write(chunk, 0, chunk.byteLength, part.at);
89+
part.at += chunk.byteLength;
8790
bytes += chunk.byteLength;
8891
signal.throwIfAborted();
8992
options.onProgress?.(bytes, total);
9093
}
91-
if (length !== undefined && written !== length) {
92-
throw new Error(
93-
`Truncated download: got ${written} bytes, expected ${length} (content-length). ` +
94-
`The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
95-
);
94+
};
95+
const fetchRange = async (start: number, end: number, res?: Response) => {
96+
const part = { at: start };
97+
for (let attempt = 0; ; attempt++) {
98+
try {
99+
res ??= await get(`bytes=${part.at}-${end}`);
100+
assertOk(res, url);
101+
if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);
102+
await pump(res, part);
103+
if (part.at > end) return;
104+
throw new Error(`connection closed at byte ${part.at}`);
105+
} catch (error) {
106+
if (signal.aborted) throw error;
107+
if (attempt === RETRIES) {
108+
throw new Error(
109+
`Failed to download ${url}: bytes ${start}-${end} dropped ${RETRIES + 1} times`,
110+
{ cause: error },
111+
);
112+
}
113+
res?.body?.cancel().catch(() => {});
114+
res = undefined;
115+
await sleep(250 * 2 ** attempt, undefined, { signal });
116+
}
96117
}
97118
};
98119
const parts: Promise<void>[] = [];
99120
if (ranged && total !== undefined) {
100-
parts.push(writePart(first, 0, Math.min(PART_BYTES, total)));
121+
parts.push(fetchRange(0, Math.min(PART_BYTES, total) - 1, first));
101122
const rest = total - PART_BYTES;
102123
const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / PART_BYTES)));
103124
for (let start = PART_BYTES; start < total; start += size) {
104-
const end = Math.min(start + size, total) - 1;
105-
parts.push(
106-
get(`bytes=${start}-${end}`).then((res) => {
107-
assertOk(res, url);
108-
if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`);
109-
return writePart(res, start, end - start + 1);
110-
}),
111-
);
125+
parts.push(fetchRange(start, Math.min(start + size, total) - 1));
112126
}
113127
} else {
114-
parts.push(writePart(first, 0, total));
128+
// No Range support means nothing to resume from: fail fast.
129+
const part = { at: 0 };
130+
parts.push(
131+
pump(first, part).then(() => {
132+
if (total !== undefined && part.at !== total) {
133+
throw new Error(
134+
`Truncated download: got ${part.at} bytes, expected ${total} (content-length). ` +
135+
`The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`,
136+
);
137+
}
138+
}),
139+
);
115140
}
116141
await Promise.all(
117142
parts.map((part) =>

0 commit comments

Comments
 (0)