|
16 | 16 | import { chmodSync, mkdirSync, mkdtempSync, renameSync, rmSync, statSync } from "node:fs"; |
17 | 17 | import { open } from "node:fs/promises"; |
18 | 18 | import { dirname, join } from "node:path"; |
| 19 | +import { setTimeout as sleep } from "node:timers/promises"; |
19 | 20 | import { resolveWritePath } from "@hyperframes/core/atomic-file"; |
20 | 21 |
|
21 | 22 | // The first request asks for PART_BYTES; a larger file is split into up to |
22 | 23 | // CONNECTIONS ranged GETs. A server that ignores Range answers 200 and is streamed. |
23 | 24 | const PART_BYTES = 8 * 1024 * 1024; |
24 | 25 | const CONNECTIONS = 8; |
| 26 | +// A ranged part whose connection drops resumes where it stopped, after 0.25, 0.5 and 1 s. |
| 27 | +const RETRIES = 3; |
25 | 28 |
|
26 | 29 | export interface DownloadOptions { |
27 | 30 | signal?: AbortSignal; |
@@ -79,39 +82,61 @@ export async function downloadToFile( |
79 | 82 | let firstError: unknown; |
80 | 83 | try { |
81 | 84 | const file = await open(stagedFile, "wx"); |
82 | | - const writePart = async (res: Response, start: number, length: number | undefined) => { |
83 | | - let written = 0; |
| 85 | + // Writes the body at `part.at` and advances it, so a dropped range resumes there. |
| 86 | + const pump = async (res: Response, part: { at: number }) => { |
84 | 87 | for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) { |
85 | | - await file.write(chunk, 0, chunk.byteLength, start + written); |
86 | | - written += chunk.byteLength; |
| 88 | + await file.write(chunk, 0, chunk.byteLength, part.at); |
| 89 | + part.at += chunk.byteLength; |
87 | 90 | bytes += chunk.byteLength; |
88 | 91 | signal.throwIfAborted(); |
89 | 92 | options.onProgress?.(bytes, total); |
90 | 93 | } |
91 | | - if (length !== undefined && written !== length) { |
92 | | - throw new Error( |
93 | | - `Truncated download: got ${written} bytes, expected ${length} (content-length). ` + |
94 | | - `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`, |
95 | | - ); |
| 94 | + }; |
| 95 | + const fetchRange = async (start: number, end: number, res?: Response) => { |
| 96 | + const part = { at: start }; |
| 97 | + for (let attempt = 0; ; attempt++) { |
| 98 | + try { |
| 99 | + res ??= await get(`bytes=${part.at}-${end}`); |
| 100 | + assertOk(res, url); |
| 101 | + if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`); |
| 102 | + await pump(res, part); |
| 103 | + if (part.at > end) return; |
| 104 | + throw new Error(`connection closed at byte ${part.at}`); |
| 105 | + } catch (error) { |
| 106 | + if (signal.aborted) throw error; |
| 107 | + if (attempt === RETRIES) { |
| 108 | + throw new Error( |
| 109 | + `Failed to download ${url}: bytes ${start}-${end} dropped ${RETRIES + 1} times`, |
| 110 | + { cause: error }, |
| 111 | + ); |
| 112 | + } |
| 113 | + res?.body?.cancel().catch(() => {}); |
| 114 | + res = undefined; |
| 115 | + await sleep(250 * 2 ** attempt, undefined, { signal }); |
| 116 | + } |
96 | 117 | } |
97 | 118 | }; |
98 | 119 | const parts: Promise<void>[] = []; |
99 | 120 | if (ranged && total !== undefined) { |
100 | | - parts.push(writePart(first, 0, Math.min(PART_BYTES, total))); |
| 121 | + parts.push(fetchRange(0, Math.min(PART_BYTES, total) - 1, first)); |
101 | 122 | const rest = total - PART_BYTES; |
102 | 123 | const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / PART_BYTES))); |
103 | 124 | for (let start = PART_BYTES; start < total; start += size) { |
104 | | - const end = Math.min(start + size, total) - 1; |
105 | | - parts.push( |
106 | | - get(`bytes=${start}-${end}`).then((res) => { |
107 | | - assertOk(res, url); |
108 | | - if (res.status !== 206) throw new Error(`Failed to download ${url}: range ignored`); |
109 | | - return writePart(res, start, end - start + 1); |
110 | | - }), |
111 | | - ); |
| 125 | + parts.push(fetchRange(start, Math.min(start + size, total) - 1)); |
112 | 126 | } |
113 | 127 | } else { |
114 | | - parts.push(writePart(first, 0, total)); |
| 128 | + // No Range support means nothing to resume from: fail fast. |
| 129 | + const part = { at: 0 }; |
| 130 | + parts.push( |
| 131 | + pump(first, part).then(() => { |
| 132 | + if (total !== undefined && part.at !== total) { |
| 133 | + throw new Error( |
| 134 | + `Truncated download: got ${part.at} bytes, expected ${total} (content-length). ` + |
| 135 | + `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.`, |
| 136 | + ); |
| 137 | + } |
| 138 | + }), |
| 139 | + ); |
115 | 140 | } |
116 | 141 | await Promise.all( |
117 | 142 | parts.map((part) => |
|
0 commit comments