1313 * existing destination is replaced only after the download succeeds.
1414 */
1515
16- import {
17- chmodSync ,
18- createWriteStream ,
19- mkdirSync ,
20- mkdtempSync ,
21- renameSync ,
22- rmSync ,
23- statSync ,
24- } from "node:fs" ;
16+ import { chmodSync , mkdirSync , mkdtempSync , renameSync , rmSync , statSync } from "node:fs" ;
17+ import { open } from "node:fs/promises" ;
2518import { dirname , join } from "node:path" ;
2619import { resolveWritePath } from "@hyperframes/core/atomic-file" ;
27- import { pipeline } from "node:stream/promises" ;
20+
21+ // The first request asks for PART_BYTES; a larger file is split into up to
22+ // CONNECTIONS ranged GETs. A server that ignores Range answers 200 and is streamed.
23+ const PART_BYTES = 8 * 1024 * 1024 ;
24+ const CONNECTIONS = 8 ;
2825
2926export interface DownloadOptions {
3027 signal ?: AbortSignal ;
@@ -54,44 +51,78 @@ export async function downloadToFile(
5451 options : DownloadOptions = { } ,
5552) : Promise < DownloadResult > {
5653 const fetchImpl = options . fetchImpl ?? fetch ;
57- const res = await fetchImpl ( url , { signal : options . signal } ) ;
58- if ( ! res . ok ) {
59- throw new Error ( `Failed to download ${ url } : HTTP ${ res . status } ${ res . statusText } ` ) ;
60- }
61- if ( ! res . body ) {
62- throw new Error ( `Failed to download ${ url } : empty response body` ) ;
54+ // Cancels the sibling parts when one fails.
55+ const failFast = new AbortController ( ) ;
56+ const signal = options . signal
57+ ? AbortSignal . any ( [ options . signal , failFast . signal ] )
58+ : failFast . signal ;
59+ const get = async ( range ?: string ) => fetchImpl ( url , { signal, headers : range ? { range } : { } } ) ;
60+
61+ let first = await get ( `bytes=0-${ PART_BYTES - 1 } ` ) ;
62+ // 416 (empty file) or a 206 without a total cannot be split: fetch it whole.
63+ if ( first . status === 416 || ( first . status === 206 && contentRangeTotal ( first ) === undefined ) ) {
64+ await first . body ?. cancel ( ) ;
65+ first = await get ( ) ;
6366 }
67+ assertOk ( first , url ) ;
68+ const ranged = first . status === 206 ;
69+ const total = ranged ? contentRangeTotal ( first ) : contentLength ( first ) ;
6470
6571 mkdirSync ( dirname ( destPath ) , { recursive : true } ) ;
6672
67- const totalHeader = res . headers . get ( "content-length" ) ;
68- const total = totalHeader ? Number . parseInt ( totalHeader , 10 ) : undefined ;
69- const totalOpt = total !== undefined && Number . isFinite ( total ) ? total : undefined ;
70-
7173 // Writes through a caller-selected symlink, as before.
7274 const destination = resolveWritePath ( destPath ) ;
7375 const previous = statSync ( destination , { throwIfNoEntry : false } ) ;
7476 const stage = mkdtempSync ( join ( dirname ( destination ) , ".hf-download-" ) ) ;
7577 const stagedFile = join ( stage , "download" ) ;
7678 let bytes = 0 ;
79+ let firstError : unknown ;
7780 try {
78- await pipeline (
79- async function * ( ) {
80- for await ( const chunk of res . body as unknown as AsyncIterable < Uint8Array > ) {
81- bytes += chunk . byteLength ;
82- options . onProgress ?.( bytes , totalOpt ) ;
83- yield chunk ;
84- }
85- if ( totalOpt !== undefined && bytes !== totalOpt ) {
86- throw new Error (
87- `Truncated download: got ${ bytes } bytes, expected ${ totalOpt } (content-length). ` +
88- `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.` ,
89- ) ;
90- }
91- } ,
92- createWriteStream ( stagedFile , { flags : "wx" } ) ,
93- { signal : options . signal } ,
81+ const file = await open ( stagedFile , "wx" ) ;
82+ const writePart = async ( res : Response , start : number , length : number | undefined ) => {
83+ let written = 0 ;
84+ for await ( const chunk of res . body as unknown as AsyncIterable < Uint8Array > ) {
85+ await file . write ( chunk , 0 , chunk . byteLength , start + written ) ;
86+ written += chunk . byteLength ;
87+ bytes += chunk . byteLength ;
88+ signal . throwIfAborted ( ) ;
89+ options . onProgress ?.( bytes , total ) ;
90+ }
91+ if ( length !== undefined && written !== length ) {
92+ throw new Error (
93+ `Truncated download: got ${ written } bytes, expected ${ length } (content-length). ` +
94+ `The presigned URL may have expired mid-transfer — refetch via \`hyperframes cloud get\`.` ,
95+ ) ;
96+ }
97+ } ;
98+ const parts : Promise < void > [ ] = [ ] ;
99+ if ( ranged && total !== undefined ) {
100+ parts . push ( writePart ( first , 0 , Math . min ( PART_BYTES , total ) ) ) ;
101+ const rest = total - PART_BYTES ;
102+ const size = Math . ceil ( rest / Math . min ( CONNECTIONS - 1 , Math . ceil ( rest / PART_BYTES ) ) ) ;
103+ for ( let start = PART_BYTES ; start < total ; start += size ) {
104+ const end = Math . min ( start + size , total ) - 1 ;
105+ parts . push (
106+ get ( `bytes=${ start } -${ end } ` ) . then ( ( res ) => {
107+ assertOk ( res , url ) ;
108+ if ( res . status !== 206 ) throw new Error ( `Failed to download ${ url } : range ignored` ) ;
109+ return writePart ( res , start , end - start + 1 ) ;
110+ } ) ,
111+ ) ;
112+ }
113+ } else {
114+ parts . push ( writePart ( first , 0 , total ) ) ;
115+ }
116+ await Promise . all (
117+ parts . map ( ( part ) =>
118+ part . catch ( ( error : unknown ) => {
119+ firstError ??= error ;
120+ failFast . abort ( error ) ;
121+ } ) ,
122+ ) ,
94123 ) ;
124+ await file . close ( ) ;
125+ if ( firstError !== undefined ) throw firstError ;
95126 options . signal ?. throwIfAborted ( ) ;
96127 if ( previous ?. isFile ( ) ) chmodSync ( stagedFile , previous . mode & 0o777 ) ;
97128 renameSync ( stagedFile , destination ) ;
@@ -104,3 +135,22 @@ export async function downloadToFile(
104135 }
105136 return { path : destPath , bytes } ;
106137}
138+
139+ function assertOk ( res : Response , url : string ) : void {
140+ if ( ! res . ok ) {
141+ throw new Error ( `Failed to download ${ url } : HTTP ${ res . status } ${ res . statusText } ` ) ;
142+ }
143+ if ( ! res . body ) {
144+ throw new Error ( `Failed to download ${ url } : empty response body` ) ;
145+ }
146+ }
147+
148+ function contentLength ( res : Response ) : number | undefined {
149+ const total = Number . parseInt ( res . headers . get ( "content-length" ) ?? "" , 10 ) ;
150+ return Number . isFinite ( total ) ? total : undefined ;
151+ }
152+
153+ function contentRangeTotal ( res : Response ) : number | undefined {
154+ const match = / ^ b y t e s \d + - \d + \/ ( \d + ) $ / . exec ( res . headers . get ( "content-range" ) ?? "" ) ;
155+ return match ? Number ( match [ 1 ] ) : undefined ;
156+ }
0 commit comments