1- import { createWriteStream , renameSync , unlinkSync } from "node:fs" ;
1+ import { randomUUID } from "node:crypto" ;
2+ import {
3+ createWriteStream ,
4+ existsSync ,
5+ mkdirSync ,
6+ readFileSync ,
7+ renameSync ,
8+ rmSync ,
9+ statSync ,
10+ unlinkSync ,
11+ utimesSync ,
12+ writeFileSync ,
13+ } from "node:fs" ;
214import { get as httpsGet } from "node:https" ;
315import { pipeline } from "node:stream/promises" ;
416
517const DEFAULT_DOWNLOAD_TIMEOUT_MS = 30_000 ;
18+ const LOCK_STALE_MS = 120_000 ;
19+ const LOCK_POLL_MS = 200 ;
20+ const LOCK_HEARTBEAT_MS = 15_000 ;
621
722export interface DownloadOptions {
823 /** Abort after this many milliseconds without network activity. */
924 timeoutMs ?: number ;
1025}
1126
27+ function isErrno ( err : unknown , code : string ) : boolean {
28+ return ( err as NodeJS . ErrnoException ) . code === code ;
29+ }
30+
1231function removePartialFile ( path : string ) : void {
1332 try {
1433 unlinkSync ( path ) ;
@@ -17,54 +36,145 @@ function removePartialFile(path: string): void {
1736 }
1837}
1938
39+ function sleep ( ms : number ) : Promise < void > {
40+ return new Promise ( ( resolve ) => setTimeout ( resolve , ms ) ) ;
41+ }
42+
43+ function readLockOwner ( lockDir : string ) : string | null {
44+ try {
45+ return readFileSync ( `${ lockDir } /owner` , "utf8" ) ;
46+ } catch ( err ) {
47+ if ( ! isErrno ( err , "ENOENT" ) ) throw err ;
48+ return null ;
49+ }
50+ }
51+
52+ function tryAcquireLock ( lockDir : string , owner : string ) : boolean {
53+ try {
54+ mkdirSync ( lockDir , { recursive : false } ) ;
55+ } catch ( err ) {
56+ if ( ! isErrno ( err , "EEXIST" ) ) throw err ;
57+ return false ;
58+ }
59+
60+ try {
61+ writeFileSync ( `${ lockDir } /owner` , owner ) ;
62+ return true ;
63+ } catch ( err ) {
64+ rmSync ( lockDir , { recursive : true , force : true } ) ;
65+ throw err ;
66+ }
67+ }
68+
69+ function reclaimStaleLock ( lockDir : string ) : void {
70+ const staleDir = `${ lockDir } .stale.${ randomUUID ( ) } ` ;
71+ try {
72+ if ( Date . now ( ) - statSync ( lockDir ) . mtimeMs > LOCK_STALE_MS ) {
73+ renameSync ( lockDir , staleDir ) ;
74+ rmSync ( staleDir , { recursive : true , force : true } ) ;
75+ }
76+ } catch ( err ) {
77+ if ( ! isErrno ( err , "ENOENT" ) ) throw err ;
78+ }
79+ }
80+
81+ async function withDownloadLock < T > ( dest : string , fn : ( waited : boolean ) => Promise < T > ) : Promise < T > {
82+ const lockDir = `${ dest } .download.lock` ;
83+ const owner = `${ process . pid } :${ randomUUID ( ) } ` ;
84+ let waited = false ;
85+
86+ for ( ; ; ) {
87+ if ( tryAcquireLock ( lockDir , owner ) ) break ;
88+ waited = true ;
89+ try {
90+ if ( Date . now ( ) - statSync ( lockDir ) . mtimeMs > LOCK_STALE_MS ) {
91+ reclaimStaleLock ( lockDir ) ;
92+ continue ;
93+ }
94+ } catch ( err ) {
95+ if ( ! isErrno ( err , "ENOENT" ) ) throw err ;
96+ }
97+ await sleep ( LOCK_POLL_MS ) ;
98+ }
99+
100+ const heartbeat = setInterval ( ( ) => {
101+ try {
102+ if ( readLockOwner ( lockDir ) !== owner ) return ;
103+ const now = new Date ( ) ;
104+ utimesSync ( lockDir , now , now ) ;
105+ } catch {
106+ // Best effort. A stale lock is still recoverable if this process dies.
107+ }
108+ } , LOCK_HEARTBEAT_MS ) ;
109+ heartbeat . unref ( ) ;
110+
111+ try {
112+ return await fn ( waited ) ;
113+ } finally {
114+ clearInterval ( heartbeat ) ;
115+ // A stale owner can resume after its directory was reclaimed. Only the
116+ // current owner may release this path, otherwise it could delete the
117+ // successor's live lock and reintroduce concurrent writes.
118+ if ( readLockOwner ( lockDir ) === owner ) {
119+ rmSync ( lockDir , { recursive : true , force : true } ) ;
120+ }
121+ }
122+ }
123+
20124/**
21125 * Download a file from a URL, following redirects.
22- * Uses atomic write (download to .tmp, rename on success) to prevent
23- * corrupt partial files from persisting in the cache on interruption .
126+ * Uses an owned cross-process lock plus atomic temp-file rename so concurrent
127+ * model warmups cannot corrupt a shared cache entry .
24128 */
25129export function downloadFile (
26130 url : string ,
27131 dest : string ,
28132 options : DownloadOptions = { } ,
29133) : Promise < void > {
30- const tmp = `${ dest } .tmp` ;
31- const timeoutMs = options . timeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS ;
32- return new Promise ( ( resolve , reject ) => {
33- const follow = ( u : string ) => {
34- const request = httpsGet ( u , ( res ) => {
35- if ( res . statusCode === 301 || res . statusCode === 302 ) {
36- const location = res . headers . location ;
37- if ( location ) {
134+ return withDownloadLock ( dest , async ( waited ) => {
135+ // Another process may have completed the same cache download while this
136+ // caller waited for the lock.
137+ if ( waited && existsSync ( dest ) ) return ;
138+
139+ const tmp = `${ dest } .${ process . pid } .${ randomUUID ( ) } .tmp` ;
140+ const timeoutMs = options . timeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS ;
141+ await new Promise < void > ( ( resolve , reject ) => {
142+ const follow = ( currentUrl : string ) => {
143+ const request = httpsGet ( currentUrl , ( res ) => {
144+ if ( res . statusCode === 301 || res . statusCode === 302 ) {
145+ const location = res . headers . location ;
146+ if ( location ) {
147+ res . resume ( ) ;
148+ follow ( location ) ;
149+ return ;
150+ }
151+ }
152+ if ( res . statusCode !== 200 ) {
38153 res . resume ( ) ;
39- follow ( location ) ;
154+ removePartialFile ( tmp ) ;
155+ reject ( new Error ( `Download failed: HTTP ${ res . statusCode } ` ) ) ;
40156 return ;
41157 }
42- }
43- if ( res . statusCode !== 200 ) {
44- res . resume ( ) ;
158+ const file = createWriteStream ( tmp ) ;
159+ pipeline ( res , file )
160+ . then ( ( ) => {
161+ renameSync ( tmp , dest ) ;
162+ resolve ( ) ;
163+ } )
164+ . catch ( ( err ) => {
165+ removePartialFile ( tmp ) ;
166+ reject ( err ) ;
167+ } ) ;
168+ } ) ;
169+ request . setTimeout ( timeoutMs , ( ) => {
170+ request . destroy ( new Error ( `Download timed out after ${ timeoutMs } ms` ) ) ;
171+ } ) ;
172+ request . on ( "error" , ( err ) => {
45173 removePartialFile ( tmp ) ;
46- reject ( new Error ( `Download failed: HTTP ${ res . statusCode } ` ) ) ;
47- return ;
48- }
49- const file = createWriteStream ( tmp ) ;
50- pipeline ( res , file )
51- . then ( ( ) => {
52- renameSync ( tmp , dest ) ;
53- resolve ( ) ;
54- } )
55- . catch ( ( err ) => {
56- removePartialFile ( tmp ) ;
57- reject ( err ) ;
58- } ) ;
59- } ) ;
60- request . setTimeout ( timeoutMs , ( ) => {
61- request . destroy ( new Error ( `Download timed out after ${ timeoutMs } ms` ) ) ;
62- } ) ;
63- request . on ( "error" , ( err ) => {
64- removePartialFile ( tmp ) ;
65- reject ( err ) ;
66- } ) ;
67- } ;
68- follow ( url ) ;
174+ reject ( err ) ;
175+ } ) ;
176+ } ;
177+ follow ( url ) ;
178+ } ) ;
69179 } ) ;
70180}
0 commit comments