Skip to content

Commit f98218a

Browse files
fix(cli): end a stalled cloud upload and send it again (#7237)
* fix(cli): end a stalled cloud upload after 60 s and send it again up to three times * fix(cli): bound the in-memory project zip upload by progress like a file, watch the socket * refactor(cli): split the upload retry and stall watch into small functions
1 parent 600bfa4 commit f98218a

2 files changed

Lines changed: 189 additions & 15 deletions

File tree

‎packages/cli/src/cloud/upload.test.ts‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { tmpdir } from "node:os";
77
import { join } from "node:path";
88
import { HyperframesApiError } from "./_gen/client.js";
99
import type { HyperframesCloudClient } from "./_gen/client.js";
10-
import { uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "./upload.js";
10+
import { putTimings, uploadAssetViaDirectUpload, uploadZipViaDirectUpload } from "./upload.js";
1111

1212
function sha256Hex(bytes: Uint8Array): string {
1313
return createHash("sha256").update(bytes).digest("hex");
@@ -371,3 +371,84 @@ describe("uploadAssetViaDirectUpload from a file on disk", () => {
371371
});
372372
});
373373
});
374+
375+
describe("a PUT that stalls or fails on the network is sent again", () => {
376+
/** A server that answers its Nth request as `plan[n]` ("hang" never answers; a number is the status) and keeps
377+
* the last answer for any later one. */
378+
async function withPlan(
379+
plan: Array<"hang" | number>,
380+
run: (url: string, requests: () => number) => Promise<void>,
381+
): Promise<void> {
382+
let n = 0;
383+
const open: Array<() => void> = [];
384+
const server = createServer(async (req, res) => {
385+
const answer = plan[Math.min(n++, plan.length - 1)]!;
386+
for await (const _ of req) void _;
387+
if (answer === "hang") return void open.push(() => res.destroy());
388+
res.writeHead(answer).end(answer === 200 ? "" : "refused");
389+
});
390+
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
391+
try {
392+
await run(`http://127.0.0.1:${(server.address() as AddressInfo).port}/put`, () => n);
393+
} finally {
394+
for (const close of open) close();
395+
server.close();
396+
}
397+
}
398+
399+
const upload = (url: string, asBytes = false) => {
400+
const dir = mkdtempSync(join(tmpdir(), "hf-upload-retry-"));
401+
onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
402+
writeFileSync(join(dir, "clip.mp4"), "bytes");
403+
const was = { ...putTimings };
404+
Object.assign(putTimings, { stallMs: 150, backoffMs: 5 });
405+
onTestFinished(() => void Object.assign(putTimings, was));
406+
return uploadAssetViaDirectUpload({
407+
client: makeClient({
408+
createAssetUpload: vi.fn(async () => ({
409+
asset_id: "asset_xyz",
410+
upload_url: url,
411+
upload_headers: {},
412+
expires_in_seconds: 3600,
413+
max_bytes: 200 * 1024 * 1024,
414+
status: "pending_upload" as const,
415+
})),
416+
} as Partial<HyperframesCloudClient>),
417+
...(asBytes ? { bytes: Buffer.from("bytes") } : { path: join(dir, "clip.mp4"), size: 5 }),
418+
filename: "clip.mp4",
419+
contentType: "video/mp4",
420+
});
421+
};
422+
423+
it("ends a try that moves no data and sends the file again", async () => {
424+
await withPlan(["hang", 200], async (url, requests) => {
425+
await upload(url);
426+
expect(requests()).toBe(2);
427+
});
428+
});
429+
430+
it("bounds the project zip, which is sent from memory, the same way", async () => {
431+
await withPlan(["hang", 200], async (url, requests) => {
432+
await upload(url, true);
433+
expect(requests()).toBe(2);
434+
});
435+
});
436+
437+
it("sends it again after a server error, and gives up after the retries with the last words", async () => {
438+
await withPlan([503, 200], async (url, requests) => {
439+
await upload(url);
440+
expect(requests()).toBe(2);
441+
});
442+
await withPlan(["hang"], async (url, requests) => {
443+
await expect(upload(url)).rejects.toThrow(/Upload stalled/);
444+
expect(requests()).toBe(1 + putTimings.retries);
445+
});
446+
});
447+
448+
it("does not send it again when S3 refuses it", async () => {
449+
await withPlan([403], async (url, requests) => {
450+
await expect(upload(url)).rejects.toThrow(/PUT failed: 403/);
451+
expect(requests()).toBe(1);
452+
});
453+
});
454+
});

‎packages/cli/src/cloud/upload.ts‎

Lines changed: 107 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@
2525

2626
import { createHash } from "node:crypto";
2727
import { createReadStream } from "node:fs";
28-
import { type IncomingMessage, request as httpRequest } from "node:http";
28+
import { type ClientRequest, type IncomingMessage, request as httpRequest } from "node:http";
2929
import { request as httpsRequest } from "node:https";
30+
import { Readable } from "node:stream";
3031
import { pipeline } from "node:stream/promises";
3132
import type { HyperframesCloudClient } from "./_gen/client.js";
3233
import { HyperframesApiError } from "./_gen/client.js";
@@ -66,6 +67,38 @@ const COMPLETE_MAX_RETRIES = 5;
6667
const COMPLETE_RETRY_BASE_MS = 500;
6768
const CONTENT_TYPE_ZIP = "application/zip";
6869

70+
/** A failed or stalled PUT is sent again to the same presigned URL (it stays valid and a PUT replaces the object). */
71+
export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000 };
72+
73+
/** S3's answer to a PUT it refused; a refusal below 500 is final (an expired signature stays expired). */
74+
class PutRefusedError extends Error {
75+
constructor(
76+
readonly status: number,
77+
message: string,
78+
) {
79+
super(message);
80+
}
81+
}
82+
83+
/** An error no second try can change: a refusal below 500, or a file that is missing or unreadable. */
84+
function isFinal(err: unknown): boolean {
85+
const code = (err as { code?: unknown }).code;
86+
return (
87+
(err instanceof PutRefusedError && err.status < 500) || code === "ENOENT" || code === "EACCES"
88+
);
89+
}
90+
91+
async function putWithRetry(attempt: () => Promise<void>, signal?: AbortSignal): Promise<void> {
92+
for (let tried = 0; ; tried++) {
93+
try {
94+
return await attempt();
95+
} catch (err) {
96+
if (isFinal(err) || signal?.aborted || tried >= putTimings.retries) throw err;
97+
await sleep(putTimings.backoffMs * 2 ** tried, signal);
98+
}
99+
}
100+
}
101+
69102
function sha256Hex(bytes: Uint8Array): string {
70103
return createHash("sha256").update(bytes).digest("hex");
71104
}
@@ -76,8 +109,11 @@ async function sha256OfFile(path: string): Promise<string> {
76109
return hash.digest("hex");
77110
}
78111

79-
function sleep(ms: number): Promise<void> {
80-
return new Promise((r) => setTimeout(r, ms));
112+
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
113+
return new Promise((resolve) => {
114+
const timer = setTimeout(resolve, ms);
115+
signal?.addEventListener("abort", () => (clearTimeout(timer), resolve()), { once: true });
116+
});
81117
}
82118

83119
// upload_headers is generated as `Record<string, unknown>` from the
@@ -99,7 +135,7 @@ function normalizeUploadHeaders(raw: Record<string, unknown>): Record<string, st
99135
// the signature (S3 returns 403). Content-Type must match the declared
100136
// content_type from step 1 (also signed).
101137
async function putBytesToPresignedUrl(
102-
fetchImpl: typeof fetch,
138+
fetchImpl: typeof fetch | undefined,
103139
uploadUrl: string,
104140
uploadHeaders: Record<string, unknown>,
105141
body: Uint8Array | UploadFileSource,
@@ -110,7 +146,22 @@ async function putBytesToPresignedUrl(
110146
"content-type": contentType,
111147
...normalizeUploadHeaders(uploadHeaders),
112148
};
113-
if (!(body instanceof Uint8Array)) return putFileToPresignedUrl(uploadUrl, headers, body, signal);
149+
return putWithRetry(
150+
() =>
151+
fetchImpl && body instanceof Uint8Array
152+
? putBytesOnce(fetchImpl, uploadUrl, headers, body, signal)
153+
: putStreamToPresignedUrl(uploadUrl, headers, body, signal),
154+
signal,
155+
);
156+
}
157+
158+
async function putBytesOnce(
159+
fetchImpl: typeof fetch,
160+
uploadUrl: string,
161+
headers: Record<string, string>,
162+
body: Uint8Array,
163+
signal?: AbortSignal,
164+
): Promise<void> {
114165
// `Uint8Array<ArrayBufferLike>` is a valid `BodyInit` at runtime but
115166
// not strictly assignable per lib.dom.d.ts — cast rather than copy,
116167
// since a 200MB buffer copy would be wasteful.
@@ -122,48 +173,90 @@ async function putBytesToPresignedUrl(
122173
});
123174
if (!res.ok) {
124175
const detail = await res.text().catch(() => "");
125-
throw new Error(
176+
throw new PutRefusedError(
177+
res.status,
126178
`Direct upload PUT failed: ${res.status} ${res.statusText}${
127179
detail ? ` — ${detail.slice(0, 300)}` : ""
128180
}`,
129181
);
130182
}
131183
}
132184

133-
// fetch holds a streamed request body in memory whole, so a file is piped through node:http(s).
134-
async function putFileToPresignedUrl(
185+
// fetch holds a streamed request body in memory whole, so a file is piped through node:http(s). A buffer goes the
186+
// same way, in slices, so a stalled link ends its try the same way (a plain fetch has no progress to watch).
187+
const SLICE_BYTES = 64 * 1024;
188+
189+
function bodyStream(body: Uint8Array | UploadFileSource): Readable {
190+
if (!(body instanceof Uint8Array)) return createReadStream(body.path);
191+
return Readable.from(
192+
(function* () {
193+
for (let at = 0; at < body.byteLength; at += SLICE_BYTES)
194+
yield body.subarray(at, at + SLICE_BYTES);
195+
})(),
196+
);
197+
}
198+
199+
/** Ends `req` when no byte has been accepted by its socket for stallMs (its buffer is full and the link drains
200+
* nothing). Returns the function that stops the watch. */
201+
function watchStall(req: ClientRequest): () => void {
202+
let written = -1;
203+
let movedAt = Date.now();
204+
const watch = setInterval(
205+
() => {
206+
const now = req.socket?.bytesWritten ?? 0;
207+
if (now !== written) [written, movedAt] = [now, Date.now()];
208+
else if (Date.now() - movedAt >= putTimings.stallMs)
209+
req.destroy(new Error(`Upload stalled: no data sent for ${putTimings.stallMs / 1000} s`));
210+
},
211+
Math.min(1000, putTimings.stallMs / 3),
212+
);
213+
return () => clearInterval(watch);
214+
}
215+
216+
async function putStreamToPresignedUrl(
135217
uploadUrl: string,
136218
headers: Record<string, string>,
137-
file: UploadFileSource,
219+
body: Uint8Array | UploadFileSource,
138220
signal?: AbortSignal,
139221
): Promise<void> {
222+
const size = body instanceof Uint8Array ? body.byteLength : body.size;
140223
const send = new URL(uploadUrl).protocol === "https:" ? httpsRequest : httpRequest;
141224
const req = send(uploadUrl, {
142225
method: "PUT",
143226
signal,
144-
headers: { ...headers, "content-length": String(file.size) },
227+
headers: { ...headers, "content-length": String(size) },
145228
});
146229
const response = new Promise<IncomingMessage>((resolve, reject) => {
147230
req.once("response", resolve).once("error", reject);
148231
});
149-
const sent = pipeline(createReadStream(file.path), req);
232+
const stopWatching = watchStall(req);
233+
const source = bodyStream(body);
234+
const sent = pipeline(source, req);
150235
let res: IncomingMessage;
151236
try {
152237
res = await response;
153238
} catch (err) {
154239
await sent.catch(() => undefined);
155240
throw err;
241+
} finally {
242+
stopWatching();
156243
}
244+
await settle(res, sent);
245+
}
246+
247+
/** Reads S3's answer: a refusal is thrown with its words; a refused upload can close the connection mid-body, so the
248+
* write error is the status's to explain. */
249+
async function settle(res: IncomingMessage, sent: Promise<void>): Promise<void> {
157250
const status = res.statusCode ?? 0;
158251
const ok = status >= 200 && status < 300;
159-
// A refused upload can close the connection mid-body; its status explains that write error.
160252
await sent.catch((err: unknown) => {
161253
if (ok) throw err;
162254
});
163255
let detail = "";
164256
for await (const chunk of res) detail += String(chunk);
165257
if (!ok) {
166-
throw new Error(
258+
throw new PutRefusedError(
259+
status,
167260
`Direct upload PUT failed: ${status} ${res.statusMessage ?? ""}${
168261
detail ? ` — ${detail.slice(0, 300)}` : ""
169262
}`,
@@ -224,7 +317,7 @@ export async function uploadAssetViaDirectUpload(
224317
const size_bytes = body instanceof Uint8Array ? body.byteLength : body.size;
225318
const checksum_sha256 =
226319
body instanceof Uint8Array ? sha256Hex(body) : await sha256OfFile(body.path);
227-
const fetchImpl = opts.fetchImpl ?? fetch;
320+
const fetchImpl = opts.fetchImpl;
228321

229322
opts.onProgress?.({ phase: "initialize" });
230323
const initialize = await opts.client.createAssetUpload({

0 commit comments

Comments
 (0)