Skip to content

Commit 555c788

Browse files
fix(cli): cloud upload tests stop flaking and two cuts at the same age count as one (heygen-com#7352)
* refactor(cli): split the upload retry decision and the test drain to keep complexity down * test(cli): drain the slow upload test server at a fixed rate whatever the chunk size * fix(cli): two cuts at the same age are the same cut whatever bytes each got through * fix(cli): sample upload progress on its own interval so a short try still reports its bytes * style(cli): format the cloud upload files
1 parent 9058fd1 commit 555c788

2 files changed

Lines changed: 62 additions & 34 deletions

File tree

‎packages/cli/src/cloud/upload.test.ts‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { describe, it, expect, onTestFinished, vi } from "vitest";
22
import { createHash } from "node:crypto";
33
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
4-
import { createServer, type IncomingHttpHeaders } from "node:http";
4+
import { createServer, type IncomingHttpHeaders, type IncomingMessage } from "node:http";
55
import type { AddressInfo } from "node:net";
66
import { tmpdir } from "node:os";
77
import { join } from "node:path";
@@ -377,7 +377,26 @@ describe("uploadAssetViaDirectUpload from a file on disk", () => {
377377
});
378378
});
379379

380-
const BIG_BYTES = 32 * 1024 * 1024;
380+
const BIG_BYTES = 16 * 1024 * 1024;
381+
382+
/** Reads the body; "cut" and "slow" take 8 MB/s, and "cut" drops the connection at 400 ms. */
383+
async function drain(req: IncomingMessage, answer: unknown): Promise<"read" | "dropped"> {
384+
const slow = answer === "cut" || answer === "slow";
385+
const born = Date.now();
386+
try {
387+
for await (const chunk of req as AsyncIterable<Buffer>) {
388+
if (!slow) continue;
389+
await new Promise((resolve) => setTimeout(resolve, Math.ceil(chunk.length / 8192)));
390+
if (answer === "cut" && Date.now() - born >= 400) {
391+
req.socket.destroy();
392+
return "dropped";
393+
}
394+
}
395+
} catch {
396+
return "dropped";
397+
}
398+
return "read";
399+
}
381400

382401
describe("a PUT that stalls or fails on the network is sent again", () => {
383402
/** Answers its Nth request as `plan[n]`, the last one for any later: "hang" never answers, "cut" drops the
@@ -390,17 +409,7 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
390409
const open: Array<() => void> = [];
391410
const server = createServer(async (req, res) => {
392411
const answer = plan[Math.min(n++, plan.length - 1)]!;
393-
const born = Date.now();
394-
try {
395-
for await (const _ of req) {
396-
void _;
397-
if (answer !== "cut" && answer !== "slow") continue;
398-
await new Promise((resolve) => setTimeout(resolve, 1));
399-
if (answer === "cut" && Date.now() - born >= 400) return void req.socket.destroy();
400-
}
401-
} catch {
402-
return;
403-
}
412+
if ((await drain(req, answer)) === "dropped") return;
404413
if (answer === "hang") return void open.push(() => res.destroy());
405414
if (answer === "cut" || answer === "slow") return void res.writeHead(200).end();
406415
res.writeHead(answer).end(answer === 200 ? "" : "refused");
@@ -424,7 +433,12 @@ describe("a PUT that stalls or fails on the network is sent again", () => {
424433
const content = big ? Buffer.alloc(BIG_BYTES, 1) : Buffer.from("bytes");
425434
writeFileSync(join(dir, "clip.mp4"), content);
426435
const was = { ...putTimings };
427-
Object.assign(putTimings, { stallMs: 150, backoffMs: 5, progressMs: 50 });
436+
Object.assign(putTimings, {
437+
stallMs: big ? 5000 : 150,
438+
backoffMs: 5,
439+
progressMs: 50,
440+
sampleMs: 50,
441+
});
428442
onTestFinished(() => void Object.assign(putTimings, was));
429443
return uploadAssetViaDirectUpload({
430444
client: makeClient({

‎packages/cli/src/cloud/upload.ts‎

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,13 @@ const COMPLETE_RETRY_BASE_MS = 500;
7676
const CONTENT_TYPE_ZIP = "application/zip";
7777

7878
/** A failed or stalled PUT is sent again to the same presigned URL (it stays valid and a PUT replaces the object). */
79-
export const putTimings = { stallMs: 60_000, retries: 3, backoffMs: 1000, progressMs: 10_000 };
79+
export const putTimings = {
80+
stallMs: 60_000,
81+
retries: 3,
82+
backoffMs: 1000,
83+
progressMs: 10_000,
84+
sampleMs: 1000,
85+
};
8086

8187
/** What one try of the PUT got through, read by the retry loop and the progress lines. */
8288
interface PutTry {
@@ -92,10 +98,13 @@ function isConnectionCut(err: unknown): boolean {
9298
return (typeof code === "string" && CUT_CODES.has(code)) || /socket hang up/.test(String(err));
9399
}
94100

95-
/** The same cut twice: a connection that dies at the same age with the same bytes sent dies the same way again. */
96-
function isSameCut(before: PutTry & { ageMs: number }, now: PutTry & { ageMs: number }): boolean {
97-
const near = (a: number, b: number) => Math.abs(a - b) <= 0.2 * Math.max(a, b);
98-
return before.sent > 0 && near(before.ageMs, now.ageMs) && near(before.sent, now.sent);
101+
/** The same cut twice: a connection that dies at the same age dies there again, whatever it got through. */
102+
function isSameCut(before: EndedTry, now: EndedTry): boolean {
103+
return (
104+
before.sent > 0 &&
105+
now.sent > 0 &&
106+
Math.abs(before.ageMs - now.ageMs) <= 0.2 * Math.max(before.ageMs, now.ageMs)
107+
);
99108
}
100109

101110
function seconds(ms: number): string {
@@ -132,32 +141,37 @@ function isFinal(err: unknown): boolean {
132141
);
133142
}
134143

144+
type EndedTry = PutTry & { ageMs: number; cut: boolean };
145+
146+
function repeatedCut(before: EndedTry | undefined, now: EndedTry, err: unknown, total: number) {
147+
if (!before?.cut || !now.cut || !isSameCut(before, now)) return undefined;
148+
const code = (err as { code?: string }).code;
149+
return Object.assign(
150+
new Error(
151+
`The connection was cut at ${seconds(now.ageMs)} on two tries in a row, after ${Math.floor((100 * now.sent) / total)}% of the file was sent (${code ?? "connection reset"}). Every try restarts the file, so this link cannot finish it before the cut: use a faster connection or send less media.`,
152+
{ cause: err },
153+
),
154+
{ code },
155+
);
156+
}
157+
135158
async function putWithRetry(
136159
attempt: (tried: PutTry) => Promise<void>,
137160
total: number,
138161
signal?: AbortSignal,
139162
): Promise<void> {
140-
let before: (PutTry & { ageMs: number }) | undefined;
141-
let cutBefore = false;
163+
let before: EndedTry | undefined;
142164
for (let tried = 0; ; tried++) {
143165
const current: PutTry = { attempt: tried + 1, startedAt: Date.now(), sent: 0 };
144166
try {
145167
return await attempt(current);
146168
} catch (err) {
147169
if (isFinal(err) || signal?.aborted) throw err;
148-
const now = { ...current, ageMs: Date.now() - current.startedAt };
149-
const cut = isConnectionCut(err);
150-
if (cut && cutBefore && before && isSameCut(before, now)) {
151-
throw Object.assign(
152-
new Error(
153-
`The connection was cut at ${seconds(now.ageMs)} on two tries in a row, after ${Math.floor((100 * now.sent) / total)}% of the file was sent (${(err as { code?: string }).code ?? "connection reset"}). Every try restarts the file, so this link cannot finish it before the cut: use a faster connection or send less media.`,
154-
{ cause: err },
155-
),
156-
{ code: (err as { code?: unknown }).code },
157-
);
158-
}
170+
const now = { ...current, ageMs: Date.now() - current.startedAt, cut: isConnectionCut(err) };
171+
const futile = repeatedCut(before, now, err, total);
172+
if (futile) throw futile;
159173
if (tried >= putTimings.retries) throw exhausted(err, tried + 1, now, total);
160-
[before, cutBefore] = [now, cut];
174+
before = now;
161175
await sleep(putTimings.backoffMs * 2 ** tried, signal);
162176
}
163177
}
@@ -311,7 +325,7 @@ function watchStall(req: ClientRequest, onSample: (written: number) => void): ()
311325
else if (Date.now() - movedAt >= putTimings.stallMs)
312326
req.destroy(new Error(`Upload stalled: no data sent for ${putTimings.stallMs / 1000} s`));
313327
},
314-
Math.min(1000, putTimings.stallMs / 3),
328+
Math.min(putTimings.sampleMs, putTimings.stallMs / 3),
315329
);
316330
return () => clearInterval(watch);
317331
}

0 commit comments

Comments
 (0)