Skip to content

Commit 852f6f9

Browse files
test(cli): cloud download tests split small parts so a slow Windows loopback cannot time them out (#5999)
1 parent 8b410f1 commit 852f6f9

2 files changed

Lines changed: 25 additions & 21 deletions

File tree

‎packages/cli/src/cloud/download.test.ts‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,6 @@ async function serveBytes(
8181
};
8282
}
8383

84-
const MiB = 1024 * 1024;
85-
8684
function patternBytes(length: number): Buffer {
8785
const bytes = Buffer.alloc(length);
8886
for (let i = 0; i < length; i++) bytes[i] = (i * 31 + (i >> 13)) & 0xff;
@@ -286,21 +284,25 @@ describe("cloud/download", () => {
286284
afterEach(async () => {
287285
await Promise.all(servers.splice(0).map((server) => server.close()));
288286
});
287+
// Small parts keep the split the same while moving little data: a Windows runner's loopback
288+
// has carried a single connection at ~1 MB/s, past the timeout for megabyte bodies.
289+
const PART = 64 * 1024;
290+
const options = { partBytes: PART };
289291
async function serve(...args: Parameters<typeof serveBytes>) {
290292
const server = await serveBytes(...args);
291293
servers.push(server);
292294
return server;
293295
}
294296

295297
it("splits a large file into parallel ranged requests", async () => {
296-
const body = patternBytes(60 * MiB + 123);
298+
const body = patternBytes(7.5 * PART + 123);
297299
const server = await serve(body, { honourRange: true });
298300
const dest = join(dir, "out.mp4");
299-
const result = await downloadToFile(server.url, dest);
301+
const result = await downloadToFile(server.url, dest, options);
300302
expect(result.bytes).toBe(body.length);
301303
expect(readFileSync(dest).equals(body)).toBe(true);
302304
expect(server.ranges).toHaveLength(8);
303-
expect(server.ranges[0]).toBe(`bytes=0-${8 * MiB - 1}`);
305+
expect(server.ranges[0]).toBe(`bytes=0-${PART - 1}`);
304306
expect(readdirSync(dir)).toEqual(["out.mp4"]);
305307
});
306308

@@ -309,42 +311,42 @@ describe("cloud/download", () => {
309311
const body = patternBytes(size);
310312
const server = await serve(body, { honourRange: true });
311313
const dest = join(dir, `out-${size}.mp4`);
312-
await downloadToFile(server.url, dest);
314+
await downloadToFile(server.url, dest, options);
313315
expect(readFileSync(dest).equals(body)).toBe(true);
314-
expect(server.ranges[0]).toBe(`bytes=0-${8 * MiB - 1}`);
316+
expect(server.ranges[0]).toBe(`bytes=0-${PART - 1}`);
315317
expect(server.ranges.slice(1)).toEqual(size === 0 ? [undefined] : []);
316318
}
317319
});
318320

319321
it("streams the whole file in one request when the server ignores Range", async () => {
320-
const body = patternBytes(20 * MiB);
322+
const body = patternBytes(2.5 * PART);
321323
const server = await serve(body, { honourRange: false });
322324
const dest = join(dir, "out.mp4");
323-
await downloadToFile(server.url, dest);
325+
await downloadToFile(server.url, dest, options);
324326
expect(readFileSync(dest).equals(body)).toBe(true);
325-
expect(server.ranges).toEqual([`bytes=0-${8 * MiB - 1}`]);
327+
expect(server.ranges).toEqual([`bytes=0-${PART - 1}`]);
326328
});
327329

328330
it("resumes a range whose connection drops mid-body from the bytes it already has", async () => {
329-
const body = patternBytes(50 * MiB);
330-
const at = 8 * MiB + 1024;
331+
const body = patternBytes(6.25 * PART);
332+
const at = PART + 1024;
331333
const server = await serve(body, { honourRange: true, drop: { at, times: 2 } });
332334
const dest = join(dir, "out.mp4");
333-
const result = await downloadToFile(server.url, dest);
335+
const result = await downloadToFile(server.url, dest, options);
334336
expect(result.bytes).toBe(body.length);
335337
expect(readFileSync(dest).equals(body)).toBe(true);
336338
expect(server.ranges.filter((range) => range?.startsWith(`bytes=${at}-`))).toHaveLength(2);
337339
});
338340

339341
it("keeps the previous output and leaves no staged file when one part keeps dropping", async () => {
340-
const body = patternBytes(50 * MiB);
342+
const body = patternBytes(6.25 * PART);
341343
const server = await serve(body, {
342344
honourRange: true,
343-
drop: { at: 8 * MiB + 1024, times: Infinity },
345+
drop: { at: PART + 1024, times: Infinity },
344346
});
345347
const dest = join(dir, "out.mp4");
346348
writeFileSync(dest, "previous render");
347-
await expect(downloadToFile(server.url, dest)).rejects.toThrow(/dropped 4 times/);
349+
await expect(downloadToFile(server.url, dest, options)).rejects.toThrow(/dropped 4 times/);
348350
expect(readFileSync(dest, "utf8")).toBe("previous render");
349351
expect(readdirSync(dir)).toEqual(["out.mp4"]);
350352
});

‎packages/cli/src/cloud/download.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export interface DownloadOptions {
3232
fetchImpl?: typeof fetch;
3333
/** Called with (bytes downloaded, total or undefined). */
3434
onProgress?: (bytes: number, total: number | undefined) => void;
35+
partBytes?: number;
3536
}
3637

3738
export interface DownloadResult {
@@ -54,14 +55,15 @@ export async function downloadToFile(
5455
options: DownloadOptions = {},
5556
): Promise<DownloadResult> {
5657
const fetchImpl = options.fetchImpl ?? fetch;
58+
const partBytes = options.partBytes ?? PART_BYTES;
5759
// Cancels the sibling parts when one fails.
5860
const failFast = new AbortController();
5961
const signal = options.signal
6062
? AbortSignal.any([options.signal, failFast.signal])
6163
: failFast.signal;
6264
const get = async (range?: string) => fetchImpl(url, { signal, headers: range ? { range } : {} });
6365

64-
let first = await get(`bytes=0-${PART_BYTES - 1}`);
66+
let first = await get(`bytes=0-${partBytes - 1}`);
6567
// 416 (empty file) or a 206 without a total cannot be split: fetch it whole.
6668
if (first.status === 416 || (first.status === 206 && contentRangeTotal(first) === undefined)) {
6769
await first.body?.cancel();
@@ -118,10 +120,10 @@ export async function downloadToFile(
118120
};
119121
const parts: Promise<void>[] = [];
120122
if (ranged && total !== undefined) {
121-
parts.push(fetchRange(0, Math.min(PART_BYTES, total) - 1, first));
122-
const rest = total - PART_BYTES;
123-
const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / PART_BYTES)));
124-
for (let start = PART_BYTES; start < total; start += size) {
123+
parts.push(fetchRange(0, Math.min(partBytes, total) - 1, first));
124+
const rest = total - partBytes;
125+
const size = Math.ceil(rest / Math.min(CONNECTIONS - 1, Math.ceil(rest / partBytes)));
126+
for (let start = partBytes; start < total; start += size) {
125127
parts.push(fetchRange(start, Math.min(start + size, total) - 1));
126128
}
127129
} else {

0 commit comments

Comments
 (0)