Skip to content

Commit 26916b7

Browse files
authored
fix(cli): validate captured Lottie archives and previews (#3811)
* fix(cli): validate captured Lottie archives and previews * fix(cli): bound Lottie discovery before buffering * fix(cli): bound Lottie discovery lifecycle and archive paths * fix(cli): retain highest-priority Lottie candidates
1 parent 1aa5b9e commit 26916b7

11 files changed

Lines changed: 596 additions & 75 deletions

‎packages/cli/src/capture/index.ts‎

Lines changed: 12 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { LottieDiscovery } from "./lottieDiscovery.js";
12
import { createCaptureDownloadBudget } from "./readBoundedResponse.js";
23
/**
34
* Website capture orchestrator.
@@ -84,6 +85,7 @@ export async function captureWebsite(
8485
onPhase,
8586
} = opts;
8687

88+
const downloadByteBudget = createCaptureDownloadBudget();
8789
const warnings: string[] = [];
8890
const progress = (stage: string, detail?: string) => {
8991
onProgress?.(stage, detail);
@@ -204,51 +206,20 @@ export async function captureWebsite(
204206

205207
// Intercept network responses to detect Lottie JSON files
206208
const discoveredLotties: DiscoveredLottie[] = [];
209+
const lottieDiscovery = new LottieDiscovery();
207210
// Layer 1 (passive video discovery): every direct-video URL the page fetches
208211
// over the whole session (load / scroll / carousel rotation), independent of
209212
// whether a <video> for it exists at snapshot time. captureVideoManifest
210213
// downloads these (guarded) and merges them into the manifest.
211214
const discoveredVideoUrls = new Set<string>();
212215
// fallow-ignore-next-line complexity
213-
page1.on("response", async (response) => {
216+
page1.on("response", (response) => {
214217
try {
215218
const responseUrl = response.url();
216219
if (/\.(mp4|webm|mov|m4v)(\?|#|$)/i.test(responseUrl)) {
217220
discoveredVideoUrls.add(responseUrl);
218221
}
219-
const contentType = response.headers()["content-type"] || "";
220-
const isJsonUrl = responseUrl.endsWith(".json");
221-
const isLottieUrl = responseUrl.endsWith(".lottie");
222-
const isJson =
223-
contentType.includes("application/json") || contentType.includes("text/plain");
224-
225-
if (isLottieUrl) {
226-
discoveredLotties.push({ url: responseUrl });
227-
return;
228-
}
229-
230-
if (isJsonUrl || isJson) {
231-
// Check Content-Length before downloading to avoid OOM on huge responses
232-
const cl = parseInt(response.headers()["content-length"] || "0", 10);
233-
if (cl > 5_000_000) return;
234-
const buffer = await response.buffer();
235-
if (buffer.length < 100 || buffer.length > 5_000_000) return; // Skip tiny or huge
236-
const text = buffer.toString("utf-8");
237-
const json = JSON.parse(text);
238-
// Validate Lottie structure: must have version, in/out points, layers, dimensions, framerate
239-
if (
240-
json &&
241-
typeof json === "object" &&
242-
["v", "ip", "op", "layers", "w", "h", "fr"].every((k: string) => k in json)
243-
) {
244-
discoveredLotties.push({
245-
url: responseUrl,
246-
data: json,
247-
dimensions: { w: json.w, h: json.h },
248-
frameRate: json.fr,
249-
});
250-
}
251-
}
222+
lottieDiscovery.collect(response);
252223
} catch {
253224
/* not JSON or parse error — skip */
254225
}
@@ -382,10 +353,16 @@ export async function captureWebsite(
382353
/* DOM scan failed — non-critical */
383354
}
384355

356+
for (const found of await lottieDiscovery.run(downloadByteBudget, remainingMs)) {
357+
const existing = discoveredLotties.findIndex((item) => item.url === found.url);
358+
if (existing < 0) discoveredLotties.push(found);
359+
else discoveredLotties[existing] = found;
360+
}
361+
385362
if (discoveredLotties.length > 0 && remainingMs() > 0) {
386363
const lottieDir = join(outputDir, "assets", "lottie");
387364
mkdirSync(lottieDir, { recursive: true });
388-
const lottieBudget = { remainingMs };
365+
const lottieBudget = { remainingMs, byteBudget: downloadByteBudget };
389366
const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir, lottieBudget);
390367
// Generate manifest + preview thumbnails so the agent can SEE what each animation is
391368
if (savedCount > 0 && remainingMs() > 0) {
@@ -609,7 +586,6 @@ export async function captureWebsite(
609586
// `budget-exhausted` for every one of them replaces a warning string that could only ever
610587
// say "some". A zero budget means it breaks on the first url, so this costs no network.
611588
phase("fonts", "started");
612-
const downloadByteBudget = createCaptureDownloadBudget();
613589
const fontPass = await downloadAndRewriteFonts(extracted.headHtml, outputDir, {
614590
remainingMs,
615591
byteBudget: downloadByteBudget,
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { describe, expect, it, vi } from "vitest";
2+
import { readLottieArchive } from "./lottieValidation.js";
3+
4+
const getEntries = vi.hoisted(() => vi.fn(() => []));
5+
vi.mock("adm-zip", () => ({
6+
default: class {
7+
getEntryCount() {
8+
return 1000;
9+
}
10+
getEntries = getEntries;
11+
},
12+
}));
13+
14+
describe("Lottie archive entry allocation", () => {
15+
it("rejects the EOCD count without materializing entries", () => {
16+
expect(readLottieArchive(Buffer.from("archive metadata supplied by test"))).toBeNull();
17+
expect(getEntries).not.toHaveBeenCalled();
18+
});
19+
});
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/** Bound raw central-directory names before adm-zip synthesizes their parent folders. */
2+
export function preflightLottieArchive(bytes: Buffer): boolean {
3+
const end = findDirectoryEnd(bytes);
4+
if (end < 0) return false;
5+
if (!unambiguousFooter(bytes, end)) return false;
6+
const count = bytes.readUInt16LE(end + 10);
7+
if (count > 256 || bytes.readUInt16LE(end + 8) !== count) return false;
8+
if (bytes.readUInt32LE(end + 4) !== 0) return false; // no multi-disk archives
9+
let offset = bytes.readUInt32LE(end + 16);
10+
const directoryEnd = offset + bytes.readUInt32LE(end + 12);
11+
if (directoryEnd > end) return false;
12+
for (let i = 0; i < count; i++) {
13+
const next = nextBoundedEntry(bytes, offset, directoryEnd);
14+
if (next === null) return false;
15+
offset = next;
16+
}
17+
return offset === directoryEnd;
18+
}
19+
20+
function unambiguousFooter(bytes: Buffer, end: number): boolean {
21+
const signature = Buffer.from([0x50, 0x4b, 0x05, 0x06]);
22+
if (bytes.lastIndexOf(signature) !== end) return false;
23+
// adm-zip scans just before EOCD for alternate/ZIP64 records. This bounded
24+
// archive format does not need those; do not let it select another directory.
25+
const preceding = bytes.subarray(Math.max(0, end - 76), end);
26+
return ![
27+
signature,
28+
Buffer.from([0x50, 0x4b, 0x06, 0x06]),
29+
Buffer.from([0x50, 0x4b, 0x06, 0x07]),
30+
].some((marker) => preceding.includes(marker));
31+
}
32+
33+
function findDirectoryEnd(bytes: Buffer): number {
34+
for (let at = bytes.length - 22; at >= Math.max(0, bytes.length - 65557); at--) {
35+
if (
36+
bytes.readUInt32LE(at) === 0x06054b50 &&
37+
at + 22 + bytes.readUInt16LE(at + 20) === bytes.length
38+
)
39+
return at;
40+
}
41+
return -1;
42+
}
43+
44+
function nextBoundedEntry(bytes: Buffer, offset: number, end: number): number | null {
45+
if (offset + 46 > end || bytes.readUInt32LE(offset) !== 0x02014b50) return null;
46+
const length = bytes.readUInt16LE(offset + 28);
47+
const next =
48+
offset + 46 + length + bytes.readUInt16LE(offset + 30) + bytes.readUInt16LE(offset + 32);
49+
if (length > 512 || next > end) return null;
50+
const name = bytes.toString("utf8", offset + 46, offset + 46 + length);
51+
return safeLottieArchivePath(name) ? next : null;
52+
}
53+
54+
export function safeLottieArchivePath(path: string): boolean {
55+
if (path.length > 512 || path.split("/").length > 16) return false;
56+
if (path.startsWith("/") || /[\\:]/.test(path)) return false;
57+
if (Array.from(path).some((char) => char.charCodeAt(0) < 32)) return false;
58+
return path.split("/").every((segment) => segment !== ".." && segment !== ".");
59+
}
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import { afterEach, describe, expect, it, vi } from "vitest";
2+
import { discoverLottieResponse, LottieDiscovery } from "./lottieDiscovery.js";
3+
4+
describe("bounded Lottie discovery", () => {
5+
it("awaits delayed candidates and ignores response events after its save boundary", async () => {
6+
const discovery = new LottieDiscovery();
7+
const response = { url: () => "https://public.example/a.json", headers: () => ({}) };
8+
discovery.collect(response);
9+
discovery.collect(response);
10+
let finish!: (response: Response) => void;
11+
const fetchMock = vi.fn(
12+
() =>
13+
new Promise<Response>((resolve) => {
14+
finish = resolve;
15+
}),
16+
);
17+
vi.stubGlobal("fetch", fetchMock);
18+
const budget = { remainingBytes: 1000 };
19+
let settled = false;
20+
const pending = discovery
21+
.run(budget, () => 10000)
22+
.then((result) => {
23+
settled = true;
24+
return result;
25+
});
26+
await Promise.resolve();
27+
expect(settled).toBe(false);
28+
finish(
29+
new Response(JSON.stringify({ v: "5", w: 100, h: 100, fr: 30, ip: 0, op: 30, layers: [] })),
30+
);
31+
expect(await pending).toHaveLength(1);
32+
const remaining = budget.remainingBytes;
33+
discovery.collect({ url: () => "https://public.example/late.json", headers: () => ({}) });
34+
expect(await discovery.run(budget, () => 10000)).toEqual([]);
35+
expect(fetchMock).toHaveBeenCalledTimes(1);
36+
expect(budget.remainingBytes).toBe(remaining);
37+
});
38+
39+
it("caps ordinary JSON candidates and their aggregate byte consumption", async () => {
40+
const discovery = new LottieDiscovery();
41+
for (let n = 0; n < 100; n++)
42+
discovery.collect({
43+
url: () => `https://public.example/api/${n}`,
44+
headers: () => ({ "content-type": "application/json" }),
45+
});
46+
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
47+
vi.stubGlobal("fetch", fetchMock);
48+
const budget = { remainingBytes: 1000 };
49+
expect(await discovery.run(budget, () => 10000)).toEqual([]);
50+
expect(fetchMock).toHaveBeenCalledTimes(32);
51+
expect(budget.remainingBytes).toBe(936);
52+
});
53+
it("retains explicit JSON when a later archive displaces a generic candidate", async () => {
54+
const discovery = new LottieDiscovery();
55+
discovery.collect({ url: () => "https://public.example/real.json", headers: () => ({}) });
56+
for (let n = 0; n < 31; n++)
57+
discovery.collect({
58+
url: () => `https://public.example/api/${n}`,
59+
headers: () => ({ "content-type": "application/json" }),
60+
});
61+
discovery.collect({ url: () => "https://public.example/anim.lottie", headers: () => ({}) });
62+
const fetchMock = vi.fn(async (_url: string | URL | Request) => new Response("{}"));
63+
vi.stubGlobal("fetch", fetchMock);
64+
await discovery.run({ remainingBytes: 1000 }, () => 10000);
65+
const urls = fetchMock.mock.calls.map((call) => call[0]);
66+
expect(urls).toContain("https://public.example/real.json");
67+
expect(urls).not.toContain("https://public.example/api/0");
68+
expect(urls).toHaveLength(31);
69+
});
70+
afterEach(() => vi.unstubAllGlobals());
71+
it.each([undefined, "1"])(
72+
"does not trust intercepted Content-Length %s or materialize Puppeteer bodies",
73+
async (length) => {
74+
const buffer = vi.fn(() => {
75+
throw new Error("unbounded body read");
76+
});
77+
const response = {
78+
url: () => "https://public.example/animation.json",
79+
headers: () => ({
80+
"content-type": "application/json",
81+
...(length ? { "content-length": length } : {}),
82+
}),
83+
buffer,
84+
};
85+
const cancel = vi.fn();
86+
vi.stubGlobal(
87+
"fetch",
88+
vi.fn(
89+
async () =>
90+
new Response(
91+
new ReadableStream({
92+
pull(controller) {
93+
controller.enqueue(new Uint8Array(4));
94+
},
95+
cancel,
96+
}),
97+
{ headers: length ? { "content-length": length } : {} },
98+
),
99+
),
100+
);
101+
expect(await discoverLottieResponse(response, { remainingBytes: 5 })).toBeNull();
102+
expect(buffer).not.toHaveBeenCalled();
103+
expect(cancel).toHaveBeenCalledOnce();
104+
},
105+
);
106+
it("retains valid discovered JSON with its existing byte accounting", async () => {
107+
const data = { v: "5.12.2", w: 100, h: 100, layers: [], fr: 30, ip: 0, op: 30 };
108+
const body = JSON.stringify(data);
109+
vi.stubGlobal(
110+
"fetch",
111+
vi.fn(async () => new Response(body)),
112+
);
113+
const budget = { remainingBytes: body.length };
114+
const found = await discoverLottieResponse(
115+
{ url: () => "https://public.example/animation.json?version=1", headers: () => ({}) },
116+
budget,
117+
);
118+
expect(found?.data).toEqual(data);
119+
expect(found?.dataBudget).toBe(budget);
120+
expect(budget.remainingBytes).toBe(0);
121+
});
122+
});

0 commit comments

Comments
 (0)