Skip to content

Commit db42e60

Browse files
miguel-heygenmiguel-heygen
authored
perf(studio): a clip's filmstrip decodes 9 frames instead of 272 and reaches the clip's end (#4526)
* perf(studio): build a video clip's filmstrip from its keyframes * fix(studio): a filmstrip slot keeps its own time when its keyframe is far away * perf(studio): a WebM filmstrip downloads each cluster once Each keyframe lookup now runs just before its frame is decoded, from a generator the canvas sink pulls, instead of all lookups before the first decode. A WebM lookup loads its whole cluster, so resolving every lookup first pushed the early clusters out of the source cache and the decode pass fetched them again. A cancel during a lookup ends the generator quietly so the decoder still closes; the cancel is raised after the loop. * fix(studio): a trimmed video with no reported length fills its filmstrip When the file reports no duration (a fragmented MP4), the source was assumed to end at the trim length, so a trim that starts later collapsed to zero length and every tile showed the trim start. It now ends at the trim start plus the trim length. * fix(studio): a video filmstrip reaches the end of the clip The strip requests a power of two of frames spread over the whole clip, but tile i showed frame i, so fewer tiles than frames showed only the start of the clip and more tiles repeated the strip. Each tile now shows the frame nearest its own place along the clip.
1 parent 5a74db9 commit db42e60

4 files changed

Lines changed: 136 additions & 12 deletions

File tree

‎packages/studio/src/player/components/VideoThumbnail.test.tsx‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,19 @@ describe("VideoThumbnail", () => {
8686
expect(host.querySelectorAll("img").length).toBeGreaterThan(0);
8787
});
8888

89+
it("spreads the frames across every tile so the strip reaches the clip's end", async () => {
90+
const urls = Array.from({ length: 8 }, (_, index) => `blob:${index}`);
91+
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
92+
value: { kind: "filmstrip", urls, aspect: 16 / 9 },
93+
weight: 256,
94+
});
95+
96+
await render(500);
97+
98+
const tiles = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));
99+
expect(tiles).toEqual(["blob:0", "blob:2", "blob:4", "blob:5", "blob:7"]);
100+
});
101+
89102
it("issues a single decode job for a narrow clip", async () => {
90103
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
91104
value: { kind: "image", url: "blob:poster", aspect: 16 / 9 },

‎packages/studio/src/player/components/VideoThumbnail.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({
146146
{urls.length > 0 && (
147147
<div className="absolute inset-0 flex">
148148
{Array.from({ length: frameCount }, (_, index) => {
149-
const src = urls[index % urls.length];
149+
const src = urls[Math.round((index * (urls.length - 1)) / Math.max(1, frameCount - 1))];
150150
return (
151151
<div
152152
key={index}

‎packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts‎

Lines changed: 102 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { decodeVideoThumbnail, videoThumbnailTimestamps } from "./thumbnailVideo
55

66
const dispose = vi.fn();
77
const canvasesAtTimestamps = vi.fn();
8+
const getKeyPacket = vi.fn(async (_time: number) => null as { timestamp: number } | null);
89
const input = {
910
getPrimaryVideoTrack: vi.fn(),
1011
dispose,
@@ -22,10 +23,25 @@ vi.mock("mediabunny", () => ({
2223
CanvasSink: class {
2324
canvasesAtTimestamps = canvasesAtTimestamps;
2425
},
26+
EncodedPacketSink: class {
27+
getKeyPacket = getKeyPacket;
28+
},
2529
}));
2630

31+
function recordDecodes(decoded: number[][]): void {
32+
canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
33+
const run: number[] = [];
34+
decoded.push(run);
35+
for await (const time of timestamps) {
36+
run.push(time);
37+
yield { canvas: document.createElement("canvas") };
38+
}
39+
});
40+
}
41+
2742
beforeEach(() => {
2843
vi.clearAllMocks();
44+
getKeyPacket.mockImplementation(async () => null);
2945
vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two");
3046
vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {});
3147
HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {
@@ -52,17 +68,14 @@ describe("videoThumbnailTimestamps", () => {
5268

5369
describe("decodeVideoThumbnail", () => {
5470
it("extracts sparse frames, returns object URLs, and disposes once", async () => {
55-
const canvas = document.createElement("canvas");
56-
canvasesAtTimestamps.mockImplementation(async function* (timestamps: number[]) {
57-
expect(timestamps).toEqual([2, 8]);
58-
yield { canvas, timestamp: 2, duration: 1 };
59-
yield { canvas, timestamp: 8, duration: 1 };
60-
});
71+
const decoded: number[][] = [];
72+
recordDecodes(decoded);
6173
const result = await decodeVideoThumbnail(
6274
{ source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },
6375
new AbortController().signal,
6476
);
6577

78+
expect(decoded).toEqual([[2, 8]]);
6679
expect(result.value).toEqual({
6780
kind: "filmstrip",
6881
urls: ["blob:one", "blob:two"],
@@ -74,6 +87,89 @@ describe("decodeVideoThumbnail", () => {
7487
expect(dispose).toHaveBeenCalledTimes(1);
7588
});
7689

90+
it("decodes each strip frame at its keyframe unless that keyframe is before the clip's range", async () => {
91+
getKeyPacket.mockImplementation(async (time) => ({ timestamp: Math.floor(time / 4) * 4 }));
92+
const decoded: number[][] = [];
93+
recordDecodes(decoded);
94+
await decodeVideoThumbnail(
95+
{ source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 8, frameCount: 3 },
96+
new AbortController().signal,
97+
);
98+
expect(decoded).toEqual([[2, 4, 8]]);
99+
});
100+
101+
it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => {
102+
getKeyPacket.mockImplementation(async () => ({ timestamp: 0 }));
103+
const decoded: number[][] = [];
104+
recordDecodes(decoded);
105+
await decodeVideoThumbnail(
106+
{ source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
107+
new AbortController().signal,
108+
);
109+
await decodeVideoThumbnail(
110+
{ source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
111+
new AbortController().signal,
112+
);
113+
expect(decoded).toEqual([[0, 5, 10], [5]]);
114+
});
115+
116+
it("looks up each keyframe just before decoding it", async () => {
117+
const events: string[] = [];
118+
getKeyPacket.mockImplementation(async (time) => {
119+
events.push(`key ${time}`);
120+
return { timestamp: time };
121+
});
122+
canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
123+
for await (const time of timestamps) {
124+
events.push(`frame ${time}`);
125+
yield { canvas: document.createElement("canvas") };
126+
}
127+
});
128+
await decodeVideoThumbnail(
129+
{ source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
130+
new AbortController().signal,
131+
);
132+
expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", "key 10", "frame 10"]);
133+
});
134+
135+
it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => {
136+
const controller = new AbortController();
137+
getKeyPacket.mockImplementation(async () => (controller.abort(), null));
138+
const decoded: number[] = [];
139+
let timesEnded = false;
140+
canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
141+
for await (const time of timestamps) {
142+
decoded.push(time);
143+
yield { canvas: document.createElement("canvas") };
144+
}
145+
timesEnded = true;
146+
});
147+
await expect(
148+
decodeVideoThumbnail(
149+
{ source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
150+
controller.signal,
151+
),
152+
).rejects.toThrow("Aborted");
153+
expect(getKeyPacket).toHaveBeenCalledTimes(1);
154+
expect(decoded).toEqual([]);
155+
expect(timesEnded).toBe(true);
156+
});
157+
158+
it("spreads a trimmed strip across its range when the file reports no duration", async () => {
159+
input.getPrimaryVideoTrack.mockResolvedValue({
160+
getDisplayWidth: vi.fn(async () => 1080),
161+
getDisplayHeight: vi.fn(async () => 1920),
162+
getDurationFromMetadata: vi.fn(async () => null),
163+
});
164+
const decoded: number[][] = [];
165+
recordDecodes(decoded);
166+
await decodeVideoThumbnail(
167+
{ source: "/clip.mp4", sourceStart: 5, sourceRangeDuration: 5, frameCount: 3 },
168+
new AbortController().signal,
169+
);
170+
expect(decoded).toEqual([[5, 7.5, 10]]);
171+
});
172+
77173
it("releases input and degrades when the source has no video track", async () => {
78174
input.getPrimaryVideoTrack.mockResolvedValue(null);
79175
await expect(

‎packages/studio/src/player/lib/thumbnailVideoDecoder.ts‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ interface DecodedResources {
4444

4545
interface ThumbnailCanvasSink {
4646
canvasesAtTimestamps(
47-
timestamps: number[],
47+
timestamps: AsyncIterable<number>,
4848
): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;
4949
}
5050

@@ -77,7 +77,7 @@ function targetDimensions(
7777

7878
async function decodeFrames(
7979
sink: ThumbnailCanvasSink,
80-
timestamps: number[],
80+
timestamps: AsyncIterable<number>,
8181
signal: AbortSignal,
8282
resources: DecodedResources,
8383
): Promise<void> {
@@ -89,6 +89,7 @@ async function decodeFrames(
8989
throwIfAborted(signal);
9090
resources.urls.push(URL.createObjectURL(blob));
9191
}
92+
throwIfAborted(signal);
9293
}
9394

9495
function loadedResult(
@@ -138,8 +139,12 @@ export async function decodeVideoThumbnail(
138139
}
139140
const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });
140141
throwIfAborted(signal);
141-
const sourceDuration = Math.max(0, metadataDuration ?? request.sourceRangeDuration ?? 0);
142-
const sourceStart = Math.min(Math.max(0, request.sourceStart ?? 0), sourceDuration);
142+
const requestedStart = Math.max(0, request.sourceStart ?? 0);
143+
const sourceDuration = Math.max(
144+
0,
145+
metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0),
146+
);
147+
const sourceStart = Math.min(requestedStart, sourceDuration);
143148
const requestedDuration =
144149
request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);
145150
const duration = Math.min(
@@ -151,6 +156,16 @@ export async function decodeVideoThumbnail(
151156
duration,
152157
Math.min(request.frameCount, budgets.richPreviewFrameCount),
153158
);
159+
const keys = new mediabunny.EncodedPacketSink(track);
160+
const maxKeyframeLead = duration / Math.max(2, timestamps.length - 1) / 2;
161+
async function* decodeTimesAtNearbyKeyframes() {
162+
for (const time of timestamps) {
163+
const key = await keys.getKeyPacket(time, { metadataOnly: true });
164+
if (signal.aborted) return;
165+
const near = key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead;
166+
yield near ? key.timestamp : time;
167+
}
168+
}
154169
const aspect = displayWidth / displayHeight;
155170
const target = targetDimensions(aspect, budgets);
156171
const sink = new mediabunny.CanvasSink(track, {
@@ -159,7 +174,7 @@ export async function decodeVideoThumbnail(
159174
fit: request.fit ?? "cover",
160175
poolSize: 1,
161176
});
162-
await decodeFrames(sink, timestamps, signal, resources);
177+
await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), signal, resources);
163178
return loadedResult(resources, aspect, target.width, target.height);
164179
} catch (error) {
165180
releaseDecodedResources(resources);

0 commit comments

Comments
 (0)