Skip to content

Commit 079f45b

Browse files
Matej Ukmarclaude
andcommitted
fix: size chat videos from their video track and loaded metadata
The MP4 parser now takes the size of a track whose handler is `vide`, so a subtitle or chapter track with its own size can no longer set the shape. The chat player prefers the loaded video's size over the server's header read, which can be stale after the file is overwritten in place; sizes written in Markdown still win. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 72d3d13 commit 079f45b

4 files changed

Lines changed: 72 additions & 24 deletions

File tree

‎apps/server/src/assets/videoDimensions.test.ts‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,21 @@ function tkhd(
4040
return box("tkhd", body);
4141
}
4242

43+
// Handler `vide`, `soun`, or a text type such as `sbtl` or `text`.
44+
const hdlr = (handler: string) => {
45+
const body = new Uint8Array(25);
46+
body.set(new TextEncoder().encode(handler), 8);
47+
return box("hdlr", body);
48+
};
49+
50+
function trak(
51+
width: number,
52+
height: number,
53+
options: Parameters<typeof tkhd>[2] & { handler?: string } = {},
54+
) {
55+
return box("trak", tkhd(width, height, options), box("mdia", hdlr(options.handler ?? "vide")));
56+
}
57+
4358
const ftyp = box("ftyp", new TextEncoder().encode("isom"));
4459
const mdat = box("mdat", new Uint8Array(4096));
4560

@@ -76,7 +91,7 @@ describe("readVideoDimensions", () => {
7691
it.each([
7792
{
7893
name: "an MP4 whose moov follows the media data",
79-
bytes: concat(ftyp, mdat, box("moov", box("trak", tkhd(1080, 1920)))),
94+
bytes: concat(ftyp, mdat, box("moov", trak(1080, 1920))),
8095
expected: { width: 1080, height: 1920 },
8196
},
8297
{
@@ -86,25 +101,30 @@ describe("readVideoDimensions", () => {
86101
box(
87102
"moov",
88103
box("mvhd", new Uint8Array(100)),
89-
box("trak", tkhd(0, 0)),
90-
box("trak", tkhd(1920, 1080)),
104+
trak(0, 0, { handler: "soun" }),
105+
trak(1920, 1080),
91106
),
92107
),
93108
expected: { width: 1920, height: 1080 },
94109
},
110+
{
111+
name: "the video track after a subtitle track with its own size",
112+
bytes: concat(ftyp, box("moov", trak(1920, 200, { handler: "sbtl" }), trak(1080, 1920))),
113+
expected: { width: 1080, height: 1920 },
114+
},
95115
{
96116
name: "a version 1 track header",
97-
bytes: concat(ftyp, box("moov", box("trak", tkhd(640, 480, { version: 1 })))),
117+
bytes: concat(ftyp, box("moov", trak(640, 480, { version: 1 }))),
98118
expected: { width: 640, height: 480 },
99119
},
100120
{
101121
name: "a track header that declares a large body",
102-
bytes: concat(ftyp, box("moov", box("trak", tkhd(1080, 1920, { padding: 1 << 20 })))),
122+
bytes: concat(ftyp, box("moov", trak(1080, 1920, { padding: 1 << 20 }))),
103123
expected: { width: 1080, height: 1920 },
104124
},
105125
{
106126
name: "a phone recording rotated a quarter turn",
107-
bytes: concat(ftyp, box("moov", box("trak", tkhd(1920, 1080, { matrix: QUARTER_TURN })))),
127+
bytes: concat(ftyp, box("moov", trak(1920, 1080, { matrix: QUARTER_TURN }))),
108128
expected: { width: 1080, height: 1920 },
109129
},
110130
{
@@ -142,12 +162,12 @@ describe("readVideoDimensions", () => {
142162
},
143163
{
144164
name: "an MP4 with no video track",
145-
bytes: concat(ftyp, box("moov", box("trak", tkhd(0, 0)))),
165+
bytes: concat(ftyp, box("moov", trak(0, 0, { handler: "soun" }))),
146166
expected: null,
147167
},
148168
{
149169
name: "a box that claims more bytes than the file holds",
150-
bytes: concat(ftyp, box("moov", box("trak", tkhd(1080, 1920))).subarray(0, 40)),
170+
bytes: concat(ftyp, box("moov", trak(1080, 1920)).subarray(0, 40)),
151171
expected: null,
152172
},
153173
{

‎apps/server/src/assets/videoDimensions.ts‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,20 +65,35 @@ async function readIsoBmff(read: ReadAt, fileSize: number): Promise<ImageDimensi
6565
if (box.type !== "moov") continue;
6666
for await (const track of isoBoxes(read, box.bodyStart, box.end)) {
6767
if (track.type !== "trak") continue;
68+
let dimensions: ImageDimensions | null = null;
69+
let handler: string | null = null;
6870
for await (const child of isoBoxes(read, track.bodyStart, track.end)) {
69-
if (child.type !== "tkhd") continue;
70-
const dimensions = readTrackHeader(
71-
await read(child.bodyStart, Math.min(TRACK_HEADER_BYTES, child.end - child.bodyStart)),
72-
);
73-
if (dimensions) return dimensions;
74-
break;
71+
if (child.type === "tkhd") {
72+
dimensions = readTrackHeader(
73+
await read(child.bodyStart, Math.min(TRACK_HEADER_BYTES, child.end - child.bodyStart)),
74+
);
75+
} else if (child.type === "mdia") {
76+
handler = await readHandlerType(read, child);
77+
}
7578
}
79+
// Subtitle and chapter text tracks can carry a nonzero size of their own.
80+
if (handler === "vide" && dimensions) return dimensions;
7681
}
7782
return null;
7883
}
7984
return null;
8085
}
8186

87+
async function readHandlerType(read: ReadAt, media: Box): Promise<string | null> {
88+
for await (const child of isoBoxes(read, media.bodyStart, media.end)) {
89+
if (child.type !== "hdlr") continue;
90+
// Version and flags, then `pre_defined`, then the four-character handler type.
91+
const handler = await read(child.bodyStart + 8, 4);
92+
return handler.length === 4 ? ascii(handler, 0, 4) : null;
93+
}
94+
return null;
95+
}
96+
8297
// A version 1 `tkhd` ends its width and height 96 bytes into the body.
8398
const TRACK_HEADER_BYTES = 96;
8499

‎apps/web/src/components/chat/ChatMarkdownMedia.browser.test.tsx‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,23 @@ class HiddenIntersectionObserver {
9393
}
9494

9595
describe("ChatMarkdownVideo", () => {
96-
it("sizes a portrait video to its own shape within the height cap", async () => {
96+
it("sizes a video from the server's size, then from its own loaded size", async () => {
9797
using _observer = stubGlobal("IntersectionObserver", HiddenIntersectionObserver);
98+
// A file overwritten in place can load with a shape the server no longer describes.
9899
await using screen = await render(
99-
<ChatMarkdownVideo src="/asset/portrait.mp4" alt="portrait" copyMarkdown={undefined} />,
100+
<ChatMarkdownVideo
101+
src="/asset/portrait.mp4"
102+
alt="portrait"
103+
copyMarkdown={undefined}
104+
knownSize={{ width: 1920, height: 1080 }}
105+
/>,
100106
);
101107
const video = screen.getByLabelText("portrait").element() as HTMLVideoElement;
108+
const knownBox = video.getBoundingClientRect();
109+
expect({ width: knownBox.width, height: knownBox.height }).toEqual({
110+
width: 414,
111+
height: 232.875,
112+
});
102113
Object.defineProperties(video, {
103114
videoWidth: { value: 1080 },
104115
videoHeight: { value: 1920 },

‎apps/web/src/components/chat/ChatMarkdownMedia.tsx‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -285,11 +285,12 @@ export function ChatMarkdownVideo(props: {
285285
readonly mediaIdentity?: string | undefined;
286286
readonly actionsSource?: MediaActionSource | undefined;
287287
readonly onRetry?: (() => Promise<unknown>) | undefined;
288+
/** The server's read of the file header; the loaded video's own size replaces it. */
289+
readonly knownSize?: { readonly width: number; readonly height: number } | undefined;
288290
}) {
289291
const [videoSize, setVideoSize] = useState<{ width: number; height: number } | null>(null);
290-
const style =
291-
props.style ??
292-
(videoSize ? authoredImageSizeStyle(videoSize.width, videoSize.height) : undefined);
292+
const size = videoSize ?? props.knownSize;
293+
const style = props.style ?? (size ? authoredImageSizeStyle(size.width, size.height) : undefined);
293294
return (
294295
<MediaVideoPlayer
295296
key={props.mediaIdentity ?? props.copyMarkdown ?? props.src}
@@ -380,13 +381,13 @@ export const ChatMarkdownAssetImage = memo(function ChatMarkdownAssetImage(props
380381
: fallbackSrc + (props.srcFragment ?? "");
381382
const knownSize = assetUrl._tag === "Success" ? assetUrl.imageDimensions : undefined;
382383
const maxHeightRem = props.maxHeightRem ?? 30;
383-
const sizedStyle =
384+
const style =
384385
props.style ??
385386
(knownSize
386387
? authoredImageSizeStyle(knownSize.width, knownSize.height, maxHeightRem)
387-
: undefined);
388-
const style =
389-
sizedStyle ?? (maxHeightRem !== 30 ? { maxHeight: `${maxHeightRem}rem` } : undefined);
388+
: maxHeightRem !== 30
389+
? { maxHeight: `${maxHeightRem}rem` }
390+
: undefined);
390391
const actionsSource: MediaActionSource = {
391392
kind: props.kind ?? "image",
392393
name: props.alt || (props.kind ?? "image"),
@@ -416,7 +417,8 @@ export const ChatMarkdownAssetImage = memo(function ChatMarkdownAssetImage(props
416417
alt={props.alt}
417418
copyMarkdown={props.copyMarkdown}
418419
originalUrl={props.originalUrl}
419-
style={sizedStyle}
420+
style={props.style}
421+
knownSize={knownSize}
420422
mediaIdentity={JSON.stringify([props.environmentId, props.resource, props.srcFragment])}
421423
onRetry={refreshAssetUrl}
422424
actionsSource={actionsSource}

0 commit comments

Comments
 (0)