Skip to content

Commit 659ef56

Browse files
fix(producer): export sizes an unsized phone video as it plays, not as stored (heygen-com#7228)
* fix(producer): export sizes an unsized phone video as it plays, not as stored * fix(engine): keep the stored-size check on the video layer path
1 parent ada71da commit 659ef56

5 files changed

Lines changed: 95 additions & 9 deletions

File tree

‎packages/engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,7 @@ export { readWebGlVendorInfoFromCanvas } from "./utils/readWebGlVendorInfoFromCa
381381

382382
export {
383383
extractMediaMetadata,
384+
displaySize,
384385
extractVideoMetadata,
385386
extractFinalVideoFrameTimestamp,
386387
extractAudioMetadata,

‎packages/engine/src/utils/ffprobe.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,63 @@ describe("extractMediaMetadata", () => {
7777
});
7878
});
7979

80+
describe("a phone clip's display size", () => {
81+
afterEach(() => {
82+
vi.resetModules();
83+
});
84+
85+
async function probed(stream: Record<string, unknown>) {
86+
const { spawn } = createSpawnSpy([
87+
{
88+
kind: "exit",
89+
code: 0,
90+
stdout: JSON.stringify({
91+
streams: [
92+
{
93+
codec_type: "video",
94+
codec_name: "hevc",
95+
width: 1920,
96+
height: 1080,
97+
r_frame_rate: "30/1",
98+
avg_frame_rate: "30/1",
99+
...stream,
100+
},
101+
],
102+
format: { duration: "3" },
103+
}),
104+
},
105+
]);
106+
vi.resetModules();
107+
vi.doMock("child_process", () => ({ spawn }));
108+
const { extractMediaMetadata: mocked, displaySize } = await import("./ffprobe.js");
109+
const dir = mkdtempSync(`${tmpdir()}/hf-turned-`);
110+
try {
111+
writeFileSync(`${dir}/clip.mov`, "");
112+
return displaySize(await mocked(`${dir}/clip.mov`));
113+
} finally {
114+
rmSync(dir, { recursive: true, force: true });
115+
}
116+
}
117+
118+
it("is portrait for an iPhone clip stored landscape with a -90 display matrix", async () => {
119+
const shown = await probed({ side_data_list: [{ rotation: -90 }] });
120+
expect(shown).toEqual({ width: 1080, height: 1920 });
121+
});
122+
123+
it("reads an older muxer's rotate tag, and keeps a half turn's size", async () => {
124+
expect(await probed({ tags: { rotate: "270" } })).toEqual({ width: 1080, height: 1920 });
125+
expect(await probed({ side_data_list: [{ rotation: 180 }] })).toEqual({
126+
width: 1920,
127+
height: 1080,
128+
});
129+
});
130+
131+
it("widens anamorphic pixels, then turns, as Chrome lays the element out", async () => {
132+
const shown = await probed({ sample_aspect_ratio: "4:3", side_data_list: [{ rotation: 90 }] });
133+
expect(shown).toEqual({ width: 1080, height: 2560 });
134+
});
135+
});
136+
80137
describe("extractMediaMetadata nb_frames duration cross-check", () => {
81138
afterEach(() => {
82139
vi.resetModules();

‎packages/engine/src/utils/ffprobe.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ export interface VideoMetadata {
225225
colorRange?: string;
226226
pixelFormat?: string;
227227
sampleAspectRatio?: number;
228+
rotation?: number;
228229
/** Decoded frame count from the video stream's `nb_frames`. Omitted when the
229230
* container does not surface a reliable count (still images, malformed
230231
* streams, or muxes that require `-count_packets` to populate). Callers
@@ -274,6 +275,7 @@ interface FFProbeStream {
274275
color_primaries?: string;
275276
color_space?: string;
276277
tags?: Record<string, string>;
278+
side_data_list?: { rotation?: number }[];
277279
disposition?: { attached_pic?: number };
278280
}
279281

@@ -855,6 +857,24 @@ export function parseFrameRate(frameRateStr: string | undefined): number {
855857
return rounded > 0 ? rounded : 0.01;
856858
}
857859

860+
function displayRotation(stream: FFProbeStream): { rotation?: number } {
861+
const turn = Number(
862+
stream.side_data_list?.find((data) => data.rotation !== undefined)?.rotation ??
863+
stream.tags?.rotate ??
864+
0,
865+
);
866+
return Number.isFinite(turn) && turn % 360 !== 0 ? { rotation: turn } : {};
867+
}
868+
869+
export function displaySize(
870+
metadata: Pick<VideoMetadata, "width" | "height" | "sampleAspectRatio" | "rotation">,
871+
): { width: number; height: number } {
872+
const width = Math.round(metadata.width * (metadata.sampleAspectRatio ?? 1));
873+
return Math.abs(metadata.rotation ?? 0) % 180 === 90
874+
? { width: metadata.height, height: width }
875+
: { width, height: metadata.height };
876+
}
877+
858878
export function parseSampleAspectRatio(value: string | undefined): number | undefined {
859879
const [num, den] = (value ?? "").split(":").map(Number);
860880
if (!num || !den || !Number.isFinite(num) || !Number.isFinite(den)) return undefined;
@@ -1052,6 +1072,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
10521072
colorRange: videoStream.color_range,
10531073
pixelFormat,
10541074
...(sampleAspectRatio === undefined ? {} : { sampleAspectRatio }),
1075+
...displayRotation(videoStream),
10551076
frames,
10561077
};
10571078
})();

‎packages/producer/src/services/renderOrchestrator.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1327,6 +1327,19 @@ describe("collectVideoMetadataHints", () => {
13271327
{ id: "video2", width: 1080, height: 1920 },
13281328
]);
13291329
});
1330+
1331+
it("sizes a phone's portrait clip as it plays, not as it is stored", () => {
1332+
const stored = { width: 1920, height: 1080, durationSeconds: 3 };
1333+
expect(
1334+
collectVideoMetadataHints([
1335+
{ videoId: "portrait", metadata: { ...stored, rotation: -90 } },
1336+
{ videoId: "upside-down", metadata: { ...stored, rotation: 180 } },
1337+
]),
1338+
).toEqual([
1339+
{ id: "portrait", width: 1080, height: 1920 },
1340+
{ id: "upside-down", width: 1920, height: 1080 },
1341+
]);
1342+
});
13301343
});
13311344

13321345
describe("resolveRenderWorkerCount", () => {

‎packages/producer/src/services/renderOrchestrator.ts‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ import {
109109
getDrawElementVerificationDetails,
110110
augmentProtocolTimeoutError,
111111
augmentPageNavigationTimeoutError,
112+
displaySize,
112113
type MotionBlurOptions,
113114
} from "@hyperframes/engine";
114115
import {
@@ -954,10 +955,7 @@ export function collectVideoReadinessSkipIds(
954955

955956
interface ExtractedVideoReadinessInput {
956957
videoId: string;
957-
metadata: {
958-
width: number;
959-
height: number;
960-
};
958+
metadata: Parameters<typeof displaySize>[0];
961959
}
962960

963961
function hasUsableVideoDimensions(metadata: ExtractedVideoReadinessInput["metadata"]) {
@@ -974,11 +972,7 @@ export function collectVideoMetadataHints(
974972
): CaptureVideoMetadataHint[] {
975973
return extractedVideos
976974
.filter((video) => hasUsableVideoDimensions(video.metadata))
977-
.map((video) => ({
978-
id: video.videoId,
979-
width: video.metadata.width,
980-
height: video.metadata.height,
981-
}))
975+
.map((video) => ({ id: video.videoId, ...displaySize(video.metadata) }))
982976
.sort((a, b) => a.id.localeCompare(b.id));
983977
}
984978

0 commit comments

Comments
 (0)