Skip to content

Commit e8371a7

Browse files
feat(media): alpha-capable authoring proxies (#2598)
* feat(media): alpha-capable authoring proxies Alpha sources were refused a proxy before the codec map ever asked whether the browser could decode them, so a ProRes 4444 alpha file (which no browser previews at all) rendered black forever, while an alpha WebM (which previews fine) was already covered by the browser-safe check on the next line. The alpha veto earned nothing and cost the one case that needed help. Alpha is now a target-codec choice rather than a veto: alpha sources transcode to VP9 + yuva420p in WebM, everything else keeps the existing H.264/MP4 path byte for byte. Only files no browser can preview are proxied, which is the rule the runtime already followed everywhere else. WebM cannot carry AAC, so the VP9 path uses Opus and drops the MP4-only faststart flag. PROXY_PARAMS_VERSION moves to v3 so clients stop serving the previously cached proxies. Safari does not decode VP9 alpha and still shows black for alpha sources, as it does today: this is better on Chromium and Firefox and no worse anywhere. * fix(media): infer proxy variant for rescue * fix(media): preserve alpha proxy hardening after restack
1 parent 8c1b6c5 commit e8371a7

21 files changed

Lines changed: 600 additions & 239 deletions

‎packages/cli/src/commands/play.test.ts‎

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,20 @@ const mediaMocks = vi.hoisted(() => ({
5959
representativeMime: "video/mp4",
6060
hasAlpha: false,
6161
})),
62-
decideMediaProxyEligibility: vi.fn(
63-
(facts: { hasAlpha: boolean; browserHostile: boolean } | null) =>
64-
facts?.hasAlpha ? { eligible: false, reason: "alpha_source" } : { eligible: true },
62+
decideMediaProxyEligibility: vi.fn((facts: { browserHostile: boolean } | null) =>
63+
facts?.browserHostile ? { eligible: true } : { eligible: false, reason: "browser_safe_codec" },
6564
),
65+
isProxyVariant: (value: string) => value === "h264" || value === "vp9",
66+
isProxyVariantRequest: (value: string) => value === "auto" || value === "h264" || value === "vp9",
67+
proxyVariantFor: (facts: { hasAlpha?: boolean }) => (facts.hasAlpha ? "vp9" : "h264"),
68+
resolveProxyVariantRequest: (request: "auto" | "h264" | "vp9", facts: { hasAlpha?: boolean }) => {
69+
const expected = facts.hasAlpha ? "vp9" : "h264";
70+
return request === "auto" || request === expected ? expected : null;
71+
},
72+
PROXY_VARIANT_CONFIG: {
73+
h264: { extension: ".mp4", contentType: "video/mp4" },
74+
vp9: { extension: ".webm", contentType: "video/webm" },
75+
},
6676
}));
6777

6878
vi.mock("@hyperframes/studio-server/proxy-transcoder", () => ({
@@ -109,7 +119,7 @@ afterEach(() => {
109119
dir = undefined;
110120
});
111121

112-
it("rejects direct proxy requests for alpha sources", async () => {
122+
it("serves direct VP9 proxy requests for alpha sources", async () => {
113123
const project = tmpProject();
114124
writeFileSync(join(project.dir, "clip.mov"), "alpha-prores");
115125
mediaMocks.probeAssetCodec.mockResolvedValueOnce({
@@ -118,17 +128,20 @@ it("rejects direct proxy requests for alpha sources", async () => {
118128
representativeMime: "video/quicktime",
119129
hasAlpha: true,
120130
});
121-
mediaMocks.decideMediaProxyEligibility.mockReturnValueOnce({
122-
eligible: false,
123-
reason: "alpha_source",
124-
});
131+
const proxyPath = join(project.dir, "proxy.webm");
132+
writeFileSync(proxyPath, "vp9-alpha-proxy");
133+
mocks.resolveProxy.mockResolvedValueOnce(proxyPath);
125134
const app = await buildApp(project, true);
126135

127-
const res = await app.request("/composition/clip.mov?hf-proxy=h264");
136+
const res = await app.request("/composition/clip.mov?hf-proxy=auto");
128137

129-
expect(res.status).toBe(422);
130-
expect(await res.text()).toContain("alpha_source");
131-
expect(mocks.resolveProxy).not.toHaveBeenCalled();
138+
expect(res.status).toBe(200);
139+
expect(res.headers.get("content-type")).toBe("video/webm");
140+
expect(mocks.resolveProxy).toHaveBeenCalledWith(
141+
project.dir,
142+
join(project.dir, "clip.mov"),
143+
"vp9",
144+
);
132145
});
133146

134147
async function buildApp(project: ProjectDir, autoProxy: boolean): Promise<Hono> {
@@ -163,7 +176,11 @@ describe("registerCompositionRoute", () => {
163176

164177
expect(res.status).toBe(200);
165178
expect(await res.text()).toBe("transcoded-h264-bytes");
166-
expect(mocks.resolveProxy).toHaveBeenCalledWith(project.dir, join(project.dir, "clip.mp4"));
179+
expect(mocks.resolveProxy).toHaveBeenCalledWith(
180+
project.dir,
181+
join(project.dir, "clip.mp4"),
182+
"h264",
183+
);
167184
});
168185

169186
it("serves ?hf-proxy=h264 for a .mov hostile asset as Content-Type video/mp4 (the proxy IS mp4)", async () => {

‎packages/cli/src/commands/play.ts‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@ import {
4444
} from "@hyperframes/studio-server/proxy-transcoder";
4545
import {
4646
decideMediaProxyEligibility,
47+
isProxyVariantRequest,
4748
probeAssetCodec,
49+
resolveProxyVariantRequest,
50+
PROXY_VARIANT_CONFIG,
4851
} from "@hyperframes/studio-server/media-codec-map";
4952

5053
export default defineCommand({
@@ -72,7 +75,7 @@ export default defineCommand({
7275
proxy: {
7376
type: "boolean",
7477
description:
75-
"Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached H.264 proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)",
78+
"Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached authoring proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)",
7679
negativeDescription: "Disable auto-proxying of browser-hostile video codecs",
7780
},
7881
},
@@ -192,8 +195,8 @@ export default defineCommand({
192195
* Registers the `/composition/*` route: serves composition HTML (runtime +
193196
* `__HF_MEDIA_CODEC_MAP__` injected) and asset files, with byte-Range support
194197
* (`play` previously did a whole-file `readFileSync`, so seeking/duration
195-
* probing on media elements never worked) and a `?hf-proxy=h264` branch that
196-
* serves the cached H.264 authoring proxy for a browser-hostile video asset
198+
* probing on media elements never worked) and a `?hf-proxy=` branch that
199+
* serves the alpha-aware authoring proxy for a browser-hostile video asset
197200
* (per docs/plans/2026-07-14-002-feat-transparent-media-proxies-plan.md,
198201
* unit U4). Exported standalone (rather than inlined in `run()`) so tests can
199202
* exercise it via `app.request(...)` without booting a real HTTP listener,
@@ -228,19 +231,28 @@ export async function registerCompositionRoute(
228231
}
229232

230233
const contentType = assetContentType(filePath);
231-
if (ctx.req.query("hf-proxy") === "h264") {
234+
const proxyParam = ctx.req.query("hf-proxy");
235+
if (proxyParam !== undefined && isProxyVariantRequest(proxyParam)) {
232236
// Opt-out (or a non-video asset) 404s the param without attempting a
233237
// transcode; a missing asset already 404'd above.
234238
if (!autoProxy || !contentType.startsWith("video/")) return ctx.text("Not found", 404);
235239
try {
236-
const eligibility = decideMediaProxyEligibility(await probeAssetCodec(filePath));
240+
const facts = await probeAssetCodec(filePath);
241+
const eligibility = decideMediaProxyEligibility(facts);
237242
if (!eligibility.eligible) {
238243
return ctx.text(`Media proxy unavailable: ${eligibility.reason}`, 422);
239244
}
240-
const proxyPath = await resolveProxy(project.dir, filePath);
241-
// The proxy IS an mp4 regardless of the source's extension (.mov,
242-
// .mkv, ...) — serve its real type, matching the preview route.
243-
return buildRangeResponse(proxyPath, "video/mp4", ctx.req.header("Range"));
245+
if (!facts) return ctx.text("Media proxy unavailable: unknown_codec", 422);
246+
const proxyVariant = resolveProxyVariantRequest(proxyParam, facts);
247+
if (!proxyVariant) {
248+
return ctx.text("Media proxy variant does not match asset", 422);
249+
}
250+
const proxyPath = await resolveProxy(project.dir, filePath, proxyVariant);
251+
return buildRangeResponse(
252+
proxyPath,
253+
PROXY_VARIANT_CONFIG[proxyVariant].contentType,
254+
ctx.req.header("Range"),
255+
);
244256
} catch (err) {
245257
if (err instanceof ProxyCapacityError) {
246258
return ctx.text(`Proxy transcode deferred: ${err.message}`, 503, {

‎packages/cli/src/commands/preview.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ export default defineCommand({
192192
proxy: {
193193
type: "boolean",
194194
description:
195-
"Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached H.264 proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)",
195+
"Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached authoring proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)",
196196
negativeDescription: "Disable auto-proxying of browser-hostile video codecs",
197197
},
198198
},

‎packages/cli/src/utils/checkBrowser.test.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ vi.mock("./staticProjectServer.js", () => ({
6969
vi.mock("@hyperframes/studio-server/media-codec-map", async (importOriginal) => ({
7070
...(await importOriginal<typeof import("@hyperframes/studio-server/media-codec-map")>()),
7171
scanProjectMediaCodecMap: mocks.scanProjectMediaCodecMap,
72+
proxyVariantFor: (facts: { hasAlpha?: boolean }) => (facts.hasAlpha ? "vp9" : "h264"),
7273
}));
7374
vi.mock("@hyperframes/studio-server/proxy-transcoder", () => ({
7475
resolveProxy: mocks.resolveProxy,
@@ -341,7 +342,7 @@ it("surfaces the runtime's media-proxy-fallback console.info line as an info fin
341342
const fallbackMessage = fakeConsoleMessage(
342343
"info",
343344
'[hyperframes] runtime_media_proxy_fallback: "assets/clip.mp4" uses a codec (hevc) this browser can\'t decode; ' +
344-
"auto-swapped to an H.264 proxy for this preview only. Render output is unaffected.",
345+
"auto-swapped to an authoring proxy for this preview only. Render output is unaffected.",
345346
);
346347
const unrelatedInfo = fakeConsoleMessage("info", "[hyperframes] render runtime fps 30");
347348
const authorInfo = fakeConsoleMessage("info", "debug runtime_media_proxy_probe");
@@ -455,7 +456,11 @@ describe("preResolveHostileMediaProxies", () => {
455456
await Promise.resolve();
456457
await Promise.resolve();
457458
expect(mocks.resolveProxy).toHaveBeenCalledTimes(1);
458-
expect(mocks.resolveProxy).toHaveBeenCalledWith(projectDir, join(projectDir, "clip.mp4"));
459+
expect(mocks.resolveProxy).toHaveBeenCalledWith(
460+
projectDir,
461+
join(projectDir, "clip.mp4"),
462+
"h264",
463+
);
459464
expect(settled).toBe(false); // still waiting on the hostile entry's transcode
460465

461466
resolveTranscode?.();
@@ -500,10 +505,10 @@ describe("preResolveHostileMediaProxies", () => {
500505
it("does not pre-resolve a hostile asset rejected by the shared proxy policy", async () => {
501506
const projectDir = mkProjectDir();
502507
mocks.scanProjectMediaCodecMap.mockResolvedValue({
503-
"/alpha.mov": {
504-
codecName: "prores",
508+
"/alpha.webm": {
509+
codecName: "vp9",
505510
browserHostile: true,
506-
representativeMime: null,
511+
representativeMime: 'video/webm; codecs="vp09.00.10.08"',
507512
hasAlpha: true,
508513
},
509514
});

‎packages/cli/src/utils/checkBrowser.ts‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import { serveStaticProjectHtml } from "./staticProjectServer.js";
2121
import { resolveAutoProxy } from "./projectConfig.js";
2222
import {
2323
decideMediaProxyEligibility,
24+
proxyVariantFor,
2425
scanProjectMediaCodecMap,
2526
} from "@hyperframes/studio-server/media-codec-map";
2627
import { resolveProxy } from "@hyperframes/studio-server/proxy-transcoder";
@@ -115,15 +116,19 @@ export async function preResolveHostileMediaProxies(
115116
);
116117
return;
117118
}
118-
const hostilePathnames = Object.entries(codecMap)
119-
.filter(([, facts]) => decideMediaProxyEligibility(facts).eligible)
120-
.map(([pathname]) => pathname);
121-
if (hostilePathnames.length === 0) return;
119+
const hostileEntries = Object.entries(codecMap).filter(
120+
([, facts]) => decideMediaProxyEligibility(facts).eligible,
121+
);
122+
if (hostileEntries.length === 0) return;
122123

123124
const startedAt = Date.now();
124125
const results = await Promise.allSettled(
125-
hostilePathnames.map((pathname) =>
126-
resolveProxy(projectDir, resolve(projectDir, pathname.replace(/^\/+/, ""))),
126+
hostileEntries.map(([pathname, facts]) =>
127+
resolveProxy(
128+
projectDir,
129+
resolve(projectDir, pathname.replace(/^\/+/, "")),
130+
proxyVariantFor(facts),
131+
),
127132
),
128133
);
129134
const failed = results.filter((result) => result.status === "rejected").length;

‎packages/cli/src/utils/projectConfig.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export interface ProjectConfigPaths {
2626
export interface ProjectConfigMedia {
2727
/**
2828
* Auto-transcode browser-hostile video codecs (e.g. HEVC) to a cached
29-
* H.264 proxy for supported preview surfaces. Render always uses the
29+
* alpha-aware authoring proxy for supported preview surfaces. Render always uses the
3030
* original file regardless of this setting. Default true.
3131
*/
3232
autoProxy?: boolean;

‎packages/cli/src/utils/publishProxyBake.test.ts‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ vi.mock("@hyperframes/studio-server/proxy-transcoder", () => ({
4949

5050
vi.mock("@hyperframes/studio-server/media-codec-map", () => ({
5151
scanProjectMediaCodecMap: mocks.scanProjectMediaCodecMap,
52+
proxyVariantFor: (facts: { hasAlpha?: boolean }) => (facts.hasAlpha ? "vp9" : "h264"),
5253
}));
5354

5455
const { bakeMediaProxies, PROXY_ARCHIVE_PREFIX } = await import("./publishProxyBake.js");
@@ -60,10 +61,10 @@ const { bakeMediaProxies, PROXY_ARCHIVE_PREFIX } = await import("./publishProxyB
6061
const PROJECT_DIR = resolve("/project");
6162

6263
const tempDirs: string[] = [];
63-
function tmpProxyFile(content: string): string {
64+
function tmpProxyFile(content: string, extension = ".mp4"): string {
6465
const dir = mkdtempSync(join(tmpdir(), "hf-publish-proxy-bake-"));
6566
tempDirs.push(dir);
66-
const path = join(dir, "proxy.mp4");
67+
const path = join(dir, `proxy${extension}`);
6768
writeFileSync(path, content, "utf-8");
6869
return path;
6970
}
@@ -108,7 +109,11 @@ describe("bakeMediaProxies", () => {
108109
expect(html).toContain(proxyEntries[0]!);
109110
expect(html).not.toContain('src="clip.mp4"');
110111

111-
expect(mocks.resolveProxy).toHaveBeenCalledWith(PROJECT_DIR, join(PROJECT_DIR, "clip.mp4"));
112+
expect(mocks.resolveProxy).toHaveBeenCalledWith(
113+
PROJECT_DIR,
114+
join(PROJECT_DIR, "clip.mp4"),
115+
"h264",
116+
);
112117
expect(mocks.waitForProxy).toHaveBeenCalledWith(expect.any(Promise), 15 * 60 * 1000);
113118
expect(manifest).toEqual({ proxied: ["/clip.mp4"], skippedAlpha: [], failed: [] });
114119
});
@@ -191,7 +196,7 @@ describe("bakeMediaProxies", () => {
191196
expect(html).not.toContain("assets/my%20clip.mp4");
192197
});
193198

194-
it("reports an alpha-bearing hostile asset as skipped while keeping HTML on the original", async () => {
199+
it("bakes an alpha-bearing hostile asset as a VP9 WebM proxy", async () => {
195200
mocks.scanProjectMediaCodecMap.mockResolvedValue({
196201
"/clip.mov": {
197202
codecName: "prores",
@@ -200,19 +205,27 @@ describe("bakeMediaProxies", () => {
200205
hasAlpha: true,
201206
},
202207
});
208+
const proxyPath = tmpProxyFile("PROXY_VP9_ALPHA_BYTES", ".webm");
209+
mocks.resolveProxy.mockResolvedValue(proxyPath);
203210
const fileContents = new Map<string, Buffer>([
204211
["index.html", indexHtml(`<video src="clip.mov" muted></video>`)],
205212
["clip.mov", Buffer.from("ORIGINAL_PRORES_4444_BYTES", "utf-8")],
206213
]);
207214

208215
const manifest = await bakeMediaProxies(PROJECT_DIR, fileContents);
209216

210-
expect(mocks.resolveProxy).not.toHaveBeenCalled();
211-
expect([...fileContents.keys()].some((k) => k.startsWith(`${PROXY_ARCHIVE_PREFIX}/`))).toBe(
212-
false,
217+
expect(mocks.resolveProxy).toHaveBeenCalledWith(
218+
PROJECT_DIR,
219+
join(PROJECT_DIR, "clip.mov"),
220+
"vp9",
221+
);
222+
const proxyEntries = [...fileContents.keys()].filter((key) =>
223+
key.startsWith(`${PROXY_ARCHIVE_PREFIX}/`),
213224
);
214-
expect(fileContents.get("index.html")?.toString("utf-8")).toContain('src="clip.mov"');
215-
expect(manifest).toEqual({ proxied: [], skippedAlpha: ["/clip.mov"], failed: [] });
225+
expect(proxyEntries).toEqual([`${PROXY_ARCHIVE_PREFIX}/proxy.webm`]);
226+
expect(fileContents.get(proxyEntries[0]!)?.toString("utf-8")).toBe("PROXY_VP9_ALPHA_BYTES");
227+
expect(fileContents.get("index.html")?.toString("utf-8")).toContain(proxyEntries[0]!);
228+
expect(manifest).toEqual({ proxied: ["/clip.mov"], skippedAlpha: [], failed: [] });
216229
});
217230

218231
it("returns deterministic manifest ordering across concurrent transcodes", async () => {

‎packages/cli/src/utils/publishProxyBake.ts‎

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Publish-time proxy baking (U6 of
33
* docs/plans/2026-07-14-002-feat-transparent-media-proxies-plan.md).
44
*
5-
* Published pages are static (no server), so the on-demand `?hf-proxy=h264`
5+
* Published pages are static (no server), so the on-demand `?hf-proxy=`
66
* negotiation the preview/play surfaces use (U3/U4) isn't possible there.
77
* Instead this scans the archive's HTML entries for local `<video src>`
88
* references to browser-hostile codecs (HEVC, ProRes, ...), transcodes each
@@ -20,8 +20,8 @@
2020
* `cloud render` never calls this: it uses `createPublishArchive` directly,
2121
* which has no baking hook (R2 in the plan).
2222
*
23-
* Alpha-bearing sources remain explicit skips because H.264 would destroy
24-
* transparency. A failed opaque-hostile transcode aborts publish with a
23+
* Alpha-bearing sources bake as VP9/WebM so transparency survives. A failed
24+
* hostile transcode aborts publish with a
2525
* structured manifest rather than silently shipping an unplayable asset.
2626
*/
2727

@@ -35,6 +35,7 @@ import {
3535
resolveLocalAssetCandidates,
3636
} from "@hyperframes/parsers/asset-resolution";
3737
import {
38+
proxyVariantFor,
3839
scanProjectMediaCodecMap,
3940
type HtmlSourceLike,
4041
} from "@hyperframes/studio-server/media-codec-map";
@@ -51,6 +52,7 @@ export const PROXY_ARCHIVE_PREFIX = "_proxy";
5152

5253
export interface ProxyBakeManifest {
5354
proxied: string[];
55+
/** @deprecated Alpha sources are proxied as VP9; retained as an always-empty compatibility field. */
5456
skippedAlpha: string[];
5557
failed: Array<{ path: string; error: string }>;
5658
}
@@ -75,11 +77,11 @@ function isHtmlEntry(path: string): boolean {
7577
}
7678

7779
/**
78-
* Mutates `fileContents` in place: adds a `_proxy/<hash>.mp4` entry for every
80+
* Mutates `fileContents` in place: adds a variant-specific `_proxy/<hash>` entry for every
7981
* browser-hostile local video asset referenced from the archive's HTML, and
8082
* rewrites those HTML entries' matching `<video src>` attributes to point at
8183
* the proxy. Returns a structured manifest; throws ProxyBakeError when any
82-
* required opaque proxy cannot be prepared.
84+
* required proxy cannot be prepared.
8385
*/
8486
export async function bakeMediaProxies(
8587
projectDir: string,
@@ -97,17 +99,7 @@ export async function bakeMediaProxies(
9799

98100
const codecMap = await scanProjectMediaCodecMap(absProjectDir, htmlSources);
99101
const hostileEntries = Object.entries(codecMap).filter(([, facts]) => facts.browserHostile);
100-
const hostilePathnames: string[] = [];
101-
for (const [pathname, facts] of hostileEntries) {
102-
if (facts.hasAlpha) {
103-
// Alpha sources are never proxied: an H.264 proxy would destroy the
104-
// transparency (e.g. ProRes 4444 alpha). Keep the original in place.
105-
manifest.skippedAlpha.push(pathname);
106-
continue;
107-
}
108-
hostilePathnames.push(pathname);
109-
}
110-
if (hostilePathnames.length === 0) return manifest;
102+
if (hostileEntries.length === 0) return manifest;
111103

112104
// Absolute source path -> archive path of its baked proxy. Built by
113105
// resolving each map key back to an absolute path the same way
@@ -117,11 +109,11 @@ export async function bakeMediaProxies(
117109
const proxyByAbsolutePath = new Map<string, string>();
118110

119111
await Promise.all(
120-
hostilePathnames.map(async (pathname) => {
112+
hostileEntries.map(async ([pathname, facts]) => {
121113
const absoluteSourcePath = resolve(absProjectDir, pathname.replace(/^\/+/, ""));
122114
try {
123115
const proxyPath = await waitForProxy(
124-
resolveProxy(absProjectDir, absoluteSourcePath),
116+
resolveProxy(absProjectDir, absoluteSourcePath, proxyVariantFor(facts)),
125117
TRANSCODE_TIMEOUT_MS,
126118
);
127119
const archivePath = `${PROXY_ARCHIVE_PREFIX}/${basename(proxyPath)}`;

0 commit comments

Comments
 (0)