Skip to content

Commit 39b588c

Browse files
feat(core): swap undecodable video to its proxy at runtime (#2592)
* feat(studio-server): serve H.264 proxies from the preview route Wires the codec manifest and the transcoder into the preview surface: the route negotiates a proxy via a query param and serves it through the existing range and ETag machinery, composition HTML carries a codec map for the runtime, and hostile assets pre-warm so a first play does not wait on a cold transcode. Exposes the three subpath exports the CLI surfaces consume upstack. Drops the TEMP fallow entry added with the transcoder: it has real importers now. * fix(studio-server): publish media proxy exports * fix(parsers): scan HTML comments linearly * feat(cli): let projects opt out of automatic proxying Adds media.autoProxy to hyperframes.json plus --proxy/--no-proxy flags, and forwards the resolved value into the studio and preview servers and the vite adapter. Lands before the runtime slice that turns auto-proxying on, so the switch exists before there is any behavior to switch off. * fix(cli): align media config schema * feat(core): swap undecodable video to its proxy at runtime Adds the browser-side half: before first load the runtime consults the injected codec map and swaps a hostile source to its proxy, and if a video still reports zero decodable width it rescues it reactively. An HEVC file carrying AAC fires no error event, so zero videoWidth, not the error event, is the reliable signal. Audio elements and alpha sources are never proxied, render mode never proxies, and each swap evicts the element's stale sync state and reports once. This completes the loop: auto-proxying is live for preview and studio from here. The opt-out (media.autoProxy, --no-proxy) shipped in the previous slice.
1 parent 6458807 commit 39b588c

4 files changed

Lines changed: 956 additions & 4 deletions

File tree

‎packages/core/src/runtime/init.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import {
2424
resolveRuntimeMediaClipDuration,
2525
syncRuntimeMedia,
2626
} from "./media";
27+
import { handleErrorForProxy, handleMetadataForProxy, maybeProxyProactively } from "./mediaProxy";
2728
import { probeAndCacheElementVolume, type VolumeKeyframe } from "./mediaVolumeEnvelope.js";
2829
import { createPickerModule } from "./picker";
2930
import { createRuntimePlayer, type RuntimePlayerTransport } from "./player";
@@ -1644,10 +1645,29 @@ export function initSandboxRuntimeModular(): void {
16441645
}, METADATA_REBIND_DEBOUNCE_MS);
16451646
};
16461647

1648+
// Reactive/tertiary undecodable-media triggers (see mediaProxy.ts). Wrapped
1649+
// as event listeners here — rather than exported directly — because
1650+
// `addEventListener` hands the listener an `Event`, not the element;
1651+
// `event.currentTarget` recovers it. Bound/unbound alongside the metadata
1652+
// listeners below, reusing `metadataBoundMedia` as the once-per-element
1653+
// dedupe (no separate tracking set needed).
1654+
const onMediaLoadedMetadataForProxy = (event: Event) => {
1655+
if (event.currentTarget instanceof HTMLMediaElement) {
1656+
handleMetadataForProxy(event.currentTarget);
1657+
}
1658+
};
1659+
const onMediaErrorForProxy = (event: Event) => {
1660+
if (event.currentTarget instanceof HTMLMediaElement) {
1661+
handleErrorForProxy(event.currentTarget);
1662+
}
1663+
};
1664+
16471665
const unbindMediaMetadataListeners = () => {
16481666
for (const mediaEl of metadataBoundMedia) {
16491667
mediaEl.removeEventListener("loadedmetadata", scheduleMetadataDurationHydration);
16501668
mediaEl.removeEventListener("durationchange", scheduleMetadataDurationHydration);
1669+
mediaEl.removeEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
1670+
mediaEl.removeEventListener("error", onMediaErrorForProxy);
16511671
}
16521672
metadataBoundMedia.clear();
16531673
};
@@ -1664,6 +1684,17 @@ export function initSandboxRuntimeModular(): void {
16641684
}
16651685
mediaEl.addEventListener("loadedmetadata", scheduleMetadataDurationHydration);
16661686
mediaEl.addEventListener("durationchange", scheduleMetadataDurationHydration);
1687+
// Reactive (zero-videoWidth) + tertiary (error event) proxy-fallback
1688+
// triggers. Inert in render mode / when the codec map is absent /
1689+
// for <audio> — all guarded inside mediaProxy.ts itself.
1690+
mediaEl.addEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
1691+
mediaEl.addEventListener("error", onMediaErrorForProxy);
1692+
1693+
// Proactive proxy-fallback trigger: consult the codec map and swap
1694+
// BEFORE the eager load() below, so a known-hostile asset never even
1695+
// attempts to load (and error-flash) the original. No-op in render
1696+
// mode, for <audio>, or when the codec map is absent.
1697+
maybeProxyProactively(mediaEl);
16671698

16681699
// Eagerly preload media data so audio/video is buffered before the user
16691700
// clicks play. Without this, the first play() call fires on un-fetched

‎packages/core/src/runtime/media.ts‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,33 @@ function clampVolume(volume: number): number {
152152
return Math.max(0, Math.min(1, volume));
153153
}
154154

155+
/**
156+
* Drop every per-source sync baseline tracked for `el` — offset drift
157+
* samples, the seek-past-buffered-range retry latch, and the last
158+
* runtime-applied volume — so the next `syncRuntimeMedia` tick treats it as
159+
* a first tick (hard resync, fresh drift baseline) instead of comparing
160+
* against state computed for a different file. Used both when a clip leaves
161+
* its active window (below) and by the runtime's proxy-swap helper
162+
* (mediaProxy.ts) right after an in-place `src` swap, which points the same
163+
* element at a different file without ever leaving its active window.
164+
*/
165+
export function evictMediaSyncState(el: HTMLMediaElement): void {
166+
lastOffset.delete(el);
167+
strictDriftSamples.delete(el);
168+
seekLoadRetried.delete(el);
169+
lastRuntimeAppliedVolume.delete(el);
170+
}
171+
172+
/** Test-only seam: whether any per-source sync state is still tracked for `el`. */
173+
export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
174+
return (
175+
lastOffset.has(el) ||
176+
strictDriftSamples.has(el) ||
177+
seekLoadRetried.has(el) ||
178+
lastRuntimeAppliedVolume.has(el)
179+
);
180+
}
181+
155182
// fallow-ignore-next-line complexity
156183
export function syncRuntimeMedia(params: {
157184
clips: RuntimeMediaClip[];
@@ -400,10 +427,7 @@ export function syncRuntimeMedia(params: {
400427
}
401428
// Clip left its active window — drop the offset baseline so the next
402429
// activation (e.g. re-entering a sub-composition) gets a hard resync.
403-
lastOffset.delete(el);
404-
strictDriftSamples.delete(el);
405-
seekLoadRetried.delete(el);
406-
lastRuntimeAppliedVolume.delete(el);
430+
evictMediaSyncState(el);
407431
if (!el.paused) el.pause();
408432
}
409433
}

0 commit comments

Comments
 (0)