Skip to content

Commit d83bf99

Browse files
feat(cli): check catches audio the render would lose or play as silence (#6439)
* feat(cli): check catches audio the render would lose or play as silence check gains an Audio section, read from the media list the render mixes: - error: a clip's source has no audio the render can read (its mix fails, the film ships without sound) - warning: a clip starts past the end of its file, or the part it plays is silent - info: the film has no sound at all The bundle and media list are now built once and shared with the HDR inspection; the audio pass runs while the browser pass does. Fixes #2775 * fix(cli): check reports an unreadable audio source the way render handles it A source with no audio track is refused by render's own media-type rule, now exported from the producer and called by check; a source that does not decode as audio fails the mix and stops the render. * perf(cli): check samples a long clip at a few spots before reading all of it for silence Sound at any spot ends the decode; a clip is only called silent after its whole played part is read. * chore(cli): drop a comment the shared finding helper's name already says
1 parent 0979e32 commit d83bf99

13 files changed

Lines changed: 596 additions & 80 deletions

File tree

‎docs/packages/cli.mdx‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1022,7 +1022,7 @@ did, in one command and one browser session.
10221022

10231023
```bash
10241024
npx hyperframes check [dir]
1025-
npx hyperframes check [dir] --json # {ok, browserSkipped, scope, lint, runtime, layout, motion, contrast, craft, hdr, snapshots}
1025+
npx hyperframes check [dir] --json # {ok, browserSkipped, scope, lint, runtime, layout, motion, contrast, craft, audio, hdr, snapshots}
10261026
npx hyperframes check [dir] --snapshots
10271027
npx hyperframes check [dir] --at 1.5,4,7.25
10281028
npx hyperframes check [dir] --strict # warnings fail too
@@ -1035,6 +1035,11 @@ layout defects (overflow, clipping, held overlaps, occlusion, coordinate-frame
10351035
drift), `*.motion.json` assertions, WCAG AA contrast, and craft warnings
10361036
(see [Craft warnings](#craft-warnings)).
10371037

1038+
The `audio` section reads every clip the render mixes, while the browser runs:
1039+
a source with no audio track or that cannot be read as audio (error: render
1040+
stops), a clip that starts past the end of its file, a clip whose played part is
1041+
silent (warnings), and a film with no sound at all (info).
1042+
10381043
When the browser never ran (lint errors, a linter crash, or a browser launch
10391044
failure), `browserSkipped` is `true` and the `layout`, `motion` and `contrast`
10401045
sections are empty, not clean.

‎packages/cli/src/commands/check.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,7 @@ function dependencies(
312312
captureFindingCrops?: CheckDependencies["captureFindingCrops"];
313313
captureFindingFrames?: CheckDependencies["captureFindingFrames"];
314314
inspectHdrAutoPromotion?: NonNullable<CheckDependencies["inspectHdrAutoPromotion"]>;
315+
inspectAudio?: NonNullable<CheckDependencies["inspectAudio"]>;
315316
} = {},
316317
): { deps: CheckDependencies; runBrowserCheck: ReturnType<typeof vi.fn> } {
317318
const runBrowserCheck = vi.fn(
@@ -338,7 +339,9 @@ function dependencies(
338339
captureFindingCrops: options.captureFindingCrops ?? vi.fn(async () => []),
339340
captureFindingFrames:
340341
options.captureFindingFrames ?? vi.fn(async () => ({ written: [], inspected: [] })),
342+
collectMedia: vi.fn(async () => ({ videos: [], images: [], audios: [], html: "" })),
341343
inspectHdrAutoPromotion: options.inspectHdrAutoPromotion ?? vi.fn(async () => null),
344+
inspectAudio: options.inspectAudio ?? vi.fn(async () => []),
342345
};
343346
return { deps, runBrowserCheck };
344347
}
@@ -719,6 +722,69 @@ it("distinguishes unavailable HDR inspection from no promotion", async () => {
719722
expect(report.hdr).toEqual({ autoPromotion: null, inspection: "unavailable" });
720723
});
721724

725+
it("fails the check on an audio error and reports it in its own section", async () => {
726+
const unreadable: CheckFinding = {
727+
code: "audio_source_unreadable",
728+
severity: "error",
729+
message: "voice.wav has no audio the render can read",
730+
selector: "#voice",
731+
dataAttributes: {},
732+
sourceFile: "index.html",
733+
bbox: { x: 0, y: 0, width: 0, height: 0 },
734+
time: 1,
735+
};
736+
const { report } = await runScenario(
737+
fakeDriver(),
738+
{},
739+
{ inspectAudio: vi.fn(async () => [unreadable]) },
740+
);
741+
742+
expect(report.audio).toMatchObject({ errorCount: 1, findings: [unreadable] });
743+
expect(report.ok).toBe(false);
744+
});
745+
746+
it("still checks audio when lint errors skip the browser pass", async () => {
747+
const { report, browser } = await runScenario(
748+
fakeDriver(),
749+
{},
750+
{
751+
lint: lintWith("error", "missing_timeline_registry", "No timeline"),
752+
inspectAudio: vi.fn(async () => [
753+
{
754+
code: "audio_none",
755+
severity: "info" as const,
756+
message: "The film has no sound",
757+
selector: "[data-composition-id]",
758+
dataAttributes: {},
759+
sourceFile: "index.html",
760+
bbox: { x: 0, y: 0, width: 0, height: 0 },
761+
time: 0,
762+
},
763+
]),
764+
},
765+
);
766+
767+
expect(browser).not.toHaveBeenCalled();
768+
expect(report.audio).toMatchObject({ infoCount: 1, findings: [{ code: "audio_none" }] });
769+
});
770+
771+
it("reports an audio check that could not run instead of failing the check", async () => {
772+
const { report } = await runScenario(
773+
fakeDriver(),
774+
{},
775+
{ inspectAudio: vi.fn(async () => Promise.reject(new Error("ffprobe unavailable"))) },
776+
);
777+
778+
expect(report.audio.findings).toMatchObject([
779+
{
780+
code: "audio_check_unavailable",
781+
severity: "info",
782+
message: expect.stringContaining("ffprobe unavailable"),
783+
},
784+
]);
785+
expect(report.ok).toBe(true);
786+
});
787+
722788
it("threads --no-proxy into the browser check options", async () => {
723789
const { report } = await runScenario(fakeDriver());
724790
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
@@ -1276,6 +1342,7 @@ function reportWithFindings(overrides: Partial<CheckReport> = {}): CheckReport {
12761342
motion: { ...emptySection(), enabled: false, samples: 0 },
12771343
contrast: { ...emptySection(), enabled: true, samples: [], checked: 0, passed: 0 },
12781344
craft: { ...emptySection(), enabled: false, step: 0, samples: 0 },
1345+
audio: emptySection(),
12791346
hdr: { autoPromotion: null, inspection: "available" },
12801347
snapshots: { enabled: false, files: [], times: [], findingFiles: [] },
12811348
...overrides,

‎packages/cli/src/commands/check.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -646,6 +646,7 @@ function printHumanReport(report: CheckReport): void {
646646
printSection("Motion", report.motion);
647647
printContrastSection(report);
648648
printCraftSection(report);
649+
printSection("Audio", report.audio);
649650
printSnapshotSection(report);
650651
console.log();
651652
const label = report.ok ? c.success("Check passed") : c.error("Check failed");
Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { afterAll, describe, expect, it } from "vitest";
5+
import type { AudioElement } from "@hyperframes/engine";
6+
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
7+
import { inspectAudio } from "./checkAudio.js";
8+
import type { ProjectMedia } from "./projectMedia.js";
9+
10+
const RATE = 48_000;
11+
const dir = mkdtempSync(join(tmpdir(), "hf-check-audio-"));
12+
afterAll(() => rmSync(dir, { recursive: true, force: true }));
13+
14+
/** A mono 16-bit WAV of `seconds`, each sample from `fill`. */
15+
function wav(name: string, seconds: number, fill: (t: number) => number): string {
16+
const frames = Math.round(seconds * RATE);
17+
const data = Buffer.alloc(frames * 2);
18+
for (let i = 0; i < frames; i++) data.writeInt16LE(Math.round(fill(i / RATE) * 32767), i * 2);
19+
const header = Buffer.alloc(44);
20+
header.write("RIFF", 0);
21+
header.writeUInt32LE(36 + data.length, 4);
22+
header.write("WAVEfmt ", 8);
23+
header.writeUInt32LE(16, 16);
24+
header.writeUInt16LE(1, 20);
25+
header.writeUInt16LE(1, 22);
26+
header.writeUInt32LE(RATE, 24);
27+
header.writeUInt32LE(RATE * 2, 28);
28+
header.writeUInt16LE(2, 32);
29+
header.writeUInt16LE(16, 34);
30+
header.write("data", 36);
31+
header.writeUInt32LE(data.length, 40);
32+
writeFileSync(join(dir, name), Buffer.concat([header, data]));
33+
return name;
34+
}
35+
36+
const tone = (t: number) => 0.5 * Math.sin(2 * Math.PI * 440 * t);
37+
const silent = () => 0;
38+
/** Sound only after `at` seconds. */
39+
const lateTone = (at: number) => (t: number) => (t < at ? 0 : tone(t));
40+
41+
function clip(src: string, overrides: Partial<AudioElement> = {}): AudioElement {
42+
return {
43+
id: src.replace(/\W/g, "-"),
44+
src,
45+
start: 0,
46+
end: 2,
47+
mediaStart: 0,
48+
layer: 0,
49+
volume: 1,
50+
type: "audio",
51+
...overrides,
52+
};
53+
}
54+
55+
function media(
56+
audios: AudioElement[],
57+
videos: ProjectMedia["videos"] = [],
58+
html = "<html></html>",
59+
): ProjectMedia {
60+
return { audios, videos, images: [], html };
61+
}
62+
63+
const codes = async (project: ProjectMedia) =>
64+
(await inspectAudio(dir, "index.html", project)).map((f) => `${f.code}:${f.selector}`);
65+
66+
const ffmpeg = findFfBinary("ffmpeg") && findFfBinary("ffprobe");
67+
68+
describe.skipIf(!ffmpeg)("inspectAudio", () => {
69+
wav("tone.wav", 3, tone);
70+
wav("silent.wav", 3, silent);
71+
wav("late.wav", 4, lateTone(2.5));
72+
// Long enough to be sampled at spots first; sound only between the spots.
73+
wav("long-silent.wav", 20, silent);
74+
wav("long-gap.wav", 20, (t) => (t > 5 && t < 6 ? tone(t) : 0));
75+
// Not audio at all: the render's audio probe rejects it and the whole mix fails.
76+
writeFileSync(join(dir, "notes.wav"), "RIFF but not really a wave file");
77+
// A 1x1 PNG: it probes, but as a still image, so render refuses it as an <audio> source.
78+
writeFileSync(
79+
join(dir, "cover.png"),
80+
Buffer.from(
81+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
82+
"base64",
83+
),
84+
);
85+
86+
it("passes a clip that plays sound", async () => {
87+
expect(await codes(media([clip("tone.wav")]))).toEqual([]);
88+
});
89+
90+
it("reports a film with no audio clips as having no sound", async () => {
91+
const findings = await inspectAudio(dir, "index.html", media([]));
92+
expect(findings).toMatchObject([{ code: "audio_none", severity: "info" }]);
93+
});
94+
95+
it("counts an unmuted video with an audio track as sound, like render does", async () => {
96+
const probed: string[] = [];
97+
const findings = await inspectAudio(
98+
dir,
99+
"index.html",
100+
media(
101+
[],
102+
[
103+
{ src: "muted.mp4", hasAudio: false },
104+
{ src: "talk.mp4", hasAudio: true },
105+
],
106+
),
107+
{
108+
renderAccepts: () => Promise.reject(new Error("no audio clips to probe")),
109+
probeAudio: () => Promise.reject(new Error("no audio clips to probe")),
110+
isSilent: () => Promise.reject(new Error("no audio clips to decode")),
111+
videoHasAudio: async (path) => {
112+
probed.push(path);
113+
return true;
114+
},
115+
},
116+
);
117+
expect(findings).toEqual([]);
118+
expect(probed).toEqual([join(dir, "talk.mp4")]);
119+
});
120+
121+
it("errors on a source the render cannot read as audio", async () => {
122+
const [finding] = await inspectAudio(dir, "index.html", media([clip("notes.wav")]));
123+
expect(finding).toMatchObject({
124+
code: "audio_source_unreadable",
125+
severity: "error",
126+
selector: "#notes-wav",
127+
time: 0,
128+
timeRange: [0, 2],
129+
});
130+
});
131+
132+
it("errors when render's media-type check refuses the source, naming the video for its track", async () => {
133+
expect(
134+
await codes(
135+
media([clip("cover.png"), clip("cover.png", { id: "camera-audio", type: "video" })]),
136+
),
137+
).toEqual(["audio_source_rejected:#cover-png", "audio_source_rejected:#camera"]);
138+
});
139+
140+
it("warns when a clip starts past the end of its file", async () => {
141+
expect(await codes(media([clip("tone.wav", { mediaStart: 3.5 })]))).toEqual([
142+
"audio_starts_past_source_end:#tone-wav",
143+
]);
144+
});
145+
146+
it("warns on a clip whose played part is silent, and only that clip", async () => {
147+
expect(
148+
await codes(media([clip("tone.wav"), clip("silent.wav", { id: "narration-2" })])),
149+
).toEqual(["audio_clip_silent:#narration-2"]);
150+
});
151+
152+
it("names the composition file a mounted clip was written in", async () => {
153+
const bundled = `<div data-composition-id="main">
154+
<div data-composition-file="compositions/scene-2.html"><audio id="vo-2"></audio></div>
155+
</div>`;
156+
const [finding] = await inspectAudio(
157+
dir,
158+
"index.html",
159+
media([clip("silent.wav", { id: "vo-2" })], [], bundled),
160+
);
161+
expect(finding).toMatchObject({ selector: "#vo-2", sourceFile: "compositions/scene-2.html" });
162+
});
163+
164+
it("only calls a long clip silent after reading all of it", async () => {
165+
expect(await codes(media([clip("long-silent.wav", { end: 20 })]))).toEqual([
166+
"audio_clip_silent:#long-silent-wav",
167+
]);
168+
expect(await codes(media([clip("long-gap.wav", { end: 20 })]))).toEqual([]);
169+
});
170+
171+
it("judges only the part of the file the clip plays", async () => {
172+
expect(await codes(media([clip("late.wav", { end: 2 })]))).toEqual([
173+
"audio_clip_silent:#late-wav",
174+
]);
175+
expect(await codes(media([clip("late.wav", { mediaStart: 2, end: 2 })]))).toEqual([]);
176+
// Played at double speed, 2 s of film reach 4 s into the file, past the silent start.
177+
expect(await codes(media([clip("late.wav", { end: 2, playbackRate: 2 })]))).toEqual([]);
178+
});
179+
});

0 commit comments

Comments
 (0)