Skip to content

Commit b37db11

Browse files
feat(cli): audio --read --layer reads a one-layer file whole, no separation (#7345)
* feat(cli): audio --read --layer music|sfx|voice reads a one-layer file whole A caller that knows a file holds a single layer can declare it: the file is read whole as that layer with no stem separation and nothing uploaded. Without the flag the read is unchanged. An unknown layer is rejected. Consumer: heygen-com/experiment-framework#66322. Proposal OK'd by Rory (C0BEH487Q7R, 2026-11-07). Co-Authored-By: Jerrai * chore(cli): drop added comments from audio read layer Co-Authored-By: Jerrai
1 parent 1ddd425 commit b37db11

6 files changed

Lines changed: 179 additions & 2 deletions

File tree

‎docs/packages/cli.mdx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -885,6 +885,12 @@ Sentences come from a transcript: `--transcript <file>`, or
885885
`<recording>.transcript.json` beside the recording. Without one, voice lines
886886
carry no words.
887887

888+
A file you know holds one layer is read whole with `--layer music|sfx|voice`:
889+
no stems are fetched and nothing leaves the machine, whatever `--quality` says.
890+
`music` reads the file as the music bed (sections, tempo, levels), `voice` reads
891+
words and pauses from it, `sfx` finds its effects. The JSON result records the
892+
declared `layer`; without the flag the read is unchanged.
893+
888894
Effects and music are read from stems. At `--quality best` the
889895
recording is separated into vocals, drums, bass, guitar, piano and other by
890896
HeyGen's `/v3/audio/stems`, with your HeyGen credential (see

‎packages/cli/src/audio/read/read.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,3 +495,60 @@ describe.skipIf(!ffmpeg)("readAudio", () => {
495495
expect(analysis.warnings[1]).toContain("no transcript and no stems");
496496
});
497497
});
498+
499+
describe.skipIf(!ffmpeg)("readAudio with a declared layer", () => {
500+
const read = (path: string, name: string, cache: string, layer?: "music" | "sfx" | "voice") => {
501+
vi.stubEnv("HEYGEN_API_KEY", "test-key");
502+
return readAudio(path, name, { cacheDir: join(dir, cache), quality: "best", layer });
503+
};
504+
505+
it("reads a music file whole, without separating it, and keeps its cache apart from a mix read", async () => {
506+
const path = wav("layer-music.wav", 8, bed);
507+
const callsBefore = stemCalls;
508+
const first = await read(path, "layer-music.wav", "cache-layer-music", "music");
509+
expect(stemCalls).toBe(callsBefore);
510+
expect(first.analysis).toMatchObject({ separation: "mix", layer: "music", voice: null });
511+
expect(first.analysis.music!.segments.length).toBeGreaterThan(0);
512+
expect(first.analysis.bpm).toBe(first.analysis.music!.segments[0]!.bpm);
513+
expect((await read(path, "layer-music.wav", "cache-layer-music", "music")).cached).toBe(true);
514+
const asMix = await read(path, "layer-music.wav", "cache-layer-music");
515+
expect(asMix.cached).toBe(false);
516+
expect(asMix.analysis.layer).toBeUndefined();
517+
});
518+
519+
it("reads a voice file whole: words and pauses from the file itself", async () => {
520+
const path = wav("layer-voice.wav", 6, voiced);
521+
writeFileSync(
522+
`${path}.transcript.json`,
523+
JSON.stringify({
524+
words: [
525+
{ text: "one", start: 0.5, end: 2 },
526+
{ text: "two", start: 3, end: 4.5 },
527+
],
528+
}),
529+
);
530+
const callsBefore = stemCalls;
531+
const { analysis } = await read(path, "layer-voice.wav", "cache-layer-voice", "voice");
532+
expect(stemCalls).toBe(callsBefore);
533+
expect(analysis).toMatchObject({ separation: "mix", layer: "voice", effects: null });
534+
expect(analysis.voice!.sentences.map((s) => s.text)).toEqual(["one", "two"]);
535+
expect(analysis.voice!.pauses.length).toBeGreaterThan(0);
536+
});
537+
538+
it("reads a sound-effect file whole, with no music read", async () => {
539+
const path = wav("layer-sfx.wav", 5, boom);
540+
const callsBefore = stemCalls;
541+
const { analysis } = await read(path, "layer-sfx.wav", "cache-layer-sfx", "sfx");
542+
expect(stemCalls).toBe(callsBefore);
543+
expect(analysis).toMatchObject({ separation: "mix", layer: "sfx", voice: null, music: null });
544+
expect(analysis.effects!.events.length).toBeGreaterThan(0);
545+
});
546+
547+
it("records no layer when none is declared", async () => {
548+
const path = wav("layer-none.wav", 3, voiced);
549+
const { analysis } = await readAudio(path, "layer-none.wav", {
550+
cacheDir: join(dir, "cache-layer-none"),
551+
});
552+
expect("layer" in analysis).toBe(false);
553+
});
554+
});

‎packages/cli/src/audio/read/read.ts‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,14 @@ import {
2828
spanDb,
2929
} from "./signal.js";
3030
import { DRUM_STEM, VOICE_STEM, cachedStems, fetchStems, stemsAccess } from "./stems.js";
31-
import { READ_VERSION, type AudibleStem, type Music, type ReadAudio, type Voice } from "./types.js";
31+
import {
32+
READ_VERSION,
33+
type AudibleStem,
34+
type Layer,
35+
type Music,
36+
type ReadAudio,
37+
type Voice,
38+
} from "./types.js";
3239
import { voiceFromActivity, voiceFromWords, type TimedWord } from "./voice.js";
3340

3441
export interface ReadOptions {
@@ -37,6 +44,7 @@ export interface ReadOptions {
3744
/** A transcript of the recording; `<file>.transcript.json` next to it is used when absent. */
3845
transcript?: string;
3946
quality?: AnalysisQuality;
47+
layer?: Layer;
4048
}
4149

4250
export interface ReadResult {
@@ -379,6 +387,46 @@ async function analyze(
379387
return { ...about, voice, effects, music: null, ambience: null, warnings };
380388
}
381389

390+
async function analyzeLayer(
391+
path: string,
392+
file: string,
393+
sha: string,
394+
transcript: string | null,
395+
layer: Layer,
396+
): Promise<ReadAudio> {
397+
const samples = await decodeMono(path);
398+
const about = {
399+
...base(file, sha, round(samples.length / READ_RATE, 3), null, transcript),
400+
layer,
401+
};
402+
const none = { voice: null, effects: null, music: null, ambience: null, warnings: [] };
403+
if (layer === "music") {
404+
const { music, ambience } = musicOf({ music: samples }, null, samples);
405+
return { ...about, ...none, music, ambience, bpm: tempoOf(music) };
406+
}
407+
if (layer === "sfx") {
408+
const effects = findEffects({
409+
sources: [{ samples }],
410+
sampleRate: READ_RATE,
411+
mix: samples,
412+
music: null,
413+
beats: [],
414+
speech: [],
415+
});
416+
return { ...about, ...none, effects };
417+
}
418+
const words = transcript ? wordsOf(transcript) : [];
419+
const pitch = pitchTrack(await decodeMono(path, PITCH_RATE));
420+
const levelOf = (span: Span) => spanDb(samples, READ_RATE, span.start, span.end);
421+
const voice = words.length
422+
? voiceFromWords(words, pitch, levelOf)
423+
: voiceFromActivity(samples, READ_RATE, pitch, levelOf);
424+
const warnings = words.length
425+
? []
426+
: [`no transcript: voice lines carry no words; ${transcribeHint(file)}`];
427+
return { ...about, ...none, voice, warnings };
428+
}
429+
382430
function base(
383431
file: string,
384432
sha: string,
@@ -408,6 +456,7 @@ export async function readAudio(
408456
options: ReadOptions,
409457
): Promise<ReadResult> {
410458
const quality = options.quality ?? "local";
459+
const { layer } = options;
411460
const sha = await sha256(path);
412461
const transcript = transcriptFor(path, options.transcript);
413462
const transcriptKey = fingerprint(transcript);
@@ -419,12 +468,14 @@ export async function readAudio(
419468
if (existsSync(jsonPath) && existsSync(textPath)) {
420469
const entry = JSON.parse(readFileSync(jsonPath, "utf8")) as CacheEntry;
421470
const stemsNowPossible =
471+
!layer &&
422472
entry.analysis.separation === "mix" &&
423473
quality === "best" &&
424474
(cachedStems(stemDir) !== null || typeof (await stemsAccess()) !== "string");
425475
if (
426476
(entry.analysis.separation !== "stems" || cachedStems(stemDir) !== null) &&
427477
entry.analysis.version === READ_VERSION &&
478+
entry.analysis.layer === layer &&
428479
entry.revision === CACHE_REVISION &&
429480
entry.transcriptKey === transcriptKey &&
430481
!stemsNowPossible &&
@@ -437,7 +488,9 @@ export async function readAudio(
437488
)
438489
return { analysis: entry.analysis, textPath, jsonPath, cached: true };
439490
}
440-
const analysis = await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
491+
const analysis = layer
492+
? await analyzeLayer(path, file, sha, transcript, layer)
493+
: await analyze(path, file, sha, transcript, stemDir, quality, options.cacheDir);
441494
writeAtomic(
442495
jsonPath,
443496
JSON.stringify({ revision: CACHE_REVISION, analysis, transcriptKey } satisfies CacheEntry),

‎packages/cli/src/audio/read/types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ export const READ_VERSION = 7;
44

55
export type Separation = "stems" | "mix";
66

7+
export const LAYERS = ["music", "sfx", "voice"] as const;
8+
export type Layer = (typeof LAYERS)[number];
9+
710
export type RateUnit = "chars" | "words";
811

912
export type Ending = "rising" | "falling" | "level";
@@ -121,6 +124,7 @@ export interface ReadAudio {
121124
sha256: string;
122125
duration: number;
123126
separation: Separation;
127+
layer?: Layer;
124128
stems: AudibleStem[];
125129
stemsDir: string | null;
126130
voiceFreeMix: string | null;
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { runCommand } from "citty";
2+
import { beforeEach, describe, expect, it, vi } from "vitest";
3+
4+
const { readAudio } = vi.hoisted(() => ({ readAudio: vi.fn() }));
5+
6+
vi.mock("../audio/read/read.js", () => ({ readAudio }));
7+
8+
import audioCommand from "./audio.js";
9+
10+
describe("audio --read --layer", () => {
11+
beforeEach(() => {
12+
readAudio.mockReset();
13+
readAudio.mockResolvedValue({ analysis: {}, textPath: "x.txt", jsonPath: "x.json" });
14+
vi.spyOn(console, "log").mockImplementation(() => {});
15+
});
16+
17+
it.each(["music", "sfx", "voice"])("passes --layer %s to the read", async (layer) => {
18+
await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", layer, "--json"] });
19+
expect(readAudio.mock.calls[0]![2]).toMatchObject({ layer });
20+
});
21+
22+
it("passes no layer when the flag is absent", async () => {
23+
await runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--json"] });
24+
expect(readAudio.mock.calls[0]![2].layer).toBeUndefined();
25+
});
26+
27+
it("rejects an unknown layer and names the valid ones", async () => {
28+
await expect(
29+
runCommand(audioCommand, { rawArgs: ["a.wav", "--read", "--layer", "drums"] }),
30+
).rejects.toThrow("--layer is music, sfx, voice; got drums");
31+
expect(readAudio).not.toHaveBeenCalled();
32+
});
33+
34+
it("rejects --layer without --read", async () => {
35+
await expect(
36+
runCommand(audioCommand, { rawArgs: ["a.wav", "--layer", "music"] }),
37+
).rejects.toThrow("--layer only works with --read");
38+
});
39+
});

‎packages/cli/src/commands/audio.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { analyzeTargets, previewTarget } from "../audio/analysis/run.js";
1212
import { filmTargets, type AudioTarget, type FilmQuery } from "../audio/analysis/targets.js";
1313
import { formatRead } from "../audio/read/format.js";
1414
import { readAudio } from "../audio/read/read.js";
15+
import { LAYERS, type Layer } from "../audio/read/types.js";
1516
import { describeProject } from "../timeline/describeProject.js";
1617
import { CliUsageError } from "../utils/commandResult.js";
1718
import { ensureDOMParser } from "../utils/dom.js";
@@ -122,6 +123,7 @@ interface AudioFlags {
122123
"silence-db"?: string;
123124
read?: boolean;
124125
transcript?: string;
126+
layer?: string;
125127
json?: boolean;
126128
}
127129

@@ -181,13 +183,23 @@ function readQuality(files: string[], flags: AudioFlags): AnalysisQuality {
181183
return parseQuality(flags);
182184
}
183185

186+
function parseLayer(flags: AudioFlags): Layer | undefined {
187+
if (flags.layer === undefined) return undefined;
188+
if (!flags.read) throw new CliUsageError("--layer only works with --read");
189+
if (!LAYERS.includes(flags.layer as Layer))
190+
throw new CliUsageError(`--layer is ${LAYERS.join(", ")}; got ${flags.layer}`);
191+
return flags.layer as Layer;
192+
}
193+
184194
async function read(files: string[], flags: AudioFlags) {
185195
const quality = readQuality(files, flags);
186196
const range = pair("range", flags.range);
197+
const layer = parseLayer(flags);
187198
const result = await readAudio(resolve(process.cwd(), files[0]!), files[0]!, {
188199
cacheDir: join(resolve(flags.dir ?? process.cwd()), ".hyperframes", "read-audio"),
189200
transcript: flags.transcript ? resolve(process.cwd(), flags.transcript) : undefined,
190201
quality,
202+
layer,
191203
});
192204
if (flags.json) {
193205
console.log(JSON.stringify(withMeta(result)));
@@ -259,6 +271,11 @@ export default defineCommand({
259271
type: "string",
260272
description: "With --read: the recording's transcript (default <file>.transcript.json)",
261273
},
274+
layer: {
275+
type: "string",
276+
description:
277+
"With --read: the file holds only this layer (music, sfx or voice); it is read whole, with no separation",
278+
},
262279
json: { type: "boolean", description: "Output as JSON", default: false },
263280
},
264281
setup({ rawArgs, cmd }) {
@@ -268,6 +285,7 @@ export default defineCommand({
268285
const flags = args as unknown as AudioFlags;
269286
const files = args._.map(String);
270287
if (flags.read) return read(files, flags);
288+
if (flags.layer !== undefined) throw new CliUsageError("--layer only works with --read");
271289
if (flags.transcript) throw new CliUsageError("--transcript only works with --read");
272290
const query = filmQuery(flags);
273291
if (files.length && query) throw new CliUsageError("give files or a film query, not both");

0 commit comments

Comments
 (0)