Skip to content

Commit 631b21d

Browse files
fix(cli): clear audio description review checks
Use portable stem path assertions and simplify response validation. Keep auth-derived URLs out of cache hashes and trim redundant comments. — Jerrai
1 parent dac0585 commit 631b21d

7 files changed

Lines changed: 87 additions & 53 deletions

File tree

‎packages/cli/src/audio/read/describe.test.ts‎

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,16 @@ describe("describeStems", () => {
7373
expect(result).toEqual({ stems: measured.stems, description: null });
7474
});
7575

76-
it.each(["api key", "access token"])(
77-
"uses the HeyGen %s for base64 audio and schema-enforced summary requests",
78-
async (credential) => {
79-
vi.stubEnv(credential === "api key" ? "HEYGEN_API_KEY" : "HEYGEN_ACCESS_TOKEN", "signed-in");
76+
it.each([
77+
{ credential: "HEYGEN_API_KEY", headers: { "x-api-key": "signed-in" } },
78+
{
79+
credential: "HEYGEN_ACCESS_TOKEN",
80+
headers: { authorization: "Bearer signed-in", "X-HeyGen-Source": "cli" },
81+
},
82+
])(
83+
"uses $credential for base64 audio and schema-enforced summary requests",
84+
async ({ credential, headers }) => {
85+
vi.stubEnv(credential, "signed-in");
8086
vi.stubEnv("HYPERFRAMES_AUDIO_API_KEY", "not-the-description-key");
8187
vi.stubEnv("HYPERFRAMES_AUDIO_STEMS_URL", "https://other.test/stems");
8288
vi.stubEnv("HYPERFRAMES_READ_AUDIO_MODEL", "not-the-model");
@@ -91,9 +97,7 @@ describe("describeStems", () => {
9197
expect(init?.headers).toEqual({
9298
"content-type": "application/json",
9399
"X-HeyGen-Client-Source": "hyperframes",
94-
...(credential === "api key"
95-
? { "x-api-key": "signed-in" }
96-
: { authorization: "Bearer signed-in", "X-HeyGen-Source": "cli" }),
100+
...headers,
97101
});
98102
const request = JSON.parse(String(init?.body));
99103
expect(request.model).toBe("gemini-4-argon");
@@ -148,7 +152,7 @@ describe("describeStems", () => {
148152
]);
149153
await describeAudio(warnings);
150154
expect(fetch).toHaveBeenCalledTimes(3);
151-
vi.stubEnv(credential === "api key" ? "HEYGEN_API_KEY" : "HEYGEN_ACCESS_TOKEN", "");
155+
vi.stubEnv(credential, "");
152156
expect((await describeAudio(warnings)).description).toBeNull();
153157
expect(fetch).toHaveBeenCalledTimes(3);
154158
expect(warnings).toHaveLength(1);
@@ -229,6 +233,12 @@ describe("describeStems", () => {
229233
JSON.stringify({ overall: "Words" }),
230234
JSON.stringify({ overall: "Words", bgm_prompt: 7 }),
231235
JSON.stringify({ overall: "", bgm_prompt: null }),
236+
JSON.stringify({ overall: " ", bgm_prompt: null }),
237+
JSON.stringify({ overall: "Words", bgm_prompt: " " }),
238+
JSON.stringify({ overall: 7, bgm_prompt: null }),
239+
JSON.stringify({ bgm_prompt: null }),
240+
"null",
241+
"[]",
232242
])("rejects a summary outside the schema instead of guessing (%s)", async (answer) => {
233243
vi.stubEnv("HEYGEN_API_KEY", "test-key");
234244
const fetch = vi
@@ -244,6 +254,54 @@ describe("describeStems", () => {
244254
expect(fetch).toHaveBeenCalledTimes(3);
245255
});
246256

257+
it.each([null, [], {}, { content: null }, { content: [null, {}, { type: "text", text: 7 }] }])(
258+
"rejects a response without text (%j)",
259+
async (answer) => {
260+
vi.stubEnv("HEYGEN_API_KEY", "test-key");
261+
vi.spyOn(globalThis, "fetch").mockImplementation(async () => Response.json(answer));
262+
const warnings: string[] = [];
263+
expect(await describeAudio(warnings)).toEqual({ stems: measured.stems, description: null });
264+
expect(warnings).toEqual(["stems not described: the description service returned no text"]);
265+
},
266+
);
267+
268+
it("joins only text blocks in a mixed response", async () => {
269+
vi.stubEnv("HEYGEN_API_KEY", "test-key");
270+
vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => {
271+
if (JSON.parse(String(init?.body)).output_config)
272+
return response(JSON.stringify({ overall: "Speech.", bgm_prompt: null }));
273+
return Response.json({
274+
content: [
275+
null,
276+
{},
277+
{ type: "audio", text: "ignored" },
278+
{ type: "text", text: 7 },
279+
{ type: "text", text: "Spoken " },
280+
{ type: "text", text: "words." },
281+
],
282+
});
283+
});
284+
const warnings: string[] = [];
285+
const result = await describeAudio(warnings);
286+
expect(result.stems.map((stem) => stem.description)).toEqual([
287+
"Spoken words.",
288+
"Spoken words.",
289+
]);
290+
expect(warnings).toEqual([]);
291+
});
292+
293+
it("keys cached descriptions on content rather than the credential or endpoint", async () => {
294+
vi.stubEnv("HEYGEN_API_KEY", "first-key");
295+
const fetch = mockMessages();
296+
const warnings: string[] = [];
297+
const first = await describeAudio(warnings);
298+
vi.stubEnv("HEYGEN_API_KEY", "second-key");
299+
vi.stubEnv("HEYGEN_API_URL", "https://another.heygen.test");
300+
expect(await describeAudio(warnings)).toEqual(first);
301+
expect(fetch).toHaveBeenCalledTimes(3);
302+
expect(warnings).toEqual([]);
303+
});
304+
247305
it("does not describe silent stems", async () => {
248306
const fetch = vi.spyOn(globalThis, "fetch");
249307
const warnings: string[] = [];

‎packages/cli/src/audio/read/describe.ts‎

Lines changed: 20 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -65,22 +65,22 @@ function overallPrompt({ duration, bpm, stems, breaks }: Measured): string {
6565
].join("\n");
6666
}
6767

68+
function isRecord(value: unknown): value is Record<string, unknown> {
69+
return value !== null && typeof value === "object" && !Array.isArray(value);
70+
}
71+
72+
function requiredText(value: unknown): string {
73+
if (typeof value !== "string" || !value.trim())
74+
throw new Error("the soundtrack answer did not match the required fields");
75+
return value.trim();
76+
}
77+
6878
function descriptionOf(text: string): Description {
6979
const value: unknown = JSON.parse(text);
70-
if (
71-
!value ||
72-
typeof value !== "object" ||
73-
!("overall" in value) ||
74-
typeof value.overall !== "string" ||
75-
!value.overall.trim() ||
76-
!("bgm_prompt" in value) ||
77-
!(value.bgm_prompt === null || typeof value.bgm_prompt === "string") ||
78-
(typeof value.bgm_prompt === "string" && !value.bgm_prompt.trim())
79-
)
80-
throw new Error("the soundtrack answer did not match the required fields");
80+
if (!isRecord(value)) throw new Error("the soundtrack answer did not match the required fields");
8181
return {
82-
overall: value.overall.trim(),
83-
bgmPrompt: value.bgm_prompt === null ? null : value.bgm_prompt.trim(),
82+
overall: requiredText(value.overall),
83+
bgmPrompt: value.bgm_prompt === null ? null : requiredText(value.bgm_prompt),
8484
};
8585
}
8686

@@ -104,27 +104,16 @@ async function mp3Of(path: string, scratch: string): Promise<Buffer> {
104104

105105
const retryable = (status: number) => status === 429 || status >= 500;
106106

107+
function textPart(part: unknown): string {
108+
if (!isRecord(part) || part.type !== "text") return "";
109+
return typeof part.text === "string" ? part.text : "";
110+
}
111+
107112
function textOf(answer: unknown): string {
108-
if (
109-
!answer ||
110-
typeof answer !== "object" ||
111-
!("content" in answer) ||
112-
!Array.isArray(answer.content)
113-
)
113+
if (!isRecord(answer) || !Array.isArray(answer.content))
114114
throw new Error("the description service returned no text");
115115
const content: unknown[] = answer.content;
116-
const texts: string[] = [];
117-
for (const part of content)
118-
if (
119-
part &&
120-
typeof part === "object" &&
121-
"type" in part &&
122-
part.type === "text" &&
123-
"text" in part &&
124-
typeof part.text === "string"
125-
)
126-
texts.push(part.text);
127-
const text = texts.join("").trim();
116+
const text = content.map(textPart).join("").trim();
128117
if (!text) throw new Error("the description service returned no text");
129118
return text;
130119
}
@@ -182,7 +171,6 @@ async function answers(asks: Ask[], cacheDir: string, access: ServiceAccess) {
182171
asks.map(async ({ path, prompt, structured = false }) => {
183172
const audio = path ? await mp3Of(path, scratch) : null;
184173
const hash = createHash("sha256")
185-
.update(access.url)
186174
.update(MODEL)
187175
.update(prompt)
188176
.update(structured ? JSON.stringify(OVERALL_SCHEMA) : "plain");

‎packages/cli/src/audio/read/effects.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -243,10 +243,8 @@ export interface EffectSource {
243243
samples: Float32Array;
244244
/** Where this track plays music or ambience: a sound that starts or peaks inside one belongs to it, not an effect. */
245245
heldBy?: readonly Span[];
246-
/** While music plays here, a sound that keeps recurring in this track is the music's own, unless it stands out. */
247246
playsIn?: readonly Span[];
248247
stem?: string;
249-
/** The stems mixed into this track, to name the one a sound came from. */
250248
parts?: Record<string, Float32Array>;
251249
}
252250

@@ -373,7 +371,6 @@ function alikeSounds(a: Measured, b: Measured): boolean {
373371
return ratio >= 0.5 && ratio <= 2 && cosine(a.print, b.print) >= LIKE_SOUND;
374372
}
375373

376-
/** Sounds in a track while its music plays that recur there, and do not stand out over the hits around them: the music's own. */
377374
function musicHits(sounds: readonly Measured[]): Set<Measured> {
378375
const playing = sounds.filter((m) => m.inMusic);
379376
const own = new Set<Measured>();

‎packages/cli/src/audio/read/format.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,6 @@ const ENTRY: Record<Entry, string> = {
105105

106106
const EDGE = 0.5;
107107

108-
/** Each stem in the segment: its name alone when it plays throughout, else where it plays. */
109108
function stemsPlaying(segment: MusicSegment): string {
110109
const byStem = new Map<string, StemSpan[]>();
111110
for (const span of segment.stems) byStem.set(span.name, [...(byStem.get(span.name) ?? []), span]);
@@ -217,7 +216,6 @@ function stemBlock(stem: AudibleStem, width: number): string[] {
217216
const DESCRIPTIONS_NOTE =
218217
"(descriptions are a model listening to each stem: what the sound is and what its effects sound like. Their times are approximate; the measured spans above and the timed lines below are exact.)";
219218

220-
/** With stems: the whole first, then each stem with where it plays and what it holds. */
221219
function byStem(analysis: ReadAudio): string[] {
222220
const { description } = analysis;
223221
const width = Math.max(0, ...analysis.stems.map((stem) => stem.name.length));

‎packages/cli/src/audio/read/music.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ interface Run {
4747
end: number;
4848
}
4949

50-
/** Where a stem is heard: quarter-second windows over -45 dBFS, gaps under a second closed, blips under half a second dropped. */
5150
export function activeSpans(samples: Float32Array, sampleRate: number): Run[] {
5251
const hop = Math.round(ACTIVE_HOP * sampleRate);
5352
const spans: Run[] = [];

‎packages/cli/src/audio/read/read.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ describe.skipIf(!ffmpeg)("readAudio", () => {
200200
spans: expect.any(Array),
201201
description: null,
202202
})),
203-
stemsDir: expect.stringMatching(/stems\/\w{12}\/$/),
203+
stemsDir: expect.stringMatching(/stems[\\/]\w{12}[\\/]$/),
204204
voiceFreeMix: expect.stringMatching(/no-vocals\.wav$/),
205205
warnings: [
206206
expect.stringContaining("no transcript"),

‎packages/cli/src/audio/read/types.ts‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ export interface Effect {
4949
overMusicDb: number | null;
5050
/** More than one when a quick run of the same sound is reported as one train. */
5151
count: number;
52-
/** The stem the sound was heard in; null without stems. */
5352
stem: string | null;
5453
}
5554

@@ -75,7 +74,6 @@ export interface MusicSegment {
7574
bpm: number | null;
7675
/** Where a fade out begins, when the segment ends on one. */
7776
fadeFrom: number | null;
78-
/** Where each stem plays inside the segment, loudest stem first. */
7977
stems: StemSpan[];
8078
}
8179

@@ -108,15 +106,12 @@ export interface Ambience {
108106
export interface AudibleStem {
109107
name: string;
110108
db: number;
111-
/** Where the stem is heard. */
112109
spans: Span[];
113-
/** What the stem holds, in a model's words; null when it was not described. */
114110
description: string | null;
115111
}
116112

117113
export interface Description {
118114
overall: string;
119-
/** The music alone as a prompt for a music generator or a library search: no voice, no seconds; null without music. */
120115
bgmPrompt: string | null;
121116
}
122117

@@ -127,7 +122,6 @@ export interface ReadAudio {
127122
duration: number;
128123
separation: Separation;
129124
stems: AudibleStem[];
130-
/** The stems' folder and every stem but the voice mixed into one file, relative to the project. */
131125
stemsDir: string | null;
132126
voiceFreeMix: string | null;
133127
bpm: number | null;

0 commit comments

Comments
 (0)