Skip to content

Commit 65a7ce6

Browse files
authored
fix(media-use): report a failed transcription as an anomaly
1 parent f7db6ba commit 65a7ce6

3 files changed

Lines changed: 142 additions & 1 deletion

File tree

‎skills-manifest.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"files": 13
1919
},
2020
"media-use": {
21-
"hash": "f61a33f8b0f8e260",
21+
"hash": "d41e06920f6769bf",
2222
"files": 116
2323
},
2424
"remotion-to-hyperframes": {

‎skills/media-use/audio/scripts/audio.mjs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,12 @@ if (only.has("tts") && lines.length) {
172172
anomalies.push(`line ${id}: bad voice duration — omitted`);
173173
return null;
174174
}
175+
// Empty: the transcription failed (e.g. no whisper-cpp/Parakeet) or found no words, or the
176+
// provider (heygen) returned no usable word_timestamps.
177+
if (!wordArr?.length)
178+
anomalies.push(
179+
`line ${id}: no word timings — transcription failed or found no words (run \`npx hyperframes transcribe ${rel}\` to see why)`,
180+
);
175181
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
176182
};
177183
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);

‎skills/media-use/audio/scripts/gemini-pipeline.test.mjs‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,138 @@ fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JS
129129
});
130130
}
131131
}
132+
133+
test("Gemini engine reports a failed transcription instead of silently writing no words", (t) => {
134+
const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-"));
135+
t.after(() => rmSync(dir, { recursive: true, force: true }));
136+
const bin = join(dir, "bin");
137+
mkdirSync(bin);
138+
const preload = join(dir, "fetch.mjs");
139+
writeFileSync(
140+
preload,
141+
`
142+
globalThis.fetch = async () => {
143+
const bytes = Buffer.alloc(48);
144+
bytes.write('RIFF'); bytes.write('WAVE', 8);
145+
return Response.json({status:'completed', steps:[{type:'model_output', content:[
146+
{type:'audio', mime_type:'audio/wav', data:bytes.toString('base64')}
147+
]}]});
148+
};
149+
`,
150+
);
151+
// `hyperframes transcribe` without an ASR engine: prints a skip notice and exits 1.
152+
writeFileSync(
153+
join(bin, "npx"),
154+
`#!${process.execPath}\nconsole.error('Captions skipped — whisper-cpp not found');\nprocess.exit(1);\n`,
155+
{ mode: 0o755 },
156+
);
157+
writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 });
158+
writeFileSync(join(dir, ".env"), "");
159+
writeFileSync(
160+
join(dir, "audio_request.json"),
161+
JSON.stringify({
162+
provider: "gemini",
163+
lines: [{ id: "intro", text: "Hello there." }],
164+
bgm: { mode: "none" },
165+
}),
166+
);
167+
const result = spawnSync(
168+
process.execPath,
169+
[
170+
"--import",
171+
preload,
172+
new URL("./audio.mjs", import.meta.url).pathname,
173+
"--hyperframes",
174+
dir,
175+
"--only",
176+
"tts",
177+
],
178+
{
179+
encoding: "utf8",
180+
env: {
181+
...process.env,
182+
HEYGEN_CONFIG_DIR: join(dir, "heygen"),
183+
HEYGEN_API_KEY: "",
184+
HYPERFRAMES_API_KEY: "",
185+
GEMINI_API_KEY: "fixture-key",
186+
PATH: `${bin}:${process.env.PATH}`,
187+
},
188+
},
189+
);
190+
assert.equal(result.status, 0, result.stderr);
191+
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
192+
assert.deepEqual(meta.voices[0].words, []);
193+
assert.match(
194+
result.stdout,
195+
/anomalies \(non-fatal\):\n\s+- line intro: no word timings — transcription failed/,
196+
);
197+
});
198+
199+
test("HeyGen engine reports a response without word timings instead of silently writing no words", (t) => {
200+
const dir = mkdtempSync(join(tmpdir(), "hf-heygen-pipeline-"));
201+
t.after(() => rmSync(dir, { recursive: true, force: true }));
202+
const bin = join(dir, "bin");
203+
mkdirSync(bin);
204+
const preload = join(dir, "fetch.mjs");
205+
writeFileSync(
206+
preload,
207+
`
208+
globalThis.fetch = async (url) => {
209+
if (String(url).endsWith('/voices/speech'))
210+
return Response.json({data:{audio_url:'https://media.example.test/a.mp3'}});
211+
return new Response(Buffer.alloc(48), {status: 200, headers: {'content-type': 'audio/mpeg'}});
212+
};
213+
`,
214+
);
215+
// The response above has no word_timestamps; the transcription fallback has no ASR engine.
216+
writeFileSync(
217+
join(bin, "npx"),
218+
`#!${process.execPath}\nconsole.error('Captions skipped — whisper-cpp not found');\nprocess.exit(1);\n`,
219+
{ mode: 0o755 },
220+
);
221+
writeFileSync(
222+
join(bin, "ffmpeg"),
223+
`#!${process.execPath}\nrequire('node:fs').writeFileSync(process.argv[process.argv.length - 1], 'wav');\n`,
224+
{ mode: 0o755 },
225+
);
226+
writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 });
227+
writeFileSync(join(dir, ".env"), "");
228+
writeFileSync(
229+
join(dir, "audio_request.json"),
230+
JSON.stringify({
231+
provider: "heygen",
232+
lines: [{ id: "intro", text: "Hello there." }],
233+
bgm: { mode: "none" },
234+
}),
235+
);
236+
const result = spawnSync(
237+
process.execPath,
238+
[
239+
"--import",
240+
preload,
241+
new URL("./audio.mjs", import.meta.url).pathname,
242+
"--hyperframes",
243+
dir,
244+
"--only",
245+
"tts",
246+
],
247+
{
248+
encoding: "utf8",
249+
env: {
250+
...process.env,
251+
HEYGEN_CONFIG_DIR: join(dir, "heygen"),
252+
HEYGEN_API_BASE: "",
253+
HEYGEN_API_KEY: "fixture-key",
254+
HYPERFRAMES_API_KEY: "",
255+
PATH: `${bin}:${process.env.PATH}`,
256+
},
257+
},
258+
);
259+
assert.equal(result.status, 0, result.stderr);
260+
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
261+
assert.deepEqual(meta.voices[0].words, []);
262+
assert.match(
263+
result.stdout,
264+
/anomalies \(non-fatal\):\n\s+- line intro: no word timings — transcription failed/,
265+
);
266+
});

0 commit comments

Comments
 (0)