Skip to content

Commit 7b30a50

Browse files
fix(cli): transcribe reads whisper output with nan or inf values (heygen-com#7110)
* fix(cli): transcribe reads whisper output with nan or inf values * fix(cli): transcribe also reads a Windows whisper's -nan(ind) as no value
1 parent 3a24da8 commit 7b30a50

3 files changed

Lines changed: 48 additions & 4 deletions

File tree

‎packages/cli/src/whisper/normalize.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,41 @@ describe("loadTranscript", () => {
124124
);
125125
});
126126

127+
it("reads whisper-cli output whose floats are nan, -nan, inf or MSVC's -nan(ind)", () => {
128+
// Token objects as whisper.cpp's cli.cpp writes them: tab-indented, floats via iostream.
129+
const token = (text: string, from: number, to: number, p: string) =>
130+
[
131+
"\t\t\t{",
132+
`\t\t\t\t"text": "${text}",`,
133+
`\t\t\t\t"timestamps": {`,
134+
`\t\t\t\t\t"from": "00:00:00.000",`,
135+
`\t\t\t\t\t"to": "00:00:00.500"`,
136+
`\t\t\t\t},`,
137+
`\t\t\t\t"offsets": {`,
138+
`\t\t\t\t\t"from": ${from},`,
139+
`\t\t\t\t\t"to": ${to}`,
140+
`\t\t\t\t},`,
141+
`\t\t\t\t"id": 50364,`,
142+
`\t\t\t\t"p": ${p},`,
143+
`\t\t\t\t"t_dtw": -1`,
144+
"\t\t\t}",
145+
].join("\n");
146+
const raw = `{\n\t"transcription": [\n\t\t{\n\t\t\t"tokens": [\n${[
147+
token(" nan", 0, 500, "nan"),
148+
token(" world", 600, 1200, "-nan"),
149+
token(" again", 1300, 1800, "inf"),
150+
token(" later", 1900, 2400, "-nan(ind)"),
151+
].join(",\n")}\n\t\t\t]\n\t\t}\n\t]\n}\n`;
152+
const { words, format } = loadTranscript(tmpFile("transcript.json", raw));
153+
expect(format).toBe("whisper-cpp");
154+
expect(words).toEqual([
155+
{ text: "nan", start: 0, end: 0.5, id: "w0" },
156+
{ text: "world", start: 0.6, end: 1.2, id: "w1" },
157+
{ text: "again", start: 1.3, end: 1.8, id: "w2" },
158+
{ text: "later", start: 1.9, end: 2.4, id: "w3" },
159+
]);
160+
});
161+
127162
it("parses whisper-cpp JSON with punctuation merging", () => {
128163
const path = tmpFile(
129164
"transcript.json",

‎packages/cli/src/whisper/normalize.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ export function detectFormat(filePath: string): TranscriptFormat {
4141
const ext = extname(filePath).toLowerCase();
4242
if (ext === ".srt") return "srt";
4343
if (ext === ".vtt") return "vtt";
44-
if (ext === ".json") return detectJsonFormat(JSON.parse(readJsonFileText(filePath)));
44+
if (ext === ".json") return detectJsonFormat(parseWhisperJson(readJsonFileText(filePath)));
4545
throw new Error(`Unsupported transcript file extension: ${ext}. Use .json, .srt, or .vtt`);
4646
}
4747

@@ -499,6 +499,15 @@ export function formatVtt(words: Word[], opts?: WordsToCuesOptions): string {
499499
// Public API
500500
// ---------------------------------------------------------------------------
501501

502+
const BARE_NAN_OR_INF_OUTSIDE_STRINGS =
503+
/"(?:[^"\\]|\\.)*"|-?\b(?:nan(?:\([^)]*\))?|inf(?:inity)?)(?!\w)/gi;
504+
505+
export function parseWhisperJson(text: string): any {
506+
return JSON.parse(
507+
text.replace(BARE_NAN_OR_INF_OUTSIDE_STRINGS, (m) => (m.startsWith('"') ? m : "null")),
508+
);
509+
}
510+
502511
/**
503512
* Load and normalize a transcript file to a standard word array.
504513
*
@@ -523,7 +532,7 @@ export function loadTranscript(filePath: string): { words: Word[]; format: Trans
523532
}
524533

525534
// JSON formats — parse once, detect, then extract words
526-
const parsed = JSON.parse(content);
535+
const parsed = parseWhisperJson(content);
527536
const format = detectJsonFormat(parsed);
528537

529538
// JSON parsers never set id — assign w{index} like the srt/vtt branches above

‎packages/cli/src/whisper/transcribe.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import { sha256File } from "./modelFile.js";
2222
import { initialModelForLanguage } from "./modelForLanguage.js";
2323
import { TRANSCRIPT_FILE } from "./transcriptFile.js";
2424
import { findWavChunk } from "./wav.js";
25-
import type { Word } from "./normalize.js";
25+
import { parseWhisperJson, type Word } from "./normalize.js";
2626
import { emitWords } from "./progress.js";
2727
import { probeAudio } from "../audio/analysis/decode.js";
2828
import { decodableAudioStream } from "@hyperframes/parsers/media-streams";
@@ -578,7 +578,7 @@ export async function transcribe(
578578
throw new Error("Whisper did not produce output. Check the input file.");
579579
}
580580

581-
const transcript = JSON.parse(readFileSync(transcriptPath, "utf-8"));
581+
const transcript = parseWhisperJson(readFileSync(transcriptPath, "utf-8"));
582582
const reportedLanguage: unknown = transcript.result?.language;
583583
const detectedLanguage =
584584
automaticLanguage && typeof reportedLanguage === "string" && reportedLanguage.length > 0

0 commit comments

Comments
 (0)