Skip to content

Commit 156e877

Browse files
feat(cli): stream heard words on the transcribe --json progress stream (#5127)
* feat(cli): stream heard words on the transcribe --json progress stream * test(cli): share the streaming transcribe helper * fix(cli): keep install and download stoppable and whisper errors short while streaming
1 parent 755ab7d commit 156e877

11 files changed

Lines changed: 371 additions & 21 deletions

‎packages/cli/src/commands/transcribe.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,21 @@ describe("transcribe command", () => {
219219
return { engine: out.engine, model: out.model, word: words[0]?.text, start: words[0]?.start };
220220
}
221221

222+
it("installs whisper's stop handling only when whisper starts, after any install or download", async () => {
223+
const run = transcribeMock.getMockImplementation()!;
224+
const handlers: number[] = [];
225+
let signal: unknown;
226+
transcribeMock.mockImplementationOnce(async (input, outputDir, options) => {
227+
handlers.push(process.listenerCount("SIGTERM"));
228+
signal = options.startCancellation();
229+
handlers.push(process.listenerCount("SIGTERM"));
230+
return run(input, outputDir, options);
231+
});
232+
await transcribeWith("whisper", { sherpa: false, mlx: false });
233+
expect(signal).toBeInstanceOf(AbortSignal);
234+
expect(handlers[1]).toBeGreaterThan(handlers[0]!);
235+
});
236+
222237
it("auto and parakeet prefer sherpa-onnx, then parakeet-mlx, then whisper", async () => {
223238
const sherpa = {
224239
engine: "parakeet",

‎packages/cli/src/commands/transcribe.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@ async function transcribeAudio(
353353
const onEvent = opts.json ? createProgressWriter(process.stderr) : undefined;
354354
let wavPath = inputPath;
355355
// Before audio prep: under --json no spinner listens for SIGINT, so Ctrl-C would kill Node.
356-
const cancellation = runner === "sherpa" ? createRenderCancellationScope() : null;
356+
let cancellation = runner === "sherpa" ? createRenderCancellationScope() : null;
357357
const run = (r: Runner) => {
358358
switch (r) {
359359
case "sherpa":
@@ -375,6 +375,7 @@ async function transcribeAudio(
375375
onEvent,
376376
timeoutMs: opts.timeoutMs,
377377
installRuntime: opts.installRuntime,
378+
startCancellation: () => (cancellation ??= createRenderCancellationScope()).signal,
378379
});
379380
default: {
380381
const unreachable: never = r;
@@ -401,7 +402,7 @@ async function transcribeAudio(
401402
try {
402403
result = await run(runner);
403404
} catch (fallbackErr) {
404-
// Whisper runs synchronously, so Ctrl-C shows as its child's signal before any listener runs.
405+
// Ctrl-C reaches whisper too, so it can stop on its own signal before the scope aborts it.
405406
if (stoppedByCancelSignal(fallbackErr as { signal?: string })) {
406407
throw new DecodeCancelled("Transcription cancelled");
407408
}

‎packages/cli/src/whisper/parakeet.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ export const PARAKEET_MODEL_LABEL = "parakeet-tdt-0.6b-v3";
2525
/** Stdout/stderr line prefixes of the sherpa-onnx decode worker. */
2626
export const SHERPA_RESULT_PREFIX = "HYPERFRAMES_PARAKEET_RESULT:";
2727
export const SHERPA_ERROR_PREFIX = "HYPERFRAMES_PARAKEET_ERROR:";
28+
export const SHERPA_WINDOW_PREFIX = "HYPERFRAMES_PARAKEET_WINDOW:";
2829

2930
/** Parakeet TDT 0.6B v3's languages, as listed on its model card. */
3031
export const PARAKEET_LANGUAGES =

‎packages/cli/src/whisper/progress.test.ts‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,30 @@
11
import { Writable } from "node:stream";
22
import { setImmediate } from "node:timers/promises";
33
import { expect, it } from "vitest";
4-
import { createProgressWriter } from "./progress.js";
4+
import { createProgressWriter, emitWords } from "./progress.js";
5+
import type { TranscribeProgress } from "./transcribe.js";
6+
7+
it("splits a long run of words into lines under 16 KB and keeps every word in order", () => {
8+
const events: TranscribeProgress[] = [];
9+
const words = Array.from({ length: 2000 }, (_, i) => ({
10+
text: `word${i}`,
11+
start: i,
12+
end: i + 0.5,
13+
}));
14+
emitWords((e) => events.push(e), "tiny", words, 2100);
15+
16+
expect(events.length).toBeGreaterThan(1);
17+
for (const event of events)
18+
expect(Buffer.byteLength(`${JSON.stringify(event)}\n`)).toBeLessThan(16_384);
19+
expect(events.flatMap((e) => (e.type === "words" ? e.words : []))).toEqual(words);
20+
expect(events.at(-1)).toMatchObject({ through: 2100 });
21+
});
22+
23+
it("reports progress through a window with no words", () => {
24+
const events: TranscribeProgress[] = [];
25+
emitWords((e) => events.push(e), "tiny", [], 30);
26+
expect(events).toEqual([{ type: "words", model: "tiny", words: [], through: 30 }]);
27+
});
528

629
it("coalesces byte updates behind a slow reader while preserving phase records", async () => {
730
const lines: string[] = [];

‎packages/cli/src/whisper/progress.ts‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,42 @@
11
import type { Writable } from "node:stream";
2+
import type { Word } from "./normalize.js";
23
import type { TranscribeProgress } from "./transcribe.js";
34

5+
const DESKTOP_MAX_LINE_BYTES = 16_000;
6+
7+
export function emitWords(
8+
onEvent: (event: TranscribeProgress) => void,
9+
model: string,
10+
words: Word[],
11+
through: number,
12+
): void {
13+
const empty = Buffer.byteLength(JSON.stringify({ type: "words", model, words: [], through })) + 1;
14+
let chunk: Word[] = [];
15+
let bytes = empty;
16+
for (const word of words) {
17+
const size = Buffer.byteLength(JSON.stringify(word)) + 1;
18+
if (chunk.length > 0 && bytes + size > DESKTOP_MAX_LINE_BYTES) {
19+
onEvent({ type: "words", model, words: chunk, through: chunk.at(-1)!.end });
20+
chunk = [];
21+
bytes = empty;
22+
}
23+
chunk.push(word);
24+
bytes += size;
25+
}
26+
onEvent({ type: "words", model, words: chunk, through });
27+
}
28+
429
export function createProgressWriter(stream: Writable): (event: TranscribeProgress) => void {
530
let blocked = false;
631
const pending: TranscribeProgress[] = [];
732
const write = (event: TranscribeProgress): void => {
833
if (blocked) {
934
const previous = pending.at(-1);
1035
if (
36+
event.type === "progress" &&
1137
event.phase === "download" &&
12-
previous?.phase === "download" &&
38+
previous?.type === "progress" &&
39+
previous.phase === "download" &&
1340
previous.model === event.model
1441
) {
1542
pending[pending.length - 1] = event;

‎packages/cli/src/whisper/sherpa.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,10 @@ module.exports = {
226226
const result = JSON.parse(readFileSync(join(out, "transcript.json"), "utf-8"));
227227
expect(result).toEqual([{ text: "hello", start: 0, end: 1 }]);
228228
const event = { type: "progress", phase: "transcription", model: "parakeet-tdt-0.6b-v3" };
229+
// The worker's one 1 s window streams its words before the final transcript.
229230
expect(onEvent.mock.calls).toEqual([
230-
[{ ...event, status: "started" }],
231+
[{ ...event, status: "started", durationSeconds: null }],
232+
[{ type: "words", model: event.model, words: result, through: 1 }],
231233
[{ ...event, status: "completed" }],
232234
]);
233235
expect(existsSync(join(root, "loaded-in-parent"))).toBe(false);

‎packages/cli/src/whisper/sherpa.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { createHash, randomUUID } from "node:crypto";
33
import { createReadStream, mkdirSync, renameSync, rmSync, statSync } from "node:fs";
44
import { homedir } from "node:os";
55
import { join } from "node:path";
6+
import { createInterface } from "node:readline";
67
import { fileURLToPath } from "node:url";
78
import { downloadToFile } from "../cloud/download.js";
89
import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
@@ -20,10 +21,12 @@ import {
2021
mergeWindowsToWords,
2122
SHERPA_ERROR_PREFIX,
2223
SHERPA_RESULT_PREFIX,
24+
SHERPA_WINDOW_PREFIX,
2325
PARAKEET_MODEL_LABEL,
2426
writeParakeetTranscript,
2527
type SherpaWindow,
2628
} from "./parakeet.js";
29+
import { emitWords } from "./progress.js";
2730
import {
2831
getPreparedWavDurationSeconds,
2932
prepareWav,
@@ -394,13 +397,14 @@ function decode(
394397
wavPath: string,
395398
runtimePath: string,
396399
signal: AbortSignal,
400+
onWindow?: (window: SherpaWindow, through: number) => void,
397401
): Promise<SherpaWindow[]> {
398402
const sourceMode = import.meta.url.endsWith(".ts");
399403
const worker = new URL(sourceMode ? "./sherpaWorker.ts" : "./sherpaWorker.js", import.meta.url);
400404
const args = [...(sourceMode ? ["--import", "tsx"] : []), fileURLToPath(worker)];
401405
const input = { wavPath, runtimePath, config: recognizerConfig() };
402406
return new Promise((resolve, reject) => {
403-
execFile(
407+
const child = execFile(
404408
process.execPath,
405409
args,
406410
{
@@ -419,6 +423,18 @@ function decode(
419423
else reject(new Error(`Parakeet decoder ${failureReason(err, stderr)}`));
420424
},
421425
);
426+
if (onWindow && child.stdout) {
427+
createInterface({ input: child.stdout }).on("line", (l) => {
428+
if (!l.startsWith(SHERPA_WINDOW_PREFIX)) return;
429+
let streamed: { window: SherpaWindow; through: number };
430+
try {
431+
streamed = JSON.parse(l.slice(SHERPA_WINDOW_PREFIX.length));
432+
} catch {
433+
return;
434+
}
435+
onWindow(streamed.window, streamed.through);
436+
});
437+
}
422438
});
423439
}
424440

@@ -455,6 +471,7 @@ export async function transcribeWithSherpa(
455471
phase: "transcription",
456472
model: PARAKEET_MODEL_LABEL,
457473
status: "started",
474+
durationSeconds: getPreparedWavDurationSeconds(wavPath),
458475
});
459476
if (options.signal.aborted) throw new DecodeCancelled("Transcription cancelled");
460477
const selected = await selectRuntime(
@@ -468,7 +485,15 @@ export async function transcribeWithSherpa(
468485
`Parakeet runtime does not load (${selected.error}). Run hyperframes models install parakeet to repair it, or use --engine whisper.`,
469486
);
470487
}
471-
const windows = await decode(wavPath, selected.path, options.signal);
488+
const onEvent = options.onEvent;
489+
const windows = await decode(
490+
wavPath,
491+
selected.path,
492+
options.signal,
493+
onEvent &&
494+
((window, through) =>
495+
emitWords(onEvent, PARAKEET_MODEL_LABEL, mergeWindowsToWords([window]), through)),
496+
);
472497
const result = writeParakeetTranscript(dir, mergeWindowsToWords(windows));
473498
options.onEvent?.({
474499
type: "progress",

‎packages/cli/src/whisper/sherpaWorker.test.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
44
import { join } from "node:path";
55
import { fileURLToPath } from "node:url";
66
import { afterEach, describe, expect, it } from "vitest";
7-
import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX } from "./parakeet.js";
7+
import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX, SHERPA_WINDOW_PREFIX } from "./parakeet.js";
88

99
const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url));
1010

@@ -82,6 +82,15 @@ describe("sherpaWorker", () => {
8282
]);
8383
});
8484

85+
it("prints each window as it is decoded, with the audio seconds done", async () => {
86+
const { stdout } = await runWorker(fakeRuntime(), "speech.wav");
87+
const streamed = stdout
88+
.split("\n")
89+
.filter((l) => l.startsWith(SHERPA_WINDOW_PREFIX))
90+
.map((l) => JSON.parse(l.slice(SHERPA_WINDOW_PREFIX.length)));
91+
expect(streamed).toEqual([{ window: windowsOf(stdout)[0], through: 3 }]);
92+
});
93+
8594
it("re-decodes the window with leading silence when the gap alone gives nothing", async () => {
8695
const { code, stdout } = await runWorker(fakeRuntime(), "hum.wav");
8796
expect(code).toBe(0);

‎packages/cli/src/whisper/sherpaWorker.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
spliceGap,
66
SHERPA_ERROR_PREFIX,
77
SHERPA_RESULT_PREFIX,
8+
SHERPA_WINDOW_PREFIX,
89
type SherpaWindow,
910
} from "./parakeet.js";
1011

@@ -77,7 +78,10 @@ try {
7778
const retry = decodeWindow(recognizer, slice, RETRY_PAD_SECONDS);
7879
if (retry.tokens.length > decoded.tokens.length) decoded = retry;
7980
}
80-
windows.push({ offset: cuts[k]! / wave.sampleRate, ...decoded });
81+
const window = { offset: cuts[k]! / wave.sampleRate, ...decoded };
82+
windows.push(window);
83+
const through = cuts[k + 1]! / wave.sampleRate;
84+
process.stdout.write(`${SHERPA_WINDOW_PREFIX}${JSON.stringify({ window, through })}\n`);
8185
}
8286
process.stdout.write(`${SHERPA_RESULT_PREFIX}${JSON.stringify(windows)}\n`);
8387
} catch (err) {

0 commit comments

Comments
 (0)