Skip to content

Commit d02fa46

Browse files
committed
fix(cli): read the WAV in JS so Built-in transcription runs inside Electron
1 parent 156e877 commit d02fa46

7 files changed

Lines changed: 121 additions & 32 deletions

File tree

‎packages/cli/src/whisper/sherpa.test.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import {
2525
transcribeWithSherpa,
2626
type ModelFile,
2727
} from "./sherpa.js";
28+
import { encodeWav } from "./wav.test-helpers.js";
2829

2930
const file = (name: string, content: string): ModelFile => ({
3031
name,
@@ -183,6 +184,7 @@ describe("a pinned Sherpa copy beside the CLI", () => {
183184
let root: string;
184185
beforeEach(() => {
185186
root = mkdtempSync(join(tmpdir(), "hf-sherpa-beside-"));
187+
writeFileSync(join(root, "speech.wav"), encodeWav(new Float32Array(100).fill(0.5), 100));
186188
});
187189
afterEach(() => {
188190
vi.restoreAllMocks();
@@ -204,7 +206,7 @@ describe("a pinned Sherpa copy beside the CLI", () => {
204206
return `
205207
if (process.pid === ${process.pid}) require("node:fs").writeFileSync(${JSON.stringify(parentMarker)}, "");
206208
module.exports = {
207-
readWave() { return { sampleRate: 100, samples: new Float32Array(100).fill(0.5) }; },
209+
readWave() { throw new Error("External buffers are not allowed"); },
208210
OfflineRecognizer: class {
209211
createStream() { return { acceptWaveform() {} }; }
210212
decode() {}
@@ -228,7 +230,8 @@ module.exports = {
228230
const event = { type: "progress", phase: "transcription", model: "parakeet-tdt-0.6b-v3" };
229231
// The worker's one 1 s window streams its words before the final transcript.
230232
expect(onEvent.mock.calls).toEqual([
231-
[{ ...event, status: "started", durationSeconds: null }],
233+
// The started line sizes the WAV as 16 kHz mono: 44 header bytes and 100 samples here.
234+
[{ ...event, status: "started", durationSeconds: (44 + 2 * 100) / 32_000 }],
232235
[{ type: "words", model: event.model, words: result, through: 1 }],
233236
[{ ...event, status: "completed" }],
234237
]);

‎packages/cli/src/whisper/sherpaWorker.test.ts‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,20 @@ import { join } from "node:path";
55
import { fileURLToPath } from "node:url";
66
import { afterEach, describe, expect, it } from "vitest";
77
import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX, SHERPA_WINDOW_PREFIX } from "./parakeet.js";
8+
import { encodeWav } from "./wav.test-helpers.js";
89

910
const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url));
1011

1112
// Stand-in sherpa-onnx-node, 3 s of sound: the whole window (300 samples) drops the first 2 s; the
1213
// padded window (350) hears it, and for speech.wav so does the padded gap +-0.5 s (308) alone.
1314
const FAKE_SHERPA = `
14-
let path;
15+
const path = JSON.parse(process.env.HYPERFRAMES_PARAKEET_INPUT).wavPath;
1516
module.exports = {
16-
readWave(p) {
17-
path = p;
18-
if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path);
19-
if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n ../a.node\\n ./b.node\\n");
20-
if (path.endsWith("long.wav")) return { sampleRate: 100, samples: new Float32Array(60000) };
21-
return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) };
22-
},
17+
// What Electron's V8 sandbox throws for this reader's native buffer: the worker reads the WAV itself.
18+
readWave() { throw new Error("External buffers are not allowed"); },
2319
OfflineRecognizer: class {
2420
constructor() {
21+
if (path.endsWith("lines.wav")) throw new Error("Could not find it. Tried\\n\\n ../a.node\\n ./b.node\\n");
2522
if (process.env.STARTED) require("fs").writeFileSync(process.env.STARTED, "");
2623
}
2724
createStream() { return { acceptWaveform(w) { this.w = w; } }; }
@@ -69,21 +66,28 @@ describe("sherpaWorker", () => {
6966
return root;
7067
}
7168

69+
/** 3 s of steady sound at 100 Hz, as the stand-in decoder expects; long.wav is 600 s of silence. */
70+
function wav(name: string): string {
71+
const samples = name === "long.wav" ? new Float32Array(60000) : new Float32Array(300).fill(0.5);
72+
writeFileSync(join(root, name), encodeWav(samples, 100));
73+
return join(root, name);
74+
}
75+
7276
function windowsOf(stdout: string) {
7377
const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!;
7478
return JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length));
7579
}
7680

7781
it("decodes a gap that skipped loud audio on its own and splices in its new tokens", async () => {
78-
const { code, stdout } = await runWorker(fakeRuntime(), "speech.wav");
82+
const { code, stdout } = await runWorker(fakeRuntime(), wav("speech.wav"));
7983
expect(code).toBe(0);
8084
expect(windowsOf(stdout)).toEqual([
8185
{ offset: 0, tokens: [" ask", " not"], timestamps: [0.5, 2.08], durations: [0.4, 0.4] },
8286
]);
8387
});
8488

8589
it("prints each window as it is decoded, with the audio seconds done", async () => {
86-
const { stdout } = await runWorker(fakeRuntime(), "speech.wav");
90+
const { stdout } = await runWorker(fakeRuntime(), wav("speech.wav"));
8791
const streamed = stdout
8892
.split("\n")
8993
.filter((l) => l.startsWith(SHERPA_WINDOW_PREFIX))
@@ -92,17 +96,19 @@ describe("sherpaWorker", () => {
9296
});
9397

9498
it("re-decodes the window with leading silence when the gap alone gives nothing", async () => {
95-
const { code, stdout } = await runWorker(fakeRuntime(), "hum.wav");
99+
const { code, stdout } = await runWorker(fakeRuntime(), wav("hum.wav"));
96100
expect(code).toBe(0);
97101
expect(windowsOf(stdout)).toEqual([
98102
{ offset: 0, tokens: [" ask", " not"], timestamps: [0.25, 1], durations: [0.4, 0.4] },
99103
]);
100104
});
101105

102106
it("exits non-zero with one prefixed error line when it cannot read the audio", async () => {
103-
const { code, stderr } = await runWorker(fakeRuntime(), "broken.wav");
107+
const broken = join(fakeRuntime(), "broken.wav");
108+
writeFileSync(broken, "not audio");
109+
const { code, stderr } = await runWorker(root, broken);
104110
expect(code).toBe(1);
105-
expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}Failed to read broken.wav`);
111+
expect(stderr).toContain(`${SHERPA_ERROR_PREFIX}${broken} is not a 16-bit mono PCM WAV`);
106112
});
107113

108114
it("keeps every line of a multi-line error on its one prefixed line", async () => {
@@ -116,7 +122,7 @@ describe("sherpaWorker", () => {
116122
async () => {
117123
const started = join(fakeRuntime(), "started");
118124
const input = JSON.stringify({
119-
wavPath: "long.wav",
125+
wavPath: wav("long.wav"),
120126
runtimePath: join(root, "node_modules", "sherpa-onnx-node", "index.js"),
121127
config: {},
122128
});

‎packages/cli/src/whisper/sherpaWorker.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ import {
88
SHERPA_WINDOW_PREFIX,
99
type SherpaWindow,
1010
} from "./parakeet.js";
11+
import { readWav } from "./wav.js";
1112

1213
interface Wave {
1314
samples: Float32Array;
1415
sampleRate: number;
1516
}
1617

1718
interface SherpaOnnx {
18-
readWave(path: string): Wave;
1919
OfflineRecognizer: new (config: object) => {
2020
createStream(): { acceptWaveform(wave: Wave): void };
2121
decode(stream: unknown): void;
@@ -49,7 +49,7 @@ function decodeWindow(
4949
try {
5050
const sherpa = createRequire(import.meta.url)(runtimePath) as SherpaOnnx;
5151
const recognizer = new sherpa.OfflineRecognizer(config);
52-
const wave = sherpa.readWave(wavPath);
52+
const wave = readWav(wavPath);
5353
const cuts = silenceCuts(wave.samples, wave.sampleRate);
5454
const windows: SherpaWindow[] = [];
5555
for (let k = 0; k + 1 < cuts.length; k++) {

‎packages/cli/src/whisper/transcribe.ts‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,10 @@ import { randomUUID } from "node:crypto";
1010
import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
1111
import { stoppedByCancelSignal } from "../utils/renderCancellation.js";
1212
import { ensureWhisper, ensureModel, hasFFmpeg, DEFAULT_MODEL } from "./manager.js";
13+
import { findWavChunk } from "./wav.js";
1314
import type { Word } from "./normalize.js";
1415
import { emitWords } from "./progress.js";
1516

16-
function findWavDataChunk(buf: Buffer): { offset: number; size: number } | null {
17-
if (buf.length < 12) return null;
18-
let pos = 12; // skip RIFF header
19-
while (pos + 8 < buf.length) {
20-
const id = buf.toString("ascii", pos, pos + 4);
21-
const size = buf.readUInt32LE(pos + 4);
22-
if (id === "data") return { offset: pos + 8, size: Math.min(size, buf.length - pos - 8) };
23-
pos += 8 + size;
24-
if (size % 2 !== 0) pos++; // RIFF chunks are word-aligned
25-
}
26-
return null;
27-
}
28-
2917
const WHISPER_TIMEOUT_FLOOR_MS = 300_000;
3018
const WHISPER_TIMEOUT_PER_AUDIO_SECOND_MS = 10_000;
3119
const WHISPER_TIMEOUT_CAP_MS = 43_200_000;
@@ -196,7 +184,7 @@ export function detectSpeechOnset(wavPath: string): number | null {
196184

197185
try {
198186
const buf = readFileSync(wavPath);
199-
const dataChunk = findWavDataChunk(buf);
187+
const dataChunk = findWavChunk(buf, "data");
200188
if (!dataChunk) return null;
201189
const pcm = new Int16Array(buf.buffer, buf.byteOffset + dataChunk.offset, dataChunk.size / 2);
202190
const totalWindows = Math.floor(pcm.length / WINDOW_SAMPLES);
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/** A 16-bit mono PCM WAV of `samples` (-1..1), as prepareWav writes one. */
2+
export function encodeWav(samples: ArrayLike<number>, sampleRate: number): Buffer {
3+
const data = Buffer.alloc(samples.length * 2);
4+
for (let i = 0; i < samples.length; i++) {
5+
data.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(samples[i]! * 32768))), 2 * i);
6+
}
7+
const head = Buffer.alloc(44);
8+
head.write("RIFF", 0, "ascii");
9+
head.writeUInt32LE(36 + data.length, 4);
10+
head.write("WAVEfmt ", 8, "ascii");
11+
head.writeUInt32LE(16, 16);
12+
head.writeUInt16LE(1, 20);
13+
head.writeUInt16LE(1, 22);
14+
head.writeUInt32LE(sampleRate, 24);
15+
head.writeUInt32LE(sampleRate * 2, 28);
16+
head.writeUInt16LE(2, 32);
17+
head.writeUInt16LE(16, 34);
18+
head.write("data", 36, "ascii");
19+
head.writeUInt32LE(data.length, 40);
20+
return Buffer.concat([head, data]);
21+
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { afterEach, beforeEach, expect, it } from "vitest";
5+
import { readWav } from "./wav.js";
6+
import { encodeWav } from "./wav.test-helpers.js";
7+
8+
let dir: string;
9+
beforeEach(() => {
10+
dir = mkdtempSync(join(tmpdir(), "hf-wav-"));
11+
});
12+
afterEach(() => rmSync(dir, { recursive: true, force: true }));
13+
const file = (name: string, bytes: Buffer) => {
14+
writeFileSync(join(dir, name), bytes);
15+
return join(dir, name);
16+
};
17+
18+
it("reads a 16-bit mono WAV's samples and rate in JS", () => {
19+
const wav = readWav(file("a.wav", encodeWav([0, 0.5, -0.5, -1], 16_000)));
20+
expect(wav.sampleRate).toBe(16_000);
21+
expect([...wav.samples]).toEqual([0, 0.5, -0.5, -1]);
22+
});
23+
24+
it("names a file that is not a 16-bit mono WAV", () => {
25+
const stereo = encodeWav([0, 0], 16_000);
26+
stereo.writeUInt16LE(2, 22);
27+
expect(() => readWav(file("text.wav", Buffer.from("not audio")))).toThrow(
28+
"text.wav is not a 16-bit mono PCM WAV",
29+
);
30+
expect(() => readWav(file("stereo.wav", stereo))).toThrow("is not a 16-bit mono PCM WAV");
31+
});

‎packages/cli/src/whisper/wav.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import { readFileSync } from "node:fs";
2+
3+
/** Where a RIFF chunk's bytes start and how many there are, or null when the file has none. */
4+
export function findWavChunk(buf: Buffer, want: string): { offset: number; size: number } | null {
5+
if (buf.length < 12) return null;
6+
let pos = 12; // skip RIFF header
7+
while (pos + 8 < buf.length) {
8+
const id = buf.toString("ascii", pos, pos + 4);
9+
const size = buf.readUInt32LE(pos + 4);
10+
if (id === want) return { offset: pos + 8, size: Math.min(size, buf.length - pos - 8) };
11+
pos += 8 + size;
12+
if (size % 2 !== 0) pos++; // RIFF chunks are word-aligned
13+
}
14+
return null;
15+
}
16+
17+
const PCM = 1;
18+
const EXTENSIBLE = 0xfffe;
19+
20+
/** A prepared 16-bit mono WAV, read in JS: sherpa's readWave returns memory Electron refuses. */
21+
export function readWav(path: string): { samples: Float32Array; sampleRate: number } {
22+
const buf = readFileSync(path);
23+
const riff = buf.toString("ascii", 0, 4) === "RIFF" && buf.toString("ascii", 8, 12) === "WAVE";
24+
const fmt = riff ? findWavChunk(buf, "fmt ") : null;
25+
const data = fmt && findWavChunk(buf, "data");
26+
const format = fmt && buf.readUInt16LE(fmt.offset);
27+
if (
28+
!fmt ||
29+
!data ||
30+
(format !== PCM && format !== EXTENSIBLE) ||
31+
buf.readUInt16LE(fmt.offset + 2) !== 1 ||
32+
buf.readUInt16LE(fmt.offset + 14) !== 16
33+
) {
34+
throw new Error(`${path} is not a 16-bit mono PCM WAV`);
35+
}
36+
const samples = new Float32Array(data.size >> 1);
37+
for (let i = 0; i < samples.length; i++)
38+
samples[i] = buf.readInt16LE(data.offset + 2 * i) / 32768;
39+
return { samples, sampleRate: buf.readUInt32LE(fmt.offset + 4) };
40+
}

0 commit comments

Comments
 (0)