Skip to content

Commit c2b782b

Browse files
committed
fix(cli): recover a clause Parakeet drops after a pause by decoding the gap alone
Each loud gap without tokens is decoded on its own with 0.5 s either side, and only its new tokens are spliced in. The padded whole-window retry stays as the fallback. A window with no tokens at all is no longer decoded twice.
1 parent 966815b commit c2b782b

4 files changed

Lines changed: 133 additions & 30 deletions

File tree

‎packages/cli/src/whisper/parakeet.test.ts‎

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import { describe, expect, it } from "vitest";
22
import {
3-
hasDroppedSpeech,
3+
droppedSpeechGaps,
44
mergeTokensToWords,
55
mergeWindowsToWords,
66
silenceCuts,
7+
spliceGap,
78
} from "./parakeet.js";
89

910
describe("mergeTokensToWords", () => {
@@ -75,28 +76,54 @@ describe("silenceCuts", () => {
7576
});
7677
});
7778

78-
describe("hasDroppedSpeech", () => {
79+
describe("droppedSpeechGaps", () => {
7980
const rate = 100;
8081
const tokens = (timestamps: number[]) => ({
8182
tokens: timestamps.map(() => " w"),
8283
timestamps,
8384
durations: timestamps.map(() => 0.2),
8485
});
8586

86-
it("flags a second of loud audio before the first token", () => {
87+
it("finds a second of loud audio before the first token and between tokens", () => {
8788
const speech = new Float32Array(6 * rate).fill(0.5);
88-
expect(hasDroppedSpeech(speech, rate, tokens([2.08, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]))).toBe(true);
89+
expect(droppedSpeechGaps(speech, rate, tokens([2.08, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]))).toEqual([
90+
[0, 2.08],
91+
]);
92+
expect(droppedSpeechGaps(speech, rate, tokens([0, 0.5, 1, 3.5, 4, 4.5, 5, 5.5]))).toEqual([
93+
[1.2, 3.5],
94+
]);
8995
expect(
90-
hasDroppedSpeech(speech, rate, tokens([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5])),
91-
).toBe(false);
96+
droppedSpeechGaps(speech, rate, tokens([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5])),
97+
).toEqual([]);
9298
});
9399

94-
it("ignores a quiet pause and silence with no tokens", () => {
100+
it("ignores a quiet pause, silence, and a loud window with no tokens at all", () => {
95101
const pause = new Float32Array(6 * rate).fill(0.5);
96102
pause.fill(0, 2 * rate, 4 * rate);
97-
expect(hasDroppedSpeech(pause, rate, tokens([0.1, 0.6, 1.1, 1.6, 4.1, 4.6, 5.1, 5.6]))).toBe(
98-
false,
99-
);
100-
expect(hasDroppedSpeech(new Float32Array(3 * rate), rate, tokens([]))).toBe(false);
103+
expect(
104+
droppedSpeechGaps(pause, rate, tokens([0.1, 0.6, 1.1, 1.6, 4.1, 4.6, 5.1, 5.6])),
105+
).toEqual([]);
106+
expect(droppedSpeechGaps(new Float32Array(3 * rate), rate, tokens([]))).toEqual([]);
107+
expect(droppedSpeechGaps(new Float32Array(3 * rate).fill(0.5), rate, tokens([]))).toEqual([]);
108+
});
109+
});
110+
111+
describe("spliceGap", () => {
112+
it("adds, in order, the re-decoded tokens inside the gap, minus its re-heard neighbours", () => {
113+
const decoded = {
114+
tokens: [" so", " A", "sk"],
115+
timestamps: [9.5, 13, 13.25],
116+
durations: [0.25, 0.25, 0.25],
117+
};
118+
const patch = {
119+
tokens: [" so", " ask", " not", ",", " a", "sk"],
120+
timestamps: [0.5, 0.75, 2.75, 3.25, 3.5, 3.75],
121+
durations: [0.25, 0.25, 0.25, 0.25, 0.25, 0.25],
122+
};
123+
expect(spliceGap(decoded, patch, 9.25, [9.75, 13])).toEqual({
124+
tokens: [" so", " ask", " not", ",", " A", "sk"],
125+
timestamps: [9.5, 10, 12, 12.5, 13, 13.25],
126+
durations: [0.25, 0.25, 0.25, 0.25, 0.25, 0.25],
127+
});
101128
});
102129
});

‎packages/cli/src/whisper/parakeet.ts‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -137,23 +137,68 @@ function meanEnergy(samples: Float32Array, from: number, to: number): number {
137137
}
138138

139139
/**
140-
* True when a stretch of at least 1 s without tokens is about as loud as the window: the model
140+
* Stretches (s) of at least 1 s without tokens that are about as loud as the window: the model
141141
* skipped speech there. Measured: a 61.91 s window lost its first 9 words this way (ratio 0.94).
142+
* A window with no tokens at all is music or noise, not a skip.
142143
*/
143-
export function hasDroppedSpeech(
144+
export function droppedSpeechGaps(
144145
samples: Float32Array,
145146
sampleRate: number,
146147
decoded: Omit<SherpaWindow, "offset">,
147-
): boolean {
148+
): [number, number][] {
149+
if (decoded.tokens.length === 0) return [];
148150
const loud = 0.3 * meanEnergy(samples, 0, samples.length);
149151
const { timestamps, durations } = decoded;
150152
const gapStarts = [0, ...timestamps.map((t, i) => t + (durations?.[i] ?? 0))];
151153
const gapEnds = [...timestamps, samples.length / sampleRate];
152-
return gapStarts.some((from, i) => {
153-
const to = gapEnds[i]!;
154-
const at = (seconds: number) => Math.round(seconds * sampleRate);
155-
return to - from >= 1 && meanEnergy(samples, at(from), at(to)) > loud;
154+
const at = (seconds: number) => Math.round(seconds * sampleRate);
155+
return gapStarts
156+
.map((from, i): [number, number] => [from, gapEnds[i]!])
157+
.filter(([from, to]) => to - from >= 1 && meanEnergy(samples, at(from), at(to)) > loud);
158+
}
159+
160+
/** Number of trailing `a` entries that equal the leading `b` entries. */
161+
function overlap(a: string[], b: string[]): number {
162+
for (let n = Math.min(a.length, b.length); n > 0; n--) {
163+
if (a.slice(-n).every((text, i) => text === b[i])) return n;
164+
}
165+
return 0;
166+
}
167+
168+
/**
169+
* Adds the `patch` tokens, decoded from `offset` s into the window, that start inside the gap.
170+
* The patch re-hears the gap's neighbours up to 0.3 s off, so those go by letters (not case).
171+
*/
172+
export function spliceGap(
173+
decoded: Omit<SherpaWindow, "offset">,
174+
patch: Omit<SherpaWindow, "offset">,
175+
offset: number,
176+
[from, to]: [number, number],
177+
): Omit<SherpaWindow, "offset"> {
178+
const token = (w: Omit<SherpaWindow, "offset">, i: number, shift: number) => ({
179+
text: w.tokens[i]!,
180+
start: shift + w.timestamps[i]!,
181+
duration: w.durations?.[i] ?? 0,
156182
});
183+
const kept = decoded.tokens.map((_, i) => token(decoded, i, 0));
184+
const texts = (tokens: typeof kept) => tokens.map((t) => t.text);
185+
const letters = (tokens: typeof kept) =>
186+
tokens.map((t) => t.text.toLowerCase().replace(/[^\p{L}\p{N}]/gu, ""));
187+
const added = patch.tokens
188+
.map((_, i) => token(patch, i, offset))
189+
.filter(({ start }) => start >= from && start < to);
190+
const before = letters(kept.filter((t) => t.start < from));
191+
const after = letters(kept.filter((t) => t.start >= to));
192+
const fresh = added.slice(
193+
overlap(before, letters(added)),
194+
added.length - overlap(letters(added), after),
195+
);
196+
const merged = [...kept, ...fresh].sort((a, b) => a.start - b.start);
197+
return {
198+
tokens: texts(merged),
199+
timestamps: merged.map((t) => t.start),
200+
durations: merged.map((t) => t.duration),
201+
};
157202
}
158203

159204
function quietestBlockCenter(samples: Float32Array, lo: number, hi: number, block: number): number {

‎packages/cli/src/whisper/sherpaWorker.test.ts‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,28 @@ import { SHERPA_ERROR_PREFIX, SHERPA_RESULT_PREFIX } from "./parakeet.js";
88

99
const WORKER = fileURLToPath(new URL("./sherpaWorker.ts", import.meta.url));
1010

11-
// A stand-in sherpa-onnx-node: 3 s of steady sound. Unpadded, the recognizer "drops" the first
12-
// 2 s the way the real model did on a 61.91 s window; with leading silence it hears everything.
11+
// A stand-in sherpa-onnx-node: 3 s of steady sound. The whole window drops the first 2 s, as the real
12+
// model did on a 61.91 s window; a padded window, or the gap alone for speech.wav, hears it all.
1313
const FAKE_SHERPA = `
14+
let path;
1415
module.exports = {
15-
readWave(path) {
16+
readWave(p) {
17+
path = p;
1618
if (path.endsWith("broken.wav")) throw new Error("Failed to read " + path);
1719
return { sampleRate: 100, samples: new Float32Array(300).fill(0.5) };
1820
},
1921
OfflineRecognizer: class {
2022
createStream() { return { acceptWaveform(w) { this.w = w; } }; }
2123
decode() {}
2224
getResult(stream) {
23-
const padded = stream.w.samples[0] === 0;
24-
return padded
25-
? { tokens: [" ask", " not"], timestamps: [0.75, 1.5], durations: [0.4, 0.4] }
26-
: { tokens: [" not"], timestamps: [2.08], durations: [0.4] };
25+
const samples = stream.w.samples;
26+
if (samples[0] === 0) {
27+
return { tokens: [" ask", " not"], timestamps: [0.75, 1.5], durations: [0.4, 0.4] };
28+
}
29+
if (samples.length === 300) return { tokens: [" not"], timestamps: [2.08], durations: [0.4] };
30+
return path.endsWith("speech.wav")
31+
? { tokens: [" ask", " not"], timestamps: [0.5, 2.08], durations: [0.4, 0.4] }
32+
: { tokens: [], timestamps: [], durations: [] };
2733
}
2834
},
2935
};
@@ -54,11 +60,23 @@ describe("sherpaWorker", () => {
5460
return root;
5561
}
5662

57-
it("re-decodes a window that skipped loud audio, with leading silence", async () => {
63+
function windowsOf(stdout: string) {
64+
const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!;
65+
return JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length));
66+
}
67+
68+
it("decodes a gap that skipped loud audio on its own and splices in its new tokens", async () => {
5869
const { code, stdout } = await runWorker(fakeRuntime(), "speech.wav");
5970
expect(code).toBe(0);
60-
const line = stdout.split("\n").find((l) => l.startsWith(SHERPA_RESULT_PREFIX))!;
61-
expect(JSON.parse(line.slice(SHERPA_RESULT_PREFIX.length))).toEqual([
71+
expect(windowsOf(stdout)).toEqual([
72+
{ offset: 0, tokens: [" ask", " not"], timestamps: [0.5, 2.08], durations: [0.4, 0.4] },
73+
]);
74+
});
75+
76+
it("re-decodes the window with leading silence when the gap alone gives nothing", async () => {
77+
const { code, stdout } = await runWorker(fakeRuntime(), "hum.wav");
78+
expect(code).toBe(0);
79+
expect(windowsOf(stdout)).toEqual([
6280
{ offset: 0, tokens: [" ask", " not"], timestamps: [0.25, 1], durations: [0.4, 0.4] },
6381
]);
6482
});

‎packages/cli/src/whisper/sherpaWorker.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import { loadInstalled } from "../utils/optionalPackages.js";
22
import {
3-
hasDroppedSpeech,
3+
droppedSpeechGaps,
44
silenceCuts,
5+
spliceGap,
56
SHERPA_ERROR_PREFIX,
67
SHERPA_RESULT_PREFIX,
78
type SherpaWindow,
@@ -25,6 +26,8 @@ const { wavPath, runtimeDir, config } = JSON.parse(process.env.HYPERFRAMES_PARAK
2526

2627
/** Leading silence moves the frame grid; 0.5 s recovered the dropped clause at every length tried. */
2728
const RETRY_PAD_SECONDS = 0.5;
29+
/** Audio kept on each side of a skipped gap when it is decoded on its own. */
30+
const GAP_MARGIN_SECONDS = 0.5;
2831

2932
function decodeWindow(
3033
recognizer: InstanceType<SherpaOnnx["OfflineRecognizer"]>,
@@ -54,7 +57,17 @@ try {
5457
samples: wave.samples.subarray(cuts[k], cuts[k + 1]),
5558
};
5659
let decoded = decodeWindow(recognizer, slice);
57-
if (hasDroppedSpeech(slice.samples, slice.sampleRate, decoded)) {
60+
const gaps = droppedSpeechGaps(slice.samples, slice.sampleRate, decoded);
61+
let spliced = decoded;
62+
for (const [from, to] of gaps) {
63+
const lo = Math.max(0, from - GAP_MARGIN_SECONDS);
64+
const at = (seconds: number) => Math.round(seconds * slice.sampleRate);
65+
const samples = slice.samples.subarray(at(lo), at(to + GAP_MARGIN_SECONDS));
66+
const patch = decodeWindow(recognizer, { sampleRate: slice.sampleRate, samples });
67+
spliced = spliceGap(spliced, patch, lo, [from, to]);
68+
}
69+
if (spliced.tokens.length > decoded.tokens.length) decoded = spliced;
70+
else if (gaps.length > 0) {
5871
const retry = decodeWindow(recognizer, slice, RETRY_PAD_SECONDS);
5972
if (retry.tokens.length > decoded.tokens.length) decoded = retry;
6073
}

0 commit comments

Comments
 (0)