Skip to content

Commit 6b02cbf

Browse files
fix(speech): match reference benchmarks to transcription hardware (#31956)
Co-authored-by: maria-rcks <[email protected]>
1 parent d7c52f9 commit 6b02cbf

15 files changed

Lines changed: 501 additions & 138 deletions

‎apps/mobile/src/features/settings/VoiceModelSection.tsx‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,12 +143,15 @@ function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly la
143143
],
144144
] as const)
145145
: []),
146-
...(data && data.secondsPer30s !== null
146+
...(data && data.secondsPer30s.accelerated !== null
147147
? ([
148148
[
149149
"speed",
150150
<>
151-
<Text className="font-t3-medium text-foreground">{data.secondsPer30s} s</Text> / 30 s
151+
<Text className="font-t3-medium text-foreground">
152+
{data.secondsPer30s.accelerated} s
153+
</Text>{" "}
154+
/ 30 s
152155
</>,
153156
],
154157
] as const)

‎apps/mobile/src/features/voice-input/voiceDestination.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ it("keeps local as the default and never reuses consent for an offline or resele
158158
acceleration: "cpu",
159159
modelUnloadTimeout: "never",
160160
gpuDevices: [],
161+
benchmarkTier: "cpu",
161162
}),
162163
)
163164
.mockResolvedValueOnce(Response.json({ text: "hello" }));

‎apps/server/scripts/speech-catalog.test.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,24 @@ it("reproduces the bundled registry byte for byte without upstream requests", as
4545
);
4646
expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
4747
accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
48-
secondsPer30s: null,
48+
secondsPer30s: { cpu: null, accelerated: null },
4949
});
50-
expect(generate().models[0]?.benchmark.secondsPer30s).toBeCloseTo(1.19, 2);
50+
expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2);
51+
for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) {
52+
const result = generateSpeechCatalog(
53+
{
54+
...catalog.data,
55+
models: [parakeet],
56+
results: { [parakeet.id]: { accuracy: {}, performance } },
57+
},
58+
"0.2.3",
59+
"0.4.1",
60+
);
61+
expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
62+
cpu: performance.cpu ? 15 : null,
63+
accelerated: performance.accelerated ? 3 : null,
64+
});
65+
}
5166
});
5267

5368
it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {

‎apps/server/scripts/speech-catalog.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,9 @@ export function generateSpeechCatalog(
8181
ids.add(model.repository);
8282
filenames.add(file.filename);
8383
const languages = [...new Set(model.languages.map(canonicalLanguage))];
84-
const realtimeFactor = data.results[model.id]?.performance.accelerated;
84+
const performance = data.results[model.id]?.performance;
85+
const secondsPer30s = (realtimeFactor: number | undefined) =>
86+
realtimeFactor ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2)) : null;
8587
const accuracy = Object.fromEntries(
8688
languages.flatMap((language) => {
8789
const cell = data.results[model.id]?.accuracy[languageIdentity(language)];
@@ -109,9 +111,10 @@ export function generateSpeechCatalog(
109111
languages,
110112
benchmark: {
111113
accuracy,
112-
secondsPer30s: realtimeFactor
113-
? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2))
114-
: null,
114+
secondsPer30s: {
115+
cpu: secondsPer30s(performance?.cpu),
116+
accelerated: secondsPer30s(performance?.accelerated),
117+
},
115118
},
116119
recommended: RECOMMENDED.has(model.id),
117120
supportsStreaming: model.capabilities.streaming,

‎apps/server/src/speech/SpeechService.test.ts‎

Lines changed: 50 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,10 @@ const modelDefinitions = vi.hoisted(() => [
5858
size: 4,
5959
languages: ["en", "fr"],
6060
accuracy: 1,
61-
benchmark: { accuracy: { en: { grade: "A-", errorPercent: 4 } }, secondsPer30s: 1.2 },
61+
benchmark: {
62+
accuracy: { en: { grade: "A-", errorPercent: 4 } },
63+
secondsPer30s: { cpu: 3, accelerated: 1.2 },
64+
},
6265
speed: 1,
6366
recommended: true,
6467
supportsStreaming: true,
@@ -146,31 +149,40 @@ it("rejects empty, misaligned, non-finite, and oversized PCM as invalid audio",
146149
}
147150
});
148151

149-
it.effect("prepares a batch model before audio arrives and reuses it for transcription", () =>
150-
Effect.gen(function* () {
151-
const speech = yield* SpeechService.SpeechService;
152-
yield* speech.selectModel("fallback-model");
153-
yield* speech.prepareModel;
154-
expect(loadNative).toHaveBeenCalledTimes(1);
155-
expect(native.transcribe).not.toHaveBeenCalled();
156-
native.supportsStreaming = false;
157-
yield* Effect.gen(function* () {
158-
const stream = yield* speech.startStream({
159-
...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
160-
speechLanguage: "fr",
161-
speechTranslateToEnglish: true,
162-
speechSilenceFiltering: false,
152+
it.effect.each(["CPU", "Vulkan0"])(
153+
"prepares a batch model on %s before audio arrives and reuses it for transcription",
154+
(backend) =>
155+
Effect.gen(function* () {
156+
const speech = yield* SpeechService.SpeechService;
157+
yield* speech.selectModel("fallback-model");
158+
loadNative.mockImplementationOnce(async () => {
159+
native.backend = backend;
160+
return native;
163161
});
164-
expect(native.begin).toHaveBeenLastCalledWith(
165-
{ timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
166-
false,
167-
"test.gguf",
168-
);
169-
expect(yield* stream.finish).toBe("hello");
170-
}).pipe(Effect.scoped);
171-
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
172-
expect(loadNative).toHaveBeenCalledTimes(1);
173-
}).pipe(Effect.provide(layer)),
162+
yield* speech.prepareModel;
163+
expect(yield* speech.status).toMatchObject({
164+
benchmarkTier: backend === "CPU" ? "cpu" : "accelerated",
165+
});
166+
expect(loadNative).toHaveBeenCalledTimes(1);
167+
expect(native.transcribe).not.toHaveBeenCalled();
168+
native.supportsStreaming = false;
169+
yield* Effect.gen(function* () {
170+
const stream = yield* speech.startStream({
171+
...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
172+
speechLanguage: "fr",
173+
speechTranslateToEnglish: true,
174+
speechSilenceFiltering: false,
175+
});
176+
expect(native.begin).toHaveBeenLastCalledWith(
177+
{ timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
178+
false,
179+
"test.gguf",
180+
);
181+
expect(yield* stream.finish).toBe("hello");
182+
}).pipe(Effect.scoped);
183+
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
184+
expect(loadNative).toHaveBeenCalledTimes(1);
185+
}).pipe(Effect.provide(layer)),
174186
);
175187

176188
it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
@@ -290,7 +302,7 @@ it.effect("rejects model mutations outside the offered registry", () =>
290302
expect(current).not.toHaveProperty("benchmark");
291303
expect(
292304
(yield* speech.models(true)).models.find((model) => model.active)?.benchmark?.secondsPer30s,
293-
).toBe(1.2);
305+
).toEqual({ cpu: 3, accelerated: 1.2 });
294306
}).pipe(Effect.provide(layer)),
295307
);
296308

@@ -665,11 +677,19 @@ it.effect.each([{ devices: [] }, { devices: [{ id: "gpu", name: "Test GPU" }] }]
665677
const speech = yield* SpeechService.SpeechService;
666678
const discover = vi.mocked(listNativeSpeechGpuDevices).mockReset();
667679
discover.mockRejectedValueOnce(new Error("driver not ready")).mockResolvedValueOnce(devices);
668-
expect(yield* speech.status).toMatchObject({ gpuDevices: [] });
680+
expect(yield* speech.status).toMatchObject({ gpuDevices: [], benchmarkTier: "cpu" });
669681
yield* speech.status;
670682
expect(discover).toHaveBeenCalledOnce();
671683
yield* TestClock.adjust("1 minute");
672-
expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
684+
expect(yield* speech.status).toMatchObject({
685+
gpuDevices: devices,
686+
benchmarkTier: devices.length ? "accelerated" : "cpu",
687+
});
688+
expect(yield* speech.updateAcceleration("cpu")).toMatchObject({ benchmarkTier: "cpu" });
689+
expect(yield* speech.updateAcceleration('gpu:["vulkan","gpu-1"]')).toMatchObject({
690+
benchmarkTier: "accelerated",
691+
});
692+
yield* speech.updateAcceleration("auto");
673693
yield* TestClock.adjust("1 minute");
674694
expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
675695
expect(discover).toHaveBeenCalledTimes(2);
@@ -700,9 +720,11 @@ it.effect(
700720
],
701721
};
702722
const speech = yield* SpeechService.SpeechService;
723+
expect(yield* speech.status).toMatchObject({ benchmarkTier: "accelerated" });
703724
expect(yield* speech.transcribe(pcm(), options)).toBe(
704725
cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
705726
);
727+
expect(yield* speech.status).toMatchObject({ benchmarkTier: "cpu" });
706728
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe(
707729
"hello",
708730
);

‎apps/server/src/speech/SpeechService.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,13 @@ const make = Effect.gen(function* () {
452452
if (unsupportedReason) return { supported: false, reason: unsupportedReason };
453453
const definition = selectedModel(settings);
454454
const ready = yield* store.isReady(definition);
455+
const gpuDevices = yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => []));
456+
const autoUsesCpu =
457+
gpuDevices.length === 0 ||
458+
autoCpuModel === definition.id ||
459+
(loadedModelId === definition.id &&
460+
loadedAcceleration === "auto" &&
461+
model?.backend.toLowerCase() === "cpu");
455462
return {
456463
supported: true,
457464
state: activeTranscriptions > 0 ? "transcribing" : ready ? "ready" : "missing-model",
@@ -464,8 +471,13 @@ const make = Effect.gen(function* () {
464471
languages: definition.languages,
465472
supportsLanguageDetection: definition.supportsLanguageDetection,
466473
acceleration: settings.speechAcceleration,
474+
benchmarkTier:
475+
settings.speechAcceleration === "cpu" ||
476+
(settings.speechAcceleration === "auto" && autoUsesCpu)
477+
? "cpu"
478+
: "accelerated",
467479
modelUnloadTimeout: settings.speechModelUnloadTimeout,
468-
gpuDevices: yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => [])),
480+
gpuDevices,
469481
};
470482
});
471483

0 commit comments

Comments
 (0)