Skip to content

Commit 422bdb9

Browse files
fix(speech): match reference benchmarks to transcription hardware
1 parent 202d5b5 commit 422bdb9

12 files changed

Lines changed: 315 additions & 114 deletions

File tree

‎apps/mobile/src/features/settings/VoiceModelSection.tsx‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,12 +143,15 @@ function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly la
143143
],
144144
] as const)
145145
: []),
146-
...(data && data.secondsPer30s !== null
146+
...(data && data.secondsPer30s.accelerated !== null
147147
? ([
148148
[
149149
"speed",
150150
<>
151-
<Text className="font-t3-medium text-foreground">{data.secondsPer30s} s</Text> / 30 s
151+
<Text className="font-t3-medium text-foreground">
152+
{data.secondsPer30s.accelerated} s
153+
</Text>{" "}
154+
/ 30 s
152155
</>,
153156
],
154157
] as const)

‎apps/mobile/src/features/voice-input/voiceDestination.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ it("keeps local as the default and never reuses consent for an offline or resele
149149
acceleration: "cpu",
150150
modelUnloadTimeout: "never",
151151
gpuDevices: [],
152+
benchmarkTier: "cpu",
152153
}),
153154
)
154155
.mockResolvedValueOnce(Response.json({ text: "hello" }));

‎apps/server/scripts/speech-catalog.test.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,24 @@ it("reproduces the bundled registry byte for byte without upstream requests", as
4545
);
4646
expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
4747
accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
48-
secondsPer30s: null,
48+
secondsPer30s: { cpu: null, accelerated: null },
4949
});
50-
expect(generate().models[0]?.benchmark.secondsPer30s).toBeCloseTo(1.19, 2);
50+
expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2);
51+
for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) {
52+
const result = generateSpeechCatalog(
53+
{
54+
...catalog.data,
55+
models: [parakeet],
56+
results: { [parakeet.id]: { accuracy: {}, performance } },
57+
},
58+
"0.2.3",
59+
"0.4.1",
60+
);
61+
expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
62+
cpu: performance.cpu ? 15 : null,
63+
accelerated: performance.accelerated ? 3 : null,
64+
});
65+
}
5166
});
5267

5368
it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {

‎apps/server/scripts/speech-catalog.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,9 @@ export function generateSpeechCatalog(
8181
ids.add(model.repository);
8282
filenames.add(file.filename);
8383
const languages = [...new Set(model.languages.map(canonicalLanguage))];
84-
const realtimeFactor = data.results[model.id]?.performance.accelerated;
84+
const performance = data.results[model.id]?.performance;
85+
const secondsPer30s = (realtimeFactor: number | undefined) =>
86+
realtimeFactor ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2)) : null;
8587
const accuracy = Object.fromEntries(
8688
languages.flatMap((language) => {
8789
const cell = data.results[model.id]?.accuracy[languageIdentity(language)];
@@ -109,9 +111,10 @@ export function generateSpeechCatalog(
109111
languages,
110112
benchmark: {
111113
accuracy,
112-
secondsPer30s: realtimeFactor
113-
? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2))
114-
: null,
114+
secondsPer30s: {
115+
cpu: secondsPer30s(performance?.cpu),
116+
accelerated: secondsPer30s(performance?.accelerated),
117+
},
115118
},
116119
recommended: RECOMMENDED.has(model.id),
117120
supportsStreaming: model.capabilities.streaming,

‎apps/server/src/speech/SpeechService.test.ts‎

Lines changed: 50 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,10 @@ const modelDefinitions = vi.hoisted(() => [
5757
size: 4,
5858
languages: ["en", "fr"],
5959
accuracy: 1,
60-
benchmark: { accuracy: { en: { grade: "A-", errorPercent: 4 } }, secondsPer30s: 1.2 },
60+
benchmark: {
61+
accuracy: { en: { grade: "A-", errorPercent: 4 } },
62+
secondsPer30s: { cpu: 3, accelerated: 1.2 },
63+
},
6164
speed: 1,
6265
recommended: true,
6366
supportsStreaming: true,
@@ -144,31 +147,40 @@ it("rejects empty, misaligned, non-finite, and oversized PCM as invalid audio",
144147
}
145148
});
146149

147-
it.effect("prepares a batch model before audio arrives and reuses it for transcription", () =>
148-
Effect.gen(function* () {
149-
const speech = yield* SpeechService.SpeechService;
150-
yield* speech.selectModel("fallback-model");
151-
yield* speech.prepareModel;
152-
expect(loadNative).toHaveBeenCalledTimes(1);
153-
expect(native.transcribe).not.toHaveBeenCalled();
154-
native.supportsStreaming = false;
155-
yield* Effect.gen(function* () {
156-
const stream = yield* speech.startStream({
157-
...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
158-
speechLanguage: "fr",
159-
speechTranslateToEnglish: true,
160-
speechSilenceFiltering: false,
150+
it.effect.each(["CPU", "Vulkan0"])(
151+
"prepares a batch model on %s before audio arrives and reuses it for transcription",
152+
(backend) =>
153+
Effect.gen(function* () {
154+
const speech = yield* SpeechService.SpeechService;
155+
yield* speech.selectModel("fallback-model");
156+
loadNative.mockImplementationOnce(async () => {
157+
native.backend = backend;
158+
return native;
161159
});
162-
expect(native.begin).toHaveBeenLastCalledWith(
163-
{ timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
164-
false,
165-
"test.gguf",
166-
);
167-
expect(yield* stream.finish).toBe("hello");
168-
}).pipe(Effect.scoped);
169-
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
170-
expect(loadNative).toHaveBeenCalledTimes(1);
171-
}).pipe(Effect.provide(layer)),
160+
yield* speech.prepareModel;
161+
expect(yield* speech.status).toMatchObject({
162+
benchmarkTier: backend === "CPU" ? "cpu" : "accelerated",
163+
});
164+
expect(loadNative).toHaveBeenCalledTimes(1);
165+
expect(native.transcribe).not.toHaveBeenCalled();
166+
native.supportsStreaming = false;
167+
yield* Effect.gen(function* () {
168+
const stream = yield* speech.startStream({
169+
...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
170+
speechLanguage: "fr",
171+
speechTranslateToEnglish: true,
172+
speechSilenceFiltering: false,
173+
});
174+
expect(native.begin).toHaveBeenLastCalledWith(
175+
{ timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
176+
false,
177+
"test.gguf",
178+
);
179+
expect(yield* stream.finish).toBe("hello");
180+
}).pipe(Effect.scoped);
181+
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
182+
expect(loadNative).toHaveBeenCalledTimes(1);
183+
}).pipe(Effect.provide(layer)),
172184
);
173185

174186
it.effect.each(["cancellation", "timeout"] as const)(
@@ -234,7 +246,7 @@ it.effect("rejects model mutations outside the offered registry", () =>
234246
expect(current).not.toHaveProperty("benchmark");
235247
expect(
236248
(yield* speech.models(true)).models.find((model) => model.active)?.benchmark?.secondsPer30s,
237-
).toBe(1.2);
249+
).toEqual({ cpu: 3, accelerated: 1.2 });
238250
}).pipe(Effect.provide(layer)),
239251
);
240252

@@ -609,11 +621,19 @@ it.effect.each([{ devices: [] }, { devices: [{ id: "gpu", name: "Test GPU" }] }]
609621
const speech = yield* SpeechService.SpeechService;
610622
const discover = vi.mocked(listNativeSpeechGpuDevices).mockReset();
611623
discover.mockRejectedValueOnce(new Error("driver not ready")).mockResolvedValueOnce(devices);
612-
expect(yield* speech.status).toMatchObject({ gpuDevices: [] });
624+
expect(yield* speech.status).toMatchObject({ gpuDevices: [], benchmarkTier: "cpu" });
613625
yield* speech.status;
614626
expect(discover).toHaveBeenCalledOnce();
615627
yield* TestClock.adjust("1 minute");
616-
expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
628+
expect(yield* speech.status).toMatchObject({
629+
gpuDevices: devices,
630+
benchmarkTier: devices.length ? "accelerated" : "cpu",
631+
});
632+
expect(yield* speech.updateAcceleration("cpu")).toMatchObject({ benchmarkTier: "cpu" });
633+
expect(yield* speech.updateAcceleration('gpu:["vulkan","gpu-1"]')).toMatchObject({
634+
benchmarkTier: "accelerated",
635+
});
636+
yield* speech.updateAcceleration("auto");
617637
yield* TestClock.adjust("1 minute");
618638
expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
619639
expect(discover).toHaveBeenCalledTimes(2);
@@ -644,9 +664,11 @@ it.effect(
644664
],
645665
};
646666
const speech = yield* SpeechService.SpeechService;
667+
expect(yield* speech.status).toMatchObject({ benchmarkTier: "accelerated" });
647668
expect(yield* speech.transcribe(pcm(), options)).toBe(
648669
cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
649670
);
671+
expect(yield* speech.status).toMatchObject({ benchmarkTier: "cpu" });
650672
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe(
651673
"hello",
652674
);

‎apps/server/src/speech/SpeechService.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,13 @@ const make = Effect.gen(function* () {
451451
if (unsupportedReason) return { supported: false, reason: unsupportedReason };
452452
const definition = selectedModel(settings);
453453
const ready = yield* store.isReady(definition);
454+
const gpuDevices = yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => []));
455+
const autoUsesCpu =
456+
gpuDevices.length === 0 ||
457+
autoCpuModel === definition.id ||
458+
(loadedModelId === definition.id &&
459+
loadedAcceleration === "auto" &&
460+
model?.backend.toLowerCase() === "cpu");
454461
return {
455462
supported: true,
456463
state: activeTranscriptions > 0 ? "transcribing" : ready ? "ready" : "missing-model",
@@ -463,8 +470,13 @@ const make = Effect.gen(function* () {
463470
languages: definition.languages,
464471
supportsLanguageDetection: definition.supportsLanguageDetection,
465472
acceleration: settings.speechAcceleration,
473+
benchmarkTier:
474+
settings.speechAcceleration === "cpu" ||
475+
(settings.speechAcceleration === "auto" && autoUsesCpu)
476+
? "cpu"
477+
: "accelerated",
466478
modelUnloadTimeout: settings.speechModelUnloadTimeout,
467-
gpuDevices: yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => [])),
479+
gpuDevices,
468480
};
469481
});
470482

0 commit comments

Comments
 (0)