Skip to content

Commit 06b501f

Browse files
Merge the voice model browser into annotation dictation
2 parents 6fc3fff + 8c9f3ad commit 06b501f

19 files changed

Lines changed: 2181 additions & 366 deletions

‎apps/server/scripts/speech-catalog.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,21 @@ it("reproduces the bundled registry byte for byte without upstream requests", as
3333
supportsTranslation: source.capabilities.translate,
3434
});
3535
}
36+
const parakeet = catalog.models[0]!;
37+
const withoutFrenchOrSpeed = generateSpeechCatalog(
38+
{
39+
...catalog.data,
40+
models: [{ ...parakeet, languages: ["en", "fr"] }],
41+
results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance: {} } },
42+
},
43+
"0.2.3",
44+
"0.4.1",
45+
);
46+
expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
47+
accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
48+
secondsPer30s: null,
49+
});
50+
expect(generate().models[0]?.benchmark.secondsPer30s).toBeCloseTo(1.19, 2);
3651
});
3752

3853
it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {

‎apps/server/scripts/speech-catalog.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import {
2+
accuracyGrade,
23
canonicalLanguage,
34
compareEngineVersions,
5+
BENCHMARK_DICTATION_SECONDS,
6+
languageIdentity,
47
modelFile,
58
type CatalogData,
69
} from "@handy-computer/models";
@@ -77,6 +80,21 @@ export function generateSpeechCatalog(
7780
}
7881
ids.add(model.repository);
7982
filenames.add(file.filename);
83+
const languages = [...new Set(model.languages.map(canonicalLanguage))];
84+
const realtimeFactor = data.results[model.id]?.performance.accelerated;
85+
const accuracy = Object.fromEntries(
86+
languages.flatMap((language) => {
87+
const cell = data.results[model.id]?.accuracy[languageIdentity(language)];
88+
return cell
89+
? [
90+
[
91+
language,
92+
{ grade: accuracyGrade(cell.error).label, errorPercent: cell.error },
93+
] as const,
94+
]
95+
: [];
96+
}),
97+
);
8098
return [
8199
{
82100
id: model.repository,
@@ -88,7 +106,13 @@ export function generateSpeechCatalog(
88106
sha256: file.sha256,
89107
license: model.license,
90108
// T3 selects base languages; the engine's prompt dictionary accepts their short aliases.
91-
languages: [...new Set(model.languages.map(canonicalLanguage))],
109+
languages,
110+
benchmark: {
111+
accuracy,
112+
secondsPer30s: realtimeFactor
113+
? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2))
114+
: null,
115+
},
92116
recommended: RECOMMENDED.has(model.id),
93117
supportsStreaming: model.capabilities.streaming,
94118
supportsLanguageDetection: model.capabilities.languageDetection,

‎apps/server/src/speech/SpeechService.test.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ const modelDefinitions = vi.hoisted(() => [
5757
size: 4,
5858
languages: ["en", "fr"],
5959
accuracy: 1,
60+
benchmark: { accuracy: { en: { grade: "A-", errorPercent: 4 } }, secondsPer30s: 1.2 },
6061
speed: 1,
6162
recommended: true,
6263
supportsStreaming: true,
@@ -228,7 +229,12 @@ it.effect("rejects model mutations outside the offered registry", () =>
228229
}
229230
}
230231
expect(download).not.toHaveBeenCalled();
231-
expect((yield* speech.models).models.find((model) => model.active)?.id).toBe("test-model");
232+
const current = (yield* speech.models(false)).models.find((model) => model.active);
233+
expect(current?.id).toBe("test-model");
234+
expect(current).not.toHaveProperty("benchmark");
235+
expect(
236+
(yield* speech.models(true)).models.find((model) => model.active)?.benchmark?.secondsPer30s,
237+
).toBe(1.2);
232238
}).pipe(Effect.provide(layer)),
233239
);
234240

‎apps/server/src/speech/SpeechService.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ export class SpeechService extends Context.Service<
132132
SpeechService,
133133
{
134134
readonly status: Effect.Effect<EnvironmentSpeechStatus, SpeechError>;
135-
readonly models: Effect.Effect<SpeechModels, SpeechError>;
135+
readonly models: (includeBenchmarks: boolean) => Effect.Effect<SpeechModels, SpeechError>;
136136
readonly downloadModel: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>;
137137
readonly selectModel: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>;
138138
readonly cancelDownload: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>;
@@ -469,6 +469,7 @@ const make = Effect.gen(function* () {
469469

470470
const listModels = Effect.fn("SpeechService.listModels")(function* (
471471
settings: SettingsSnapshot,
472+
includeBenchmarks: boolean,
472473
): Effect.fn.Return<SpeechModels> {
473474
const selected = selectedModel(settings);
474475
const models = yield* Effect.forEach(
@@ -485,6 +486,7 @@ const make = Effect.gen(function* () {
485486
languages: definition.languages,
486487
accuracy: definition.accuracy,
487488
speed: definition.speed,
489+
...(includeBenchmarks ? { benchmark: definition.benchmark } : {}),
488490
recommended: definition.recommended,
489491
supportsStreaming: definition.supportsStreaming,
490492
supportsLanguageDetection: definition.supportsLanguageDetection,
@@ -508,7 +510,11 @@ const make = Effect.gen(function* () {
508510

509511
const freshStatus = (operation: string) =>
510512
readSettings(operation).pipe(Effect.flatMap(currentStatus));
511-
const modelsEffect = readSettings("model listing").pipe(Effect.flatMap(listModels));
513+
const readModels = (includeBenchmarks: boolean) =>
514+
readSettings("model listing").pipe(
515+
Effect.flatMap((settings) => listModels(settings, includeBenchmarks)),
516+
);
517+
const modelsEffect = readModels(false);
512518

513519
const requireSupported = Effect.suspend(() =>
514520
unsupportedReason
@@ -857,7 +863,7 @@ const make = Effect.gen(function* () {
857863

858864
return SpeechService.of({
859865
status: freshStatus("status"),
860-
models: modelsEffect,
866+
models: readModels,
861867
prepareModel,
862868
transcribe,
863869
startStream: Effect.fn("SpeechService.startStream")(function* (options) {

0 commit comments

Comments
 (0)