Skip to content

Commit f1e284e

Browse files
donjor-agent[bot]donjor
authored andcommitted
feat(voice-input): send per-project speech terms to the dictation service (#1882)
## Problem The speech service had no knowledge of project names. It misheard terms such as `kubectl`, and only agent cleanup could fix them later. ## Change - **Speech terms:** per project, in Text processing - **Format:** one per line, up to four words - **Sent as:** standard `prompt`, comma list - **Over four words:** left out of `prompt` - **Spelling:** `heard, misheard -> Term` - **Spelling applied:** by T3, on every service - **Live:** previews and final both spelled - **Paths:** batch and live, via `?projectId=` - **Resolution:** in `VoiceInput`; routes stay thin Terms are per project, not per environment, because names differ by repository. One entry over four words makes RSI-Software/promptletariat#159 read the whole `prompt` as prose and drop every hotword, so T3 leaves such entries out. ## Scope and approval Fork-only dictation (`voice-input` domain), stacked on the base URL PR. T3 sends only OpenAI-standard fields; the earlier Promptletariat extensions switch is gone. ## Verification - Server voice-input tests → pass - Web voice-input and rpc tests → pass - `vp run check:changed` → exit 0 - `vp run fork:ci` → exit 0 - Chrome: no extensions switch; save works ![dictation-settings-terms.png](https://github.com/user-attachments/assets/5131e649-35ef-4bac-b8dd-2b68bbc6a5bc) Claude Opus 5.5 via Claude Code in T3 Code. ## Fork trailers Squashes: - 2f35a07 feat(voice-input): send per-project speech terms to the dictation service (#1882) (#1882) - 274db9b fixup! feat(voice-input): send per-project speech terms to the dictation service (#1882) (#1882) Fork-Domain: voice-input Fork-Tier: core Co-authored-by: donjor <[email protected]>
1 parent b6dd169 commit f1e284e

20 files changed

Lines changed: 395 additions & 105 deletions

‎apps/server/src/voice-input/VoiceInput.fork.test.ts‎

Lines changed: 121 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
1-
import type { VoiceInputConfigUpdateFork } from "@t3tools/contracts";
1+
import type { VoiceInputConfigUpdateFork, VoiceTextSettingsFork } from "@t3tools/contracts";
22
import { describe, expect, it } from "@effect/vitest";
33
import * as Effect from "effect/Effect";
44
import * as Deferred from "effect/Deferred";
55
import * as Fiber from "effect/Fiber";
66
import * as TestClock from "effect/testing/TestClock";
77
import * as Layer from "effect/Layer";
88
import * as Option from "effect/Option";
9+
import * as Queue from "effect/Queue";
910
import * as Socket from "effect/socket/Socket";
1011
import * as HttpClient from "effect/http/HttpClient";
1112
import * as HttpClientRequest from "effect/http/HttpClientRequest";
1213
import * as HttpClientResponse from "effect/http/HttpClientResponse";
1314
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
1415
import * as VoiceInput from "./VoiceInput.fork.ts";
16+
import * as VoiceText from "./VoiceText.fork.ts";
1517

1618
function recording(rate = 16000) {
1719
const audio = new Uint8Array(44 + rate * 2);
@@ -57,6 +59,7 @@ function harness(
5759
customClient?: HttpClient.HttpClient,
5860
) {
5961
const stored = new Map<string, Uint8Array>();
62+
const speech = { defaults: { terms: "" }, projects: {} as Record<string, { terms: string }> };
6063
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
6164
const secrets = ServerSecretStore.ServerSecretStore.of({
6265
get: (name) => Effect.sync(() => Option.fromUndefinedOr(stored.get(name))),
@@ -88,13 +91,16 @@ function harness(
8891
Layer.mergeAll(
8992
Layer.succeed(ServerSecretStore.ServerSecretStore, secrets),
9093
Layer.succeed(HttpClient.HttpClient, customClient ?? client),
94+
Layer.mock(VoiceText.VoiceText)({
95+
settings: Effect.sync(() => speech as unknown as VoiceTextSettingsFork),
96+
}),
9197
),
9298
),
9399
);
94100
const run = <A, E, R = never>(
95101
use: (service: VoiceInput.VoiceInput["Service"]) => Effect.Effect<A, E, R>,
96102
) => Effect.flatMap(VoiceInput.VoiceInput, use).pipe(Effect.provide(layer));
97-
return { stored, requests, run };
103+
return { stored, requests, speech, run };
98104
}
99105

100106
describe("external dictation service", () => {
@@ -245,6 +251,41 @@ describe("external dictation service", () => {
245251
expect(body.formData.get("file")).toBeInstanceOf(Blob);
246252
}),
247253
);
254+
it.effect("sends project terms as the prompt and applies their spelling rules", () =>
255+
Effect.gen(function* () {
256+
const h = harness({
257+
text: " Cube control, then CUBECTOL, cube and cubectoland, dollar dollar, οδοσ. ",
258+
});
259+
h.speech.defaults.terms = "shared";
260+
h.speech.projects.p1 = {
261+
terms:
262+
"cube -> Kube\ncube control, cubectol -> kubectl\n\n Hyprland \nVec<T>\ndollar dollar -> $$\nΟΔΟΣ -> Odos\nfive words is too long",
263+
};
264+
const texts = yield* h.run((s) =>
265+
Effect.gen(function* () {
266+
yield* s.configure(openai);
267+
return [yield* s.transcribe(recording(), "p1"), yield* s.transcribe(recording(), "gone")];
268+
}),
269+
);
270+
const [project, shared] = h.requests.map((request) =>
271+
request.body._tag === "FormData" ? request.body.formData : new FormData(),
272+
);
273+
// Only standard fields; one entry over four words would make the list read as prose.
274+
expect([...(project?.keys() ?? [])].toSorted()).toEqual([
275+
"file",
276+
"model",
277+
"prompt",
278+
"response_format",
279+
]);
280+
expect(project?.get("prompt")).toBe("Kube, kubectl, Hyprland, Vec<T>, $$, Odos");
281+
expect(shared?.get("prompt")).toBe("shared");
282+
expect(texts).toEqual([
283+
// The longest alias wins, targets are inserted literally, and case folding is Unicode's.
284+
"kubectl, then kubectl, Kube and cubectoland, $$, Odos.",
285+
"Cube control, then CUBECTOL, cube and cubectoland, dollar dollar, οδοσ.",
286+
]);
287+
}),
288+
);
248289
it.effect("retains keys only for the same destination and supports removal", () =>
249290
Effect.gen(function* () {
250291
const h = harness();
@@ -328,7 +369,7 @@ describe("external dictation service", () => {
328369
config: { ...openai, live: true },
329370
url: "ws://127.0.0.1:8788/v1/realtime?intent=transcription",
330371
headers: { authorization: "Bearer stream-key" },
331-
start: '"type":"session.update"',
372+
start: '"transcription":{"model":"nemo","prompt":"kubectl"}',
332373
},
333374
{
334375
config: meta,
@@ -339,6 +380,7 @@ describe("external dictation service", () => {
339380
] as const)("opens $config.provider's streaming socket with its key", ({ config, ...expected }) =>
340381
Effect.gen(function* () {
341382
const h = harness();
383+
h.speech.defaults.terms = "kubectl";
342384
yield* h.run((s) => s.configure({ ...config, apiKey: "stream-key" }));
343385
const opened = Promise.withResolvers<{ url: string; options: unknown; start: unknown }>();
344386
const upstream = Layer.succeed(Socket.WebSocketConstructor, (url, options) => ({
@@ -360,6 +402,82 @@ describe("external dictation service", () => {
360402
expect(start).toContain(expected.start);
361403
}),
362404
);
405+
const transcription = "conversation.item.input_audio_transcription";
406+
it.effect.each([
407+
{
408+
config: { ...openai, live: true },
409+
start: '"prompt":"kubectl"',
410+
ready: '{"type":"session.updated"}',
411+
previews: [
412+
`{"type":"${transcription}.delta","delta":"cube"}`,
413+
`{"type":"${transcription}.delta","delta":" control"}`,
414+
],
415+
end: '{"type":"input_audio_buffer.commit"}',
416+
final: `{"type":"${transcription}.completed","transcript":"Cube control."}`,
417+
},
418+
{
419+
config: meta,
420+
start: '"accessToken"',
421+
ready: '{"sessionId":"session"}',
422+
previews: [
423+
'{"type":"transcript","transcript":"cube","final":false}',
424+
'{"type":"transcript","transcript":"cube control","final":false}',
425+
],
426+
end: '{"type":"endStream"}',
427+
final: '{"type":"transcript","transcript":"Cube control.","final":true}',
428+
},
429+
] as const)(
430+
"spells $config.provider live previews and the final transcript with the project's terms",
431+
({ config, start, ready, previews, end, final }) =>
432+
Effect.gen(function* () {
433+
const h = harness();
434+
h.speech.projects.p1 = { terms: "cube control -> kubectl" };
435+
yield* h.run((s) => s.configure({ ...config, apiKey: "stream-key" }));
436+
const fromClient = yield* Queue.make<string>();
437+
const toClient = yield* Queue.make<string>();
438+
const toUpstream = yield* Queue.make<string>();
439+
let deliver: (event: { data: string }) => void = () => {};
440+
const client = Socket.make({
441+
reader: Effect.succeed({
442+
pull: Queue.take(fromClient).pipe(Effect.map((frame) => [frame] as const)),
443+
upgrade: () => Effect.void,
444+
}),
445+
writer: Effect.succeed({
446+
write: (frame) => Queue.offer(toClient, String(frame)).pipe(Effect.asVoid),
447+
writeAll: () => Effect.void,
448+
}),
449+
});
450+
const upstream = Layer.succeed(Socket.WebSocketConstructor, () => ({
451+
readyState: 1,
452+
addEventListener: (type: string, listener: typeof deliver) => {
453+
if (type === "message") deliver = listener;
454+
},
455+
removeEventListener: () => {},
456+
close: () => {},
457+
send: (frame: string) => Queue.offerUnsafe(toUpstream, frame),
458+
}));
459+
const fiber = yield* h
460+
.run((s) => s.stream(client, "p1"))
461+
.pipe(Effect.scoped, Effect.provide(upstream), Effect.forkChild);
462+
expect(yield* Queue.take(toUpstream)).toContain(start);
463+
deliver({ data: ready });
464+
expect(yield* Queue.take(toClient)).toBe('{"type":"ready"}');
465+
for (const data of previews) deliver({ data });
466+
expect(yield* Queue.take(toClient)).toBe(
467+
'{"type":"transcript","text":"cube","final":false}',
468+
);
469+
expect(yield* Queue.take(toClient)).toBe(
470+
'{"type":"transcript","text":"kubectl","final":false}',
471+
);
472+
yield* Queue.offer(fromClient, '{"type":"endStream"}');
473+
expect(yield* Queue.take(toUpstream)).toBe(end);
474+
deliver({ data: final });
475+
expect(yield* Queue.take(toClient)).toBe(
476+
'{"type":"transcript","text":"kubectl.","final":true}',
477+
);
478+
yield* Fiber.join(fiber);
479+
}),
480+
);
363481
it.effect("refuses a live socket while live preview is off", () =>
364482
Effect.gen(function* () {
365483
const h = harness();

‎apps/server/src/voice-input/VoiceInput.fork.ts‎

Lines changed: 96 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@ import * as Socket from "effect/socket/Socket";
1717
import type * as Scope from "effect/Scope";
1818

1919
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
20+
import * as VoiceText from "./VoiceText.fork.ts";
2021
import { isVoiceRecordingFork } from "./recording.fork.ts";
2122
import { metaVoiceRequestFork, metaVoiceStreamFork, metaVoiceTranscriptFork } from "./meta.fork.ts";
2223
import { openAiVoiceStreamFork } from "./openaiRealtime.fork.ts";
23-
import { relayVoiceStreamFork } from "./realtime.fork.ts";
24+
import { relayVoiceStreamFork, type VoiceStreamAdapterFork } from "./realtime.fork.ts";
2425

2526
export class VoiceInputError extends Schema.TaggedError<VoiceInputError>()("VoiceInputError", {
2627
reason: Schema.Literals([
@@ -122,24 +123,85 @@ function voiceInputUrlsFork(config: Pick<VoiceInputConfigFork, "provider" | "bas
122123
return { upload: base + upload, realtime: base.replace(/^http/, "ws") + realtime };
123124
}
124125

126+
/**
127+
* Project terms, as lines of `term` or `heard, misheard -> Term`. Terms go out as the standard
128+
* `prompt`, a comma list some services decode as hotwords; an entry over four words reads as prose
129+
* and disables them all, so it stays out. Spelling rules run here, so every service gets them.
130+
*/
131+
function voiceTermsFork(terms: string) {
132+
const canonical = new Set<string>();
133+
const spellings = new Map<string, string>();
134+
for (const line of terms.split("\n")) {
135+
const [heard, term = heard] = line.split("->").map((part) => part.trim());
136+
if (!term) continue;
137+
if (term.split(/\s+/u).length <= 4) canonical.add(term);
138+
if (heard === term) continue;
139+
for (const from of (heard ?? "").split(",")) {
140+
const words = from.trim().split(/\s+/u).filter(Boolean);
141+
if (words.length > 0) spellings.set(words.map(escapeRegExp).join("\\s+"), term);
142+
}
143+
}
144+
let prompt = "";
145+
for (const term of canonical) {
146+
const next = prompt ? `${prompt}, ${term}` : term;
147+
if (next.length > 4096) break;
148+
prompt = next;
149+
}
150+
// One pass, longest alias first, so a replacement is never rescanned by another rule. Each alias
151+
// is its own group, so the target follows the regex's case folding rather than a second one.
152+
const rules = [...spellings].toSorted(([a], [b]) => b.length - a.length);
153+
const pattern =
154+
rules.length > 0
155+
? new RegExp(
156+
`(?<![\\p{L}\\p{N}])(?:${rules.map(([from]) => `(${from})`).join("|")})(?![\\p{L}\\p{N}])`,
157+
"giu",
158+
)
159+
: undefined;
160+
return {
161+
prompt,
162+
spell: (text: string) =>
163+
pattern
164+
? text.replace(pattern, (match, ...groups: Array<unknown>) => {
165+
const rule = groups.slice(0, rules.length).findIndex((group) => group !== undefined);
166+
return rules[rule]?.[1] ?? match;
167+
})
168+
: text,
169+
};
170+
}
171+
const escapeRegExp = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
172+
125173
export class VoiceInput extends Context.Service<
126174
VoiceInput,
127175
{
128176
readonly settings: Effect.Effect<VoiceInputSettingsFork, VoiceInputError>;
129177
readonly configure: (
130178
input: VoiceInputConfigUpdateFork,
131179
) => Effect.Effect<VoiceInputSettingsFork, VoiceInputError>;
132-
readonly transcribe: (audio: Uint8Array) => Effect.Effect<string, VoiceInputError>;
180+
/** `projectId` selects the dictating project's speech terms; the defaults' otherwise. */
181+
readonly transcribe: (
182+
audio: Uint8Array,
183+
projectId?: string,
184+
) => Effect.Effect<string, VoiceInputError>;
133185
readonly stream: (
134186
socket: Socket.Socket,
187+
projectId?: string,
135188
) => Effect.Effect<void, VoiceInputError, Scope.Scope | Socket.WebSocketConstructor>;
136189
}
137190
>()("t3/voice-input/VoiceInput.fork/VoiceInput") {}
138191

139192
const make = Effect.gen(function* () {
140193
const secrets = yield* ServerSecretStore.ServerSecretStore;
141194
const client = yield* HttpClient.HttpClient;
195+
const text = yield* VoiceText.VoiceText;
142196
const writes = yield* Semaphore.make(1);
197+
const projectTerms = (projectId: string | undefined) =>
198+
text.settings.pipe(
199+
Effect.map(({ projects, defaults }) =>
200+
voiceTermsFork(((projectId && projects[projectId]) || defaults).terms),
201+
),
202+
// Terms only steer recognition; dictation still works without them.
203+
Effect.orElseSucceed(() => voiceTermsFork("")),
204+
);
143205

144206
const read = secrets.get(SECRET_NAME).pipe(
145207
Effect.mapError((cause) => new VoiceInputError({ reason: "storage", cause })),
@@ -190,11 +252,15 @@ const make = Effect.gen(function* () {
190252
return publicSettings(saved);
191253
});
192254

193-
const transcribe = Effect.fn("VoiceInput.transcribe")(function* (audio: Uint8Array) {
255+
const transcribe = Effect.fn("VoiceInput.transcribe")(function* (
256+
audio: Uint8Array,
257+
projectId: string | undefined,
258+
) {
194259
const config = yield* read;
195260
if (!config.enabled) return yield* new VoiceInputError({ reason: "disabled" });
196261
if (!isVoiceRecordingFork(audio)) return yield* new VoiceInputError({ reason: "audio" });
197262
const { upload } = voiceInputUrlsFork(config);
263+
const { prompt, spell } = yield* projectTerms(projectId);
198264
let request: HttpClientRequest.HttpClientRequest;
199265
if (config.provider === "meta") {
200266
request = metaVoiceRequestFork(upload, config.model, audio);
@@ -203,6 +269,7 @@ const make = Effect.gen(function* () {
203269
form.set("file", new Blob([new Uint8Array(audio)], { type: "audio/wav" }), "dictation.wav");
204270
form.set("model", config.model);
205271
form.set("response_format", "json");
272+
if (prompt) form.set("prompt", prompt);
206273
request = HttpClientRequest.post(upload).pipe(HttpClientRequest.bodyFormData(form));
207274
}
208275
if (config.apiKey)
@@ -220,18 +287,35 @@ const make = Effect.gen(function* () {
220287
Effect.map((body) => body.text),
221288
)
222289
).pipe(Effect.mapError((cause) => new VoiceInputError({ reason: "response", cause })));
223-
return text.trim();
290+
return spell(text.trim());
224291
});
225292
return VoiceInput.of({
226293
settings: read.pipe(Effect.map(publicSettings)),
227-
configure: (input) => writes.withPermits(1)(configure(input)),
294+
// `configure` and `transcribe` own their spans.
295+
configure: Effect.fnUntraced(function* (input) {
296+
return yield* writes.withPermits(1)(configure(input));
297+
}),
228298
stream: Effect.fn("VoiceInput.stream")(
229-
function* (socket) {
299+
function* (socket, projectId) {
230300
const config = yield* read;
231301
if (!config.enabled) return yield* new VoiceInputError({ reason: "disabled" });
232302
if (!config.live) return yield* new VoiceInputError({ reason: "live" });
233-
const adapter =
234-
config.provider === "meta" ? metaVoiceStreamFork(config) : openAiVoiceStreamFork(config);
303+
const { prompt, spell } = yield* projectTerms(projectId);
304+
const wire =
305+
config.provider === "meta"
306+
? metaVoiceStreamFork(config)
307+
: openAiVoiceStreamFork(config, prompt);
308+
const adapter: VoiceStreamAdapterFork = {
309+
...wire,
310+
receive: (frame) =>
311+
wire
312+
.receive(frame)
313+
.pipe(
314+
Effect.map((update) =>
315+
update?.type === "transcript" ? { ...update, text: spell(update.text) } : update,
316+
),
317+
),
318+
};
235319
const upstream = yield* Socket.makeWebSocket(voiceInputUrlsFork(config).realtime, {
236320
openTimeout: "10 seconds",
237321
...(adapter.headers ? { headers: adapter.headers } : {}),
@@ -242,15 +326,16 @@ const make = Effect.gen(function* () {
242326
isVoiceInputError(cause) ? cause : new VoiceInputError({ reason: "upstream", cause }),
243327
),
244328
),
245-
transcribe: (audio) =>
246-
transcribe(audio).pipe(
329+
transcribe: Effect.fnUntraced(function* (audio, projectId) {
330+
return yield* transcribe(audio, projectId).pipe(
247331
Effect.timeout("60 seconds"),
248332
Effect.mapError((cause) =>
249333
cause._tag === "TimeoutError"
250334
? new VoiceInputError({ reason: "upstream", cause })
251335
: cause,
252336
),
253-
),
337+
);
338+
}),
254339
});
255340
});
256341

‎apps/server/src/voice-input/VoiceText.fork.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const config: VoiceTextConfigFork = {
3838
formattingMinWords: 50,
3939
recentMessageCount: 0,
4040
projectContext: "React, Zustand, Hyprland",
41+
terms: "",
4142
...VOICE_TEXT_STARTER_PROMPTS_FORK,
4243
};
4344

0 commit comments

Comments
 (0)