Skip to content

Commit 24529cb

Browse files
donjor-agent[bot]donjor
authored andcommitted
feat(server): transcribe recordings with Meta Muse (#1673)
Fork-Domain: voice-input Fork-Tier: core Co-authored-by: donjor <[email protected]>
1 parent 92d6058 commit 24529cb

4 files changed

Lines changed: 138 additions & 14 deletions

File tree

‎apps/server/src/voice-input/VoiceInput.fork.test.ts‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import type { VoiceInputConfigUpdateFork } from "@t3tools/contracts";
22
import { describe, expect, it } from "@effect/vitest";
33
import * as Effect from "effect/Effect";
4+
import * as Deferred from "effect/Deferred";
5+
import * as Fiber from "effect/Fiber";
6+
import * as TestClock from "effect/testing/TestClock";
47
import * as Layer from "effect/Layer";
58
import * as Option from "effect/Option";
69
import * as HttpClient from "effect/http/HttpClient";
@@ -35,7 +38,11 @@ const local: VoiceInputConfigUpdateFork = {
3538
model: "",
3639
};
3740

38-
function harness(body: unknown = { text: " spoken words " }, status = 200) {
41+
function harness(
42+
body: unknown = { text: " spoken words " },
43+
status = 200,
44+
customClient?: HttpClient.HttpClient,
45+
) {
3946
const stored = new Map<string, Uint8Array>();
4047
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
4148
const secrets = ServerSecretStore.ServerSecretStore.of({
@@ -67,7 +74,7 @@ function harness(body: unknown = { text: " spoken words " }, status = 200) {
6774
Layer.provide(
6875
Layer.mergeAll(
6976
Layer.succeed(ServerSecretStore.ServerSecretStore, secrets),
70-
Layer.succeed(HttpClient.HttpClient, client),
77+
Layer.succeed(HttpClient.HttpClient, customClient ?? client),
7178
),
7279
),
7380
);
@@ -102,6 +109,84 @@ describe("external dictation service", () => {
102109
expect(h.requests[0]?.headers.authorization).toBe("Bearer legacy-secret");
103110
}),
104111
);
112+
it.effect("bounds provider latency and interrupts pending requests", () =>
113+
Effect.gen(function* () {
114+
const started = yield* Deferred.make<void>();
115+
let released = false;
116+
const client = HttpClient.make(() =>
117+
Deferred.succeed(started, undefined).pipe(
118+
Effect.andThen(Effect.never),
119+
Effect.ensuring(
120+
Effect.sync(() => {
121+
released = true;
122+
}),
123+
),
124+
),
125+
);
126+
const h = harness(undefined, 200, client);
127+
yield* h.run((s) => s.configure(local));
128+
const pending = yield* h
129+
.run((s) => s.transcribe(recording()))
130+
.pipe(Effect.flip, Effect.forkScoped);
131+
yield* Deferred.await(started);
132+
yield* TestClock.adjust("61 seconds");
133+
expect(yield* Fiber.join(pending)).toMatchObject({ reason: "upstream" });
134+
expect(released).toBe(true);
135+
}),
136+
);
137+
it.effect("uses Meta's JSON request and WAV audio parts and reads transcript", () =>
138+
Effect.gen(function* () {
139+
const h = harness({ transcript: " meta words " });
140+
const audio = recording();
141+
const text = yield* h.run((s) =>
142+
Effect.gen(function* () {
143+
yield* s.configure({
144+
enabled: true,
145+
provider: "meta",
146+
endpoint: "https://api.meta.ai/v1/asr/transcribe",
147+
model: "muse-voice-transcribe-1.0",
148+
apiKey: "meta-key",
149+
});
150+
return yield* s.transcribe(audio);
151+
}),
152+
);
153+
expect(text).toBe("meta words");
154+
expect(h.requests[0]?.headers.authorization).toBe("Bearer meta-key");
155+
const body = h.requests[0]?.body;
156+
if (body?._tag !== "FormData") throw new Error("Expected Meta multipart upload");
157+
const request = body.formData.get("request");
158+
const file = body.formData.get("audio");
159+
if (!(request instanceof Blob) || !(file instanceof Blob))
160+
throw new Error("Expected JSON and WAV parts");
161+
expect(request.type).toBe("application/json");
162+
expect(JSON.parse(yield* Effect.promise(() => request.text()))).toEqual({
163+
model: "muse-voice-transcribe-1.0",
164+
audioEncoding: "WAV",
165+
mode: "PUSH_TO_TALK",
166+
});
167+
expect(file.type).toBe("audio/wav");
168+
expect(new Uint8Array(yield* Effect.promise(() => file.arrayBuffer()))).toEqual(audio);
169+
expect(body.formData.has("file")).toBe(false);
170+
}),
171+
);
172+
it.effect.each([{ text: "wrong response shape" }, { transcript: 42 }])(
173+
"rejects invalid Meta responses",
174+
(body) =>
175+
Effect.gen(function* () {
176+
const h = harness(body);
177+
yield* h.run((s) =>
178+
s.configure({
179+
enabled: true,
180+
provider: "meta",
181+
endpoint: "https://api.meta.ai/v1/asr/transcribe",
182+
model: "muse-voice-transcribe-1.0",
183+
}),
184+
);
185+
expect(yield* h.run((s) => s.transcribe(recording())).pipe(Effect.flip)).toMatchObject({
186+
reason: "response",
187+
});
188+
}),
189+
);
105190
it.effect("starts disabled and redacts keys", () =>
106191
Effect.gen(function* () {
107192
const h = harness();

‎apps/server/src/voice-input/VoiceInput.fork.ts‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import * as HttpClientResponse from "effect/http/HttpClientResponse";
1717
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
1818
import { voiceInputProvidersFork } from "./providers.fork.ts";
1919
import { isVoiceRecordingFork } from "./recording.fork.ts";
20+
import { metaVoiceRequestFork, metaVoiceTranscriptFork } from "./meta.fork.ts";
2021

2122
export class VoiceInputError extends Schema.TaggedError<VoiceInputError>()("VoiceInputError", {
2223
reason: Schema.Literals([
@@ -159,34 +160,48 @@ const make = Effect.gen(function* () {
159160
return yield* new VoiceInputError({ reason: "unsupported" });
160161
}
161162
if (!isVoiceRecordingFork(audio)) return yield* new VoiceInputError({ reason: "audio" });
162-
let request = HttpClientRequest.post(config.endpoint);
163+
let request =
164+
config.provider === "meta"
165+
? metaVoiceRequestFork(config.endpoint, config.model, audio)
166+
: HttpClientRequest.post(config.endpoint);
163167
if (config.apiKey)
164168
request = HttpClientRequest.setHeader(request, "authorization", `Bearer ${config.apiKey}`);
165169
if (config.provider === "local") {
166170
request = HttpClientRequest.bodyUint8Array(request, audio, "audio/wav");
167-
} else {
171+
} else if (config.provider === "openai-compatible") {
168172
const form = new FormData();
169173
form.set("file", new Blob([new Uint8Array(audio)], { type: "audio/wav" }), "dictation.wav");
170174
form.set("model", config.model);
171175
form.set("response_format", "json");
172176
request = HttpClientRequest.bodyFormData(request, form);
173177
}
174-
const response = yield* client.execute(request).pipe(
175-
Effect.timeout("60 seconds"),
176-
Effect.mapError((cause) => new VoiceInputError({ reason: "upstream", cause })),
177-
);
178+
const response = yield* client
179+
.execute(request)
180+
.pipe(Effect.mapError((cause) => new VoiceInputError({ reason: "upstream", cause })));
178181
if (response.status < 200 || response.status >= 300) {
179182
return yield* new VoiceInputError({ reason: "upstream", status: response.status });
180183
}
181-
const body = yield* HttpClientResponse.schemaBodyJson(VoiceInputTranscriptFork)(response).pipe(
182-
Effect.mapError((cause) => new VoiceInputError({ reason: "response", cause })),
183-
);
184-
return body.text.trim();
184+
const text = yield* (
185+
config.provider === "meta"
186+
? metaVoiceTranscriptFork(response)
187+
: HttpClientResponse.schemaBodyJson(VoiceInputTranscriptFork)(response).pipe(
188+
Effect.map((body) => body.text),
189+
)
190+
).pipe(Effect.mapError((cause) => new VoiceInputError({ reason: "response", cause })));
191+
return text.trim();
185192
});
186193
return VoiceInput.of({
187194
settings: read.pipe(Effect.map(publicSettings)),
188195
configure: (input) => writes.withPermits(1)(configure(input)),
189-
transcribe,
196+
transcribe: (audio) =>
197+
transcribe(audio).pipe(
198+
Effect.timeout("60 seconds"),
199+
Effect.mapError((cause) =>
200+
cause._tag === "TimeoutError"
201+
? new VoiceInputError({ reason: "upstream", cause })
202+
: cause,
203+
),
204+
),
190205
});
191206
});
192207

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import * as Effect from "effect/Effect";
2+
import * as Schema from "effect/Schema";
3+
import * as HttpClientRequest from "effect/http/HttpClientRequest";
4+
import * as HttpClientResponse from "effect/http/HttpClientResponse";
5+
6+
const MetaTranscript = Schema.Struct({ transcript: Schema.String });
7+
8+
/** Meta's push-to-talk API uses named JSON and WAV parts, unlike OpenAI uploads. */
9+
export function metaVoiceRequestFork(endpoint: string, model: string, audio: Uint8Array) {
10+
const form = new FormData();
11+
form.set(
12+
"request",
13+
new Blob([JSON.stringify({ model, audioEncoding: "WAV", mode: "PUSH_TO_TALK" })], {
14+
type: "application/json",
15+
}),
16+
);
17+
form.set("audio", new Blob([new Uint8Array(audio)], { type: "audio/wav" }), "dictation.wav");
18+
return HttpClientRequest.post(endpoint).pipe(HttpClientRequest.bodyFormData(form));
19+
}
20+
21+
export const metaVoiceTranscriptFork = (response: HttpClientResponse.HttpClientResponse) =>
22+
HttpClientResponse.schemaBodyJson(MetaTranscript)(response).pipe(
23+
Effect.map((body) => body.transcript),
24+
);

‎apps/server/src/voice-input/providers.fork.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ export const voiceInputProvidersFork = [
55
{
66
id: "meta",
77
label: "Meta",
8-
available: false,
8+
available: true,
99
endpoint: "https://api.meta.ai/v1/asr/transcribe",
1010
model: "muse-voice-transcribe-1.0",
1111
},

0 commit comments

Comments
 (0)