Skip to content

Commit d5fc29f

Browse files
fix(web): relay-connected microphone destinations authorize speech requests (#31778)
1 parent b90e554 commit d5fc29f

6 files changed

Lines changed: 172 additions & 33 deletions

File tree

‎apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx‎

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection";
1+
import { PrimaryConnectionTarget, RelayConnectionTarget } from "@t3tools/client-runtime/connection";
2+
import type * as Effect from "effect/Effect";
23
import {
34
EnvironmentId,
45
type EnvironmentSpeechModel,
@@ -19,6 +20,53 @@ import { SpeechLanguageSelect } from "./SpeechLanguageSelect";
1920
import { VoiceSettingsPanel } from "./VoiceSettingsPanel";
2021

2122
vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true });
23+
// The real connection runtime authorizes relay requests with a signed-in cloud session.
24+
vi.mock("../../connection/runtime", async () => {
25+
const [{ ManagedRelay }, rpc, { Atom }, Context, Effect, Layer] = await Promise.all([
26+
import("@t3tools/client-runtime/relay"),
27+
import("@t3tools/client-runtime/rpc"),
28+
import("effect/reactivity"),
29+
import("effect/Context"),
30+
import("effect/Effect"),
31+
import("effect/Layer"),
32+
]);
33+
type AuthorizedHttp =
34+
import("@t3tools/client-runtime/authorization").AuthorizedRemoteHttpEnvironment;
35+
// Services resolve by key, and client-runtime does not export this one.
36+
class RemoteEnvironmentAuthorization extends Context.Service<
37+
RemoteEnvironmentAuthorization,
38+
{
39+
readonly authorizeDpopHttp: (input: {
40+
readonly expectedEnvironmentId: AuthorizedHttp["environmentId"];
41+
}) => Effect.Effect<AuthorizedHttp>;
42+
}
43+
// @effect-diagnostics-next-line deterministicKeys:off
44+
>()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {}
45+
return {
46+
connectionAtomRuntime: Atom.runtime(
47+
Layer.mergeAll(
48+
rpc.layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)),
49+
Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, {
50+
thumbprint: Effect.succeed("thumbprint"),
51+
createProof: () => Effect.succeed("proof"),
52+
}),
53+
Layer.succeed(RemoteEnvironmentAuthorization, {
54+
authorizeDpopHttp: ({ expectedEnvironmentId }) =>
55+
Effect.succeed({
56+
environmentId: expectedEnvironmentId,
57+
label: "MacBook Pro",
58+
httpBaseUrl: window.location.origin,
59+
httpAuthorization: {
60+
_tag: "Dpop",
61+
accessToken: "relay-access-token",
62+
expiresAtEpochMs: 4_102_444_800_000,
63+
},
64+
}),
65+
}),
66+
),
67+
),
68+
};
69+
});
2270

2371
it("preserves the model test transcript when its model is filtered out", async () => {
2472
const target = new PrimaryConnectionTarget({
@@ -157,6 +205,81 @@ it("preserves the model test transcript when its model is filtered out", async (
157205
);
158206
});
159207

208+
it("loads the models of a relay-connected destination", async () => {
209+
const target = new RelayConnectionTarget({
210+
environmentId: EnvironmentId.make("voice-relay"),
211+
label: "MacBook Pro",
212+
});
213+
const prepared = {
214+
environmentId: target.environmentId,
215+
label: target.label,
216+
httpBaseUrl: window.location.origin,
217+
socketUrl: "ws://localhost",
218+
httpAuthorization: {
219+
_tag: "Dpop" as const,
220+
accessToken: "stale-access-token",
221+
expiresAtEpochMs: 4_102_444_800_000,
222+
},
223+
target,
224+
};
225+
const model: EnvironmentSpeechModel = {
226+
id: "tiny",
227+
name: "Tiny",
228+
description: "",
229+
size: 100,
230+
languages: ["en"],
231+
accuracy: 1,
232+
speed: 1,
233+
recommended: false,
234+
supportsStreaming: false,
235+
supportsTranslation: false,
236+
supportsLanguageDetection: false,
237+
active: false,
238+
state: "downloadable",
239+
};
240+
const status = {
241+
supported: true,
242+
state: "missing-model",
243+
modelId: model.id,
244+
model: model.name,
245+
size: model.size,
246+
supportsStreaming: false,
247+
supportsTranslation: false,
248+
acceleration: "cpu",
249+
modelUnloadTimeout: "never",
250+
gpuDevices: [],
251+
} satisfies EnvironmentSpeechStatus;
252+
const authorizations: Array<string | null> = [];
253+
const fetch = globalThis.fetch;
254+
using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
255+
const url = new URL(input instanceof Request ? input.url : input, window.location.href);
256+
if (url.pathname === "/api/voice/status") {
257+
authorizations.push(new Headers(init?.headers).get("authorization"));
258+
return Promise.resolve(Response.json(status));
259+
}
260+
if (url.pathname === "/api/voice/models")
261+
return Promise.resolve(Response.json({ models: [model] }));
262+
return fetch(input, init);
263+
});
264+
using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
265+
// Restored even when an assertion fails, unlike `withImplementation`.
266+
using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({
267+
environmentId: target.environmentId,
268+
label: target.label,
269+
grantId: "test",
270+
prepared,
271+
}));
272+
await using screen = await render(
273+
<SettingsScopeProvider search={{}} onChange={() => {}}>
274+
<VoiceSettingsPanel />
275+
</SettingsScopeProvider>,
276+
);
277+
await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0));
278+
expect(notify).not.toHaveBeenCalled();
279+
expect(authorizations).toEqual(["DPoP relay-access-token"]);
280+
await expect.element(screen.getByText("Tiny", { exact: true })).toBeVisible();
281+
});
282+
160283
it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => {
161284
const target = new PrimaryConnectionTarget({
162285
environmentId: EnvironmentId.make("voice-progress"),

‎apps/web/src/components/settings/VoiceSettingsPanel.tsx‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ import {
4545
useCleanupGrant,
4646
} from "../../speech/voiceConsent";
4747
import { updateClientSettings, useClientSettings } from "../../hooks/useSettings";
48-
import { runtime } from "../../lib/runtime";
48+
import { runEnvironmentSpeech } from "../../speech/runEnvironmentSpeech";
4949
import {
5050
getCustomModelOptionsByInstance,
5151
resolveAppModelSelectionState,
@@ -542,8 +542,8 @@ function useVoiceModels(prepared: PreparedConnection | null) {
542542
const revision = ++refreshRevision.current;
543543
try {
544544
const [nextStatus, nextModels] = await Promise.all([
545-
runtime.runPromise(getEnvironmentSpeechStatus(connection)),
546-
runtime.runPromise(getEnvironmentSpeechModels(connection, mode === "foreground")),
545+
runEnvironmentSpeech(getEnvironmentSpeechStatus(connection)),
546+
runEnvironmentSpeech(getEnvironmentSpeechModels(connection, mode === "foreground")),
547547
]);
548548
if (current !== epoch.current || revision !== refreshRevision.current) return;
549549
setStatus(nextStatus);
@@ -649,7 +649,7 @@ function AdvancedSpeechSettings({
649649
onValueChange={(value) => {
650650
if (value)
651651
void run("Could not change transcription hardware", () =>
652-
runtime.runPromise(
652+
runEnvironmentSpeech(
653653
updateEnvironmentSpeechAcceleration(prepared, value as SpeechAcceleration),
654654
),
655655
);
@@ -688,7 +688,7 @@ function AdvancedSpeechSettings({
688688
onValueChange={(value) => {
689689
if (value)
690690
void run("Could not change when the model unloads", () =>
691-
runtime.runPromise(
691+
runEnvironmentSpeech(
692692
updateEnvironmentSpeechModelUnloadTimeout(
693693
prepared,
694694
value as SpeechModelUnloadTimeout,
@@ -896,22 +896,22 @@ function SpeechModelBrowser(props: {
896896
void run(
897897
"Could not download the model",
898898
() =>
899-
runtime.runPromise(downloadEnvironmentSpeechModel(prepared, model.id)),
899+
runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, model.id)),
900900
model.id,
901901
),
902902
onSelect: () =>
903903
void run("Could not select the model", () =>
904-
runtime.runPromise(selectEnvironmentSpeechModel(prepared, model.id)),
904+
runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, model.id)),
905905
),
906906
onCancel: () =>
907907
void run("Could not cancel the download", () =>
908-
runtime.runPromise(
908+
runEnvironmentSpeech(
909909
cancelEnvironmentSpeechModelDownload(prepared, model.id),
910910
),
911911
),
912912
onDelete: () =>
913913
void run("Could not delete the model", () =>
914-
runtime.runPromise(removeEnvironmentSpeechModel(prepared, model.id)),
914+
runEnvironmentSpeech(removeEnvironmentSpeechModel(prepared, model.id)),
915915
),
916916
};
917917
return model.active ? (

‎apps/web/src/speech/browserVoiceInput.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import {
1111
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
1212
import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
1313

14-
import { runtime } from "../lib/runtime";
14+
import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
1515
import { createBrowserPcmCapture } from "./browserPcmCapture";
1616

1717
export function createBrowserVoiceInputPlatform(input: {
@@ -26,7 +26,7 @@ export function createBrowserVoiceInputPlatform(input: {
2626
}) {
2727
return createBrowserPcmCapture(input, async (options) => {
2828
const recordingOptions = input.getTranscriptionOptions();
29-
const status = await runtime.runPromise(getEnvironmentSpeechStatus(input.prepared), options);
29+
const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
3030
throwIfVoiceTranscriptionAborted(options.signal);
3131
if (!status.supported)
3232
throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
@@ -52,7 +52,7 @@ export function createBrowserVoiceInputPlatform(input: {
5252
try {
5353
const response = await fetch(uri, { signal: transcriptionSignal });
5454
const pcm = new Uint8Array(await response.arrayBuffer());
55-
const result = await runtime.runPromise(
55+
const result = await runEnvironmentSpeech(
5656
transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent),
5757
{
5858
signal: transcriptionSignal,
@@ -72,7 +72,7 @@ export function createBrowserVoiceInputPlatform(input: {
7272
if (!status.supportsStreaming && !status.supportsBufferedStreaming) {
7373
return { locale, transcribe: transcribeRecording };
7474
}
75-
const url = await runtime.runPromise(getEnvironmentSpeechStreamUrl(input.prepared), options);
75+
const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);
7676
return {
7777
locale,
7878
transcribe: transcribeRecording,
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import * as Effect from "effect/Effect";
2+
import type { HttpClient } from "effect/http";
3+
import { AtomRegistry } from "effect/reactivity";
4+
5+
import { connectionAtomRuntime } from "../connection/runtime";
6+
import { appAtomRegistry } from "../rpc/atomRegistry";
7+
8+
// Relay (DPoP) environments authorize through services only the connection runtime provides.
9+
export function runEnvironmentSpeech<A, E>(
10+
effect: Effect.Effect<A, E, HttpClient.HttpClient>,
11+
options?: Effect.RunOptions,
12+
): Promise<A> {
13+
return Effect.runPromise(
14+
AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
15+
Effect.flatMap((context) => Effect.provide(effect, context)),
16+
),
17+
options,
18+
);
19+
}

‎apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ vi.mock("../hooks/useSettings", () => ({
8888
),
8989
}),
9090
}));
91-
vi.mock("../lib/runtime", () => ({ runtime: { runPromise: Effect.runPromise } }));
91+
vi.mock("./runEnvironmentSpeech", () => ({ runEnvironmentSpeech: Effect.runPromise }));
9292
vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } }));
9393
vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
9494
...(await importOriginal<typeof import("@t3tools/client-runtime/voice-input")>()),

‎apps/web/src/speech/useEnvironmentSpeechInput.ts‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ import {
3333
useClientSettingsHydrated,
3434
useEnvironmentSettings,
3535
} from "../hooks/useSettings";
36-
import { runtime } from "../lib/runtime";
3736
import { useEnvironment } from "../state/environments";
3837
import { useEnvironmentScope, usePreparedConnection } from "../state/session";
3938
import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
39+
import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
4040
import { bindAudioGrant, readCleanupGrant, useAudioGrant } from "./voiceConsent";
4141

4242
const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
@@ -54,7 +54,7 @@ const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechSt
5454
function probeSpeechStatus(prepared: PreparedConnection, refresh = false) {
5555
const cached = refresh ? undefined : statusProbes.get(prepared);
5656
if (cached) return cached;
57-
const probe = runtime.runPromise(getEnvironmentSpeechStatus(prepared));
57+
const probe = runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared));
5858
statusProbes.set(prepared, probe);
5959
probe.catch(() => {
6060
if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared);
@@ -296,18 +296,16 @@ export function useEnvironmentSpeechInput(input: HookInput) {
296296
return Promise.reject(
297297
new Error("The draft's environment is unavailable for transcript cleanup."),
298298
);
299-
return runtime
300-
.runPromise(
301-
postProcessEnvironmentTranscript(
302-
target,
303-
transcript,
304-
cleanupContext(options.draft),
305-
preferences.cleanup,
306-
() => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
307-
),
308-
options,
309-
)
310-
.then((result) => result.text);
299+
return runEnvironmentSpeech(
300+
postProcessEnvironmentTranscript(
301+
target,
302+
transcript,
303+
cleanupContext(options.draft),
304+
preferences.cleanup,
305+
() => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
306+
),
307+
options,
308+
).then((result) => result.text);
311309
};
312310
controller = new VoiceInputController({
313311
retryFailedRecording: true,
@@ -374,8 +372,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
374372
const timer = window.setInterval(() => {
375373
if (refreshing) return;
376374
refreshing = true;
377-
void runtime
378-
.runPromise(getEnvironmentSpeechModels(prepared))
375+
void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared))
379376
.then((result) => {
380377
if (disposed) return;
381378
const model = result.models.find((candidate) => candidate.id === modelId) ?? null;
@@ -471,7 +468,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
471468
const request = ++startRequestRef.current;
472469
setSetup((current) => ({ ...current, downloading: true, error: null }));
473470
try {
474-
await runtime.runPromise(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
471+
await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
475472
if (request !== startRequestRef.current) return;
476473
const next = await probeSpeechStatus(prepared, true);
477474
if (request !== startRequestRef.current) return;
@@ -500,7 +497,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
500497
startRequestRef.current += 1;
501498
setSetup((current) => ({ ...current, open: false, downloading: false }));
502499
try {
503-
await runtime.runPromise(
500+
await runEnvironmentSpeech(
504501
cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),
505502
);
506503
} catch (error) {

0 commit comments

Comments
 (0)