Skip to content

Commit 310c408

Browse files
committed
fix(voice): retain sessions and respect navigation boundaries
1 parent 16361d9 commit 310c408

9 files changed

Lines changed: 215 additions & 30 deletions

File tree

‎apps/desktop/src/window/MainWindowPermissions.test.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import { describe, expect, it } from "vite-plus/test";
22

3-
import { decideMainWindowMediaRequest, isAppUrl } from "./MainWindowPermissions.ts";
3+
import {
4+
decideMainWindowMediaRequest,
5+
isAppUrl,
6+
isMainWindowPermissionAllowed,
7+
} from "./MainWindowPermissions.ts";
48

59
const APP_URL = "t3code://app/";
610

@@ -44,3 +48,13 @@ describe("decideMainWindowMediaRequest", () => {
4448
expect(decide({ mediaTypes: ["video"], microphoneStatus: () => "denied" })).toBe("grant");
4549
});
4650
});
51+
52+
it("denies synchronous clipboard reads for embedded origins and the app shell", () => {
53+
for (const origin of ["https://example.com/frame", APP_URL]) {
54+
expect(isMainWindowPermissionAllowed("deprecated-sync-clipboard-read", origin, APP_URL)).toBe(
55+
false,
56+
);
57+
}
58+
expect(isMainWindowPermissionAllowed("media", "https://example.com/frame", APP_URL)).toBe(false);
59+
expect(isMainWindowPermissionAllowed("media", APP_URL, APP_URL)).toBe(true);
60+
});

‎apps/desktop/src/window/MainWindowPermissions.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,18 +28,27 @@ export function decideMainWindowMediaRequest(input: {
2828
return input.microphoneStatus() === "granted" ? "grant" : "ask-microphone";
2929
}
3030

31+
export function isMainWindowPermissionAllowed(
32+
permission: string,
33+
origin: string | undefined,
34+
appUrl: string,
35+
): boolean {
36+
return permission === "media"
37+
? isAppUrl(origin, appUrl)
38+
: permission !== "deprecated-sync-clipboard-read";
39+
}
40+
3141
/**
3242
* Installs the main window session's permission handlers. Only `media` is
33-
* decided here; every other permission keeps Electron's default of allowing
34-
* it, which is what this session did before it had handlers.
43+
* decided here. Synchronous clipboard reads keep Electron's default denial.
3544
*/
3645
export function installMainWindowPermissionHandlers(
3746
session: Electron.Session,
3847
options: { readonly appUrl: string; readonly platform: NodeJS.Platform },
3948
): void {
4049
session.setPermissionRequestHandler((_webContents, permission, callback, details) => {
4150
if (permission !== "media") {
42-
callback(true);
51+
callback(isMainWindowPermissionAllowed(permission, details.requestingUrl, options.appUrl));
4352
return;
4453
}
4554
const decision = decideMainWindowMediaRequest({
@@ -58,6 +67,6 @@ export function installMainWindowPermissionHandlers(
5867
});
5968
});
6069
session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) =>
61-
permission === "media" ? isAppUrl(requestingOrigin, options.appUrl) : true,
70+
isMainWindowPermissionAllowed(permission, requestingOrigin, options.appUrl),
6271
);
6372
}

‎apps/mobile/src/features/threads/ThreadComposer.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
508508
const canToggleVoiceConversation =
509509
voiceConversationLive ||
510510
(props.connectionState === "connected" &&
511+
props.canOperateThread &&
511512
props.selectedThread.activeProviderThreadId !== null &&
512513
!voiceInput.isGloballyBusy);
513514
const toggleVoiceConversation = () => voiceMode.toggle(voiceTarget);
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import { act, createContext, useContext } from "react";
2+
import { createRoot, type Root } from "react-dom/client";
3+
import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";
4+
import { EnvironmentId, ThreadId } from "@t3tools/contracts";
5+
6+
const fixture = vi.hoisted(() => ({ active: true, currentThread: "thread-a", stop: vi.fn() }));
7+
const RouteName = createContext("Thread");
8+
const target = {
9+
environmentId: EnvironmentId.make("environment"),
10+
threadId: ThreadId.make("thread-a"),
11+
};
12+
vi.mock("@react-navigation/native", () => ({
13+
useRoute: () => ({ name: useContext(RouteName) }),
14+
useNavigationState: (
15+
select: (state: { routes: Array<{ name: string; params: { threadId: string } }> }) => unknown,
16+
) => select({ routes: [{ name: "Thread", params: { threadId: fixture.currentThread } }] }),
17+
}));
18+
vi.mock("react-native", () => ({ Alert: { alert: vi.fn() } }));
19+
vi.mock("./voiceModePresentation", () => ({ resolveVoiceModeNotice: () => null }));
20+
vi.mock("./voiceMode", () => ({
21+
voiceMode: {
22+
isActiveFor: (value: { threadId: string }) => fixture.active && value.threadId === "thread-a",
23+
stop: () => {
24+
fixture.active = false;
25+
fixture.stop();
26+
},
27+
subscribe: () => () => {},
28+
getState: () => ({ phase: "active", target }),
29+
},
30+
}));
31+
import { useVoiceModeThreadLifecycle } from "./useVoiceMode";
32+
function Lifecycle() {
33+
useVoiceModeThreadLifecycle(target);
34+
return null;
35+
}
36+
let root: Root;
37+
beforeEach(() => {
38+
fixture.active = true;
39+
fixture.currentThread = "thread-a";
40+
fixture.stop.mockClear();
41+
(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
42+
root = createRoot(document.createElement("div"));
43+
});
44+
afterEach(() => act(() => root.unmount()));
45+
const render = (overlay: boolean) =>
46+
act(() =>
47+
root.render(
48+
<>
49+
<RouteName value="Thread">
50+
<Lifecycle />
51+
</RouteName>
52+
{overlay ? (
53+
<RouteName value="ThreadFiles">
54+
<Lifecycle />
55+
</RouteName>
56+
) : null}
57+
</>,
58+
),
59+
);
60+
it("keeps voice running when Files closes and stops when the owning Thread leaves", () => {
61+
render(true);
62+
render(false);
63+
expect(fixture.stop).not.toHaveBeenCalled();
64+
act(() => root.unmount());
65+
expect(fixture.stop).toHaveBeenCalledTimes(1);
66+
});
67+
it("stops the previous thread when navigation selects a different thread", () => {
68+
render(false);
69+
fixture.currentThread = "thread-b";
70+
render(false);
71+
expect(fixture.stop).toHaveBeenCalledTimes(1);
72+
});

‎apps/mobile/src/features/voice-mode/useVoiceMode.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import type {
55
} from "@t3tools/client-runtime/voice-mode";
66
import { useCallback, useEffect, useSyncExternalStore } from "react";
77
import { Alert } from "react-native";
8-
import { useNavigationState } from "@react-navigation/native";
8+
import { useNavigationState, useRoute } from "@react-navigation/native";
99

1010
import { voiceMode } from "./voiceMode";
1111
import { resolveVoiceModeNotice } from "./voiceModePresentation";
@@ -33,6 +33,7 @@ export function useVoiceModePhase(target: VoiceSessionTarget): VoiceModePhase {
3333
*/
3434
export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void {
3535
const { environmentId, threadId } = target;
36+
const routeName = useRoute().name;
3637
const visibleThreadId = useNavigationState((state) => {
3738
const route = state.routes.findLast((route) => route.name === "Thread");
3839
const params = route?.params;
@@ -45,9 +46,10 @@ export function useVoiceModeThreadLifecycle(target: VoiceSessionTarget): void {
4546

4647
useEffect(
4748
() => () => {
48-
if (voiceMode.isActiveFor({ environmentId, threadId })) voiceMode.stop();
49+
if (routeName === "Thread" && voiceMode.isActiveFor({ environmentId, threadId }))
50+
voiceMode.stop();
4951
},
50-
[environmentId, threadId],
52+
[environmentId, threadId, routeName],
5153
);
5254

5355
useEffect(() => {

‎apps/mobile/vite.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { defineConfig } from "vite-plus";
22
import "vite-plus/test/config";
33

44
const reactTests = [
5+
"src/features/voice-mode/useVoiceMode.test.tsx",
56
"src/components/ui/Dialog.android.test.tsx",
67
"src/features/home/home-route-visibility.test.tsx",
78
"src/features/settings/SettingsServerControlsRouteScreen.test.tsx",

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
334334
readonly backgroundTasksEndWithProcess?: boolean;
335335
readonly hangSessionScopeClose?: boolean;
336336
readonly startTurn?: Effect.Effect<void>;
337+
readonly startVoiceSession?: Effect.Effect<void>;
337338
readonly beforeUnload?: Effect.Effect<void>;
338339
readonly spawnBeforeOpen?: boolean;
339340
readonly scopeCloseReached?: Deferred.Deferred<void>;
@@ -448,6 +449,15 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
448449
})),
449450
),
450451
),
452+
...(options.startVoiceSession === undefined
453+
? {}
454+
: {
455+
startVoiceSession: () =>
456+
Stream.fromEffect(options.startVoiceSession!).pipe(
457+
Stream.drain,
458+
Stream.concat(Stream.never),
459+
),
460+
}),
451461
respondToRuntimeRequest: () => Effect.void,
452462
readThreadSnapshot: () => unimplemented("readThreadSnapshot unused in test"),
453463
rollbackThread: () => unimplemented("rollbackThread unused in test"),
@@ -483,6 +493,7 @@ function layerTest(input: {
483493
readonly backgroundTasksEndWithProcess?: boolean;
484494
readonly hangSessionScopeClose?: boolean;
485495
readonly startTurn?: Effect.Effect<void>;
496+
readonly startVoiceSession?: Effect.Effect<void>;
486497
readonly beforeUnload?: Effect.Effect<void>;
487498
readonly spawnBeforeOpen?: boolean;
488499
readonly scopeCloseReached?: Deferred.Deferred<void>;
@@ -518,6 +529,9 @@ function layerTest(input: {
518529
? {}
519530
: { hangSessionScopeClose: input.hangSessionScopeClose }),
520531
...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }),
532+
...(input.startVoiceSession === undefined
533+
? {}
534+
: { startVoiceSession: input.startVoiceSession }),
521535
...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }),
522536
...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }),
523537
...(input.scopeCloseReached === undefined
@@ -5029,3 +5043,62 @@ it.effect(
50295043
assert.isFalse(denied?.capabilities?.has("device"));
50305044
}),
50315045
);
5046+
5047+
it.effect(
5048+
"protects the provider session while voice streams and restores idle release after disconnect",
5049+
() =>
5050+
Effect.gen(function* () {
5051+
const state = yield* Ref.make(emptyState);
5052+
const started = yield* Deferred.make<void>();
5053+
yield* Effect.gen(function* () {
5054+
const sink = yield* EventSink.EventSinkV2;
5055+
const allocator = yield* IdAllocator.IdAllocatorV2;
5056+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
5057+
const now = yield* DateTime.now;
5058+
const threadId = ThreadId.make("voice-idle-test");
5059+
const providerSessionId = yield* allocator.allocate.providerSession({
5060+
providerInstanceId: modelSelection.instanceId,
5061+
threadId,
5062+
});
5063+
const providerThread = makeProviderThread({
5064+
idAllocator: allocator,
5065+
threadId,
5066+
providerSessionId,
5067+
now,
5068+
});
5069+
yield* sink.write({
5070+
events: [
5071+
yield* makeThreadCreatedEvent({ idAllocator: allocator, threadId, now }),
5072+
{
5073+
id: yield* allocator.allocate.event({ threadId }),
5074+
type: "provider-thread.updated",
5075+
threadId,
5076+
driver: CODEX_DRIVER,
5077+
occurredAt: now,
5078+
payload: providerThread,
5079+
},
5080+
],
5081+
});
5082+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
5083+
const voice = yield* manager
5084+
.startVoiceSession({ threadId, offerSdp: "offer" })
5085+
.pipe(Stream.runDrain, Effect.forkChild);
5086+
yield* Deferred.await(started);
5087+
yield* TestClock.adjust("2 seconds");
5088+
assert.isTrue(Option.isSome(yield* manager.get(providerSessionId)));
5089+
assert.equal((yield* Ref.get(state)).closeCount, 0);
5090+
yield* Fiber.interrupt(voice);
5091+
yield* TestClock.adjust("2 seconds");
5092+
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
5093+
assert.equal((yield* Ref.get(state)).closeCount, 1);
5094+
}).pipe(
5095+
Effect.provide(
5096+
layerTest({
5097+
state,
5098+
idleTimeoutMs: 1000,
5099+
startVoiceSession: Deferred.succeed(started, undefined),
5100+
}),
5101+
),
5102+
);
5103+
}),
5104+
);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2104,6 +2104,7 @@ export const layerWithOptions = (
21042104
),
21052105
);
21062106

2107+
let nextVoiceOrdinal = -1;
21072108
return ProviderSessionManagerV2.of({
21082109
startVoiceSession: (input) =>
21092110
Stream.unwrap(
@@ -2128,6 +2129,18 @@ export const layerWithOptions = (
21282129
threadId: input.threadId,
21292130
detail: `Provider '${runtime.driver}' does not support voice conversations.`,
21302131
});
2132+
const providerSessionId = runtime.providerSessionId;
2133+
const voiceOrdinal = nextVoiceOrdinal--;
2134+
yield* holdThreadLoaded({
2135+
providerSessionId,
2136+
threadId: input.threadId,
2137+
providerThread,
2138+
});
2139+
yield* Effect.acquireRelease(
2140+
markBusy(providerSessionId, busyTurnKey(providerThread.id, voiceOrdinal)),
2141+
() =>
2142+
markIdle(providerSessionId, providerThread.id, voiceOrdinal).pipe(Effect.ignore),
2143+
);
21312144
return runtime.startVoiceSession({ providerThread, offerSdp: input.offerSdp });
21322145
}),
21332146
).pipe(

‎apps/web/src/components/ChatView.tsx‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6458,6 +6458,28 @@ export default function ChatView(props: ChatViewProps) {
64586458

64596459
return;
64606460
}
6461+
const parsedSlashCommand =
6462+
composerImages.length === 0 &&
6463+
composerFiles.length === 0 &&
6464+
sendableComposerTerminalContexts.length === 0 &&
6465+
composerPreviewAnnotations.length === 0 &&
6466+
composerReviewComments.length === 0 &&
6467+
composerThreadContexts.length === 0 &&
6468+
multipleModelSelections === null
6469+
? parseStandaloneComposerSlashCommand(trimmed)
6470+
: null;
6471+
if (parsedSlashCommand === "voice") {
6472+
// Providers without voice receive "/voice" as a normal message.
6473+
const voiceAvailability = resolveThreadVoiceAvailability(activeThread, providerStatuses);
6474+
// An active conversation always stops, even if the provider stopped offering voice.
6475+
if (voiceAvailability.kind !== "unsupported" || voiceMode.isActiveFor(routeThreadRef)) {
6476+
toggleThreadVoice(routeThreadRef, voiceAvailability);
6477+
promptRef.current = "";
6478+
clearComposerDraftContent(composerDraftTarget);
6479+
composerRef.current?.resetCursorState();
6480+
return;
6481+
}
6482+
}
64616483
if (
64626484
!directAnnotation &&
64636485
sendInteractionModeEnabled &&
@@ -6528,28 +6550,6 @@ export default function ChatView(props: ChatViewProps) {
65286550
}
65296551
return;
65306552
}
6531-
const parsedSlashCommand =
6532-
composerImages.length === 0 &&
6533-
composerFiles.length === 0 &&
6534-
sendableComposerTerminalContexts.length === 0 &&
6535-
composerPreviewAnnotations.length === 0 &&
6536-
composerReviewComments.length === 0 &&
6537-
composerThreadContexts.length === 0 &&
6538-
multipleModelSelections === null
6539-
? parseStandaloneComposerSlashCommand(trimmed)
6540-
: null;
6541-
if (parsedSlashCommand === "voice") {
6542-
// Providers without voice receive "/voice" as a normal message.
6543-
const voiceAvailability = resolveThreadVoiceAvailability(activeThread, providerStatuses);
6544-
// An active conversation always stops, even if the provider stopped offering voice.
6545-
if (voiceAvailability.kind !== "unsupported" || voiceMode.isActiveFor(routeThreadRef)) {
6546-
toggleThreadVoice(routeThreadRef, voiceAvailability);
6547-
promptRef.current = "";
6548-
clearComposerDraftContent(composerDraftTarget);
6549-
composerRef.current?.resetCursorState();
6550-
return;
6551-
}
6552-
}
65536553
// Providers without the legacy toggle receive their native commands unchanged.
65546554
const standaloneSlashCommand =
65556555
sendInteractionModeEnabled && parsedSlashCommand !== "voice" ? parsedSlashCommand : null;

0 commit comments

Comments
 (0)