Skip to content

Commit 084aa6e

Browse files
jakeleventhalclaude
andcommitted
feat: Codex voice conversations on web, desktop, and mobile
Codex 0.156 voice mode for T3 Code: talk to a realtime voice model that hands work to Codex in the thread and reads its answers back. Audio runs over WebRTC between the client and OpenAI; the server relays only SDP and captions over provider.voice.session, and ending the stream stops the conversation. On orchestrator V2, Codex's self-started voice turns are interrupted and resubmitted as ordinary user runs, since V2 has no provider-initiated runs. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 67c0fc0 commit 084aa6e

53 files changed

Lines changed: 2979 additions & 40 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/desktop/src/window/DesktopWindow.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,10 @@ function makeFakeBrowserWindow() {
9595
reloadIgnoringCache: vi.fn(),
9696
replaceMisspelling: vi.fn(),
9797
send: vi.fn(),
98+
session: {
99+
setPermissionCheckHandler: vi.fn(),
100+
setPermissionRequestHandler: vi.fn(),
101+
},
98102
setBackgroundThrottling: vi.fn(),
99103
setWindowOpenHandler: vi.fn(),
100104
toggleDevTools: vi.fn(),

‎apps/desktop/src/window/DesktopWindow.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts";
3333
import * as DesktopClientSettings from "../settings/DesktopClientSettings.ts";
3434
import * as ElectronApp from "../electron/ElectronApp.ts";
3535
import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts";
36+
import { installMainWindowPermissionHandlers } from "./MainWindowPermissions.ts";
3637
import { makeQuitShortcutHandler } from "./QuitHold.ts";
3738
import { copyContextMenuImage } from "./ContextMenuImage.ts";
3839

@@ -418,6 +419,10 @@ export const make = Effect.gen(function* () {
418419
if (environment.platform === "darwin") {
419420
window.setAutoHideCursor(false);
420421
}
422+
installMainWindowPermissionHandlers(window.webContents.session, {
423+
appUrl: applicationUrl,
424+
platform: environment.platform,
425+
});
421426
let boundsPersistFiber: Fiber.Fiber<void, never> | undefined;
422427
let pendingBoundsPersistFiber: Fiber.Fiber<void, never> | undefined;
423428
let boundsPersistenceEnabled = persistedBounds === null || restoredPersistedBounds;
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
3+
import { decideMainWindowMediaRequest, isAppUrl } from "./MainWindowPermissions.ts";
4+
5+
const APP_URL = "t3code://app/";
6+
7+
function decide(overrides: Partial<Parameters<typeof decideMainWindowMediaRequest>[0]> = {}) {
8+
return decideMainWindowMediaRequest({
9+
requestingUrl: "t3code://app/threads/abc",
10+
appUrl: APP_URL,
11+
mediaTypes: ["audio"],
12+
platform: "darwin",
13+
microphoneStatus: () => "granted",
14+
...overrides,
15+
});
16+
}
17+
18+
describe("isAppUrl", () => {
19+
it("matches the app shell and nothing that merely shares its prefix", () => {
20+
expect(isAppUrl("t3code://app", APP_URL)).toBe(true);
21+
expect(isAppUrl("t3code://app/settings", APP_URL)).toBe(true);
22+
expect(isAppUrl("t3code://application/", APP_URL)).toBe(false);
23+
expect(isAppUrl("https://example.com/", APP_URL)).toBe(false);
24+
expect(isAppUrl(undefined, APP_URL)).toBe(false);
25+
});
26+
});
27+
28+
describe("decideMainWindowMediaRequest", () => {
29+
it("denies media to frames outside the app shell", () => {
30+
expect(decide({ requestingUrl: "https://example.com/embed" })).toBe("deny");
31+
});
32+
33+
it("asks macOS for the microphone until the user has granted it", () => {
34+
expect(decide({ microphoneStatus: () => "not-determined" })).toBe("ask-microphone");
35+
expect(decide({ microphoneStatus: () => "denied" })).toBe("ask-microphone");
36+
expect(decide()).toBe("grant");
37+
});
38+
39+
it("grants the microphone directly off macOS", () => {
40+
expect(decide({ platform: "win32", microphoneStatus: () => "unknown" })).toBe("grant");
41+
});
42+
43+
it("keeps granting video-only capture such as preview tab recording", () => {
44+
expect(decide({ mediaTypes: ["video"], microphoneStatus: () => "denied" })).toBe("grant");
45+
});
46+
});
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import * as Electron from "electron";
2+
3+
export type MediaPermissionDecision = "grant" | "deny" | "ask-microphone";
4+
5+
/** Whether `url` belongs to the app shell served from `appUrl` (e.g. `t3code://app/`). */
6+
export function isAppUrl(url: string | undefined, appUrl: string): boolean {
7+
if (!url) return false;
8+
// Custom schemes have an opaque WHATWG origin, so compare the serialized prefix.
9+
const appOrigin = appUrl.endsWith("/") ? appUrl.slice(0, -1) : appUrl;
10+
return url === appOrigin || url.startsWith(`${appOrigin}/`);
11+
}
12+
13+
/**
14+
* Media requests come from the app shell (voice conversations use the
15+
* microphone; preview tab recording routes `getDisplayMedia` through the same
16+
* permission). Frames from any other origin never get media. On macOS the
17+
* microphone also needs the system's consent, asked for on first use.
18+
*/
19+
export function decideMainWindowMediaRequest(input: {
20+
readonly requestingUrl: string;
21+
readonly appUrl: string;
22+
readonly mediaTypes: ReadonlyArray<"audio" | "video">;
23+
readonly platform: NodeJS.Platform;
24+
readonly microphoneStatus: () => string;
25+
}): MediaPermissionDecision {
26+
if (!isAppUrl(input.requestingUrl, input.appUrl)) return "deny";
27+
if (!input.mediaTypes.includes("audio") || input.platform !== "darwin") return "grant";
28+
return input.microphoneStatus() === "granted" ? "grant" : "ask-microphone";
29+
}
30+
31+
/**
32+
* Installs the main window session's permission handlers. Only `media` is
33+
* decided here; every other permission keeps Electron's default of allowing
34+
* it, which is what this session did before it had handlers.
35+
*/
36+
export function installMainWindowPermissionHandlers(
37+
session: Electron.Session,
38+
options: { readonly appUrl: string; readonly platform: NodeJS.Platform },
39+
): void {
40+
session.setPermissionRequestHandler((_webContents, permission, callback, details) => {
41+
if (permission !== "media") {
42+
callback(true);
43+
return;
44+
}
45+
const decision = decideMainWindowMediaRequest({
46+
requestingUrl: details.requestingUrl,
47+
appUrl: options.appUrl,
48+
mediaTypes: "mediaTypes" in details ? (details.mediaTypes ?? []) : [],
49+
platform: options.platform,
50+
microphoneStatus: () => Electron.systemPreferences.getMediaAccessStatus("microphone"),
51+
});
52+
if (decision !== "ask-microphone") {
53+
callback(decision === "grant");
54+
return;
55+
}
56+
Electron.systemPreferences.askForMediaAccess("microphone").then(callback, () => {
57+
callback(false);
58+
});
59+
});
60+
session.setPermissionCheckHandler((_webContents, permission, requestingOrigin) =>
61+
permission === "media" ? isAppUrl(requestingOrigin, options.appUrl) : true,
62+
);
63+
}

‎apps/mobile/app.config.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,10 @@ const sharingPlugin: NonNullable<ExpoConfig["plugins"]>[number] = [
217217
},
218218
];
219219

220+
const MICROPHONE_PERMISSION =
221+
"Allow T3 Code to use your microphone for voice input and voice conversations.";
222+
const CAMERA_PERMISSION = "Allow T3 Code to access your camera so you can scan pairing QR codes.";
223+
220224
// These aliases match the fonts' PostScript names on iOS. Register the same
221225
// names on Android so React Native and the native composer use one set of
222226
// family names without waiting for runtime font loading.
@@ -305,6 +309,8 @@ const config: ExpoConfig = {
305309
foregroundImage: variant.assets.androidAdaptiveForeground,
306310
monochromeImage: variant.assets.androidMonochromeIcon,
307311
},
312+
// react-native-webrtc's plugin requests overlay access for screen sharing, which voice does not use.
313+
blockedPermissions: ["android.permission.SYSTEM_ALERT_WINDOW"],
308314
// Opts into OnBackInvokedCallback-based back dispatch (Android 13+).
309315
// JS back handling survives it via react-native's Android 16 shim plus
310316
// withAndroidPredictiveBackCompat on Android 13-15.
@@ -378,22 +384,30 @@ const config: ExpoConfig = {
378384
[
379385
"expo-audio",
380386
{
381-
microphonePermission: "Allow T3 Code to use your microphone for voice input.",
382-
recordAudioAndroid: false,
387+
microphonePermission: MICROPHONE_PERMISSION,
388+
recordAudioAndroid: true,
383389
enableBackgroundPlayback: false,
384390
enableBackgroundRecording: false,
385391
},
386392
],
393+
// Voice conversations run over WebRTC. Pass both usage strings so its
394+
// plugin cannot fall back to generic ones.
395+
[
396+
"@config-plugins/react-native-webrtc",
397+
{ cameraPermission: CAMERA_PERMISSION, microphonePermission: MICROPHONE_PERMISSION },
398+
],
387399
[
388400
"expo-camera",
389401
{
390-
cameraPermission: "Allow T3 Code to access your camera so you can scan pairing QR codes.",
402+
cameraPermission: CAMERA_PERMISSION,
391403
microphonePermission: false,
392404
barcodeScannerEnabled: true,
393405
recordAudioAndroid: false,
394406
},
395407
],
396-
["expo-image-picker", { photosPermission: false, microphonePermission: false }],
408+
// `microphonePermission: false` would block RECORD_AUDIO on Android for every
409+
// package, including voice conversations, so share the app's usage string.
410+
["expo-image-picker", { photosPermission: false, microphonePermission: MICROPHONE_PERMISSION }],
397411
[
398412
"expo-splash-screen",
399413
{

‎apps/mobile/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@
136136
"react-native-screens": "file:deps/react-native-screens-5.0.0-t3.29.tgz",
137137
"react-native-shiki-engine": "^0.3.12",
138138
"react-native-svg": "15.16.0",
139+
"react-native-webrtc": "124.0.8",
139140
"react-native-webview": "14.0.1",
140141
"react-native-worklets": "0.13.0",
141142
"remark-gfm": "catalog:",
@@ -148,6 +149,7 @@
148149
"devDependencies": {
149150
"@babel/core": "^7.29.7",
150151
"@effect/platform-node": "catalog:",
152+
"@config-plugins/react-native-webrtc": "15.0.2",
151153
"@effect/vitest": "catalog:",
152154
"@pierre/trees": "1.0.0-beta.6",
153155
"@types/mdast": "^4.0.4",

‎apps/mobile/src/components/ui/AppSymbol.tsx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ import IconLayoutSidebar from "@tabler/icons-react-native/IconLayoutSidebar";
8383
import IconLayoutSidebarRight from "@tabler/icons-react-native/IconLayoutSidebarRight";
8484
import IconLetterSpacing from "@tabler/icons-react-native/IconLetterSpacing";
8585
import IconMicrophone from "@tabler/icons-react-native/IconMicrophone";
86+
import IconMicrophoneOff from "@tabler/icons-react-native/IconMicrophoneOff";
8687
import IconLink from "@tabler/icons-react-native/IconLink";
8788
import IconListNumbers from "@tabler/icons-react-native/IconListNumbers";
8889
import IconMenu2 from "@tabler/icons-react-native/IconMenu2";
@@ -129,6 +130,7 @@ import IconUserQuestion from "@tabler/icons-react-native/IconUserQuestion";
129130
import IconUserPlus from "@tabler/icons-react-native/IconUserPlus";
130131
import IconUsers from "@tabler/icons-react-native/IconUsers";
131132
import IconWifi from "@tabler/icons-react-native/IconWifi";
133+
import IconWaveSine from "@tabler/icons-react-native/IconWaveSine";
132134
import IconWifiOff from "@tabler/icons-react-native/IconWifiOff";
133135
import IconWorld from "@tabler/icons-react-native/IconWorld";
134136
import IconX from "@tabler/icons-react-native/IconX";
@@ -223,6 +225,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
223225
macstudio: IconDeviceDesktop,
224226
magnifyingglass: IconSearch,
225227
mic: IconMicrophone,
228+
"mic.slash": IconMicrophoneOff,
226229
paintbrush: IconPalette,
227230
pencil: IconPencil,
228231
"person.crop.circle": IconUserCircle,
@@ -273,6 +276,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
273276
"tray.and.arrow.up": IconUpload,
274277
trash: IconTrash,
275278
wifi: IconWifi,
279+
waveform: IconWaveSine,
276280
"wifi.slash": IconWifiOff,
277281
xmark: IconX,
278282
"xmark.circle.fill": IconCircleXFilled,

‎apps/mobile/src/features/threads/ThreadComposer.tsx‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@ import {
113113
} from "../voice-input/ComposerDictationControl";
114114
import { useVoiceInputController } from "../voice-input/useVoiceInputController";
115115
import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation";
116+
import { ComposerVoiceModeButton, ComposerVoiceStrip } from "../voice-mode/ComposerVoiceMode";
117+
import { useVoiceModePhase, useVoiceModeThreadLifecycle } from "../voice-mode/useVoiceMode";
118+
import { voiceMode, voiceModeSupported } from "../voice-mode/voiceMode";
116119
import {
117120
rememberModelOptions,
118121
withRememberedModelOptions,
@@ -480,15 +483,40 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
480483
onUsageLimits:
481484
usageLimitsOffered && draftAttachments.length === 0 ? openUsageLimits : undefined,
482485
});
486+
const voiceTarget = useMemo(
487+
() => ({ environmentId: props.environmentId, threadId: props.selectedThread.id }),
488+
[props.environmentId, props.selectedThread.id],
489+
);
490+
const voicePhase = useVoiceModePhase(voiceTarget);
491+
useVoiceModeThreadLifecycle(voiceTarget);
492+
// Dictation and a voice conversation both need the microphone.
493+
const voiceConversationLive = voicePhase !== "idle";
483494
const voiceInput = useVoiceInputController({
484495
ownerKey: composerDraftKey,
485496
label: props.selectedThread.title || "Untitled thread",
486497
readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text,
487498
subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange),
488499
selection: composerMenu.selection,
500+
disabled: voiceConversationLive,
489501
onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text),
490502
onChangeSelection: composerMenu.onSelectionChange,
491503
});
504+
const offersVoiceConversation =
505+
voiceModeSupported && selectedProviderStatus?.supportsVoice === true;
506+
// The server attaches voice to the thread's provider session, which the first message creates.
507+
const canToggleVoiceConversation =
508+
voiceConversationLive ||
509+
(props.connectionState === "connected" &&
510+
props.selectedThread.latestUserMessageAt !== null &&
511+
!voiceInput.isBusy);
512+
const toggleVoiceConversation = useCallback(() => voiceMode.toggle(voiceTarget), [voiceTarget]);
513+
const voiceModeButton = offersVoiceConversation ? (
514+
<ComposerVoiceModeButton
515+
phase={voicePhase}
516+
disabled={!canToggleVoiceConversation}
517+
onPress={toggleVoiceConversation}
518+
/>
519+
) : null;
492520
const voicePresentation = resolveVoiceComposerPresentation(
493521
voiceInput.state,
494522
voiceInput.elapsedSeconds,
@@ -736,6 +764,9 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
736764
{selectedProviderStatus.compatibilityAdvisory.message}
737765
</Text>
738766
) : null}
767+
768+
{voiceConversationLive ? <ComposerVoiceStrip /> : null}
769+
739770
{modelUnavailable ? (
740771
<Button
741772
label="Model unavailable. Open model settings."
@@ -977,9 +1008,11 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
9771008
) : null}
9781009
{!isExpanded ? (
9791010
<View className="flex-row items-center">
1011+
{voiceModeButton}
9801012
<ComposerDictationStartAction
9811013
state={voiceInput.state}
9821014
isAvailable={voiceInput.isAvailable}
1015+
disabled={voiceConversationLive}
9831016
onStart={voiceInput.start}
9841017
onCancel={voiceInput.cancel}
9851018
/>
@@ -1077,10 +1110,12 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
10771110
</View>
10781111
)}
10791112
<View className="shrink-0 flex-row items-center">
1113+
{isVoiceInputPresented ? null : voiceModeButton}
10801114
<ComposerDictationPrimaryAction
10811115
state={voiceInput.state}
10821116
presentation={voicePresentation}
10831117
isAvailable={voiceInput.isAvailable}
1118+
disabled={voiceConversationLive}
10841119
onStart={voiceInput.start}
10851120
onConfirm={voiceInput.stop}
10861121
onCancel={voiceInput.cancel}

0 commit comments

Comments
 (0)