Skip to content

Commit 123d146

Browse files
mylesndavidmylesndavid
authored
feat(desktop): GPT Live calls, Voice tab and narrower resizable panel [skip ci]
OmniCode desktop can now hold a GPT Live conversation through the saved AgentOS account while its local coding agent works. Spoken and typed requests enter the normal coding session and retain its permission flow; completed work is relayed back to the call. Hanging up releases microphone and call usage independently of coding execution. The composer becomes the call controls with the Libraries.dev VoiceBeam. Hold Space while muted for push-to-talk; text fields retain ordinary spaces. An icon opens text entry during a call. Voice appears alongside Review in the existing resizable panel, whose minimum width is reduced by 80 px. Original and unsent call drafts survive navigation and hang-up. Credentials and Fly affinity stay in the private CLI bridge; renderer IPC can only manage its own calls. Includes shared native-style icons, third-party notices, and desktop documentation. Validation: desktop and CLI typechecks, 6 desktop bridge tests, 20 tab/width tests, packaged Apple Silicon app with production AgentOS/GPT Live, real coding handoff and typed result, Space down/up and blur muting, text spaces, draft restoration, Voice/Review switching, and live geometry at 1280×800 and 1200×750. Local desktop package is ad-hoc signed; no notarized public desktop release is claimed. CI skipped at owner request. Backend is already deployed by AgentOS PR anomalyco#1473.
1 parent 138629d commit 123d146

27 files changed

Lines changed: 1941 additions & 436 deletions

‎packages/app/src/agentos.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,10 @@ export type AgentOSDictation = { audio: ArrayBuffer; type: string; id: string }
1313
export type AgentOSDictationResult =
1414
| { ok: true; text: string }
1515
| { ok: false; reason: "auth" | "credits" | "empty" | "invalid" | "failed" }
16+
17+
export type AgentOSLiveRequest =
18+
| { action: "start"; sdp: string; context: string }
19+
| { action: "status" | "end"; id: string }
20+
export type AgentOSLiveResult =
21+
| { ok: true; id: string; sdp?: string; phase?: string; error?: string | null }
22+
| { ok: false; reason: "auth" | "credits" | "conflict" | "failed" }

‎packages/app/src/app.tsx‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import "@/index.css"
2+
import { OmniLiveProvider } from "@/components/omni-live"
23
import * as Sentry from "@sentry/solid"
34
import { I18nProvider } from "@opencode-ai/ui/context"
45
import { DialogProvider } from "@opencode-ai/ui/context/dialog"
@@ -316,7 +317,9 @@ function SharedProviders(props: ParentProps) {
316317
<BodyDesignClass />
317318
<CommandProvider>
318319
<DesktopCommands />
319-
<HighlightsProvider>{props.children}</HighlightsProvider>
320+
<HighlightsProvider>
321+
<OmniLiveProvider>{props.children}</OmniLiveProvider>
322+
</HighlightsProvider>
320323
</CommandProvider>
321324
</>
322325
)

‎packages/app/src/components/omni-dictation.tsx‎

Lines changed: 19 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
1-
import { createEffect, on, onCleanup, onMount, Show } from "solid-js"
1+
import { createEffect, on, onCleanup, Show } from "solid-js"
22
import { createStore } from "solid-js/store"
3+
import { Icon } from "@opencode-ai/ui/v2/icon"
34
import { IconButtonV2 } from "@opencode-ai/ui/v2/icon-button-v2"
45
import { useLanguage } from "@/context/language"
56
import { usePlatform } from "@/context/platform"
6-
import type { Root } from "react-dom/client"
7+
import { OmniVoiceEffects } from "./omni-voice-effects"
78

89
const MAX_AUDIO_BYTES = 25 * 1024 * 1024
910

1011
export function createOmniDictation(input: {
12+
disabled?: () => boolean
1113
scope: () => unknown
1214
insert: (text: string) => void
1315
working: () => boolean
@@ -41,7 +43,7 @@ export function createOmniDictation(input: {
4143
onCleanup(cancel)
4244

4345
async function start() {
44-
if (busy() || !available()) return
46+
if (busy() || input.disabled?.() || !available()) return
4547
const current = ++generation
4648
const scope = input.scope()
4749
setState({ phase: "requesting", error: "", seconds: 0 })
@@ -109,7 +111,7 @@ export function createOmniDictation(input: {
109111
function Controls() {
110112
return (
111113
<Show when={available()}>
112-
<div class="flex items-center gap-1 mr-1" data-component="omni-dictation" data-state={state.phase}>
114+
<div class="flex items-center gap-1" data-component="omni-dictation" data-state={state.phase}>
113115
<Show when={busy()}>
114116
<span role="status" class="text-[11px] text-v2-text-text-muted whitespace-nowrap px-1">
115117
{language.t(
@@ -127,110 +129,40 @@ export function createOmniDictation(input: {
127129
<IconButtonV2
128130
type="button"
129131
size="small"
130-
variant="ghost"
132+
variant="ghost-muted"
131133
aria-label={language.t("omni.voice.cancel")}
132134
title={language.t("omni.voice.cancel")}
133135
onClick={cancel}
134-
icon={
135-
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
136-
<path d="m4 4 8 8m0-8-8 8" stroke="currentColor" />
137-
</svg>
138-
}
136+
icon={<Icon name="xmark-small" />}
139137
/>
140138
</Show>
141139
<IconButtonV2
142140
type="button"
143-
variant="ghost"
144-
disabled={state.phase === "processing" || state.phase === "requesting"}
141+
variant="ghost-muted"
142+
disabled={input.disabled?.() || state.phase === "processing" || state.phase === "requesting"}
145143
aria-label={language.t(state.phase === "listening" ? "omni.voice.stop" : "omni.voice.start")}
146144
title={language.t(state.phase === "listening" ? "omni.voice.stop" : "omni.voice.start")}
147145
aria-pressed={state.phase === "listening"}
148146
onClick={() =>
149147
state.phase === "listening" ? recorder?.state === "recording" && recorder.stop() : void start()
150148
}
151-
icon={
152-
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
153-
<Show
154-
when={state.phase === "listening"}
155-
fallback={
156-
<>
157-
<rect x="5.5" y="1.5" width="5" height="8" rx="2.5" stroke="currentColor" />
158-
<path d="M3.5 7a4.5 4.5 0 0 0 9 0M8 11.5v3m-2.5 0h5" stroke="currentColor" />
159-
</>
160-
}
161-
>
162-
<rect x="4" y="4" width="8" height="8" rx="1" fill="currentColor" />
163-
</Show>
164-
</svg>
165-
}
149+
icon={<Icon name={state.phase === "listening" ? "stop" : "microphone"} />}
166150
/>
167151
</div>
168152
</Show>
169153
)
170154
}
171155

172156
function Effects() {
173-
let host!: HTMLDivElement
174-
let root: Root | undefined
175-
let disposed = false
176-
const [effects, setEffects] = createStore<{ render?: () => void }>({})
177-
onMount(async () => {
178-
if (!available()) return
179-
const [react, dom, voice, beam] = await Promise.all([
180-
import("react"),
181-
import("react-dom/client"),
182-
import("voice-glow"),
183-
import("border-beam"),
184-
])
185-
if (disposed) return
186-
root = dom.createRoot(host)
187-
setEffects("render", () => () => {
188-
const style = {
189-
position: "absolute" as const,
190-
inset: 0,
191-
width: "100%",
192-
height: "100%",
193-
borderRadius: 12,
194-
pointerEvents: "none" as const,
195-
}
196-
const child = () => react.createElement("div", { style: { width: "100%", height: "100%", borderRadius: 12 } })
197-
root!.render(
198-
react.createElement(
199-
react.Fragment,
200-
null,
201-
react.createElement(beam.BorderBeam, {
202-
style,
203-
children: child(),
204-
active: input.working() && !busy(),
205-
strength: 0.55,
206-
colorVariant: "colorful",
207-
theme: "auto",
208-
}),
209-
react.createElement(voice.VoiceBeam, {
210-
style,
211-
children: child(),
212-
stream: state.stream,
213-
active: state.phase === "listening" || state.phase === "processing",
214-
processing: state.phase === "processing",
215-
strength: 0.7,
216-
theme: "auto",
217-
}),
218-
),
219-
)
220-
})
221-
})
222-
createEffect(() => effects.render?.())
223-
onCleanup(() => {
224-
disposed = true
225-
root?.unmount()
226-
})
227157
return (
228-
<div
229-
ref={host}
230-
class="pointer-events-none absolute inset-0 z-20 rounded-xl"
231-
aria-hidden="true"
232-
data-component="omni-voice-effects"
233-
/>
158+
<Show when={available()}>
159+
<OmniVoiceEffects
160+
stream={state.stream}
161+
active={state.phase === "listening" || state.phase === "processing"}
162+
processing={state.phase === "processing"}
163+
working={input.working() && !busy()}
164+
/>
165+
</Show>
234166
)
235167
}
236168
return { available, busy, Controls, Effects, error: () => state.error }

‎packages/app/src/components/omni-live.css‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
[data-component="omni-live-call"] {
2+
position: relative;
3+
width: 100%;
4+
min-height: 174px;
5+
border-radius: 12px;
6+
background: var(--v2-background-bg-base);
7+
box-shadow: var(--v2-elevation-raised);
8+
color: var(--v2-text-text-base);
9+
padding: 12px 16px 28px;
10+
outline: none;
11+
isolation: isolate;
12+
}
13+
[data-slot="live-header"],
14+
[data-slot="live-title"],
15+
[data-slot="live-options"],
16+
[data-slot="live-actions"] {
17+
display: flex;
18+
align-items: center;
19+
}
20+
[data-slot="live-header"] {
21+
justify-content: space-between;
22+
gap: 12px;
23+
}
24+
[data-slot="live-title"] {
25+
gap: 8px;
26+
font-size: 13px;
27+
font-weight: 500;
28+
}
29+
[data-slot="live-title"] > svg {
30+
color: var(--v2-text-text-muted);
31+
}
32+
[data-slot="live-timer"] {
33+
color: var(--v2-text-text-muted);
34+
font-size: 12px;
35+
font-variant-numeric: tabular-nums;
36+
padding-left: 4px;
37+
font-weight: 400;
38+
}
39+
[data-slot="live-options"] {
40+
gap: 8px;
41+
}
42+
[data-slot="live-center"] {
43+
display: flex;
44+
flex-direction: column;
45+
align-items: center;
46+
gap: 16px;
47+
padding-top: 20px;
48+
}
49+
[data-slot="live-status"] {
50+
font-size: 13px;
51+
color: var(--v2-text-text-muted);
52+
min-height: 20px;
53+
text-align: center;
54+
}
55+
[data-slot="live-actions"] {
56+
gap: 16px;
57+
}
58+
[data-component="omni-live-call"] [data-slot="live-actions"] > [data-component="icon-button-v2"] {
59+
width: 34px;
60+
height: 34px;
61+
border-radius: 50%;
62+
}
63+
[data-component="omni-live-call"] [data-slot="live-actions"] > [data-component="button-v2"] {
64+
height: 34px;
65+
border-radius: 18px;
66+
padding: 0 13px;
67+
gap: 7px;
68+
font-size: 12px;
69+
}
70+
[data-component="omni-live-call"] [data-slot="live-actions"] > [data-component="button-v2"] > svg {
71+
color: var(--v2-state-fg-danger);
72+
}
73+
[data-slot="live-text"] {
74+
position: relative;
75+
margin-top: 22px;
76+
border-top: 1px solid var(--v2-border-border-base);
77+
padding-top: 14px;
78+
display: flex;
79+
align-items: flex-end;
80+
gap: 12px;
81+
}
82+
[data-slot="live-text"] textarea {
83+
resize: none;
84+
flex: 1;
85+
width: 100%;
86+
min-width: 0;
87+
max-height: 160px;
88+
background: transparent;
89+
outline: none;
90+
font-size: 13px;
91+
line-height: 20px;
92+
color: inherit;
93+
}
94+
[data-slot="live-text"] textarea::placeholder {
95+
color: var(--v2-text-text-faint);
96+
}
97+
[data-slot="live-text"] > button {
98+
width: 30px;
99+
height: 30px;
100+
flex: none;
101+
}
102+
[data-slot="live-error"] {
103+
font-size: 12px;
104+
line-height: 18px;
105+
color: var(--v2-text-text-muted);
106+
margin-top: 16px;
107+
}
108+
[data-component="omni-live-transcript"] {
109+
display: flex;
110+
height: 100%;
111+
min-height: 0;
112+
flex-direction: column;
113+
}
114+
[data-slot="live-transcript-scroll"] {
115+
flex: 1;
116+
min-height: 0;
117+
padding: 8px 24px 28px 20px;
118+
overflow-y: auto;
119+
overscroll-behavior: contain;
120+
overflow-wrap: anywhere;
121+
}
122+
[data-slot="live-transcript-scroll"] article {
123+
margin-bottom: 24px;
124+
}
125+
[data-slot="live-transcript-scroll"] article > span {
126+
font-size: 11px;
127+
color: var(--v2-text-text-muted);
128+
}
129+
[data-slot="live-transcript-scroll"] article > p {
130+
margin-top: 6px;
131+
font-size: 13px;
132+
line-height: 21px;
133+
white-space: pre-wrap;
134+
color: var(--v2-text-text-base);
135+
user-select: text;
136+
}
137+
[data-slot="live-transcript-empty"] {
138+
font-size: 13px;
139+
line-height: 21px;
140+
color: var(--v2-text-text-muted);
141+
}

0 commit comments

Comments
 (0)