Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
191 commits
Select commit Hold shift + click to select a range
ab94d90
feat(desktop): add local voice input
t3-code-bot[bot] Aug 23, 2026
a3a0c5b
fix(desktop): align speech service and controls
t3-code-bot[bot] Aug 23, 2026
0d8b6f5
fix(desktop): harden speech cancellation and models
t3-code-bot[bot] Aug 23, 2026
daf2f22
fix(desktop): close speech input edge cases
t3-code-bot[bot] Aug 23, 2026
b5d9835
fix(desktop): resolve speech data path from app identity
t3-code-bot[bot] Aug 23, 2026
a6549e3
fix(desktop): inject speech model http client
t3-code-bot[bot] Aug 23, 2026
94c041b
fix(web): align voice input feedback
t3-code-bot[bot] Aug 23, 2026
a02e0ec
fix(web): respect reduced motion for voice level
t3-code-bot[bot] Aug 23, 2026
35766b5
fix(web): preserve composer focus for voice controls
t3-code-bot[bot] Aug 23, 2026
39f3567
fix(desktop): follow redirects when downloading the speech model
UtkarshUsername Aug 29, 2026
393b235
fix(desktop): run voice transcription on the CPU backend
UtkarshUsername Aug 29, 2026
df9b610
fix(desktop): surface a clear error when voice capture is silent
UtkarshUsername Aug 29, 2026
bb16fbe
feat(desktop): add voice settings and microphone selection
UtkarshUsername Aug 31, 2026
22789be
refactor(desktop): share voice input orchestration
UtkarshUsername Aug 31, 2026
fb4612b
refactor(voice): transcribe on the environment
UtkarshUsername Sep 5, 2026
56bac28
fix(voice): guard asynchronous recording state
UtkarshUsername Sep 5, 2026
486d8e4
fix(server): preserve speech service types
UtkarshUsername Sep 5, 2026
1d63dfb
fix(web): hide unsupported voice input
UtkarshUsername Sep 5, 2026
6c4fe1b
fix: address environment speech review findings
t3-code-bot[bot] Sep 5, 2026
7ec4c05
fix(web): refine voice input composer controls
UtkarshUsername Sep 9, 2026
c1ed314
fix(web): keep voice cancel control in the composer footer
UtkarshUsername Sep 9, 2026
a768ff7
fix(web): keep speech controls aligned in composer footer
UtkarshUsername Sep 9, 2026
55a86c8
fix(voice): preserve integration review fixes
t3-code-bot[bot] Sep 9, 2026
e0fd22c
fix: align speech exports with current unused-code checks
t3-code-bot[bot] Sep 9, 2026
4522d2b
fix: address voice transport lifecycle and composer reviews
t3-code-bot[bot] Sep 9, 2026
9ed5151
fix(server): ignore unrelated speech process messages
UtkarshUsername Sep 10, 2026
3ee3d6c
fix(server): use current speech error schemas
UtkarshUsername Sep 10, 2026
bd2e1f4
fix: preserve voice authentication and running controls
UtkarshUsername Sep 10, 2026
c01377c
test(client-runtime): type voice request fetch arguments
UtkarshUsername Sep 10, 2026
191ef37
feat(speech): add selectable local transcription models
UtkarshUsername Sep 10, 2026
ffcc018
fix(voice): satisfy model checks
UtkarshUsername Sep 10, 2026
589c767
fix(voice): recover model operations safely
UtkarshUsername Sep 10, 2026
d8183b3
refine voice model settings
UtkarshUsername Sep 10, 2026
ed9f6a8
fix(voice): treat download cancellation as success
UtkarshUsername Sep 10, 2026
95b2fdc
docs(voice): describe transcription model selection
UtkarshUsername Sep 10, 2026
3628a70
fix(voice): serialize model progress polling
UtkarshUsername Sep 10, 2026
7ed9c26
refine voice settings search ordering
UtkarshUsername Sep 10, 2026
adbe832
fix(voice): preserve valid active model selection
UtkarshUsername Sep 10, 2026
e91334b
feat(voice): select transcription environment
UtkarshUsername Sep 12, 2026
42d9258
fix: handle voice cancellation errors and clean up test frames
t3-code-bot[bot] Sep 10, 2026
8887e78
refine voice recording controls
UtkarshUsername Sep 12, 2026
337dfac
fix(voice): hide speech controls during confirmation
UtkarshUsername Sep 12, 2026
8506853
fix(voice): keep attachment controls visible during speech
UtkarshUsername Sep 12, 2026
e057eba
fix(web): show voice option labels
UtkarshUsername Sep 12, 2026
35715af
save
UtkarshUsername Sep 12, 2026
b52a84e
feat(voice): connect streaming capture and live transcript previews
UtkarshUsername Sep 12, 2026
6091d58
fix(voice): typecheck
UtkarshUsername Sep 12, 2026
27acb13
fix(voice): preserve streaming review guards
t3-code-bot[bot] Sep 12, 2026
29dd478
fix: discard stale voice model lists and retain ios batch types
t3-code-bot[bot] Sep 12, 2026
f705987
refactor(voice): keep streaming optional
UtkarshUsername Sep 12, 2026
fb28688
fix(voice): decode streamed audio frames exactly
UtkarshUsername Sep 14, 2026
2b73cfc
fix(voice): let streaming inference catch up
UtkarshUsername Sep 14, 2026
275bd67
perf(speech): use accelerated transcription backends
UtkarshUsername Sep 14, 2026
f08ccb8
perf(web): start recording while speech loads
UtkarshUsername Sep 14, 2026
50f877d
perf(speech): report transcription timings
UtkarshUsername Sep 14, 2026
a4b5b32
perf(web): batch speech stream audio
UtkarshUsername Sep 14, 2026
386b1ef
fix(speech): recover from accelerated backend crashes
UtkarshUsername Sep 14, 2026
00f4059
feat(speech): add custom words for local transcription
UtkarshUsername Sep 15, 2026
47172c7
feat(speech): remove filler words from transcriptions
UtkarshUsername Sep 15, 2026
6bc0e23
feat(speech): add configurable transcript post-processing
UtkarshUsername Sep 17, 2026
66c14b5
fix(speech): resolve main integration conflicts
UtkarshUsername Sep 19, 2026
178211a
fix(speech): adapt to current server runtime
UtkarshUsername Sep 19, 2026
8b95d19
test(speech): use managed effect runtime
UtkarshUsername Sep 19, 2026
1038121
fix(mobile): present speech post-processing state
UtkarshUsername Sep 19, 2026
e849ed7
fix(web): generate speech prompt ids through shared utility
UtkarshUsername Sep 19, 2026
d6b39f6
test(speech): provide HTTP layers together
UtkarshUsername Sep 19, 2026
296ca08
fix(web): add sm-pill button size for skip action
UtkarshUsername Sep 19, 2026
0705dee
fix(speech): keep settings access in Effect without runPromise
UtkarshUsername Sep 19, 2026
91482d2
fix(web): move voice post-processing settings to Voice
UtkarshUsername Sep 23, 2026
1b7942c
fix(voice): keep post-processing out of mobile
UtkarshUsername Sep 23, 2026
3e60147
fix(web): use button variants for voice controls
UtkarshUsername Sep 23, 2026
7232024
fix(web): use current model picker props
UtkarshUsername Sep 23, 2026
6762bac
fix(server): preserve streaming speech errors
UtkarshUsername Sep 23, 2026
df811de
fix(web): shorten speech post-processing status
UtkarshUsername Sep 23, 2026
bde040b
feat(web): enable voice post-processing by default
UtkarshUsername Sep 23, 2026
06bd88f
Merge branch 'main' into feat/local-speech-to-text
UtkarshUsername Sep 23, 2026
6607a55
feat(web): set voice processing reasoning default
UtkarshUsername Sep 23, 2026
c073aec
test(server): disable speech post-processing in fallback case
UtkarshUsername Sep 23, 2026
97d7bc2
rename voice custom words to dictionary
UtkarshUsername Sep 23, 2026
cc1c40c
feat(voice): support spoken correction cues
UtkarshUsername Sep 23, 2026
089b1d9
fix(server): encode speech correction cues safely
UtkarshUsername Sep 23, 2026
729844b
fix(speech): preserve domain errors and use Effect service
UtkarshUsername Sep 23, 2026
2f70bbf
feat(speech): recognize the configured correction word
UtkarshUsername Sep 23, 2026
aad8efa
feat(voice): let users choose transcription acceleration
UtkarshUsername Sep 23, 2026
9715736
fix(voice): complete speech HTTP test fixture
UtkarshUsername Sep 23, 2026
734c2a8
fix(voice): ignore unrelated GPU probe messages
UtkarshUsername Sep 23, 2026
ed118e5
fix(voice): preserve audio until stream is ready
UtkarshUsername Sep 23, 2026
e073d1a
feat(web): organize voice settings and model browsing
UtkarshUsername Sep 23, 2026
694a59f
feat(web): order voice languages by speaker count
UtkarshUsername Sep 23, 2026
8158471
feat(web): search voice languages and prioritize local models
UtkarshUsername Sep 23, 2026
ad1bc14
feat(speech): offer Handy transcription model catalog
UtkarshUsername Sep 23, 2026
f3b3e3f
fix(speech): assert catalog fixture exists
UtkarshUsername Sep 23, 2026
a118726
feat(speech): choose transcription language
UtkarshUsername Sep 23, 2026
c9d5ed9
feat(web): search transcription models
UtkarshUsername Sep 23, 2026
639e8e4
fix(web): align voice prompt controls
UtkarshUsername Sep 23, 2026
91823e2
fix(web): align voice model search fields
UtkarshUsername Sep 23, 2026
a7fbc52
fix(web): divide voice model search headers
UtkarshUsername Sep 23, 2026
3305d04
fix(web): remove voice search dividers
UtkarshUsername Sep 23, 2026
aa93514
feat(web): edit voice cleanup presets inline
UtkarshUsername Sep 23, 2026
9888475
fix(speech): raise Koffi stack for Vulkan
UtkarshUsername Sep 24, 2026
4ae55fa
fix(speech): prepare batch models when recording starts
UtkarshUsername Sep 24, 2026
d0880d4
feat(speech): unload idle transcription models
UtkarshUsername Sep 24, 2026
6480c72
fix(voice): clear active state when the last model is deleted
UtkarshUsername Sep 24, 2026
cdf1f21
feat(web): guide first-time voice input setup
UtkarshUsername Sep 24, 2026
20f6293
feat(web): choose microphone during voice setup
UtkarshUsername Sep 24, 2026
cdd8257
fix(voice): resume interrupted model downloads
UtkarshUsername Sep 24, 2026
87be8e2
fix(web): show retry after voice model download fails
UtkarshUsername Sep 24, 2026
14fd7a9
fix(web): show voice setup download progress
UtkarshUsername Sep 24, 2026
9825a3a
fix(web): keep voice setup settings links in content
UtkarshUsername Sep 24, 2026
b09f2e4
feat(web): sort speech models by editorial rank
UtkarshUsername Sep 24, 2026
fa77d49
feat(web): sort unranked speech models by quality
UtkarshUsername Sep 24, 2026
7cbe6bd
merge: integrate latest main into speech branch
UtkarshUsername Sep 24, 2026
4f38551
fix(voice): satisfy lint and speech contract checks
UtkarshUsername Sep 24, 2026
31c40f8
test(web): keep settings search ranking assertion focused
UtkarshUsername Sep 24, 2026
5211636
perf(speech): pipeline bounded streaming audio
UtkarshUsername Sep 24, 2026
ab65f0d
fix(voice): preserve meaning in transcript cleanup prompt
UtkarshUsername Sep 25, 2026
e577b5e
fix(voice): clarify explicit correction cue settings
UtkarshUsername Sep 25, 2026
6d19bc8
feat(voice): add dictionary aliases for speech transcription
UtkarshUsername Sep 25, 2026
bb094e3
fix(voice): preserve transcript whitespace during fuzzy correction
UtkarshUsername Sep 25, 2026
457b19e
merge: integrate latest main into speech branch
UtkarshUsername Sep 25, 2026
a4a2d46
feat(web): simplify voice dictionary editing
UtkarshUsername Sep 25, 2026
beff56e
fix(web): clarify voice dictionary hints
UtkarshUsername Sep 25, 2026
9b39959
feat(web): collapse voice dictionary entries
UtkarshUsername Sep 25, 2026
3b0cd87
feat(web): add dictation keyboard shortcuts
UtkarshUsername Sep 25, 2026
dba3591
fix(web): clarify dictation shortcut setting
UtkarshUsername Sep 25, 2026
165319b
fix(web): group voice input settings
UtkarshUsername Sep 25, 2026
367aa6b
fix(web): keep dictation cancel contextual
UtkarshUsername Sep 25, 2026
ff66a2f
fix(server): reuse speech model after canceling a stream
UtkarshUsername Sep 26, 2026
3697e88
fix(web): start dictation after cancellation cleanup
UtkarshUsername Sep 26, 2026
f82b586
fix(server): bound speech stream reset and release early-cancelled st…
UtkarshUsername Sep 26, 2026
12aee36
fix(server): give speech stream reset headroom over feed drain
UtkarshUsername Sep 26, 2026
6f7ba65
fix(web): queue dictation start until previous stream drains
UtkarshUsername Sep 26, 2026
874abb9
fix(server): free cancelled batch transcription and stop native work
UtkarshUsername Sep 26, 2026
64d5517
fix(server): keep cancelled batch work warm, preempt on next use
UtkarshUsername Sep 26, 2026
49e7063
merge: update voice transcription branch from main
t3-code-bot[bot] Sep 26, 2026
ce865f8
fix(voice): recover failed workers and reset replaced controllers
t3-code-bot[bot] Sep 26, 2026
51843c5
fix(voice): use effect deadlines and stabilize keybinding order
t3-code-bot[bot] Sep 26, 2026
69d1592
refactor(voice): compose cleanup deadlines in the caller effect
t3-code-bot[bot] Sep 26, 2026
dc1c87f
fix(voice): retain ownership until cancelled warmup settles
t3-code-bot[bot] Sep 26, 2026
41182f7
fix(voice): scope unload timers and clean up uninitialized streams
t3-code-bot[bot] Sep 26, 2026
8dc1ac1
refactor(voice): remove compatibility for obsolete response shapes
t3-code-bot[bot] Sep 26, 2026
7551a3c
feat(web): add bulk voice dictionary entry
UtkarshUsername Sep 27, 2026
2904ebe
feat(web): show dictionary words as editable tags
UtkarshUsername Sep 27, 2026
41a8eed
refactor(voice): replace prompt presets with custom instructions
UtkarshUsername Sep 27, 2026
aaf2b3a
fix(voice): keep built-in prompt until custom instructions exist
UtkarshUsername Sep 28, 2026
35a444a
feat(voice): translate speech to English
UtkarshUsername Sep 28, 2026
8cb7d21
fix(voice): avoid nested reset runtime
UtkarshUsername Sep 28, 2026
1be5b71
fix(voice): allow adapter reset timer
UtkarshUsername Sep 28, 2026
16797fe
feat(voice): include composer context in transcription cleanup
UtkarshUsername Sep 29, 2026
eab0d57
refactor(voice): remove transcript output placeholder
UtkarshUsername Sep 29, 2026
b048c3c
feat(voice): use project names for transcription vocabulary
UtkarshUsername Sep 29, 2026
f75eef1
fix(voice): preserve tagged inference errors
UtkarshUsername Sep 29, 2026
61f598d
refactor(voice): yield tagged inference errors directly
UtkarshUsername Sep 29, 2026
8471068
docs(voice): mention automatic project name hint
UtkarshUsername Sep 29, 2026
630f94d
fix(voice): use composer terms to correct transcriptions
UtkarshUsername Sep 29, 2026
b057ad3
fix(voice): use composer context for transcript ambiguity
UtkarshUsername Sep 29, 2026
e63c320
fix(voice): describe transcript insertion at cursor
UtkarshUsername Sep 29, 2026
b2703f1
refactor(web): improve transcription model cards
UtkarshUsername Sep 29, 2026
fd39ea3
chore: merge latest main into voice transcription branch
UtkarshUsername Sep 30, 2026
a7e6854
fix(voice): support transcription processing with managed Codex
UtkarshUsername Sep 30, 2026
520c638
feat(web): add dictation to agent comment editors
UtkarshUsername Sep 30, 2026
721b17b
test(desktop): signal annotation receipts without a runtime
UtkarshUsername Sep 30, 2026
b4b244d
refactor(shared): keep shortcut matching private
UtkarshUsername Sep 30, 2026
d79905f
fix(shared): preserve Apple keyboard modifier matching
UtkarshUsername Sep 30, 2026
eca8b9b
fix(desktop): keep annotation preloads self-contained
UtkarshUsername Sep 30, 2026
5e695e4
fix(web): enable the mic for custom agent answers
UtkarshUsername Sep 30, 2026
7c592e9
fix(desktop): match annotation voice controls to the composer
UtkarshUsername Sep 30, 2026
36234ae
fix(web): preserve upstream usage shortcut ordering
UtkarshUsername Sep 30, 2026
e469575
fix(desktop): show the annotation microphone button
UtkarshUsername Oct 1, 2026
e69298c
fix(web): update voice waveforms without waiting for transcription ch…
UtkarshUsername Oct 1, 2026
4e663a4
fix(desktop): match the annotation microphone icon
UtkarshUsername Oct 1, 2026
0800689
fix(desktop): show dictation shortcuts in annotation tooltips
UtkarshUsername Oct 1, 2026
673a9f5
feat(web): add dictation to pull request comments and reviews
UtkarshUsername Oct 1, 2026
a42f40c
fix(web): group pull request composer actions
UtkarshUsername Oct 1, 2026
796f24f
fix(web): close citation popover during voice setup
UtkarshUsername Oct 1, 2026
fe0c27b
test(web): include voice setup state in citation mock
UtkarshUsername Oct 1, 2026
fc84dc6
fix(web): clarify voice input setup
UtkarshUsername Oct 1, 2026
5d90672
fix(web): simplify correction cue punctuation
UtkarshUsername Oct 1, 2026
cc23927
fix(web): smooth voice waveform levels
UtkarshUsername Oct 1, 2026
113758a
fix(voice): add provider defaults and fallback for transcript cleanup
UtkarshUsername Oct 1, 2026
e46b76e
fix(voice): handle absent provider defaults in fallback resolution
UtkarshUsername Oct 1, 2026
53b712c
fix(voice): match text generation provider fallback order
UtkarshUsername Oct 1, 2026
b2a2040
fix(voice): use a branded provider ID in fallback test
UtkarshUsername Oct 1, 2026
1235ace
feat(web): add microphone recording and playback test
UtkarshUsername Oct 2, 2026
1b54266
chore: merge latest main into voice transcription branch
UtkarshUsername Oct 2, 2026
8bfa6b4
feat(web): add voice model transcription test
UtkarshUsername Oct 2, 2026
51709ce
fix(web): show recording dot in microphone test
UtkarshUsername Oct 2, 2026
0011e45
feat(web): add All to transcription model language filters
UtkarshUsername Oct 2, 2026
9760ec1
test(web): cover browsing all transcription models
UtkarshUsername Oct 2, 2026
ebb115a
feat(voice): add project-specific dictionary words
UtkarshUsername Oct 2, 2026
0d90b62
test(server): update expected voice CORS headers
UtkarshUsername Oct 2, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/desktop/src/ipc/channels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,6 @@ export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input"

export const RECEIVE_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:receive-provider-auth-callback";
export const CANCEL_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:cancel-provider-auth-callback";

export const PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL = "desktop:preview-annotation-voice-event";
export const PREVIEW_ANNOTATION_VOICE_STATE_CHANNEL = "desktop:preview-annotation-voice-state";
19 changes: 16 additions & 3 deletions apps/desktop/src/ipc/methods/preview.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import {
DesktopPreviewAnnotationThemeInputSchema,
DesktopPreviewAnnotationVoiceInputSchema,
DesktopPreviewAnnotationVoiceStateSchema,
DesktopPreviewArtifactInputSchema,
DesktopPreviewAutomationClickInputSchema,
DesktopPreviewAutomationEvaluateInputSchema,
Expand Down Expand Up @@ -354,11 +356,11 @@ export const setAnnotationTheme = DesktopIpc.makeIpcMethod({

export const pickElement = DesktopIpc.makeIpcMethod({
channel: IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL,
payload: DesktopPreviewTabInputSchema,
payload: DesktopPreviewAnnotationVoiceInputSchema,
result: Schema.NullOr(PreviewAnnotationSubmissionResultSchema),
handler: Effect.fn("desktop.ipc.preview.pickElement")(function* ({ tabId }) {
handler: Effect.fn("desktop.ipc.preview.pickElement")(function* ({ tabId, voice }) {
const manager = yield* PreviewManager.PreviewManager;
return yield* manager.pickElement(tabId);
return yield* manager.pickElement(tabId, voice);
}),
});

Expand Down Expand Up @@ -482,7 +484,18 @@ export const saveRecording = DesktopIpc.makeIpcMethod({
}),
});

export const updateAnnotationVoice = DesktopIpc.makeIpcMethod({
channel: IpcChannels.PREVIEW_ANNOTATION_VOICE_STATE_CHANNEL,
payload: DesktopPreviewAnnotationVoiceStateSchema,
result: Schema.Void,
handler: Effect.fn("desktop.ipc.preview.updateAnnotationVoice")(function* (state) {
const manager = yield* PreviewManager.PreviewManager;
yield* manager.updateAnnotationVoice(state);
}),
});

export const methods = [
updateAnnotationVoice,
createTab,
closeTab,
registerWebview,
Expand Down
16 changes: 15 additions & 1 deletion apps/desktop/src/preload.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type {
DesktopBridge,
DesktopPreviewAnnotationVoiceEvent,
DesktopPreviewPointerEvent,
DesktopPreviewRecordingInputEvent,
DesktopPreviewRecordingFrame,
Expand Down Expand Up @@ -321,7 +322,20 @@ contextBridge.exposeInMainWorld("desktopBridge", {
ipcRenderer.invoke(IpcChannels.PREVIEW_GET_CONFIG_CHANNEL, { environmentId, profileId }),
setAnnotationTheme: (theme) =>
ipcRenderer.invoke(IpcChannels.PREVIEW_SET_ANNOTATION_THEME_CHANNEL, { theme }),
pickElement: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, { tabId }),
annotationVoice: {
onEvent: (listener) => {
const onEvent = (_event: Electron.IpcRendererEvent, tabId: string, value: unknown) => {
if (typeof value !== "object" || value === null) return;
listener(tabId, value as DesktopPreviewAnnotationVoiceEvent);
};
ipcRenderer.on(IpcChannels.PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL, onEvent);
return () => ipcRenderer.off(IpcChannels.PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL, onEvent);
},
update: (state) =>
ipcRenderer.invoke(IpcChannels.PREVIEW_ANNOTATION_VOICE_STATE_CHANNEL, state),
},
pickElement: (tabId, voice) =>
ipcRenderer.invoke(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, { tabId, voice }),
cancelPickElement: (tabId) =>
ipcRenderer.invoke(IpcChannels.PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL, { tabId }),
captureScreenshot: (tabId) =>
Expand Down
6 changes: 6 additions & 0 deletions apps/desktop/src/preview/Annotation.css
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
--t3-popover-foreground: oklch(0.269 0 0);
--t3-primary: oklch(0.488 0.217 264);
--t3-primary-foreground: white;
--t3-destructive: oklch(0.577 0.245 27.325);
--t3-muted: rgb(0 0 0 / 4%);
--t3-muted-foreground: oklch(0.556 0 0);
--t3-accent: rgb(0 0 0 / 4%);
Expand Down Expand Up @@ -66,3 +67,8 @@ textarea:focus-visible {
outline: 2px solid color-mix(in srgb, var(--t3-ring) 72%, transparent);
outline-offset: 1px;
}

/* Utility display classes must not override the native hidden state. */
[hidden] {
display: none !important;
}
2 changes: 1 addition & 1 deletion apps/desktop/src/preview/AnnotationStyles.generated.ts

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions apps/desktop/src/preview/GuestProtocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ export const RECORDING_POINTER_CHANNEL = "preview:recording-pointer";
export const RECORDING_KEY_CHANNEL = "preview:recording-key";
export const RECORDING_INPUT_CHANNEL = "preview:recording-input";
export const RECORDING_CONTROLLER_CHANNEL = "preview:recording-controller";

export const ANNOTATION_VOICE_EVENT_CHANNEL = "preview:annotation-voice-event";
export const ANNOTATION_VOICE_STATE_CHANNEL = "preview:annotation-voice-state";
65 changes: 65 additions & 0 deletions apps/desktop/src/preview/Manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3810,6 +3810,71 @@ describe("PreviewManager", () => {
),
);

effectIt.effect("annotation voice updates cannot reach a replaced or cancelled pick", () =>
withManager((manager) =>
Effect.gen(function* () {
const firstReady = yield* Deferred.make<void>();
const secondReady = yield* Deferred.make<void>();
let registrations = 0;
const guest = {
...makeTestPreviewWebContents(async () => ({
toPNG: () => Buffer.from("unused"),
toJPEG: () => Buffer.from("unused"),
getSize: () => ({ width: 1, height: 1 }),
})),
isFocused: () => true,
focus: vi.fn(),
send: (channel: string, ...args: unknown[]) => {
webviewSend(channel, ...args);
if (channel === "preview:start-pick") {
Deferred.doneUnsafe(registrations++ === 0 ? firstReady : secondReady, Effect.void);
}
},
once: vi.fn(),
ipc: { on: vi.fn(), off: vi.fn(), removeListener: vi.fn() },
};
fromId.mockReturnValue(guest as never);
yield* manager.createTab("tab_1");
yield* manager.registerWebview("tab_1", 42);
const first = yield* manager
.pickElement("tab_1", { sessionId: "first", keybindings: [] })
.pipe(Effect.forkChild);
yield* Deferred.await(firstReady);
const state = {
tabId: "tab_1",
sessionId: "first",
available: true,
phase: "recording" as const,
status: "Recording",
errorAction: null,
preview: null,
freezesEditor: true,
blocksSubmission: true,
level: 0,
shortcutLabel: "Ctrl+D",
draft: { text: "Tighten spacing", cursor: 15 },
};
yield* manager.updateAnnotationVoice(state);
expect(webviewSend).toHaveBeenCalledWith("preview:annotation-voice-state", state);
const second = yield* manager
.pickElement("tab_1", { sessionId: "second", keybindings: [] })
.pipe(Effect.forkChild);
yield* Deferred.await(secondReady);
expect(yield* Fiber.join(first)).toBeNull();
webviewSend.mockClear();
yield* manager.updateAnnotationVoice(state);
expect(webviewSend).not.toHaveBeenCalled();
yield* manager.updateAnnotationVoice({ ...state, sessionId: "second" });
expect(webviewSend).toHaveBeenCalledOnce();
yield* manager.cancelPickElement("tab_1");
expect(yield* Fiber.join(second)).toBeNull();
webviewSend.mockClear();
yield* manager.updateAnnotationVoice({ ...state, sessionId: "second" });
expect(webviewSend).not.toHaveBeenCalled();
}),
),
);

effectIt.effect("a stale capture from a replaced pick never touches the next pick", () =>
withManager((manager) =>
Effect.gen(function* () {
Expand Down
68 changes: 64 additions & 4 deletions apps/desktop/src/preview/Manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@
import * as NodeCrypto from "node:crypto";
import {
DesktopPreviewRecordingInputSchema,
DesktopPreviewAnnotationVoiceEventSchema,
DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER,
} from "@t3tools/contracts";
import type {
DesktopPreviewAnnotationTheme,
DesktopPreviewAnnotationVoiceConfig,
DesktopPreviewAnnotationVoiceState,
DesktopPreviewAutomationStatus,
DesktopPreviewColorScheme,
DesktopPreviewFavicon,
Expand Down Expand Up @@ -66,10 +69,15 @@ import * as Scope from "effect/Scope";
import * as SynchronizedRef from "effect/SynchronizedRef";

import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
import { PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "../ipc/channels.ts";
import {
PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL,
PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL,
} from "../ipc/channels.ts";
import * as BrowserSession from "./BrowserSession.ts";
import {
ANNOTATION_CAPTURED_CHANNEL,
ANNOTATION_VOICE_EVENT_CHANNEL,
ANNOTATION_VOICE_STATE_CHANNEL,
ANNOTATION_THEME_CHANNEL,
CANCEL_PICK_CHANNEL,
ELEMENT_PICKED_CHANNEL,
Expand All @@ -92,6 +100,8 @@ import {
import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts";
import { DEFAULT_RECORDING_INPUT_OPTIONS, type RecordingInputOptions } from "./RecordingInput.ts";

const decodeAnnotationVoice = Schema.decodeUnknownOption(DesktopPreviewAnnotationVoiceEventSchema);

export type PreviewNavStatus =
| { kind: "Idle" }
| { kind: "Loading"; url: string; title: string }
Expand Down Expand Up @@ -470,6 +480,7 @@ interface PendingRecording {
}

interface PickSession {
readonly voiceSessionId?: string | undefined;
readonly cancel: Effect.Effect<void>;
}

Expand Down Expand Up @@ -2578,18 +2589,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
);
});

const pickElement = Effect.fn("PreviewManager.pickElement")(function* (tabId: string) {
const updateAnnotationVoice = Effect.fn("PreviewManager.updateAnnotationVoice")(function* (
state: DesktopPreviewAnnotationVoiceState,
) {
const sessions = yield* Ref.get(pickSessionsRef);
if (sessions.get(state.tabId)?.voiceSessionId !== state.sessionId) return;
const wc = yield* requireWebContents(state.tabId);
yield* attempt({ operation: "annotationVoice.update", tabId: state.tabId }, () =>
wc.send(ANNOTATION_VOICE_STATE_CHANNEL, state),
);
});

const pickElement = Effect.fn("PreviewManager.pickElement")(function* (
tabId: string,
voice?: DesktopPreviewAnnotationVoiceConfig,
) {
const wc = yield* requireWebContents(tabId);
yield* cancelPickElement(tabId);
const annotationTheme = yield* Ref.get(annotationThemeRef);
return yield* Effect.callback<PreviewAnnotationSubmissionResult | null, PreviewManagerError>(
(resume) => {
// Declared first so cleanup can check slot ownership by identity
// without a type cycle through the cancel effect it builds.
const session: PickSession = { cancel: Effect.suspend(() => cancelPickSession()) };
const session: PickSession = {
voiceSessionId: voice?.sessionId,
cancel: Effect.suspend(() => cancelPickSession()),
};
const cleanup = Effect.fn("PreviewManager.cleanupPickElement")(function* () {
yield* attempt({ operation: "pickElement.cleanup", tabId, webContentsId: wc.id }, () => {
wc.ipc.removeListener(ELEMENT_PICKED_CHANNEL, onMessage);
wc.ipc.removeListener(ANNOTATION_VOICE_EVENT_CHANNEL, onVoiceEvent);
wc.off("destroyed", onDestroyed);
wc.off("did-start-navigation", onNavigated);
}).pipe(Effect.ignore);
Expand Down Expand Up @@ -2687,6 +2716,30 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
),
);
};
const onVoiceEvent = (_event: Electron.IpcMainEvent, value: unknown) => {
const decoded = decodeAnnotationVoice(value);
if (
settled ||
!voice ||
Option.isNone(decoded) ||
decoded.value.sessionId !== voice.sessionId
)
return;
runFork(
Effect.gen(function* () {
const window = yield* Ref.get(mainWindowRef);
if (Option.isSome(window) && !window.value.isDestroyed()) {
yield* attempt({ operation: "annotationVoice.forward", tabId }, () =>
window.value.webContents.send(
PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL,
tabId,
decoded.value,
),
).pipe(Effect.ignore);
}
}),
);
};
const onDestroyed = () => settle(null);
const onNavigated = (
event: Electron.Event<Electron.WebContentsDidStartNavigationEventParams>,
Expand All @@ -2711,10 +2764,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
if (settled) return;
yield* attempt({ operation: "pickElement.register", tabId, webContentsId: wc.id }, () => {
wc.ipc.on(ELEMENT_PICKED_CHANNEL, onMessage);
wc.ipc.on(ANNOTATION_VOICE_EVENT_CHANNEL, onVoiceEvent);
wc.once("destroyed", onDestroyed);
wc.on("did-start-navigation", onNavigated);
if (!wc.isFocused()) wc.focus();
wc.send(START_PICK_CHANNEL, annotationTheme);
wc.send(START_PICK_CHANNEL, annotationTheme, voice);
});
});
runFork(
Expand Down Expand Up @@ -4675,6 +4729,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
openPictureInPicture,
openDevTools,
pickElement,
updateAnnotationVoice,
reapplyZoom,
refresh,
registerWebview,
Expand Down Expand Up @@ -5047,7 +5102,11 @@ export class PreviewManager extends Context.Service<
) => Effect.Effect<void, PreviewManagerError>;
readonly pickElement: (
tabId: string,
voice?: DesktopPreviewAnnotationVoiceConfig,
) => Effect.Effect<PreviewAnnotationSubmissionResult | null, PreviewManagerError>;
readonly updateAnnotationVoice: (
state: DesktopPreviewAnnotationVoiceState,
) => Effect.Effect<void, PreviewManagerError>;
readonly cancelPickElement: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
readonly captureScreenshot: (
tabId: string,
Expand Down Expand Up @@ -5178,6 +5237,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
),
setAnnotationTheme: operations.setAnnotationTheme,
pickElement: operations.pickElement,
updateAnnotationVoice: operations.updateAnnotationVoice,
cancelPickElement: operations.cancelPickElement,
captureScreenshot: operations.captureScreenshot,
revealArtifact: operations.revealArtifact,
Expand Down
Loading
Loading