Skip to content

Commit 6257c11

Browse files
donjor-agent[bot]donjor
authored andcommitted
feat(desktop): record dictation into editable chat drafts (#1675)
Desktop dictation through a selected microphone and external speech endpoint. Stop to insert editable text; X or Escape cancels. Uses mobile's layout and motion, existing desktop controls, and a dedicated Dictation settings page. **Recording and cancellation** https://github.com/user-attachments/assets/7c86ac75-c643-44d2-8eab-9e7504e5fab4 **Compact** ![Compact recording](https://github.com/user-attachments/assets/84bf8925-6d15-4d2f-ba77-0873341926ea) **Expanded** ![Draft remains visible while recording](https://github.com/user-attachments/assets/bbcfdd8a-87a0-4dda-8233-c6c801d33c85) **Settings: before** ![Previous settings](https://github.com/user-attachments/assets/f51525d1-ef70-4fcb-bb07-ca337add9a7f) **Settings: after** ![Saved key uses dots and inline removal](https://github.com/user-attachments/assets/7fabf71e-9d5b-42c3-8831-e969609369dc) **Verified:** 11 recorder/settings tests, web typecheck/build, targeted lint, fork CI, and Electron interactions. Real microphone capture; fixture transcription. Reduced motion supported. Live Meta recognition and packaged macOS permission remain untested. Evidence: `9fd003ebc6683b15dc066baba8748b84c8e798b0`; clip is 4.97s, composer crop only. Final head: `79fffbab2acd1ca4b725a6318afcc46c21a32719`. Rebased onto current `hyprws`; dictation source is unchanged, with a user-guide correction. Model: GPT-6.1-sol; harness: Codex in T3 Code. Squashes: - 19c9a0e feat(desktop): record dictation into editable chat drafts (#1675) (#1675) - cebe9ea fixup! feat(desktop): record dictation into editable chat drafts (#1675) (#1675) - 254439e1ca feat(desktop): record dictation into editable chat drafts (#1675) (#1675) - 1c650938cc fixup! feat(desktop): record dictation into editable chat drafts (#1675) (#1675) - da09a65f2f feat(desktop): record dictation into editable chat drafts (#1675) (#1675) - 6d649ab186 fixup! feat(desktop): record dictation into editable chat drafts (#1675) (#1675) - 4f0886d feat(desktop): record dictation into editable chat drafts (#1675) (#1675) - c1186aa fixup! feat(desktop): record dictation into editable chat drafts (#1675) (#1675) Fork-Domain: voice-input <!-- Keep the fork trailer block contiguous. --> Fork-Tier: core Co-authored-by: donjor <[email protected]>
1 parent 24529cb commit 6257c11

30 files changed

Lines changed: 1755 additions & 75 deletions

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,7 @@ Full docs live in [docs/](./docs). There's no docs site yet.
296296
- [Browser bookmarks](./docs/fork/user/browser.md)
297297
- [Project settings](./docs/user/project-settings.md)
298298
- [Appearance preferences](./docs/user/appearance.md)
299+
- [Desktop dictation](./docs/fork/user/dictation.md)
299300
- [Remote access from a phone or another machine](./docs/user/remote-access.md)
300301
- [Connect Claude Code, Codex, ChatGPT and other agents over MCP](./docs/user/outside-agents.md)
301302
- [Keeping app and server in sync](./docs/user/updating.md)

‎apps/desktop/scripts/electron-launcher.mjs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import * as NodeOS from "node:os";
55
import * as NodePath from "node:path";
66
import * as NodeURL from "node:url";
77
import { ensureElectronRuntime } from "./ensure-electron-runtime.mjs";
8+
import { voiceInputMacPurposeFork } from "./voice-input.fork.mjs"; // fork-hook: voice-input/dev-mic-import
89

910
const isDevelopment = Boolean(process.env.VITE_DEV_SERVER_URL);
1011
const __dirname = NodePath.dirname(NodeURL.fileURLToPath(import.meta.url));
@@ -270,6 +271,7 @@ function resolveMacBundleInfoPlistStrings(executableName) {
270271
NSScreenCaptureUsageDescription:
271272
"T3 Code captures the active window when you use the snapshot shortcut.",
272273
NSDocumentsFolderUsageDescription: "T3 Code reads project files you open in the desktop app.",
274+
...voiceInputMacPurposeFork, // fork-hook: voice-input/dev-mic-purpose
273275
};
274276
}
275277

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
export const voiceInputMacPurposeFork = Object.freeze({
2+
NSMicrophoneUsageDescription: "T3 Code records speech when you start dictation.",
3+
});

‎apps/server/src/voice-input/http.fork.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,13 @@ describe("dictation HTTP boundary", () => {
5353
it.effect("refuses unauthenticated audio", () =>
5454
Effect.gen(function* () {
5555
const h = yield* harness([], false);
56-
expect((yield* Effect.promise(() => h.handler(upload()))).status).toBe(401);
56+
const response = yield* Effect.promise(() => h.handler(upload()));
57+
expect(response.status).toBe(401);
58+
expect(yield* Effect.promise(() => response.json())).toMatchObject({
59+
_tag: "EnvironmentAuthInvalidError",
60+
code: "auth_invalid",
61+
reason: "missing_credential",
62+
});
5763
expect(h.calls()).toBe(0);
5864
}),
5965
);
@@ -96,7 +102,7 @@ describe("dictation HTTP boundary", () => {
96102
const response = yield* Effect.promise(() =>
97103
h.handler(
98104
new Request(`http://env.local${VOICE_INPUT_ROUTE_FORK}/settings`, {
99-
method: "PUT",
105+
method: "POST",
100106
headers: { "content-type": "application/json" },
101107
body: "{}",
102108
}),

‎apps/server/src/voice-input/http.fork.ts‎

Lines changed: 65 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@ import * as Layer from "effect/Layer";
1212
import * as Schema from "effect/Schema";
1313
import * as Stream from "effect/Stream";
1414
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http";
15+
import * as HttpServerRespondable from "effect/http/HttpServerRespondable";
1516

1617
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
1718
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
1819
import * as VoiceInput from "./VoiceInput.fork.ts";
20+
import { failEnvironmentAuthInvalid } from "../auth/http.ts";
1921
const isVoiceInputError = Schema.is(VoiceInput.VoiceInputError);
2022
const decodeSettingsJson = Schema.decodeEffect(Schema.fromJsonString(VoiceInputConfigUpdateFork));
2123

@@ -24,56 +26,52 @@ const json = (body: unknown, status = 200) =>
2426
status,
2527
headers: { "cache-control": "no-store" },
2628
});
27-
const authorize = (scope: AuthEnvironmentScope) =>
28-
Effect.gen(function* () {
29-
const request = yield* HttpServerRequest.HttpServerRequest;
30-
const auth = yield* EnvironmentAuth.EnvironmentAuth;
31-
return yield* auth.authenticateHttpRequest(request).pipe(
32-
Effect.map((session) =>
33-
session.scopes.includes(scope)
34-
? null
35-
: json({ error: "Permission required.", requiredScope: scope }, 403),
36-
),
37-
Effect.catch((error) =>
38-
Effect.succeed(
39-
json(
40-
{ error: "Authentication failed." },
41-
EnvironmentAuth.isServerAuthCredentialError(error) ? 401 : 500,
42-
),
43-
),
44-
),
45-
);
46-
});
29+
const authorize = Effect.fn("http.authorize")(function* (scope: AuthEnvironmentScope) {
30+
const request = yield* HttpServerRequest.HttpServerRequest;
31+
const auth = yield* EnvironmentAuth.EnvironmentAuth;
32+
return yield* auth.authenticateHttpRequest(request).pipe(
33+
Effect.map((session) =>
34+
session.scopes.includes(scope)
35+
? null
36+
: json({ error: "Permission required.", requiredScope: scope }, 403),
37+
),
38+
Effect.catch((error) =>
39+
EnvironmentAuth.isServerAuthCredentialError(error)
40+
? failEnvironmentAuthInvalid(
41+
EnvironmentAuth.serverAuthCredentialReason(error),
42+
EnvironmentAuth.serverAuthDpopFailureReason(error),
43+
).pipe(Effect.catch(HttpServerRespondable.toResponse))
44+
: Effect.succeed(json({ error: "Authentication failed." }, 500)),
45+
),
46+
);
47+
});
4748

4849
/** Bound untrusted uploads before allocating the joined recording. */
49-
const readBody = (maxBytes: number) =>
50-
Effect.gen(function* () {
51-
const request = yield* HttpServerRequest.HttpServerRequest;
52-
const body = yield* Stream.runFoldEffect(
53-
request.stream,
54-
() => ({ size: 0, chunks: [] as Array<Uint8Array> }),
55-
(body, chunk) => {
56-
if (body.size + chunk.byteLength > maxBytes)
57-
return Effect.fail(new VoiceInput.VoiceInputError({ reason: "audio" }));
58-
body.size += chunk.byteLength;
59-
body.chunks.push(chunk);
60-
return Effect.succeed(body);
61-
},
62-
).pipe(
63-
Effect.mapError((cause) =>
64-
isVoiceInputError(cause)
65-
? cause
66-
: new VoiceInput.VoiceInputError({ reason: "audio", cause }),
67-
),
68-
);
69-
const bytes = new Uint8Array(body.size);
70-
let at = 0;
71-
for (const chunk of body.chunks) {
72-
bytes.set(chunk, at);
73-
at += chunk.byteLength;
74-
}
75-
return bytes;
76-
});
50+
const readBody = Effect.fn("http.readBody")(function* (maxBytes: number) {
51+
const request = yield* HttpServerRequest.HttpServerRequest;
52+
const body = yield* Stream.runFoldEffect(
53+
request.stream,
54+
() => ({ size: 0, chunks: [] as Array<Uint8Array> }),
55+
(body, chunk) => {
56+
if (body.size + chunk.byteLength > maxBytes)
57+
return Effect.fail(new VoiceInput.VoiceInputError({ reason: "audio" }));
58+
body.size += chunk.byteLength;
59+
body.chunks.push(chunk);
60+
return Effect.succeed(body);
61+
},
62+
).pipe(
63+
Effect.mapError((cause) =>
64+
isVoiceInputError(cause) ? cause : new VoiceInput.VoiceInputError({ reason: "audio", cause }),
65+
),
66+
);
67+
const bytes = new Uint8Array(body.size);
68+
let at = 0;
69+
for (const chunk of body.chunks) {
70+
bytes.set(chunk, at);
71+
at += chunk.byteLength;
72+
}
73+
return bytes;
74+
});
7775

7876
const failure = (error: VoiceInput.VoiceInputError) =>
7977
Effect.succeed(
@@ -87,6 +85,22 @@ const failure = (error: VoiceInput.VoiceInputError) =>
8785
),
8886
);
8987

88+
const configureSettings = (service: VoiceInput.VoiceInput["Service"]) =>
89+
Effect.gen(function* () {
90+
const refusal = yield* authorize(AuthSettingsWriteScope);
91+
if (refusal) return refusal;
92+
const request = yield* HttpServerRequest.HttpServerRequest;
93+
if (request.headers["content-type"]?.split(";")[0]?.trim() !== "application/json")
94+
return json({ error: "Expected JSON settings." }, 415);
95+
return yield* readBody(16384).pipe(
96+
Effect.flatMap((bytes) => decodeSettingsJson(new TextDecoder().decode(bytes))),
97+
Effect.mapError((cause) => new VoiceInput.VoiceInputError({ reason: "settings", cause })),
98+
Effect.flatMap(service.configure),
99+
Effect.map((settings) => json(settings)),
100+
Effect.catch(failure),
101+
);
102+
});
103+
90104
export const routes = (service: VoiceInput.VoiceInput["Service"]) =>
91105
Layer.mergeAll(
92106
HttpRouter.add(
@@ -101,24 +115,9 @@ export const routes = (service: VoiceInput.VoiceInput["Service"]) =>
101115
);
102116
}),
103117
),
104-
HttpRouter.add(
105-
"PUT",
106-
`${VOICE_INPUT_ROUTE_FORK}/settings`,
107-
Effect.gen(function* () {
108-
const refusal = yield* authorize(AuthSettingsWriteScope);
109-
if (refusal) return refusal;
110-
const request = yield* HttpServerRequest.HttpServerRequest;
111-
if (request.headers["content-type"]?.split(";")[0]?.trim() !== "application/json")
112-
return json({ error: "Expected JSON settings." }, 415);
113-
return yield* readBody(16384).pipe(
114-
Effect.flatMap((bytes) => decodeSettingsJson(new TextDecoder().decode(bytes))),
115-
Effect.mapError((cause) => new VoiceInput.VoiceInputError({ reason: "settings", cause })),
116-
Effect.flatMap(service.configure),
117-
Effect.map((settings) => json(settings)),
118-
Effect.catch(failure),
119-
);
120-
}),
121-
),
118+
HttpRouter.add("PUT", `${VOICE_INPUT_ROUTE_FORK}/settings`, configureSettings(service)),
119+
// POST follows the existing browser/desktop CORS policy; PUT remains compatible.
120+
HttpRouter.add("POST", `${VOICE_INPUT_ROUTE_FORK}/settings`, configureSettings(service)),
122121
HttpRouter.add(
123122
"POST",
124123
`${VOICE_INPUT_ROUTE_FORK}/transcribe`,

‎apps/web/src/components/chat/ChatComposer.tsx‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText";
44
import { runtimeModeConfig, runtimeModeOptions as runtimeModes } from "./runtimeModeConfig";
55
import { isLocalEnvironmentDisabled } from "../../localEnvironment";
66
import { usePrimaryEnvironmentId } from "../../state/environments";
7+
import { useDesktopVoiceInputFork } from "../../voice-input/useVoiceInput.fork"; // fork-hook: voice-input/composer-import
8+
import { VoiceComposerFooterFork } from "../../voice-input/VoiceComposerControls.fork"; // fork-hook: voice-input/footer-import
79
import { useRightPanelStore } from "~/rightPanelStore";
810
import { importPastedComposerText, readPastedComposerContext } from "../composerInlineTokenPaste";
911
import {
@@ -945,6 +947,24 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
945947
selectedProviderEntry?.snapshot,
946948
selectedModel,
947949
);
950+
const desktopVoiceFork = useDesktopVoiceInputFork({
951+
environmentId,
952+
ownerKey: composerDraftTargetKey,
953+
enabled: canOperateThread && !activePendingProgress && pendingApprovals.length === 0,
954+
readDraft: () => ({
955+
ownerKey: composerDraftTargetKeyRef.current,
956+
text: promptRef.current,
957+
selection: composerEditorRef.current?.readSelectionRange() ?? {
958+
start: promptRef.current.length,
959+
end: promptRef.current.length,
960+
},
961+
}),
962+
commitDraft: (text, cursor) => {
963+
const collapsedCursor = collapseExpandedComposerCursor(text, cursor);
964+
onPromptChange(text, collapsedCursor, cursor, false, collectInlineContextIds(text));
965+
requestAnimationFrame(() => composerEditorRef.current?.focusAt(collapsedCursor));
966+
},
967+
}); // fork-hook: voice-input/composer-controller
948968
const sendDisabledReason =
949969
externalSendDisabledReason ??
950970
(multipleModelSelections?.length === 0 ? "Select at least one model." : null) ??
@@ -1658,6 +1678,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
16581678
) => {
16591679
expandComposerForEditorChange();
16601680
refreshWorkspaceSnapshotOnTyping();
1681+
desktopVoiceFork.markDraftChanged(); // fork-hook: voice-input/draft-revision
16611682
if (activePendingProgress?.activeQuestion && pendingUserInputs.length > 0) {
16621683
if (activePendingProgress.activeQuestion.allowCustomAnswer === false) return;
16631684
setComposerCursor(nextCursor);
@@ -2119,6 +2140,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
21192140
dispatchMode?: ComposerDispatchMode,
21202141
submissionIntent?: ComposerSubmissionIntent,
21212142
) => {
2143+
if (desktopVoiceFork.blocksSubmission()) {
2144+
return;
2145+
} // fork-hook: voice-input/submission-guard
21222146
if (
21232147
noProviderAvailable ||
21242148
isSendDisabled ||
@@ -3430,6 +3454,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
34303454
}}
34313455
className="mx-auto w-full min-w-0 max-w-(--chat-content-max-width)"
34323456
data-chat-composer-form="true"
3457+
{...{
3458+
"data-voice-input-presented-fork": desktopVoiceFork.presented ? "true" : undefined,
3459+
"data-voice-input-compact-fork": isComposerResting ? "true" : undefined,
3460+
}} /* fork-hook: voice-input/presentation */
34333461
{...threadContextDropTargetProps()}
34343462
>
34353463
{composerControlsCollapsed && restingControlsHost
@@ -3439,6 +3467,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
34393467
data-chat-composer-resting-controls="true"
34403468
aria-hidden={restingControlsVisible ? undefined : true}
34413469
inert={restingControlsVisible ? undefined : true}
3470+
{...(desktopVoiceFork.presented
3471+
? { "data-voice-input-hidden-fork": "true", "aria-hidden": true, inert: true }
3472+
: {})} /* fork-hook: voice-input/resting-presentation */
34423473
className={cn(
34433474
"relative flex w-max min-w-0 max-w-full items-center gap-1 font-normal text-muted-foreground/70 [&_button]:text-xs!",
34443475
!restingControlsVisible && "invisible",
@@ -3600,6 +3631,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
36003631
};
36013632
}}
36023633
data-chat-composer-body="true"
3634+
{...{
3635+
inert:
3636+
desktopVoiceFork.busy || (desktopVoiceFork.presented && isComposerResting)
3637+
? true
3638+
: undefined,
3639+
"aria-hidden": desktopVoiceFork.presented && isComposerResting ? true : undefined,
3640+
}} /* fork-hook: voice-input/editor-presentation */
36033641
className={cn(
36043642
"relative px-3 pb-2 sm:px-4",
36053643
"pt-3.5 sm:pt-4",
@@ -3764,6 +3802,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
37643802
promptHasText={false}
37653803
isSendBusy={isSendBusy}
37663804
sendDisabledReason={sendDisabledReason}
3805+
{...desktopVoiceFork.sendProps} /* fork-hook: voice-input/mobile-send */
37673806
isConnecting={isConnecting}
37683807
isEnvironmentUnavailable={
37693808
environmentUnavailable !== null ||
@@ -3789,6 +3828,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
37893828
{isComposerCollapsedMobile ? null : (
37903829
<div
37913830
data-chat-composer-footer="true"
3831+
{...{
3832+
inert: desktopVoiceFork.presented ? true : undefined,
3833+
"aria-hidden": desktopVoiceFork.presented ? true : undefined,
3834+
}} /* fork-hook: voice-input/footer-presentation */
37923835
data-chat-composer-footer-compact={isComposerFooterCompact ? "true" : "false"}
37933836
className={cn(
37943837
"flex min-w-0 flex-nowrap items-center justify-between gap-2 overflow-visible px-3 pb-3 sm:px-4 sm:pb-4",
@@ -3823,6 +3866,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
38233866
className="flex shrink-0 flex-nowrap items-center justify-end gap-2"
38243867
>
38253868
{composerAttachButton}
3869+
{/* fork-hook: voice-input/composer-controls */}
3870+
{desktopVoiceFork.controls}
3871+
{/* fork-hook-end */}
38263872
<ComposerFooterPrimaryActions
38273873
compact={isComposerResting || isComposerPrimaryActionsCompact}
38283874
canOperateThread={canOperateThread}
@@ -3857,6 +3903,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
38573903
promptHasText={prompt.trim().length > 0}
38583904
isSendBusy={isSendBusy}
38593905
sendDisabledReason={sendDisabledReason}
3906+
{...desktopVoiceFork.sendProps} /* fork-hook: voice-input/primary-send */
38603907
isConnecting={isConnecting}
38613908
isEnvironmentUnavailable={
38623909
environmentUnavailable !== null ||
@@ -3888,7 +3935,12 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
38883935
</div>
38893936
)}
38903937
{showInlineRestingControls ? (
3891-
<div className="h-8">
3938+
<div
3939+
className="h-8"
3940+
{...{
3941+
"data-voice-input-resting-fork": "true",
3942+
}} /* fork-hook: voice-input/inline-resting */
3943+
>
38923944
<div
38933945
ref={setInlineRestingControlsHost}
38943946
className="absolute bottom-2 inset-x-4 min-w-0"
@@ -3908,6 +3960,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
39083960
</div>
39093961
</div>
39103962
) : null}
3963+
{/* fork-hook: voice-input/footer */}
3964+
<VoiceComposerFooterFork
3965+
toolbar={desktopVoiceFork.toolbar}
3966+
compact={isComposerResting}
3967+
presented={desktopVoiceFork.presented}
3968+
/>
3969+
{/* fork-hook-end */}
39113970
</div>
39123971
</ComposerSurface.Main>
39133972
</div>

‎apps/web/src/components/settings/SettingsSidebarNav.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ import {
5656
} from "./settingsSearch";
5757
import { useAvailableSettingsSearchItems } from "./useAvailableSettingsSearchItems";
5858
import { validateSettingsScopeSearch } from "./settingsScope";
59+
import { MicIcon } from "lucide-react"; // fork-hook: voice-input/settings-icon-import
60+
import { isElectron } from "../../env"; // fork-hook: voice-input/settings-desktop-import
5961

6062
const SnapShotIcon = createLucideIcon("snap-shot", [
6163
[
@@ -85,6 +87,7 @@ const SETTINGS_SECTION_ICONS: Readonly<
8587
> = {
8688
"/settings/general": Settings2Icon,
8789
"/settings/diagnostics": WrenchIcon,
90+
"/settings/dictation": MicIcon, // fork-hook: voice-input/settings-icon
8891
"/settings/appearance": PaletteIcon,
8992
"/settings/chat": MessageSquareTextIcon,
9093
"/settings/diffs": FileDiffIcon,

0 commit comments

Comments
 (0)