Skip to content

Commit e1fad55

Browse files
donjor-agent[bot]rsi-pr-agent
authored andcommitted
fix(web): dictation fills AskUserQuestion custom answers (#1803)
Squashes: - 144f2ca fix(web): dictation fills AskUserQuestion custom answers (#1803) (#1803) - a862e5e fixup! fix(web): dictation fills AskUserQuestion custom answers (#1803) (#1803) Fork-Domain: voice-input Fork-Tier: bugfix Fork-Upstreamable: no
1 parent b7af052 commit e1fad55

3 files changed

Lines changed: 152 additions & 64 deletions

File tree

‎apps/web/src/components/chat/ChatComposer.tsx‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText";
44
import { runtimeModeConfig, runtimeModeOptions as runtimeModes } from "./runtimeModeConfig";
55
import { isLocalEnvironmentDisabled } from "../../localEnvironment";
66
import { usePrimaryEnvironmentId } from "../../state/environments";
7-
import { useDesktopVoiceInputFork } from "../../voice-input/useVoiceInput.fork"; // fork-hook: voice-input/composer-import
7+
import {
8+
useDesktopVoiceInputFork,
9+
voicePendingAnswerFork,
10+
} from "../../voice-input/useVoiceInput.fork"; // fork-hook: voice-input/composer-import
811
import { VoiceComposerFooterFork } from "../../voice-input/VoiceComposerControls.fork"; // fork-hook: voice-input/footer-import
912
import { useRightPanelStore } from "~/rightPanelStore";
1013
import { importPastedComposerText, readPastedComposerContext } from "../composerInlineTokenPaste";
@@ -954,7 +957,12 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
954957
ownerKey: composerDraftTargetKey,
955958
draftTarget: composerDraftTarget, // fork-hook: voice-input/draft-target
956959
label: activeThread?.title || "New thread", // fork-hook: voice-input/draft-label
957-
enabled: canOperateThread && !activePendingProgress && pendingApprovals.length === 0,
960+
enabled: canOperateThread && pendingApprovals.length === 0, // fork-hook: voice-input/pending-answer-enabled
961+
pendingAnswer: voicePendingAnswerFork(
962+
pendingUserInputs[0]?.requestId,
963+
activePendingProgress,
964+
onChangeActivePendingUserInputCustomAnswer,
965+
), // fork-hook: voice-input/pending-answer
958966
readSelection: () =>
959967
composerEditorRef.current?.readSelectionRange() ?? {
960968
start: promptRef.current.length,

‎apps/web/src/voice-input/useVoiceInput.fork.browser.test.tsx‎

Lines changed: 82 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
import { act, useLayoutEffect, useSyncExternalStore } from "react";
2-
import { createRoot, type Root } from "react-dom/client";
3-
import { afterEach, describe, expect, it, vi } from "vite-plus/test";
1+
import { useLayoutEffect, useSyncExternalStore } from "react";
2+
import { describe, expect, it, vi } from "vite-plus/test";
43
import * as Option from "effect/Option";
54
import {
65
EnvironmentId,
@@ -9,13 +8,15 @@ import {
98
ThreadId,
109
VOICE_TEXT_STARTER_PROMPTS_FORK,
1110
} from "@t3tools/contracts";
11+
import { act, render, type Screen } from "../../test/render";
1212
import { useComposerDraftStore } from "../composerDraftStore";
1313
import { DesktopVoiceSessionFork } from "./session.fork";
1414
import { useDesktopVoiceInputFork } from "./useVoiceInput.fork";
1515

1616
const mocks = vi.hoisted(() => ({
1717
request: vi.fn(),
1818
draft: "",
19+
stored: [] as string[],
1920
listeners: new Set<() => void>(),
2021
}));
2122
const environmentId = EnvironmentId.make("test");
@@ -63,6 +64,7 @@ vi.mock("./draft.fork", () => ({
6364
readVoiceDraftTextFork: () => mocks.draft,
6465
commitVoiceDraftFork: (_target: unknown, text: string) => {
6566
mocks.draft = text;
67+
mocks.stored.push(text);
6668
mocks.listeners.forEach((listener) => listener());
6769
},
6870
}));
@@ -74,8 +76,8 @@ vi.mock("./VoiceInputProvider.fork", () => ({
7476
})) as never,
7577
}));
7678

77-
let root: Root | undefined;
7879
let actions: ReturnType<typeof useDesktopVoiceInputFork>;
80+
let pendingAnswer: { key: string; text: string; commit: (text: string) => void } | undefined;
7981
function Probe() {
8082
const value = useDesktopVoiceInputFork({
8183
environmentId,
@@ -90,87 +92,90 @@ function Probe() {
9092
mocks.draft = text;
9193
actions.markDraftChanged();
9294
},
95+
pendingAnswer,
9396
});
9497
useLayoutEffect(() => {
9598
actions = value;
9699
});
97100
// The composer hides the normal controls when the dictation toolbar is presented.
98101
return value.presented ? value.toolbar : value.controls;
99102
}
100-
let host: HTMLDivElement | undefined;
103+
let screen: Screen | undefined;
101104
function container() {
102-
return host!;
105+
return screen!.container;
103106
}
104107
function button(label: string) {
105108
const match = container().querySelector<HTMLButtonElement>(`button[aria-label="${label}"]`);
106109
if (!match) throw new Error(`No ${label} button.`);
107110
return match;
108111
}
109-
afterEach(async () => {
110-
await act(() => root?.unmount());
111-
root = undefined;
112-
host?.remove();
113-
host = undefined;
114-
runtime?.session.dispose();
115-
vi.restoreAllMocks();
116-
vi.unstubAllGlobals();
117-
mocks.listeners.clear();
118-
});
112+
function mountRuntime(transcript: string) {
113+
const stack = new AsyncDisposableStack();
114+
stack.defer(() => {
115+
mocks.listeners.clear();
116+
pendingAnswer = undefined;
117+
});
118+
stack.use(vi.spyOn(console, "error").mockImplementation(() => {}));
119+
mocks.draft = "";
120+
stack.use(
121+
vi.spyOn(useComposerDraftStore, "subscribe").mockImplementation((listener) => {
122+
const notify = () =>
123+
listener(useComposerDraftStore.getState(), useComposerDraftStore.getState());
124+
mocks.listeners.add(notify);
125+
return () => mocks.listeners.delete(notify);
126+
}),
127+
);
128+
const recorded = Promise.withResolvers<void>();
129+
const recorder = {
130+
uri: "memory:recording",
131+
prepareToRecordAsync: async () => {},
132+
record: () => recorded.resolve(),
133+
stop: async () => {},
134+
readLevel: () => 0,
135+
};
136+
runtime = {
137+
recorder,
138+
session: new DesktopVoiceSessionFork({
139+
recorder,
140+
requestPermission: async () => ({ granted: true, canAskAgain: true }),
141+
configureRecording: async () => {},
142+
releaseRecording: async () => {},
143+
deleteRecording: () => {},
144+
getTranscriber: () => ({
145+
prepare: async () => ({ locale: "en-US", transcribe: async () => transcript }),
146+
}),
147+
}),
148+
};
149+
stack.defer(() => runtime.session.dispose());
150+
return {
151+
recorded: recorded.promise,
152+
async mount() {
153+
screen = stack.use(await render(<Probe />, { router: false }));
154+
},
155+
async [Symbol.asyncDispose]() {
156+
await stack.disposeAsync();
157+
screen = undefined;
158+
},
159+
};
160+
}
119161

120162
describe("automatic cleanup in the active dictation toolbar", () => {
121163
it.each(["button", "Escape"])(
122164
"cancels through %s and commits raw speech without formatting",
123165
async (cancelWith) => {
124-
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
125-
vi.spyOn(console, "error").mockImplementation(() => {});
126-
mocks.draft = "";
127-
vi.spyOn(useComposerDraftStore, "subscribe").mockImplementation((listener) => {
128-
const notify = () =>
129-
listener(useComposerDraftStore.getState(), useComposerDraftStore.getState());
130-
mocks.listeners.add(notify);
131-
return () => mocks.listeners.delete(notify);
132-
});
133-
const recorded = Promise.withResolvers<void>();
166+
await using fixture = mountRuntime("raw use affect words");
134167
const cleanupStarted = Promise.withResolvers<AbortSignal>();
135-
const recorder = {
136-
uri: "memory:recording",
137-
prepareToRecordAsync: async () => {},
138-
record: () => recorded.resolve(),
139-
stop: async () => {},
140-
readLevel: () => 0,
141-
};
142-
runtime = {
143-
recorder,
144-
session: new DesktopVoiceSessionFork({
145-
recorder,
146-
requestPermission: async () => ({ granted: true, canAskAgain: true }),
147-
configureRecording: async () => {},
148-
releaseRecording: async () => {},
149-
deleteRecording: () => {},
150-
getTranscriber: () => ({
151-
prepare: async () => ({
152-
locale: "en-US",
153-
transcribe: async () => "raw use affect words",
154-
}),
155-
}),
156-
}),
157-
};
158168
mocks.request.mockReset().mockImplementation(
159169
(_input, signal: AbortSignal) =>
160170
new Promise((_resolve, reject) => {
161171
cleanupStarted.resolve(signal);
162172
signal.addEventListener("abort", () => reject(new Error("cancelled")), { once: true });
163173
}),
164174
);
165-
await act(() => {
166-
host = document.createElement("div");
167-
document.body.append(host);
168-
root = createRoot(host);
169-
root.render(<Probe />);
170-
});
175+
await fixture.mount();
171176
await act(async () => {
172177
button("Record dictation").click();
173-
await recorded.promise;
178+
await fixture.recorded;
174179
});
175180
let finishing: Promise<void>;
176181
await act(async () => {
@@ -183,7 +188,7 @@ describe("automatic cleanup in the active dictation toolbar", () => {
183188
status.textContent?.includes("Cleaning up"),
184189
),
185190
).toBe(true);
186-
const cancelDictation = vi.spyOn(runtime.session.controller, "cancel");
191+
using cancelDictation = vi.spyOn(runtime.session.controller, "cancel");
187192
await act(async () => {
188193
if (cancelWith === "button") button("Cancel text processing").click();
189194
else
@@ -198,3 +203,23 @@ describe("automatic cleanup in the active dictation toolbar", () => {
198203
},
199204
);
200205
});
206+
207+
describe("dictating a pending question answer", () => {
208+
it("commits to the answer, never the hidden composer draft", async () => {
209+
await using fixture = mountRuntime("yes");
210+
mocks.stored = [];
211+
mocks.request.mockReset().mockRejectedValue(new Error("offline"));
212+
const commit = vi.fn((text: string) => {
213+
pendingAnswer = { key: "request:question", text, commit };
214+
});
215+
pendingAnswer = { key: "request:question", text: "", commit };
216+
await fixture.mount();
217+
await act(async () => {
218+
button("Record dictation").click();
219+
await fixture.recorded;
220+
});
221+
await act(() => runtime.session.controller.stop());
222+
expect(commit).toHaveBeenCalledWith("yes");
223+
expect(mocks.stored).toEqual([]);
224+
});
225+
});

‎apps/web/src/voice-input/useVoiceInput.fork.tsx‎

Lines changed: 60 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,65 @@ interface DesktopVoiceInputFork {
3333
enabled: boolean;
3434
readSelection: () => { start: number; end: number };
3535
commitDraft: (text: string, cursor: number) => void;
36+
/**
37+
* The active question's custom answer replaces the composer draft while it is shown.
38+
* `null` means a question is shown that takes no custom answer, so dictation is off.
39+
*/
40+
pendingAnswer?: { key: string; text: string; commit: (text: string) => void } | null | undefined;
41+
}
42+
43+
/** Adapts the upstream pending-question state to a dictation target. */
44+
export function voicePendingAnswerFork(
45+
requestId: string | undefined,
46+
progress: {
47+
activeQuestion: { id: string; allowCustomAnswer?: boolean | undefined } | null;
48+
customAnswer: string;
49+
} | null,
50+
setAnswer: (
51+
questionId: string,
52+
text: string,
53+
cursor: number,
54+
end: number,
55+
mention: false,
56+
) => void,
57+
): DesktopVoiceInputFork["pendingAnswer"] {
58+
if (!progress) return undefined;
59+
const question = progress.activeQuestion;
60+
if (!requestId || !question || question.allowCustomAnswer === false) return null;
61+
return {
62+
key: `${requestId}:${question.id}`,
63+
text: progress.customAnswer,
64+
commit: (text) => setAnswer(question.id, text, text.length, text.length, false),
65+
};
66+
}
67+
68+
/** Text a dictation session reads and commits once it can outlive the visible editor. */
69+
function voiceDraftSourceFork(latest: { current: DesktopVoiceInputFork }) {
70+
const { draftTarget, pendingAnswer } = latest.current;
71+
if (!pendingAnswer) {
72+
return {
73+
read: () => readVoiceDraftTextFork(draftTarget),
74+
commit: (text: string) => commitVoiceDraftFork(draftTarget, text),
75+
subscribe: (onChange: () => void) => useComposerDraftStore.subscribe(onChange),
76+
};
77+
}
78+
// Answers live in React state; a transcript for an answered or changed question is dropped.
79+
const current = () =>
80+
latest.current.pendingAnswer?.key === pendingAnswer.key ? latest.current.pendingAnswer : null;
81+
return {
82+
read: () => current()?.text ?? null,
83+
commit: (text: string) => current()?.commit(text),
84+
subscribe: () => () => {},
85+
};
3686
}
3787

3888
const IDLE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
3989

40-
export function useDesktopVoiceInputFork(input: DesktopVoiceInputFork) {
90+
export function useDesktopVoiceInputFork(composerInput: DesktopVoiceInputFork) {
91+
const input = {
92+
...composerInput,
93+
enabled: composerInput.enabled && composerInput.pendingAnswer !== null,
94+
};
4195
const { settings } = useVoiceSettingsFork(input.environmentId);
4296
const { session, recorder, snapshot, waiting } = useDesktopVoiceRuntimeFork();
4397
const controller = session.controller;
@@ -57,7 +111,7 @@ export function useDesktopVoiceInputFork(input: DesktopVoiceInputFork) {
57111
enabled: input.enabled && !voiceInputBlocksSubmission(state),
58112
readDraft: (): VoiceDraftSnapshot => ({
59113
ownerKey: input.ownerKey,
60-
text: readVoiceDraftTextFork(input.draftTarget) ?? "",
114+
text: voiceDraftSourceFork(latest).read() ?? "",
61115
selection: input.readSelection(),
62116
revision: revision.current,
63117
}),
@@ -103,6 +157,7 @@ export function useDesktopVoiceInputFork(input: DesktopVoiceInputFork) {
103157
const captured = latest.current;
104158
if (!captured.enabled || voiceInputBlocksSubmission(controller.currentState)) return;
105159
const selection = captured.readSelection();
160+
const source = voiceDraftSourceFork(latest);
106161
void session.start({
107162
ownerKey: captured.ownerKey,
108163
environmentId: captured.environmentId,
@@ -112,11 +167,11 @@ export function useDesktopVoiceInputFork(input: DesktopVoiceInputFork) {
112167
? { kind: "draft", draftId: captured.draftTarget }
113168
: { kind: "server", threadRef: captured.draftTarget },
114169
readDraft: () => {
115-
const text = readVoiceDraftTextFork(captured.draftTarget);
170+
const text = source.read();
116171
return text === null ? null : { ownerKey: captured.ownerKey, text, selection };
117172
},
118-
commitDraft: (text) => commitVoiceDraftFork(captured.draftTarget, text),
119-
subscribe: (onChange) => useComposerDraftStore.subscribe(onChange),
173+
commitDraft: (text) => source.commit(text),
174+
subscribe: source.subscribe,
120175
});
121176
};
122177
const unavailableReason =

0 commit comments

Comments
 (0)