Skip to content

Commit f141af7

Browse files
steipetesteipete
authored
fix(ui): keep streamed talk voice-turn transcripts readable (#102573)
* fix(ui): keep streamed talk voice-turn transcripts readable Talk voice turns collapsed to one character per line in the macOS app (WKWebView) while streaming: WebKit never re-runs shrink-to-fit intrinsic sizing for a grid flex-item as its content grows, so the turn froze at first-delta width. Convert the turn to a flex row (same visual layout). Assistant transcript deltas also merged through the user-ASR word-boundary spacing heuristic, mangling verbatim fragments ("Chat G PT"). Assistant entries now concatenate fragments verbatim, space only sentence-punctuation joins, and accept both full-transcript and tail-fragment finals. Fixes #102556 * fix(ui): keep assistant talk fragments strictly verbatim Drop the sentence-punctuation separator: it invented characters in real content (Version 1. 2, docs.openclaw. ai) and Google Live tail-finals could never heal it. The punctuation-spacing heuristic remains user-only.
1 parent 3ac7729 commit f141af7

4 files changed

Lines changed: 192 additions & 8 deletions

File tree

‎ui/src/e2e/browser-talk-start-stop.e2e.test.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,74 @@ describeControlUiE2e("Control UI browser Talk", () => {
281281
}
282282
});
283283

284+
it("renders streamed relay assistant transcript deltas as readable text", async () => {
285+
const browser = await chromium.launch({ executablePath: chromiumExecutablePath });
286+
const context = await browser.newContext({ permissions: ["microphone"] });
287+
const page = await context.newPage();
288+
const relaySessionId = "relay-e2e-transcript";
289+
const gateway = await installMockGateway(page, {
290+
methodResponses: {
291+
"talk.client.create": {
292+
provider: "openai",
293+
transport: "gateway-relay",
294+
relaySessionId,
295+
audio: {
296+
inputEncoding: "pcm16",
297+
inputSampleRateHz: 16_000,
298+
outputEncoding: "pcm16",
299+
outputSampleRateHz: 24_000,
300+
},
301+
},
302+
"talk.session.appendAudio": {},
303+
"talk.session.close": {},
304+
},
305+
});
306+
await installTalkBrowserFixtures(page);
307+
308+
try {
309+
await page.goto(`${server.baseUrl}chat`);
310+
await page.setViewportSize({ width: 1366, height: 900 });
311+
await page.getByRole("button", { name: "Start voice input" }).click();
312+
await gateway.waitForRequest("talk.client.create");
313+
await gateway.emitGatewayEvent("talk.event", { relaySessionId, type: "ready" });
314+
await gateway.emitGatewayEvent("talk.event", {
315+
relaySessionId,
316+
type: "transcript",
317+
role: "user",
318+
text: "Hey, what model are you using?",
319+
final: true,
320+
});
321+
// Assistant audio transcripts stream as verbatim fragments that can split
322+
// words ("I","'m"," Chat","G","PT"); regression coverage for #102556 where
323+
// the merge injected spaces mid-word and the turn collapsed while streaming.
324+
const assistantText =
325+
"I'm ChatGPT, a conversational AI model designed to help answer questions, brainstorm ideas, and chat about pretty much anything you want to talk about today.";
326+
for (const char of assistantText) {
327+
await gateway.emitGatewayEvent("talk.event", {
328+
relaySessionId,
329+
type: "transcript",
330+
role: "assistant",
331+
text: char,
332+
final: false,
333+
});
334+
}
335+
336+
const assistantTurnText = page.locator(
337+
".agent-chat__voice-turn--assistant .agent-chat__voice-turn-text",
338+
);
339+
await expect.poll(() => assistantTurnText.textContent()).toBe(assistantText);
340+
341+
const turnBounds = await page.locator(".agent-chat__voice-turn--assistant").boundingBox();
342+
expect(turnBounds).not.toBeNull();
343+
// A collapsed turn renders one character per line (tall, sliver-wide box).
344+
expect(turnBounds?.width ?? 0).toBeGreaterThanOrEqual(500);
345+
expect(turnBounds?.height ?? Number.POSITIVE_INFINITY).toBeLessThanOrEqual(120);
346+
} finally {
347+
await context.close();
348+
await browser.close();
349+
}
350+
});
351+
284352
it("keeps blocked microphone guidance readable in a narrow viewport", async () => {
285353
const browser = await chromium.launch({ executablePath: chromiumExecutablePath });
286354
const context = await browser.newContext();

‎ui/src/pages/chat/realtime-talk-conversation.test.ts‎

Lines changed: 83 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,24 +28,103 @@ describe("realtime Talk conversation", () => {
2828
]);
2929
});
3030

31-
it("inserts spacing after punctuation-ended transcript fragments", () => {
31+
it("inserts spacing after punctuation-ended user transcript fragments", () => {
3232
let state = createRealtimeTalkConversationState();
3333

3434
state = updateRealtimeTalkConversation(state, {
35-
role: "assistant",
35+
role: "user",
3636
text: "Ready.",
3737
final: false,
3838
nowMs: 1,
3939
});
4040
state = updateRealtimeTalkConversation(state, {
41-
role: "assistant",
41+
role: "user",
4242
text: "What next?",
4343
final: false,
4444
nowMs: 2,
4545
});
4646

4747
expect(state.entries).toMatchObject([
48-
{ role: "assistant", text: "Ready. What next?", isStreaming: true },
48+
{ role: "user", text: "Ready. What next?", isStreaming: true },
49+
]);
50+
});
51+
52+
it("concatenates streamed assistant deltas verbatim without injecting spaces", () => {
53+
let state = createRealtimeTalkConversationState();
54+
55+
const deltas = ["I", "'m", " Chat", "G", "PT", ",", " a", " con", "vers", "ational", " AI"];
56+
for (const [index, delta] of deltas.entries()) {
57+
state = updateRealtimeTalkConversation(state, {
58+
role: "assistant",
59+
text: delta,
60+
final: false,
61+
nowMs: index + 1,
62+
});
63+
}
64+
65+
expect(state.entries).toMatchObject([
66+
{ role: "assistant", text: "I'm ChatGPT, a conversational AI", isStreaming: true },
67+
]);
68+
});
69+
70+
it("keeps per-character assistant deltas verbatim across punctuation boundaries", () => {
71+
let state = createRealtimeTalkConversationState();
72+
73+
for (const [index, char] of "Version 1.2 is on docs.openclaw.ai today.".split("").entries()) {
74+
state = updateRealtimeTalkConversation(state, {
75+
role: "assistant",
76+
text: char,
77+
final: false,
78+
nowMs: index + 1,
79+
});
80+
}
81+
82+
expect(state.entries).toMatchObject([
83+
{ role: "assistant", text: "Version 1.2 is on docs.openclaw.ai today.", isStreaming: true },
84+
]);
85+
});
86+
87+
it("replaces streamed assistant text with the authoritative final transcript", () => {
88+
let state = createRealtimeTalkConversationState();
89+
90+
for (const delta of ["I'm Chat", "GPT."]) {
91+
state = updateRealtimeTalkConversation(state, {
92+
role: "assistant",
93+
text: delta,
94+
final: false,
95+
nowMs: 1,
96+
});
97+
}
98+
state = updateRealtimeTalkConversation(state, {
99+
role: "assistant",
100+
text: "I'm ChatGPT, nice to meet you.",
101+
final: true,
102+
nowMs: 2,
103+
});
104+
105+
expect(state.entries).toMatchObject([
106+
{ role: "assistant", text: "I'm ChatGPT, nice to meet you.", isStreaming: false },
107+
]);
108+
});
109+
110+
it("appends a final assistant fragment that only carries the transcript tail", () => {
111+
let state = createRealtimeTalkConversationState();
112+
113+
state = updateRealtimeTalkConversation(state, {
114+
role: "assistant",
115+
text: "Sure, the lights are ",
116+
final: false,
117+
nowMs: 1,
118+
});
119+
state = updateRealtimeTalkConversation(state, {
120+
role: "assistant",
121+
text: "off now.",
122+
final: true,
123+
nowMs: 2,
124+
});
125+
126+
expect(state.entries).toMatchObject([
127+
{ role: "assistant", text: "Sure, the lights are off now.", isStreaming: false },
49128
]);
50129
});
51130

‎ui/src/pages/chat/realtime-talk-conversation.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,10 @@ function upsertRealtimeConversationEntry(
112112
return upsertRealtimeConversationEntry(state, role, null, text, isFinal, nowMs);
113113
}
114114
const entry = state.entries[targetIndex];
115-
const updatedText = mergeRealtimeTranscriptText(entry.text, text, isFinal);
115+
const updatedText =
116+
role === "assistant"
117+
? mergeAssistantTranscriptText(entry.text, text, isFinal)
118+
: mergeRealtimeTranscriptText(entry.text, text, isFinal);
116119
const entries =
117120
entry.text === updatedText && entry.isStreaming === !isFinal
118121
? state.entries
@@ -201,6 +204,34 @@ function shouldStartNewRealtimeUserEntry(
201204
return true;
202205
}
203206

207+
// Assistant transcripts are verbatim fragment streams: providers concatenate
208+
// deltas into the exact transcript (OpenAI audio-transcript deltas, Google Live
209+
// outputTranscription chunks). Never synthesize characters between fragments —
210+
// the user-side ASR spacing heuristic would mangle "ChatGPT" into "Chat G PT"
211+
// and "Version 1.2" into "Version 1. 2".
212+
function mergeAssistantTranscriptText(
213+
existing: string,
214+
incoming: string,
215+
isFinal: boolean,
216+
): string {
217+
if (existing.trim() === "") {
218+
return incoming.trimStart();
219+
}
220+
if (!isFinal) {
221+
return `${existing}${incoming}`;
222+
}
223+
// Final shape differs by provider: OpenAI-style finals carry the full
224+
// transcript (replace), Google Live finals carry only the last fragment
225+
// (append). Replace only when incoming restates what already streamed.
226+
if (incoming === existing || incoming.startsWith(existing)) {
227+
return incoming;
228+
}
229+
if (looksLikeTranscriptReplacement(existing, incoming)) {
230+
return incoming;
231+
}
232+
return `${existing}${incoming}`;
233+
}
234+
204235
function mergeRealtimeTranscriptText(existing: string, incoming: string, isFinal: boolean): string {
205236
if (existing.trim() === "") {
206237
return incoming.trimStart();

‎ui/src/styles/chat/layout.css‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1644,9 +1644,11 @@ openclaw-chat-page {
16441644
background: transparent;
16451645
}
16461646

1647+
/* Flex row, not grid: WebKit (Mac/iOS WKWebView) never re-runs shrink-to-fit
1648+
intrinsic sizing for a grid flex-item while streamed transcript text grows,
1649+
freezing the turn at first-delta width and wrapping one character per line. */
16471650
.agent-chat__voice-turn {
1648-
display: grid;
1649-
grid-template-columns: minmax(56px, max-content) minmax(0, 1fr) 12px;
1651+
display: flex;
16501652
gap: 8px;
16511653
align-items: baseline;
16521654
max-width: min(100%, 780px);
@@ -1666,7 +1668,9 @@ openclaw-chat-page {
16661668
}
16671669

16681670
.agent-chat__voice-turn-speaker {
1669-
min-width: 0;
1671+
flex: 0 0 auto;
1672+
min-width: 56px;
1673+
max-width: 200px;
16701674
overflow: hidden;
16711675
color: var(--muted);
16721676
font-size: 12px;
@@ -1676,6 +1680,7 @@ openclaw-chat-page {
16761680
}
16771681

16781682
.agent-chat__voice-turn-text {
1683+
flex: 1 1 auto;
16791684
min-width: 0;
16801685
overflow-wrap: anywhere;
16811686
color: var(--text);
@@ -1685,6 +1690,7 @@ openclaw-chat-page {
16851690
}
16861691

16871692
.agent-chat__voice-turn-stream {
1693+
flex: 0 0 auto;
16881694
width: 6px;
16891695
height: 6px;
16901696
border-radius: 999px;

0 commit comments

Comments
 (0)