Skip to content

Commit 40c3682

Browse files
shivamhwpclaude
andauthored
feat(web): screen readers can hear the agent's messages and steps during a turn (#33900)
Co-authored-by: Claude Fable 5.2 <[email protected]>
1 parent 950c342 commit 40c3682

11 files changed

Lines changed: 935 additions & 13 deletions

‎apps/web/src/components/chat/MessagesTimeline.logic.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ import {
7979
} from "@t3tools/shared/toolActivity";
8080
import { isWindowsAbsolutePath } from "@t3tools/shared/path";
8181

82-
function timelineEntryRunId(entry: TimelineEntry): RunId | null {
82+
export function timelineEntryRunId(entry: TimelineEntry): RunId | null {
8383
if (entry.kind === "message") {
8484
return entry.message.role === "assistant" ? (entry.message.runId ?? null) : null;
8585
}
@@ -917,7 +917,7 @@ function timelineEntryIsSteer(entry: TimelineEntry): boolean {
917917
);
918918
}
919919

920-
function timelineEntryStartsResponse(entry: TimelineEntry): boolean {
920+
export function timelineEntryStartsResponse(entry: TimelineEntry): boolean {
921921
return (
922922
(entry.kind === "message" && entry.message.role === "user" && !timelineEntryIsSteer(entry)) ||
923923
(entry.kind === "work" && entry.entry.itemType === "notification")

‎apps/web/src/components/chat/MessagesTimeline.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ const TIMELINE_MAINTAIN_SCROLL_AT_END = {
200200
},
201201
} as const satisfies MaintainScrollAtEndOptions;
202202
const EMPTY_RUN_IDS: ReadonlySet<RunId> = new Set();
203+
const EMPTY_TIMELINE_ENTRIES: ReadonlyArray<TimelineEntry> = [];
203204
const TIMELINE_MAINTAIN_SCROLL_AT_END_SMOOTH = {
204205
...TIMELINE_MAINTAIN_SCROLL_AT_END,
205206
animated: true,
@@ -552,6 +553,9 @@ const ConversationTimeline = memo(function ConversationTimeline({
552553
}
553554

554555
const preferredToolCallVisibility = useClientSettings((settings) => settings.toolCallVisibility);
556+
const screenReaderAnnouncements = useClientSettings(
557+
(settings) => settings.screenReaderAnnouncements,
558+
);
555559
const toolCallVisibility =
556560
findActive && preferredToolCallVisibility === "zen" ? "hideAll" : preferredToolCallVisibility;
557561
const revealedMessageId = citationRequest?.citation.messageId ?? null;
@@ -1078,6 +1082,9 @@ const ConversationTimeline = memo(function ConversationTimeline({
10781082
runId={announcedRun?.runId ?? null}
10791083
runStatus={announcedRun?.status ?? null}
10801084
detailBehind={detailBehindShell}
1085+
level={screenReaderAnnouncements}
1086+
entries={screenReaderAnnouncements === "status" ? EMPTY_TIMELINE_ENTRIES : timelineEntries}
1087+
workspaceRoot={workspaceRoot}
10811088
/>
10821089
);
10831090
const listHeaderLeadingContent =
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
import { MessageId, RunId } from "@t3tools/contracts";
2+
import { type ComponentProps } from "react";
3+
import { describe, expect, it, vi } from "vite-plus/test";
4+
5+
import { act, fakeTimers, render } from "../../../test/render";
6+
import type { TimelineEntry } from "../../session-logic";
7+
import { ThreadActivityAnnouncer } from "./ThreadActivityAnnouncer";
8+
9+
const run = RunId.make("run-1");
10+
const at = "2026-01-01T00:00:00.000Z";
11+
12+
const message = (id: string, role: "user" | "assistant", streaming = false): TimelineEntry => ({
13+
kind: "message",
14+
id,
15+
createdAt: at,
16+
message: {
17+
id: MessageId.make(id),
18+
role,
19+
text: id,
20+
runId: role === "assistant" ? run : null,
21+
streaming,
22+
createdAt: at,
23+
updatedAt: at,
24+
},
25+
});
26+
const command = (program: string): TimelineEntry => ({
27+
kind: "work",
28+
id: program,
29+
createdAt: at,
30+
entry: {
31+
id: program,
32+
createdAt: at,
33+
runId: run,
34+
label: program,
35+
tone: "tool",
36+
itemType: "command_execution",
37+
command: program,
38+
toolLifecycleStatus: "inProgress",
39+
},
40+
});
41+
const prompt = message("prompt", "user");
42+
43+
const announcer = (props: Partial<ComponentProps<typeof ThreadActivityAnnouncer>>) => (
44+
<ThreadActivityAnnouncer
45+
threadKey="thread-1"
46+
status="working"
47+
requestId={null}
48+
runId={run}
49+
runStatus="running"
50+
detailBehind={false}
51+
level="everything"
52+
entries={[prompt]}
53+
workspaceRoot={undefined}
54+
{...props}
55+
/>
56+
);
57+
58+
const regions = (container: HTMLElement) => {
59+
const [detail, status] = container.querySelectorAll<HTMLElement>('[aria-live="polite"]');
60+
return { detail: detail!, status: status! };
61+
};
62+
63+
describe("ThreadActivityAnnouncer", () => {
64+
it("drops a waiting step once the agent has moved on", async () => {
65+
// The announcer reads no route.
66+
await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
67+
router: false,
68+
});
69+
const { detail, status } = regions(screen.container);
70+
{
71+
// Restored before the assertions, which poll on real timers.
72+
using _timers = fakeTimers();
73+
await screen.rerender(announcer({}));
74+
await screen.rerender(announcer({ entries: [prompt, command("git")] }));
75+
await screen.rerender(
76+
announcer({ entries: [prompt, command("git"), message("reply", "assistant", true)] }),
77+
);
78+
await act(() => vi.advanceTimersByTime(5_000));
79+
}
80+
81+
await expect.element(status).toHaveTextContent("Working");
82+
await expect.element(detail).toBeEmptyDOMElement();
83+
});
84+
85+
it("holds a waiting step while the thread's detail is behind, then reads it", async () => {
86+
// The announcer reads no route.
87+
await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
88+
router: false,
89+
});
90+
const { detail } = regions(screen.container);
91+
const entries = [prompt, command("git")];
92+
const heard: string[] = [];
93+
{
94+
// Restored before the assertion, which polls on real timers.
95+
using _timers = fakeTimers();
96+
await screen.rerender(announcer({}));
97+
await screen.rerender(announcer({ entries }));
98+
await screen.rerender(announcer({ entries, detailBehind: true }));
99+
await act(() => vi.advanceTimersByTime(5_000));
100+
heard.push(detail.textContent);
101+
await screen.rerender(announcer({ entries }));
102+
await act(() => vi.advanceTimersByTime(2_000));
103+
}
104+
105+
expect(heard).toEqual([""]);
106+
await expect.element(detail).toHaveTextContent("Running git");
107+
});
108+
109+
it("keeps a message beside the status that follows it, then reads the newest step", async () => {
110+
// The announcer reads no route.
111+
await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
112+
router: false,
113+
});
114+
const { detail, status } = regions(screen.container);
115+
const entries = [prompt, message("Checking the repo", "assistant"), command("git")];
116+
const heard: Array<[status: string, detail: string]> = [];
117+
using _timers = fakeTimers();
118+
for (const props of [
119+
{},
120+
{ entries },
121+
{ entries, status: "approval", requestId: "request-1" },
122+
{ entries: [...entries, command("rg")] },
123+
] satisfies ReadonlyArray<Partial<ComponentProps<typeof ThreadActivityAnnouncer>>>) {
124+
await screen.rerender(announcer(props));
125+
heard.push([status.textContent, detail.textContent]);
126+
}
127+
await act(() => vi.advanceTimersByTime(2_000));
128+
heard.push([status.textContent, detail.textContent]);
129+
130+
expect(heard).toEqual([
131+
["Working", ""],
132+
["Working", "T3 Code: Checking the repo"],
133+
["Needs approval", "T3 Code: Checking the repo"],
134+
["Needs approval", "T3 Code: Checking the repo"],
135+
["Needs approval", "Running rg"],
136+
]);
137+
});
138+
139+
it("reads a status that lands with a message as the message's last sentence", async () => {
140+
// The announcer reads no route.
141+
await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), {
142+
router: false,
143+
});
144+
const { detail, status } = regions(screen.container);
145+
await screen.rerender(announcer({}));
146+
await screen.rerender(
147+
announcer({
148+
entries: [prompt, message("Checking the repo", "assistant"), command("git")],
149+
status: "approval",
150+
requestId: "request-1",
151+
}),
152+
);
153+
154+
await expect.element(detail).toHaveTextContent("T3 Code: Checking the repo. Needs approval");
155+
await expect.element(status).toHaveTextContent("Working");
156+
});
157+
});

0 commit comments

Comments
 (0)