Skip to content

Commit 3f9cbb4

Browse files
committed
fix(web): cite text inside inline HTML renders
1 parent d720210 commit 3f9cbb4

14 files changed

Lines changed: 1166 additions & 23 deletions

‎apps/web/src/components/chat/AssistantCitationSource.tsx‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import {
66
type AssistantCitationSourceAnchor,
77
} from "~/lib/assistantTextSelection";
88
import { toastManager } from "../ui/toast";
9+
import { htmlSelectionCommand } from "~/lib/htmlRenderSelection";
10+
import { observeHtmlRenderCitationSource } from "./htmlRenderCitationSource";
911

1012
const CITATION_PULSE_DURATION_MS = 650;
1113
// The second pulse settles into a held highlight so late glances still find the quote.
@@ -28,6 +30,20 @@ export function observeAssistantCitationCommentSource({
2830
citation: AssistantCitation;
2931
onUnavailable: () => void;
3032
}): () => void {
33+
if (anchor.htmlRender) {
34+
const frame = anchor.htmlRender;
35+
const validate = () => {
36+
if (!anchor.source.isConnected || !anchor.viewport.contains(frame)) onUnavailable();
37+
};
38+
const observer = new MutationObserver(validate);
39+
observer.observe(anchor.viewport, { childList: true, subtree: true });
40+
htmlSelectionCommand(frame, "mark", citation);
41+
validate();
42+
return () => {
43+
observer.disconnect();
44+
htmlSelectionCommand(frame, "unmark");
45+
};
46+
}
3147
const { source, range, viewport } = anchor;
3248
const registry = typeof CSS !== "undefined" ? CSS.highlights : undefined;
3349
let highlight: Highlight | null = null;
@@ -113,6 +129,9 @@ export function observeAssistantCitationSource({
113129
request: AssistantCitationTarget;
114130
list: LegendListRef;
115131
}) {
132+
if (root.dataset.htmlCitationSource === "true") {
133+
return observeHtmlRenderCitationSource({ root, itemKey, request, list });
134+
}
116135
const activation = request.activationRef.current;
117136
if (activation.dismissed) return;
118137
const scrollNode = list.getScrollableNode();
@@ -349,13 +368,15 @@ export function AssistantCitationSource({
349368
request,
350369
listRef,
351370
children,
371+
htmlRender = false,
352372
}: {
353373
messageId: MessageId;
354374
threadRef?: ScopedThreadRef;
355375
itemKey: string;
356376
request: AssistantCitationTarget | null;
357377
listRef: RefObject<LegendListRef | null>;
358378
children: ReactNode;
379+
htmlRender?: boolean;
359380
}) {
360381
const rootRef = useRef<HTMLDivElement>(null);
361382
useEffect(() => {
@@ -369,6 +390,7 @@ export function AssistantCitationSource({
369390
<div
370391
ref={rootRef}
371392
data-assistant-citation-source={messageId}
393+
data-html-citation-source={htmlRender || undefined}
372394
data-assistant-citation-environment={threadRef?.environmentId}
373395
data-assistant-citation-thread={threadRef?.threadId}
374396
>
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// @vitest-environment jsdom
2+
import { EnvironmentId, MessageId, ThreadId, type AssistantCitation } from "@t3tools/contracts";
3+
import type { AssistantCitationSourceAnchor } from "~/lib/assistantTextSelection";
4+
import { act } from "react";
5+
import { createRoot, type Root } from "react-dom/client";
6+
import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
7+
import { AssistantSelectionToolbar } from "./AssistantSelectionToolbar";
8+
9+
let root: Root | undefined;
10+
beforeEach(() => vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true));
11+
afterEach(() => {
12+
act(() => root?.unmount());
13+
root = undefined;
14+
document.body.replaceChildren();
15+
vi.restoreAllMocks();
16+
vi.unstubAllGlobals();
17+
});
18+
19+
function setup() {
20+
const viewport = document.createElement("div");
21+
const source = document.createElement("div");
22+
source.dataset.assistantCitationSource = "render-tool-item";
23+
const frame = document.createElement("iframe");
24+
frame.dataset.htmlSelectionBridge = "true";
25+
source.append(frame);
26+
viewport.append(source);
27+
const mount = document.createElement("div");
28+
document.body.append(viewport, mount);
29+
vi.spyOn(viewport, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 600, 500));
30+
vi.spyOn(frame, "getBoundingClientRect").mockReturnValue(new DOMRect(100, 40, 400, 300));
31+
const onCite = vi.fn<
32+
(citation: AssistantCitation, anchor: AssistantCitationSourceAnchor) => boolean
33+
>(() => true);
34+
root = createRoot(mount);
35+
act(() =>
36+
root!.render(
37+
<AssistantSelectionToolbar
38+
viewport={viewport}
39+
threadRef={{ environmentId: EnvironmentId.make("env"), threadId: ThreadId.make("thread") }}
40+
onCite={onCite}
41+
/>,
42+
),
43+
);
44+
frame.focus();
45+
const quote = { text: "quoted text", start: 7, end: 18, prefix: "Before ", suffix: " after." };
46+
const data = {
47+
jsonrpc: "2.0",
48+
method: "t3/selection",
49+
params: {
50+
selector: quote,
51+
rect: { left: 10, top: 20, width: 80, height: 15 },
52+
pointer: { x: 70, y: 35 },
53+
},
54+
};
55+
const send = (payload: unknown = data, origin: Window | null = frame.contentWindow) =>
56+
act(() => {
57+
window.dispatchEvent(new MessageEvent("message", { source: origin, data: payload }));
58+
});
59+
return { frame, onCite, quote, data, send };
60+
}
61+
62+
describe("HTML selection Cite toolbar", () => {
63+
it("adds the iframe quote to the composer with a source anchor in client coordinates", () => {
64+
const { frame, onCite, quote, send } = setup();
65+
const post = vi.spyOn(frame.contentWindow!, "postMessage");
66+
send();
67+
const button = document.querySelector<HTMLButtonElement>(
68+
'[aria-label="Cite selection in composer"]',
69+
)!;
70+
expect(button).not.toBeNull();
71+
expect(button.style.left).toBe("170px");
72+
expect(button.style.top).toBe("75px");
73+
act(() => button.click());
74+
expect(onCite).toHaveBeenCalledOnce();
75+
const [citation, anchor] = onCite.mock.calls[0]!;
76+
expect(citation).toEqual({
77+
version: 1,
78+
environmentId: "env",
79+
threadId: "thread",
80+
messageId: MessageId.make("render-tool-item"),
81+
...quote,
82+
});
83+
expect(anchor.htmlRender).toBe(frame);
84+
expect(anchor.range.getBoundingClientRect()).toEqual(new DOMRect(110, 60, 80, 15));
85+
expect(post).toHaveBeenCalledWith(
86+
{ method: "t3/selection-command", params: { action: "clear", selector: undefined } },
87+
"*",
88+
);
89+
expect(document.querySelector('[aria-label="Cite selection in composer"]')).toBeNull();
90+
});
91+
92+
it("ignores messages from unrelated or unfocused frames", () => {
93+
const { frame, data, send } = setup();
94+
send(data, window);
95+
expect(document.querySelector('[aria-label="Cite selection in composer"]')).toBeNull();
96+
frame.blur();
97+
send();
98+
expect(document.querySelector('[aria-label="Cite selection in composer"]')).toBeNull();
99+
});
100+
101+
it("offers shortening for oversized selections and dismisses on iframe interaction", () => {
102+
const { onCite, data, send } = setup();
103+
send({ ...data, params: { tooLong: true, rect: data.params.rect, pointer: null } });
104+
const button = document.querySelector<HTMLButtonElement>(
105+
'[aria-label="Selection is too long to cite"]',
106+
)!;
107+
expect(button.disabled).toBe(true);
108+
act(() => button.click());
109+
expect(onCite).not.toHaveBeenCalled();
110+
send({ ...data, params: null });
111+
expect(document.querySelector('[aria-label="Selection is too long to cite"]')).toBeNull();
112+
});
113+
});

‎apps/web/src/components/chat/AssistantSelectionToolbar.tsx‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ import {
1717
type SelectionActionPoint,
1818
} from "~/lib/selectionActions";
1919
import { Button } from "../ui/button";
20+
import {
21+
htmlSelectionClientRect,
22+
htmlSelectionCommand,
23+
htmlSelectionParams,
24+
readHtmlSelection,
25+
readHtmlSelectionRect,
26+
} from "~/lib/htmlRenderSelection";
2027

2128
export function AssistantSelectionToolbar({
2229
viewport,
@@ -31,6 +38,7 @@ export function AssistantSelectionToolbar({
3138
citation: AssistantCitation;
3239
position: SelectionActionPoint;
3340
sourceAnchor: AssistantCitationSourceAnchor;
41+
tooLong?: boolean;
3442
} | null>(null);
3543
const toolbarRef = useRef<HTMLButtonElement>(null);
3644
const actionsRef = useRef<ReturnType<typeof observeSelectionActions> | null>(null);
@@ -106,7 +114,78 @@ export function AssistantSelectionToolbar({
106114
};
107115
document.addEventListener("keydown", focusActions, true);
108116
document.addEventListener("selectionchange", actions.selectionChanged);
117+
const htmlSelection = (event: MessageEvent) => {
118+
const params = htmlSelectionParams(event.data);
119+
if (params === undefined) return;
120+
const frame = [
121+
...viewport.querySelectorAll<HTMLIFrameElement>("iframe[data-html-selection-bridge]"),
122+
].find((frame) => frame.contentWindow === event.source);
123+
const source = frame?.closest<HTMLElement>("[data-assistant-citation-source]");
124+
const messageId = source?.dataset.assistantCitationSource;
125+
if (!frame || !source || !messageId) return;
126+
if (params === null) {
127+
setSelection((previous) =>
128+
document.activeElement === frame || previous?.sourceAnchor.htmlRender === frame
129+
? null
130+
: previous,
131+
);
132+
return;
133+
}
134+
if (params.focus === true) {
135+
if (document.activeElement === frame) toolbarRef.current?.focus({ preventScroll: true });
136+
return;
137+
}
138+
if ("target" in params) return;
139+
if (document.activeElement !== frame) return;
140+
const captured = readHtmlSelection(event.data);
141+
const localRect =
142+
captured?.rect ?? (params.tooLong === true ? readHtmlSelectionRect(params.rect) : null);
143+
if (!localRect) {
144+
clear();
145+
return;
146+
}
147+
const rect = htmlSelectionClientRect(frame, localRect);
148+
const bounds = viewport.getBoundingClientRect();
149+
const frameBounds = frame.getBoundingClientRect();
150+
if (
151+
rect.bottom < bounds.top ||
152+
rect.top > bounds.bottom ||
153+
rect.bottom < frameBounds.top ||
154+
rect.top > frameBounds.bottom
155+
) {
156+
clear();
157+
return;
158+
}
159+
const selector = captured?.selector ?? { text: "", start: 0, end: 1, prefix: "", suffix: "" };
160+
setSelection({
161+
tooLong: params.tooLong === true,
162+
citation: { version: 1, ...threadRef, messageId: MessageId.make(messageId), ...selector },
163+
sourceAnchor: {
164+
source,
165+
viewport,
166+
htmlRender: frame,
167+
range: {
168+
getBoundingClientRect: () => htmlSelectionClientRect(frame, localRect),
169+
getClientRects: () =>
170+
Object.assign([htmlSelectionClientRect(frame, localRect)], {
171+
item: (index: number) =>
172+
index === 0 ? htmlSelectionClientRect(frame, localRect) : null,
173+
}),
174+
},
175+
},
176+
position: resolveSelectionActionPosition({
177+
bounds,
178+
selectionRect: rect,
179+
pointer: captured?.pointer
180+
? { x: frameBounds.left + captured.pointer.x, y: frameBounds.top + captured.pointer.y }
181+
: null,
182+
viewport: { width: window.innerWidth, height: window.innerHeight },
183+
}),
184+
});
185+
};
186+
window.addEventListener("message", htmlSelection);
109187
return () => {
188+
window.removeEventListener("message", htmlSelection);
110189
document.removeEventListener("keydown", focusActions, true);
111190
document.removeEventListener("selectionchange", actions.selectionChanged);
112191
actions.dispose();
@@ -115,14 +194,17 @@ export function AssistantSelectionToolbar({
115194
}, [threadRef, viewport]);
116195

117196
if (!selection) return null;
118-
const tooLong = selection.citation.text.length > ASSISTANT_CITATION_MAX_TEXT_LENGTH;
197+
const tooLong =
198+
selection.tooLong || selection.citation.text.length > ASSISTANT_CITATION_MAX_TEXT_LENGTH;
119199
const dismiss = () => {
120200
actionsRef.current?.cancel();
121201
setSelection(null);
122202
};
123203
const cite = () => {
124204
if (tooLong || !onCite(selection.citation, selection.sourceAnchor)) return false;
125-
window.getSelection()?.removeAllRanges();
205+
if (selection.sourceAnchor.htmlRender) {
206+
htmlSelectionCommand(selection.sourceAnchor.htmlRender, "clear");
207+
} else window.getSelection()?.removeAllRanges();
126208
dismiss();
127209
return true;
128210
};
@@ -142,6 +224,8 @@ export function AssistantSelectionToolbar({
142224
event.stopPropagation();
143225
if (event.key === "Escape" && !event.nativeEvent.isComposing) {
144226
event.preventDefault();
227+
if (selection.sourceAnchor.htmlRender)
228+
htmlSelectionCommand(selection.sourceAnchor.htmlRender, "dismiss");
145229
dismiss();
146230
}
147231
}}

‎apps/web/src/components/chat/HtmlRenderFrame.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ export function HtmlRenderFrame(props: {
100100
title={title}
101101
className="block size-full"
102102
onContentHeight={setContentHeight}
103+
selectionBridge
103104
/>
104105
<div className="absolute end-2 top-2 opacity-0 transition-opacity duration-150 focus-within:opacity-100 group-hover/html-render:opacity-100 pointer-coarse:opacity-100">
105106
<Tooltip>
@@ -130,7 +131,10 @@ export function HtmlRenderFrame(props: {
130131
</div>
131132
</>
132133
) : failed ? (
133-
<p className="flex size-full items-center justify-center text-muted-foreground text-xs">
134+
<p
135+
data-html-render-error
136+
className="flex size-full items-center justify-center text-muted-foreground text-xs"
137+
>
134138
Unable to load {title}
135139
</p>
136140
) : null}

‎apps/web/src/components/chat/MessagesTimeline.tsx‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
ORCHESTRATION_V2_WORKSPACE_PREPARATION_FAILURE_CODE,
2222
type AssistantCitation,
2323
type EnvironmentId,
24-
type MessageId,
24+
MessageId,
2525
type OrchestrationV2TurnItem,
2626
type RunAttemptId,
2727
type ScopedThreadRef,
@@ -2773,13 +2773,22 @@ function HtmlRenderTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "htm
27732773

27742774
return (
27752775
<div className="min-w-0 px-1">
2776-
<HtmlRenderFrame
2777-
// A recycled row must not keep another page's frozen frame.
2778-
key={row.htmlRender.attachmentId}
2779-
environmentId={ctx.activeThreadEnvironmentId}
2780-
htmlRender={row.htmlRender}
2781-
onOpen={ctx.onFileOpen}
2782-
/>
2776+
<AssistantCitationSource
2777+
messageId={MessageId.make(row.id)}
2778+
{...(ctx.threadRef ? { threadRef: ctx.threadRef } : {})}
2779+
itemKey={row.id}
2780+
request={ctx.citationRequest}
2781+
listRef={ctx.listRef}
2782+
htmlRender
2783+
>
2784+
<HtmlRenderFrame
2785+
// A recycled row must not keep another page's frozen frame.
2786+
key={row.htmlRender.attachmentId}
2787+
environmentId={ctx.activeThreadEnvironmentId}
2788+
htmlRender={row.htmlRender}
2789+
onOpen={ctx.onFileOpen}
2790+
/>
2791+
</AssistantCitationSource>
27832792
</div>
27842793
);
27852794
}

0 commit comments

Comments
 (0)