Skip to content

Commit 9601a9b

Browse files
committed
feat(web): suggest the next message as ghost text in the composer
A finished Claude turn now leaves a prediction of what the user will type next. The server forks the thread's session right after the turn commits, so the provider's prompt cache covers the conversation and only the instruction is new input; a cold, unreported, or oversized cache predicts nothing. Tab, Right arrow, or the keycap fills the empty draft without sending; Escape hides the suggestion until the next turn. Off by default in Settings -> Labs.
1 parent 57f4ae2 commit 9601a9b

26 files changed

Lines changed: 1396 additions & 1 deletion

‎apps/server/src/auth/RpcAuthorization.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ export const RPC_REQUIRED_SCOPES = {
5959
[ORCHESTRATION_V2_WS_METHODS.subscribeShell]: AuthOrchestrationReadScope,
6060
[ORCHESTRATION_V2_WS_METHODS.subscribeThread]: AuthOrchestrationReadScope,
6161
[ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: AuthOrchestrationReadScope,
62+
[ORCHESTRATION_V2_WS_METHODS.suggestComposerReply]: AuthOrchestrationReadScope,
6263
[WS_METHODS.projectsMutate]: AuthOrchestrationOperateScope,
6364
[WS_METHODS.serverProbe]: AuthOrchestrationReadScope,
6465
[WS_METHODS.serverGetConfig]: AuthOrchestrationReadScope,
Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import {
2+
type MessageId,
3+
OrchestrationV2SuggestComposerReplyError,
4+
type OrchestrationV2SuggestComposerReplyResult,
5+
type RunId,
6+
type ThreadId,
7+
} from "@t3tools/contracts";
8+
import * as Cause from "effect/Cause";
9+
import * as Context from "effect/Context";
10+
import * as DateTime from "effect/DateTime";
11+
import * as Effect from "effect/Effect";
12+
import * as FiberMap from "effect/FiberMap";
13+
import * as Layer from "effect/Layer";
14+
import * as Ref from "effect/Ref";
15+
import type * as Scope from "effect/Scope";
16+
import * as Stream from "effect/Stream";
17+
18+
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
19+
import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
20+
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
21+
import { forkParked } from "../serverActivation.ts";
22+
import * as ServerSettings from "../serverSettings.ts";
23+
import { composerSuggestionTarget, lastAssistantReply } from "./decide.ts";
24+
import { cleanComposerSuggestion } from "./filter.ts";
25+
import { COMPOSER_SUGGESTION_INSTRUCTION } from "./instruction.ts";
26+
27+
/**
28+
* Predicts the user's next message after each completed turn by forking the finished session, so
29+
* the provider's prompt cache covers the conversation and only the instruction is new input. It
30+
* lives in memory, never in the event log: chrome around the next turn, not conversation.
31+
*/
32+
const SUGGESTION_TTL_MS = 15 * 60_000;
33+
34+
/** Threads kept in memory; older entries fall out first. */
35+
const MAX_CACHED_THREADS = 200;
36+
37+
interface CachedSuggestion {
38+
readonly messageId: MessageId;
39+
readonly text: string;
40+
readonly storedAt: number;
41+
}
42+
43+
const replyError = (threadId: ThreadId, cause: unknown) =>
44+
new OrchestrationV2SuggestComposerReplyError({
45+
threadId,
46+
message: "Failed to read the composer suggestion",
47+
cause,
48+
});
49+
50+
/**
51+
* Predicts the user's next message after each completed turn by forking the finished session, so
52+
* the provider's prompt cache covers the conversation and only the instruction is new input. It
53+
* lives in memory, never in the event log: chrome around the next turn, not conversation.
54+
*/
55+
export class ComposerSuggestionService extends Context.Service<
56+
ComposerSuggestionService,
57+
{
58+
readonly read: (input: {
59+
readonly threadId: ThreadId;
60+
}) => Effect.Effect<
61+
OrchestrationV2SuggestComposerReplyResult,
62+
OrchestrationV2SuggestComposerReplyError
63+
>;
64+
readonly start: () => Effect.Effect<void, never, Scope.Scope>;
65+
}
66+
>()("t3/composerSuggestion/ComposerSuggestionService") {}
67+
68+
const make = Effect.gen(function* () {
69+
const orchestrator = yield* Orchestrator.OrchestratorV2;
70+
const projections = yield* ProjectionStore.ProjectionStoreV2;
71+
const settings = yield* ServerSettings.ServerSettingsService;
72+
const instances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry;
73+
const cache = yield* Ref.make(new Map<ThreadId, CachedSuggestion>());
74+
75+
const store = (threadId: ThreadId, messageId: MessageId, text: string, nowMs: number) =>
76+
Ref.update(cache, (current) => {
77+
const next = new Map(current);
78+
next.delete(threadId);
79+
next.set(threadId, { messageId, text, storedAt: nowMs });
80+
while (next.size > MAX_CACHED_THREADS) {
81+
const oldest = next.keys().next().value;
82+
if (oldest === undefined) break;
83+
next.delete(oldest);
84+
}
85+
return next;
86+
});
87+
88+
const read: ComposerSuggestionService["Service"]["read"] = Effect.fn(
89+
"ComposerSuggestionService.read",
90+
)(function* (input) {
91+
const current = yield* settings.getSettings.pipe(
92+
Effect.mapError((cause) => replyError(input.threadId, cause)),
93+
);
94+
if (!current.composerReplySuggestions) return { text: "" };
95+
const records = yield* projections
96+
.getThreadRecords(input.threadId, ["messages"], { messageRoles: ["assistant"] })
97+
.pipe(Effect.mapError((cause) => replyError(input.threadId, cause)));
98+
const reply = lastAssistantReply(records.messages);
99+
if (reply === undefined) return { text: "" };
100+
const cached = (yield* Ref.get(cache)).get(input.threadId);
101+
if (cached === undefined || cached.messageId !== reply.id) return { text: "" };
102+
const nowMs = DateTime.toEpochMillis(yield* DateTime.now);
103+
if (nowMs - cached.storedAt > SUGGESTION_TTL_MS) return { text: "" };
104+
return { text: cached.text };
105+
});
106+
107+
const suggest = Effect.fn("ComposerSuggestionService.suggest")(function* (input: {
108+
readonly threadId: ThreadId;
109+
readonly runId: RunId;
110+
}) {
111+
const current = yield* settings.getSettings;
112+
if (!current.composerReplySuggestions) return;
113+
const records = yield* projections.getThreadRecords(
114+
input.threadId,
115+
["messages", "providerTurns", "providerThreads"],
116+
{ messageRoles: ["assistant"] },
117+
);
118+
const nowMs = DateTime.toEpochMillis(yield* DateTime.now);
119+
const target = composerSuggestionTarget(records, input.runId, nowMs);
120+
if (target === null) return;
121+
const providerThread = records.providerThreads.find(
122+
(candidate) => candidate.id === target.providerThreadId,
123+
);
124+
if (providerThread === undefined) return;
125+
const instance = yield* instances.getInstance(records.thread.providerInstanceId);
126+
const adapter = instance?.orchestrationAdapter;
127+
const suggestReply = adapter?.suggestComposerReply;
128+
if (adapter === undefined || suggestReply === undefined) return;
129+
const result = yield* suggestReply.call(adapter, {
130+
threadId: input.threadId,
131+
providerThread,
132+
instruction: COMPOSER_SUGGESTION_INSTRUCTION,
133+
});
134+
const verdict = cleanComposerSuggestion(result.text);
135+
if (verdict.kind !== "accepted") {
136+
return yield* Effect.logDebug("composer suggestion declined", {
137+
threadId: input.threadId,
138+
verdict: verdict.kind,
139+
...(verdict.kind === "rejected" ? { reason: verdict.reason } : {}),
140+
});
141+
}
142+
yield* store(input.threadId, target.reply.id, verdict.text, nowMs);
143+
});
144+
145+
const start: ComposerSuggestionService["Service"]["start"] = Effect.fn(
146+
"ComposerSuggestionService.start",
147+
)(function* () {
148+
// Keyed by thread: a newer completed run supersedes a prediction still in flight.
149+
const predictions = yield* FiberMap.make<ThreadId>();
150+
yield* forkParked(
151+
Stream.runForEach(orchestrator.streamDomainEvents, (event) =>
152+
event.type === "run.updated" && event.payload.status === "completed"
153+
? FiberMap.run(
154+
predictions,
155+
event.threadId,
156+
suggest({ threadId: event.threadId, runId: event.payload.id }).pipe(
157+
Effect.catchCause((cause) =>
158+
Cause.hasInterruptsOnly(cause)
159+
? Effect.interrupt
160+
: Effect.logDebug("composer suggestion skipped", {
161+
threadId: event.threadId,
162+
cause,
163+
}),
164+
),
165+
),
166+
)
167+
: Effect.void,
168+
),
169+
);
170+
});
171+
172+
return ComposerSuggestionService.of({ read, start });
173+
});
174+
175+
export const layer = Layer.effect(ComposerSuggestionService, make);
Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
import { MessageId, ProviderThreadId, RunId } from "@t3tools/contracts";
3+
4+
import {
5+
composerSuggestionCacheIsWarm,
6+
composerSuggestionTarget,
7+
lastAssistantReply,
8+
type ComposerSuggestionMessage,
9+
type ComposerSuggestionRecords,
10+
type ComposerSuggestionTurn,
11+
} from "./decide.ts";
12+
13+
const NOW_MS = Date.parse("2026-11-03T12:00:00.000Z");
14+
15+
const runId = RunId.make("run-1");
16+
const otherRunId = RunId.make("run-2");
17+
const providerThreadId = ProviderThreadId.make("provider-thread");
18+
19+
const userMessage: ComposerSuggestionMessage = {
20+
id: MessageId.make("message-user"),
21+
role: "user",
22+
runId,
23+
streaming: false,
24+
text: "Fix the parser and run the tests.",
25+
};
26+
27+
const reply = (over: Partial<ComposerSuggestionMessage> = {}): ComposerSuggestionMessage => ({
28+
id: MessageId.make("message-reply"),
29+
role: "assistant",
30+
runId,
31+
streaming: false,
32+
text: "Fixed the parser. Want me to run the tests?",
33+
...over,
34+
});
35+
36+
const turn = (over: Partial<ComposerSuggestionTurn> = {}): ComposerSuggestionTurn => ({
37+
providerThreadId,
38+
status: "completed",
39+
ordinal: 1,
40+
tokenUsage: {
41+
inputTokens: 400,
42+
cachedInputTokens: 30_000,
43+
promptCacheExpiresAt: "2026-11-03T12:02:00.000Z",
44+
updatedAt: "2026-11-03T12:00:00.000Z",
45+
},
46+
...over,
47+
});
48+
49+
const records = (
50+
over: Partial<Omit<ComposerSuggestionRecords, "thread">> & {
51+
readonly thread?: Partial<ComposerSuggestionRecords["thread"]>;
52+
} = {},
53+
): ComposerSuggestionRecords => ({
54+
thread: {
55+
archivedAt: null,
56+
activeProviderThreadId: providerThreadId,
57+
lineage: { relationshipToParent: null },
58+
...over.thread,
59+
},
60+
messages: over.messages ?? [userMessage, reply()],
61+
providerTurns: over.providerTurns ?? [turn()],
62+
providerThreads: over.providerThreads ?? [
63+
{ id: providerThreadId, nativeThreadRef: { nativeId: "session-1" } },
64+
],
65+
});
66+
67+
const target = (over?: Parameters<typeof records>[0], currentRunId = runId, nowMs = NOW_MS) =>
68+
composerSuggestionTarget(records(over), currentRunId, nowMs);
69+
70+
describe("composerSuggestionTarget", () => {
71+
it("follows the reply that closed the run", () => {
72+
expect(target()?.reply.id).toBe("message-reply");
73+
});
74+
75+
it("ignores a thread with nothing to follow", () => {
76+
expect(target({ messages: [userMessage] })).toBeNull();
77+
expect(target({ messages: [userMessage, reply({ streaming: true })] })).toBeNull();
78+
expect(target({ messages: [userMessage, reply({ text: " " })] })).toBeNull();
79+
expect(target({ messages: [userMessage, reply({ role: "user" })] })).toBeNull();
80+
});
81+
82+
it("only follows the run that just finished", () => {
83+
expect(target(undefined, otherRunId)).toBeNull();
84+
});
85+
86+
it("skips archived threads and subagent threads", () => {
87+
expect(target({ thread: { archivedAt: "2026-11-03T12:00:00.000Z" } })).toBeNull();
88+
expect(target({ thread: { lineage: { relationshipToParent: "subagent" } } })).toBeNull();
89+
});
90+
91+
it("needs the thread's live native session", () => {
92+
expect(target({ providerThreads: [] })).toBeNull();
93+
expect(
94+
target({ providerThreads: [{ id: providerThreadId, nativeThreadRef: null }] }),
95+
).toBeNull();
96+
expect(
97+
target({ thread: { activeProviderThreadId: ProviderThreadId.make("other-thread") } }),
98+
).toBeNull();
99+
});
100+
101+
it("needs a completed turn with reported usage", () => {
102+
expect(target({ providerTurns: [] })).toBeNull();
103+
expect(target({ providerTurns: [turn({ status: "failed" })] })).toBeNull();
104+
expect(target({ providerTurns: [turn({ tokenUsage: undefined })] })).toBeNull();
105+
});
106+
107+
it("reads the latest completed turn, not the first", () => {
108+
const earlier = turn({
109+
ordinal: 1,
110+
tokenUsage: {
111+
inputTokens: 1,
112+
cachedInputTokens: 1,
113+
updatedAt: "2026-11-03T12:00:00.000Z",
114+
},
115+
});
116+
const latest = turn({
117+
ordinal: 2,
118+
tokenUsage: {
119+
inputTokens: 400,
120+
cachedInputTokens: 30_000,
121+
promptCacheExpiresAt: "2026-11-03T12:02:00.000Z",
122+
updatedAt: "2026-11-03T12:00:00.000Z",
123+
},
124+
});
125+
expect(target({ providerTurns: [latest, earlier] })?.reply.id).toBe("message-reply");
126+
expect(composerSuggestionCacheIsWarm(turn({ ordinal: 2 }), NOW_MS + 10 * 60_000)).toBe(false);
127+
});
128+
});
129+
130+
describe("composerSuggestionCacheIsWarm", () => {
131+
it("needs a reported cache hit", () => {
132+
expect(
133+
composerSuggestionCacheIsWarm(
134+
turn({
135+
tokenUsage: {
136+
inputTokens: 30_000,
137+
cachedInputTokens: 0,
138+
updatedAt: "2026-11-03T12:00:00.000Z",
139+
},
140+
}),
141+
NOW_MS,
142+
),
143+
).toBe(false);
144+
});
145+
146+
it("stops at the prefix ceiling", () => {
147+
expect(
148+
composerSuggestionCacheIsWarm(
149+
turn({
150+
tokenUsage: {
151+
inputTokens: 10_000,
152+
cachedInputTokens: 60_001,
153+
promptCacheExpiresAt: "2026-11-03T12:02:00.000Z",
154+
updatedAt: "2026-11-03T12:00:00.000Z",
155+
},
156+
}),
157+
NOW_MS,
158+
),
159+
).toBe(false);
160+
});
161+
162+
it("stops once the reported cache expires", () => {
163+
expect(composerSuggestionCacheIsWarm(turn(), NOW_MS + 180_000)).toBe(false);
164+
});
165+
166+
it("falls back to the five-minute default when no expiry is reported", () => {
167+
const usage = {
168+
inputTokens: 400,
169+
cachedInputTokens: 30_000,
170+
updatedAt: "2026-11-03T12:00:00.000Z",
171+
};
172+
expect(composerSuggestionCacheIsWarm(turn({ tokenUsage: usage }), NOW_MS + 60_000)).toBe(true);
173+
expect(composerSuggestionCacheIsWarm(turn({ tokenUsage: usage }), NOW_MS + 6 * 60_000)).toBe(
174+
false,
175+
);
176+
});
177+
178+
it("treats an unparseable expiry as cold", () => {
179+
expect(
180+
composerSuggestionCacheIsWarm(
181+
turn({
182+
tokenUsage: {
183+
inputTokens: 1,
184+
cachedInputTokens: 1,
185+
promptCacheExpiresAt: "not a date",
186+
updatedAt: "also not a date",
187+
},
188+
}),
189+
NOW_MS,
190+
),
191+
).toBe(false);
192+
});
193+
});
194+
195+
describe("lastAssistantReply", () => {
196+
it("returns the final assistant reply with text", () => {
197+
expect(
198+
lastAssistantReply([userMessage, reply(), reply({ id: MessageId.make("later") })])?.id,
199+
).toBe("later");
200+
expect(lastAssistantReply([reply({ streaming: true })])).toBeUndefined();
201+
});
202+
});

0 commit comments

Comments
 (0)