|
| 1 | +import { |
| 2 | + type MessageId, |
| 3 | + OrchestrationV2SuggestComposerReplyError, |
| 4 | + type OrchestrationV2SuggestComposerReplyResult, |
| 5 | + type RunId, |
| 6 | + type ThreadId, |
| 7 | +} from "@t3tools/contracts"; |
| 8 | +import * as Cause from "effect/Cause"; |
| 9 | +import * as Context from "effect/Context"; |
| 10 | +import * as DateTime from "effect/DateTime"; |
| 11 | +import * as Effect from "effect/Effect"; |
| 12 | +import * as FiberMap from "effect/FiberMap"; |
| 13 | +import * as Layer from "effect/Layer"; |
| 14 | +import * as Ref from "effect/Ref"; |
| 15 | +import type * as Scope from "effect/Scope"; |
| 16 | +import * as Stream from "effect/Stream"; |
| 17 | + |
| 18 | +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; |
| 19 | +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; |
| 20 | +import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; |
| 21 | +import { forkParked } from "../serverActivation.ts"; |
| 22 | +import * as ServerSettings from "../serverSettings.ts"; |
| 23 | +import { composerSuggestionTarget, lastAssistantReply } from "./decide.ts"; |
| 24 | +import { cleanComposerSuggestion } from "./filter.ts"; |
| 25 | +import { COMPOSER_SUGGESTION_INSTRUCTION } from "./instruction.ts"; |
| 26 | + |
| 27 | +/** |
| 28 | + * Predicts the user's next message after each completed turn by forking the finished session, so |
| 29 | + * the provider's prompt cache covers the conversation and only the instruction is new input. It |
| 30 | + * lives in memory, never in the event log: chrome around the next turn, not conversation. |
| 31 | + */ |
| 32 | +const SUGGESTION_TTL_MS = 15 * 60_000; |
| 33 | + |
| 34 | +/** Threads kept in memory; older entries fall out first. */ |
| 35 | +const MAX_CACHED_THREADS = 200; |
| 36 | + |
| 37 | +interface CachedSuggestion { |
| 38 | + readonly messageId: MessageId; |
| 39 | + readonly text: string; |
| 40 | + readonly storedAt: number; |
| 41 | +} |
| 42 | + |
| 43 | +const replyError = (threadId: ThreadId, cause: unknown) => |
| 44 | + new OrchestrationV2SuggestComposerReplyError({ |
| 45 | + threadId, |
| 46 | + message: "Failed to read the composer suggestion", |
| 47 | + cause, |
| 48 | + }); |
| 49 | + |
| 50 | +/** |
| 51 | + * Predicts the user's next message after each completed turn by forking the finished session, so |
| 52 | + * the provider's prompt cache covers the conversation and only the instruction is new input. It |
| 53 | + * lives in memory, never in the event log: chrome around the next turn, not conversation. |
| 54 | + */ |
| 55 | +export class ComposerSuggestionService extends Context.Service< |
| 56 | + ComposerSuggestionService, |
| 57 | + { |
| 58 | + readonly read: (input: { |
| 59 | + readonly threadId: ThreadId; |
| 60 | + }) => Effect.Effect< |
| 61 | + OrchestrationV2SuggestComposerReplyResult, |
| 62 | + OrchestrationV2SuggestComposerReplyError |
| 63 | + >; |
| 64 | + readonly start: () => Effect.Effect<void, never, Scope.Scope>; |
| 65 | + } |
| 66 | +>()("t3/composerSuggestion/ComposerSuggestionService") {} |
| 67 | + |
| 68 | +const make = Effect.gen(function* () { |
| 69 | + const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 70 | + const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 71 | + const settings = yield* ServerSettings.ServerSettingsService; |
| 72 | + const instances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; |
| 73 | + const cache = yield* Ref.make(new Map<ThreadId, CachedSuggestion>()); |
| 74 | + |
| 75 | + const store = (threadId: ThreadId, messageId: MessageId, text: string, nowMs: number) => |
| 76 | + Ref.update(cache, (current) => { |
| 77 | + const next = new Map(current); |
| 78 | + next.delete(threadId); |
| 79 | + next.set(threadId, { messageId, text, storedAt: nowMs }); |
| 80 | + while (next.size > MAX_CACHED_THREADS) { |
| 81 | + const oldest = next.keys().next().value; |
| 82 | + if (oldest === undefined) break; |
| 83 | + next.delete(oldest); |
| 84 | + } |
| 85 | + return next; |
| 86 | + }); |
| 87 | + |
| 88 | + const read: ComposerSuggestionService["Service"]["read"] = Effect.fn( |
| 89 | + "ComposerSuggestionService.read", |
| 90 | + )(function* (input) { |
| 91 | + const current = yield* settings.getSettings.pipe( |
| 92 | + Effect.mapError((cause) => replyError(input.threadId, cause)), |
| 93 | + ); |
| 94 | + if (!current.composerReplySuggestions) return { text: "" }; |
| 95 | + const records = yield* projections |
| 96 | + .getThreadRecords(input.threadId, ["messages"], { messageRoles: ["assistant"] }) |
| 97 | + .pipe(Effect.mapError((cause) => replyError(input.threadId, cause))); |
| 98 | + const reply = lastAssistantReply(records.messages); |
| 99 | + if (reply === undefined) return { text: "" }; |
| 100 | + const cached = (yield* Ref.get(cache)).get(input.threadId); |
| 101 | + if (cached === undefined || cached.messageId !== reply.id) return { text: "" }; |
| 102 | + const nowMs = DateTime.toEpochMillis(yield* DateTime.now); |
| 103 | + if (nowMs - cached.storedAt > SUGGESTION_TTL_MS) return { text: "" }; |
| 104 | + return { text: cached.text }; |
| 105 | + }); |
| 106 | + |
| 107 | + const suggest = Effect.fn("ComposerSuggestionService.suggest")(function* (input: { |
| 108 | + readonly threadId: ThreadId; |
| 109 | + readonly runId: RunId; |
| 110 | + }) { |
| 111 | + const current = yield* settings.getSettings; |
| 112 | + if (!current.composerReplySuggestions) return; |
| 113 | + const records = yield* projections.getThreadRecords( |
| 114 | + input.threadId, |
| 115 | + ["messages", "providerTurns", "providerThreads"], |
| 116 | + { messageRoles: ["assistant"] }, |
| 117 | + ); |
| 118 | + const nowMs = DateTime.toEpochMillis(yield* DateTime.now); |
| 119 | + const target = composerSuggestionTarget(records, input.runId, nowMs); |
| 120 | + if (target === null) return; |
| 121 | + const providerThread = records.providerThreads.find( |
| 122 | + (candidate) => candidate.id === target.providerThreadId, |
| 123 | + ); |
| 124 | + if (providerThread === undefined) return; |
| 125 | + const instance = yield* instances.getInstance(records.thread.providerInstanceId); |
| 126 | + const adapter = instance?.orchestrationAdapter; |
| 127 | + const suggestReply = adapter?.suggestComposerReply; |
| 128 | + if (adapter === undefined || suggestReply === undefined) return; |
| 129 | + const result = yield* suggestReply.call(adapter, { |
| 130 | + threadId: input.threadId, |
| 131 | + providerThread, |
| 132 | + instruction: COMPOSER_SUGGESTION_INSTRUCTION, |
| 133 | + }); |
| 134 | + const verdict = cleanComposerSuggestion(result.text); |
| 135 | + if (verdict.kind !== "accepted") { |
| 136 | + return yield* Effect.logDebug("composer suggestion declined", { |
| 137 | + threadId: input.threadId, |
| 138 | + verdict: verdict.kind, |
| 139 | + ...(verdict.kind === "rejected" ? { reason: verdict.reason } : {}), |
| 140 | + }); |
| 141 | + } |
| 142 | + yield* store(input.threadId, target.reply.id, verdict.text, nowMs); |
| 143 | + }); |
| 144 | + |
| 145 | + const start: ComposerSuggestionService["Service"]["start"] = Effect.fn( |
| 146 | + "ComposerSuggestionService.start", |
| 147 | + )(function* () { |
| 148 | + // Keyed by thread: a newer completed run supersedes a prediction still in flight. |
| 149 | + const predictions = yield* FiberMap.make<ThreadId>(); |
| 150 | + yield* forkParked( |
| 151 | + Stream.runForEach(orchestrator.streamDomainEvents, (event) => |
| 152 | + event.type === "run.updated" && event.payload.status === "completed" |
| 153 | + ? FiberMap.run( |
| 154 | + predictions, |
| 155 | + event.threadId, |
| 156 | + suggest({ threadId: event.threadId, runId: event.payload.id }).pipe( |
| 157 | + Effect.catchCause((cause) => |
| 158 | + Cause.hasInterruptsOnly(cause) |
| 159 | + ? Effect.interrupt |
| 160 | + : Effect.logDebug("composer suggestion skipped", { |
| 161 | + threadId: event.threadId, |
| 162 | + cause, |
| 163 | + }), |
| 164 | + ), |
| 165 | + ), |
| 166 | + ) |
| 167 | + : Effect.void, |
| 168 | + ), |
| 169 | + ); |
| 170 | + }); |
| 171 | + |
| 172 | + return ComposerSuggestionService.of({ read, start }); |
| 173 | +}); |
| 174 | + |
| 175 | +export const layer = Layer.effect(ComposerSuggestionService, make); |
0 commit comments