Skip to content

Commit 64f9d72

Browse files
fix(server): validate citations under the thread dispatch lock
1 parent 88988d5 commit 64f9d72

4 files changed

Lines changed: 398 additions & 331 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
127127
import * as RunExecutionService from "./RunExecutionService.ts";
128128
import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts";
129129
import { ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts";
130+
import { providerUserMessageText } from "./ProviderMessageText.ts";
130131
import { isAutomaticCompletionRun, queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
131132
import { RuntimePolicyV2 } from "./RuntimePolicy.ts";
132133
import {
@@ -11207,6 +11208,61 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1120711208
if (shell !== null) yield* refuseAboveDispatchModeLimit(command, threadId, shell);
1120811209
}
1120911210

11211+
if (
11212+
command.type === "message.dispatch" ||
11213+
command.type === "queued-run.edit" ||
11214+
command.type === "queued-message.promote-to-steer"
11215+
) {
11216+
let message: Pick<OrchestrationV2ConversationMessage, "text" | "context"> | undefined;
11217+
if (command.type === "message.dispatch") {
11218+
message = command;
11219+
} else if (command.type !== "queued-run.edit" || command.text.trim().length > 0) {
11220+
const projection = yield* loadProjectionForCommand(
11221+
command,
11222+
command.type === "queued-run.edit"
11223+
? ["runs", "messages"]
11224+
: ["runs", "messages", "nodes", "attempts"],
11225+
{ messageRoles: ["user"] },
11226+
);
11227+
const runId = command.type === "queued-run.edit" ? command.runId : command.queuedRunId;
11228+
const run = projection.runs.find((candidate) => candidate.id === runId);
11229+
const storedMessage = projection.messages.find(
11230+
(candidate) => candidate.id === run?.userMessageId,
11231+
);
11232+
// Leave invalid queue targets to the orchestrator's existing command errors.
11233+
if (
11234+
run?.status === "queued" &&
11235+
storedMessage !== undefined &&
11236+
storedMessage.delegatedCompletion === undefined &&
11237+
storedMessage.notification === undefined
11238+
) {
11239+
if (command.type === "queued-run.edit") {
11240+
message = { text: command.text, context: command.context ?? storedMessage.context };
11241+
} else if (
11242+
command.type === "queued-message.promote-to-steer" &&
11243+
projection.thread.archivedAt === null &&
11244+
projection.thread.deletedAt === null &&
11245+
projection.nodes.some((candidate) => candidate.id === run.rootNodeId) &&
11246+
projection.attempts.some((candidate) => candidate.id === run.activeAttemptId)
11247+
) {
11248+
message = storedMessage;
11249+
}
11250+
}
11251+
}
11252+
if (message !== undefined) {
11253+
yield* providerUserMessageText(message).pipe(
11254+
Effect.mapError(
11255+
(cause) =>
11256+
new OrchestratorCommandRejectedError({
11257+
commandId: command.commandId,
11258+
commandType: command.type,
11259+
cause: cause.message,
11260+
}),
11261+
),
11262+
);
11263+
}
11264+
}
11265+
1121011266
const plan = yield* dispatchOnce(command).pipe(
1121111267
Effect.flatMap((planned) =>
1121211268
planned.events.length > 0 ||

‎apps/server/src/orchestration-v2/ThreadManagementService.test.ts‎

Lines changed: 0 additions & 274 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
import { expect, it } from "@effect/vitest";
22
import {
33
CommandId,
4-
EnvironmentId,
5-
EventId,
6-
PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
74
MessageId,
85
NodeId,
96
type OrchestrationV2Command,
@@ -19,10 +16,8 @@ import {
1916
ProjectId,
2017
ProviderInstanceId,
2118
RunId,
22-
RunAttemptId,
2319
ThreadId,
2420
} from "@t3tools/contracts";
25-
import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
2621
import * as Deferred from "effect/Deferred";
2722
import * as DateTime from "effect/DateTime";
2823
import * as Option from "effect/Option";
@@ -38,15 +33,12 @@ import * as TestClock from "effect/testing/TestClock";
3833
import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts";
3934
import * as ServerSettings from "../serverSettings.ts";
4035
import * as SqlitePersistence from "../persistence/Sqlite.ts";
41-
import { dispatchFailure } from "../mcp/threadAccess.ts";
4236
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
4337
import * as Orchestrator from "./Orchestrator.ts";
44-
import { emptyProjection } from "./ProjectionStore.ts";
4538
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
4639
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
4740
import * as ThreadManagementService from "./ThreadManagementService.ts";
4841
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
49-
import { userFacingDispatchErrorMessage } from "./UserFacingErrors.ts";
5042
import { DispatchModeLimit, type DispatchModeRefusal } from "./DispatchModeLimit.ts";
5143

5244
it("stamps authoritative provenance on commands that create threads or messages", () => {
@@ -435,272 +427,6 @@ it.effect(
435427
},
436428
);
437429

438-
it.effect("rejects an oversized expanded steer to the caller before committing it", () => {
439-
const dispatched: Array<string> = [];
440-
const layerTest = layerDispatchChokepointTest(
441-
ServerSettings.layerTest({
442-
providerInstances: {
443-
[ProviderInstanceId.make("codex_work")]: { driver: "codex", enabled: true },
444-
},
445-
}),
446-
)(dispatched);
447-
const citation = serializeAssistantCitation({
448-
version: 1,
449-
environmentId: EnvironmentId.make("environment:steer-citation"),
450-
threadId: ThreadId.make("thread:steer-citation:source"),
451-
messageId: MessageId.make("message:steer-citation:source"),
452-
text: "Quoted claim",
453-
start: 0,
454-
end: 12,
455-
prefix: "",
456-
suffix: "",
457-
});
458-
const text = `${"x".repeat(PROVIDER_SEND_TURN_MAX_INPUT_CHARS - citation.length - 1)} ${citation}`;
459-
const command = {
460-
type: "message.dispatch",
461-
commandId: CommandId.make("command:steer-citation:oversized"),
462-
threadId: ThreadId.make("thread:thread-management:chokepoint"),
463-
messageId: MessageId.make("message:steer-citation:oversized"),
464-
text,
465-
attachments: [],
466-
dispatchMode: { type: "steer_active", targetRunId: RunId.make("run:active") },
467-
createdBy: "user",
468-
creationSource: "web",
469-
} satisfies OrchestrationV2Command;
470-
471-
return Effect.gen(function* () {
472-
const threads = yield* ThreadManagementService.ThreadManagementService;
473-
const result = yield* Effect.result(threads.dispatch(command));
474-
475-
expect(result._tag).toBe("Failure");
476-
if (result._tag === "Failure") {
477-
expect(userFacingDispatchErrorMessage(result.failure)).toMatch(
478-
/over the .*character limit.*quotes are expanded/,
479-
);
480-
}
481-
expect(dispatched).toEqual([]);
482-
483-
const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
484-
yield* receipts.insertIfAbsent({
485-
commandId: command.commandId,
486-
threadId: command.threadId,
487-
commandType: command.type,
488-
acceptedAt: yield* DateTime.now,
489-
resultSequence: 1,
490-
status: "accepted",
491-
error: null,
492-
});
493-
yield* threads.dispatch(command);
494-
expect(dispatched).toEqual(["message.dispatch"]);
495-
}).pipe(Effect.provide(layerTest));
496-
});
497-
498-
it.effect(
499-
"rejects oversized queue edits and promotion of older oversized messages before committing",
500-
() =>
501-
Effect.gen(function* () {
502-
const now = yield* DateTime.now;
503-
const threadId = ThreadId.make("thread:queue-citation");
504-
const providerInstanceId = ProviderInstanceId.make("codex");
505-
const providerThreadId = ProviderThreadId.make("provider-thread:queue-citation");
506-
const modelSelection = { instanceId: providerInstanceId, model: "test" };
507-
const runs: Array<OrchestrationV2Run> = (["running", "queued"] as const).map(
508-
(status, index) => ({
509-
id: RunId.make(`run:queue-citation:${status}`),
510-
threadId,
511-
ordinal: index + 1,
512-
providerInstanceId,
513-
modelSelection,
514-
providerThreadId,
515-
userMessageId: MessageId.make(`message:queue-citation:${status}`),
516-
rootNodeId: NodeId.make(`node:queue-citation:${status}`),
517-
activeAttemptId: RunAttemptId.make(`attempt:queue-citation:${status}`),
518-
status,
519-
queuePosition: status === "queued" ? 1 : null,
520-
requestedAt: now,
521-
startedAt: status === "running" ? now : null,
522-
completedAt: null,
523-
checkpointId: null,
524-
contextHandoffId: null,
525-
}),
526-
);
527-
let projection = emptyProjection({
528-
id: EventId.make("event:queue-citation:created"),
529-
type: "thread.created",
530-
threadId,
531-
occurredAt: now,
532-
payload: {
533-
id: threadId,
534-
projectId: ProjectId.make("project:queue-citation"),
535-
title: "Queued citations",
536-
providerInstanceId,
537-
modelSelection,
538-
createdBy: "user",
539-
creationSource: "web",
540-
runtimeMode: "full-access",
541-
interactionMode: "default",
542-
branch: null,
543-
worktreePath: null,
544-
activeProviderThreadId: providerThreadId,
545-
lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
546-
forkedFrom: null,
547-
createdAt: now,
548-
updatedAt: now,
549-
archivedAt: null,
550-
settledOverride: null,
551-
settledAt: null,
552-
lastVisitedAt: null,
553-
deletedAt: null,
554-
},
555-
});
556-
projection = {
557-
...projection,
558-
runs,
559-
nodes: runs.map((run) => ({
560-
id: run.rootNodeId!,
561-
threadId,
562-
runId: run.id,
563-
parentNodeId: null,
564-
rootNodeId: run.rootNodeId!,
565-
kind: "root_turn",
566-
status: run.status === "running" ? "running" : "pending",
567-
countsForRun: true,
568-
providerThreadId,
569-
providerTurnId: null,
570-
nativeItemRef: null,
571-
runtimeRequestId: null,
572-
checkpointScopeId: null,
573-
startedAt: run.startedAt,
574-
completedAt: null,
575-
})),
576-
attempts: runs.map((run) => ({
577-
id: run.activeAttemptId!,
578-
runId: run.id,
579-
attemptOrdinal: 1,
580-
rootNodeId: run.rootNodeId!,
581-
providerInstanceId,
582-
providerThreadId,
583-
providerTurnId: null,
584-
reason: "initial",
585-
status: run.status === "running" ? "running" : "pending",
586-
startedAt: run.startedAt,
587-
completedAt: null,
588-
})),
589-
messages: runs.map((run) => ({
590-
id: run.userMessageId,
591-
threadId,
592-
runId: run.id,
593-
nodeId: run.rootNodeId,
594-
role: "user",
595-
text: "Keep this message",
596-
attachments: [],
597-
streaming: false,
598-
createdBy: "user",
599-
creationSource: "web",
600-
createdAt: now,
601-
updatedAt: now,
602-
})),
603-
};
604-
const citation = serializeAssistantCitation({
605-
version: 1,
606-
environmentId: EnvironmentId.make("environment:queue-citation"),
607-
threadId,
608-
messageId: MessageId.make("message:queue-citation:source"),
609-
text: "Quoted claim",
610-
start: 0,
611-
end: 12,
612-
prefix: "",
613-
suffix: "",
614-
});
615-
const text = `${"x".repeat(PROVIDER_SEND_TURN_MAX_INPUT_CHARS - citation.length - 1)} ${citation}`;
616-
const dispatched: Array<string> = [];
617-
const layerTest = ThreadManagementService.layer.pipe(
618-
Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
619-
Layer.provide(
620-
Layer.mock(Orchestrator.OrchestratorV2)({
621-
getThreadRecords: () => Effect.succeed(projection),
622-
dispatch: (command) =>
623-
Effect.sync(() => {
624-
dispatched.push(command.type);
625-
return { sequence: 1, storedEvents: [] };
626-
}),
627-
}),
628-
),
629-
Layer.provide(ServerSettings.layerTest()),
630-
Layer.provideMerge(layerReceiptsTest),
631-
);
632-
yield* Effect.gen(function* () {
633-
const threads = yield* ThreadManagementService.ThreadManagementService;
634-
const edit = {
635-
type: "queued-run.edit",
636-
commandId: CommandId.make("command:queue-citation:edit"),
637-
threadId,
638-
runId: runs[1]!.id,
639-
text,
640-
} satisfies OrchestrationV2Command;
641-
const editError = yield* threads.dispatch(edit).pipe(Effect.flip);
642-
expect(editError).toBeInstanceOf(Orchestrator.OrchestratorCommandRejectedError);
643-
expect(userFacingDispatchErrorMessage(editError)).toMatch(
644-
/over the .*character limit.*quotes are expanded/,
645-
);
646-
expect(dispatchFailure(editError)).toMatchObject({
647-
code: "orchestration_error",
648-
message: expect.stringMatching(
649-
/over the .*character limit.*quotes are expanded.*Shorten it or quote less/,
650-
),
651-
});
652-
expect(projection.messages.map((message) => message.text)).toEqual([
653-
"Keep this message",
654-
"Keep this message",
655-
]);
656-
expect(dispatched).toEqual([]);
657-
658-
// Simulate a message persisted before citation validation existed.
659-
projection = {
660-
...projection,
661-
messages: projection.messages.map((message) =>
662-
message.runId === edit.runId ? { ...message, text } : message,
663-
),
664-
};
665-
const promote = {
666-
type: "queued-message.promote-to-steer",
667-
commandId: CommandId.make("command:queue-citation:promote"),
668-
threadId,
669-
queuedRunId: edit.runId,
670-
targetRunId: runs[0]!.id,
671-
} satisfies OrchestrationV2Command;
672-
const promoteError = yield* threads.dispatch(promote).pipe(Effect.flip);
673-
expect(promoteError).toBeInstanceOf(Orchestrator.OrchestratorCommandRejectedError);
674-
expect(userFacingDispatchErrorMessage(promoteError)).toMatch(
675-
/over the .*character limit.*quotes are expanded/,
676-
);
677-
expect(dispatchFailure(promoteError)).toMatchObject({
678-
code: "orchestration_error",
679-
message: expect.stringMatching(
680-
/over the .*character limit.*quotes are expanded.*Shorten it or quote less/,
681-
),
682-
});
683-
expect(projection.runs.map((run) => run.status)).toEqual(["running", "queued"]);
684-
expect(dispatched).toEqual([]);
685-
686-
const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
687-
for (const command of [edit, promote]) {
688-
yield* receipts.insertIfAbsent({
689-
commandId: command.commandId,
690-
threadId,
691-
commandType: command.type,
692-
acceptedAt: now,
693-
resultSequence: 1,
694-
status: "accepted",
695-
error: null,
696-
});
697-
yield* threads.dispatch(command);
698-
}
699-
expect(dispatched).toEqual([edit.type, promote.type]);
700-
}).pipe(Effect.provide(layerTest));
701-
}),
702-
);
703-
704430
it.effect(
705431
"a replay of an already-accepted command id dispatches despite the instance now being disabled",
706432
() => {

0 commit comments

Comments
 (0)