Skip to content

Commit ce5c2c1

Browse files
committed
fix(server): validate concurrent no-op command receipts
1 parent 91f9d97 commit ce5c2c1

2 files changed

Lines changed: 108 additions & 45 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.receipt-race.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { vi } from "vite-plus/test";
12
import { assert, it } from "@effect/vitest";
23
import {
34
CommandId,
@@ -109,6 +110,50 @@ it.effect("a command id reused concurrently for two threads succeeds for only on
109110
assert.equal(models[winner], "gpt-6");
110111
assert.equal(models[loser], "gpt-5.1-codex");
111112

113+
const noOpArrived = yield* Ref.make(0);
114+
const bothNoOpsArrived = yield* Deferred.make<void>();
115+
const getThreadRecords = projections.getThreadRecords;
116+
using _records = vi.spyOn(projections, "getThreadRecords").mockImplementation((...args) =>
117+
Effect.gen(function* () {
118+
if ((yield* Ref.updateAndGet(noOpArrived, (count) => count + 1)) === 2) {
119+
yield* Deferred.succeed(bothNoOpsArrived, undefined);
120+
}
121+
yield* Deferred.await(bothNoOpsArrived);
122+
return yield* getThreadRecords(...args);
123+
}),
124+
);
125+
const closeShare = (threadId: ThreadId) =>
126+
orchestrator.dispatch({
127+
type: "thread.guest-share.close",
128+
commandId: CommandId.make("command:shared-noop"),
129+
threadId,
130+
});
131+
const noOpExits = yield* Effect.forEach(
132+
threadIds,
133+
(threadId) => Effect.exit(closeShare(threadId)),
134+
{
135+
concurrency: "unbounded",
136+
},
137+
);
138+
const noOpWinner = noOpExits.findIndex(Exit.isSuccess);
139+
const noOpLoser = noOpExits.findIndex(Exit.isFailure);
140+
assert.isAtLeast(noOpWinner, 0);
141+
assert.isAtLeast(noOpLoser, 0);
142+
const noOpFailure = noOpExits[noOpLoser]!;
143+
if (Exit.isFailure(noOpFailure)) {
144+
const error = Cause.findErrorOption(noOpFailure.cause);
145+
assert.isTrue(error._tag === "Some");
146+
if (error._tag === "Some") {
147+
assert.equal(error.value._tag, "OrchestratorCommandIdConflictError");
148+
}
149+
}
150+
const noOpReplay = yield* closeShare(threadIds[noOpWinner]!);
151+
const noOpSuccess = noOpExits[noOpWinner]!;
152+
if (Exit.isSuccess(noOpSuccess)) {
153+
assert.equal(noOpReplay.sequence, noOpSuccess.value.sequence);
154+
assert.deepEqual(noOpReplay.storedEvents, []);
155+
}
156+
112157
// The winner's own thread still replays its receipt.
113158
const replay = yield* setModel(threadIds[winner]!);
114159
if (Exit.isSuccess(winnerExit)) {

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 63 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ import {
104104
isNativeMaintenanceCommand,
105105
resolveMessageDispatchIntent,
106106
} from "./CommandPolicy.ts";
107-
import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
107+
import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts";
108108
import {
109109
ContextHandoffServiceV2,
110110
readForkHistoryItems,
@@ -13239,6 +13239,54 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1323913239
};
1324013240
});
1324113241

13242+
const replayCommandReceipt = Effect.fn("orchestrationV2.dispatch.replayReceipt")(function* (
13243+
command: OrchestrationV2ServerCommand,
13244+
receipt: CommandReceiptV2,
13245+
) {
13246+
if (receipt.status === "rejected") {
13247+
return yield* new OrchestratorCommandPreviouslyRejectedError({
13248+
commandId: command.commandId,
13249+
commandType: command.type,
13250+
detail: receipt.error ?? "Previously rejected.",
13251+
});
13252+
}
13253+
const dispatchThreadId = commandThreadId(command);
13254+
// Receipts only prove execution for the thread they were recorded against.
13255+
if (receipt.threadId !== dispatchThreadId) {
13256+
return yield* new OrchestratorCommandIdConflictError({
13257+
commandId: command.commandId,
13258+
commandType: command.type,
13259+
receiptThreadId: receipt.threadId,
13260+
commandThreadId: dispatchThreadId,
13261+
});
13262+
}
13263+
const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe(
13264+
Stream.runCollect,
13265+
Effect.map((events): ReadonlyArray<OrchestrationV2StoredEvent> => Array.from(events)),
13266+
Effect.mapError(
13267+
(cause) =>
13268+
new OrchestratorDispatchError({
13269+
commandId: command.commandId,
13270+
commandType: command.type,
13271+
cause,
13272+
}),
13273+
),
13274+
);
13275+
if (command.type === "queue.resume" || command.type === "queued-run.start") {
13276+
yield* mapDispatchError(command)(
13277+
startNextQueuedRun(
13278+
command.threadId,
13279+
command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
13280+
),
13281+
);
13282+
}
13283+
return {
13284+
sequence: receipt.resultSequence,
13285+
storedEvents,
13286+
replayed: true,
13287+
} satisfies OrchestratorV2DispatchResult;
13288+
});
13289+
1324213290
/** `sideChatIds` are the live side chats a `thread.delete` cascades to, read under their locks. */
1324313291
const dispatchWithReceiptEffect = Effect.fn("orchestrationV2.dispatch.withReceipt")(function* (
1324413292
command: OrchestrationV2ServerCommand,
@@ -13262,49 +13310,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1326213310
);
1326313311

1326413312
if (Option.isSome(existingReceipt)) {
13265-
const receipt = existingReceipt.value;
13266-
if (receipt.status === "rejected") {
13267-
return yield* new OrchestratorCommandPreviouslyRejectedError({
13268-
commandId: command.commandId,
13269-
commandType: command.type,
13270-
detail: receipt.error ?? "Previously rejected.",
13271-
});
13272-
}
13273-
const dispatchThreadId = commandThreadId(command);
13274-
// Receipts only prove execution for the thread they were recorded against.
13275-
if (receipt.threadId !== dispatchThreadId) {
13276-
return yield* new OrchestratorCommandIdConflictError({
13277-
commandId: command.commandId,
13278-
commandType: command.type,
13279-
receiptThreadId: receipt.threadId,
13280-
commandThreadId: dispatchThreadId,
13281-
});
13282-
}
13283-
const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe(
13284-
Stream.runCollect,
13285-
Effect.map((events): ReadonlyArray<OrchestrationV2StoredEvent> => Array.from(events)),
13286-
Effect.mapError(
13287-
(cause) =>
13288-
new OrchestratorDispatchError({
13289-
commandId: command.commandId,
13290-
commandType: command.type,
13291-
cause,
13292-
}),
13293-
),
13294-
);
13295-
if (command.type === "queue.resume" || command.type === "queued-run.start") {
13296-
yield* mapDispatchError(command)(
13297-
startNextQueuedRun(
13298-
command.threadId,
13299-
command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
13300-
),
13301-
);
13302-
}
13303-
return {
13304-
sequence: receipt.resultSequence,
13305-
storedEvents,
13306-
replayed: true,
13307-
} satisfies OrchestratorV2DispatchResult;
13313+
return yield* replayCommandReceipt(command, existingReceipt.value);
1330813314
}
1330913315

1331013316
// Recheck under the lock; the user may have raised modes since sender preflight.
@@ -13399,10 +13405,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1339913405
}),
1340013406
),
1340113407
);
13408+
if (!inserted) {
13409+
const receipt = yield* mapDispatchError(command)(
13410+
commandReceipts.getByCommandId(command.commandId),
13411+
);
13412+
if (Option.isNone(receipt)) {
13413+
return yield* new OrchestratorDispatchError({
13414+
commandId: command.commandId,
13415+
commandType: command.type,
13416+
cause: "Conflicting receipt is not a thread command receipt.",
13417+
});
13418+
}
13419+
return yield* replayCommandReceipt(command, receipt.value);
13420+
}
1340213421
return {
1340313422
sequence: resultSequence,
1340413423
storedEvents: [],
13405-
...(inserted ? {} : { replayed: true as const }),
1340613424
} satisfies OrchestratorV2DispatchResult;
1340713425
}
1340813426
const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now);

0 commit comments

Comments
 (0)