Skip to content

Commit 7a64505

Browse files
fix(server): preserve message order across lineage reads (pingdotgg#24442)
1 parent c7618f6 commit 7a64505

3 files changed

Lines changed: 113 additions & 15 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ import {
1919
TurnItemId,
2020
} from "@t3tools/contracts";
2121
import * as DateTime from "effect/DateTime";
22+
import * as Deferred from "effect/Deferred";
2223
import * as Effect from "effect/Effect";
24+
import * as Fiber from "effect/Fiber";
2325
import * as Layer from "effect/Layer";
2426
import * as Metric from "effect/Metric";
2527
import * as Result from "effect/Result";
@@ -2083,3 +2085,76 @@ it.effect("dispatch records the command outcome and its duration", () =>
20832085
assert.strictEqual((yield* durations).count - durationsBefore, 1);
20842086
}).pipe(Effect.provide(layerTest)),
20852087
);
2088+
2089+
it.effect("a user message dispatched first stays run 1 when its lineage read is slow", () => {
2090+
const threadId = ThreadId.make("thread:dispatch-order");
2091+
// A file database serves projection reads from a reader worker, so the lineage read a user
2092+
// dispatch makes before taking the thread lock can yield. This gate holds it open instead.
2093+
const lineageReadEntered = Deferred.makeUnsafe<void>();
2094+
const lineageReadRelease = Deferred.makeUnsafe<void>();
2095+
let gateArmed = false;
2096+
const layerGatedLineageRead = ProviderReplayHarness.layerWithRegistry(
2097+
{ name: "dispatch-order" },
2098+
ProviderAdapterRegistry.layerFromAdapters([adapter]),
2099+
{
2100+
runEffectWorker: false,
2101+
wrapProjectionStore: (store) => ({
2102+
...store,
2103+
getThread: (id) => {
2104+
if (!gateArmed || id !== threadId) return store.getThread(id);
2105+
gateArmed = false;
2106+
return Deferred.succeed(lineageReadEntered, undefined).pipe(
2107+
Effect.andThen(Deferred.await(lineageReadRelease)),
2108+
Effect.andThen(store.getThread(id)),
2109+
);
2110+
},
2111+
}),
2112+
},
2113+
);
2114+
return Effect.gen(function* () {
2115+
const orchestrator = yield* Orchestrator.OrchestratorV2;
2116+
yield* orchestrator.dispatch({
2117+
type: "thread.create",
2118+
commandId: CommandId.make("create-dispatch-order"),
2119+
threadId,
2120+
projectId: ProjectId.make("project:dispatch-order"),
2121+
title: "Dispatch order",
2122+
modelSelection,
2123+
runtimeMode: "full-access",
2124+
interactionMode: "default",
2125+
branch: null,
2126+
worktreePath: null,
2127+
createdBy: "user",
2128+
creationSource: "web",
2129+
});
2130+
const message = (id: string, createdBy: "user" | "agent") =>
2131+
orchestrator.dispatch({
2132+
type: "message.dispatch",
2133+
commandId: CommandId.make(id),
2134+
threadId,
2135+
messageId: MessageId.make(id),
2136+
text: id,
2137+
attachments: [],
2138+
dispatchMode: { type: "defer_start" },
2139+
createdBy,
2140+
creationSource: createdBy === "user" ? "web" : "server",
2141+
});
2142+
2143+
gateArmed = true;
2144+
const user = yield* message("message:user-first", "user").pipe(Effect.forkChild);
2145+
yield* Deferred.await(lineageReadEntered);
2146+
// Agent messages skip the lineage read, so this one would take a free thread lock first.
2147+
const agent = yield* message("message:agent-second", "agent").pipe(
2148+
Effect.forkChild({ startImmediately: true }),
2149+
);
2150+
yield* Deferred.succeed(lineageReadRelease, undefined);
2151+
yield* Fiber.join(agent);
2152+
yield* Fiber.join(user);
2153+
2154+
const { runs } = yield* orchestrator.getThreadProjection(threadId);
2155+
assert.deepEqual(
2156+
runs.toSorted((left, right) => left.ordinal - right.ordinal).map((run) => run.userMessageId),
2157+
[MessageId.make("message:user-first"), MessageId.make("message:agent-second")],
2158+
);
2159+
}).pipe(Effect.provide(layerGatedLineageRead));
2160+
});

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ import {
7777
TrimmedNonEmptyString,
7878
managedRuntimeModeCeilingMessage,
7979
} from "@t3tools/contracts";
80+
import * as KeyedLock from "@t3tools/shared/KeyedLock";
8081
import { modelSelectionsEqual } from "@t3tools/shared/model";
8182
import { modelSwitchSince } from "./modelSwitch.ts";
8283
import { recordThreadOpen } from "@t3tools/shared/paletteSearch";
@@ -1313,6 +1314,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13131314
const runtimePolicy = yield* RuntimePolicyV2;
13141315
const threadForkService = yield* ThreadForkServiceV2;
13151316
const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor;
1317+
const messageIntake = yield* KeyedLock.make<ThreadId>();
13161318
// Read once: the policy is fixed for the server's life, and dispatch takes it as an input.
13171319
const { managedPolicy } = serverSettings;
13181320
const runtimeModeCeiling: RuntimeModeCeiling | undefined =
@@ -14674,6 +14676,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1467414676
} satisfies OrchestratorV2DispatchResult;
1467514677
}, withCommandMetrics);
1467614678

14679+
// Follow-up intake and Stop serialize on the parent before taking the child lock.
14680+
const withSubagentParentLock = <A, E, R>(threadId: ThreadId, dispatch: Effect.Effect<A, E, R>) =>
14681+
appOwnedSubagentParentThreadId(threadId).pipe(
14682+
Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
14683+
Effect.flatMap((parentThreadId) =>
14684+
parentThreadId === undefined ? dispatch : threadDispatch.withLock(parentThreadId, dispatch),
14685+
),
14686+
);
14687+
1467714688
// dispatchWithReceiptEffect owns the dispatch span; this wrapper selects the thread locks.
1467814689
const dispatchWithReceipt = Effect.fnUntraced(function* (command: OrchestrationV2ServerCommand) {
1467914690
const dispatch = threadDispatch.withLock(
@@ -14706,21 +14717,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1470614717
),
1470714718
),
1470814719
);
14720+
// Taken before the lineage read, so a message whose read yields keeps its
14721+
// place ahead of later messages to the same thread.
14722+
if (command.type === "message.dispatch")
14723+
return yield* messageIntake.withLock(
14724+
command.threadId,
14725+
mayStartSubagentFollowup(command)
14726+
? withSubagentParentLock(command.threadId, dispatch)
14727+
: dispatch,
14728+
);
1470914729
// Answering a message-mode question can also start a follow-up.
14710-
if (
14711-
command.type !== "runtime-request.respond" &&
14712-
(command.type !== "message.dispatch" || !mayStartSubagentFollowup(command))
14713-
)
14714-
return yield* dispatch;
14715-
const parentThreadId = yield* appOwnedSubagentParentThreadId(command.threadId).pipe(
14716-
Effect.mapError(
14717-
(cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
14718-
),
14719-
);
14720-
// Follow-up intake and Stop serialize on the parent before taking the child lock.
14721-
return yield* parentThreadId === undefined
14722-
? dispatch
14723-
: threadDispatch.withLock(parentThreadId, dispatch);
14730+
return yield* command.type === "runtime-request.respond"
14731+
? withSubagentParentLock(command.threadId, dispatch)
14732+
: dispatch;
1472414733
});
1472514734

1472614735
const handleRunUpdate = Effect.fn("Orchestrator.handleRunUpdate")(

‎apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,10 @@ export function layerWithRegistry<Error>(
282282
readonly runContinuationWorker?: boolean;
283283
readonly recoverOnStartup?: boolean;
284284
readonly continueThreadsAfterServerUpdate?: boolean;
285+
/** Lets a test hold a projection read open to order concurrent dispatches. */
286+
readonly wrapProjectionStore?: (
287+
store: ProjectionStore.ProjectionStoreV2Shape,
288+
) => ProjectionStore.ProjectionStoreV2Shape;
285289
readonly managedPolicy?: ManagedPolicyDocument;
286290
} = {},
287291
): Layer.Layer<
@@ -322,9 +326,19 @@ export function layerWithRegistry<Error>(
322326
),
323327
),
324328
).pipe(Layer.orDie);
329+
const wrapProjectionStore = options.wrapProjectionStore;
330+
const layerProjectionStore =
331+
wrapProjectionStore === undefined
332+
? ProjectionStore.layer
333+
: Layer.effect(
334+
ProjectionStore.ProjectionStoreV2,
335+
Effect.gen(function* () {
336+
return wrapProjectionStore(yield* ProjectionStore.ProjectionStoreV2);
337+
}),
338+
).pipe(Layer.provide(ProjectionStore.layer));
325339
const layerStores = Layer.mergeAll(
326340
EventStore.layer,
327-
ProjectionStore.layer,
341+
layerProjectionStore,
328342
ProjectStore.layer,
329343
CommandReceiptStore.layer,
330344
EffectOutbox.layer,

0 commit comments

Comments
 (0)