Skip to content

Commit 529d806

Browse files
voltcrashclaude
andauthored
fix(server): queued messages land after a resume that ran before them (pingdotgg#17773)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent b5b6c8c commit 529d806

3 files changed

Lines changed: 153 additions & 10 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -380,8 +380,9 @@ export class OrchestratorV2 extends Context.Service<OrchestratorV2, Orchestrator
380380
"t3/orchestration-v2/Orchestrator/OrchestratorV2",
381381
) {}
382382

383+
/** Run ordinals can skip values: a queued run moves past newer runs when it starts. */
383384
function nextRunOrdinal(projection: Pick<OrchestrationV2ThreadProjection, "runs">): number {
384-
return projection.runs.length + 1;
385+
return projection.runs.reduce((max, run) => Math.max(max, run.ordinal), 0) + 1;
385386
}
386387

387388
/**
@@ -1341,6 +1342,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13411342
);
13421343
return;
13431344
}
1345+
// Run ordinals order the transcript, checkpoints, and the latest run. A
1346+
// run that started while this one waited (a resume of a held queue, a
1347+
// reorder, a delivery that jumps the queue) has a newer ordinal, so this
1348+
// run moves after it. A started run has its prompt in the transcript (a
1349+
// wake's prompt is a notification); a message cancelled from the queue has none.
1350+
const newerRunIds = projection.runs
1351+
.filter((run) => run.status !== "queued" && run.ordinal > queuedRun.ordinal)
1352+
.map((run) => run.id);
1353+
const startedAhead =
1354+
newerRunIds.length > 0 &&
1355+
(yield* projectionStore.getThreadRecords(threadId, ["turnItems"], {
1356+
turnItemTypes: ["user_message", "notification"],
1357+
turnItemRunIds: newerRunIds,
1358+
})).turnItems.length > 0;
1359+
const ordinal = startedAhead ? nextRunOrdinal(projection) : queuedRun.ordinal;
13441360
const rootNodeId = queuedRun.rootNodeId;
13451361
const attemptId = queuedRun.activeAttemptId;
13461362
const providerThreadId = queuedRun.providerThreadId;
@@ -1614,8 +1630,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16141630
...deliveryProviderThread,
16151631
providerSessionId,
16161632
status: "not_loaded",
1617-
firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? queuedRun.ordinal,
1618-
lastRunOrdinal: queuedRun.ordinal,
1633+
firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? ordinal,
1634+
lastRunOrdinal: ordinal,
16191635
handoffIds: appendContextHandoffId(
16201636
appendContextHandoffId(queuedProviderThread.handoffIds, handoff?.id ?? null),
16211637
legacyImportRecoveryHandoff?.id ?? null,
@@ -1624,6 +1640,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16241640
};
16251641
const startingRun: OrchestrationV2Run = {
16261642
...queuedRun,
1643+
ordinal,
16271644
status: "starting",
16281645
queuePosition: null,
16291646
startedAt: null,
@@ -1644,7 +1661,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16441661
providerTurnId: null,
16451662
nativeItemRef: null,
16461663
parentItemId: null,
1647-
ordinal: queuedRun.ordinal * 100,
1664+
ordinal: ordinal * 100,
16481665
status: "completed",
16491666
title: null,
16501667
type: "user_message",
@@ -1681,7 +1698,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16811698
providerTurnId: null,
16821699
nativeItemRef: null,
16831700
parentItemId: null,
1684-
ordinal: queuedRun.ordinal * 100 - 1,
1701+
ordinal: ordinal * 100 - 1,
16851702
status: "completed",
16861703
title: handoff === null ? "Imported context" : "Provider handoff",
16871704
startedAt: now,

‎apps/server/src/orchestration-v2/ThreadStop.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -520,8 +520,9 @@ it.effect.each(["thread.stop", "run.interrupt"] as const)(
520520
creationSource: "server",
521521
restartContinuationOfRunId: source.id,
522522
});
523+
// The held run resumed after the cancelled source ran, so it sorts after it.
523524
assert.deepEqual(yield* threadState(threadId), {
524-
runs: ["completed", "interrupted", "cancelled"],
525+
runs: ["completed", "cancelled", "interrupted"],
525526
watched: [],
526527
});
527528
}).pipe(Effect.provide(layerTest.pipe(Layer.provideMerge(TestClock.layer())))),

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 129 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3972,6 +3972,16 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
39723972
yield* settle("queued", queuedStartedAt);
39733973
assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("late-wake")).id);
39743974
assert.equal(millis((yield* runFor("late-wake")).workStartedAt), millis(queuedStartedAt));
3975+
3976+
// Runs sort in the order they started, including after a wake that ran ahead.
3977+
const ordinals = yield* Effect.forEach(
3978+
["prompt", "early-wake", "queued", "late-wake"],
3979+
(key) => Effect.map(runFor(key), (run) => run.ordinal),
3980+
);
3981+
assert.deepEqual(
3982+
ordinals,
3983+
ordinals.toSorted((left, right) => left - right),
3984+
);
39753985
}),
39763986
);
39773987

@@ -4164,6 +4174,115 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
41644174
}),
41654175
);
41664176

4177+
it.effect("places held queued messages after a continuation that started first", () =>
4178+
Effect.gen(function* () {
4179+
const orchestrator = yield* Orchestrator.OrchestratorV2;
4180+
const eventSink = yield* EventSink.EventSinkV2;
4181+
const threadId = ThreadId.make("runtime-layer-continuation-before-queue");
4182+
const messageId = (key: string) => MessageId.make(`${threadId}:${key}`);
4183+
yield* orchestrator.dispatch({
4184+
type: "thread.create",
4185+
createdBy: "user",
4186+
creationSource: "web",
4187+
commandId: CommandId.make(`${threadId}:create`),
4188+
threadId,
4189+
projectId: ProjectId.make(`${threadId}:project`),
4190+
title: "Continuation before queue",
4191+
modelSelection,
4192+
runtimeMode: "full-access",
4193+
interactionMode: "default",
4194+
branch: null,
4195+
worktreePath: process.cwd(),
4196+
});
4197+
const dispatch = (key: string, start: boolean) =>
4198+
orchestrator.dispatch({
4199+
type: "message.dispatch",
4200+
createdBy: "user",
4201+
creationSource: "web",
4202+
commandId: CommandId.make(`${threadId}:message:${key}`),
4203+
threadId,
4204+
messageId: messageId(key),
4205+
text: key,
4206+
attachments: [],
4207+
modelSelection,
4208+
dispatchMode: { type: start ? "start_immediately" : "queue_after_active" },
4209+
});
4210+
const runFor = (key: string) =>
4211+
Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => {
4212+
const run = runs.find((candidate) => candidate.userMessageId === messageId(key));
4213+
assert.isDefined(run);
4214+
return run;
4215+
});
4216+
yield* dispatch("active", true);
4217+
yield* dispatch("first-queued", false);
4218+
yield* dispatch("second-queued", false);
4219+
yield* orchestrator.dispatch({
4220+
type: "run.interrupt",
4221+
commandId: CommandId.make(`${threadId}:interrupt`),
4222+
threadId,
4223+
runId: (yield* runFor("active")).id,
4224+
holdQueue: true,
4225+
});
4226+
// The continuation starts while the older messages wait in the held queue.
4227+
yield* dispatch("continue", true);
4228+
yield* orchestrator.dispatch({
4229+
type: "queue.resume",
4230+
commandId: CommandId.make(`${threadId}:resume`),
4231+
threadId,
4232+
});
4233+
4234+
const startedRunIds = yield* Queue.unbounded<RunId>();
4235+
const afterSequence = yield* orchestrator.getThreadEventSequence(threadId);
4236+
yield* eventSink.stream({ threadId, afterSequence }).pipe(
4237+
Stream.runForEach((stored) =>
4238+
stored.event.type === "run.updated" && stored.event.payload.status === "starting"
4239+
? Queue.offer(startedRunIds, stored.event.payload.id)
4240+
: Effect.void,
4241+
),
4242+
Effect.forkScoped,
4243+
);
4244+
yield* Effect.yieldNow;
4245+
const complete = (key: string) =>
4246+
Effect.gen(function* () {
4247+
const run = yield* runFor(key);
4248+
const now = yield* DateTime.now;
4249+
yield* eventSink.write({
4250+
events: [
4251+
{
4252+
id: EventId.make(`${threadId}:${key}:completed`),
4253+
type: "run.updated",
4254+
threadId,
4255+
runId: run.id,
4256+
...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
4257+
providerInstanceId: run.providerInstanceId,
4258+
occurredAt: now,
4259+
payload: { ...run, status: "completed", startedAt: now, completedAt: now },
4260+
},
4261+
],
4262+
});
4263+
});
4264+
4265+
yield* complete("continue");
4266+
assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("first-queued")).id);
4267+
yield* complete("first-queued");
4268+
assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("second-queued")).id);
4269+
4270+
const keys = ["active", "continue", "first-queued", "second-queued"];
4271+
const runs = yield* Effect.forEach(keys, runFor);
4272+
const ordinals = runs.map((run) => run.ordinal);
4273+
assert.deepEqual(
4274+
ordinals,
4275+
ordinals.toSorted((left, right) => left - right),
4276+
);
4277+
const { turnItems } = yield* orchestrator.getThreadProjection(threadId);
4278+
const userItems = turnItems.filter((item) => item.type === "user_message");
4279+
assert.deepEqual(
4280+
userItems.map((item) => item.messageId),
4281+
keys.map(messageId),
4282+
);
4283+
}),
4284+
);
4285+
41674286
it.effect.each(["startup", "shutdown"] as const)(
41684287
"preserves and holds queued messages across %s until explicitly resumed",
41694288
(trigger) =>
@@ -5698,8 +5817,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
56985817
: 0),
56995818
);
57005819
if (scenario === "queued-resume") {
5701-
assert.equal(after.runs[1]?.status, "queued");
5702-
assert.isTrue(after.runs[1]?.queueHeld);
5820+
const held = after.runs[1]!;
5821+
assert.equal(held.status, "queued");
5822+
assert.isTrue(held.queueHeld);
57035823
const continuation = after.runs[2]!;
57045824
const completedAt = yield* DateTime.now;
57055825
yield* events.write({
@@ -5720,8 +5840,13 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
57205840
threadId,
57215841
});
57225842
const resumed = yield* orchestrator.getThreadProjection(threadId);
5723-
assert.equal(resumed.runs[1]?.status, "starting");
5724-
assert.isFalse(resumed.runs[1]?.queueHeld);
5843+
// The held message ran after the continuation, so it now sorts after it.
5844+
assert.deepEqual(
5845+
resumed.runs.map((run) => run.id),
5846+
[run.id, continuation.id, held.id],
5847+
);
5848+
assert.equal(resumed.runs[2]?.status, "starting");
5849+
assert.isFalse(resumed.runs[2]?.queueHeld);
57255850
}
57265851
}),
57275852
);

0 commit comments

Comments
 (0)