Skip to content

Commit 47c9d94

Browse files
voltcrashclaude
andcommitted
fix(server): queued messages land after a resume that ran before them
A resumed turn (after a usage limit or a stop) starts while older queued messages wait in the held queue. It takes a newer run ordinal, and the queued runs keep their older ones, so once they run they render above the resume and its replies while their live output streams at the bottom. When a queued run starts after a newer run already started, give it the next ordinal so the transcript, checkpoints, and latest-run logic follow the order runs actually ran in. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent bd2346e commit 47c9d94

3 files changed

Lines changed: 143 additions & 10 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -370,8 +370,9 @@ export class OrchestratorV2 extends Context.Service<OrchestratorV2, Orchestrator
370370
"t3/orchestration-v2/Orchestrator/OrchestratorV2",
371371
) {}
372372

373+
/** Run ordinals can skip values: a queued run moves past newer runs when it starts. */
373374
function nextRunOrdinal(projection: Pick<OrchestrationV2ThreadProjection, "runs">): number {
374-
return projection.runs.length + 1;
375+
return projection.runs.reduce((max, run) => Math.max(max, run.ordinal), 0) + 1;
375376
}
376377

377378
/**
@@ -1326,6 +1327,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13261327
);
13271328
return;
13281329
}
1330+
// Run ordinals order the transcript, checkpoints, and the latest run. A
1331+
// run that started while this one waited (a resume of a held queue, a
1332+
// reorder, a delivery that jumps the queue) has a newer ordinal, so this
1333+
// run moves after it. A message cancelled from the queue never started.
1334+
const startedRunIds = new Set(
1335+
projection.turnItems.flatMap((item) =>
1336+
item.type === "user_message" && item.runId !== null ? [item.runId] : [],
1337+
),
1338+
);
1339+
const ordinal = projection.runs.some(
1340+
(run) =>
1341+
run.status !== "queued" && run.ordinal > queuedRun.ordinal && startedRunIds.has(run.id),
1342+
)
1343+
? nextRunOrdinal(projection)
1344+
: queuedRun.ordinal;
13291345
const rootNodeId = queuedRun.rootNodeId;
13301346
const attemptId = queuedRun.activeAttemptId;
13311347
const providerThreadId = queuedRun.providerThreadId;
@@ -1599,8 +1615,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15991615
...deliveryProviderThread,
16001616
providerSessionId,
16011617
status: "not_loaded",
1602-
firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? queuedRun.ordinal,
1603-
lastRunOrdinal: queuedRun.ordinal,
1618+
firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? ordinal,
1619+
lastRunOrdinal: ordinal,
16041620
handoffIds: appendContextHandoffId(
16051621
appendContextHandoffId(queuedProviderThread.handoffIds, handoff?.id ?? null),
16061622
legacyImportRecoveryHandoff?.id ?? null,
@@ -1609,6 +1625,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16091625
};
16101626
const startingRun: OrchestrationV2Run = {
16111627
...queuedRun,
1628+
ordinal,
16121629
status: "starting",
16131630
queuePosition: null,
16141631
startedAt: null,
@@ -1629,7 +1646,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16291646
providerTurnId: null,
16301647
nativeItemRef: null,
16311648
parentItemId: null,
1632-
ordinal: queuedRun.ordinal * 100,
1649+
ordinal: ordinal * 100,
16331650
status: "completed",
16341651
title: null,
16351652
type: "user_message",
@@ -1666,7 +1683,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
16661683
providerTurnId: null,
16671684
nativeItemRef: null,
16681685
parentItemId: null,
1669-
ordinal: queuedRun.ordinal * 100 - 1,
1686+
ordinal: ordinal * 100 - 1,
16701687
status: "completed",
16711688
title: handoff === null ? "Imported context" : "Provider handoff",
16721689
startedAt: now,

‎apps/server/src/orchestration-v2/ThreadStop.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,9 @@ it.effect.each(["thread.stop", "run.interrupt"] as const)(
464464
creationSource: "server",
465465
restartContinuationOfRunId: source.id,
466466
});
467+
// The held run resumed after the cancelled source ran, so it sorts after it.
467468
assert.deepEqual(yield* threadState(threadId), {
468-
runs: ["completed", "interrupted", "cancelled"],
469+
runs: ["completed", "cancelled", "interrupted"],
469470
watched: [],
470471
});
471472
}).pipe(Effect.provide(layerTest.pipe(Layer.provideMerge(TestClock.layer())))),

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 119 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4180,6 +4180,115 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
41804180
}),
41814181
);
41824182

4183+
it.effect("places held queued messages after a continuation that started first", () =>
4184+
Effect.gen(function* () {
4185+
const orchestrator = yield* Orchestrator.OrchestratorV2;
4186+
const eventSink = yield* EventSink.EventSinkV2;
4187+
const threadId = ThreadId.make("runtime-layer-continuation-before-queue");
4188+
const messageId = (key: string) => MessageId.make(`${threadId}:${key}`);
4189+
yield* orchestrator.dispatch({
4190+
type: "thread.create",
4191+
createdBy: "user",
4192+
creationSource: "web",
4193+
commandId: CommandId.make(`${threadId}:create`),
4194+
threadId,
4195+
projectId: ProjectId.make(`${threadId}:project`),
4196+
title: "Continuation before queue",
4197+
modelSelection,
4198+
runtimeMode: "full-access",
4199+
interactionMode: "default",
4200+
branch: null,
4201+
worktreePath: process.cwd(),
4202+
});
4203+
const dispatch = (key: string, start: boolean) =>
4204+
orchestrator.dispatch({
4205+
type: "message.dispatch",
4206+
createdBy: "user",
4207+
creationSource: "web",
4208+
commandId: CommandId.make(`${threadId}:message:${key}`),
4209+
threadId,
4210+
messageId: messageId(key),
4211+
text: key,
4212+
attachments: [],
4213+
modelSelection,
4214+
dispatchMode: { type: start ? "start_immediately" : "queue_after_active" },
4215+
});
4216+
const runFor = (key: string) =>
4217+
Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => {
4218+
const run = runs.find((candidate) => candidate.userMessageId === messageId(key));
4219+
assert.isDefined(run);
4220+
return run;
4221+
});
4222+
yield* dispatch("active", true);
4223+
yield* dispatch("first-queued", false);
4224+
yield* dispatch("second-queued", false);
4225+
yield* orchestrator.dispatch({
4226+
type: "run.interrupt",
4227+
commandId: CommandId.make(`${threadId}:interrupt`),
4228+
threadId,
4229+
runId: (yield* runFor("active")).id,
4230+
holdQueue: true,
4231+
});
4232+
// The continuation starts while the older messages wait in the held queue.
4233+
yield* dispatch("continue", true);
4234+
yield* orchestrator.dispatch({
4235+
type: "queue.resume",
4236+
commandId: CommandId.make(`${threadId}:resume`),
4237+
threadId,
4238+
});
4239+
4240+
const startedRunIds = yield* Queue.unbounded<RunId>();
4241+
const afterSequence = yield* orchestrator.getThreadEventSequence(threadId);
4242+
yield* eventSink.stream({ threadId, afterSequence }).pipe(
4243+
Stream.runForEach((stored) =>
4244+
stored.event.type === "run.updated" && stored.event.payload.status === "starting"
4245+
? Queue.offer(startedRunIds, stored.event.payload.id)
4246+
: Effect.void,
4247+
),
4248+
Effect.forkScoped,
4249+
);
4250+
yield* Effect.yieldNow;
4251+
const complete = (key: string) =>
4252+
Effect.gen(function* () {
4253+
const run = yield* runFor(key);
4254+
const now = yield* DateTime.now;
4255+
yield* eventSink.write({
4256+
events: [
4257+
{
4258+
id: EventId.make(`${threadId}:${key}:completed`),
4259+
type: "run.updated",
4260+
threadId,
4261+
runId: run.id,
4262+
...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
4263+
providerInstanceId: run.providerInstanceId,
4264+
occurredAt: now,
4265+
payload: { ...run, status: "completed", startedAt: now, completedAt: now },
4266+
},
4267+
],
4268+
});
4269+
});
4270+
4271+
yield* complete("continue");
4272+
assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("first-queued")).id);
4273+
yield* complete("first-queued");
4274+
assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("second-queued")).id);
4275+
4276+
const keys = ["active", "continue", "first-queued", "second-queued"];
4277+
const runs = yield* Effect.forEach(keys, runFor);
4278+
const ordinals = runs.map((run) => run.ordinal);
4279+
assert.deepEqual(
4280+
ordinals,
4281+
ordinals.toSorted((left, right) => left - right),
4282+
);
4283+
const { turnItems } = yield* orchestrator.getThreadProjection(threadId);
4284+
const userItems = turnItems.filter((item) => item.type === "user_message");
4285+
assert.deepEqual(
4286+
userItems.map((item) => item.messageId),
4287+
keys.map(messageId),
4288+
);
4289+
}),
4290+
);
4291+
41834292
it.effect.each(["startup", "shutdown"] as const)(
41844293
"preserves and holds queued messages across %s until explicitly resumed",
41854294
(trigger) =>
@@ -5419,8 +5528,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
54195528
: 0),
54205529
);
54215530
if (scenario === "queued-resume") {
5422-
assert.equal(after.runs[1]?.status, "queued");
5423-
assert.isTrue(after.runs[1]?.queueHeld);
5531+
const held = after.runs[1]!;
5532+
assert.equal(held.status, "queued");
5533+
assert.isTrue(held.queueHeld);
54245534
const continuation = after.runs[2]!;
54255535
const completedAt = yield* DateTime.now;
54265536
yield* events.write({
@@ -5441,8 +5551,13 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
54415551
threadId,
54425552
});
54435553
const resumed = yield* orchestrator.getThreadProjection(threadId);
5444-
assert.equal(resumed.runs[1]?.status, "starting");
5445-
assert.isFalse(resumed.runs[1]?.queueHeld);
5554+
// The held message ran after the continuation, so it now sorts after it.
5555+
assert.deepEqual(
5556+
resumed.runs.map((run) => run.id),
5557+
[run.id, continuation.id, held.id],
5558+
);
5559+
assert.equal(resumed.runs[2]?.status, "starting");
5560+
assert.isFalse(resumed.runs[2]?.queueHeld);
54465561
}
54475562
}),
54485563
);

0 commit comments

Comments
 (0)