Skip to content

Commit 737afe2

Browse files
authored
fix(server): retry unstarted runs after recovery (pingdotgg#30560)
1 parent eb1db29 commit 737afe2

5 files changed

Lines changed: 233 additions & 39 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2376,7 +2376,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
23762376
},
23772377
})),
23782378
{
2379-
id: `effect:${commandId}:provider-turn.start:${queuedRun.id}`,
2379+
// A restart can return this run to the queue after its previous start effect was cancelled.
2380+
id: `effect:${commandId}:provider-turn.start:${queuedRun.id}:${yield* idAllocator.allocate.event({ threadId, commandId })}`,
23802381
commandId,
23812382
threadId,
23822383
request: { type: "provider-turn.start", runId: queuedRun.id },

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,19 @@ it.effect("recovery closes a secret card's form but leaves a link card open", ()
501501
}).pipe(Effect.provide(layer));
502502
});
503503

504-
it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart", (status) => {
504+
it.effect.each([
505+
"queued",
506+
"preparing",
507+
"starting",
508+
"starting-other-request",
509+
"starting-own-request",
510+
"starting-scheduled",
511+
] as const)("recovers $0 work after restart", (scenario) => {
512+
const status = scenario.startsWith("starting") ? "starting" : scenario;
513+
const terminalized =
514+
status === "preparing" ||
515+
scenario === "starting-own-request" ||
516+
scenario === "starting-scheduled";
505517
const threadId = ThreadId.make("thread_queued_restart");
506518
const runId = RunId.make("run_queued_restart");
507519
const attemptId = RunAttemptId.make("attempt_queued_restart");
@@ -510,14 +522,28 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
510522
null;
511523
const projection = {
512524
thread: { id: threadId },
513-
runtimeRequests: [],
525+
runtimeRequests: scenario.endsWith("request")
526+
? [
527+
{
528+
id: RuntimeRequestId.make("pending-recovery-request"),
529+
nodeId:
530+
scenario === "starting-own-request" ? rootNodeId : NodeId.make("other-run-request"),
531+
kind: "user_input",
532+
status: "pending",
533+
responseCapability: { type: "message" },
534+
},
535+
]
536+
: [],
514537
providerSessions: [],
515538
providerThreads: [],
516539
providerTurns: [],
517540
runs: [
518541
{
519542
id: runId,
520543
status,
544+
startedAt: null,
545+
activeAttemptId: attemptId,
546+
userMessageId: MessageId.make("message_queued_restart"),
521547
queuePosition: 1,
522548
providerInstanceId: ProviderInstanceId.make("codex"),
523549
...(status === "preparing"
@@ -540,14 +566,32 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
540566
},
541567
],
542568
nodes: [
569+
...(scenario === "starting-other-request"
570+
? [
571+
{
572+
id: NodeId.make("other-run-request"),
573+
runId: RunId.make("earlier-run"),
574+
status: "waiting",
575+
},
576+
]
577+
: []),
543578
{
544579
id: rootNodeId,
545580
runId,
546581
status: "pending",
547582
},
548583
],
549584
subagents: [],
550-
messages: [],
585+
messages:
586+
status === "starting"
587+
? [
588+
{
589+
id: MessageId.make("message_queued_restart"),
590+
createdBy: "user",
591+
...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),
592+
},
593+
]
594+
: [],
551595
turnItems: [],
552596
} as unknown as OrchestrationV2ThreadProjection;
553597
const layer = ProviderRuntimeRecovery.layer.pipe(
@@ -579,7 +623,7 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
579623
return Effect.gen(function* () {
580624
const summary =
581625
yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
582-
assert.equal(summary.terminalizedRuns, status === "preparing" ? 1 : 0);
626+
assert.equal(summary.terminalizedRuns, terminalized ? 1 : 0);
583627
const command = committedInput;
584628
assert.isNotNull(command);
585629
if (command === null) return;
@@ -588,17 +632,17 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
588632
const nodeEvent = command.events.find((event) => event.type === "node.updated");
589633
assert.equal(
590634
runEvent?.type === "run.updated" ? runEvent.payload.status : null,
591-
status === "preparing" ? "cancelled" : "queued",
635+
terminalized ? "cancelled" : "queued",
592636
);
593637
assert.isEmpty(command.effects);
594-
if (status === "preparing") {
638+
if (terminalized) {
595639
assert.equal(
596640
attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,
597641
"cancelled",
598642
);
599643
assert.equal(
600644
nodeEvent?.type === "node.updated" ? nodeEvent.payload.status : null,
601-
"cancelled",
645+
scenario === "starting-own-request" ? null : "cancelled",
602646
);
603647
} else {
604648
assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 81 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
2525
import * as ServerSettings from "../serverSettings.ts";
2626
import { restartContinuationRun } from "./RestartContinuation.ts";
2727
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
28+
import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
2829
import {
2930
endOrphanedNativeSubagent,
3031
endRunlessRootTurns,
@@ -210,7 +211,7 @@ export const make = Effect.gen(function* () {
210211
stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
211212
) {
212213
const now = yield* DateTime.now;
213-
const runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
214+
let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
214215
for (const run of nonterminalRuns(projection)) {
215216
if (run.status === "waiting") {
216217
const checkpointEffects = yield* outbox
@@ -325,13 +326,25 @@ export const make = Effect.gen(function* () {
325326
cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);
326327
}
327328
};
329+
const unstartedRuns = runs.filter(
330+
(run) =>
331+
run.status === "starting" &&
332+
neverStarted(run) &&
333+
!projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) &&
334+
!projection.runtimeRequests.some(
335+
(request) =>
336+
request.status === "pending" &&
337+
projection.nodes.some((node) => node.id === request.nodeId && node.runId === run.id),
338+
),
339+
);
328340
const queuedRuns = projection.runs.filter((run) => run.status === "queued");
341+
const pendingRuns = [...unstartedRuns, ...queuedRuns];
329342
const queuedMessages =
330-
queuedRuns.length === 0
343+
pendingRuns.length === 0
331344
? []
332345
: (yield* projections
333346
.getThreadRecords(projection.thread.id, ["messages"], {
334-
messageIds: queuedRuns.map((run) => run.userMessageId),
347+
messageIds: pendingRuns.map((run) => run.userMessageId),
335348
})
336349
.pipe(
337350
Effect.mapError(
@@ -346,20 +359,82 @@ export const make = Effect.gen(function* () {
346359
const bufferedMessageIds = new Set(
347360
queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
348361
);
349-
for (const run of queuedRuns) {
362+
const retryableMessageIds = new Set(
363+
queuedMessages
364+
.filter(
365+
(message) =>
366+
message.createdBy === "user" &&
367+
message.scheduledTaskId === undefined &&
368+
message.notification === undefined &&
369+
message.delegatedCompletion === undefined,
370+
)
371+
.map((message) => message.id),
372+
);
373+
// A saved user message the provider never accepted stays available through the held queue.
374+
const retryableRuns = unstartedRuns.filter((run) =>
375+
retryableMessageIds.has(run.userMessageId),
376+
);
377+
const retryableRunIds = new Set(retryableRuns.map((run) => run.id));
378+
const retryableItems =
379+
retryableRuns.length === 0
380+
? []
381+
: (yield* projections
382+
.getThreadRecords(projection.thread.id, ["turnItems"], {
383+
turnItemTypes: ["user_message"],
384+
turnItemRunIds: retryableRuns.map((run) => run.id),
385+
})
386+
.pipe(
387+
Effect.mapError(
388+
(cause) =>
389+
new ProviderRuntimeRecoveryError({
390+
operation: "read-projections",
391+
threadId: projection.thread.id,
392+
cause,
393+
}),
394+
),
395+
)).turnItems;
396+
for (const item of retryableItems) {
397+
if (item.type !== "user_message" || !retryableMessageIds.has(item.messageId)) continue;
398+
events.push({
399+
id: yield* allocateEventId(),
400+
type: "turn-item.updated",
401+
threadId: projection.thread.id,
402+
...(item.runId === null ? {} : { runId: item.runId }),
403+
occurredAt: now,
404+
payload: {
405+
...item,
406+
inputIntent: "queued_turn",
407+
status: "pending",
408+
startedAt: null,
409+
completedAt: null,
410+
updatedAt: now,
411+
},
412+
});
413+
}
414+
runs = runs.filter((run) => !retryableRunIds.has(run.id));
415+
const heldRuns = [
416+
...retryableRuns,
417+
...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
418+
];
419+
for (const [index, run] of heldRuns.entries()) {
350420
if (bufferedMessageIds.has(run.userMessageId)) {
351421
runs.push(run);
352422
continue;
353423
}
354-
if (run.queueHeld === true) continue;
424+
if (run.queueHeld === true && retryableRuns.length === 0) continue;
355425
events.push({
356426
id: yield* allocateEventId(),
357427
type: "run.updated",
358428
threadId: projection.thread.id,
359429
runId: run.id,
360430
providerInstanceId: run.providerInstanceId,
361431
occurredAt: now,
362-
payload: { ...run, queueHeld: true },
432+
payload: {
433+
...run,
434+
status: "queued",
435+
queueHeld: true,
436+
...(retryableRuns.length === 0 ? {} : { queuePosition: index + 1 }),
437+
},
363438
});
364439
}
365440
for (const question of questions) {

0 commit comments

Comments
 (0)