Skip to content

Commit 4b8f398

Browse files
committed
fix(server): retry early recovery after pending requests clear
1 parent b0b0027 commit 4b8f398

2 files changed

Lines changed: 50 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5066,8 +5066,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
50665066
...(recovery?.autoResume &&
50675067
recovery.requestId === command.usageLimitRecoveryRequestId &&
50685068
recovery.runId === command.usageLimitContinuationOfRunId &&
5069-
projection.thread.snoozedUntil != null &&
5070-
DateTime.toEpochMillis(projection.thread.snoozedUntil) > DateTime.toEpochMillis(now)
5069+
(projection.runtimeRequests.some((request) => request.status === "pending") ||
5070+
(projection.thread.snoozedUntil != null &&
5071+
DateTime.toEpochMillis(projection.thread.snoozedUntil) >
5072+
DateTime.toEpochMillis(now)))
50715073
? { limitRecovery: { ...recovery, requestId: command.commandId } }
50725074
: {}),
50735075
},

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6360,6 +6360,52 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63606360
threadId,
63616361
limitRecovery: { runId: run.id, resetAt, autoResume: true },
63626362
});
6363+
const ready = (yield* projections.getLimitRecoveryCandidates({
6364+
now: checkedNow,
6365+
autoResume: true,
6366+
snooze: false,
6367+
})).find((row) => row.id === threadId)!;
6368+
const racedResume = limitRecoveryCommand(ready, true, nowMs, false, reading(checkedNow, 0));
6369+
assert(racedResume?.type === "message.dispatch");
6370+
const nodeId = run.rootNodeId;
6371+
assert(nodeId !== null);
6372+
const request = {
6373+
id: RuntimeRequestId.make("recovery:pending-race"),
6374+
nodeId,
6375+
providerTurnId: null,
6376+
nativeRequestRef: null,
6377+
kind: "user_input" as const,
6378+
status: "pending" as const,
6379+
responseCapability: { type: "message" as const },
6380+
createdAt: checkedNow,
6381+
resolvedAt: null,
6382+
};
6383+
yield* events.write({
6384+
events: [
6385+
{
6386+
id: EventId.make("recovery:pending-race"),
6387+
type: "runtime-request.updated",
6388+
threadId,
6389+
nodeId,
6390+
occurredAt: checkedNow,
6391+
payload: request,
6392+
},
6393+
],
6394+
});
6395+
yield* orchestrator.dispatch(racedResume);
6396+
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1);
6397+
yield* events.write({
6398+
events: [
6399+
{
6400+
id: EventId.make("recovery:pending-race:resolved"),
6401+
type: "runtime-request.updated",
6402+
threadId,
6403+
nodeId,
6404+
occurredAt: checkedNow,
6405+
payload: { ...request, status: "resolved", resolvedAt: checkedNow },
6406+
},
6407+
],
6408+
});
63636409
yield* sweep(reading(checkedNow, 0));
63646410
const resumed = yield* orchestrator.getThreadProjection(threadId);
63656411
assert.lengthOf(resumed.runs, 2);

0 commit comments

Comments
 (0)