Skip to content

Commit 8d26079

Browse files
committed
fix(server): bound delegated completion delivery retries
1 parent efecd3c commit 8d26079

3 files changed

Lines changed: 139 additions & 5 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderContinuationService.test.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ import {
99
type OrchestrationV2Notification,
1010
} from "@t3tools/contracts";
1111
import * as Deferred from "effect/Deferred";
12+
import * as Cause from "effect/Cause";
1213
import * as DateTime from "effect/DateTime";
1314
import * as Effect from "effect/Effect";
15+
import * as Exit from "effect/Exit";
1416
import * as Layer from "effect/Layer";
1517
import * as Option from "effect/Option";
1618
import * as Queue from "effect/Queue";
@@ -103,6 +105,106 @@ function testLayer(input: {
103105
}
104106

105107
describe("ProviderContinuationService", () => {
108+
it.effect.each(["dispatch", "projection"] as const)(
109+
"bounds delegated completion retries after persistent %s failure",
110+
(failure) =>
111+
Effect.gen(function* () {
112+
const attempts = yield* Ref.make(0);
113+
const attemptEvents = yield* Queue.unbounded<number>();
114+
const fail = Ref.updateAndGet(attempts, (count) => count + 1).pipe(
115+
Effect.tap((count) => Queue.offer(attemptEvents, count)),
116+
Effect.andThen(Effect.die("persistent continuation failure")),
117+
);
118+
const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({
119+
getThreadRecords: () =>
120+
failure === "projection" ? fail : Effect.succeed(delegatedProjection()),
121+
dispatch: () => fail,
122+
});
123+
const worker = ProviderContinuationService.workerLive.pipe(
124+
Layer.provide(
125+
Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, threads),
126+
),
127+
);
128+
yield* Effect.gen(function* () {
129+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
130+
yield* requests.offer({
131+
threadId,
132+
providerThreadId,
133+
driver,
134+
detail: null,
135+
delegatedCompletion: { parentRunId, generation: 1, messageId: delegatedMessageId },
136+
});
137+
assert.equal(yield* Queue.take(attemptEvents), 1);
138+
for (const delay of [100, 200, 400, 800, 1600, 3200, 5000, 5000]) {
139+
yield* TestClock.adjust(`${delay} millis`);
140+
if (failure === "projection") yield* Queue.take(attemptEvents);
141+
yield* Queue.take(attemptEvents);
142+
}
143+
const exhaustedCount = yield* Ref.get(attempts);
144+
assert.equal(exhaustedCount, failure === "projection" ? 17 : 9);
145+
yield* TestClock.adjust("1 day");
146+
assert.equal(yield* Ref.get(attempts), exhaustedCount);
147+
// A durable mailbox re-offer starts another bounded retry chain.
148+
yield* requests.offer({
149+
threadId,
150+
providerThreadId,
151+
driver,
152+
detail: null,
153+
delegatedCompletion: { parentRunId, generation: 1, messageId: delegatedMessageId },
154+
});
155+
assert.equal(yield* Queue.take(attemptEvents), exhaustedCount + 1);
156+
yield* TestClock.adjust("100 millis");
157+
if (failure === "projection") yield* Queue.take(attemptEvents);
158+
yield* Queue.take(attemptEvents);
159+
assert.equal(
160+
yield* Ref.get(attempts),
161+
exhaustedCount + (failure === "projection" ? 3 : 2),
162+
);
163+
}).pipe(
164+
Effect.provide(Layer.merge(ProviderContinuationRequests.layer, worker)),
165+
Effect.scoped,
166+
);
167+
}),
168+
);
169+
it.effect("interrupts an in-flight delegated delivery when the worker scope closes", () =>
170+
Effect.gen(function* () {
171+
const attempts = yield* Ref.make(0);
172+
const started = yield* Deferred.make<void>();
173+
const stopped = yield* Deferred.make<Exit.Exit<never>>();
174+
const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({
175+
getThreadRecords: () => Effect.succeed(delegatedProjection()),
176+
dispatch: () =>
177+
Ref.update(attempts, (count) => count + 1).pipe(
178+
Effect.andThen(Deferred.succeed(started, undefined)),
179+
Effect.andThen(Effect.never),
180+
Effect.onExit((exit) => Deferred.succeed(stopped, exit)),
181+
),
182+
});
183+
const worker = ProviderContinuationService.workerLive.pipe(
184+
Layer.provide(
185+
Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, threads),
186+
),
187+
);
188+
yield* Effect.gen(function* () {
189+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
190+
yield* requests.offer({
191+
threadId,
192+
providerThreadId,
193+
driver,
194+
detail: null,
195+
delegatedCompletion: { parentRunId, generation: 1, messageId: delegatedMessageId },
196+
});
197+
yield* Deferred.await(started);
198+
}).pipe(
199+
Effect.provide(Layer.merge(ProviderContinuationRequests.layer, worker)),
200+
Effect.scoped,
201+
);
202+
const exit = yield* Deferred.await(stopped);
203+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
204+
yield* TestClock.adjust("1 day");
205+
assert.equal(yield* Ref.get(attempts), 1);
206+
}),
207+
);
106208
it.effect("recovers an unaccepted persisted steer using the same delivery identity", () =>
107209
Effect.gen(function* () {
108210
const dispatched = yield* Queue.unbounded<unknown>();

‎apps/server/src/orchestration-v2/ProviderContinuationService.ts‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,15 @@ import { CommandId, type OrchestrationV2ThreadProjection } from "@t3tools/contra
22
import * as Effect from "effect/Effect";
33
import * as Layer from "effect/Layer";
44
import * as Ref from "effect/Ref";
5+
import * as Cause from "effect/Cause";
56

67
import * as IdAllocator from "./IdAllocator.ts";
78
import * as ProviderContinuationRequests from "./ProviderContinuationRequests.ts";
89
import * as ThreadManagementService from "./ThreadManagementService.ts";
910
import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts";
1011

1112
const CONTINUATION_MESSAGE_TEXT = "Background task completed.";
13+
const MAX_DELEGATED_COMPLETION_RETRIES = 8;
1214

1315
function delegatedCompletionText(taskIds: ReadonlyArray<string>): string {
1416
const taskList = taskIds.join(", ");
@@ -76,6 +78,10 @@ export const workerLive = Layer.effectDiscard(
7678
Ref.modify(retryAttempts, (current) => {
7779
const attempt = current.get(key) ?? 0;
7880
const updated = new Map(current);
81+
if (attempt >= MAX_DELEGATED_COMPLETION_RETRIES) {
82+
updated.delete(key);
83+
return [null, updated] as const;
84+
}
7985
updated.set(key, attempt + 1);
8086
return [Math.min(100 * 2 ** Math.min(attempt, 6), 5_000), updated] as const;
8187
});
@@ -185,6 +191,7 @@ export const workerLive = Layer.effectDiscard(
185191
dispatchContinuation(request).pipe(
186192
Effect.catchCause((cause) =>
187193
Effect.gen(function* () {
194+
if (Cause.hasInterruptsOnly(cause)) return yield* Effect.failCause(cause);
188195
yield* Effect.logWarning("orchestration-v2.provider-continuation.dispatch-failed", {
189196
threadId: request.threadId,
190197
providerThreadId: request.providerThreadId,
@@ -194,6 +201,19 @@ export const workerLive = Layer.effectDiscard(
194201
const completion = request.delegatedCompletion;
195202
const retryKey = delegatedCompletionRetryKey(request, completion);
196203
const retryDelay = yield* nextRetryDelay(retryKey);
204+
if (retryDelay === null) {
205+
yield* Effect.logError(
206+
"orchestration-v2.provider-continuation.retries-exhausted",
207+
{
208+
threadId: request.threadId,
209+
providerThreadId: request.providerThreadId,
210+
parentRunId: completion.parentRunId,
211+
generation: completion.generation,
212+
messageId: completion.messageId,
213+
},
214+
);
215+
return;
216+
}
197217
yield* Effect.gen(function* () {
198218
yield* Effect.sleep(`${retryDelay} millis`);
199219
const projection = yield* threads.getThreadRecords(
@@ -213,11 +233,16 @@ export const workerLive = Layer.effectDiscard(
213233
}
214234
}).pipe(
215235
Effect.catchCause((retryCause) =>
216-
Effect.logWarning("orchestration-v2.provider-continuation.retry-check-failed", {
217-
threadId: request.threadId,
218-
providerThreadId: request.providerThreadId,
219-
cause: retryCause,
220-
}).pipe(Effect.andThen(requests.offer(request))),
236+
Cause.hasInterruptsOnly(retryCause)
237+
? Effect.failCause(retryCause)
238+
: Effect.logWarning(
239+
"orchestration-v2.provider-continuation.retry-check-failed",
240+
{
241+
threadId: request.threadId,
242+
providerThreadId: request.providerThreadId,
243+
cause: retryCause,
244+
},
245+
).pipe(Effect.andThen(requests.offer(request))),
221246
),
222247
Effect.forkScoped,
223248
);

‎docs/internals/providers.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,13 @@ client connections and provider-instance rebuilds. Releases are immutable, with
4949
selecting the version for new processes. Running processes hold leases on their version. Updates
5050
and removal must respect those leases instead of replacing executables under a running agent.
5151

52+
## Delegated completion delivery
53+
54+
Exhausting a [completion delivery](../../apps/server/src/orchestration-v2/ProviderContinuationService.ts)
55+
retry chain does not consume its durable mailbox offer. Recovery or a later re-offer
56+
can start a fresh retry budget, including in the same process; exhaustion is not a
57+
permanent delivery-failure marker.
58+
5259
## Setup must not happen as a health-check side effect
5360

5461
Opening a provider session can start MCP servers, run hooks, or launch a login browser.

0 commit comments

Comments
 (0)