Skip to content

Commit 6b75141

Browse files
refactor(session): coordinate provider health as a runner service
Address review on anomalyco#55090: drop the pre-attempt gate, keep the per-session retry pipeline authoritative, and fold the shared wait in as a floor. - provider health becomes a service built by `make` (like SessionStep), one instance per runner layer; no module-level state and no $reset - `SessionRunnerHealth.coordinate` wraps the `retry.decide` result: the plugin session.retry hook, the retry budget, and the assistant identity all stay in the path; the shared wait can only raise the effective delay - the cooldown now requires failures from distinct sessions (threshold 5), so a single session keeps its original 2-10s backoff and ~84s give-up - rate limits no longer count toward the cooldown: they only share the provider-requested Retry-After window (the core of anomalyco#55075) - step.ts keeps only a `clean` flag on the retry callback so partial-stream failures never blame the provider; the gate and the pre-attempt error type are gone - tests rebuilt around the service: single-session immunity, rate limits never opening the breaker, distinct-session threshold, cooldown doubling, success clearing, partial-stream exclusion, per-route isolation
1 parent bb14a6c commit 6b75141

4 files changed

Lines changed: 254 additions & 179 deletions

File tree

‎packages/core/src/session/runner/llm.ts‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ export * as SessionRunnerLLM from "./llm.js"
22

33
import { Message } from "@opencode/ai/effect"
44
import { and, desc, eq, inArray, sql } from "drizzle-orm"
5-
import { Cause, Effect, Exit, FiberMap, Layer } from "effect"
5+
import { Cause, Clock, Effect, Exit, FiberMap, Layer } from "effect"
66
import { Database } from "../../database/database.js"
77
import { Bus } from "../../bus.js"
88
import { LocationLifecycle } from "../../location-lifecycle.js"
@@ -28,6 +28,7 @@ import { makeLocationNode } from "@opencode/util/effect/app-node"
2828
import { llmClient } from "../../effect/app-node-platform.js"
2929
import { StepFailedError } from "../error.js"
3030
import { SessionRunnerRetry } from "./retry.js"
31+
import { SessionRunnerHealth } from "./provider-health.js"
3132
import { SessionStep } from "./step.js"
3233
import { ToolOutput } from "../../tool-output.js"
3334
import { Plugin } from "../../plugin.js"
@@ -57,6 +58,8 @@ const layer = Layer.effect(
5758
const jobs = yield* Job.Service
5859
const title = yield* SessionTitle.Service
5960
const steps = yield* SessionStep.make
61+
// Cross-session provider coordination (#55075); one instance per runner layer.
62+
const health = yield* SessionRunnerHealth.make
6063
// Title generation starts once input is visible and must not delay model execution.
6164
const titles = yield* FiberMap.make<SessionSchema.ID, void, never>()
6265

@@ -234,6 +237,10 @@ const layer = Layer.effect(
234237
const sessionID = first.session.id
235238
let assistantMessageID = SessionMessage.ID.create()
236239
const retry = yield* SessionRunnerRetry.make(bus, sessionID)
240+
const healthKey = SessionRunnerHealth.key(
241+
first.model.model.provider,
242+
first.model.model.route.providerMetadataKey,
243+
)
237244
let initial: SessionContext.Loaded | undefined = first
238245
let recoverOverflow = true
239246
let recoverContinuation = true
@@ -294,14 +301,26 @@ const layer = Layer.effect(
294301
// A tool call can arrive during a reload; its permission check reads the session's agent.
295302
executeTool: (call) => plugins.awaitActivation.pipe(Effect.andThen(prepared.executeTool(call))),
296303
},
297-
retry: (cause, error, proposed) =>
298-
retry.decide({
299-
cause,
300-
error,
301-
agent: loaded.agent.id,
302-
model: loaded.model.ref,
303-
hook: prepared.retry,
304-
retry: proposed,
304+
retry: (cause, error, proposed, clean) =>
305+
Effect.gen(function* () {
306+
const decision = yield* retry.decide({
307+
cause,
308+
error,
309+
agent: loaded.agent.id,
310+
model: loaded.model.ref,
311+
hook: prepared.retry,
312+
retry: proposed,
313+
})
314+
// The shared health record only raises the wait (floor semantics), so the
315+
// plugin hook and the per-session retry budget above stay authoritative.
316+
return health.coordinate({
317+
key: healthKey,
318+
sessionID,
319+
cause,
320+
decision,
321+
clean,
322+
now: yield* Clock.currentTimeMillis,
323+
})
305324
}),
306325
recoverContinuation,
307326
recoverOverflow: Effect.suspend(() =>
@@ -314,6 +333,8 @@ const layer = Layer.effect(
314333
})
315334
const completed = yield* SessionStep.Outcome.$match(outcome, {
316335
Completed: Effect.fnUntraced(function* (outcome) {
336+
// A settled provider response clears the route's health record (#55075).
337+
health.recordSuccess(healthKey)
317338
// The Session is about to go idle, which leaves the background summary time to finish.
318339
if (!outcome.needsContinuation)
319340
yield* compaction.precompute(
Lines changed: 84 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,119 @@
11
export * as SessionRunnerHealth from "./provider-health.js"
22

33
import type { AIError } from "@opencode/ai"
4+
import { Effect } from "effect"
5+
import { SessionSchema } from "../schema.js"
6+
import type { SessionRunnerRetry } from "./retry.js"
47

58
/**
6-
* Cross-session provider health, shared within one server process.
9+
* Cross-session provider health, shared by the sessions of one runner.
710
*
811
* Each session's retry policy is deliberately scoped to that session, so concurrent
912
* sessions and subagents retry the same failing provider independently and every
10-
* attempt still burns a request. This module tracks failures per provider key so
11-
* those sessions coordinate:
13+
* attempt still burns a request. `coordinate` folds a per-session retry decision
14+
* into a per-route health record so those sessions coordinate:
15+
*
1216
* - a 429's `Retry-After` yields one shared next-allowed time for every session,
1317
* instead of each session discovering it by failing on its own;
14-
* - sustained 5xx / connection failures open a circuit so new attempts fail fast
15-
* until a cooldown elapses, then one probe request is let through;
16-
* - in-flight streams are never touched: the gate only decides whether a new
17-
* attempt may start.
18-
*
19-
* See https://github.com/anomalyco/opencode/issues/55075 for the field data.
18+
* - once failures come from several distinct sessions, the shared wait grows a
19+
* cooldown so the sessions stop pounding the route in overlapping bursts —
20+
* after each cooldown the next attempt acts as the probe, and it succeeds
21+
* (record clears) or fails (cooldown doubles) without extra bookkeeping;
22+
* - rate limits never count toward that cooldown: they only share the
23+
* provider-requested delay, which is what #55075 actually asks for;
24+
* - the shared wait is a floor on the per-session decision, so the plugin
25+
* `session.retry` hook and the session's retry budget stay in the path.
2026
*/
2127

22-
/** Consecutive provider-level failures before the circuit opens: roughly half of one exhausted per-session policy, so a single unlucky session cannot trip it alone while 2-3 concurrent sessions can. */
28+
/** Distinct failing sessions before the shared cooldown kicks in. */
2329
const OPEN_THRESHOLD = 5
24-
/** Initial open-state cooldown, then doubled per further failure up to the cap. The per-session policy burns ~84s over its 10 attempts, so 60s opens the circuit around the time sessions would give up anyway. */
25-
const OPEN_COOLDOWN_MS = 60_000
26-
const OPEN_COOLDOWN_MAX_MS = 15 * 60_000
27-
/** A half-open probe has this long to settle (via recordSuccess/recordFailure) before another probe is allowed, so a probe lost to cancellation cannot wedge the half-open state. */
28-
const PROBE_WINDOW_MS = 30_000
29-
/** A hostile or buggy retry-after cannot stall new attempts for hours (mirrors the per-session cap in retry.ts). */
30+
/** Initial shared cooldown, then doubled per further failure up to the cap. The per-session policy burns ~84s over its 10 attempts, so 60s engages around the time sessions would give up anyway. */
31+
const COOLDOWN_MS = 60_000
32+
const COOLDOWN_MAX_MS = 15 * 60_000
33+
/** A hostile or buggy retry-after cannot stall attempts for hours (mirrors the per-session cap in retry.ts). */
3034
const RETRY_AFTER_MAX_MS = 15 * 60_000
3135

3236
interface State {
33-
consecutiveFailures: number
37+
sessions: Set<SessionSchema.ID>
3438
openUntil: number
35-
probeUntil: number
39+
opens: number
3640
nextAllowedAt: number
3741
}
3842

39-
// One map per server process; sessions reach it through the attempt gate and the
40-
// retry path, which is all the coordination the storm fix needs.
41-
const states = new Map<string, State>()
43+
export interface CoordinateInput {
44+
readonly key: string
45+
readonly sessionID: SessionSchema.ID
46+
/** The provider failure that produced `decision`, or undefined when there is none. */
47+
readonly cause: AIError | undefined
48+
/** Decision from the per-session policy; `coordinate` may only raise its delay. */
49+
readonly decision: { readonly retry: false } | SessionRunnerRetry.Decision
50+
/** Settle a clean pre-output failure only; partial-stream failures must not blame the provider. */
51+
readonly clean: boolean
52+
readonly now: number
53+
}
4254

43-
const state = (key: string): State => {
44-
let s = states.get(key)
45-
if (!s) {
46-
s = { consecutiveFailures: 0, openUntil: 0, probeUntil: 0, nextAllowedAt: 0 }
47-
states.set(key, s)
48-
}
49-
return s
55+
export interface Service {
56+
coordinate: (input: CoordinateInput) => { readonly retry: false } | SessionRunnerRetry.Decision
57+
recordSuccess: (key: string) => void
5058
}
5159

52-
/** One health bucket per provider route; routes (and their credentials) share fate because they are load-balanced together. */
60+
/** One health record per provider route; routes (and their credentials) share fate because they are load-balanced together. */
5361
export const key = (provider: string, routeKey: string | undefined): string => routeKey ?? provider
5462

55-
/** Provider-requested retry delay, only where the provider actually signals one. */
56-
export const retryAfterMs = (cause: AIError): number | undefined => {
63+
const retryAfterMs = (cause: AIError | undefined): number | undefined => {
64+
if (cause === undefined) return undefined
5765
if (cause.reason._tag === "RateLimit" || cause.reason._tag === "ProviderInternal")
5866
return cause.reason.retryAfterMs
5967
return undefined
6068
}
6169

62-
export const canAttempt = (key: string, now: number): { allowed: boolean; retryInMs: number } => {
63-
const s = states.get(key)
64-
if (!s) return { allowed: true, retryInMs: 0 }
65-
if (now < s.nextAllowedAt) return { allowed: false, retryInMs: s.nextAllowedAt - now }
66-
if (now < s.openUntil) return { allowed: false, retryInMs: s.openUntil - now }
67-
if (s.consecutiveFailures >= OPEN_THRESHOLD) {
68-
// Half-open: let this attempt probe the provider while holding others back
69-
// until the probe settles or the window expires.
70-
s.openUntil = now + PROBE_WINDOW_MS
71-
s.probeUntil = s.openUntil
70+
/**
71+
* Builds an isolated health instance. The runner layer constructs one per server;
72+
* tests construct one per case, which keeps state out of the module scope.
73+
*/
74+
export const make = Effect.gen(function* () {
75+
const states = new Map<string, State>()
76+
77+
const state = (key: string): State => {
78+
let s = states.get(key)
79+
if (!s) {
80+
s = { sessions: new Set(), openUntil: 0, opens: 0, nextAllowedAt: 0 }
81+
states.set(key, s)
82+
}
83+
return s
7284
}
73-
return { allowed: true, retryInMs: 0 }
74-
}
7585

76-
export const recordFailure = (key: string, now: number, cause: AIError | undefined): void => {
77-
const s = state(key)
78-
s.probeUntil = 0
79-
s.consecutiveFailures++
80-
const after = cause === undefined ? undefined : retryAfterMs(cause)
81-
if (after !== undefined) s.nextAllowedAt = now + Math.min(Math.max(0, after), RETRY_AFTER_MAX_MS)
82-
if (s.consecutiveFailures >= OPEN_THRESHOLD) {
83-
const doubling = Math.min(2 ** Math.min(s.consecutiveFailures - OPEN_THRESHOLD, 4), 16)
84-
s.openUntil = now + Math.min(OPEN_COOLDOWN_MS * doubling, OPEN_COOLDOWN_MAX_MS)
86+
const coordinate: Service["coordinate"] = (input) => {
87+
if (!input.decision.retry) return input.decision
88+
const s = state(input.key)
89+
// 5xx and connection failures come from the provider side; rate limits are
90+
// expected under concurrency and only share the requested delay.
91+
const counted =
92+
input.clean &&
93+
input.cause !== undefined &&
94+
(input.cause.reason._tag === "Transport" || input.cause.reason._tag === "ProviderInternal")
95+
if (counted) s.sessions.add(input.sessionID)
96+
const after = retryAfterMs(input.cause)
97+
if (input.clean && after !== undefined)
98+
s.nextAllowedAt = Math.max(s.nextAllowedAt, input.now + Math.min(Math.max(0, after), RETRY_AFTER_MAX_MS))
99+
100+
let waitUntil = 0
101+
if (s.sessions.size >= OPEN_THRESHOLD) {
102+
if (input.now >= s.openUntil) {
103+
// First opening, or the previous cooldown elapsed and this failure was the probe.
104+
s.opens++
105+
s.openUntil = input.now + Math.min(COOLDOWN_MS * 2 ** Math.min(s.opens - 1, 4), COOLDOWN_MAX_MS)
106+
}
107+
waitUntil = s.openUntil
108+
}
109+
waitUntil = Math.max(waitUntil, s.nextAllowedAt)
110+
const floor = Math.max(0, waitUntil - input.now)
111+
return floor > input.decision.delay ? { ...input.decision, delay: floor } : input.decision
85112
}
86-
}
87113

88-
export const recordSuccess = (key: string): void => {
89-
states.delete(key)
90-
}
114+
const recordSuccess: Service["recordSuccess"] = (key) => {
115+
states.delete(key)
116+
}
91117

92-
/** Test seam: drops all shared state between tests. */
93-
export const $reset = (): void => {
94-
states.clear()
95-
}
118+
return { coordinate, recordSuccess }
119+
})

‎packages/core/src/session/runner/step.ts‎

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import { SessionSchema } from "../schema.js"
2424
import { toSessionError } from "../to-session-error.js"
2525
import { SessionUsage } from "../usage.js"
2626
import { SessionRunnerModel } from "./model.js"
27-
import { SessionRunnerHealth } from "./provider-health.js"
2827
import { createLLMEventPublisher } from "./publish-llm-event.js"
2928
import { SessionRunnerRetry } from "./retry.js"
3029

@@ -52,6 +51,8 @@ interface Input {
5251
cause: AIError,
5352
error: SessionError.Error,
5453
retry: boolean,
54+
/** Settled without any provider output: a clean provider-side failure. */
55+
clean: boolean,
5556
) => Effect.Effect<{ readonly retry: false } | SessionRunnerRetry.Decision>
5657
readonly recoverContinuation: boolean
5758
/** The runner owns compaction policy; the attempt invokes it only before durable output. */
@@ -70,23 +71,6 @@ export const make = Effect.gen(function* () {
7071
const toolOutput = yield* ToolOutput.Service
7172

7273
const attempt = Effect.fn("SessionStep.attempt")(function* (input: Input) {
73-
// Provider health is shared across sessions (#55075): while the circuit is open
74-
// or a shared Retry-After is in effect, this attempt waits in the existing retry
75-
// channel instead of burning another doomed request. The gate never records the
76-
// outcome of the attempt it blocks; in-flight attempts settle their own record.
77-
const healthKey = SessionRunnerHealth.key(
78-
input.model.model.provider,
79-
input.model.model.route.providerMetadataKey,
80-
)
81-
const gate = SessionRunnerHealth.canAttempt(healthKey, yield* Clock.currentTimeMillis)
82-
if (!gate.allowed)
83-
return Outcome.Retry({
84-
error: {
85-
type: "provider.unavailable",
86-
message: `Provider temporarily unavailable, retrying in ${Math.ceil(gate.retryInMs / 1000)}s`,
87-
},
88-
decision: { retry: true, attempt: 1, delay: gate.retryInMs },
89-
})
9074
// The start snapshot only has to exist before local tools run, which cannot happen before Step.Started,
9175
// so it is captured while the provider request is in flight instead of delaying it.
9276
const pendingStartSnapshot = yield* snapshots.capture().pipe(Effect.forkScoped)
@@ -223,16 +207,13 @@ export const make = Effect.gen(function* () {
223207
llmError,
224208
SessionRunnerRetry.isRetryable(llmFailure) ||
225209
(recorded.outputStarted && isInterruptedStream(llmFailure)),
210+
!recorded.outputStarted,
226211
),
227212
)
228213
: undefined
229214
if (llmFailure && llmError && retry?.retry && !recorded.outputStarted) {
230215
// Retry state projects onto the existing assistant, even before it has produced output.
231216
yield* publisher.startAssistant()
232-
// A clean pre-output failure is a provider-level signal; partial-stream
233-
// failures (Continue/RecoverFull) are not recorded, since a client-side
234-
// transport break would otherwise blame a healthy provider.
235-
SessionRunnerHealth.recordFailure(healthKey, yield* Clock.currentTimeMillis, llmFailure)
236217
return Outcome.Retry({ error: llmError, decision: retry })
237218
}
238219
if (llmError) yield* publisher.failAssistant(llmError)
@@ -278,9 +259,7 @@ export const make = Effect.gen(function* () {
278259
: undefined
279260
const usage = record.finish ? SessionUsage.record(record.finish.usage, input.model.cost) : undefined
280261
if (record.failure) yield* publisher.publishStepFailure({ ...usage, snapshot, files })
281-
if (record.finish && usage && !record.failure) {
282-
// A settled provider response clears the health record (#55075).
283-
SessionRunnerHealth.recordSuccess(healthKey)
262+
if (record.finish && usage && !record.failure)
284263
yield* bus.publish(SessionEvent.Step.Ended, {
285264
sessionID: input.sessionID,
286265
assistantMessageID: yield* publisher.startAssistant(),
@@ -291,7 +270,6 @@ export const make = Effect.gen(function* () {
291270
snapshot,
292271
files,
293272
})
294-
}
295273
}
296274

297275
if (

0 commit comments

Comments
 (0)