Skip to content

Commit 838135f

Browse files
feat(session): share provider health across concurrent sessions
Concurrent sessions and subagents each run their own retry policy, so a failing provider keeps receiving doomed attempts from N directions at overlapping times, and every attempt publishes session-scoped events to all clients (anomalyco#55075). Add a process-wide provider health record keyed by provider route: - a 429's Retry-After yields one shared next-allowed time for every session, instead of each session discovering it by failing; - sustained 5xx / connection failures open a circuit (threshold 5, 60s cooldown doubling to a 15min cap) so new attempts wait in the existing retry channel without burning another request; - after the cooldown one half-open probe is let through per provider route, with a self-healing 30s window; - in-flight streams are untouched: the gate only decides whether a new attempt may start, and partial-stream failures are never recorded. The gate projects onto the existing Retry outcome, so the UI shows the same retry countdown it already does - no new error surface.
1 parent 57f4a36 commit 838135f

3 files changed

Lines changed: 236 additions & 1 deletion

File tree

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
export * as SessionRunnerHealth from "./provider-health.js"
2+
3+
import type { AIError } from "@opencode/ai"
4+
5+
/**
6+
* Cross-session provider health, shared within one server process.
7+
*
8+
* Each session's retry policy is deliberately scoped to that session, so concurrent
9+
* sessions and subagents retry the same failing provider independently and every
10+
* attempt still burns a request. This module tracks failures per provider key so
11+
* those sessions coordinate:
12+
* - a 429's `Retry-After` yields one shared next-allowed time for every session,
13+
* instead of each session discovering it by failing on its own;
14+
* - sustained 5xx / connection failures open a circuit so new attempts fail fast
15+
* until a cooldown elapses, then one probe request is let through;
16+
* - in-flight streams are never touched: the gate only decides whether a new
17+
* attempt may start.
18+
*
19+
* See https://github.com/anomalyco/opencode/issues/55075 for the field data.
20+
*/
21+
22+
/** Consecutive provider-level failures before the circuit opens: roughly half of one exhausted per-session policy, so a single unlucky session cannot trip it alone while 2-3 concurrent sessions can. */
23+
const OPEN_THRESHOLD = 5
24+
/** Initial open-state cooldown, then doubled per further failure up to the cap. The per-session policy burns ~84s over its 10 attempts, so 60s opens the circuit around the time sessions would give up anyway. */
25+
const OPEN_COOLDOWN_MS = 60_000
26+
const OPEN_COOLDOWN_MAX_MS = 15 * 60_000
27+
/** A half-open probe has this long to settle (via recordSuccess/recordFailure) before another probe is allowed, so a probe lost to cancellation cannot wedge the half-open state. */
28+
const PROBE_WINDOW_MS = 30_000
29+
/** A hostile or buggy retry-after cannot stall new attempts for hours (mirrors the per-session cap in retry.ts). */
30+
const RETRY_AFTER_MAX_MS = 15 * 60_000
31+
32+
interface State {
33+
consecutiveFailures: number
34+
openUntil: number
35+
probeUntil: number
36+
nextAllowedAt: number
37+
}
38+
39+
// One map per server process; sessions reach it through the attempt gate and the
40+
// retry path, which is all the coordination the storm fix needs.
41+
const states = new Map<string, State>()
42+
43+
const state = (key: string): State => {
44+
let s = states.get(key)
45+
if (!s) {
46+
s = { consecutiveFailures: 0, openUntil: 0, probeUntil: 0, nextAllowedAt: 0 }
47+
states.set(key, s)
48+
}
49+
return s
50+
}
51+
52+
/** One health bucket per provider route; routes (and their credentials) share fate because they are load-balanced together. */
53+
export const key = (provider: string, routeKey: string | undefined): string => routeKey ?? provider
54+
55+
/** Provider-requested retry delay, only where the provider actually signals one. */
56+
export const retryAfterMs = (cause: AIError): number | undefined => {
57+
if (cause.reason._tag === "RateLimit" || cause.reason._tag === "ProviderInternal")
58+
return cause.reason.retryAfterMs
59+
return undefined
60+
}
61+
62+
export const canAttempt = (key: string, now: number): { allowed: boolean; retryInMs: number } => {
63+
const s = states.get(key)
64+
if (!s) return { allowed: true, retryInMs: 0 }
65+
if (now < s.nextAllowedAt) return { allowed: false, retryInMs: s.nextAllowedAt - now }
66+
if (now < s.openUntil) return { allowed: false, retryInMs: s.openUntil - now }
67+
if (s.consecutiveFailures >= OPEN_THRESHOLD) {
68+
// Half-open: let this attempt probe the provider while holding others back
69+
// until the probe settles or the window expires.
70+
s.openUntil = now + PROBE_WINDOW_MS
71+
s.probeUntil = s.openUntil
72+
}
73+
return { allowed: true, retryInMs: 0 }
74+
}
75+
76+
export const recordFailure = (key: string, now: number, cause: AIError | undefined): void => {
77+
const s = state(key)
78+
s.probeUntil = 0
79+
s.consecutiveFailures++
80+
const after = cause === undefined ? undefined : retryAfterMs(cause)
81+
if (after !== undefined) s.nextAllowedAt = now + Math.min(Math.max(0, after), RETRY_AFTER_MAX_MS)
82+
if (s.consecutiveFailures >= OPEN_THRESHOLD) {
83+
const doubling = Math.min(2 ** Math.min(s.consecutiveFailures - OPEN_THRESHOLD, 4), 16)
84+
s.openUntil = now + Math.min(OPEN_COOLDOWN_MS * doubling, OPEN_COOLDOWN_MAX_MS)
85+
}
86+
}
87+
88+
export const recordSuccess = (key: string): void => {
89+
states.delete(key)
90+
}
91+
92+
/** Test seam: drops all shared state between tests. */
93+
export const $reset = (): void => {
94+
states.clear()
95+
}

‎packages/core/src/session/runner/step.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import { SessionSchema } from "../schema.js"
2626
import { toSessionError } from "../to-session-error.js"
2727
import { SessionUsage } from "../usage.js"
2828
import { SessionRunnerModel } from "./model.js"
29+
import { SessionRunnerHealth } from "./provider-health.js"
2930
import { createLLMEventPublisher } from "./publish-llm-event.js"
3031
import { SessionRunnerRetry } from "./retry.js"
3132

@@ -71,6 +72,23 @@ export const make = Effect.gen(function* () {
7172
const toolOutput = yield* ToolOutput.Service
7273

7374
const attempt = Effect.fn("SessionStep.attempt")(function* (input: Input) {
75+
// Provider health is shared across sessions (#55075): while the circuit is open
76+
// or a shared Retry-After is in effect, this attempt waits in the existing retry
77+
// channel instead of burning another doomed request. The gate never records the
78+
// outcome of the attempt it blocks; in-flight attempts settle their own record.
79+
const healthKey = SessionRunnerHealth.key(
80+
input.model.model.provider,
81+
input.model.model.route.providerMetadataKey,
82+
)
83+
const gate = SessionRunnerHealth.canAttempt(healthKey, yield* Clock.currentTimeMillis)
84+
if (!gate.allowed)
85+
return Outcome.Retry({
86+
error: {
87+
type: "provider.unavailable",
88+
message: `Provider temporarily unavailable, retrying in ${Math.ceil(gate.retryInMs / 1000)}s`,
89+
},
90+
decision: { retry: true, attempt: 1, delay: gate.retryInMs },
91+
})
7492
// The start snapshot only has to exist before local tools run, which cannot happen before Step.Started,
7593
// so it is captured while the provider request is in flight instead of delaying it.
7694
const pendingStartSnapshot = yield* snapshots.capture().pipe(Effect.forkScoped)
@@ -200,6 +218,10 @@ export const make = Effect.gen(function* () {
200218
if (llmFailure && llmError && retry?.retry && !recorded.outputStarted) {
201219
// Retry state projects onto the existing assistant, even before it has produced output.
202220
yield* publisher.startAssistant()
221+
// A clean pre-output failure is a provider-level signal; partial-stream
222+
// failures (Continue/RecoverFull) are not recorded, since a client-side
223+
// transport break would otherwise blame a healthy provider.
224+
SessionRunnerHealth.recordFailure(healthKey, yield* Clock.currentTimeMillis, llmFailure)
203225
return Outcome.Retry({ error: llmError, decision: retry })
204226
}
205227
if (llmError) yield* publisher.failAssistant(llmError)
@@ -247,7 +269,9 @@ export const make = Effect.gen(function* () {
247269
? { cost: SessionUsage.calculateCost(input.model.cost, record.finish.tokens), tokens: record.finish.tokens }
248270
: undefined
249271
if (record.failure) yield* publisher.publishStepFailure({ ...usage, snapshot, files })
250-
if (record.finish && usage && !record.failure)
272+
if (record.finish && usage && !record.failure) {
273+
// A settled provider response clears the health record (#55075).
274+
SessionRunnerHealth.recordSuccess(healthKey)
251275
yield* bus.publish(SessionEvent.Step.Ended, {
252276
sessionID: input.sessionID,
253277
assistantMessageID: yield* publisher.startAssistant(),
@@ -258,6 +282,7 @@ export const make = Effect.gen(function* () {
258282
snapshot,
259283
files,
260284
})
285+
}
261286
}
262287

263288
if (
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import { describe, expect, it } from "bun:test"
2+
import { AIError, ProviderInternalError, RateLimitError, TransportError } from "@opencode/ai"
3+
import { SessionRunnerHealth } from "@opencode/core/session/runner/provider-health"
4+
5+
const rateLimit = (retryAfterMs?: number) =>
6+
new AIError({ reason: new RateLimitError({ message: "rate limited", retryAfterMs }) })
7+
const internal = new AIError({ reason: new ProviderInternalError({ message: "internal" }) })
8+
const timeout = new AIError({
9+
reason: new TransportError({ message: "Timed out", transport: "http", operation: "request", code: "Timeout" }),
10+
})
11+
12+
const KEY = "provider"
13+
const OTHER = "other"
14+
15+
describe("SessionRunnerHealth", () => {
16+
describe("canAttempt", () => {
17+
it("allows when the provider has no recorded failures", () => {
18+
SessionRunnerHealth.$reset()
19+
expect(SessionRunnerHealth.canAttempt(KEY, 1000)).toEqual({ allowed: true, retryInMs: 0 })
20+
})
21+
22+
it("fails fast while the circuit is open", () => {
23+
SessionRunnerHealth.$reset()
24+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, internal)
25+
const gate = SessionRunnerHealth.canAttempt(KEY, 2000)
26+
expect(gate.allowed).toBe(false)
27+
expect(gate.retryInMs).toBeGreaterThan(0)
28+
})
29+
30+
it("keeps unrelated providers open while one is failing", () => {
31+
SessionRunnerHealth.$reset()
32+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, internal)
33+
expect(SessionRunnerHealth.canAttempt(OTHER, 2000).allowed).toBe(true)
34+
})
35+
})
36+
37+
describe("circuit breaker", () => {
38+
it("opens after the failure threshold and lets one probe through after the cooldown", () => {
39+
SessionRunnerHealth.$reset()
40+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, internal)
41+
// Still open well before the cooldown elapses.
42+
expect(SessionRunnerHealth.canAttempt(KEY, 10_000).allowed).toBe(false)
43+
// Half-open after 60s: the first attempt is the probe.
44+
expect(SessionRunnerHealth.canAttempt(KEY, 61_000).allowed).toBe(true)
45+
// While the probe window runs, other attempts wait.
46+
expect(SessionRunnerHealth.canAttempt(KEY, 62_000).allowed).toBe(false)
47+
})
48+
49+
it("resets when the probe succeeds", () => {
50+
SessionRunnerHealth.$reset()
51+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, internal)
52+
expect(SessionRunnerHealth.canAttempt(KEY, 61_000).allowed).toBe(true)
53+
SessionRunnerHealth.recordSuccess(KEY)
54+
expect(SessionRunnerHealth.canAttempt(KEY, 62_000)).toEqual({ allowed: true, retryInMs: 0 })
55+
})
56+
57+
it("reopens with a longer cooldown when the probe fails", () => {
58+
SessionRunnerHealth.$reset()
59+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, internal)
60+
expect(SessionRunnerHealth.canAttempt(KEY, 61_000).allowed).toBe(true)
61+
SessionRunnerHealth.recordFailure(KEY, 62_000, internal)
62+
// Second opening doubles the cooldown: still closed 60s later, open at ~120s.
63+
expect(SessionRunnerHealth.canAttempt(KEY, 100_000).allowed).toBe(false)
64+
expect(SessionRunnerHealth.canAttempt(KEY, 183_000).allowed).toBe(true)
65+
})
66+
67+
it("re-opens the probe window when a probe never settles", () => {
68+
SessionRunnerHealth.$reset()
69+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, internal)
70+
expect(SessionRunnerHealth.canAttempt(KEY, 61_000).allowed).toBe(true)
71+
// The probe attempt is cancelled and never records; after the window the
72+
// next attempt becomes a fresh probe instead of wedging half-open.
73+
expect(SessionRunnerHealth.canAttempt(KEY, 61_000 + 30_001).allowed).toBe(true)
74+
})
75+
})
76+
77+
describe("shared Retry-After", () => {
78+
it("blocks all sessions until the provider-requested delay elapses", () => {
79+
SessionRunnerHealth.$reset()
80+
SessionRunnerHealth.recordFailure(KEY, 1000, rateLimit(10_000))
81+
expect(SessionRunnerHealth.canAttempt(KEY, 5000)).toEqual({ allowed: false, retryInMs: 6000 })
82+
expect(SessionRunnerHealth.canAttempt(KEY, 11_000)).toEqual({ allowed: true, retryInMs: 0 })
83+
})
84+
85+
it("counts rate limits toward opening the circuit", () => {
86+
SessionRunnerHealth.$reset()
87+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, rateLimit(1000))
88+
const gate = SessionRunnerHealth.canAttempt(KEY, 1500)
89+
expect(gate.allowed).toBe(false)
90+
// The shared Retry-After (last recorded) governs the wait, not the cooldown.
91+
expect(gate.retryInMs).toBe(500)
92+
})
93+
94+
it("caps a hostile retry-after at 15 minutes", () => {
95+
SessionRunnerHealth.$reset()
96+
SessionRunnerHealth.recordFailure(KEY, 0, rateLimit(24 * 60 * 60 * 1000))
97+
expect(SessionRunnerHealth.canAttempt(KEY, 15 * 60_000)).toEqual({ allowed: true, retryInMs: 0 })
98+
expect(SessionRunnerHealth.canAttempt(KEY, 15 * 60_000 - 1).allowed).toBe(false)
99+
})
100+
101+
it("does not record a shared next-allowed time for provider-internal failures", () => {
102+
SessionRunnerHealth.$reset()
103+
SessionRunnerHealth.recordFailure(KEY, 1000, internal)
104+
expect(SessionRunnerHealth.canAttempt(KEY, 1000).allowed).toBe(true)
105+
})
106+
})
107+
108+
describe("transport failures", () => {
109+
it("count toward the circuit like any other failure", () => {
110+
SessionRunnerHealth.$reset()
111+
for (let i = 0; i < 5; i++) SessionRunnerHealth.recordFailure(KEY, 1000, timeout)
112+
expect(SessionRunnerHealth.canAttempt(KEY, 2000).allowed).toBe(false)
113+
})
114+
})
115+
})

0 commit comments

Comments
 (0)