|
1 | 1 | export * as SessionRunnerHealth from "./provider-health.js" |
2 | 2 |
|
3 | 3 | import type { AIError } from "@opencode/ai" |
| 4 | +import { Effect } from "effect" |
| 5 | +import { SessionSchema } from "../schema.js" |
| 6 | +import type { SessionRunnerRetry } from "./retry.js" |
4 | 7 |
|
5 | 8 | /** |
6 | | - * Cross-session provider health, shared within one server process. |
| 9 | + * Cross-session provider health, shared by the sessions of one runner. |
7 | 10 | * |
8 | 11 | * Each session's retry policy is deliberately scoped to that session, so concurrent |
9 | 12 | * sessions and subagents retry the same failing provider independently and every |
10 | | - * attempt still burns a request. This module tracks failures per provider key so |
11 | | - * those sessions coordinate: |
| 13 | + * attempt still burns a request. `coordinate` folds a per-session retry decision |
| 14 | + * into a per-route health record so those sessions coordinate: |
| 15 | + * |
12 | 16 | * - a 429's `Retry-After` yields one shared next-allowed time for every session, |
13 | 17 | * instead of each session discovering it by failing on its own; |
14 | | - * - sustained 5xx / connection failures open a circuit so new attempts fail fast |
15 | | - * until a cooldown elapses, then one probe request is let through; |
16 | | - * - in-flight streams are never touched: the gate only decides whether a new |
17 | | - * attempt may start. |
18 | | - * |
19 | | - * See https://github.com/anomalyco/opencode/issues/55075 for the field data. |
| 18 | + * - once failures come from several distinct sessions, the shared wait grows a |
| 19 | + * cooldown so the sessions stop pounding the route in overlapping bursts — |
| 20 | + * after each cooldown the next attempt acts as the probe, and it succeeds |
| 21 | + * (record clears) or fails (cooldown doubles) without extra bookkeeping; |
| 22 | + * - rate limits never count toward that cooldown: they only share the |
| 23 | + * provider-requested delay, which is what #55075 actually asks for; |
| 24 | + * - the shared wait is a floor on the per-session decision, so the plugin |
| 25 | + * `session.retry` hook and the session's retry budget stay in the path. |
20 | 26 | */ |
21 | 27 |
|
22 | | -/** Consecutive provider-level failures before the circuit opens: roughly half of one exhausted per-session policy, so a single unlucky session cannot trip it alone while 2-3 concurrent sessions can. */ |
| 28 | +/** Distinct failing sessions before the shared cooldown kicks in. */ |
23 | 29 | const OPEN_THRESHOLD = 5 |
24 | | -/** Initial open-state cooldown, then doubled per further failure up to the cap. The per-session policy burns ~84s over its 10 attempts, so 60s opens the circuit around the time sessions would give up anyway. */ |
25 | | -const OPEN_COOLDOWN_MS = 60_000 |
26 | | -const OPEN_COOLDOWN_MAX_MS = 15 * 60_000 |
27 | | -/** A half-open probe has this long to settle (via recordSuccess/recordFailure) before another probe is allowed, so a probe lost to cancellation cannot wedge the half-open state. */ |
28 | | -const PROBE_WINDOW_MS = 30_000 |
29 | | -/** A hostile or buggy retry-after cannot stall new attempts for hours (mirrors the per-session cap in retry.ts). */ |
| 30 | +/** Initial shared cooldown, then doubled per further failure up to the cap. The per-session policy burns ~84s over its 10 attempts, so 60s engages around the time sessions would give up anyway. */ |
| 31 | +const COOLDOWN_MS = 60_000 |
| 32 | +const COOLDOWN_MAX_MS = 15 * 60_000 |
| 33 | +/** A hostile or buggy retry-after cannot stall attempts for hours (mirrors the per-session cap in retry.ts). */ |
30 | 34 | const RETRY_AFTER_MAX_MS = 15 * 60_000 |
31 | 35 |
|
32 | 36 | interface State { |
33 | | - consecutiveFailures: number |
| 37 | + sessions: Set<SessionSchema.ID> |
34 | 38 | openUntil: number |
35 | | - probeUntil: number |
| 39 | + opens: number |
36 | 40 | nextAllowedAt: number |
37 | 41 | } |
38 | 42 |
|
39 | | -// One map per server process; sessions reach it through the attempt gate and the |
40 | | -// retry path, which is all the coordination the storm fix needs. |
41 | | -const states = new Map<string, State>() |
| 43 | +export interface CoordinateInput { |
| 44 | + readonly key: string |
| 45 | + readonly sessionID: SessionSchema.ID |
| 46 | + /** The provider failure that produced `decision`, or undefined when there is none. */ |
| 47 | + readonly cause: AIError | undefined |
| 48 | + /** Decision from the per-session policy; `coordinate` may only raise its delay. */ |
| 49 | + readonly decision: { readonly retry: false } | SessionRunnerRetry.Decision |
| 50 | + /** Settle a clean pre-output failure only; partial-stream failures must not blame the provider. */ |
| 51 | + readonly clean: boolean |
| 52 | + readonly now: number |
| 53 | +} |
42 | 54 |
|
43 | | -const state = (key: string): State => { |
44 | | - let s = states.get(key) |
45 | | - if (!s) { |
46 | | - s = { consecutiveFailures: 0, openUntil: 0, probeUntil: 0, nextAllowedAt: 0 } |
47 | | - states.set(key, s) |
48 | | - } |
49 | | - return s |
| 55 | +export interface Service { |
| 56 | + coordinate: (input: CoordinateInput) => { readonly retry: false } | SessionRunnerRetry.Decision |
| 57 | + recordSuccess: (key: string) => void |
50 | 58 | } |
51 | 59 |
|
52 | | -/** One health bucket per provider route; routes (and their credentials) share fate because they are load-balanced together. */ |
| 60 | +/** One health record per provider route; routes (and their credentials) share fate because they are load-balanced together. */ |
53 | 61 | export const key = (provider: string, routeKey: string | undefined): string => routeKey ?? provider |
54 | 62 |
|
55 | | -/** Provider-requested retry delay, only where the provider actually signals one. */ |
56 | | -export const retryAfterMs = (cause: AIError): number | undefined => { |
| 63 | +const retryAfterMs = (cause: AIError | undefined): number | undefined => { |
| 64 | + if (cause === undefined) return undefined |
57 | 65 | if (cause.reason._tag === "RateLimit" || cause.reason._tag === "ProviderInternal") |
58 | 66 | return cause.reason.retryAfterMs |
59 | 67 | return undefined |
60 | 68 | } |
61 | 69 |
|
62 | | -export const canAttempt = (key: string, now: number): { allowed: boolean; retryInMs: number } => { |
63 | | - const s = states.get(key) |
64 | | - if (!s) return { allowed: true, retryInMs: 0 } |
65 | | - if (now < s.nextAllowedAt) return { allowed: false, retryInMs: s.nextAllowedAt - now } |
66 | | - if (now < s.openUntil) return { allowed: false, retryInMs: s.openUntil - now } |
67 | | - if (s.consecutiveFailures >= OPEN_THRESHOLD) { |
68 | | - // Half-open: let this attempt probe the provider while holding others back |
69 | | - // until the probe settles or the window expires. |
70 | | - s.openUntil = now + PROBE_WINDOW_MS |
71 | | - s.probeUntil = s.openUntil |
| 70 | +/** |
| 71 | + * Builds an isolated health instance. The runner layer constructs one per server; |
| 72 | + * tests construct one per case, which keeps state out of the module scope. |
| 73 | + */ |
| 74 | +export const make = Effect.gen(function* () { |
| 75 | + const states = new Map<string, State>() |
| 76 | + |
| 77 | + const state = (key: string): State => { |
| 78 | + let s = states.get(key) |
| 79 | + if (!s) { |
| 80 | + s = { sessions: new Set(), openUntil: 0, opens: 0, nextAllowedAt: 0 } |
| 81 | + states.set(key, s) |
| 82 | + } |
| 83 | + return s |
72 | 84 | } |
73 | | - return { allowed: true, retryInMs: 0 } |
74 | | -} |
75 | 85 |
|
76 | | -export const recordFailure = (key: string, now: number, cause: AIError | undefined): void => { |
77 | | - const s = state(key) |
78 | | - s.probeUntil = 0 |
79 | | - s.consecutiveFailures++ |
80 | | - const after = cause === undefined ? undefined : retryAfterMs(cause) |
81 | | - if (after !== undefined) s.nextAllowedAt = now + Math.min(Math.max(0, after), RETRY_AFTER_MAX_MS) |
82 | | - if (s.consecutiveFailures >= OPEN_THRESHOLD) { |
83 | | - const doubling = Math.min(2 ** Math.min(s.consecutiveFailures - OPEN_THRESHOLD, 4), 16) |
84 | | - s.openUntil = now + Math.min(OPEN_COOLDOWN_MS * doubling, OPEN_COOLDOWN_MAX_MS) |
| 86 | + const coordinate: Service["coordinate"] = (input) => { |
| 87 | + if (!input.decision.retry) return input.decision |
| 88 | + const s = state(input.key) |
| 89 | + // 5xx and connection failures come from the provider side; rate limits are |
| 90 | + // expected under concurrency and only share the requested delay. |
| 91 | + const counted = |
| 92 | + input.clean && |
| 93 | + input.cause !== undefined && |
| 94 | + (input.cause.reason._tag === "Transport" || input.cause.reason._tag === "ProviderInternal") |
| 95 | + if (counted) s.sessions.add(input.sessionID) |
| 96 | + const after = retryAfterMs(input.cause) |
| 97 | + if (input.clean && after !== undefined) |
| 98 | + s.nextAllowedAt = Math.max(s.nextAllowedAt, input.now + Math.min(Math.max(0, after), RETRY_AFTER_MAX_MS)) |
| 99 | + |
| 100 | + let waitUntil = 0 |
| 101 | + if (s.sessions.size >= OPEN_THRESHOLD) { |
| 102 | + if (input.now >= s.openUntil) { |
| 103 | + // First opening, or the previous cooldown elapsed and this failure was the probe. |
| 104 | + s.opens++ |
| 105 | + s.openUntil = input.now + Math.min(COOLDOWN_MS * 2 ** Math.min(s.opens - 1, 4), COOLDOWN_MAX_MS) |
| 106 | + } |
| 107 | + waitUntil = s.openUntil |
| 108 | + } |
| 109 | + waitUntil = Math.max(waitUntil, s.nextAllowedAt) |
| 110 | + const floor = Math.max(0, waitUntil - input.now) |
| 111 | + return floor > input.decision.delay ? { ...input.decision, delay: floor } : input.decision |
85 | 112 | } |
86 | | -} |
87 | 113 |
|
88 | | -export const recordSuccess = (key: string): void => { |
89 | | - states.delete(key) |
90 | | -} |
| 114 | + const recordSuccess: Service["recordSuccess"] = (key) => { |
| 115 | + states.delete(key) |
| 116 | + } |
91 | 117 |
|
92 | | -/** Test seam: drops all shared state between tests. */ |
93 | | -export const $reset = (): void => { |
94 | | - states.clear() |
95 | | -} |
| 118 | + return { coordinate, recordSuccess } |
| 119 | +}) |
0 commit comments