Skip to content

Commit 123bb39

Browse files
CodePomclaude
andcommitted
fix(session): cap retry attempts and individual retry waits
SessionRetry.policy() retried retryable errors (429/529/overloaded) forever: no attempt cap and no bound on per-attempt sleep, so persistent provider failures kept sessions in 'retry' status indefinitely and hard quota limits (retry-after of hours/days) put sessions into multi-hour silent slumber. - add RETRY_MAX_ATTEMPTS = 3, checked at the top of the policy() step; overridable per call via opts.maxAttempts - add RETRY_MAX_WAIT = 60s: every computed wait is capped (Math.min), so a 6h retry-after burns its remaining attempts in minutes and surfaces session.error instead of slumbering. Capping (vs aborting) keeps retry status updates flowing, tolerates legitimate ~60-120s retry-after values, and bounds the uncapped backoff on the headers-without-retry-hints path. - hoist the pure delay() computation out of the Effect.gen block - tests: cap honored, maxAttempts override, 6h retry-after capped with status still published, 61s retry-after not aborted Fixes anomalyco#21960. Supersedes the auto-closed anomalyco#26369 by also addressing the hard-quota slumber gap raised in its review comments. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 9976269 commit 123bb39

2 files changed

Lines changed: 89 additions & 2 deletions

File tree

‎packages/opencode/src/session/retry.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ export const RETRY_INITIAL_DELAY = 2000
2727
export const RETRY_BACKOFF_FACTOR = 2
2828
export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
2929
export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout
30+
export const RETRY_MAX_ATTEMPTS = 3
31+
// Hard quota limits (e.g. Go weekly/monthly) send retry-after of hours or days.
32+
// Sleeping that long inside a session is indistinguishable from a hang, so cap
33+
// every individual wait: worst case the schedule burns its remaining attempts
34+
// in minutes and surfaces session.error instead of slumbering.
35+
export const RETRY_MAX_WAIT = 60_000 // 1 minute
3036

3137
function cap(ms: number) {
3238
return Math.min(ms, RETRY_MAX_DELAY)
@@ -177,14 +183,19 @@ export function policy(opts: {
177183
provider: string
178184
parse: (error: unknown) => Err
179185
set: (input: { attempt: number; message: string; action?: Retryable["action"]; next: number }) => Effect.Effect<void>
186+
maxAttempts?: number
180187
}) {
181188
return Schedule.fromStepWithMetadata(
182189
Effect.succeed((meta: Schedule.InputMetadata<unknown>) => {
190+
if (meta.attempt > (opts.maxAttempts ?? RETRY_MAX_ATTEMPTS)) return Cause.done(meta.attempt)
183191
const error = opts.parse(meta.input)
184192
const retry = retryable(error, opts.provider)
185193
if (!retry) return Cause.done(meta.attempt)
194+
const wait = Math.min(
195+
delay(meta.attempt, SessionV1.APIError.isInstance(error) ? error : undefined),
196+
RETRY_MAX_WAIT,
197+
)
186198
return Effect.gen(function* () {
187-
const wait = delay(meta.attempt, SessionV1.APIError.isInstance(error) ? error : undefined)
188199
const now = yield* Clock.currentTimeMillis
189200
yield* opts.set({
190201
attempt: meta.attempt,

‎packages/opencode/test/session/retry.test.ts‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { SessionV1 } from "@opencode-ai/core/v1/session"
44
import type { NamedError } from "@opencode-ai/core/util/error"
55
import { APICallError } from "ai"
66
import { setTimeout as sleep } from "node:timers/promises"
7-
import { Effect, Schedule, Schema } from "effect"
7+
import { Effect, Fiber, Schedule, Schema } from "effect"
88
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
99
import { SessionRetry } from "../../src/session/retry"
1010
import { MessageV2 } from "../../src/session/message-v2"
@@ -117,6 +117,82 @@ describe("session.retry.delay", () => {
117117
)
118118
})
119119

120+
describe("session.retry.policy.limits", () => {
121+
function policyStep(seen: { count: number; next?: number }, maxAttempts?: number) {
122+
return Schedule.toStepWithMetadata(
123+
SessionRetry.policy({
124+
provider: "test",
125+
parse: Schema.decodeUnknownSync(SessionV1.APIError.Schema),
126+
set: (info) =>
127+
Effect.sync(() => {
128+
seen.count++
129+
seen.next = info.next
130+
}),
131+
maxAttempts,
132+
}),
133+
)
134+
}
135+
136+
// the schedule step sleeps its computed wait, so for long waits we fork the
137+
// step, assert the published status, and interrupt instead of sleeping
138+
function firstWait(seen: { count: number; next?: number }, error: SessionV1.APIError) {
139+
return Effect.gen(function* () {
140+
const step = yield* policyStep(seen)
141+
const before = Date.now()
142+
const fiber = yield* Effect.forkChild(step(error))
143+
yield* Effect.promise(() => sleep(100))
144+
yield* Fiber.interrupt(fiber)
145+
expect(seen.count).toBe(1)
146+
return seen.next! - before
147+
})
148+
}
149+
150+
it.instance("stops after RETRY_MAX_ATTEMPTS", () =>
151+
Effect.gen(function* () {
152+
const error = apiError({ "retry-after-ms": "0" })
153+
const seen = { count: 0 }
154+
const step = yield* policyStep(seen)
155+
// steps beyond the cap terminate the schedule (Cause.done surfaces as a
156+
// failure in the step Effect), so absorb with Effect.ignore
157+
for (let i = 0; i < SessionRetry.RETRY_MAX_ATTEMPTS + 2; i++) {
158+
yield* Effect.ignore(step(error))
159+
}
160+
expect(seen.count).toBe(SessionRetry.RETRY_MAX_ATTEMPTS)
161+
}),
162+
)
163+
164+
it.instance("maxAttempts option overrides the default cap", () =>
165+
Effect.gen(function* () {
166+
const error = apiError({ "retry-after-ms": "0" })
167+
const seen = { count: 0 }
168+
const step = yield* policyStep(seen, 1)
169+
for (let i = 0; i < 3; i++) {
170+
yield* Effect.ignore(step(error))
171+
}
172+
expect(seen.count).toBe(1)
173+
}),
174+
)
175+
176+
it.instance("caps individual waits at RETRY_MAX_WAIT", () =>
177+
Effect.gen(function* () {
178+
// hard quota limits send retry-after of hours; never slumber the session
179+
const seen = { count: 0 }
180+
const wait = yield* firstWait(seen, apiError({ "retry-after": "21600" })) // 6 hours
181+
expect(wait).toBeLessThanOrEqual(SessionRetry.RETRY_MAX_WAIT + 1000)
182+
expect(wait).toBeGreaterThan(SessionRetry.RETRY_MAX_WAIT - 5000)
183+
}),
184+
)
185+
186+
it.instance("legitimate retry-after just above the cap is not aborted", () =>
187+
Effect.gen(function* () {
188+
const seen = { count: 0 }
189+
const wait = yield* firstWait(seen, apiError({ "retry-after": "61" }))
190+
// capped to RETRY_MAX_WAIT, still retried (status update published)
191+
expect(wait).toBeLessThanOrEqual(SessionRetry.RETRY_MAX_WAIT + 1000)
192+
}),
193+
)
194+
})
195+
120196
describe("session.retry.retryable", () => {
121197
test("maps too_many_requests json messages", () => {
122198
const error = wrap(JSON.stringify({ type: "error", error: { type: "too_many_requests" } }))

0 commit comments

Comments
 (0)