Skip to content

Commit 86c7f7c

Browse files
authored
fix(core): retry buffered generations with shared policy (#56497)
1 parent 8c173a8 commit 86c7f7c

7 files changed

Lines changed: 227 additions & 52 deletions

File tree

‎packages/core/src/generate.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { makeLocationNode } from "@opencode/util/effect/app-node"
77
import { llmClient } from "./effect/app-node-platform.js"
88
import { ModelResolver } from "./model-resolver.js"
99
import { Model } from "./model.js"
10+
import { ModelRetry } from "./model-retry.js"
1011

1112
export interface TextInput {
1213
readonly prompt: string
@@ -71,6 +72,7 @@ export const layer = Layer.effect(
7172
}),
7273
)
7374
.pipe(
75+
ModelRetry.retry,
7476
Effect.mapError(
7577
(error: AIError) =>
7678
new UnavailableError({

‎packages/core/src/model-retry.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
export * as ModelRetry from "./model-retry.js"
2+
3+
import { AIError, isRetryable } from "@opencode/ai"
4+
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
5+
6+
interface Input {
7+
readonly cause: AIError
8+
readonly retry: boolean
9+
}
10+
11+
/** Bound provider-requested delays so a hostile or buggy retry-after cannot stall a call for hours. */
12+
const RETRY_AFTER_MAX = Duration.toMillis("15 minutes")
13+
14+
const retryAfter = (input: Input) => {
15+
if (input.cause.reason._tag === "RateLimit" || input.cause.reason._tag === "ProviderInternal")
16+
return input.cause.reason.retryAfterMs === undefined
17+
? undefined
18+
: Math.min(input.cause.reason.retryAfterMs, RETRY_AFTER_MAX)
19+
return undefined
20+
}
21+
22+
// Exponential from 2s capped at 10s per gap, for 10 retries: 2, 4, 8, then 10 × 7, about 84s of
23+
// waiting when every attempt fails (67–101s with jitter). `min` takes the faster schedule, so the
24+
// cap applies per gap; `max` with `recurs` bounds the count.
25+
const schedule = Schedule.max([
26+
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("10 seconds")]),
27+
Schedule.recurs(10),
28+
]).pipe(
29+
Schedule.jittered,
30+
Schedule.setInputType<Input>(),
31+
Schedule.modifyDelay(({ input, duration: delay }) => {
32+
const minimum = retryAfter(input)
33+
const duration = minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))
34+
return Effect.succeed(Duration.millis(Math.ceil(Duration.toMillis(duration))))
35+
}),
36+
)
37+
38+
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
39+
// dead provider hold a call for most of an hour. Cap those attempts well below it.
40+
const MAX_TIMEOUT_RETRIES = 3
41+
42+
export const policy = Effect.gen(function* () {
43+
const step = yield* Schedule.toStep(schedule)
44+
let attempt = 1
45+
let timeouts = 0
46+
return (input: Input) =>
47+
Effect.gen(function* () {
48+
const now = yield* Clock.currentTimeMillis
49+
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
50+
if (!next) return undefined
51+
const [, duration] = next
52+
attempt++
53+
if (input.cause.reason._tag === "Transport" && input.cause.reason.code === "Timeout") timeouts++
54+
return {
55+
attempt,
56+
delay: Math.ceil(Duration.toMillis(duration)),
57+
retry: input.retry && timeouts <= MAX_TIMEOUT_RETRIES,
58+
}
59+
})
60+
})
61+
62+
/** Retry a buffered model call from scratch; failed attempts never contribute to its returned output. */
63+
export const retry = <A, R>(effect: Effect.Effect<A, AIError, R>) =>
64+
Effect.gen(function* () {
65+
const decide = yield* policy
66+
while (true) {
67+
const result = yield* Effect.result(effect)
68+
if (result._tag === "Success") return result.success
69+
const decision = yield* decide({ cause: result.failure, retry: isRetryable(result.failure) })
70+
if (!decision?.retry) return yield* result.failure
71+
yield* Effect.sleep(Duration.millis(decision.delay))
72+
}
73+
})

‎packages/core/src/session/generate.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import type { AgentNotFoundError } from "./error.js"
1212
import { SessionHistory } from "./history.js"
1313
import { SessionProviderContext } from "./provider-context.js"
1414
import { SessionModelRequest } from "./model-request.js"
15+
import { ModelRetry } from "../model-retry.js"
1516
import type { SessionRunnerModel } from "./runner/model.js"
1617
import type { SessionSchema } from "./schema.js"
1718

@@ -66,7 +67,7 @@ export const generate = Effect.fn("SessionGenerate.generate")(function* (input:
6667
providerID: model.ref.providerID,
6768
modelID: model.ref.id,
6869
})
69-
const response = yield* llm.generate(prepared.request, prepared.options)
70+
const response = yield* llm.generate(prepared.request, prepared.options).pipe(ModelRetry.retry)
7071
yield* Effect.logInfo("session generation usage diagnostic", { usage: response.usage })
7172
return response.text
7273
}).pipe(instances.provide(input.session))

‎packages/core/src/session/runner/retry.ts‎

Lines changed: 10 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ import { AIError, isRetryable } from "@opencode/ai"
44
import { Agent } from "@opencode/schema/agent"
55
import { Model } from "@opencode/schema/model"
66
import { SessionError } from "@opencode/schema/session-error"
7-
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
7+
import { Clock, Duration, Effect } from "effect"
88
import { Bus } from "../../bus.js"
9+
import { ModelRetry } from "../../model-retry.js"
910
import type { PluginHooks } from "../../plugin/hooks.js"
1011
import { SessionEvent } from "../event.js"
1112
import { SessionMessage } from "../message.js"
@@ -28,66 +29,28 @@ export interface Decision {
2829
readonly delay: number
2930
}
3031

31-
/** Bound provider-requested delays so a hostile or buggy retry-after cannot stall a session for hours. */
32-
const RETRY_AFTER_MAX = Duration.toMillis("15 minutes")
33-
34-
const retryAfter = (input: Input) => {
35-
if (input.cause.reason._tag === "RateLimit" || input.cause.reason._tag === "ProviderInternal")
36-
return input.cause.reason.retryAfterMs === undefined
37-
? undefined
38-
: Math.min(input.cause.reason.retryAfterMs, RETRY_AFTER_MAX)
39-
return undefined
40-
}
41-
42-
// Exponential from 2s capped at 10s per gap, for 10 retries: 2, 4, 8, then 10 × 7, about 84s of
43-
// waiting when every attempt fails (67–101s with jitter). `min` takes the faster schedule, so the
44-
// cap applies per gap; `max` with `recurs` bounds the count.
45-
const schedule = Schedule.max([
46-
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("10 seconds")]),
47-
Schedule.recurs(10),
48-
]).pipe(
49-
Schedule.jittered,
50-
Schedule.setInputType<Input>(),
51-
Schedule.modifyDelay(({ input, duration: delay }) => {
52-
const minimum = retryAfter(input)
53-
const duration = minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))
54-
return Effect.succeed(Duration.millis(Math.ceil(Duration.toMillis(duration))))
55-
}),
56-
)
57-
58-
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
59-
// dead provider hold a step for most of an hour. Cap those attempts well below it.
60-
const MAX_TIMEOUT_RETRIES = 3
61-
62-
const isTimeout = (error: AIError) => error.reason._tag === "Transport" && error.reason.code === "Timeout"
63-
6432
export const policy = (sessionID: SessionSchema.ID) =>
6533
Effect.gen(function* () {
66-
const step = yield* Schedule.toStep(schedule)
67-
let attempt = 1
68-
let timeouts = 0
34+
const decide = yield* ModelRetry.policy
6935
return (input: Input) =>
7036
Effect.gen(function* () {
71-
const now = yield* Clock.currentTimeMillis
72-
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
37+
const next = yield* decide(input)
7338
if (!next) return { retry: false as const }
74-
const [, duration] = next
75-
attempt++
76-
if (isTimeout(input.cause)) timeouts++
77-
const delay = Math.ceil(Duration.toMillis(duration))
7839
const event: PluginHooks.Domains["session"]["retry"] = {
7940
sessionID,
8041
agent: input.agent,
8142
model: input.model,
8243
error: input.error,
83-
attempt,
84-
decision: input.retry && timeouts <= MAX_TIMEOUT_RETRIES ? { retry: true, delay } : { retry: false },
44+
attempt: next.attempt,
45+
decision: next.retry ? { retry: true, delay: next.delay } : { retry: false },
8546
}
8647
yield* input.hook(event)
8748
if (!event.decision.retry) return event.decision
8849
const normalized =
89-
Number.isFinite(event.decision.delay) && event.decision.delay >= 0 ? Math.ceil(event.decision.delay) : delay
90-
return { retry: true as const, attempt, delay: normalized }
50+
Number.isFinite(event.decision.delay) && event.decision.delay >= 0
51+
? Math.ceil(event.decision.delay)
52+
: next.delay
53+
return { retry: true as const, attempt: next.attempt, delay: normalized }
9154
})
9255
})
9356

‎packages/core/test/generate.test.ts‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
import { expect } from "bun:test"
2-
import { LanguageModel, LLMClient } from "@opencode/ai"
2+
import {
3+
AIError,
4+
AuthenticationError,
5+
InvalidRequestError,
6+
LanguageModel,
7+
LLMClient,
8+
LLMEvent,
9+
ProviderInternalError,
10+
QuotaExceededError,
11+
RateLimitError,
12+
TransportError,
13+
} from "@opencode/ai"
314
import { RequestExecutor } from "@opencode/ai/route"
415
import { OpenAIChat } from "@opencode/ai/protocols"
516
import { TestLLM } from "@opencode/ai/testing"
@@ -10,7 +21,8 @@ import { ModelResolver } from "@opencode/core/model-resolver"
1021
import { ID, Info, Model, Ref } from "@opencode/core/model"
1122
import { Provider } from "@opencode/core/provider"
1223
import { Npm } from "@opencode/util/npm"
13-
import { Effect, Layer } from "effect"
24+
import { Effect, Exit, Fiber, Layer } from "effect"
25+
import { TestClock } from "effect/testing"
1426
import { HttpClient, HttpClientResponse } from "effect/http"
1527
import { testEffect } from "./lib/effect"
1628

@@ -19,6 +31,7 @@ const selected = Info.make({
1931
package: Provider.aisdk("@ai-sdk/perplexity"),
2032
})
2133
const runtime = LanguageModel.make({ id: "gemini", provider: "test-provider", route: OpenAIChat.route })
34+
const reference = Ref.make({ providerID: selected.providerID, id: selected.id })
2235

2336
const providers = Layer.mock(Provider.Service, {
2437
get: () => Effect.undefined,
@@ -66,6 +79,7 @@ const client = TestLLM.testLayer({ fallback: TestLLM.text("OK", "generate") })
6679
const resolver = ModelResolver.layer.pipe(Layer.provide(Layer.mergeAll(providers, models, integrations, npm, aisdk)))
6780
const it = testEffect(Generate.layer.pipe(Layer.provide(Layer.merge(resolver, client))))
6881
const resolverIt = testEffect(resolver)
82+
const retryIt = testEffect(Generate.layer.pipe(Layer.provideMerge(Layer.merge(resolver, TestLLM.testLayer()))))
6983

7084
it.effect("loads dynamic AI SDK models", () =>
7185
Effect.gen(function* () {
@@ -79,6 +93,91 @@ it.effect("loads dynamic AI SDK models", () =>
7993
}),
8094
)
8195

96+
for (const operation of ["request", "read"] as const) {
97+
retryIt.effect(`regenerates the full stateless response after a ${operation} failure`, () =>
98+
Effect.gen(function* () {
99+
const generate = yield* Generate.Service
100+
const llm = yield* TestLLM.Test
101+
yield* llm.push(
102+
TestLLM.failAfter(
103+
new AIError({ reason: new TransportError({ message: "Disconnected", transport: "http", operation }) }),
104+
...(operation === "read"
105+
? [LLMEvent.textStart({ id: "failed" }), LLMEvent.textDelta({ id: "failed", text: "Discard this" })]
106+
: []),
107+
),
108+
TestLLM.text("Fresh answer", "success"),
109+
)
110+
const result = yield* generate
111+
.text({ prompt: "Answer privately", model: reference })
112+
.pipe(Effect.forkChild({ startImmediately: true }))
113+
yield* llm.wait(1)
114+
yield* TestClock.adjust("3 seconds")
115+
expect(yield* Fiber.join(result)).toBe("Fresh answer")
116+
const requests = yield* llm.requests()
117+
expect(requests).toHaveLength(2)
118+
expect(requests[1]).toBe(requests[0])
119+
}),
120+
)
121+
}
122+
123+
for (const reason of [
124+
new AuthenticationError({ message: "Invalid credentials" }),
125+
new QuotaExceededError({ message: "No quota" }),
126+
new InvalidRequestError({ message: "Invalid prompt" }),
127+
]) {
128+
retryIt.effect(`does not retry stateless ${reason._tag} failures`, () =>
129+
Effect.gen(function* () {
130+
const generate = yield* Generate.Service
131+
const llm = yield* TestLLM.Test
132+
yield* llm.push(TestLLM.failAfter(new AIError({ reason })))
133+
const result = yield* generate.text({ prompt: "Answer privately", model: reference }).pipe(Effect.flip)
134+
expect(result).toBeInstanceOf(Generate.UnavailableError)
135+
expect(result.message).toBe(reason.message)
136+
expect(yield* llm.requests()).toHaveLength(1)
137+
}),
138+
)
139+
}
140+
141+
retryIt.effect("honors provider backoff and cancels without another generation", () =>
142+
Effect.gen(function* () {
143+
const generate = yield* Generate.Service
144+
const llm = yield* TestLLM.Test
145+
yield* llm.always(
146+
TestLLM.failAfter(new AIError({ reason: new RateLimitError({ message: "Wait", retryAfterMs: 5_000 }) })),
147+
)
148+
const result = yield* generate
149+
.text({ prompt: "Answer privately", model: reference })
150+
.pipe(Effect.forkChild({ startImmediately: true }))
151+
yield* llm.wait(1)
152+
yield* TestClock.adjust("4999 millis")
153+
expect(yield* llm.requests()).toHaveLength(1)
154+
yield* Fiber.interrupt(result)
155+
yield* TestClock.adjust("1 minute")
156+
expect(Exit.hasInterrupts(yield* Fiber.await(result))).toBe(true)
157+
expect(yield* llm.requests()).toHaveLength(1)
158+
}),
159+
)
160+
161+
for (const [reason, attempts] of [
162+
[new ProviderInternalError({ message: "Unavailable" }), 11],
163+
[new TransportError({ message: "Timed out", transport: "http", operation: "request", code: "Timeout" }), 4],
164+
] as const) {
165+
retryIt.effect(`bounds stateless ${reason._tag} retries`, () =>
166+
Effect.gen(function* () {
167+
const generate = yield* Generate.Service
168+
const llm = yield* TestLLM.Test
169+
yield* llm.always(TestLLM.failAfter(new AIError({ reason })))
170+
const result = yield* generate
171+
.text({ prompt: "Answer privately", model: reference })
172+
.pipe(Effect.flip, Effect.forkChild({ startImmediately: true }))
173+
yield* llm.wait(1)
174+
yield* TestClock.adjust("2 minutes")
175+
expect(yield* Fiber.join(result)).toBeInstanceOf(Generate.UnavailableError)
176+
expect(yield* llm.requests()).toHaveLength(attempts)
177+
}),
178+
)
179+
}
180+
82181
resolverIt.effect("resolves dynamic models with their catalog metadata", () =>
83182
Effect.gen(function* () {
84183
const resolver = yield* ModelResolver.Service

‎packages/core/test/session-generate.test.ts‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
import { expect } from "bun:test"
22
import {
3+
AIError,
34
LLMClient,
45
LLMEvent,
56
LLMResponse,
67
LanguageModel,
78
Message,
89
SystemPart,
910
ToolDefinition,
11+
TransportError,
1012
type LLMRequest,
1113
} from "@opencode/ai"
1214
import { OpenAIChat } from "@opencode/ai/protocols"
15+
import { TestLLM } from "@opencode/ai/testing"
1316
import type { StreamOptions } from "@opencode/ai/route"
1417
import { Agent } from "@opencode/core/agent"
1518
import { Database } from "@opencode/core/database/database"
@@ -51,7 +54,8 @@ import { PluginHooks } from "@opencode/core/plugin/hooks"
5154
import { PluginSupervisor } from "@opencode/core/plugin/supervisor"
5255
import { Tool } from "@opencode/core/tool"
5356
import { asc, eq } from "drizzle-orm"
54-
import { Effect, Layer, Schema, Stream } from "effect"
57+
import { Effect, Fiber, Layer, Schema, Stream } from "effect"
58+
import { TestClock } from "effect/testing"
5559
import { testEffect } from "./lib/effect"
5660

5761
const requests: LLMRequest[] = []
@@ -394,6 +398,39 @@ it.effect(
394398
{ timeout: 15_000 },
395399
)
396400

401+
it.effect("regenerates a failed partial answer without mutating Session history", () =>
402+
Effect.gen(function* () {
403+
instruction = "Initial context"
404+
const { db, bus, instructions, session, instances } = yield* setup
405+
yield* InstructionState.prepare(db, bus, instructions, sessionID)
406+
const before = yield* durableState(db, sessionID)
407+
const llm = yield* TestLLM.Test
408+
yield* llm.push(
409+
TestLLM.failAfter(
410+
new AIError({
411+
reason: new TransportError({ message: "Disconnected", transport: "http", operation: "read" }),
412+
}),
413+
LLMEvent.textStart({ id: "failed" }),
414+
LLMEvent.textDelta({ id: "failed", text: "Discard this" }),
415+
),
416+
TestLLM.text("Fresh answer", "success"),
417+
)
418+
const result = yield* SessionGenerate.generate({ session, prompt: "Summarize privately" }).pipe(
419+
Effect.provideService(Instance.Service, instances),
420+
Effect.provideService(LLMClient.Service, llm),
421+
Effect.forkChild({ startImmediately: true }),
422+
)
423+
yield* llm.wait(1)
424+
yield* TestClock.adjust("3 seconds")
425+
expect(yield* Fiber.join(result)).toBe("Fresh answer")
426+
const requests = yield* llm.requests()
427+
expect(requests).toHaveLength(2)
428+
expect(requests[1]).toBe(requests[0])
429+
expect(userTexts(requests[1])).toContain("Summarize privately")
430+
expect(yield* durableState(db, sessionID)).toEqual(before)
431+
}).pipe(Effect.provide(TestLLM.testLayer())),
432+
)
433+
397434
it.effect(
398435
"blocks unavailable initial instructions before generation",
399436
() =>

0 commit comments

Comments
 (0)