Repository navigation
Expand file tree
/
Copy pathretry.ts
More file actions
115 lines (104 loc) · 4.47 KB
/
Copy pathretry.ts
File metadata and controls
115 lines (104 loc) · 4.47 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
export * as SessionRunnerRetry from "./retry.js"
import { AIError, isRetryable } from "@opencode/ai"
import { Agent } from "@opencode/schema/agent"
import { Model } from "@opencode/schema/model"
import { SessionError } from "@opencode/schema/session-error"
import { Clock, Duration, Effect, Pull, Schedule } from "effect"
import { Bus } from "../../bus.js"
import type { PluginHooks } from "../../plugin/hooks.js"
import { SessionEvent } from "../event.js"
import { SessionMessage } from "../message.js"
import { SessionSchema } from "../schema.js"
export { isRetryable }
interface Input {
readonly cause: AIError
readonly error: SessionError.Error
readonly agent: Agent.ID
readonly model: Model.Ref
readonly hook: (event: PluginHooks.Domains["session"]["retry"]) => Effect.Effect<void>
readonly retry: boolean
}
export interface Decision {
readonly retry: true
readonly attempt: number
readonly delay: number
}
/** Bound provider-requested delays so a hostile or buggy retry-after cannot stall a session for hours. */
const RETRY_AFTER_MAX = Duration.toMillis("15 minutes")
const retryAfter = (input: Input) => {
if (input.cause.reason._tag === "RateLimit" || input.cause.reason._tag === "ProviderInternal")
return input.cause.reason.retryAfterMs === undefined
? undefined
: Math.min(input.cause.reason.retryAfterMs, RETRY_AFTER_MAX)
return undefined
}
// Exponential from 2s capped at 10s per gap, for 10 retries: 2, 4, 8, then 10 × 7, about 84s of
// waiting when every attempt fails (67–101s with jitter). `min` takes the faster schedule, so the
// cap applies per gap; `max` with `recurs` bounds the count.
const schedule = Schedule.max([
Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("10 seconds")]),
Schedule.recurs(10),
]).pipe(
Schedule.jittered,
Schedule.setInputType<Input>(),
Schedule.modifyDelay(({ input, duration: delay }) => {
const minimum = retryAfter(input)
const duration = minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))
return Effect.succeed(Duration.millis(Math.ceil(Duration.toMillis(duration))))
}),
)
// A timed-out attempt already waited minutes before failing, so the general allowance would let a
// dead provider hold a step for most of an hour. Cap those attempts well below it.
const MAX_TIMEOUT_RETRIES = 3
const isTimeout = (error: AIError) => error.reason._tag === "Transport" && error.reason.code === "Timeout"
export const policy = (sessionID: SessionSchema.ID) =>
Effect.gen(function* () {
const step = yield* Schedule.toStep(schedule)
let attempt = 1
let timeouts = 0
return (input: Input) =>
Effect.gen(function* () {
const now = yield* Clock.currentTimeMillis
const next = yield* step(now, input).pipe(Pull.catchDone(() => Effect.succeed(undefined)))
if (!next) return { retry: false as const }
const [, duration] = next
attempt++
if (isTimeout(input.cause)) timeouts++
const delay = Math.ceil(Duration.toMillis(duration))
const event: PluginHooks.Domains["session"]["retry"] = {
sessionID,
agent: input.agent,
model: input.model,
error: input.error,
attempt,
decision: input.retry && timeouts <= MAX_TIMEOUT_RETRIES ? { retry: true, delay } : { retry: false },
}
yield* input.hook(event)
if (!event.decision.retry) return event.decision
const normalized =
Number.isFinite(event.decision.delay) && event.decision.delay >= 0 ? Math.ceil(event.decision.delay) : delay
return { retry: true as const, attempt, delay: normalized }
})
})
export const make = (bus: Bus.Interface, sessionID: SessionSchema.ID) =>
Effect.gen(function* () {
const decide = yield* policy(sessionID)
const wait = (input: {
readonly decision: Decision
readonly assistantMessageID: SessionMessage.ID
readonly error: SessionError.Error
}) =>
Effect.gen(function* () {
const scheduled = yield* Clock.currentTimeMillis
yield* bus.publish(SessionEvent.RetryScheduled, {
sessionID,
assistantMessageID: input.assistantMessageID,
attempt: input.decision.attempt,
at: scheduled + input.decision.delay,
error: input.error,
})
const remaining = Math.max(0, scheduled + input.decision.delay - (yield* Clock.currentTimeMillis))
yield* Effect.sleep(Duration.millis(remaining))
})
return { decide, wait }
})