Skip to content

Commit 679669e

Browse files
author
engineer
committed
fix(session): retry unexpected abort errors (anomalyco#17717)
1 parent c2ca149 commit 679669e

4 files changed

Lines changed: 205 additions & 2 deletions

File tree

‎packages/opencode/src/session/processor.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ export namespace SessionProcessor {
364364
error,
365365
})
366366
} else {
367-
const retry = SessionRetry.retryable(error)
367+
const retry = SessionRetry.retryable(error, input.abort)
368368
if (retry !== undefined) {
369369
attempt++
370370
const delay = SessionRetry.delay(attempt, error.name === "APIError" ? error : undefined)

‎packages/opencode/src/session/retry.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,13 @@ export namespace SessionRetry {
5858
return Math.min(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), RETRY_MAX_DELAY_NO_HEADERS)
5959
}
6060

61-
export function retryable(error: ReturnType<NamedError["toObject"]>) {
61+
export function retryable(error: ReturnType<NamedError["toObject"]>, abort?: AbortSignal) {
6262
// context overflow errors should not be retried
6363
if (MessageV2.ContextOverflowError.isInstance(error)) return undefined
64+
if (MessageV2.AbortedError.isInstance(error)) {
65+
if (abort?.aborted) return undefined
66+
return error.data.message
67+
}
6468
if (MessageV2.APIError.isInstance(error)) {
6569
if (!error.data.isRetryable) return undefined
6670
if (error.data.responseBody?.includes("FreeUsageLimitError"))
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
import { describe, expect, test, spyOn } from "bun:test"
2+
import type { Agent } from "../../src/agent/agent"
3+
import { Agent as AgentApi } from "../../src/agent/agent"
4+
import { Instance } from "../../src/project/instance"
5+
import type { Provider } from "../../src/provider/provider"
6+
import { ModelID, ProviderID } from "../../src/provider/schema"
7+
import { Session } from "../../src/session"
8+
import { MessageV2 } from "../../src/session/message-v2"
9+
import * as LLMModule from "../../src/session/llm"
10+
import { SessionProcessor } from "../../src/session/processor"
11+
import * as RetryModule from "../../src/session/retry"
12+
import { MessageID } from "../../src/session/schema"
13+
import { Log } from "../../src/util/log"
14+
import { tmpdir } from "../fixture/fixture"
15+
16+
Log.init({ print: false })
17+
18+
type Stream = Awaited<ReturnType<typeof LLMModule.LLM.stream>>
19+
20+
function model() {
21+
return {
22+
id: ModelID.make("test-model"),
23+
providerID: ProviderID.make("test"),
24+
} as unknown as Provider.Model
25+
}
26+
27+
function user(sessionID: Session.Info["id"]) {
28+
return {
29+
id: MessageID.ascending(),
30+
sessionID,
31+
role: "user" as const,
32+
time: { created: Date.now() },
33+
agent: "build",
34+
model: {
35+
providerID: ProviderID.make("test"),
36+
modelID: ModelID.make("test-model"),
37+
},
38+
tools: {},
39+
} as MessageV2.User
40+
}
41+
42+
function assistant(sessionID: Session.Info["id"], parentID: MessageV2.User["id"], dir: string) {
43+
return {
44+
id: MessageID.ascending(),
45+
sessionID,
46+
parentID,
47+
role: "assistant" as const,
48+
mode: "build",
49+
agent: "build",
50+
path: {
51+
cwd: dir,
52+
root: dir,
53+
},
54+
cost: 0,
55+
tokens: {
56+
input: 0,
57+
output: 0,
58+
reasoning: 0,
59+
cache: { read: 0, write: 0 },
60+
},
61+
modelID: ModelID.make("test-model"),
62+
providerID: ProviderID.make("test"),
63+
time: {
64+
created: Date.now(),
65+
},
66+
} as MessageV2.Assistant
67+
}
68+
69+
function output(values: unknown[]) {
70+
return {
71+
fullStream: (async function* () {
72+
for (const value of values) yield value
73+
})(),
74+
} as unknown as Stream
75+
}
76+
77+
async function run(
78+
fn: (input: {
79+
agent: Agent.Info
80+
msg: MessageV2.Assistant
81+
session: Session.Info
82+
usr: MessageV2.User
83+
}) => Promise<void>,
84+
) {
85+
await using tmp = await tmpdir({ git: true })
86+
await Instance.provide({
87+
directory: tmp.path,
88+
fn: async () => {
89+
const session = await Session.create({})
90+
const agent = await AgentApi.get("build")
91+
const usr = (await Session.updateMessage(user(session.id))) as MessageV2.User
92+
const msg = (await Session.updateMessage(assistant(session.id, usr.id, tmp.path))) as MessageV2.Assistant
93+
await fn({ agent, msg, session, usr })
94+
await Session.remove(session.id)
95+
},
96+
})
97+
}
98+
99+
describe("session.processor", () => {
100+
test("retries unexpected aborts when the session is still active", async () => {
101+
await run(async ({ agent, msg, session, usr }) => {
102+
const ctl = new AbortController()
103+
const stream = spyOn(LLMModule.LLM, "stream")
104+
const sleep = spyOn(RetryModule.SessionRetry, "sleep").mockResolvedValue()
105+
let calls = 0
106+
107+
try {
108+
stream.mockImplementation(async (): Promise<Stream> => {
109+
calls++
110+
if (calls === 1) throw new DOMException("The operation was aborted.", "AbortError")
111+
return output([{ type: "start" }])
112+
})
113+
114+
const result = await SessionProcessor.create({
115+
assistantMessage: msg,
116+
sessionID: session.id,
117+
model: model(),
118+
abort: ctl.signal,
119+
}).process({
120+
user: usr,
121+
agent,
122+
abort: ctl.signal,
123+
sessionID: session.id,
124+
system: [],
125+
messages: [],
126+
tools: {},
127+
model: model(),
128+
})
129+
130+
const stored = await MessageV2.get({ sessionID: session.id, messageID: msg.id })
131+
if (stored.info.role !== "assistant") throw new Error("expected assistant")
132+
expect(result).toBe("continue")
133+
expect(stream).toHaveBeenCalledTimes(2)
134+
expect(sleep).toHaveBeenCalledTimes(1)
135+
expect(stored.info.error).toBeUndefined()
136+
expect(stored.info.time.completed).toBeDefined()
137+
} finally {
138+
stream.mockRestore()
139+
sleep.mockRestore()
140+
}
141+
})
142+
})
143+
144+
test("stops on aborts after session cancellation", async () => {
145+
await run(async ({ agent, msg, session, usr }) => {
146+
const ctl = new AbortController()
147+
ctl.abort()
148+
const stream = spyOn(LLMModule.LLM, "stream")
149+
const sleep = spyOn(RetryModule.SessionRetry, "sleep").mockResolvedValue()
150+
151+
try {
152+
stream.mockImplementation(async (): Promise<Stream> => {
153+
throw new DOMException("The operation was aborted.", "AbortError")
154+
})
155+
156+
const result = await SessionProcessor.create({
157+
assistantMessage: msg,
158+
sessionID: session.id,
159+
model: model(),
160+
abort: ctl.signal,
161+
}).process({
162+
user: usr,
163+
agent,
164+
abort: ctl.signal,
165+
sessionID: session.id,
166+
system: [],
167+
messages: [],
168+
tools: {},
169+
model: model(),
170+
})
171+
172+
const stored = await MessageV2.get({ sessionID: session.id, messageID: msg.id })
173+
if (stored.info.role !== "assistant") throw new Error("expected assistant")
174+
expect(result).toBe("stop")
175+
expect(stream).toHaveBeenCalledTimes(1)
176+
expect(sleep).not.toHaveBeenCalled()
177+
expect(stored.info.error?.name).toBe("MessageAbortedError")
178+
} finally {
179+
stream.mockRestore()
180+
sleep.mockRestore()
181+
}
182+
})
183+
})
184+
})

‎packages/opencode/test/session/retry.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,21 @@ describe("session.retry.retryable", () => {
125125

126126
expect(SessionRetry.retryable(error)).toBeUndefined()
127127
})
128+
129+
test("retries aborted errors when the session is still active", () => {
130+
const ctl = new AbortController()
131+
const error = new MessageV2.AbortedError({ message: "The operation was aborted." }).toObject()
132+
133+
expect(SessionRetry.retryable(error, ctl.signal)).toBe("The operation was aborted.")
134+
})
135+
136+
test("does not retry aborted errors after session cancellation", () => {
137+
const ctl = new AbortController()
138+
ctl.abort()
139+
const error = new MessageV2.AbortedError({ message: "The operation was aborted." }).toObject()
140+
141+
expect(SessionRetry.retryable(error, ctl.signal)).toBeUndefined()
142+
})
128143
})
129144

130145
describe("session.message-v2.fromError", () => {

0 commit comments

Comments
 (0)