Skip to content

Commit d03f46c

Browse files
committed
fix(api): pass abortSignal to streamText() for OpenAI Compatible provider (Zoo-Code-Org#404)
When user clicks stop during streaming, the HTTP request to the provider continues running because Vercel AI SDK's streamText() doesn't receive an abort signal. This wastes API tokens/compute on the provider side. Changes: - Add abortSignal?: AbortSignal to ApiHandlerCreateMessageMetadata interface - Pass Task.ts's AbortController.signal through metadata to createMessage() - Use signal in openai-compatible.ts streamText() options This fix affects all OpenAICompatibleHandler subclasses (LiteLLM, OpenRouter, Vercel AI Gateway, etc.) that use the Vercel AI SDK. Fixes Zoo-Code-Org#404
1 parent f1f7cb4 commit d03f46c

20 files changed

Lines changed: 1199 additions & 31 deletions

‎ABORT_ANALYSIS.md‎

Lines changed: 442 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
// Tests for ApiHandlerCreateMessageMetadata type definition.
2+
// Verifies that the interface includes abortSignal and maintains backward compatibility.
3+
4+
import type { ApiHandlerCreateMessageMetadata } from "../index"
5+
6+
describe("ApiHandlerCreateMessageMetadata", () => {
7+
describe("type structure", () => {
8+
it("should include taskId as a required field", () => {
9+
const metadata: ApiHandlerCreateMessageMetadata = {
10+
taskId: "test-task-id",
11+
}
12+
expect(metadata.taskId).toBe("test-task-id")
13+
})
14+
15+
it("should include abortSignal as an optional field", () => {
16+
// Should compile without error — abortSignal is optional
17+
const metadataWithoutSignal: ApiHandlerCreateMessageMetadata = {
18+
taskId: "test-task-id",
19+
}
20+
expect(metadataWithoutSignal.taskId).toBe("test-task-id")
21+
expect((metadataWithoutSignal as any).abortSignal).toBeUndefined()
22+
23+
// Should also compile with abortSignal provided
24+
const controller = new AbortController()
25+
const metadataWithSignal: ApiHandlerCreateMessageMetadata = {
26+
taskId: "test-task-id",
27+
abortSignal: controller.signal,
28+
}
29+
expect(metadataWithSignal.abortSignal).toBe(controller.signal)
30+
})
31+
32+
it("should support all documented optional fields", () => {
33+
const controller = new AbortController()
34+
const metadata: ApiHandlerCreateMessageMetadata = {
35+
taskId: "test-task-id",
36+
mode: "code",
37+
suppressPreviousResponseId: true,
38+
store: false,
39+
tools: [],
40+
tool_choice: "auto",
41+
parallelToolCalls: true,
42+
allowedFunctionNames: ["read_file"],
43+
abortSignal: controller.signal,
44+
}
45+
46+
expect(metadata.taskId).toBe("test-task-id")
47+
expect(metadata.mode).toBe("code")
48+
expect(metadata.suppressPreviousResponseId).toBe(true)
49+
expect(metadata.store).toBe(false)
50+
expect(metadata.tools).toEqual([])
51+
expect(metadata.tool_choice).toBe("auto")
52+
expect(metadata.parallelToolCalls).toBe(true)
53+
expect(metadata.allowedFunctionNames).toEqual(["read_file"])
54+
expect(metadata.abortSignal).toBe(controller.signal)
55+
})
56+
57+
it("should be backward compatible — existing callers without abortSignal should work", () => {
58+
// Simulate an existing caller that doesn't know about abortSignal
59+
const legacyMetadata: ApiHandlerCreateMessageMetadata = {
60+
taskId: "legacy-task",
61+
mode: "architect",
62+
}
63+
64+
expect(legacyMetadata.taskId).toBe("legacy-task")
65+
expect(legacyMetadata.mode).toBe("architect")
66+
expect((legacyMetadata as any).abortSignal).toBeUndefined()
67+
})
68+
69+
it("should allow undefined abortSignal explicitly", () => {
70+
const metadata: ApiHandlerCreateMessageMetadata = {
71+
taskId: "test-task-id",
72+
abortSignal: undefined,
73+
}
74+
expect(metadata.abortSignal).toBeUndefined()
75+
})
76+
})
77+
78+
describe("signal type correctness", () => {
79+
it("should accept AbortController.signal as abortSignal value", () => {
80+
const controller = new AbortController()
81+
const signal: AbortSignal = controller.signal
82+
83+
const metadata: ApiHandlerCreateMessageMetadata = {
84+
taskId: "test-task-id",
85+
abortSignal: signal,
86+
}
87+
88+
expect(metadata.abortSignal).toBe(signal)
89+
expect(controller.signal.aborted).toBe(false)
90+
91+
controller.abort()
92+
expect(controller.signal.aborted).toBe(true)
93+
})
94+
95+
it("should accept AbortSignal.timeout() as abortSignal value", () => {
96+
const timeoutSignal = AbortSignal.timeout(5000)
97+
98+
const metadata: ApiHandlerCreateMessageMetadata = {
99+
taskId: "test-task-id",
100+
abortSignal: timeoutSignal,
101+
}
102+
103+
expect(metadata.abortSignal).toBe(timeoutSignal)
104+
})
105+
})
106+
})

‎src/api/index.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,12 @@ export interface ApiHandlerCreateMessageMetadata {
8989
* Only applies to providers that support function calling restrictions (e.g., Gemini).
9090
*/
9191
allowedFunctionNames?: string[]
92+
/**
93+
* Abort signal for cancelling the HTTP request mid-stream.
94+
* Passed through to AI SDK's streamText() so the underlying HTTP request is aborted
95+
* when the user clicks stop, preventing wasted API tokens/compute on the provider side.
96+
*/
97+
abortSignal?: AbortSignal
9298
}
9399

94100
export interface ApiHandler {
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
// Tests for OpenAICompatibleHandler's abortSignal passing to streamText()
2+
// Verifies that when createMessage() is called with metadata containing an abortSignal,
3+
// the signal is passed through to AI SDK's streamText() so HTTP requests can be aborted.
4+
5+
const { mockStreamText } = vi.hoisted(() => ({
6+
mockStreamText: vi.fn(),
7+
}))
8+
9+
vi.mock("ai", async (importOriginal) => {
10+
const actual = await importOriginal<typeof import("ai")>()
11+
return {
12+
...actual,
13+
streamText: mockStreamText,
14+
}
15+
})
16+
17+
vi.mock("@ai-sdk/openai-compatible", () => ({
18+
createOpenAICompatible: vi.fn(() => {
19+
return vi.fn(() => ({
20+
modelId: "test-model",
21+
provider: "test-provider",
22+
}))
23+
}),
24+
}))
25+
26+
import type { Anthropic } from "@anthropic-ai/sdk"
27+
28+
import { OpenAICompatibleHandler, OpenAICompatibleConfig } from "../openai-compatible"
29+
import type { ApiHandlerOptions } from "../../../shared/api"
30+
import type { ModelInfo } from "@roo-code/types"
31+
32+
// Concrete test subclass of the abstract OpenAICompatibleHandler
33+
class TestOpenAiCompatibleHandler extends OpenAICompatibleHandler {
34+
constructor(options: ApiHandlerOptions, config: OpenAICompatibleConfig) {
35+
super(options, config)
36+
}
37+
38+
override getModel(): { id: string; info: ModelInfo } {
39+
return { id: this.config.modelId, info: this.config.modelInfo }
40+
}
41+
}
42+
43+
describe("OpenAICompatibleHandler abort signal", () => {
44+
let handler: TestOpenAiCompatibleHandler
45+
const mockOptions: ApiHandlerOptions = {}
46+
const config: OpenAICompatibleConfig = {
47+
providerName: "test-provider",
48+
baseURL: "https://api.test.com/v1",
49+
apiKey: "test-key",
50+
modelId: "test-model",
51+
modelInfo: { maxTokens: 8192, contextWindow: 128000, supportsImages: false, supportsPromptCache: true },
52+
}
53+
54+
beforeEach(() => {
55+
vi.clearAllMocks()
56+
handler = new TestOpenAiCompatibleHandler(mockOptions, config)
57+
})
58+
59+
describe("createMessage abortSignal passing", () => {
60+
const systemPrompt = "You are a helpful assistant."
61+
const messages: Anthropic.Messages.MessageParam[] = [
62+
{ role: "user", content: [{ type: "text" as const, text: "Hello!" }] },
63+
]
64+
65+
it("should pass abortSignal to streamText when provided in metadata", async () => {
66+
const controller = new AbortController()
67+
const mockAbortSignal = controller.signal
68+
69+
async function* mockFullStream() {
70+
yield { type: "text-delta", text: "Test response" }
71+
}
72+
73+
function createMockStream(yieldValue) {
74+
return {
75+
fullStream: (async function* () { yield yieldValue; })(),
76+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
77+
}
78+
}
79+
80+
const mockUsage = Promise.resolve({ inputTokens: 10, outputTokens: 5 })
81+
82+
mockStreamText.mockReturnValue({
83+
fullStream: mockFullStream(),
84+
usage: mockUsage,
85+
})
86+
87+
await handler.createMessage(systemPrompt, messages, {
88+
taskId: "test-task",
89+
abortSignal: mockAbortSignal,
90+
}).next()
91+
92+
expect(mockStreamText).toHaveBeenCalledWith(
93+
expect.objectContaining({
94+
signal: mockAbortSignal,
95+
}),
96+
)
97+
})
98+
99+
it("should pass undefined signal to streamText when abortSignal is not provided", async () => {
100+
async function* mockFullStream() {
101+
yield { type: "text-delta", text: "Test response" }
102+
}
103+
104+
function createMockStream(yieldValue) {
105+
return {
106+
fullStream: (async function* () { yield yieldValue; })(),
107+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
108+
}
109+
}
110+
111+
const mockUsage = Promise.resolve({ inputTokens: 10, outputTokens: 5 })
112+
113+
mockStreamText.mockReturnValue({
114+
fullStream: mockFullStream(),
115+
usage: mockUsage,
116+
})
117+
118+
await handler.createMessage(systemPrompt, messages, {
119+
taskId: "test-task",
120+
}).next()
121+
122+
expect(mockStreamText).toHaveBeenCalledWith(
123+
expect.objectContaining({
124+
signal: undefined,
125+
}),
126+
)
127+
})
128+
129+
it("should pass signal to streamText when metadata is undefined", async () => {
130+
async function* mockFullStream() {
131+
yield { type: "text-delta", text: "Test response" }
132+
}
133+
134+
function createMockStream(yieldValue) {
135+
return {
136+
fullStream: (async function* () { yield yieldValue; })(),
137+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
138+
}
139+
}
140+
141+
const mockUsage = Promise.resolve({ inputTokens: 10, outputTokens: 5 })
142+
143+
mockStreamText.mockReturnValue({
144+
fullStream: mockFullStream(),
145+
usage: mockUsage,
146+
})
147+
148+
await handler.createMessage(systemPrompt, messages).next()
149+
150+
expect(mockStreamText).toHaveBeenCalledWith(
151+
expect.objectContaining({
152+
signal: undefined,
153+
}),
154+
)
155+
})
156+
157+
it("should pass the correct signal when it fires during streaming", async () => {
158+
const controller = new AbortController()
159+
const mockAbortSignal = controller.signal
160+
161+
let capturedOptions: any = null
162+
163+
mockStreamText.mockImplementation((options) => {
164+
capturedOptions = options
165+
return {
166+
fullStream: (async function* () {
167+
yield { type: "text-delta", text: "Partial" }
168+
})(),
169+
usage: Promise.resolve({ inputTokens: 5, outputTokens: 3 }),
170+
}
171+
})
172+
173+
const stream = handler.createMessage(systemPrompt, messages, {
174+
taskId: "test-task",
175+
abortSignal: mockAbortSignal,
176+
})
177+
178+
// Verify the signal was captured before aborting
179+
expect(capturedOptions).toBeDefined()
180+
expect(capturedOptions.signal).toBe(mockAbortSignal)
181+
182+
// Now abort - this should cause streamText to receive an aborted signal
183+
controller.abort()
184+
expect(controller.signal.aborted).toBe(true)
185+
})
186+
187+
it("should pass all other request options alongside the signal", async () => {
188+
const controller = new AbortController()
189+
190+
let capturedOptions: any = null
191+
192+
mockStreamText.mockImplementation((options) => {
193+
capturedOptions = options
194+
return {
195+
fullStream: (async function* () {
196+
yield { type: "text-delta", text: "Test" }
197+
})(),
198+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
199+
}
200+
})
201+
202+
await handler.createMessage(systemPrompt, messages, {
203+
taskId: "test-task",
204+
abortSignal: controller.signal,
205+
}).next()
206+
207+
expect(capturedOptions).toHaveProperty("model")
208+
expect(capturedOptions).toHaveProperty("system", systemPrompt)
209+
expect(capturedOptions).toHaveProperty("messages")
210+
expect(capturedOptions).toHaveProperty("signal", controller.signal)
211+
})
212+
})
213+
})

‎src/api/providers/base-openai-compatible-provider.ts‎

100644100755
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
104104
}
105105

106106
try {
107-
return this.client.chat.completions.create(params, requestOptions)
107+
return this.client.chat.completions.create(params, { ...requestOptions, signal: metadata?.abortSignal })
108108
} catch (error) {
109109
throw handleOpenAIError(error, this.providerName)
110110
}

‎src/api/providers/deepseek.ts‎

100644100755
Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,10 @@ export class DeepSeekHandler extends OpenAiHandler {
133133
try {
134134
stream = await this.client.chat.completions.create(
135135
requestOptions as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming,
136-
isAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {},
136+
{
137+
...(isAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}),
138+
signal: metadata?.abortSignal,
139+
},
137140
)
138141
} catch (error) {
139142
const { handleOpenAIError } = await import("./utils/openai-error-handler")

‎src/api/providers/lite-llm.ts‎

100644100755
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,9 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa
223223
}
224224

225225
try {
226-
const { data: completion } = await this.client.chat.completions.create(requestOptions).withResponse()
226+
const { data: completion } = await this.client.chat.completions
227+
.create(requestOptions, { signal: metadata?.abortSignal })
228+
.withResponse()
227229

228230
let lastUsage
229231

‎src/api/providers/lm-studio.ts‎

100644100755
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
9999

100100
let results
101101
try {
102-
results = await this.client.chat.completions.create(params)
102+
results = await this.client.chat.completions.create(params, { signal: metadata?.abortSignal })
103103
} catch (error) {
104104
throw handleOpenAIError(error, this.providerName)
105105
}

‎src/api/providers/mimo.ts‎

100644100755
Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,10 @@ export class MimoHandler extends OpenAiHandler {
9999

100100
let stream: AsyncIterable<OpenAI.Chat.Completions.ChatCompletionChunk>
101101
try {
102-
stream = (await this.client.chat.completions.create(params as any)) as any
102+
stream = (await this.client.chat.completions.create(
103+
params as any,
104+
{ signal: metadata?.abortSignal } as any,
105+
)) as any
103106
} catch (error) {
104107
throw handleProviderError(error, "MiMo")
105108
}

‎src/api/providers/openai-compatible.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ export abstract class OpenAICompatibleHandler extends BaseProvider implements Si
174174
maxOutputTokens: this.getMaxOutputTokens(),
175175
tools: aiSdkTools,
176176
toolChoice: this.mapToolChoice(metadata?.tool_choice),
177+
abortSignal: metadata?.abortSignal,
177178
}
178179

179180
// Use streamText for streaming responses

0 commit comments

Comments
 (0)