Skip to content

Commit 8783455

Browse files
committed
fix(api): pass abortSignal to streamText() for OpenAI Compatible provider (Zoo-Code-Org#404)
When user clicks stop during streaming, the HTTP request to the provider continues running because Vercel AI SDK's streamText() doesn't receive an abort signal. This wastes API tokens/compute on the provider side. Changes: - Add abortSignal?: AbortSignal to ApiHandlerCreateMessageMetadata interface - Pass Task.ts's AbortController.signal through metadata to createMessage() - Use signal in openai-compatible.ts streamText() options This fix affects all OpenAICompatibleHandler subclasses (LiteLLM, OpenRouter, Vercel AI Gateway, etc.) that use the Vercel AI SDK. Fixes Zoo-Code-Org#404
1 parent f1f7cb4 commit 8783455

7 files changed

Lines changed: 1149 additions & 7 deletions

File tree

‎ABORT_ANALYSIS.md‎

Lines changed: 442 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
// Tests for ApiHandlerCreateMessageMetadata type definition.
2+
// Verifies that the interface includes abortSignal and maintains backward compatibility.
3+
4+
import type { ApiHandlerCreateMessageMetadata } from "../index"
5+
6+
describe("ApiHandlerCreateMessageMetadata", () => {
7+
describe("type structure", () => {
8+
it("should include taskId as a required field", () => {
9+
const metadata: ApiHandlerCreateMessageMetadata = {
10+
taskId: "test-task-id",
11+
}
12+
expect(metadata.taskId).toBe("test-task-id")
13+
})
14+
15+
it("should include abortSignal as an optional field", () => {
16+
// Should compile without error — abortSignal is optional
17+
const metadataWithoutSignal: ApiHandlerCreateMessageMetadata = {
18+
taskId: "test-task-id",
19+
}
20+
expect(metadataWithoutSignal.taskId).toBe("test-task-id")
21+
expect((metadataWithoutSignal as any).abortSignal).toBeUndefined()
22+
23+
// Should also compile with abortSignal provided
24+
const controller = new AbortController()
25+
const metadataWithSignal: ApiHandlerCreateMessageMetadata = {
26+
taskId: "test-task-id",
27+
abortSignal: controller.signal,
28+
}
29+
expect(metadataWithSignal.abortSignal).toBe(controller.signal)
30+
})
31+
32+
it("should support all documented optional fields", () => {
33+
const controller = new AbortController()
34+
const metadata: ApiHandlerCreateMessageMetadata = {
35+
taskId: "test-task-id",
36+
mode: "code",
37+
suppressPreviousResponseId: true,
38+
store: false,
39+
tools: [],
40+
tool_choice: "auto",
41+
parallelToolCalls: true,
42+
allowedFunctionNames: ["read_file"],
43+
abortSignal: controller.signal,
44+
}
45+
46+
expect(metadata.taskId).toBe("test-task-id")
47+
expect(metadata.mode).toBe("code")
48+
expect(metadata.suppressPreviousResponseId).toBe(true)
49+
expect(metadata.store).toBe(false)
50+
expect(metadata.tools).toEqual([])
51+
expect(metadata.tool_choice).toBe("auto")
52+
expect(metadata.parallelToolCalls).toBe(true)
53+
expect(metadata.allowedFunctionNames).toEqual(["read_file"])
54+
expect(metadata.abortSignal).toBe(controller.signal)
55+
})
56+
57+
it("should be backward compatible — existing callers without abortSignal should work", () => {
58+
// Simulate an existing caller that doesn't know about abortSignal
59+
const legacyMetadata: ApiHandlerCreateMessageMetadata = {
60+
taskId: "legacy-task",
61+
mode: "architect",
62+
}
63+
64+
expect(legacyMetadata.taskId).toBe("legacy-task")
65+
expect(legacyMetadata.mode).toBe("architect")
66+
expect((legacyMetadata as any).abortSignal).toBeUndefined()
67+
})
68+
69+
it("should allow undefined abortSignal explicitly", () => {
70+
const metadata: ApiHandlerCreateMessageMetadata = {
71+
taskId: "test-task-id",
72+
abortSignal: undefined,
73+
}
74+
expect(metadata.abortSignal).toBeUndefined()
75+
})
76+
})
77+
78+
describe("signal type correctness", () => {
79+
it("should accept AbortController.signal as abortSignal value", () => {
80+
const controller = new AbortController()
81+
const signal: AbortSignal = controller.signal
82+
83+
const metadata: ApiHandlerCreateMessageMetadata = {
84+
taskId: "test-task-id",
85+
abortSignal: signal,
86+
}
87+
88+
expect(metadata.abortSignal).toBe(signal)
89+
expect(controller.signal.aborted).toBe(false)
90+
91+
controller.abort()
92+
expect(controller.signal.aborted).toBe(true)
93+
})
94+
95+
it("should accept AbortSignal.timeout() as abortSignal value", () => {
96+
const timeoutSignal = AbortSignal.timeout(5000)
97+
98+
const metadata: ApiHandlerCreateMessageMetadata = {
99+
taskId: "test-task-id",
100+
abortSignal: timeoutSignal,
101+
}
102+
103+
expect(metadata.abortSignal).toBe(timeoutSignal)
104+
})
105+
})
106+
})

‎src/api/index.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,12 @@ export interface ApiHandlerCreateMessageMetadata {
8989
* Only applies to providers that support function calling restrictions (e.g., Gemini).
9090
*/
9191
allowedFunctionNames?: string[]
92+
/**
93+
* Abort signal for cancelling the HTTP request mid-stream.
94+
* Passed through to AI SDK's streamText() so the underlying HTTP request is aborted
95+
* when the user clicks stop, preventing wasted API tokens/compute on the provider side.
96+
*/
97+
abortSignal?: AbortSignal
9298
}
9399

94100
export interface ApiHandler {
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
// Tests for OpenAICompatibleHandler's abortSignal passing to streamText()
2+
// Verifies that when createMessage() is called with metadata containing an abortSignal,
3+
// the signal is passed through to AI SDK's streamText() so HTTP requests can be aborted.
4+
5+
const { mockStreamText } = vi.hoisted(() => ({
6+
mockStreamText: vi.fn(),
7+
}))
8+
9+
vi.mock("ai", async (importOriginal) => {
10+
const actual = await importOriginal<typeof import("ai")>()
11+
return {
12+
...actual,
13+
streamText: mockStreamText,
14+
}
15+
})
16+
17+
vi.mock("@ai-sdk/openai-compatible", () => ({
18+
createOpenAICompatible: vi.fn(() => {
19+
return vi.fn(() => ({
20+
modelId: "test-model",
21+
provider: "test-provider",
22+
}))
23+
}),
24+
}))
25+
26+
import type { Anthropic } from "@anthropic-ai/sdk"
27+
28+
import { OpenAICompatibleHandler, OpenAICompatibleConfig } from "../openai-compatible"
29+
import type { ApiHandlerOptions } from "../../../shared/api"
30+
import type { ModelInfo } from "@roo-code/types"
31+
32+
// Concrete test subclass of the abstract OpenAICompatibleHandler
33+
class TestOpenAiCompatibleHandler extends OpenAICompatibleHandler {
34+
constructor(options: ApiHandlerOptions, config: OpenAICompatibleConfig) {
35+
super(options, config)
36+
}
37+
38+
override getModel(): { id: string; info: ModelInfo } {
39+
return { id: this.config.modelId, info: this.config.modelInfo }
40+
}
41+
}
42+
43+
describe("OpenAICompatibleHandler abort signal", () => {
44+
let handler: TestOpenAiCompatibleHandler
45+
const mockOptions: ApiHandlerOptions = {}
46+
const config: OpenAICompatibleConfig = {
47+
providerName: "test-provider",
48+
baseURL: "https://api.test.com/v1",
49+
apiKey: "test-key",
50+
modelId: "test-model",
51+
modelInfo: { maxTokens: 8192, contextWindow: 128000, supportsImages: false, supportsPromptCache: true },
52+
}
53+
54+
beforeEach(() => {
55+
vi.clearAllMocks()
56+
handler = new TestOpenAiCompatibleHandler(mockOptions, config)
57+
})
58+
59+
describe("createMessage abortSignal passing", () => {
60+
const systemPrompt = "You are a helpful assistant."
61+
const messages: Anthropic.Messages.MessageParam[] = [
62+
{ role: "user", content: [{ type: "text" as const, text: "Hello!" }] },
63+
]
64+
65+
it("should pass abortSignal to streamText when provided in metadata", async () => {
66+
const controller = new AbortController()
67+
const mockAbortSignal = controller.signal
68+
69+
async function* mockFullStream() {
70+
yield { type: "text-delta", text: "Test response" }
71+
}
72+
73+
function createMockStream(yieldValue) {
74+
return {
75+
fullStream: (async function* () { yield yieldValue; })(),
76+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
77+
}
78+
}
79+
80+
const mockUsage = Promise.resolve({ inputTokens: 10, outputTokens: 5 })
81+
82+
mockStreamText.mockReturnValue({
83+
fullStream: mockFullStream(),
84+
usage: mockUsage,
85+
})
86+
87+
await handler.createMessage(systemPrompt, messages, {
88+
taskId: "test-task",
89+
abortSignal: mockAbortSignal,
90+
}).next()
91+
92+
expect(mockStreamText).toHaveBeenCalledWith(
93+
expect.objectContaining({
94+
signal: mockAbortSignal,
95+
}),
96+
)
97+
})
98+
99+
it("should pass undefined signal to streamText when abortSignal is not provided", async () => {
100+
async function* mockFullStream() {
101+
yield { type: "text-delta", text: "Test response" }
102+
}
103+
104+
function createMockStream(yieldValue) {
105+
return {
106+
fullStream: (async function* () { yield yieldValue; })(),
107+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
108+
}
109+
}
110+
111+
const mockUsage = Promise.resolve({ inputTokens: 10, outputTokens: 5 })
112+
113+
mockStreamText.mockReturnValue({
114+
fullStream: mockFullStream(),
115+
usage: mockUsage,
116+
})
117+
118+
await handler.createMessage(systemPrompt, messages, {
119+
taskId: "test-task",
120+
}).next()
121+
122+
expect(mockStreamText).toHaveBeenCalledWith(
123+
expect.objectContaining({
124+
signal: undefined,
125+
}),
126+
)
127+
})
128+
129+
it("should pass signal to streamText when metadata is undefined", async () => {
130+
async function* mockFullStream() {
131+
yield { type: "text-delta", text: "Test response" }
132+
}
133+
134+
function createMockStream(yieldValue) {
135+
return {
136+
fullStream: (async function* () { yield yieldValue; })(),
137+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
138+
}
139+
}
140+
141+
const mockUsage = Promise.resolve({ inputTokens: 10, outputTokens: 5 })
142+
143+
mockStreamText.mockReturnValue({
144+
fullStream: mockFullStream(),
145+
usage: mockUsage,
146+
})
147+
148+
await handler.createMessage(systemPrompt, messages).next()
149+
150+
expect(mockStreamText).toHaveBeenCalledWith(
151+
expect.objectContaining({
152+
signal: undefined,
153+
}),
154+
)
155+
})
156+
157+
it("should pass the correct signal when it fires during streaming", async () => {
158+
const controller = new AbortController()
159+
const mockAbortSignal = controller.signal
160+
161+
let capturedOptions: any = null
162+
163+
mockStreamText.mockImplementation((options) => {
164+
capturedOptions = options
165+
return {
166+
fullStream: (async function* () {
167+
yield { type: "text-delta", text: "Partial" }
168+
})(),
169+
usage: Promise.resolve({ inputTokens: 5, outputTokens: 3 }),
170+
}
171+
})
172+
173+
const stream = handler.createMessage(systemPrompt, messages, {
174+
taskId: "test-task",
175+
abortSignal: mockAbortSignal,
176+
})
177+
178+
// Verify the signal was captured before aborting
179+
expect(capturedOptions).toBeDefined()
180+
expect(capturedOptions.signal).toBe(mockAbortSignal)
181+
182+
// Now abort - this should cause streamText to receive an aborted signal
183+
controller.abort()
184+
expect(controller.signal.aborted).toBe(true)
185+
})
186+
187+
it("should pass all other request options alongside the signal", async () => {
188+
const controller = new AbortController()
189+
190+
let capturedOptions: any = null
191+
192+
mockStreamText.mockImplementation((options) => {
193+
capturedOptions = options
194+
return {
195+
fullStream: (async function* () {
196+
yield { type: "text-delta", text: "Test" }
197+
})(),
198+
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
199+
}
200+
})
201+
202+
await handler.createMessage(systemPrompt, messages, {
203+
taskId: "test-task",
204+
abortSignal: controller.signal,
205+
}).next()
206+
207+
expect(capturedOptions).toHaveProperty("model")
208+
expect(capturedOptions).toHaveProperty("system", systemPrompt)
209+
expect(capturedOptions).toHaveProperty("messages")
210+
expect(capturedOptions).toHaveProperty("signal", controller.signal)
211+
})
212+
})
213+
})

‎src/api/providers/openai-compatible.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ export abstract class OpenAICompatibleHandler extends BaseProvider implements Si
174174
maxOutputTokens: this.getMaxOutputTokens(),
175175
tools: aiSdkTools,
176176
toolChoice: this.mapToolChoice(metadata?.tool_choice),
177+
abortSignal: metadata?.abortSignal,
177178
}
178179

179180
// Use streamText for streaming responses

‎src/core/task/Task.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4149,19 +4149,20 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
41494149
// Include tools whenever they are present.
41504150
...(shouldIncludeTools
41514151
? {
4152-
tools: allTools,
4153-
tool_choice: "auto",
4154-
parallelToolCalls: true,
4155-
// When mode restricts tools, provide allowedFunctionNames so providers
4156-
// like Gemini can see all tools in history but only call allowed ones
4157-
...(allowedFunctionNames ? { allowedFunctionNames } : {}),
4158-
}
4152+
tools: allTools,
4153+
tool_choice: "auto",
4154+
parallelToolCalls: true,
4155+
// When mode restricts tools, provide allowedFunctionNames so providers
4156+
// like Gemini can see all tools in history but only call allowed ones
4157+
...(allowedFunctionNames ? { allowedFunctionNames } : {}),
4158+
}
41594159
: {}),
41604160
}
41614161

41624162
// Create an AbortController to allow cancelling the request mid-stream
41634163
this.currentRequestAbortController = new AbortController()
41644164
const abortSignal = this.currentRequestAbortController.signal
4165+
metadata.abortSignal = abortSignal
41654166
// Reset the flag after using it
41664167
this.skipPrevResponseIdOnce = false
41674168

0 commit comments

Comments
 (0)