Skip to content

Commit 4497268

Browse files
committed
feat(api): abort-signal wiring for lm-studio and qwen-code (round 2)
1 parent e61feb1 commit 4497268

5 files changed

Lines changed: 901 additions & 95 deletions

File tree

‎src/api/providers/__tests__/lm-studio-timeout.spec.ts‎

Lines changed: 269 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,33 @@ vitest.mock("../utils/timeout-config", () => ({
1111
import { getApiRequestTimeout } from "../utils/timeout-config"
1212

1313
import { clearAllMocks } from "../../../test-utils/reset"
14+
import { asyncStreamFrom } from "../../../test-utils/stream"
1415

15-
// Mock OpenAI
16+
interface MockOpenAiClient {
17+
chat: {
18+
completions: {
19+
create: ReturnType<typeof vitest.fn>
20+
}
21+
}
22+
}
23+
24+
// Mock OpenAI (records each created client so tests can drive its create call)
1625
const mockOpenAIConstructor = vitest.fn()
26+
const createdClients: MockOpenAiClient[] = []
1727
vitest.mock("openai", () => {
1828
return {
1929
__esModule: true,
2030
default: vitest.fn().mockImplementation(function (config) {
21-
mockOpenAIConstructor(config)
22-
return {
31+
const client: MockOpenAiClient = {
2332
chat: {
2433
completions: {
2534
create: vitest.fn(),
2635
},
2736
},
2837
}
38+
createdClients.push(client)
39+
mockOpenAIConstructor(config)
40+
return client
2941
}),
3042
}
3143
})
@@ -36,7 +48,7 @@ describe("LmStudioHandler timeout configuration", () => {
3648
})
3749

3850
it("should use default timeout of 600 seconds when no configuration is set", () => {
39-
;(getApiRequestTimeout as any).mockReturnValue(600000)
51+
vitest.mocked(getApiRequestTimeout).mockReturnValue(600000)
4052

4153
const options: ApiHandlerOptions = {
4254
apiModelId: "llama2",
@@ -57,7 +69,7 @@ describe("LmStudioHandler timeout configuration", () => {
5769
})
5870

5971
it("should use custom timeout when configuration is set", () => {
60-
;(getApiRequestTimeout as any).mockReturnValue(1200000) // 20 minutes
72+
vitest.mocked(getApiRequestTimeout).mockReturnValue(1200000) // 20 minutes
6173

6274
const options: ApiHandlerOptions = {
6375
apiModelId: "llama2",
@@ -75,7 +87,7 @@ describe("LmStudioHandler timeout configuration", () => {
7587
})
7688

7789
it("should handle zero timeout (no timeout)", () => {
78-
;(getApiRequestTimeout as any).mockReturnValue(0)
90+
vitest.mocked(getApiRequestTimeout).mockReturnValue(0)
7991

8092
const options: ApiHandlerOptions = {
8193
apiModelId: "llama2",
@@ -91,3 +103,254 @@ describe("LmStudioHandler timeout configuration", () => {
91103
)
92104
})
93105
})
106+
107+
describe("LmStudioHandler abort signal wiring", () => {
108+
let options: ApiHandlerOptions
109+
110+
// Mirror the OpenAI SDK's APIUserAbortError shape: name "Error", message
111+
// "Request was aborted." It does not satisfy the Task.ts abort contract
112+
// (message must end in "aborted"), so the provider must normalize it.
113+
const sdkAbortError = (): Error => {
114+
const err = new Error("Request was aborted.")
115+
err.name = "Error"
116+
return err
117+
}
118+
119+
const waitForCreateCall = async (create: { mock: { calls: unknown[][] } }, timeoutMs = 5000): Promise<void> => {
120+
const start = Date.now()
121+
while (create.mock.calls.length === 0) {
122+
if (Date.now() - start > timeoutMs) {
123+
throw new Error("timed out waiting for the SDK create call")
124+
}
125+
await new Promise((resolve) => setTimeout(resolve, 5))
126+
}
127+
}
128+
129+
const waitForSignalAbort = (signal: AbortSignal | undefined): Promise<void> => {
130+
return new Promise((resolve, reject) => {
131+
if (!signal) {
132+
reject(new Error("SDK create was called without a signal"))
133+
return
134+
}
135+
if (signal.aborted) {
136+
resolve()
137+
return
138+
}
139+
signal.addEventListener("abort", () => resolve(), { once: true })
140+
})
141+
}
142+
143+
const lastCreate = (): MockOpenAiClient["chat"]["completions"]["create"] => {
144+
const client = createdClients[createdClients.length - 1]
145+
if (!client) {
146+
throw new Error("no OpenAI client was created")
147+
}
148+
return client.chat.completions.create
149+
}
150+
151+
beforeEach(() => {
152+
clearAllMocks()
153+
vitest.mocked(getApiRequestTimeout).mockReturnValue(600000)
154+
options = {
155+
apiModelId: "llama2",
156+
lmStudioModelId: "llama2",
157+
lmStudioBaseUrl: "http://localhost:1234",
158+
}
159+
})
160+
161+
describe("createMessage", () => {
162+
it("should pass a request-local AbortSignal to the SDK and bridge the external signal", async () => {
163+
const handler = new LmStudioHandler(options)
164+
vitest.spyOn(handler, "countTokens").mockResolvedValue(1)
165+
const create = lastCreate()
166+
create.mockResolvedValue(asyncStreamFrom([]))
167+
168+
const external = new AbortController()
169+
const stream = handler.createMessage("system", [], { taskId: "t1", abortSignal: external.signal })
170+
await stream.next()
171+
172+
const opts = create.mock.calls[0][1]
173+
expect(opts?.signal).toBeInstanceOf(AbortSignal)
174+
expect(opts.signal).not.toBe(external.signal) // request-local, not the external signal
175+
expect(opts.signal.aborted).toBe(false)
176+
177+
external.abort()
178+
expect(opts.signal.aborted).toBe(true) // the external abort is bridged to the SDK signal
179+
180+
await stream.next() // drain the generator
181+
})
182+
183+
it("should fast-fail with a normalized AbortError when the signal is pre-aborted", async () => {
184+
const handler = new LmStudioHandler(options)
185+
const create = lastCreate()
186+
const external = new AbortController()
187+
external.abort()
188+
189+
const stream = handler.createMessage("system", [], { taskId: "t1", abortSignal: external.signal })
190+
let caught: unknown
191+
try {
192+
await stream.next()
193+
} catch (error) {
194+
caught = error
195+
}
196+
197+
expect(caught).toBeInstanceOf(Error)
198+
expect((caught as Error).name).toBe("AbortError")
199+
expect((caught as Error).message).toMatch(/aborted$/)
200+
expect(create).not.toHaveBeenCalled()
201+
})
202+
203+
it("should abort the in-flight SDK request when the external signal fires", async () => {
204+
const handler = new LmStudioHandler(options)
205+
vitest.spyOn(handler, "countTokens").mockResolvedValue(1)
206+
const create = lastCreate()
207+
// Simulate the OpenAI SDK: reject with its abort error when the signal aborts.
208+
create.mockImplementation((_params: unknown, opts?: { signal?: AbortSignal }) => {
209+
return new Promise((_resolve, reject) => {
210+
if (!opts?.signal) {
211+
reject(new Error("SDK create was called without a signal"))
212+
return
213+
}
214+
opts.signal.addEventListener("abort", () => reject(sdkAbortError()), { once: true })
215+
})
216+
})
217+
218+
const external = new AbortController()
219+
const stream = handler.createMessage("system", [], { taskId: "t1", abortSignal: external.signal })
220+
const pending = stream.next()
221+
await waitForCreateCall(create)
222+
external.abort()
223+
224+
let caught: unknown
225+
try {
226+
await pending
227+
} catch (error) {
228+
caught = error
229+
}
230+
231+
expect(caught).toBeInstanceOf(Error)
232+
expect((caught as Error).name).toBe("AbortError")
233+
expect((caught as Error).message).toMatch(/aborted$/)
234+
})
235+
236+
it("should normalize an abort error thrown mid-stream", async () => {
237+
const handler = new LmStudioHandler(options)
238+
vitest.spyOn(handler, "countTokens").mockResolvedValue(1)
239+
const create = lastCreate()
240+
const external = new AbortController()
241+
// Simulate the OpenAI SDK stream: yield once, then reject with its
242+
// abort error once the request-local signal is aborted.
243+
create.mockImplementation((_params: unknown, opts?: { signal?: AbortSignal }) => {
244+
return (async function* () {
245+
yield { choices: [{ delta: { content: "partial" } }] }
246+
await waitForSignalAbort(opts?.signal)
247+
throw sdkAbortError()
248+
})()
249+
})
250+
251+
const stream = handler.createMessage("system", [], { taskId: "t1", abortSignal: external.signal })
252+
const chunks: { type: string; text?: string }[] = []
253+
let caught: unknown
254+
try {
255+
for await (const chunk of stream) {
256+
chunks.push(chunk)
257+
if (chunk.type === "text") {
258+
external.abort()
259+
}
260+
}
261+
} catch (error) {
262+
caught = error
263+
}
264+
265+
expect(chunks).toContainEqual({ type: "text", text: "partial" })
266+
expect(caught).toBeInstanceOf(Error)
267+
expect((caught as Error).name).toBe("AbortError")
268+
expect((caught as Error).message).toMatch(/aborted$/)
269+
})
270+
})
271+
272+
describe("completePrompt", () => {
273+
it("should pass the external signal through, and nothing without a signal or with a zero timeout", async () => {
274+
const handler = new LmStudioHandler(options)
275+
const create = lastCreate()
276+
create.mockResolvedValue({ choices: [{ message: { content: "ok" } }] })
277+
const external = new AbortController()
278+
279+
expect(await handler.completePrompt("hi")).toBe("ok")
280+
expect(create.mock.calls[0][1]).toBeUndefined() // no signal, no timeout: nothing reaches the SDK
281+
282+
expect(await handler.completePrompt("hi", { abortSignal: external.signal })).toBe("ok")
283+
// no timeout: the merged signal is the external signal itself
284+
expect(create.mock.calls[1][1]?.signal).toBe(external.signal)
285+
286+
// timeoutMs <= 0 means "no explicit timeout": nothing may reach the SDK
287+
expect(await handler.completePrompt("hi", { timeoutMs: 0 })).toBe("ok")
288+
expect(create.mock.calls[2][1]).toBeUndefined()
289+
})
290+
291+
it("should merge the external signal with a positive timeoutMs", async () => {
292+
const handler = new LmStudioHandler(options)
293+
const create = lastCreate()
294+
create.mockImplementation((_params: unknown, opts?: { signal?: AbortSignal }) => {
295+
return new Promise((_resolve, reject) => {
296+
const signal = opts?.signal
297+
if (!signal) {
298+
reject(new Error("SDK create was called without a signal"))
299+
return
300+
}
301+
signal.addEventListener("abort", () => reject(sdkAbortError()), { once: true })
302+
})
303+
})
304+
const external = new AbortController()
305+
306+
const pending = handler.completePrompt("hi", { abortSignal: external.signal, timeoutMs: 60_000 })
307+
const opts = create.mock.calls[0][1]
308+
expect(opts?.signal).toBeInstanceOf(AbortSignal)
309+
expect(opts.signal).not.toBe(external.signal) // merged via AbortSignal.any
310+
expect(opts.signal.aborted).toBe(false)
311+
312+
external.abort()
313+
let caught: unknown
314+
try {
315+
await pending
316+
} catch (error) {
317+
caught = error
318+
}
319+
320+
expect((caught as Error).name).toBe("AbortError")
321+
expect((caught as Error).message).toMatch(/aborted$/)
322+
})
323+
324+
it("should normalize SDK abort errors instead of wrapping them", async () => {
325+
const handler = new LmStudioHandler(options)
326+
const create = lastCreate()
327+
create.mockRejectedValue(sdkAbortError())
328+
329+
let caught: unknown
330+
try {
331+
await handler.completePrompt("hi")
332+
} catch (error) {
333+
caught = error
334+
}
335+
336+
expect((caught as Error).name).toBe("AbortError")
337+
expect((caught as Error).message).toMatch(/aborted$/)
338+
})
339+
340+
it("should keep wrapping non-abort errors in the LM Studio debug message", async () => {
341+
const handler = new LmStudioHandler(options)
342+
const create = lastCreate()
343+
create.mockRejectedValue(new Error("boom"))
344+
345+
let caught: unknown
346+
try {
347+
await handler.completePrompt("hi")
348+
} catch (error) {
349+
caught = error
350+
}
351+
352+
expect(caught).toBeInstanceOf(Error)
353+
expect((caught as Error).message).toContain("Please check the LM Studio developer logs")
354+
})
355+
})
356+
})

0 commit comments

Comments
 (0)