Skip to content

Commit d8129ca

Browse files
authored
fix(ai): raise default chunk timeout to fifteen minutes (anomalyco#55032)
1 parent b4f19bf commit d8129ca

5 files changed

Lines changed: 23 additions & 9 deletions

File tree

‎packages/ai/src/route/transport/http.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import type { HttpMiddleware, Transport, TransportPrepareInput } from "./index.j
77
import * as ProviderShared from "../../protocols/shared.js"
88
import {
99
AIError,
10+
DEFAULT_CHUNK_TIMEOUT_MS,
1011
DEFAULT_HTTP_TIMEOUT_MS,
1112
mergeJsonRecords,
1213
TransportError,
@@ -124,7 +125,7 @@ export const httpJson = <Body, Frame>(input: HttpJsonInput<Body, Frame>): HttpJs
124125
frames: prepared.framing.frame(
125126
RequestExecutor.responseStream(response).pipe(
126127
Stream.timeoutOrElse({
127-
duration: timeoutDuration(request.http?.chunkTimeout),
128+
duration: timeoutDuration(request.http?.chunkTimeout, DEFAULT_CHUNK_TIMEOUT_MS),
128129
orElse: () => Stream.fail(timeout("read", "Timed out waiting for response data", http)),
129130
}),
130131
Stream.interruptWhen(
@@ -140,8 +141,8 @@ export const httpJson = <Body, Frame>(input: HttpJsonInput<Body, Frame>): HttpJs
140141
}),
141142
})
142143

143-
const timeoutDuration = (value: HttpTimeout | undefined) =>
144-
value === false ? Duration.infinity : Duration.millis(value ?? DEFAULT_HTTP_TIMEOUT_MS)
144+
const timeoutDuration = (value: HttpTimeout | undefined, fallback = DEFAULT_HTTP_TIMEOUT_MS) =>
145+
value === false ? Duration.infinity : Duration.millis(value ?? fallback)
145146

146147
export const sseJson = {
147148
id: "http-json/sse",

‎packages/ai/src/schema/options.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,15 @@ export const mergeProviderOptions = (
5252
export const HttpTimeout = Schema.Union([Schema.Number.check(Schema.isGreaterThan(0)), Schema.Literal(false)])
5353
export type HttpTimeout = Schema.Schema.Type<typeof HttpTimeout>
5454

55-
/** Applied to `headerTimeout` and `chunkTimeout` when a request leaves them unset. */
55+
/** Applied to `headerTimeout` when a request leaves it unset. */
5656
export const DEFAULT_HTTP_TIMEOUT_MS = 300_000
5757

58+
/**
59+
* Applied to `chunkTimeout` when a request leaves it unset. Some providers, such as DeepSeek, have specifically asked
60+
* clients to allow at least 15 minutes, because their models can stream nothing for minutes while they reason.
61+
*/
62+
export const DEFAULT_CHUNK_TIMEOUT_MS = 900_000
63+
5864
/** `Content-Encoding` for JSON request bodies; `none` sends plain JSON. */
5965
export const RequestEncoding = Schema.Literals(["zstd", "gzip", "none"])
6066
export type RequestEncoding = Schema.Schema.Type<typeof RequestEncoding>

‎packages/ai/test/http-timeout.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ describe("HTTP transport timeouts", () => {
6969
}),
7070
)
7171

72-
it.effect("fails when the response body stalls for five minutes", () =>
72+
it.effect("fails when the response body stalls for fifteen minutes", () =>
7373
Effect.gen(function* () {
7474
const server = yield* stalledServer
7575
const fiber = yield* LLMClient.generate(LLM.request({ model, prompt: "Hello" })).pipe(
@@ -79,7 +79,7 @@ describe("HTTP transport timeouts", () => {
7979
)
8080
yield* Deferred.await(server.stalled)
8181
yield* Effect.yieldNow
82-
yield* TestClock.adjust("5 minutes")
82+
yield* TestClock.adjust("15 minutes")
8383
const error = yield* Fiber.join(fiber)
8484

8585
expect(error.reason).toMatchObject({ _tag: "Transport", transport: "http", operation: "read", code: "Timeout" })

‎packages/core/src/provider.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,9 +166,16 @@ export function modelSettings(settings: Settings | undefined) {
166166

167167
const decodeTimeout = Schema.decodeUnknownOption(HttpTimeout)
168168

169-
/** Applied to `headerTimeout` and `chunkTimeout` when provider settings leave them unset or invalid. */
169+
/** Applied to `headerTimeout` when provider settings leave it unset or invalid. */
170170
export const DEFAULT_TIMEOUT_MS = 300_000
171171

172+
/**
173+
* Applied to `chunkTimeout` when provider settings leave it unset or invalid. Some providers, such as DeepSeek, have
174+
* specifically asked clients to allow at least 15 minutes, because their models can stream nothing for minutes while
175+
* they reason.
176+
*/
177+
export const DEFAULT_CHUNK_TIMEOUT_MS = 900_000
178+
172179
/** One timeout setting without a default; invalid values are dropped. */
173180
export function timeout(value: unknown) {
174181
return Option.getOrUndefined(decodeTimeout(value))
@@ -179,7 +186,7 @@ export function timeouts(settings: Readonly<Record<string, unknown>>) {
179186
return {
180187
timeout: timeout(settings.timeout),
181188
headerTimeout: timeout(settings.headerTimeout) ?? DEFAULT_TIMEOUT_MS,
182-
chunkTimeout: timeout(settings.chunkTimeout) ?? DEFAULT_TIMEOUT_MS,
189+
chunkTimeout: timeout(settings.chunkTimeout) ?? DEFAULT_CHUNK_TIMEOUT_MS,
183190
}
184191
}
185192

‎packages/web/src/content/docs/config.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ Provider options can include `timeout`, `headerTimeout`, `chunkTimeout`, and `se
393393

394394
- `timeout` - Request timeout in milliseconds (default: 300000). Set to `false` to disable.
395395
- `headerTimeout` - Timeout in milliseconds to wait for response headers (default: 300000, or 5 minutes). This timer stops once headers arrive and does not limit the streamed response body. Set to `false` to disable.
396-
- `chunkTimeout` - Timeout in milliseconds between streamed response chunks (default: 300000, or 5 minutes). If no chunk arrives in time, the request is aborted. Set to `false` to disable.
396+
- `chunkTimeout` - Timeout in milliseconds between streamed response chunks (default: 900000, or 15 minutes). If no chunk arrives in time, the request is aborted. Set to `false` to disable.
397397
- `setCacheKey` - Ensure a cache key is always set for designated provider.
398398

399399
You can also configure [local models](/docs/models#local). [Learn more](/docs/models).

0 commit comments

Comments
 (0)