Skip to content

Commit c0fd5c4

Browse files
committed
fix(core): count websocket stream failures
1 parent ad15173 commit c0fd5c4

2 files changed

Lines changed: 123 additions & 11 deletions

File tree

‎packages/core/src/session/model-transport.ts‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ export const makeLayer = (connector: WebSocketConnector) =>
166166
owner: State,
167167
channel: Channel,
168168
error: AIError,
169+
countFailure: boolean,
169170
) {
170171
if (owner.channel === channel) owner.channel = undefined
171172
if (channel.closing) return
@@ -175,16 +176,14 @@ export const makeLayer = (connector: WebSocketConnector) =>
175176
code: error.reason._tag === "Transport" ? error.reason.code : error.reason._tag,
176177
active: channel.active !== undefined,
177178
})
178-
if (channel.active) {
179-
Queue.failCauseUnsafe(channel.active.queue, Cause.fail(error))
180-
yield* streamFailure(owner)
181-
}
179+
if (channel.active) Queue.failCauseUnsafe(channel.active.queue, Cause.fail(error))
180+
if (countFailure) yield* streamFailure(owner)
182181
yield* metric("protocol_failure")
183182
yield* channel.connection.close
184183
})
185184

186-
// A socket that keeps dying mid-exchange costs a retry every step; after enough consecutive
187-
// losses the Session stays on HTTP.
185+
// A WebSocket exchange that keeps failing before a terminal event costs a retry every step;
186+
// after enough consecutive losses the Session stays on HTTP.
188187
const streamFailure = Effect.fn("SessionModelTransport.streamFailure")(function* (owner: State) {
189188
owner.streamFailures++
190189
if (owner.streamFailures < MAX_STREAM_FAILURES) return
@@ -273,6 +272,7 @@ export const makeLayer = (connector: WebSocketConnector) =>
273272
? "rejected"
274273
: "ambiguous",
275274
}),
275+
channel.active !== undefined,
276276
),
277277
),
278278
Effect.forkIn(scope, { startImmediately: true }),
@@ -441,12 +441,12 @@ export const makeLayer = (connector: WebSocketConnector) =>
441441
),
442442
Stream.takeUntil(observationTerminal),
443443
Stream.mapEffect(observationFrame),
444-
Stream.ensuring(
444+
Stream.onExit((exit) =>
445445
Effect.gen(function* () {
446-
if (channel.active === active) channel.active = undefined
447446
const pending = yield* Queue.size(active.queue)
448447
yield* Queue.shutdown(active.queue)
449448
if (terminal && pending === 0) {
449+
if (channel.active === active) channel.active = undefined
450450
owner.streamFailures = 0
451451
yield* metric("terminal", { type: terminal.type })
452452
if (terminal.type === "rejected") yield* metric("rejection", { recovery: terminal.recovery })
@@ -456,7 +456,9 @@ export const makeLayer = (connector: WebSocketConnector) =>
456456
if (terminal.type !== "completed" && terminal.type !== "incomplete") yield* closeChannel(owner, channel)
457457
return
458458
}
459-
yield* metric("cancellation")
459+
const failed = exit._tag === "Failure" && !Cause.hasInterruptsOnly(exit.cause)
460+
if (channel.active === active) channel.active = undefined
461+
if (!failed) yield* metric("cancellation")
460462
channel.checkpoint = undefined
461463
channel.pending = undefined
462464
const error = terminal
@@ -474,7 +476,7 @@ export const makeLayer = (connector: WebSocketConnector) =>
474476
phase: "receive",
475477
delivery: active.delivery === "provider-observed" ? "accepted" : "ambiguous",
476478
})
477-
yield* poison(owner, channel, error)
479+
yield* poison(owner, channel, error, failed)
478480
}),
479481
),
480482
Stream.catch((error) => {

‎packages/core/test/session-model-transport.test.ts‎

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, test } from "bun:test"
2-
import { AIError, HttpContext, InvalidRequestError, TransportError } from "@opencode/ai"
2+
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError, TransportError } from "@opencode/ai"
33
import type {
44
ChannelObservation,
55
WebSocketChannelExchange,
@@ -634,6 +634,50 @@ describe("SessionModelTransport", () => {
634634
)
635635
})
636636

637+
test("keeps the Session on HTTP after repeated idle timeouts", async () => {
638+
const sent = queue<void>()
639+
let opens = 0
640+
const connector: WebSocketConnector = {
641+
open: () =>
642+
Effect.gen(function* () {
643+
opens++
644+
const messages = yield* Queue.unbounded<string | Uint8Array, AIError>()
645+
return {
646+
sendText: () =>
647+
Queue.offer(sent, undefined).pipe(
648+
Effect.andThen(opens > 5 ? Queue.offer(messages, "completed") : Effect.void),
649+
Effect.asVoid,
650+
),
651+
messages: Stream.fromQueue(messages),
652+
close: Queue.shutdown(messages).pipe(Effect.asVoid),
653+
}
654+
}),
655+
}
656+
657+
await runWithTestClock(
658+
connector,
659+
Effect.gen(function* () {
660+
const transport = yield* SessionModelTransport.Service
661+
const executor = transport.bind(session, undefined, 1_000)
662+
for (let attempt = 0; attempt < 5; attempt++) {
663+
const running = yield* collect(executor, exchange(`attempt-${attempt}`)).pipe(
664+
Effect.result,
665+
Effect.forkChild({ startImmediately: true }),
666+
)
667+
yield* Queue.take(sent)
668+
yield* TestClock.adjust("1 second")
669+
expect(yield* Fiber.join(running)).toMatchObject({
670+
_tag: "Failure",
671+
failure: { reason: { _tag: "Transport", code: "idle-timeout" } },
672+
})
673+
}
674+
675+
expect(yield* collect(executor, exchange("sixth"))).toEqual(["fallback:sixth"])
676+
expect(opens).toBe(5)
677+
}),
678+
)
679+
})
680+
637681
test("closes a newly opened connection when request creation is interrupted", async () => {
638682
const opened = Deferred.makeUnsafe<void>()
639683
const messages = queue<string | Uint8Array, AIError>()
@@ -765,6 +809,72 @@ describe("SessionModelTransport", () => {
765809
)
766810
})
767811

812+
test("keeps the Session on HTTP after repeated response parser failures", async () => {
813+
const fixture = automatic()
814+
const item = (id: string): WebSocketChannelExchange => ({
815+
...exchange(id),
816+
driver: {
817+
create: () => Effect.succeed({ message: id, mode: "full" }),
818+
observe: () =>
819+
Effect.fail(new AIError({ reason: new InvalidProviderOutputError({ message: "Invalid response frame" }) })),
820+
},
821+
})
822+
823+
await run(
824+
fixture.connector,
825+
Effect.gen(function* () {
826+
const transport = yield* SessionModelTransport.Service
827+
const executor = transport.bind(session)
828+
for (let attempt = 0; attempt < 5; attempt++) {
829+
const result = yield* Effect.result(collect(executor, item(`attempt-${attempt}`)))
830+
expect(result._tag).toBe("Failure")
831+
}
832+
expect(fixture.connections).toHaveLength(5)
833+
expect(yield* collect(executor, exchange("sixth"))).toEqual(["fallback:sixth"])
834+
expect(fixture.connections).toHaveLength(5)
835+
}),
836+
)
837+
})
838+
839+
test("does not count interrupted exchanges as stream failures", async () => {
840+
const sent = queue<string>()
841+
let opens = 0
842+
const connector: WebSocketConnector = {
843+
open: () =>
844+
Effect.gen(function* () {
845+
opens++
846+
const messages = yield* Queue.unbounded<string | Uint8Array, AIError>()
847+
return {
848+
sendText: (message) =>
849+
Queue.offer(sent, message).pipe(
850+
Effect.andThen(message === "sixth" ? Queue.offer(messages, `completed:${message}`) : Effect.void),
851+
Effect.asVoid,
852+
),
853+
messages: Stream.fromQueue(messages),
854+
close: Queue.shutdown(messages).pipe(Effect.asVoid),
855+
}
856+
}),
857+
}
858+
859+
await run(
860+
connector,
861+
Effect.gen(function* () {
862+
const transport = yield* SessionModelTransport.Service
863+
const executor = transport.bind(session)
864+
for (let attempt = 0; attempt < 5; attempt++) {
865+
const running = yield* collect(executor, exchange(`attempt-${attempt}`)).pipe(
866+
Effect.forkChild({ startImmediately: true }),
867+
)
868+
expect(yield* Queue.take(sent)).toBe(`attempt-${attempt}`)
869+
yield* Fiber.interrupt(running)
870+
}
871+
872+
expect(yield* collect(executor, exchange("sixth"))).toEqual(["completed:sixth"])
873+
expect(opens).toBe(6)
874+
}),
875+
)
876+
})
877+
768878
test("falls back to HTTP after close code 1009 and keeps the Session on HTTP", async () => {
769879
const messages = queue<string | Uint8Array, AIError>()
770880
let opened = 0

0 commit comments

Comments
 (0)