Skip to content

Commit e72d00f

Browse files
committed
fix(client): reconnect transient stalls instead of replacing the service (#52049)
1 parent d21b079 commit e72d00f

7 files changed

Lines changed: 75 additions & 9 deletions

File tree

‎packages/client/src/effect/service.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,9 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
7777
info,
7878
count: timeouts !== undefined && same(timeouts.info, info) ? timeouts.count + 1 : 1,
7979
}
80-
if (timeouts.count >= 3) {
80+
// Require sustained unresponsiveness before evicting: transient Windows probe stalls must
81+
// reconnect, never kill a healthy service and abort its sessions.
82+
if (timeouts.count >= 5) {
8183
yield* announce("missing")
8284
yield* Effect.logWarning("Background service is unresponsive; recovery cannot preserve persistent terminals")
8385
yield* Effect.tryPromise(() => PtyHandoff.clear(options.file ?? fallback()))
@@ -107,7 +109,10 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
107109

108110
const failed = pool.reap()
109111
if (failed !== undefined) return yield* Effect.fail(failed)
110-
if (pool.shouldRecruit(info !== undefined)) {
112+
// Keep one candidate plus one lock probe so a pre-lock stall cannot block recovery. While health
113+
// probes against the registered service keep timing out, wait for eviction instead: a contender
114+
// that registers first would take over and leak the hung process terminate() no longer matches.
115+
if (timeouts === undefined && pool.shouldRecruit(info !== undefined)) {
111116
yield* announce("missing")
112117
pool.add(yield* spawnContender)
113118
}

‎packages/client/src/promise/service.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
5757
info: registration.info,
5858
count: timeouts !== undefined && same(timeouts.info, registration.info) ? timeouts.count + 1 : 1,
5959
}
60-
if (timeouts.count >= 3) {
60+
// Require sustained unresponsiveness before evicting: transient Windows probe stalls must
61+
// reconnect, never kill a healthy service and abort its sessions.
62+
if (timeouts.count >= 5) {
6163
announce("missing")
6264
console.warn("Background service is unresponsive; recovery cannot preserve persistent terminals")
6365
await PtyHandoff.clear(options.file ?? fallback())
@@ -87,7 +89,10 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
8789
} else {
8890
const failed = pool.reap()
8991
if (failed !== undefined) throw failed
90-
if (pool.shouldRecruit(registration.info !== undefined)) {
92+
// Keep one candidate plus one lock probe so a pre-lock stall cannot block recovery. While health
93+
// probes against the registered service keep timing out, wait for eviction instead: a contender
94+
// that registers first would take over and leak the hung process terminate() no longer matches.
95+
if (timeouts === undefined && pool.shouldRecruit(registration.info !== undefined)) {
9196
announce("missing")
9297
pool.add(await spawnContender())
9398
}

‎packages/client/src/solid/connection.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ export type ClientConnectionOptions = {
2020
readonly pageLifecycle?: boolean
2121
/**
2222
* Abort and reconnect a stream that receives no bytes for this long. The server writes a keepalive
23-
* comment every 15 seconds, so a quiet but healthy stream never trips this.
23+
* comment every 15 seconds, so a quiet but healthy stream never trips this. The default allows six
24+
* missed keepalives so transient Windows stalls never look like a dead stream.
2425
*/
2526
readonly idleTimeout?: number
2627
readonly log?: {
@@ -32,7 +33,7 @@ export type ClientConnectionOptions = {
3233
const connectTimeout = 2_000
3334
const reconnectDelay = 1_000
3435
const connectionHistoryLimit = 50
35-
export const defaultIdleTimeout = 45_000
36+
export const defaultIdleTimeout = 90_000
3637
// Longer than one server keepalive interval: a stream that is silent this long when the page
3738
// returns to the foreground is probably half-open after the device slept.
3839
export const foregroundIdleThreshold = 20_000

‎packages/client/test/fixture/service.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ const server = Bun.serve({
7979
await appendFile(registration + ".requests", process.pid + "\n")
8080
return new Promise<Response>(() => {})
8181
}
82+
if (mode === "flaky") {
83+
await appendFile(registration + ".requests", process.pid + "\n")
84+
if (requests <= Number(delay ?? "3")) return new Promise<Response>(() => {})
85+
}
8286
if (mode === "modern" && requests === 1) {
8387
await writeFile(registration + ".first-request", "")
8488
while (!(await Bun.file(registration + ".release").exists())) await Bun.sleep(5)

‎packages/client/test/promise-service.test.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ test("evicts an unresponsive registered service before starting its replacement"
212212
const replacement = await Bun.file(registration).json()
213213
fixture.track(replacement.pid)
214214

215-
expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(3)
215+
expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(5)
216216
expect(await existing.exited).toBe(0)
217217
expect(replacement.pid).not.toBe(original.pid)
218218
expect(endpoint.url).toBe(replacement.url)
@@ -272,6 +272,27 @@ test("stops the registered service even when terminal handoff fails", async () =
272272
expect((await Bun.file(registration + ".pty-handoff").json()).handoff).toBeNull()
273273
})
274274

275+
test("keeps a registered service that recovers after transient probe stalls", async () => {
276+
await using fixture = await serviceFixture()
277+
const registration = fixture.registration
278+
const existing = fixture.spawn("flaky", "3")
279+
await fixture.waitForFile()
280+
const original = await Bun.file(registration).json()
281+
282+
const endpoint = await ensure({
283+
file: registration,
284+
version: "test",
285+
command: fixture.command("delayed", "10"),
286+
})
287+
const current = await Bun.file(registration).json()
288+
fixture.track(current.pid)
289+
290+
expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(4)
291+
expect(existing.exitCode).toBe(null)
292+
expect(current.pid).toBe(original.pid)
293+
expect(endpoint.url).toBe(original.url)
294+
})
295+
275296
test("signals the registered service process", async () => {
276297
await using fixture = await serviceFixture()
277298
const registration = fixture.registration

‎packages/client/test/service.test.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,13 +186,37 @@ test("evicts an unresponsive registered service before starting its replacement"
186186
const replacement = await Bun.file(registration).json()
187187
fixture.track(replacement.pid)
188188

189-
expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(3)
189+
expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(5)
190190
expect(await existing.exited).toBe(0)
191191
expect(replacement.pid).not.toBe(original.pid)
192192
expect(endpoint.url).toBe(replacement.url)
193193
expect(await status(endpoint.url)).toMatchObject({ version: "test", pid: replacement.pid })
194194
})
195195

196+
test("keeps a registered service that recovers after transient probe stalls", async () => {
197+
await using fixture = await serviceFixture()
198+
const registration = fixture.registration
199+
const existing = fixture.spawn("flaky", "3")
200+
await fixture.waitForFile()
201+
const original = await Bun.file(registration).json()
202+
203+
const endpoint = await run(
204+
ensure({
205+
file: registration,
206+
version: "test",
207+
command: fixture.command("delayed", "10"),
208+
}),
209+
)
210+
const current = await Bun.file(registration).json()
211+
fixture.track(current.pid)
212+
213+
expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(4)
214+
expect(existing.exitCode).toBe(null)
215+
expect(current.pid).toBe(original.pid)
216+
expect(endpoint.url).toBe(original.url)
217+
expect(await status(endpoint.url)).toMatchObject({ version: "test", pid: original.pid })
218+
})
219+
196220
test("signals an unresponsive registered service process", async () => {
197221
await using fixture = await serviceFixture()
198222
const registration = fixture.registration

‎packages/client/test/solid-connection.test.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { expect, test } from "bun:test"
22
import { createRoot } from "solid-js"
3-
import { createClientConnection } from "../src/solid"
3+
import { createClientConnection, defaultIdleTimeout } from "../src/solid"
44
import { OpenCode, type OpenCodeEvent } from "../src/promise"
55

66
const connected = { id: "evt_connected", created: 1, type: "server.connected", data: {} }
@@ -179,3 +179,9 @@ test("a reconnect does not ask the volatile event stream to replay", async () =>
179179
ctx.dispose()
180180
}
181181
})
182+
183+
test("the default idle timeout spans several server keepalives", () => {
184+
// The server writes a keepalive comment every 15 seconds. The default must cover several missed
185+
// keepalives so a transient Windows stall triggers a reconnect, never a service replacement.
186+
expect(defaultIdleTimeout).toBeGreaterThanOrEqual(6 * 15_000)
187+
})

0 commit comments

Comments
 (0)