Skip to content

Commit 05cdc54

Browse files
committed
fix(client): ensure never stops a service it did not start for answering probes slowly
Service.ensure terminated the registered service after three 2 s probe timeouts (upstream #40896, still on dev). On a loaded host a booting or busy service misses them, so every opencode api call could kill the live service; the workbox restarted about once a minute. Now a slow registration this call did not start is reported (onWait) and waited for, bounded by the ensure deadline, then reported as left running; no contender races it. Only an attempt this call spawned is still evicted when it stops answering.
1 parent f0c6c66 commit 05cdc54

4 files changed

Lines changed: 45 additions & 4 deletions

File tree

‎packages/client/src/effect/service.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { contenderPool, spawnServiceContender } from "../service-contender.js"
44
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
55
import { matchesVersion } from "../service-version.js"
66
import { PtyHandoff } from "../pty-handoff.js"
7-
import { decide, fallback, headers, probeResult, same } from "../service-probe.js"
7+
import { decide, fallback, headers, probeResult, same, unanswered } from "../service-probe.js"
88

99
export * from "../service.js"
1010
export { headers }
@@ -50,6 +50,7 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
5050
const timing = ensureTiming(options)
5151
const pool = contenderPool(timing)
5252
let timeouts: { readonly info: Info; readonly count: number } | undefined
53+
let unresponsive: Info | undefined
5354
let announced = false
5455
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) =>
5556
Effect.sync(() => {
@@ -79,7 +80,15 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
7980
const registration = yield* registered(options.file, timing.requestTimeout)
8081
const info = registration.info
8182
const service = registration.service
83+
unresponsive = undefined
8284
if (registration.timedOut && info !== undefined) {
85+
// A slow service this call did not start is busy or booting as often as hung: it is reported
86+
// and waited for (bounded by promiseTimeout), never stopped or raced by a contender.
87+
if (!pool.owns(info.pid)) {
88+
unresponsive = info
89+
yield* waiting(info.url)
90+
return Option.none()
91+
}
8392
timeouts = {
8493
info,
8594
count: timeouts !== undefined && same(timeouts.info, info) ? timeouts.count + 1 : 1,
@@ -131,7 +140,11 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
131140
Effect.ensuring(Effect.sync(() => pool.releaseAll())),
132141
)
133142
if (Option.isNone(found))
134-
return yield* Effect.fail(pool.failure() ?? new Error("Timed out waiting for the background service to start"))
143+
return yield* Effect.fail(
144+
unresponsive !== undefined
145+
? unanswered(unresponsive, timing.promiseTimeout)
146+
: (pool.failure() ?? new Error("Timed out waiting for the background service to start")),
147+
)
135148
return found.value.endpoint
136149
})
137150

‎packages/client/src/promise/service.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { contenderPool, spawnServiceContender } from "../service-contender.js"
44
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
55
import { matchesVersion } from "../service-version.js"
66
import { PtyHandoff } from "../pty-handoff.js"
7-
import { decide, fallback, headers, probeResult, same } from "../service-probe.js"
7+
import { decide, fallback, headers, probeResult, same, unanswered } from "../service-probe.js"
88

99
export * from "../service.js"
1010
export { fallback, headers }
@@ -30,6 +30,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
3030
const deadline = Date.now() + timing.promiseTimeout
3131
const pool = contenderPool(timing)
3232
let timeouts: { readonly info: Info; readonly count: number } | undefined
33+
let unresponsive: Info | undefined
3334
let announced = false
3435

3536
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
@@ -56,8 +57,19 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
5657
try {
5758
while (true) {
5859
if (Date.now() >= deadline)
59-
throw pool.failure() ?? new Error("Timed out waiting for the background service to start")
60+
throw unresponsive !== undefined
61+
? unanswered(unresponsive, timing.promiseTimeout)
62+
: (pool.failure() ?? new Error("Timed out waiting for the background service to start"))
6063
const registration = await registered(options.file, timing.requestTimeout)
64+
unresponsive = undefined
65+
// A slow service this call did not start is busy or booting as often as hung: it is reported
66+
// and waited for (bounded by promiseTimeout), never stopped or raced by a contender.
67+
if (registration.timedOut && registration.info !== undefined && !pool.owns(registration.info.pid)) {
68+
unresponsive = registration.info
69+
waiting(registration.info.url)
70+
await delay(timing.pollInterval)
71+
continue
72+
}
6173
if (registration.timedOut && registration.info !== undefined) {
6274
timeouts = {
6375
info: registration.info,

‎packages/client/src/service-contender.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ export function contenderPool(timing: EnsureTiming) {
107107
// survivors finish without recruiting replacements that could hide the error indefinitely.
108108
return failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay
109109
},
110+
/** Whether `pid` is a live attempt this call started: the only service it may stop for not answering. */
111+
owns(pid: number) {
112+
return [...contenders].some((item) => item.child.pid === pid && !contenderFinished(item))
113+
},
110114
add(contender: ServiceContender) {
111115
contenders.add(contender)
112116
lastSpawn = Date.now()

‎packages/client/src/service-probe.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,18 @@ export function decide(service: LocalService, options: DiscoverOptions) {
3131
return { _tag: "wait" } as const
3232
}
3333

34+
/**
35+
* Why `ensure()` gave up on a registered service it did not start that kept not answering its
36+
* probes. Such a service is busy or still starting as often as it is hung, so ensure never stops
37+
* it; it waits for it, bounded, and leaves the decision to restart it to the user.
38+
*/
39+
export function unanswered(info: Info, waitedMs: number) {
40+
return new Error(
41+
`Background service at ${info.url} (pid ${info.pid}) did not answer within ${Math.round(waitedMs / 1000)}s. ` +
42+
"It was left running; if it is stuck, restart it with `opencode service restart`.",
43+
)
44+
}
45+
3446
function fail(message: string) {
3547
return { _tag: "fail" as const, error: new Error(message) }
3648
}

0 commit comments

Comments
 (0)