Skip to content

Commit 80e68c6

Browse files
tomopasCopilot
andcommitted
fix(cli): retry managed service port during shutdown
Closes anomalyco#54267 Co-authored-by: Copilot <[email protected]>
1 parent c458ad7 commit 80e68c6

7 files changed

Lines changed: 114 additions & 14 deletions

File tree

‎packages/cli/src/server-process.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,12 +151,14 @@ const processEffect = Effect.fnUntraced(function* (options: Options) {
151151
)
152152
const server = yield* launch.pipe(
153153
Effect.catch((error) => {
154-
if (findIncumbent === undefined || !addressInUse(error)) return Effect.fail(error)
154+
if (!addressInUse(error)) return Effect.fail(error)
155155
return Effect.gen(function* () {
156156
const deadline = Date.now() + 15_000
157157
while (Date.now() < deadline) {
158-
const found = yield* findIncumbent.pipe(Effect.timeoutOption(deadline - Date.now()))
159-
if (Option.isSome(found) && found.value !== undefined) return
158+
if (findIncumbent !== undefined) {
159+
const found = yield* findIncumbent.pipe(Effect.timeoutOption(deadline - Date.now()))
160+
if (Option.isSome(found) && found.value !== undefined) return
161+
}
160162
yield* Effect.sleep("100 millis")
161163
if (Date.now() >= deadline) break
162164
// Failed binds close their scope; a successful bind may take longer than this window to boot.

‎packages/cli/src/services/service-registration.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ export const register = Effect.fnUntraced(function* (options: {
2020
}) {
2121
const fs = yield* FileSystem.FileSystem
2222
const temp = options.file + "." + options.id + ".tmp"
23+
const marker = options.file + ".port"
24+
const markerTemp = marker + "." + options.id + ".tmp"
2325
yield* fs.makeDirectory(path.dirname(options.file), { recursive: true })
2426
const info = {
2527
id: options.id,
@@ -36,7 +38,10 @@ export const register = Effect.fnUntraced(function* (options: {
3638
found.url === info.url &&
3739
found.pid === info.pid &&
3840
found.password === info.password
39-
yield* fs.writeFileString(temp, encoded, { mode: 0o600 }).pipe(Effect.andThen(fs.rename(temp, options.file)))
41+
yield* fs
42+
.writeFileString(temp, encoded, { mode: 0o600 })
43+
.pipe(Effect.andThen(fs.rename(temp, options.file)))
44+
yield* fs.writeFileString(markerTemp, encoded, { mode: 0o600 }).pipe(Effect.andThen(fs.rename(markerTemp, marker)))
4045
yield* current.pipe(
4146
Effect.catchCause((cause) =>
4247
Effect.logWarning("managed service registration check failed; shutting down", {
@@ -68,5 +73,13 @@ export const register = Effect.fnUntraced(function* (options: {
6873
return current.pipe(
6974
Effect.flatMap((found) => (owns(found) ? fs.remove(options.file) : Effect.void)),
7075
Effect.ignore,
76+
Effect.andThen(
77+
fs.readFileString(marker).pipe(
78+
Effect.flatMap(decodeInfo),
79+
Effect.filterOrFail(owns),
80+
Effect.andThen(fs.remove(marker)),
81+
Effect.ignore,
82+
),
83+
),
7184
)
7285
})

‎packages/cli/test/service-lifecycle.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,29 @@ serviceTest(
153153
120_000,
154154
)
155155

156+
serviceTest(
157+
"waits for an unregistered service that still owns the managed port",
158+
async () => {
159+
await using home = await serviceHome()
160+
await ensure(home)
161+
const original = await settled(home)
162+
163+
signal(original.pid, "SIGSTOP")
164+
await fs.rm(home.registration)
165+
const starting = cli(home, "service", "start")
166+
await Bun.sleep(2_500)
167+
expect(await servers(home)).toEqual([original.pid])
168+
169+
signal(original.pid, "SIGCONT")
170+
await starting
171+
const replacement = await settled(home)
172+
expect(replacement.pid).not.toBe(original.pid)
173+
await exited(original.pid)
174+
await stopped(home)
175+
},
176+
120_000,
177+
)
178+
156179
serviceTest(
157180
"a server that fails to boot reports the failure without respawning",
158181
async () => {

‎packages/client/src/effect/service.ts‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { contenderPool, spawnServiceContender } from "../service-contender.js"
44
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
55
import { matchesVersion } from "../service-version.js"
66
import { PtyHandoff } from "../pty-handoff.js"
7-
import { decide, fallback, headers, probeResult, same } from "../service-probe.js"
7+
import { decide, fallback, headers, marker, probeResult, same } from "../service-probe.js"
88

99
export * from "../service.js"
1010
export { headers }
@@ -69,10 +69,12 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
6969
})
7070
})
7171
const found = yield* Effect.gen(function* () {
72-
const registration = yield* registered(options.file, timing.requestTimeout)
72+
const registrationFile = options.file ?? fallback()
73+
const registrationPresent = (yield* read(registrationFile)) !== undefined
74+
const registration = yield* registered(registrationFile, timing.requestTimeout, marker(options.file))
7375
const info = registration.info
7476
const service = registration.service
75-
if (registration.timedOut && info !== undefined) {
77+
if (registrationPresent && registration.timedOut && info !== undefined) {
7678
timeouts = {
7779
info,
7880
count: timeouts !== undefined && same(timeouts.info, info) ? timeouts.count + 1 : 1,
@@ -105,9 +107,10 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
105107
return Option.none()
106108
}
107109

110+
if (info !== undefined && processAlive(info.pid)) return Option.none()
108111
const failed = pool.reap()
109112
if (failed !== undefined) return yield* Effect.fail(failed)
110-
if (pool.shouldRecruit(info !== undefined)) {
113+
if (pool.shouldRecruit(info !== undefined && !processAlive(info.pid))) {
111114
yield* announce("missing")
112115
pool.add(yield* spawnContender)
113116
}
@@ -157,12 +160,25 @@ const read = Effect.fnUntraced(function* (file?: string) {
157160
return yield* decode(text.value).pipe(Effect.option, Effect.map(Option.getOrUndefined))
158161
})
159162

160-
const registered = Effect.fnUntraced(function* (file?: string, timeout?: number) {
163+
const registered = Effect.fnUntraced(function* (file?: string, timeout?: number, markerFile?: string) {
161164
const info = yield* read(file)
165+
if (info === undefined && markerFile !== undefined) {
166+
const markerInfo = yield* read(markerFile)
167+
if (markerInfo !== undefined) return { info: markerInfo, ...(yield* Effect.promise(() => probeResult(markerInfo, timeout))) }
168+
}
162169
if (info === undefined) return { info: undefined, service: undefined, timedOut: false }
163170
return { info, ...(yield* Effect.promise(() => probeResult(info, timeout))) }
164171
})
165172

173+
function processAlive(pid: number) {
174+
try {
175+
process.kill(pid, 0)
176+
return true
177+
} catch {
178+
return false
179+
}
180+
}
181+
166182
// 50ms cadence bounded at ~5s, shared by stop escalation and each ensure
167183
// discovery window.
168184
const poll = (timing: EnsureTiming) =>

‎packages/client/src/promise/service.ts‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { contenderPool, spawnServiceContender } from "../service-contender.js"
44
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
55
import { matchesVersion } from "../service-version.js"
66
import { PtyHandoff } from "../pty-handoff.js"
7-
import { decide, fallback, headers, probeResult, same } from "../service-probe.js"
7+
import { decide, fallback, headers, marker, probeResult, same } from "../service-probe.js"
88

99
export * from "../service.js"
1010
export { headers }
@@ -51,8 +51,10 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
5151
while (true) {
5252
if (Date.now() >= deadline)
5353
throw pool.failure() ?? new Error("Timed out waiting for the background service to start")
54-
const registration = await registered(options.file, timing.requestTimeout)
55-
if (registration.timedOut && registration.info !== undefined) {
54+
const registrationFile = options.file ?? fallback()
55+
const registrationPresent = (await read(registrationFile)) !== undefined
56+
const registration = await registered(registrationFile, timing.requestTimeout, marker(options.file))
57+
if (registrationPresent && registration.timedOut && registration.info !== undefined) {
5658
timeouts = {
5759
info: registration.info,
5860
count: timeouts !== undefined && same(timeouts.info, registration.info) ? timeouts.count + 1 : 1,
@@ -84,7 +86,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
8486
await stop({ file: options.file, pty: decision.pty }).catch(() => undefined)
8587
pool.evict(service.info.pid)
8688
}
87-
} else {
89+
} else if (registration.info === undefined || !processAlive(registration.info.pid)) {
8890
const failed = pool.reap()
8991
if (failed !== undefined) throw failed
9092
if (pool.shouldRecruit(registration.info !== undefined)) {
@@ -121,12 +123,22 @@ async function read(file?: string) {
121123
}
122124
}
123125

124-
async function registered(file?: string, timeout?: number) {
126+
async function registered(file?: string, timeout?: number, markerFile?: string) {
125127
const info = await read(file)
128+
if (info === undefined && markerFile !== undefined) return registered(markerFile, timeout)
126129
if (info === undefined) return { info: undefined, service: undefined, timedOut: false }
127130
return { info, ...(await probeResult(info, timeout)) }
128131
}
129132

133+
function processAlive(pid: number) {
134+
try {
135+
process.kill(pid, 0)
136+
return true
137+
} catch {
138+
return false
139+
}
140+
}
141+
130142
function signal(pid: number, name: NodeJS.Signals) {
131143
try {
132144
process.kill(pid, name)

‎packages/client/src/service-probe.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ export function fallback() {
4040
return join(process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"), "opencode", "service.json")
4141
}
4242

43+
export function marker(file?: string) {
44+
return (file ?? fallback()) + ".port"
45+
}
46+
4347
/** Whether two registrations describe the same service instance. */
4448
export function same(left: Info, right: Info) {
4549
return left.id === right.id && left.version === right.version && left.url === right.url && left.pid === right.pid

‎packages/client/test/service.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,36 @@ test("replaces a crashed service whose registration names a dead process", async
157157
expect(starts).toEqual(["missing"])
158158
})
159159

160+
test("waits for a live service marker after registration disappears", async () => {
161+
await using fixture = await serviceFixture()
162+
const registration = fixture.registration
163+
const existing = fixture.spawn("hanging")
164+
await fixture.waitForFile()
165+
const original = await Bun.file(registration).json()
166+
await Bun.write(registration + ".port", JSON.stringify(original))
167+
await Bun.file(registration).delete()
168+
169+
const pending = run(
170+
ensure({
171+
file: registration,
172+
version: "test",
173+
command: fixture.command("delayed", "10"),
174+
}),
175+
)
176+
await Bun.sleep(250)
177+
expect(await Bun.file(registration + ".starts").exists()).toBe(false)
178+
expect(existing.exitCode).toBe(null)
179+
180+
existing.kill("SIGKILL")
181+
await existing.exited
182+
const endpoint = await pending
183+
const replacement = await Bun.file(registration).json()
184+
fixture.track(replacement.pid)
185+
186+
expect(replacement.pid).not.toBe(original.pid)
187+
expect(endpoint.url).toBe(replacement.url)
188+
})
189+
160190
test("reports a failed registered service without spawning", async () => {
161191
await using fixture = await serviceFixture()
162192
const registration = fixture.registration

0 commit comments

Comments
 (0)