Skip to content

Commit ce8dced

Browse files
feat(desktop): sidecar supervisor with capped backoff and a failed state
A pure supervisor (no electron import) that restarts the local server after an unexpected exit, waiting 1 s, 2 s, 5 s, then 10 s between attempts, and gives up with a visible "failed" state after 5 crashes in 10 minutes instead of looping forever. A restart reuses the previous port and credentials; a failed launch counts as a crash; a deliberate stop is never restarted; a manual restart clears the crash window. Unit tests use a fake child. The live test spawns a throwaway `opencode serve` in a temp data dir (never the running app's sidecar), creates a session, SIGKILLs the process, and reads the session back after the supervisor brings it up on the same connection. Ref anomalyco#43, LDE-35 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01GHX6CpBR14wnFxJLDahwTr
1 parent f67bf25 commit ce8dced

3 files changed

Lines changed: 402 additions & 0 deletions

File tree

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { afterAll, describe, expect, test } from "bun:test"
2+
import { spawn, type ChildProcess } from "node:child_process"
3+
import { mkdtempSync, rmSync } from "node:fs"
4+
import { createServer } from "node:net"
5+
import { tmpdir } from "node:os"
6+
import { join, resolve } from "node:path"
7+
import { createSidecarSupervisor, type SidecarConnection, type StartedSidecar } from "./sidecar-supervisor"
8+
9+
// A real, throwaway server process: the opencode CLI's `serve`, in a temp data
10+
// directory, on its own port. It is never the live dev app's sidecar. The test
11+
// kills it the way the 9/21 crash did (the process just ends), and checks that
12+
// the supervisor brings it back and that a session created before the crash is
13+
// still there after it: sessions live on disk, not in the process.
14+
15+
const opencode = resolve(import.meta.dir, "../../../opencode")
16+
const root = mkdtempSync(join(tmpdir(), "sidecar-supervisor-"))
17+
const children: ChildProcess[] = []
18+
19+
afterAll(() => {
20+
for (const child of children) child.kill()
21+
rmSync(root, { recursive: true, force: true })
22+
})
23+
24+
const freePort = () =>
25+
new Promise<number>((done, fail) => {
26+
const socket = createServer()
27+
socket.on("error", fail)
28+
socket.listen(0, "127.0.0.1", () => {
29+
const address = socket.address()
30+
const port = typeof address === "object" && address ? address.port : 0
31+
socket.close(() => done(port))
32+
})
33+
})
34+
35+
const auth = (connection: SidecarConnection) =>
36+
`Basic ${Buffer.from(`${connection.username}:${connection.password}`).toString("base64")}`
37+
38+
async function healthy(connection: SidecarConnection, deadline: number) {
39+
while (Date.now() < deadline) {
40+
const ok = await fetch(`${connection.url}/global/health`, { headers: { authorization: auth(connection) } })
41+
.then((response) => response.ok)
42+
.catch(() => false)
43+
if (ok) return
44+
await new Promise((resolve) => setTimeout(resolve, 250))
45+
}
46+
throw new Error(`server at ${connection.url} never became healthy`)
47+
}
48+
49+
async function startServer(previous: SidecarConnection | undefined): Promise<StartedSidecar> {
50+
const connection = previous ?? {
51+
url: `http://127.0.0.1:${await freePort()}`,
52+
username: "opencode",
53+
password: crypto.randomUUID(),
54+
}
55+
const port = new URL(connection.url).port
56+
const child = spawn(process.execPath, ["run", "./src/index.ts", "serve", "--hostname", "127.0.0.1", "--port", port], {
57+
cwd: opencode,
58+
env: {
59+
...process.env,
60+
OPENCODE_SERVER_PASSWORD: connection.password,
61+
OPENCODE_SERVER_USERNAME: connection.username,
62+
XDG_DATA_HOME: join(root, "share"),
63+
XDG_CACHE_HOME: join(root, "cache"),
64+
XDG_CONFIG_HOME: join(root, "config"),
65+
XDG_STATE_HOME: join(root, "state"),
66+
OPENCODE_MODELS_PATH: join(opencode, "test", "tool", "fixtures", "models-api.json"),
67+
OPENCODE_DISABLE_MODELS_FETCH: "true",
68+
OPENCODE_DISABLE_AUTOUPDATE: "true",
69+
},
70+
stdio: "ignore",
71+
})
72+
children.push(child)
73+
const exited = new Promise<number>((done) => child.once("exit", (code) => done(code ?? -1)))
74+
await Promise.race([
75+
healthy(connection, Date.now() + 90_000),
76+
exited.then((code) => {
77+
throw new Error(`server exited before it was healthy (code ${code})`)
78+
}),
79+
])
80+
return { connection, exited, stop: async () => void child.kill() }
81+
}
82+
83+
describe("sidecar supervisor with a real server process", () => {
84+
test("a crashed server is restarted and a session created before the crash is still there", async () => {
85+
const project = mkdtempSync(join(root, "project-"))
86+
const supervisor = createSidecarSupervisor({
87+
start: startServer,
88+
policy: { delays: [500], maxCrashes: 3, windowMs: 60_000 },
89+
})
90+
const first = await supervisor.start()
91+
const headers = { authorization: auth(first), "content-type": "application/json" }
92+
const created = await fetch(`${first.url}/session?directory=${encodeURIComponent(project)}`, {
93+
method: "POST",
94+
headers,
95+
body: JSON.stringify({ title: "before the crash" }),
96+
}).then((response) => response.json() as Promise<{ id: string }>)
97+
expect(created.id).toBeTruthy()
98+
99+
// the crash: the process is gone, nobody asked it to stop
100+
children.at(-1)!.kill("SIGKILL")
101+
102+
const deadline = Date.now() + 120_000
103+
while (Date.now() < deadline && !(supervisor.state().status === "running" && children.length === 2)) {
104+
await new Promise((resolve) => setTimeout(resolve, 250))
105+
}
106+
expect(supervisor.state().status).toBe("running")
107+
expect(supervisor.state().restarts).toBe(1)
108+
const after = supervisor.connection()!
109+
// same port and credentials, so clients holding the old server.json keep working
110+
expect(after).toEqual(first)
111+
112+
const read = await fetch(`${after.url}/session/${created.id}?directory=${encodeURIComponent(project)}`, {
113+
headers,
114+
})
115+
expect(read.status).toBe(200)
116+
expect(((await read.json()) as { title: string }).title).toBe("before the crash")
117+
await supervisor.stop()
118+
}, 300_000)
119+
})
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import { describe, expect, test } from "bun:test"
2+
import { createSidecarSupervisor, type SidecarState, type StartedSidecar } from "./sidecar-supervisor"
3+
4+
type Fake = { url: string; exit: (code: number) => void; stopped: boolean }
5+
6+
function harness(options: { failStarts?: number[]; urls?: string[]; maxCrashes?: number } = {}) {
7+
const started: Fake[] = []
8+
const states: SidecarState[] = []
9+
const sleeps: number[] = []
10+
let clock = 0
11+
let starts = 0
12+
const supervisor = createSidecarSupervisor({
13+
start: async (previous) => {
14+
starts += 1
15+
if (options.failStarts?.includes(starts)) throw new Error(`start ${starts} failed`)
16+
let resolveExit!: (code: number) => void
17+
const exited = new Promise<number>((resolve) => (resolveExit = resolve))
18+
const fake: Fake = {
19+
url: options.urls?.[started.length] ?? previous?.url ?? "http://127.0.0.1:4096",
20+
exit: (code) => resolveExit(code),
21+
stopped: false,
22+
}
23+
started.push(fake)
24+
return {
25+
connection: { url: fake.url, username: "opencode", password: "pw" },
26+
exited,
27+
stop: async () => {
28+
fake.stopped = true
29+
resolveExit(0)
30+
},
31+
} satisfies StartedSidecar
32+
},
33+
onState: (state) => states.push(state),
34+
sleep: async (ms) => {
35+
sleeps.push(ms)
36+
clock += ms
37+
},
38+
now: () => clock,
39+
policy: { delays: [1000, 2000, 5000, 10_000], maxCrashes: options.maxCrashes ?? 3, windowMs: 60_000 },
40+
})
41+
const settle = () => new Promise((resolve) => setTimeout(resolve, 0))
42+
return { supervisor, started, states, sleeps, settle, tick: (ms: number) => (clock += ms) }
43+
}
44+
45+
describe("sidecar supervisor", () => {
46+
test("starts once and reports running with the connection", async () => {
47+
const { supervisor, states } = harness()
48+
const connection = await supervisor.start()
49+
expect(connection.url).toBe("http://127.0.0.1:4096")
50+
expect(states.map((state) => state.status)).toEqual(["starting", "running"])
51+
})
52+
53+
test("a crash is restarted after a backoff, and the new connection is reported", async () => {
54+
const { supervisor, started, states, sleeps, settle } = harness()
55+
await supervisor.start()
56+
started[0]!.exit(3221225477) // 0xC0000005
57+
await settle()
58+
await settle()
59+
expect(started).toHaveLength(2)
60+
expect(sleeps).toEqual([1000])
61+
const statuses = states.map((state) => state.status)
62+
expect(statuses).toEqual(["starting", "running", "restarting", "starting", "running"])
63+
const restarting = states.find((state) => state.status === "restarting")!
64+
expect(restarting.lastExit?.code).toBe(3221225477)
65+
expect(restarting.restarts).toBe(1)
66+
expect(supervisor.state().status).toBe("running")
67+
})
68+
69+
test("the restart reuses the previous connection when it can", async () => {
70+
const { supervisor, started, settle } = harness()
71+
await supervisor.start()
72+
started[0]!.exit(1)
73+
await settle()
74+
await settle()
75+
expect(started[1]!.url).toBe(started[0]!.url)
76+
})
77+
78+
test("backoff follows 1s, 2s, 5s, 10s and stays at 10s", async () => {
79+
const { supervisor, started, sleeps, settle } = harness({ maxCrashes: 10 })
80+
await supervisor.start()
81+
for (let i = 0; i < 6; i++) {
82+
started.at(-1)!.exit(1)
83+
await settle()
84+
await settle()
85+
}
86+
expect(sleeps).toEqual([1000, 2000, 5000, 10_000, 10_000, 10_000])
87+
})
88+
89+
test("too many crashes inside the window stops restarting and reports failed", async () => {
90+
const { supervisor, started, states, settle } = harness()
91+
await supervisor.start()
92+
for (let i = 0; i < 3; i++) {
93+
started.at(-1)!.exit(1)
94+
await settle()
95+
await settle()
96+
}
97+
expect(supervisor.state().status).toBe("failed")
98+
expect(started).toHaveLength(3)
99+
expect(states.at(-1)?.status).toBe("failed")
100+
})
101+
102+
test("crashes spread out beyond the window keep restarting", async () => {
103+
const { supervisor, started, settle, tick } = harness()
104+
await supervisor.start()
105+
for (let i = 0; i < 5; i++) {
106+
tick(120_000)
107+
started.at(-1)!.exit(1)
108+
await settle()
109+
await settle()
110+
}
111+
expect(supervisor.state().status).toBe("running")
112+
expect(started).toHaveLength(6)
113+
})
114+
115+
test("a failed restart attempt counts as a crash and is retried", async () => {
116+
const { supervisor, started, settle } = harness({ failStarts: [2] })
117+
await supervisor.start()
118+
started[0]!.exit(1)
119+
for (let i = 0; i < 6; i++) await settle()
120+
expect(supervisor.state().status).toBe("running")
121+
expect(started).toHaveLength(2)
122+
})
123+
124+
test("a manual restart after failure starts it again", async () => {
125+
const { supervisor, started, settle } = harness()
126+
await supervisor.start()
127+
for (let i = 0; i < 3; i++) {
128+
started.at(-1)!.exit(1)
129+
await settle()
130+
await settle()
131+
}
132+
expect(supervisor.state().status).toBe("failed")
133+
await supervisor.restart()
134+
expect(supervisor.state().status).toBe("running")
135+
})
136+
137+
test("a deliberate stop is not restarted", async () => {
138+
const { supervisor, started, settle } = harness()
139+
await supervisor.start()
140+
await supervisor.stop()
141+
await settle()
142+
await settle()
143+
expect(started).toHaveLength(1)
144+
expect(supervisor.state().status).toBe("stopped")
145+
})
146+
})

0 commit comments

Comments
 (0)