|
| 1 | +import { describe, expect, test } from "bun:test" |
| 2 | +import { createSidecarSupervisor, type SidecarState, type StartedSidecar } from "./sidecar-supervisor" |
| 3 | + |
| 4 | +type Fake = { url: string; exit: (code: number) => void; stopped: boolean } |
| 5 | + |
| 6 | +function harness(options: { failStarts?: number[]; urls?: string[]; maxCrashes?: number } = {}) { |
| 7 | + const started: Fake[] = [] |
| 8 | + const states: SidecarState[] = [] |
| 9 | + const sleeps: number[] = [] |
| 10 | + let clock = 0 |
| 11 | + let starts = 0 |
| 12 | + const supervisor = createSidecarSupervisor({ |
| 13 | + start: async (previous) => { |
| 14 | + starts += 1 |
| 15 | + if (options.failStarts?.includes(starts)) throw new Error(`start ${starts} failed`) |
| 16 | + let resolveExit!: (code: number) => void |
| 17 | + const exited = new Promise<number>((resolve) => (resolveExit = resolve)) |
| 18 | + const fake: Fake = { |
| 19 | + url: options.urls?.[started.length] ?? previous?.url ?? "http://127.0.0.1:4096", |
| 20 | + exit: (code) => resolveExit(code), |
| 21 | + stopped: false, |
| 22 | + } |
| 23 | + started.push(fake) |
| 24 | + return { |
| 25 | + connection: { url: fake.url, username: "opencode", password: "pw" }, |
| 26 | + exited, |
| 27 | + stop: async () => { |
| 28 | + fake.stopped = true |
| 29 | + resolveExit(0) |
| 30 | + }, |
| 31 | + } satisfies StartedSidecar |
| 32 | + }, |
| 33 | + onState: (state) => states.push(state), |
| 34 | + sleep: async (ms) => { |
| 35 | + sleeps.push(ms) |
| 36 | + clock += ms |
| 37 | + }, |
| 38 | + now: () => clock, |
| 39 | + policy: { delays: [1000, 2000, 5000, 10_000], maxCrashes: options.maxCrashes ?? 3, windowMs: 60_000 }, |
| 40 | + }) |
| 41 | + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)) |
| 42 | + return { supervisor, started, states, sleeps, settle, tick: (ms: number) => (clock += ms) } |
| 43 | +} |
| 44 | + |
| 45 | +describe("sidecar supervisor", () => { |
| 46 | + test("starts once and reports running with the connection", async () => { |
| 47 | + const { supervisor, states } = harness() |
| 48 | + const connection = await supervisor.start() |
| 49 | + expect(connection.url).toBe("http://127.0.0.1:4096") |
| 50 | + expect(states.map((state) => state.status)).toEqual(["starting", "running"]) |
| 51 | + }) |
| 52 | + |
| 53 | + test("a crash is restarted after a backoff, and the new connection is reported", async () => { |
| 54 | + const { supervisor, started, states, sleeps, settle } = harness() |
| 55 | + await supervisor.start() |
| 56 | + started[0]!.exit(3221225477) // 0xC0000005 |
| 57 | + await settle() |
| 58 | + await settle() |
| 59 | + expect(started).toHaveLength(2) |
| 60 | + expect(sleeps).toEqual([1000]) |
| 61 | + const statuses = states.map((state) => state.status) |
| 62 | + expect(statuses).toEqual(["starting", "running", "restarting", "starting", "running"]) |
| 63 | + const restarting = states.find((state) => state.status === "restarting")! |
| 64 | + expect(restarting.lastExit?.code).toBe(3221225477) |
| 65 | + expect(restarting.restarts).toBe(1) |
| 66 | + expect(supervisor.state().status).toBe("running") |
| 67 | + }) |
| 68 | + |
| 69 | + test("the restart reuses the previous connection when it can", async () => { |
| 70 | + const { supervisor, started, settle } = harness() |
| 71 | + await supervisor.start() |
| 72 | + started[0]!.exit(1) |
| 73 | + await settle() |
| 74 | + await settle() |
| 75 | + expect(started[1]!.url).toBe(started[0]!.url) |
| 76 | + }) |
| 77 | + |
| 78 | + test("backoff follows 1s, 2s, 5s, 10s and stays at 10s", async () => { |
| 79 | + const { supervisor, started, sleeps, settle } = harness({ maxCrashes: 10 }) |
| 80 | + await supervisor.start() |
| 81 | + for (let i = 0; i < 6; i++) { |
| 82 | + started.at(-1)!.exit(1) |
| 83 | + await settle() |
| 84 | + await settle() |
| 85 | + } |
| 86 | + expect(sleeps).toEqual([1000, 2000, 5000, 10_000, 10_000, 10_000]) |
| 87 | + }) |
| 88 | + |
| 89 | + test("too many crashes inside the window stops restarting and reports failed", async () => { |
| 90 | + const { supervisor, started, states, settle } = harness() |
| 91 | + await supervisor.start() |
| 92 | + for (let i = 0; i < 3; i++) { |
| 93 | + started.at(-1)!.exit(1) |
| 94 | + await settle() |
| 95 | + await settle() |
| 96 | + } |
| 97 | + expect(supervisor.state().status).toBe("failed") |
| 98 | + expect(started).toHaveLength(3) |
| 99 | + expect(states.at(-1)?.status).toBe("failed") |
| 100 | + }) |
| 101 | + |
| 102 | + test("crashes spread out beyond the window keep restarting", async () => { |
| 103 | + const { supervisor, started, settle, tick } = harness() |
| 104 | + await supervisor.start() |
| 105 | + for (let i = 0; i < 5; i++) { |
| 106 | + tick(120_000) |
| 107 | + started.at(-1)!.exit(1) |
| 108 | + await settle() |
| 109 | + await settle() |
| 110 | + } |
| 111 | + expect(supervisor.state().status).toBe("running") |
| 112 | + expect(started).toHaveLength(6) |
| 113 | + }) |
| 114 | + |
| 115 | + test("a failed restart attempt counts as a crash and is retried", async () => { |
| 116 | + const { supervisor, started, settle } = harness({ failStarts: [2] }) |
| 117 | + await supervisor.start() |
| 118 | + started[0]!.exit(1) |
| 119 | + for (let i = 0; i < 6; i++) await settle() |
| 120 | + expect(supervisor.state().status).toBe("running") |
| 121 | + expect(started).toHaveLength(2) |
| 122 | + }) |
| 123 | + |
| 124 | + test("a manual restart after failure starts it again", async () => { |
| 125 | + const { supervisor, started, settle } = harness() |
| 126 | + await supervisor.start() |
| 127 | + for (let i = 0; i < 3; i++) { |
| 128 | + started.at(-1)!.exit(1) |
| 129 | + await settle() |
| 130 | + await settle() |
| 131 | + } |
| 132 | + expect(supervisor.state().status).toBe("failed") |
| 133 | + await supervisor.restart() |
| 134 | + expect(supervisor.state().status).toBe("running") |
| 135 | + }) |
| 136 | + |
| 137 | + test("a deliberate stop is not restarted", async () => { |
| 138 | + const { supervisor, started, settle } = harness() |
| 139 | + await supervisor.start() |
| 140 | + await supervisor.stop() |
| 141 | + await settle() |
| 142 | + await settle() |
| 143 | + expect(started).toHaveLength(1) |
| 144 | + expect(supervisor.state().status).toBe("stopped") |
| 145 | + }) |
| 146 | +}) |
0 commit comments