Skip to content

Commit 6a56c8b

Browse files
tlongwell-blocknpub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67dTyler Longwell
authored
test(desktop): live relay kill/restart reconnect gate (#2583)
Signed-off-by: Tyler Longwell <[email protected]> Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Co-authored-by: Tyler Longwell <[email protected]>
1 parent 1911c69 commit 6a56c8b

3 files changed

Lines changed: 139 additions & 3 deletions

File tree

‎desktop/playwright.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ export default defineConfig({
137137
"**/persona-sync.spec.ts",
138138
"**/team-snapshot.spec.ts",
139139
"**/agents-everywhere.live.spec.ts",
140+
"**/relay-restart.live.spec.ts",
140141
"**/parity-ancestor-island.spec.ts",
141142
],
142143
use: {

‎desktop/tests/e2e/helpers/twoRelayHarness.ts‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ type OwnedProcess = { name: string; child: ChildProcess; logPath: string };
1717

1818
export class TwoRelayHarness {
1919
readonly root: string;
20-
readonly relays: readonly [RelaySpec, RelaySpec];
20+
readonly relays: readonly RelaySpec[];
2121
private readonly processes: OwnedProcess[] = [];
2222

2323
get ownedPids(): number[] {
@@ -26,12 +26,12 @@ export class TwoRelayHarness {
2626
);
2727
}
2828

29-
private constructor(root: string, relays: readonly [RelaySpec, RelaySpec]) {
29+
private constructor(root: string, relays: readonly RelaySpec[]) {
3030
this.root = root;
3131
this.relays = relays;
3232
}
3333

34-
static async create(relays: readonly [RelaySpec, RelaySpec]) {
34+
static async create(relays: readonly RelaySpec[]) {
3535
return new TwoRelayHarness(
3636
await mkdtemp(join(tmpdir(), "buzz-ae-e2e-")),
3737
relays,
@@ -46,6 +46,45 @@ export class TwoRelayHarness {
4646
);
4747
}
4848

49+
/**
50+
* SIGTERM one relay by name and wait for it to exit on its own. Unlike
51+
* `stop()`, this never escalates to SIGKILL: the relay's graceful drain
52+
* (readiness 503 → 5s grace → 1012 close broadcast → listener drain) is
53+
* exactly what restart tests exercise, so a forced kill would invalidate
54+
* the run. Throws if the process does not exit within `timeoutMs`.
55+
*/
56+
async terminateRelayGracefully(name: string, timeoutMs = 45_000) {
57+
const owned = [...this.processes]
58+
.reverse()
59+
.find(
60+
(candidate) =>
61+
candidate.name === name &&
62+
candidate.child.exitCode === null &&
63+
candidate.child.signalCode === null,
64+
);
65+
if (!owned) throw new Error(`no live relay process named ${name}`);
66+
const exited = new Promise<void>((resolveExit) =>
67+
owned.child.once("exit", () => resolveExit()),
68+
);
69+
this.signal(owned.child, "SIGTERM");
70+
if (
71+
!(await Promise.race([exited.then(() => true), delay(timeoutMs, false)]))
72+
) {
73+
throw new Error(
74+
`${name} did not exit within ${timeoutMs}ms of SIGTERM — graceful drain is stuck`,
75+
);
76+
}
77+
}
78+
79+
/** Start a fresh process for a relay spec on its original ports. */
80+
async restartRelay(name: string, binary = process.env.BUZZ_E2E_RELAY_BIN) {
81+
if (!binary)
82+
throw new Error("BUZZ_E2E_RELAY_BIN is required for the live gate");
83+
const relay = this.relays.find((candidate) => candidate.name === name);
84+
if (!relay) throw new Error(`no relay spec named ${name}`);
85+
await this.startRelay(binary, relay);
86+
}
87+
4988
async startAcp(
5089
name: string,
5190
relayWsUrl: string,
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import { expect, test, type Page } from "@playwright/test";
2+
3+
import { installBridge } from "../helpers/bridge";
4+
import { TwoRelayHarness, type RelaySpec } from "./helpers/twoRelayHarness";
5+
6+
// Live gate: boots a REAL buzz-relay process, points the app at it, SIGTERMs
7+
// the relay mid-session, restarts it on the same port, and asserts the client
8+
// converges back to "connected". This proves the full restart story end to
9+
// end: the relay's graceful-drain 1012 close broadcast (server side) and the
10+
// client's dial-failure retry + 1012 fast-reconnect (desktop side) — the two
11+
// halves that synthetic mock-websocket specs cannot compose.
12+
//
13+
// Requires: BUZZ_E2E_RELAY_RESTART=1, BUZZ_E2E_RELAY_BIN, and
14+
// BUZZ_E2E_DATABASE_URL (plus reachable Redis and media object store, same
15+
// infra as the agents-everywhere live gate).
16+
const enabled = process.env.BUZZ_E2E_RELAY_RESTART === "1";
17+
18+
function required(name: string, value: string | undefined): string {
19+
if (!value) throw new Error(`${name} is required for the live gate`);
20+
return value;
21+
}
22+
23+
async function connectionState(page: Page): Promise<string> {
24+
return page.evaluate(() => {
25+
const win = window as Window & {
26+
__BUZZ_E2E_GET_RELAY_CONNECTION_STATE__?: () => string;
27+
};
28+
return win.__BUZZ_E2E_GET_RELAY_CONNECTION_STATE__?.() ?? "uninstalled";
29+
});
30+
}
31+
32+
test.describe("relay restart live gate", () => {
33+
test.skip(!enabled, "set BUZZ_E2E_RELAY_RESTART=1 to run live gate");
34+
35+
test("client reconnects after the relay is SIGTERMed and restarted", async ({
36+
page,
37+
}) => {
38+
test.setTimeout(180_000);
39+
const portBase = 26_000 + (process.pid % 3_000);
40+
const spec: RelaySpec = {
41+
name: "relay-restart",
42+
ports: {
43+
main: portBase,
44+
health: portBase + 3_000,
45+
metrics: portBase + 6_000,
46+
},
47+
databaseUrl: required(
48+
"BUZZ_E2E_DATABASE_URL",
49+
process.env.BUZZ_E2E_DATABASE_URL,
50+
),
51+
redisUrl:
52+
process.env.BUZZ_E2E_REDIS_RESTART ?? "redis://127.0.0.1:6379/13",
53+
};
54+
const harness = await TwoRelayHarness.create([spec]);
55+
try {
56+
await harness.startRelays();
57+
58+
const relayHttpUrl = `http://127.0.0.1:${spec.ports.main}`;
59+
await installBridge(page, {
60+
mode: "relay",
61+
user: "tyler",
62+
relayHttpUrl,
63+
relayWsUrl: `ws://127.0.0.1:${spec.ports.main}`,
64+
});
65+
await page.goto("/");
66+
67+
// Baseline: the app converges to a live authenticated session.
68+
await expect
69+
.poll(() => connectionState(page), { timeout: 60_000 })
70+
.toBe("connected");
71+
72+
// Roll the pod. Graceful drain: readiness 503 → 5s grace → 1012 close
73+
// broadcast → process exit. The client must observe the close (not a
74+
// silent stall) and start retrying.
75+
await harness.terminateRelayGracefully(spec.name);
76+
await expect
77+
.poll(() => connectionState(page), { timeout: 30_000 })
78+
.not.toBe("connected");
79+
80+
// Bring the "new pod" up on the same address, exactly like a k8s
81+
// restart behind a stable service endpoint.
82+
await harness.restartRelay(spec.name);
83+
84+
// The client's retry loop must find the fresh relay and converge back
85+
// to connected without any user interaction.
86+
await expect
87+
.poll(() => connectionState(page), { timeout: 60_000 })
88+
.toBe("connected");
89+
} catch (error) {
90+
console.error(await harness.logs());
91+
throw error;
92+
} finally {
93+
await harness.stop();
94+
}
95+
});
96+
});

0 commit comments

Comments
 (0)