Skip to content

Commit bb65de3

Browse files
committed
Own OpenCode candidate-first generation cutovers and cold session starts
Replace the independent ensure/restart launchers with one serialized owner and persisted active process identity. Prepare and verify only at idle, publish before retiring, and retry failed candidates without abandoning the serving incumbent. Preserve dd1e4ca's protection for active work and live procId reuse: SIGKILL is permitted only for an identity-checked retired generation after its grace period, never the active generation. Keep cbd4c3a's CawCo hub provenance check without waiting on unrelated cold MCP servers.
1 parent 547b3a7 commit bb65de3

3 files changed

Lines changed: 501 additions & 133 deletions

File tree

‎packages/agent/src/daemon.ts‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import { convergeDeniedTools } from "./denied-tools";
3030
import { deployRoot, latestDeploy } from "./deploy";
3131
import { rediscoverHub, toWsUrl } from "./discovery";
3232
import { harnesses } from "./harnesses";
33-
import { OPENCODE_SERVER_PROC_ID } from "./harnesses/opencode";
33+
import { isOpencodeServerProc } from "./harnesses/opencode-server";
3434
import { cache as transcriptCache } from "./harnesses/transcript-cache";
3535
import { machineId } from "./machine-id";
3636
import {
@@ -458,11 +458,9 @@ const readSessions = async () => {
458458
const held = client.procs.filter((proc) => proc.alive);
459459
return {
460460
instances: held
461-
.filter((proc) => proc.procId !== OPENCODE_SERVER_PROC_ID)
461+
.filter((proc) => !isOpencodeServerProc(proc.procId))
462462
.map((proc) => proc.procId),
463-
opencode: held.some(
464-
(proc) => proc.procId === OPENCODE_SERVER_PROC_ID
465-
),
463+
opencode: held.some((proc) => isOpencodeServerProc(proc.procId)),
466464
};
467465
} finally {
468466
client.close();
Lines changed: 354 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,354 @@
1+
import { createHash } from "node:crypto";
2+
import { homedir } from "node:os";
3+
import { join } from "node:path";
4+
import { type ProcSpec, sessiondEndpoint } from "@cawco/core/sessiond";
5+
import type { SessiondClient } from "../sessiond-client";
6+
import { readJson, writeJson } from "./fleet-common";
7+
8+
export const OPENCODE_SERVER_PROC_ID = "opencode-server";
9+
export const SERVER_CUTOVER_TIMEOUT_MS = 60_000;
10+
export const isOpencodeServerProc = (id: string): boolean =>
11+
id === OPENCODE_SERVER_PROC_ID ||
12+
id.startsWith(`${OPENCODE_SERVER_PROC_ID}-`);
13+
14+
export interface ServerIdentity {
15+
epoch: string;
16+
pid: number;
17+
procId: string;
18+
startedAt: string;
19+
url: string;
20+
}
21+
22+
interface ServerRecord {
23+
active: ServerIdentity | null;
24+
retired: ServerIdentity[];
25+
}
26+
27+
type Attach = (
28+
client: SessiondClient,
29+
procId: string,
30+
spec: ProcSpec,
31+
signal: AbortSignal
32+
) => Promise<string>;
33+
34+
function abortable<T>(pending: Promise<T>, signal: AbortSignal): Promise<T> {
35+
signal.throwIfAborted();
36+
return new Promise((resolve, reject) => {
37+
const abort = () => reject(signal.reason);
38+
signal.addEventListener("abort", abort, { once: true });
39+
pending
40+
.then(resolve, reject)
41+
.finally(() => signal.removeEventListener("abort", abort));
42+
});
43+
}
44+
45+
/** OS start identity plus sessiond epoch/procId/pid prevents signalling a reused PID. */
46+
async function processStart(pid: number): Promise<string> {
47+
if (process.platform === "linux") {
48+
const stat = await Bun.file(`/proc/${pid}/stat`)
49+
.text()
50+
.catch(() => "");
51+
return stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19] ?? "";
52+
}
53+
if (process.platform !== "darwin") {
54+
throw new Error(
55+
"OpenCode generation retirement requires OS process start identity."
56+
);
57+
}
58+
const proc = Bun.spawn(["ps", "-p", String(pid), "-o", "lstart="], {
59+
env: { ...process.env, LC_ALL: "C" },
60+
stdout: "pipe",
61+
stderr: "ignore",
62+
});
63+
const value = (await new Response(proc.stdout).text()).trim();
64+
return (await proc.exited) === 0 ? value : "";
65+
}
66+
67+
/** The only launcher/adopter. Publish a replacement before retiring its predecessor. */
68+
export class OpencodeServerOwner {
69+
readonly #path = join(
70+
homedir(),
71+
".cawco",
72+
`opencode-server-${createHash("sha256")
73+
.update(process.env.CAWCO_SESSIOND_ENDPOINT ?? sessiondEndpoint())
74+
.digest("hex")
75+
.slice(0, 16)}.json`
76+
);
77+
readonly #sessiond: () => Promise<SessiondClient>;
78+
readonly #attach: Attach;
79+
readonly #mayManage: () => Promise<boolean>;
80+
#record: ServerRecord | null = null;
81+
#transition: Promise<ServerIdentity> | null = null;
82+
#candidateProcId: string | null = null;
83+
#writes = Promise.resolve();
84+
readonly #retiring = new Map<string, Promise<void>>();
85+
readonly #retryAt = new Map<string, number>();
86+
87+
constructor(
88+
sessiond: () => Promise<SessiondClient>,
89+
attach: Attach,
90+
mayManage: () => Promise<boolean>
91+
) {
92+
this.#sessiond = sessiond;
93+
this.#attach = attach;
94+
this.#mayManage = mayManage;
95+
}
96+
97+
get active(): ServerIdentity | null {
98+
return this.#record?.active ?? null;
99+
}
100+
101+
async #load(): Promise<ServerRecord> {
102+
if (!this.#record) {
103+
const stored = await readJson<ServerRecord>(this.#path);
104+
if ((await Bun.file(this.#path).exists()) && !stored) {
105+
throw new Error("OpenCode active-generation record is unreadable.");
106+
}
107+
this.#record = stored ?? { active: null, retired: [] };
108+
}
109+
return this.#record;
110+
}
111+
112+
#save(): Promise<void> {
113+
const record = this.#record;
114+
const written = this.#writes.then(() => writeJson(this.#path, record));
115+
this.#writes = written.catch(() => undefined);
116+
return written;
117+
}
118+
119+
async #matches(identity: ServerIdentity): Promise<boolean> {
120+
const client = await this.#sessiond();
121+
const listed = await client.list();
122+
const proc = listed.procs.find((row) => row.procId === identity.procId);
123+
return Boolean(
124+
listed.epoch === identity.epoch &&
125+
proc?.alive &&
126+
proc.pid === identity.pid &&
127+
(await processStart(identity.pid)) === identity.startedAt
128+
);
129+
}
130+
131+
async #identify(
132+
client: SessiondClient,
133+
procId: string,
134+
url: string
135+
): Promise<ServerIdentity> {
136+
const listed = await client.list();
137+
const proc = listed.procs.find((row) => row.procId === procId && row.alive);
138+
const startedAt = proc ? await processStart(proc.pid) : "";
139+
if (!(proc && startedAt)) {
140+
throw new Error(`OpenCode generation ${procId} exited before adoption.`);
141+
}
142+
return { procId, pid: proc.pid, epoch: listed.epoch, startedAt, url };
143+
}
144+
145+
async #launch(spec: ProcSpec, signal: AbortSignal): Promise<ServerIdentity> {
146+
const client = await this.#sessiond();
147+
const procId = `${OPENCODE_SERVER_PROC_ID}-${crypto.randomUUID()}`;
148+
const reservation = Bun.serve({
149+
hostname: "127.0.0.1",
150+
port: 0,
151+
fetch: () => new Response(null, { status: 404 }),
152+
});
153+
const { port } = reservation;
154+
await reservation.stop(true);
155+
signal.throwIfAborted();
156+
try {
157+
const url = await this.#attach(
158+
client,
159+
procId,
160+
{
161+
...spec,
162+
args: [
163+
...spec.args.filter((arg) => !arg.startsWith("--port=")),
164+
`--port=${port}`,
165+
],
166+
},
167+
signal
168+
);
169+
const identity = await this.#identify(client, procId, url);
170+
signal.throwIfAborted();
171+
return identity;
172+
} catch (error) {
173+
// A timed-out announcement may leave a real candidate. Capture it for
174+
// retirement; never reuse that procId or signal the active generation.
175+
const orphan = await this.#identify(client, procId, "").catch(() => null);
176+
if (orphan && this.#record) {
177+
this.#record = {
178+
...this.#record,
179+
retired: [...this.#record.retired, orphan],
180+
};
181+
await this.#save();
182+
this.maintain();
183+
}
184+
throw error;
185+
}
186+
}
187+
188+
ensure(spec: ProcSpec): Promise<ServerIdentity> {
189+
if (this.#transition) {
190+
return this.#transition;
191+
}
192+
this.#transition = (async () => {
193+
const manages = await this.#mayManage();
194+
if (!manages) {
195+
this.#record = null;
196+
}
197+
const record = await this.#load();
198+
if (record.active && (await this.#matches(record.active))) {
199+
if (manages) {
200+
this.maintain();
201+
}
202+
return record.active;
203+
}
204+
const client = await this.#sessiond();
205+
const listed = await client.list();
206+
// Initial cutover adopts the existing server held under the stable name.
207+
const incumbent =
208+
!record.active &&
209+
listed.procs.some(
210+
(proc) => proc.procId === OPENCODE_SERVER_PROC_ID && proc.alive
211+
);
212+
const signal = AbortSignal.timeout(SERVER_CUTOVER_TIMEOUT_MS);
213+
if (!(manages || incumbent)) {
214+
throw new Error(
215+
"The machine's OpenCode lifecycle owner must start its server."
216+
);
217+
}
218+
const identity = incumbent
219+
? await this.#identify(
220+
client,
221+
OPENCODE_SERVER_PROC_ID,
222+
await this.#attach(client, OPENCODE_SERVER_PROC_ID, spec, signal)
223+
)
224+
: await this.#launch(spec, signal);
225+
this.#record = { ...record, active: identity };
226+
if (manages) {
227+
await this.#save();
228+
this.maintain();
229+
}
230+
return identity;
231+
})().finally(() => {
232+
this.#transition = null;
233+
});
234+
return this.#transition;
235+
}
236+
237+
replace(
238+
spec: ProcSpec,
239+
verify: (identity: ServerIdentity, signal: AbortSignal) => Promise<void>,
240+
activate: (identity: ServerIdentity) => void
241+
): Promise<ServerIdentity> {
242+
if (this.#transition) {
243+
throw new Error("OpenCode lifecycle transition already in progress.");
244+
}
245+
this.#transition = (async () => {
246+
if (!(await this.#mayManage())) {
247+
throw new Error(
248+
"Only the machine agent may replace its OpenCode generation."
249+
);
250+
}
251+
const record = await this.#load();
252+
const old = record.active;
253+
const signal = AbortSignal.timeout(SERVER_CUTOVER_TIMEOUT_MS);
254+
const candidate = await abortable(this.#launch(spec, signal), signal);
255+
this.#candidateProcId = candidate.procId;
256+
try {
257+
// An unselected candidate is durable cleanup work if the agent crashes.
258+
const preparing = await this.#load();
259+
this.#record = {
260+
...preparing,
261+
retired: [...preparing.retired, candidate],
262+
};
263+
await this.#save();
264+
await abortable(verify(candidate, signal), signal);
265+
signal.throwIfAborted();
266+
const retired = this.#record.retired.filter(
267+
(row) => row.procId !== candidate.procId
268+
);
269+
if (old) {
270+
retired.push(old);
271+
}
272+
this.#record = { active: candidate, retired };
273+
await this.#save();
274+
activate(candidate);
275+
console.info(
276+
`[opencode] candidate→active ${old?.procId ?? "none"}/${old?.pid ?? "none"} → ${candidate.procId}/${candidate.pid}`
277+
);
278+
return candidate;
279+
} catch (error) {
280+
const current = await this.#load();
281+
this.#record = { ...current, active: old };
282+
await this.#save();
283+
throw error;
284+
} finally {
285+
this.#candidateProcId = null;
286+
this.maintain();
287+
}
288+
})().finally(() => {
289+
this.#transition = null;
290+
});
291+
return this.#transition;
292+
}
293+
294+
maintain(): void {
295+
for (const identity of this.#record?.retired ?? []) {
296+
if (
297+
identity.procId === this.#candidateProcId ||
298+
this.#retiring.has(identity.procId) ||
299+
Date.now() < (this.#retryAt.get(identity.procId) ?? 0)
300+
) {
301+
continue;
302+
}
303+
const retiring = this.#retire(identity)
304+
.catch((error: unknown) => {
305+
this.#retryAt.set(identity.procId, Date.now() + 5000);
306+
console.warn(
307+
`[opencode] retired generation ${identity.procId}: ${error}; retry in 5s`
308+
);
309+
})
310+
.finally(() => this.#retiring.delete(identity.procId));
311+
this.#retiring.set(identity.procId, retiring);
312+
}
313+
}
314+
315+
async #retire(identity: ServerIdentity): Promise<void> {
316+
if (this.active?.procId === identity.procId) {
317+
throw new Error("Refusing to retire the active OpenCode generation.");
318+
}
319+
const client = await this.#sessiond();
320+
if (await this.#matches(identity)) {
321+
await client.signal(identity.procId, "SIGTERM");
322+
const deadline = Date.now() + 15_000;
323+
// biome-ignore lint/performance/noAwaitInLoops: observe this exact retired identity during its grace period
324+
while (Date.now() < deadline && (await this.#matches(identity))) {
325+
await Bun.sleep(200);
326+
}
327+
// dd1e4ca1 protects active work and live procId reuse. This generation is
328+
// retired, has no bound sessions, and its unique procId is never reused.
329+
if (
330+
this.active?.procId !== identity.procId &&
331+
(await this.#matches(identity))
332+
) {
333+
await client.signal(identity.procId, "SIGKILL");
334+
const killDeadline = Date.now() + 5000;
335+
// biome-ignore lint/performance/noAwaitInLoops: verify exit of only the retired identity
336+
while (Date.now() < killDeadline && (await this.#matches(identity))) {
337+
await Bun.sleep(100);
338+
}
339+
if (await this.#matches(identity)) {
340+
throw new Error("Retired generation remained alive after SIGKILL.");
341+
}
342+
}
343+
}
344+
if (this.#record) {
345+
this.#record = {
346+
...this.#record,
347+
retired: this.#record.retired.filter(
348+
(row) => row.procId !== identity.procId
349+
),
350+
};
351+
await this.#save();
352+
}
353+
}
354+
}

0 commit comments

Comments
 (0)