Skip to content

Commit a4e0097

Browse files
committed
fix: reconcile dead-PID jobs on every status read
Adds isProcessAlive helper and reconcileIfDead / reconcileAllJobs / markDeadPidJobFailed in job-control. buildStatusSnapshot and the handleResult/handleCancel paths now probe kill(pid, 0) on any active-state job and rewrite dead ones to failed before consuming the list. A single /opencode:status / result / cancel surfaces stuck workers without waiting for SessionEnd. markDeadPidJobFailed is race-safe: it re-reads state and refuses to downgrade terminal states or rewrite when the pid has changed. Closes #42. Port of openai/codex-plugin-cc#176 + dead-PID parts of #184.
1 parent cad21e8 commit a4e0097

4 files changed

Lines changed: 334 additions & 5 deletions

File tree

‎plugins/opencode/scripts/lib/job-control.mjs‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,89 @@
11
// Job control: query, sort, enrich, and build status snapshots.
22

33
import { tailLines } from "./fs.mjs";
4-
import { jobLogPath } from "./state.mjs";
4+
import { jobLogPath, loadState, upsertJob } from "./state.mjs";
5+
import { isProcessAlive } from "./process.mjs";
6+
7+
function isActiveJobStatus(status) {
8+
return status === "queued" || status === "running";
9+
}
10+
11+
/**
12+
* Mark a job as failed because its tracked PID is no longer alive. Re-reads
13+
* the latest persisted state before writing to guard against a legitimate
14+
* completion racing the probe.
15+
*
16+
* @param {string} workspacePath
17+
* @param {string} jobId
18+
* @param {number} pid - the pid we observed as dead
19+
* @returns {boolean} true if a write happened
20+
*/
21+
export function markDeadPidJobFailed(workspacePath, jobId, pid) {
22+
const latest = loadState(workspacePath).jobs?.find((j) => j.id === jobId);
23+
if (!latest) return false;
24+
25+
// Only overwrite active states; never downgrade terminal states.
26+
if (!isActiveJobStatus(latest.status)) return false;
27+
28+
// Only overwrite if the PID still matches what we observed as dead. Guards
29+
// against a job that legitimately restarted with a new PID between the
30+
// probe and the write.
31+
if (latest.pid !== pid) return false;
32+
33+
upsertJob(workspacePath, {
34+
id: jobId,
35+
status: "failed",
36+
phase: "failed",
37+
pid: null,
38+
errorMessage: `Tracked process PID ${pid} exited unexpectedly without writing a terminal status.`,
39+
completedAt: new Date().toISOString(),
40+
});
41+
return true;
42+
}
43+
44+
/**
45+
* If a job is still marked active but its tracked PID is dead, reconcile it
46+
* to failed and return the updated record. Otherwise return the original.
47+
*
48+
* Called from every status read path so a single status query is enough to
49+
* surface dead workers — no need to wait for SessionEnd.
50+
*
51+
* @param {string} workspacePath
52+
* @param {object} job
53+
* @returns {object}
54+
*/
55+
export function reconcileIfDead(workspacePath, job) {
56+
if (!job || !isActiveJobStatus(job.status)) return job;
57+
const pid = Number.isFinite(job.pid) ? job.pid : null;
58+
if (pid === null) return job;
59+
if (isProcessAlive(pid)) return job;
60+
61+
try {
62+
markDeadPidJobFailed(workspacePath, job.id, pid);
63+
} catch {
64+
// Never let reconciliation errors crash a status read.
65+
return job;
66+
}
67+
68+
const latest = loadState(workspacePath).jobs?.find((j) => j.id === job.id);
69+
return latest ?? job;
70+
}
71+
72+
/**
73+
* Reconcile all active jobs in the given list against live PIDs.
74+
* Returns a new list where dead-PID jobs have been rewritten to failed.
75+
*
76+
* Cheap shortcut used by handlers that otherwise operate on pure job arrays
77+
* (handleCancel, handleResult) so a single call surfaces dead workers.
78+
*
79+
* @param {object[]} jobs
80+
* @param {string} workspacePath
81+
* @returns {object[]}
82+
*/
83+
export function reconcileAllJobs(jobs, workspacePath) {
84+
if (!Array.isArray(jobs) || jobs.length === 0) return jobs;
85+
return jobs.map((j) => reconcileIfDead(workspacePath, j));
86+
}
587

688
/**
789
* Sort jobs newest first by updatedAt.
@@ -83,7 +165,10 @@ export function buildStatusSnapshot(jobs, workspacePath, opts = {}) {
83165
}
84166

85167
const sorted = sortJobsNewestFirst(filtered);
86-
const enriched = sorted.map((j) => enrichJob(j, workspacePath));
168+
// Reconcile any active jobs whose tracked PID is dead before enriching, so
169+
// a single status read surfaces stuck workers immediately.
170+
const reconciled = sorted.map((j) => reconcileIfDead(workspacePath, j));
171+
const enriched = reconciled.map((j) => enrichJob(j, workspacePath));
87172

88173
const running = enriched.filter((j) => j.status === "running");
89174
const finished = enriched.filter((j) => j.status !== "running");

‎plugins/opencode/scripts/lib/process.mjs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,23 @@ export function spawnDetached(cmd, args, opts = {}) {
9090
return child;
9191
}
9292

93+
/**
94+
* Check whether a process is still alive. Uses signal 0 which does not
95+
* affect the process — only probes existence.
96+
* @param {number | null | undefined} pid
97+
* @returns {boolean}
98+
*/
99+
export function isProcessAlive(pid) {
100+
if (pid == null || !Number.isFinite(pid) || pid <= 0) return false;
101+
try {
102+
process.kill(pid, 0);
103+
return true;
104+
} catch (err) {
105+
// ESRCH = dead. EPERM = exists but no permission.
106+
return err?.code === "EPERM";
107+
}
108+
}
109+
93110
// ------------------------------------------------------------------
94111
// OpenCode auth.json discovery
95112
// ------------------------------------------------------------------

‎plugins/opencode/scripts/opencode-companion.mjs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import { isOpencodeInstalled, getOpencodeVersion, spawnDetached, getConfiguredPr
4444
import { isServerRunning, ensureServer, createClient, connect } from "./lib/opencode-server.mjs";
4545
import { resolveWorkspace } from "./lib/workspace.mjs";
4646
import { loadState, updateState, upsertJob, generateJobId, jobDataPath } from "./lib/state.mjs";
47-
import { buildStatusSnapshot, resolveResultJob, resolveCancelableJob, enrichJob } from "./lib/job-control.mjs";
47+
import { buildStatusSnapshot, resolveResultJob, resolveCancelableJob, enrichJob, reconcileAllJobs } from "./lib/job-control.mjs";
4848
import { createJobRecord, runTrackedJob, getClaudeSessionId } from "./lib/tracked-jobs.mjs";
4949
import {
5050
renderStatus,
@@ -521,8 +521,9 @@ async function handleResult(argv) {
521521

522522
const workspace = await resolveWorkspace();
523523
const state = loadState(workspace);
524+
const reconciled = reconcileAllJobs(state.jobs ?? [], workspace);
524525

525-
const { job, ambiguous } = resolveResultJob(state.jobs ?? [], ref);
526+
const { job, ambiguous } = resolveResultJob(reconciled, ref);
526527

527528
if (ambiguous) {
528529
console.error("Ambiguous job reference. Please provide a more specific ID prefix.");
@@ -550,9 +551,10 @@ async function handleCancel(argv) {
550551
const workspace = await resolveWorkspace();
551552
const state = loadState(workspace);
552553
const sessionId = getClaudeSessionId();
554+
const reconciled = reconcileAllJobs(state.jobs ?? [], workspace);
553555

554556
const { job, ambiguous, sessionScoped } = resolveCancelableJob(
555-
state.jobs ?? [],
557+
reconciled,
556558
ref,
557559
{ sessionId }
558560
);

‎tests/dead-pid-reconcile.test.mjs‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
import { describe, it, before, after, beforeEach } from "node:test";
2+
import assert from "node:assert/strict";
3+
import fs from "node:fs";
4+
import os from "node:os";
5+
import path from "node:path";
6+
import {
7+
markDeadPidJobFailed,
8+
reconcileIfDead,
9+
reconcileAllJobs,
10+
buildStatusSnapshot,
11+
} from "../plugins/opencode/scripts/lib/job-control.mjs";
12+
import { upsertJob, loadState } from "../plugins/opencode/scripts/lib/state.mjs";
13+
import { isProcessAlive } from "../plugins/opencode/scripts/lib/process.mjs";
14+
15+
// PID 999999 is virtually guaranteed to be dead on macOS/Linux (well above
16+
// pid_max for typical configurations and any short-lived workload).
17+
const DEAD_PID = 999_999;
18+
19+
describe("isProcessAlive", () => {
20+
it("returns false for null/undefined/invalid pids", () => {
21+
assert.equal(isProcessAlive(null), false);
22+
assert.equal(isProcessAlive(undefined), false);
23+
assert.equal(isProcessAlive(0), false);
24+
assert.equal(isProcessAlive(-1), false);
25+
assert.equal(isProcessAlive(NaN), false);
26+
});
27+
28+
it("returns true for the current process", () => {
29+
assert.equal(isProcessAlive(process.pid), true);
30+
});
31+
32+
it("returns false for a clearly dead pid", () => {
33+
assert.equal(isProcessAlive(DEAD_PID), false);
34+
});
35+
});
36+
37+
describe("markDeadPidJobFailed", () => {
38+
let workspace;
39+
let previousPluginData;
40+
41+
before(() => {
42+
workspace = fs.mkdtempSync(path.join(os.tmpdir(), "opencode-deadpid-"));
43+
previousPluginData = process.env.CLAUDE_PLUGIN_DATA;
44+
process.env.CLAUDE_PLUGIN_DATA = fs.mkdtempSync(
45+
path.join(os.tmpdir(), "opencode-deadpid-data-")
46+
);
47+
});
48+
49+
after(() => {
50+
if (previousPluginData == null) {
51+
delete process.env.CLAUDE_PLUGIN_DATA;
52+
} else {
53+
process.env.CLAUDE_PLUGIN_DATA = previousPluginData;
54+
}
55+
});
56+
57+
beforeEach(() => {
58+
// Wipe state before each test.
59+
const state = loadState(workspace);
60+
state.jobs = [];
61+
upsertJob(workspace, { id: "__seed__", status: "completed" });
62+
const after = loadState(workspace);
63+
after.jobs = [];
64+
for (const j of after.jobs) {}
65+
});
66+
67+
it("rewrites a running job with a dead pid to failed", () => {
68+
upsertJob(workspace, {
69+
id: "task-1",
70+
status: "running",
71+
type: "task",
72+
pid: DEAD_PID,
73+
});
74+
const written = markDeadPidJobFailed(workspace, "task-1", DEAD_PID);
75+
assert.equal(written, true);
76+
77+
const stored = loadState(workspace).jobs.find((j) => j.id === "task-1");
78+
assert.equal(stored.status, "failed");
79+
assert.equal(stored.phase, "failed");
80+
assert.equal(stored.pid, null);
81+
assert.match(stored.errorMessage, new RegExp(`PID ${DEAD_PID}`));
82+
});
83+
84+
it("refuses to downgrade a terminal state", () => {
85+
upsertJob(workspace, {
86+
id: "task-2",
87+
status: "completed",
88+
type: "task",
89+
pid: DEAD_PID,
90+
});
91+
const written = markDeadPidJobFailed(workspace, "task-2", DEAD_PID);
92+
assert.equal(written, false);
93+
94+
const stored = loadState(workspace).jobs.find((j) => j.id === "task-2");
95+
assert.equal(stored.status, "completed");
96+
});
97+
98+
it("refuses when the pid has changed since probe", () => {
99+
upsertJob(workspace, {
100+
id: "task-3",
101+
status: "running",
102+
type: "task",
103+
pid: 12345,
104+
});
105+
// Probe observed DEAD_PID, but latest pid is 12345 → refuse.
106+
const written = markDeadPidJobFailed(workspace, "task-3", DEAD_PID);
107+
assert.equal(written, false);
108+
109+
const stored = loadState(workspace).jobs.find((j) => j.id === "task-3");
110+
assert.equal(stored.status, "running");
111+
});
112+
});
113+
114+
describe("reconcileIfDead", () => {
115+
let workspace;
116+
let previousPluginData;
117+
118+
before(() => {
119+
workspace = fs.mkdtempSync(path.join(os.tmpdir(), "opencode-reconcile-"));
120+
previousPluginData = process.env.CLAUDE_PLUGIN_DATA;
121+
process.env.CLAUDE_PLUGIN_DATA = fs.mkdtempSync(
122+
path.join(os.tmpdir(), "opencode-reconcile-data-")
123+
);
124+
});
125+
126+
after(() => {
127+
if (previousPluginData == null) {
128+
delete process.env.CLAUDE_PLUGIN_DATA;
129+
} else {
130+
process.env.CLAUDE_PLUGIN_DATA = previousPluginData;
131+
}
132+
});
133+
134+
it("leaves a running job with a live pid alone", () => {
135+
upsertJob(workspace, {
136+
id: "live-1",
137+
status: "running",
138+
type: "task",
139+
pid: process.pid,
140+
});
141+
const result = reconcileIfDead(workspace, {
142+
id: "live-1",
143+
status: "running",
144+
pid: process.pid,
145+
});
146+
assert.equal(result.status, "running");
147+
});
148+
149+
it("reconciles a running job with a dead pid to failed", () => {
150+
upsertJob(workspace, {
151+
id: "dead-1",
152+
status: "running",
153+
type: "task",
154+
pid: DEAD_PID,
155+
});
156+
const result = reconcileIfDead(workspace, {
157+
id: "dead-1",
158+
status: "running",
159+
pid: DEAD_PID,
160+
});
161+
assert.equal(result.status, "failed");
162+
assert.equal(result.phase, "failed");
163+
});
164+
165+
it("leaves running jobs with no pid alone", () => {
166+
const job = { id: "nopid", status: "running", pid: null };
167+
const result = reconcileIfDead(workspace, job);
168+
assert.equal(result, job);
169+
});
170+
171+
it("leaves terminal-state jobs alone", () => {
172+
const job = { id: "done", status: "completed", pid: DEAD_PID };
173+
const result = reconcileIfDead(workspace, job);
174+
assert.equal(result, job);
175+
});
176+
});
177+
178+
describe("buildStatusSnapshot reconciles dead pids inline", () => {
179+
let workspace;
180+
let previousPluginData;
181+
182+
before(() => {
183+
workspace = fs.mkdtempSync(path.join(os.tmpdir(), "opencode-snapshot-"));
184+
previousPluginData = process.env.CLAUDE_PLUGIN_DATA;
185+
process.env.CLAUDE_PLUGIN_DATA = fs.mkdtempSync(
186+
path.join(os.tmpdir(), "opencode-snapshot-data-")
187+
);
188+
});
189+
190+
after(() => {
191+
if (previousPluginData == null) {
192+
delete process.env.CLAUDE_PLUGIN_DATA;
193+
} else {
194+
process.env.CLAUDE_PLUGIN_DATA = previousPluginData;
195+
}
196+
});
197+
198+
it("surfaces a dead-pid running job as failed in one read", () => {
199+
const now = new Date().toISOString();
200+
const jobs = [
201+
{
202+
id: "zombie",
203+
status: "running",
204+
type: "task",
205+
pid: DEAD_PID,
206+
createdAt: now,
207+
updatedAt: now,
208+
},
209+
];
210+
upsertJob(workspace, jobs[0]);
211+
212+
const snapshot = buildStatusSnapshot(jobs, workspace);
213+
assert.equal(snapshot.running.length, 0);
214+
assert.ok(snapshot.latestFinished);
215+
assert.equal(snapshot.latestFinished.id, "zombie");
216+
assert.equal(snapshot.latestFinished.status, "failed");
217+
});
218+
});
219+
220+
describe("reconcileAllJobs", () => {
221+
it("returns input unchanged when empty", () => {
222+
assert.deepEqual(reconcileAllJobs([], "/tmp"), []);
223+
assert.equal(reconcileAllJobs(null, "/tmp"), null);
224+
});
225+
});

0 commit comments

Comments
 (0)