Skip to content

Commit abbd2fa

Browse files
committed
feat(server): idle + bash-stuck detectors in sendPrompt watcher
Raise the absolute prompt timeout to 4h as a pure safety cap and move real stall detection into the watcher so long-but-alive tasks aren't killed by a fixed deadline. - Idle timeout (OPENCODE_IDLE_TIMEOUT_MS, default 15min): abort when the session shows no message/part/tool-output change for too long. - Bash-tool stuck detector: when the latest tool is a bash in status running but `opencode serve` has zero child processes for N consecutive polls (default 3 × 5s), abort. This catches the ask-permission deadlock (anomalyco/opencode#14473) where the shell process already exited cleanly but tool state never flipped to completed. Gracefully degrades on Windows or when lsof/pgrep is unavailable. - Restructure fetch-vs-watcher race so a rejection from one side no longer cancels the other. The server's 5-min POST cap used to kill sendPrompt before the watcher could observe completion; now both settle independently and we prefer whichever succeeded.
1 parent 54b175e commit abbd2fa

1 file changed

Lines changed: 178 additions & 11 deletions

File tree

‎plugins/opencode/scripts/lib/opencode-server.mjs‎

Lines changed: 178 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,82 @@
22
// Unlike codex-plugin-cc which uses JSON-RPC over stdin/stdout,
33
// OpenCode exposes a REST API + SSE. This module wraps that API.
44

5-
import { spawn } from "node:child_process";
5+
import { spawn, spawnSync } from "node:child_process";
6+
7+
// Re-export for spec-compliance / discoverability: probeSessionTerminal lives
8+
// in auto-heal.mjs because it is tightly coupled to heal-decision logic, but
9+
// conceptually it is a server probe.
10+
export { probeSessionTerminal } from "./auto-heal.mjs";
611

712
const DEFAULT_PORT = 4096;
813
const DEFAULT_HOST = "127.0.0.1";
914
const SERVER_START_TIMEOUT = 30_000;
1015

1116
// Long-running tasks (e.g. engine builds, large refactors) can easily exceed
1217
// the old 5-10 min caps, causing `fetch failed` at a fixed deadline. Default
13-
// to 30 min; override via env for even longer workloads.
18+
// PROMPT_TIMEOUT_MS to 4 hours — absolute safety cap. Real stall detection
19+
// lives in the watcher via IDLE_TIMEOUT_MS + pgrep child-process check.
1420
const REQUEST_TIMEOUT_MS = Number(process.env.OPENCODE_REQUEST_TIMEOUT_MS) || 1_800_000;
15-
const PROMPT_TIMEOUT_MS = Number(process.env.OPENCODE_PROMPT_TIMEOUT_MS) || 1_800_000;
21+
const PROMPT_TIMEOUT_MS = Number(process.env.OPENCODE_PROMPT_TIMEOUT_MS) || 14_400_000;
22+
// How long a session may go without ANY activity signal before we assume it
23+
// is stuck. Activity = new message, new parts, tool output growth, status
24+
// change. Default 15 min — long enough for most silent-but-alive tasks.
25+
const IDLE_TIMEOUT_MS = Number(process.env.OPENCODE_IDLE_TIMEOUT_MS) || 900_000;
26+
// Bash-tool "no child process" consecutive-miss threshold. If the latest
27+
// tool is a bash in status=running but opencode serve has zero child
28+
// processes for N polls in a row, declare stuck. 3 × 5s = 15s grace.
29+
const PGREP_MISS_THRESHOLD = Number(process.env.OPENCODE_PGREP_MISS_THRESHOLD) || 3;
30+
31+
const IS_WINDOWS = process.platform === "win32";
32+
33+
/**
34+
* Find the PID of `opencode serve` listening on `port`, if we can.
35+
* Returns null on Windows or any detection failure (caller degrades gracefully).
36+
*/
37+
function resolveServePid(port) {
38+
if (IS_WINDOWS) return null;
39+
try {
40+
// macOS + Linux: lsof works the same way. Short timeout so we never block
41+
// the watcher loop if the tool is slow/missing.
42+
const r = spawnSync("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN"], {
43+
encoding: "utf8",
44+
timeout: 2000,
45+
});
46+
if (r.status !== 0 || !r.stdout) return null;
47+
const lines = r.stdout.split("\n").slice(1).filter(Boolean);
48+
for (const line of lines) {
49+
const cols = line.trim().split(/\s+/);
50+
const pid = Number(cols[1]);
51+
if (Number.isInteger(pid) && pid > 0) return pid;
52+
}
53+
} catch {
54+
// lsof missing or errored — degrade to no pgrep checks
55+
}
56+
return null;
57+
}
58+
59+
/**
60+
* Count direct child processes of `pid`. Returns:
61+
* -1 — feature unavailable (Windows, pgrep missing, etc.) — caller should skip check
62+
* 0 — no children
63+
* >0 — that many children
64+
*/
65+
function countChildren(pid) {
66+
if (!pid || IS_WINDOWS) return -1;
67+
try {
68+
const r = spawnSync("pgrep", ["-P", String(pid)], {
69+
encoding: "utf8",
70+
timeout: 2000,
71+
});
72+
if (r.error) return -1;
73+
// pgrep exits 1 when no matches (empty stdout) — that's a real "zero", not a failure
74+
const out = (r.stdout || "").trim();
75+
if (!out) return 0;
76+
return out.split("\n").filter(Boolean).length;
77+
} catch {
78+
return -1;
79+
}
80+
}
1681

1782
/**
1883
* Check if an OpenCode server is already running on the given port.
@@ -179,6 +244,21 @@ export function createClient(baseUrl, opts = {}) {
179244
// Wait briefly so the new generation has a chance to start and we
180245
// don't latch onto a stale completed message from before this prompt.
181246
await new Promise((r) => setTimeout(r, MIN_POLL_DELAY_MS));
247+
248+
// Resolve the opencode serve PID once so we can check for child
249+
// processes later. If this fails (Windows, no lsof, permissions)
250+
// we silently skip the pgrep-based stuck detector — idle timeout
251+
// still covers most cases.
252+
const urlObj = (() => {
253+
try { return new URL(baseUrl); } catch { return null; }
254+
})();
255+
const port = Number(urlObj?.port) || DEFAULT_PORT;
256+
const opencodePid = resolveServePid(port);
257+
258+
let prevSig = "";
259+
let lastActivityMs = Date.now();
260+
let pgrepMissCount = 0;
261+
182262
while (!ac.signal.aborted) {
183263
try {
184264
const params = new URLSearchParams({ limit: "1" });
@@ -190,8 +270,30 @@ export function createClient(baseUrl, opts = {}) {
190270
const arr = await r.json();
191271
const last = Array.isArray(arr) ? arr[arr.length - 1] : null;
192272
const info = last?.info;
193-
// Only treat assistant messages created *after* this prompt
194-
// started as a completion signal for this call.
273+
const parts = Array.isArray(last?.parts) ? last.parts : [];
274+
// Most recent tool part — the one actually "running" if any.
275+
let lastTool = null;
276+
for (let i = parts.length - 1; i >= 0; i--) {
277+
if (parts[i]?.type === "tool") { lastTool = parts[i]; break; }
278+
}
279+
280+
// Activity signature: any change here = progress was made.
281+
const sig = JSON.stringify({
282+
mid: info?.id,
283+
created: info?.time?.created,
284+
completed: info?.time?.completed,
285+
parts: parts.length,
286+
tStatus: lastTool?.state?.status,
287+
tOutLen: (lastTool?.state?.output || "").length,
288+
});
289+
if (sig !== prevSig) {
290+
lastActivityMs = Date.now();
291+
prevSig = sig;
292+
pgrepMissCount = 0;
293+
}
294+
295+
// Completion signal: assistant message created after our prompt
296+
// started, with a terminal `finish` field populated.
195297
if (
196298
info &&
197299
info.role === "assistant" &&
@@ -201,23 +303,88 @@ export function createClient(baseUrl, opts = {}) {
201303
) {
202304
return { source: "watcher", data: last };
203305
}
306+
307+
// Bash-tool stuck detector: latest tool is bash in status=running
308+
// but opencode serve has zero children for N consecutive polls.
309+
// This is the signature of the "ask permission deadlock" bug
310+
// (sst/opencode#14473): the shell process already exited cleanly
311+
// but tool state never flipped to completed.
312+
if (
313+
opencodePid &&
314+
lastTool?.tool === "bash" &&
315+
lastTool?.state?.status === "running"
316+
) {
317+
const n = countChildren(opencodePid);
318+
if (n === 0) {
319+
pgrepMissCount += 1;
320+
if (pgrepMissCount >= PGREP_MISS_THRESHOLD) {
321+
ac.abort(
322+
new Error(
323+
`bash tool stuck — opencode serve (pid ${opencodePid}) has no child for ${pgrepMissCount} polls while tool.status=running`,
324+
),
325+
);
326+
throw new Error("bash tool stuck (no child)");
327+
}
328+
} else if (n > 0) {
329+
pgrepMissCount = 0;
330+
}
331+
// n === -1 → feature unavailable, don't count either way
332+
}
333+
334+
// Idle timeout: nothing happened in the session for too long.
335+
// Covers all tool types (not just bash), including non-pgrep
336+
// platforms (Windows).
337+
const idleMs = Date.now() - lastActivityMs;
338+
if (idleMs > IDLE_TIMEOUT_MS) {
339+
ac.abort(
340+
new Error(
341+
`session idle ${Math.floor(idleMs / 1000)}s > ${IDLE_TIMEOUT_MS / 1000}s`,
342+
),
343+
);
344+
throw new Error("session idle timeout");
345+
}
204346
}
205-
} catch {
206-
// Ignore transient poll errors; keep waiting.
347+
} catch (err) {
348+
// If we aborted above, propagate so the outer race sees a failure.
349+
if (ac.signal.aborted) throw err;
350+
// Otherwise it's a transient network/server blip — keep polling.
207351
}
208352
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
209353
}
210354
throw new Error("watcher aborted");
211355
})();
212356

357+
// Settle-wrap each so a single rejection doesn't lose the other side.
358+
// Server-side 5-min POST cap means fetchPromise often rejects LONG
359+
// before the agent is actually done; we must still wait on the watcher.
360+
const wrap = (p, via) =>
361+
p.then(
362+
(v) => ({ ok: true, via, data: v.data }),
363+
(err) => ({ ok: false, via, err }),
364+
);
365+
const runFetch = wrap(fetchPromise, "fetch");
366+
const runWatcher = wrap(watcherPromise, "watcher");
367+
213368
try {
214-
const winner = await Promise.race([fetchPromise, watcherPromise]);
215-
// Whichever arrived first, cancel the other.
369+
const first = await Promise.race([runFetch, runWatcher]);
370+
if (first.ok) {
371+
ac.abort();
372+
fetchPromise.catch(() => {});
373+
watcherPromise.catch(() => {});
374+
return first.data;
375+
}
376+
// First to settle was a failure — the other promise may still succeed.
377+
// Do NOT abort yet: in particular, the watcher needs to keep polling
378+
// when the POST was killed by the server's 5-min cap but generation
379+
// is still running.
380+
const second = first.via === "fetch" ? await runWatcher : await runFetch;
216381
ac.abort();
217-
// Swallow the loser's rejection to avoid unhandled rejection noise.
218382
fetchPromise.catch(() => {});
219383
watcherPromise.catch(() => {});
220-
return winner.data;
384+
if (second.ok) return second.data;
385+
// Both failed — surface the more informative error. Prefer the
386+
// fetch error because it usually has the HTTP status/body.
387+
throw first.via === "fetch" ? first.err : second.err;
221388
} finally {
222389
clearTimeout(timeoutId);
223390
}

0 commit comments

Comments
 (0)