Skip to content

Commit 3305da8

Browse files
EivMeyerclaude
andcommitted
fix: prevent infinite hang when provider CLI hangs (opencode 429 bug)
Problem: - When opencode (or any provider CLI) hangs due to API errors like 429, zeroshot workers would wait forever because exec() had no timeout - The consecutive failure counter only increments when callback fires, but a hung process = callback never fires = infinite wait Solution: 1. Created src/lib/safe-exec.js wrapper that enforces timeouts (default 30s) 2. Updated agent-task-executor.js to use safe-exec 3. Updated isolation-manager.js to use safe-exec + added timeout to execInContainer() 4. Added ESLint rule to prevent direct child_process exec/execSync usage Now if a provider hangs: - 5s timeout kills the process - Callback fires with timeout error - Worker fails gracefully instead of freezing Root cause in opencode: anomalyco/opencode#8203 Co-Authored-By: Claude Opus 4.5 <[email protected]>
1 parent 7139ad5 commit 3305da8

10 files changed

Lines changed: 220 additions & 41 deletions

‎eslint.config.js‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,24 @@ export default [
7777
'no-restricted-syntax': [
7878
'error',
7979
{
80-
selector: "LogicalExpression[operator='||'][right.value=/localhost|127\\.0\\.0|^\\d{4,5}$/]",
80+
selector:
81+
"LogicalExpression[operator='||'][right.value=/localhost|127\\.0\\.0|^\\d{4,5}$/]",
8182
message: 'FORBIDDEN: Dangerous fallback. Throw error if missing instead.',
8283
},
84+
{
85+
// Catches: const { exec } = require('child_process')
86+
selector:
87+
"VariableDeclarator[init.callee.name='require'][init.arguments.0.value='child_process'] > ObjectPattern > Property[key.name='exec']",
88+
message:
89+
"FORBIDDEN: Direct exec() from child_process can hang forever. Use require('./lib/safe-exec') instead.",
90+
},
91+
{
92+
// Catches: const { execSync } = require('child_process')
93+
selector:
94+
"VariableDeclarator[init.callee.name='require'][init.arguments.0.value='child_process'] > ObjectPattern > Property[key.name='execSync']",
95+
message:
96+
"FORBIDDEN: Direct execSync() from child_process can hang forever. Use require('./lib/safe-exec') instead.",
97+
},
8398
],
8499

85100
// Style (keep existing)
@@ -100,6 +115,25 @@ export default [
100115
},
101116
},
102117
},
118+
{
119+
// Allow direct child_process in safe-exec wrapper, tests, CLI, and lib/
120+
// - safe-exec.js: IS the wrapper
121+
// - tests/: Need to test subprocess behavior directly
122+
// - cli/: Has its own timeout handling for user-facing commands
123+
// - lib/: Utility code with specific timeout requirements
124+
files: ['src/lib/safe-exec.js', 'tests/**/*.js', 'cli/**/*.js', 'lib/**/*.js'],
125+
rules: {
126+
'no-restricted-syntax': [
127+
'error',
128+
// Keep the dangerous fallback rule, just disable the exec rule
129+
{
130+
selector:
131+
"LogicalExpression[operator='||'][right.value=/localhost|127\\.0\\.0|^\\d{4,5}$/]",
132+
message: 'FORBIDDEN: Dangerous fallback. Throw error if missing instead.',
133+
},
134+
],
135+
},
136+
},
103137
{
104138
ignores: ['node_modules/**', 'dist/**', 'coverage/**', 'cluster-hooks/**', 'hooks/**'],
105139
},

‎src/agent/agent-hook-executor.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
*/
99

1010
const vm = require('vm');
11+
const { execSync } = require('../lib/safe-exec'); // Enforces timeouts
1112

1213
/**
1314
* Execute a hook
@@ -278,7 +279,6 @@ async function substituteTemplate(params) {
278279
if (taskId !== 'UNKNOWN') {
279280
console.error(`\nFetching task logs for ${taskId}...`);
280281
try {
281-
const { execSync } = require('child_process');
282282
const ctPath = agent._getClaudeTasksPath();
283283
taskLogs = execSync(`${ctPath} logs ${taskId} --lines 100`, {
284284
encoding: 'utf-8',

‎src/agent/agent-stuck-detector.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
* Threshold: stuckScore >= 3.5 = likely stuck
2424
*/
2525

26-
const { execSync } = require('child_process');
26+
const { execSync } = require('../lib/safe-exec'); // Enforces timeouts
2727
const fs = require('fs');
2828

2929
// Stuck detection thresholds

‎src/agent/agent-task-executor.js‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ const { spawn } = require('child_process');
1414
const path = require('path');
1515
const fs = require('fs');
1616
const os = require('os');
17+
const { exec, execSync } = require('../lib/safe-exec'); // Enforces timeouts - prevents infinite hangs
1718
const { getProvider, parseChunkWithProvider } = require('../providers');
1819

1920
// Schema utilities for normalizing LLM output
@@ -532,16 +533,16 @@ async function spawnClaudeTask(agent, context) {
532533
* @returns {Promise<void>}
533534
*/
534535
async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
535-
const { exec } = require('child_process');
536536
const ctPath = getClaudeTasksPath();
537537

538538
for (let i = 0; i < maxRetries; i++) {
539-
const exists = await new Promise((resolve) => {
540-
exec(`${ctPath} status ${taskId}`, (error, stdout) => {
541-
// Task exists if status doesn't return "Task not found"
542-
resolve(!error && !stdout.includes('Task not found'));
543-
});
544-
});
539+
let exists = false;
540+
try {
541+
const { stdout } = await exec(`${ctPath} status ${taskId}`, { timeout: 5000 });
542+
exists = !stdout.includes('Task not found');
543+
} catch {
544+
// Timeout or error - task not ready yet
545+
}
545546

546547
if (exists) return;
547548

@@ -562,7 +563,6 @@ async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
562563
*/
563564
function followClaudeTaskLogs(agent, taskId) {
564565
const fsModule = require('fs');
565-
const { execSync, exec } = require('child_process');
566566
const ctPath = getClaudeTasksPath();
567567
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
568568

@@ -578,6 +578,7 @@ function followClaudeTaskLogs(agent, taskId) {
578578
try {
579579
logFilePath = execSync(`${ctPath} get-log-path ${taskId}`, {
580580
encoding: 'utf-8',
581+
timeout: 5000,
581582
}).trim();
582583
agent._log(`📋 Agent ${agent.id}: Following ct logs for ${taskId}`);
583584
} catch {
@@ -675,6 +676,7 @@ function followClaudeTaskLogs(agent, taskId) {
675676
try {
676677
logFilePath = execSync(`${ctPath} get-log-path ${taskId}`, {
677678
encoding: 'utf-8',
679+
timeout: 5000,
678680
}).trim();
679681
agent._log(`📋 Agent ${agent.id}: Found log file: ${logFilePath}`);
680682
} catch {
@@ -718,7 +720,7 @@ function followClaudeTaskLogs(agent, taskId) {
718720
const MAX_CONSECUTIVE_FAILURES = 30; // 30 seconds of failures = log warning
719721

720722
statusCheckInterval = setInterval(() => {
721-
exec(`${ctPath} status ${taskId}`, (error, stdout, stderr) => {
723+
exec(`${ctPath} status ${taskId}`, { timeout: 5000 }, (error, stdout, stderr) => {
722724
if (resolved) return;
723725

724726
// Track exec failures - if status command keeps failing, something is wrong
@@ -1405,9 +1407,8 @@ function killTask(agent) {
14051407
// Also kill the underlying zeroshot task if we have a task ID
14061408
// This ensures the task process is stopped, not just our polling intervals
14071409
if (agent.currentTaskId) {
1408-
const { exec } = require('child_process');
14091410
const ctPath = getClaudeTasksPath();
1410-
exec(`${ctPath} task kill ${agent.currentTaskId}`, (error) => {
1411+
exec(`${ctPath} task kill ${agent.currentTaskId}`, { timeout: 10000 }, (error) => {
14111412
if (error) {
14121413
// Task may have already completed or been killed, ignore errors
14131414
agent._log(`Note: Could not kill task ${agent.currentTaskId}: ${error.message}`);

‎src/claude-task-runner.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
* following logs, and assembling results.
66
*/
77

8-
const { spawn, exec, execSync } = require('child_process');
8+
const { spawn } = require('child_process');
9+
const { exec, execSync } = require('./lib/safe-exec'); // Enforces timeouts
910
const fs = require('fs');
1011
const TaskRunner = require('./task-runner');
1112
const { loadSettings } = require('../lib/settings');

‎src/github.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* - Fallback to plain text input
88
*/
99

10-
const { execSync } = require('child_process');
10+
const { execSync } = require('./lib/safe-exec'); // Enforces timeouts
1111
const fs = require('fs');
1212
const path = require('path');
1313

‎src/isolation-manager.js‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
* - Container cleanup on stop/kill
99
*/
1010

11-
const { spawn, execSync } = require('child_process');
11+
const { spawn } = require('child_process');
12+
const { execSync } = require('./lib/safe-exec'); // Enforces timeouts - prevents infinite hangs
1213
const { Worker } = require('worker_threads');
1314
const crypto = require('crypto');
1415
const path = require('path');
@@ -387,6 +388,7 @@ class IsolationManager {
387388
* @param {object} [options] - Exec options
388389
* @param {boolean} [options.interactive] - Use -it flags
389390
* @param {object} [options.env] - Environment variables
391+
* @param {number} [options.timeout=30000] - Timeout in ms (0 = no timeout). Prevents infinite hangs.
390392
* @returns {Promise<{stdout: string, stderr: string, code: number}>}
391393
*/
392394
execInContainer(clusterId, command, options = {}) {
@@ -410,13 +412,26 @@ class IsolationManager {
410412

411413
args.push(containerId, ...command);
412414

415+
// Default timeout: 30 seconds (prevents infinite hangs)
416+
const timeout = options.timeout ?? 30000;
417+
413418
return new Promise((resolve, reject) => {
414419
const proc = spawn('docker', args, {
415420
stdio: options.interactive ? 'inherit' : ['pipe', 'pipe', 'pipe'],
416421
});
417422

418423
let stdout = '';
419424
let stderr = '';
425+
let timedOut = false;
426+
let timeoutId = null;
427+
428+
// Set up timeout if specified (0 = no timeout)
429+
if (timeout > 0) {
430+
timeoutId = setTimeout(() => {
431+
timedOut = true;
432+
proc.kill('SIGKILL');
433+
}, timeout);
434+
}
420435

421436
if (!options.interactive) {
422437
proc.stdout.on('data', (data) => {
@@ -428,10 +443,16 @@ class IsolationManager {
428443
}
429444

430445
proc.on('close', (code) => {
431-
resolve({ stdout, stderr, code });
446+
if (timeoutId) clearTimeout(timeoutId);
447+
if (timedOut) {
448+
reject(new Error(`Docker exec timed out after ${timeout}ms`));
449+
} else {
450+
resolve({ stdout, stderr, code });
451+
}
432452
});
433453

434454
proc.on('error', (err) => {
455+
if (timeoutId) clearTimeout(timeoutId);
435456
reject(new Error(`Docker exec error: ${err.message}`));
436457
});
437458
});

‎src/lib/safe-exec.js‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/**
2+
* Safe subprocess execution with mandatory timeouts.
3+
*
4+
* NEVER use child_process.exec() or execSync() directly.
5+
* These wrappers enforce timeouts to prevent infinite hangs.
6+
*/
7+
8+
const { exec: nodeExec, execSync: nodeExecSync } = require('child_process');
9+
10+
/** Default timeout: 30 seconds */
11+
const DEFAULT_TIMEOUT_MS = 30000;
12+
13+
/**
14+
* Execute command with mandatory timeout.
15+
* Supports both Promise and callback styles for gradual migration.
16+
*
17+
* @param {string} command - Command to execute
18+
* @param {object} [options] - Options (timeout uses default if not specified)
19+
* @param {function} [callback] - Optional callback(error, stdout, stderr)
20+
* @returns {Promise<{stdout: string, stderr: string}>|void} Promise if no callback, void if callback
21+
*
22+
* @example
23+
* // Promise style (preferred)
24+
* const { stdout } = await exec('ls -la', { timeout: 5000 });
25+
*
26+
* @example
27+
* // Callback style (for legacy code migration)
28+
* exec('ls -la', { timeout: 5000 }, (error, stdout) => { ... });
29+
*/
30+
function exec(command, options = {}, callback = null) {
31+
// Handle overloaded signature: exec(cmd, callback)
32+
if (typeof options === 'function') {
33+
callback = options;
34+
options = {};
35+
}
36+
37+
const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
38+
39+
if (timeout <= 0) {
40+
const err = new Error('exec() timeout must be > 0. Infinite waits are forbidden.');
41+
if (callback) {
42+
callback(err);
43+
return;
44+
}
45+
return Promise.reject(err);
46+
}
47+
48+
// Callback style
49+
if (callback) {
50+
nodeExec(command, { ...options, timeout }, (error, stdout, stderr) => {
51+
if (error && error.killed && error.signal === 'SIGTERM') {
52+
error.message = `Command timed out after ${timeout}ms: ${command}`;
53+
}
54+
callback(error, stdout, stderr);
55+
});
56+
return;
57+
}
58+
59+
// Promise style
60+
return new Promise((resolve, reject) => {
61+
nodeExec(command, { ...options, timeout }, (error, stdout, stderr) => {
62+
if (error) {
63+
if (error.killed && error.signal === 'SIGTERM') {
64+
error.message = `Command timed out after ${timeout}ms: ${command}`;
65+
}
66+
reject(error);
67+
} else {
68+
resolve({ stdout, stderr });
69+
}
70+
});
71+
});
72+
}
73+
74+
/**
75+
* Execute command with mandatory timeout (sync)
76+
* @param {string} command - Command to execute
77+
* @param {object} [options] - Options (timeout required or uses default)
78+
* @returns {string} stdout
79+
*/
80+
function execSync(command, options = {}) {
81+
const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
82+
83+
if (timeout <= 0) {
84+
throw new Error('execSync() timeout must be > 0. Infinite waits are forbidden.');
85+
}
86+
87+
return nodeExecSync(command, { ...options, timeout });
88+
}
89+
90+
module.exports = { exec, execSync, DEFAULT_TIMEOUT_MS };

‎src/preflight.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* Provides CLEAR, ACTIONABLE error messages with recovery instructions.
1010
*/
1111

12-
const { execSync } = require('child_process');
12+
const { execSync } = require('./lib/safe-exec'); // Enforces timeouts
1313
const path = require('path');
1414
const fs = require('fs');
1515
const os = require('os');

0 commit comments

Comments
 (0)