Skip to content

Commit 2d10049

Browse files
Bohan-JJmultica-agent
authored andcommitted
fix(agent): terminate opencode process group before closing stdout (multica-ai#4533) (multica-ai#4541)
On cancellation/timeout the opencode backend closed the stdout read end immediately, leaving the child writing into a closed pipe. Every write then returns EPIPE and, per anomalyco/opencode#33653, can spin an orphaned process at 100% CPU — surfacing as high idle CPU after a cancelled task or daemon restart (MUL-3655). Cleanup now runs opencode in its own process group and, on cancel, drives a graceful group-wide SIGTERM → grace → SIGKILL, closing the stdout pipe only as a last-resort unblock once the tree has been signalled (SIGKILL is uncatchable, so no member can write again — no EPIPE window). The group signal also reaps tool subprocesses opencode spawned instead of orphaning them. WaitDelay remains the hard backstop. Adds unix tests covering the graceful path and the SIGTERM-ignored → SIGKILL escalation, asserting the whole process group is reaped and the run never deadlocks on the scanner. Windows behaviour is unchanged (no process groups). Co-authored-by: J <[email protected]> Co-authored-by: multica-agent <[email protected]>
1 parent ccdca87 commit 2d10049

4 files changed

Lines changed: 263 additions & 4 deletions

File tree

‎server/pkg/agent/opencode.go‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,26 @@ import (
1111
"path/filepath"
1212
"runtime"
1313
"strings"
14+
"sync/atomic"
15+
"syscall"
1416
"time"
1517
)
1618

19+
// opencodeTerminateGraceNanos optionally overrides, in nanoseconds, how long a
20+
// cancelled opencode process is given to exit after SIGTERM before it (and its
21+
// whole process group) is SIGKILLed. Zero means use the default. It is atomic
22+
// so tests can shorten the grace without racing the cancellation goroutine that
23+
// reads it. See the cancellation handler in Execute for why termination must
24+
// precede closing the stdout pipe (#4533).
25+
var opencodeTerminateGraceNanos atomic.Int64
26+
27+
func opencodeTerminateGrace() time.Duration {
28+
if n := opencodeTerminateGraceNanos.Load(); n > 0 {
29+
return time.Duration(n)
30+
}
31+
return 5 * time.Second
32+
}
33+
1734
// opencodeBlockedArgs are flags hardcoded by the daemon that must not be
1835
// overridden by user-configured custom_args.
1936
var opencodeBlockedArgs = map[string]blockedArgMode{
@@ -82,6 +99,18 @@ func (b *opencodeBackend) Execute(ctx context.Context, prompt string, opts ExecO
8299

83100
cmd := exec.CommandContext(runCtx, execPath, args...)
84101
hideAgentWindow(cmd)
102+
// Run opencode in its own process group so cancellation can reach the
103+
// whole tree (opencode plus any tool subprocess it spawns), not just the
104+
// direct child — otherwise a cancelled or restarted run can orphan a
105+
// descendant that keeps spinning (#4533).
106+
configureProcessGroup(cmd)
107+
// Take over context cancellation. The default CommandContext behaviour
108+
// SIGKILLs only the leader the instant runCtx is done; we instead drive a
109+
// graceful, group-wide SIGTERM→SIGKILL from the cancellation goroutine
110+
// below and close the stdout read end only after the tree has been
111+
// signalled. Returning nil here keeps os/exec from racing us with its own
112+
// kill; WaitDelay remains the hard backstop.
113+
cmd.Cancel = func() error { return nil }
85114
b.cfg.Logger.Info("agent command", "exec", execPath, "args", args)
86115
cmd.WaitDelay = 10 * time.Second
87116
if opts.Cwd != "" {
@@ -145,9 +174,34 @@ func (b *opencodeBackend) Execute(ctx context.Context, prompt string, opts ExecO
145174
msgCh := make(chan Message, 256)
146175
resCh := make(chan Result, 1)
147176

148-
// Close stdout when the context is cancelled so the scanner unblocks.
177+
// procDone closes once cmd.Wait() returns, letting the cancellation handler
178+
// skip a process that already exited and avoid signalling a dead pid.
179+
procDone := make(chan struct{})
180+
181+
// On cancellation / timeout, terminate opencode (and the tool subprocesses
182+
// it spawned) BEFORE unblocking the scanner. The previous implementation
183+
// closed the stdout read end immediately, which left opencode writing into
184+
// a closed pipe: every write returns EPIPE and, per anomalyco/opencode#33653,
185+
// can spin the orphaned process at 100% CPU. Instead we SIGTERM the whole
186+
// process group, give it a grace period to exit cleanly, then SIGKILL it.
187+
// SIGKILL is uncatchable, so once it is delivered no group member can run
188+
// (or write) again — only then is it safe to close the stdout read end as a
189+
// last-resort unblock for a scanner that a wedged descendant still keeps
190+
// open. WaitDelay is the final backstop (#4533).
149191
go func() {
150-
<-runCtx.Done()
192+
select {
193+
case <-procDone:
194+
return // finished on its own; nothing to terminate
195+
case <-runCtx.Done():
196+
}
197+
if cmd.Process != nil {
198+
signalProcessGroup(cmd.Process, syscall.SIGTERM)
199+
select {
200+
case <-procDone: // exited within the grace window
201+
case <-time.After(opencodeTerminateGrace()):
202+
signalProcessGroup(cmd.Process, syscall.SIGKILL)
203+
}
204+
}
151205
_ = stdout.Close()
152206
}()
153207

@@ -159,8 +213,9 @@ func (b *opencodeBackend) Execute(ctx context.Context, prompt string, opts ExecO
159213
startTime := time.Now()
160214
scanResult := b.processEvents(stdout, msgCh)
161215

162-
// Wait for process exit.
216+
// Wait for process exit, then release the cancellation handler.
163217
exitErr := cmd.Wait()
218+
close(procDone)
164219
duration := time.Since(startTime)
165220

166221
if runCtx.Err() == context.DeadlineExceeded {
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
//go:build unix
2+
3+
package agent
4+
5+
import (
6+
"context"
7+
"log/slog"
8+
"os"
9+
"path/filepath"
10+
"strconv"
11+
"strings"
12+
"syscall"
13+
"testing"
14+
"time"
15+
)
16+
17+
// opencodeCancelFakeScript returns a POSIX-sh script that impersonates a
18+
// long-running `opencode`: it spawns a background grandchild, records both its
19+
// own (process-group-leader) pid and the grandchild pid, then streams stdout in
20+
// a tight loop forever. This is the shape that orphans and spins on EPIPE when
21+
// the daemon closes stdout while the process is still alive. When ignoreTerm is
22+
// true the whole group ignores SIGTERM, forcing the SIGKILL escalation path.
23+
func opencodeCancelFakeScript(ignoreTerm bool) string {
24+
trap := "trap 'exit 0' TERM\n"
25+
if ignoreTerm {
26+
trap = "trap '' TERM\n"
27+
}
28+
return "#!/bin/sh\n" + trap +
29+
`# Background grandchild so the test can assert the *whole* group is
30+
# terminated on cancellation, not just the direct child.
31+
( sleep 300 ) &
32+
child=$!
33+
if [ -n "$OPENCODE_PID_FILE" ]; then
34+
printf '%s %s\n' "$$" "$child" > "$OPENCODE_PID_FILE"
35+
fi
36+
printf '{"type":"step_start","timestamp":1,"sessionID":"ses_fake","part":{"type":"step-start"}}\n'
37+
while true; do
38+
printf '{"type":"text","timestamp":2,"sessionID":"ses_fake","part":{"type":"text","text":"tick"}}\n'
39+
sleep 0.1
40+
done
41+
`
42+
}
43+
44+
// TestOpencodeCancellationTerminatesProcessGroupGraceful verifies that
45+
// cancelling a run terminates a SIGTERM-respecting opencode and its whole
46+
// process group, returns an "aborted" result without hanging, and leaves no
47+
// orphaned descendant.
48+
func TestOpencodeCancellationTerminatesProcessGroupGraceful(t *testing.T) {
49+
runOpencodeCancellationTest(t, opencodeCancelFakeScript(false))
50+
}
51+
52+
// TestOpencodeCancellationEscalatesToSIGKILL verifies the worst case from
53+
// #4533: opencode (and its children) ignore SIGTERM and keep writing to stdout.
54+
// Cancellation must escalate to a group SIGKILL, still return promptly, and
55+
// still reap the whole group — without deadlocking on the stdout scanner or
56+
// closing the pipe under a live writer.
57+
func TestOpencodeCancellationEscalatesToSIGKILL(t *testing.T) {
58+
opencodeTerminateGraceNanos.Store(int64(300 * time.Millisecond))
59+
t.Cleanup(func() { opencodeTerminateGraceNanos.Store(0) })
60+
runOpencodeCancellationTest(t, opencodeCancelFakeScript(true))
61+
}
62+
63+
func runOpencodeCancellationTest(t *testing.T, script string) {
64+
t.Helper()
65+
66+
tempDir := t.TempDir()
67+
pidFile := filepath.Join(tempDir, "pids")
68+
fakePath := filepath.Join(tempDir, "opencode")
69+
writeTestExecutable(t, fakePath, []byte(script))
70+
71+
backend, err := New("opencode", Config{
72+
ExecutablePath: fakePath,
73+
Logger: slog.Default(),
74+
Env: map[string]string{"OPENCODE_PID_FILE": pidFile},
75+
})
76+
if err != nil {
77+
t.Fatalf("new opencode backend: %v", err)
78+
}
79+
80+
ctx, cancel := context.WithCancel(context.Background())
81+
defer cancel()
82+
83+
session, err := backend.Execute(ctx, "prompt-ignored", ExecOptions{Cwd: tempDir})
84+
if err != nil {
85+
t.Fatalf("execute: %v", err)
86+
}
87+
88+
// Drain streamed messages so processEvents never blocks on a full channel.
89+
go func() {
90+
for range session.Messages {
91+
}
92+
}()
93+
94+
pids := waitForPids(t, pidFile)
95+
96+
cancel() // user cancels the task
97+
98+
select {
99+
case res := <-session.Result:
100+
if res.Status != "aborted" {
101+
t.Errorf("status = %q, want aborted", res.Status)
102+
}
103+
case <-time.After(10 * time.Second):
104+
t.Fatal("Execute did not return after cancellation (possible scanner deadlock or unkilled process)")
105+
}
106+
107+
// The leader and the grandchild must both be gone — cancellation reaped the
108+
// whole group, leaving no orphan spinning.
109+
for _, pid := range pids {
110+
waitProcessGone(t, pid)
111+
}
112+
}
113+
114+
// waitForPids polls pidFile until it contains the space-separated pids the fake
115+
// recorded, then returns them.
116+
func waitForPids(t *testing.T, pidFile string) []int {
117+
t.Helper()
118+
deadline := time.Now().Add(5 * time.Second)
119+
for time.Now().Before(deadline) {
120+
raw, err := os.ReadFile(pidFile)
121+
if err == nil {
122+
fields := strings.Fields(string(raw))
123+
if len(fields) >= 2 {
124+
pids := make([]int, 0, len(fields))
125+
ok := true
126+
for _, f := range fields {
127+
n, perr := strconv.Atoi(f)
128+
if perr != nil || n <= 0 {
129+
ok = false
130+
break
131+
}
132+
pids = append(pids, n)
133+
}
134+
if ok {
135+
return pids
136+
}
137+
}
138+
}
139+
time.Sleep(20 * time.Millisecond)
140+
}
141+
t.Fatalf("fake opencode never recorded its pids in %s", pidFile)
142+
return nil
143+
}
144+
145+
// waitProcessGone polls until signal 0 to pid reports the process no longer
146+
// exists (ESRCH), failing if it is still alive after the deadline.
147+
func waitProcessGone(t *testing.T, pid int) {
148+
t.Helper()
149+
deadline := time.Now().Add(5 * time.Second)
150+
for time.Now().Before(deadline) {
151+
if err := syscall.Kill(pid, 0); err == syscall.ESRCH {
152+
return
153+
}
154+
time.Sleep(20 * time.Millisecond)
155+
}
156+
t.Errorf("process %d still alive after cancellation — orphaned/leaked", pid)
157+
}

‎server/pkg/agent/proc_other.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,37 @@
22

33
package agent
44

5-
import "os/exec"
5+
import (
6+
"os"
7+
"os/exec"
8+
"syscall"
9+
)
610

711
// hideAgentWindow is a no-op on non-Windows platforms.
812
func hideAgentWindow(cmd *exec.Cmd) {}
13+
14+
// configureProcessGroup puts the child into its own process group (it becomes
15+
// the group leader, so the group id equals the child pid). This lets the
16+
// daemon signal the entire tree — the agent CLI plus any tool subprocess it
17+
// spawns — in one call, instead of killing only the direct child and leaking
18+
// grandchildren that keep running (and, for opencode, spinning on EPIPE) after
19+
// a task is cancelled or the daemon restarts. See signalProcessGroup.
20+
func configureProcessGroup(cmd *exec.Cmd) {
21+
if cmd.SysProcAttr == nil {
22+
cmd.SysProcAttr = &syscall.SysProcAttr{}
23+
}
24+
cmd.SysProcAttr.Setpgid = true
25+
}
26+
27+
// signalProcessGroup sends sig to the whole process group led by p (when the
28+
// command was started with configureProcessGroup), falling back to the single
29+
// process if the group send fails. Targeting the group (negative pid) reaches
30+
// the descendants the agent spawned, not just the leader.
31+
func signalProcessGroup(p *os.Process, sig syscall.Signal) {
32+
if p == nil {
33+
return
34+
}
35+
if err := syscall.Kill(-p.Pid, sig); err != nil {
36+
_ = p.Signal(sig)
37+
}
38+
}

‎server/pkg/agent/proc_windows.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package agent
44

55
import (
6+
"os"
67
"os/exec"
78
"syscall"
89
)
@@ -30,3 +31,19 @@ func hideAgentWindow(cmd *exec.Cmd) {
3031
cmd.SysProcAttr.HideWindow = true
3132
cmd.SysProcAttr.CreationFlags |= createNewConsole
3233
}
34+
35+
// configureProcessGroup is a no-op on Windows: there is no Setpgid/process-group
36+
// signalling. Descendant cleanup relies on the hidden console group set up by
37+
// hideAgentWindow plus exec.CommandContext / WaitDelay terminating the child.
38+
func configureProcessGroup(cmd *exec.Cmd) {}
39+
40+
// signalProcessGroup terminates the process on Windows. Windows has no
41+
// SIGTERM/SIGKILL distinction or process-group signalling, so the signal is
42+
// ignored and the process is killed directly (TerminateProcess via Kill). The
43+
// caller's grace window still applies before this is invoked with SIGKILL.
44+
func signalProcessGroup(p *os.Process, _ syscall.Signal) {
45+
if p == nil {
46+
return
47+
}
48+
_ = p.Kill()
49+
}

0 commit comments

Comments
 (0)