Skip to content

Commit 180cfc3

Browse files
committed
fix(sandbox): reap process group keeper
1 parent fa631df commit 180cfc3

2 files changed

Lines changed: 214 additions & 72 deletions

File tree

‎src/agents/sandbox/sandboxes/unix_local.py‎

Lines changed: 147 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
import time
2222
import uuid
2323
from collections import deque
24-
from collections.abc import Collection, Mapping, Sequence
24+
from collections.abc import Awaitable, Collection, Mapping, Sequence
2525
from contextlib import suppress
2626
from dataclasses import dataclass, field
2727
from functools import partial
2828
from pathlib import Path
29-
from typing import Literal, cast
29+
from typing import Literal, TypeVar, cast
3030

3131
from ...logger import log_tool_action_warning
3232
from .._mount_security import redact_mount_error_data
@@ -102,56 +102,62 @@
102102

103103
logger = logging.getLogger(__name__)
104104
_PROCESS_CLEANUP_TIMEOUT_SECONDS = 1.0
105+
_PROCESS_GROUP_KEEPER_SCRIPT = """
106+
import os
107+
import sys
105108
106-
107-
def _keep_process_group_alive(control_fd: int) -> None:
108-
"""Keep the process-group identity stable without becoming a child of the command."""
109-
intermediate_pid = os.fork()
110-
if intermediate_pid == 0:
111-
try:
112-
keeper_pid = os.fork()
113-
except BaseException:
114-
os._exit(1)
115-
116-
if keeper_pid == 0:
117-
for fd in (0, 1, 2):
118-
if fd != control_fd:
119-
with suppress(OSError):
120-
os.close(fd)
121-
max_fd = os.sysconf("SC_OPEN_MAX")
122-
os.closerange(3, control_fd)
123-
os.closerange(control_fd + 1, max_fd)
124-
while True:
125-
try:
126-
if not os.read(control_fd, 4096):
127-
break
128-
except InterruptedError:
129-
continue
130-
except OSError:
131-
break
132-
os._exit(0)
133-
134-
# The keeper is now reparented away from the command when this
135-
# intermediate process exits. It remains in the command's process
136-
# group, while the command cannot observe it with waitpid(-1, ...).
137-
os._exit(0)
138-
139-
while True:
140-
try:
141-
_, status = os.waitpid(intermediate_pid, 0)
109+
control_fd = int(sys.argv[1])
110+
while True:
111+
try:
112+
if not os.read(control_fd, 4096):
142113
break
143-
except InterruptedError:
144-
continue
145-
146-
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
147-
raise ChildProcessError("failed to create the process-group keeper")
148-
149-
os.close(control_fd)
150-
114+
except InterruptedError:
115+
continue
116+
except OSError:
117+
break
118+
os._exit(0)
119+
"""
120+
_SubprocessResultT = TypeVar("_SubprocessResultT")
121+
122+
123+
def _create_process_group() -> None:
124+
os.setpgid(0, 0)
125+
126+
127+
async def _settle_subprocess_awaitable(
128+
awaitable: Awaitable[_SubprocessResultT],
129+
) -> tuple[_SubprocessResultT, bool]:
130+
task = asyncio.ensure_future(awaitable)
131+
completion = asyncio.create_task(asyncio.wait((task,)))
132+
caller_cancelled = False
133+
while not completion.done():
134+
try:
135+
await asyncio.shield(completion)
136+
except asyncio.CancelledError:
137+
caller_cancelled = True
151138

152-
async def _terminate_process_group_and_reap_process(proc: asyncio.subprocess.Process) -> None:
139+
completion.result()
140+
try:
141+
result = task.result()
142+
except BaseException:
143+
if caller_cancelled:
144+
raise asyncio.CancelledError() from None
145+
raise
146+
return result, caller_cancelled
147+
148+
149+
async def _terminate_process_group_and_reap_process(
150+
proc: asyncio.subprocess.Process,
151+
*,
152+
process_group_id: int | None = None,
153+
keeper: asyncio.subprocess.Process | None = None,
154+
) -> None:
155+
group_id = proc.pid if process_group_id is None else process_group_id
153156
with suppress(OSError):
154-
os.killpg(proc.pid, signal.SIGKILL)
157+
os.killpg(group_id, signal.SIGKILL)
158+
if keeper is not None and keeper.returncode is None:
159+
with suppress(OSError):
160+
keeper.kill()
155161
# The process group can contain descendants owned by another user when the
156162
# command was launched through sudo. Kill the direct child separately so
157163
# that a permission failure for one group member cannot leave the sudo
@@ -175,12 +181,29 @@ async def _terminate_process_group_and_reap_process(proc: asyncio.subprocess.Pro
175181
transport = getattr(proc, "_transport", None)
176182
if transport is not None:
177183
transport.close()
184+
if keeper is not None:
185+
with suppress(asyncio.TimeoutError):
186+
await asyncio.wait_for(
187+
keeper.wait(),
188+
timeout=_PROCESS_CLEANUP_TIMEOUT_SECONDS,
189+
)
178190

179191

180-
async def _terminate_process_group_and_reap(proc: asyncio.subprocess.Process) -> None:
192+
async def _terminate_process_group_and_reap(
193+
proc: asyncio.subprocess.Process,
194+
*,
195+
process_group_id: int | None = None,
196+
keeper: asyncio.subprocess.Process | None = None,
197+
) -> None:
181198
# Keep process cleanup in an owned task so a second cancellation cannot
182199
# interrupt the drain, direct-child reap, or transport close.
183-
cleanup_task = asyncio.create_task(_terminate_process_group_and_reap_process(proc))
200+
cleanup_task = asyncio.create_task(
201+
_terminate_process_group_and_reap_process(
202+
proc,
203+
process_group_id=process_group_id,
204+
keeper=keeper,
205+
)
206+
)
184207
while not cleanup_task.done():
185208
try:
186209
await asyncio.shield(cleanup_task)
@@ -377,38 +400,97 @@ async def _exec_internal(
377400
control_read_fd, control_write_fd = os.pipe()
378401
try:
379402
try:
380-
proc = await asyncio.create_subprocess_exec(
381-
*exec_command,
382-
stdout=asyncio.subprocess.PIPE,
383-
stderr=asyncio.subprocess.PIPE,
384-
cwd=process_cwd,
385-
env=env,
386-
start_new_session=True,
387-
pass_fds=(control_read_fd,),
388-
preexec_fn=lambda: _keep_process_group_alive(control_read_fd),
403+
keeper, keeper_cancelled = await _settle_subprocess_awaitable(
404+
asyncio.create_subprocess_exec(
405+
sys.executable,
406+
"-c",
407+
_PROCESS_GROUP_KEEPER_SCRIPT,
408+
str(control_read_fd),
409+
stdin=asyncio.subprocess.DEVNULL,
410+
stdout=asyncio.subprocess.DEVNULL,
411+
stderr=asyncio.subprocess.DEVNULL,
412+
cwd=process_cwd,
413+
env=env,
414+
pass_fds=(control_read_fd,),
415+
preexec_fn=_create_process_group,
416+
)
389417
)
390418
except BaseException:
391419
_close_fd_quietly(control_read_fd)
392420
raise
393421
_close_fd_quietly(control_read_fd)
394422

423+
if keeper_cancelled:
424+
_close_fd_quietly(control_write_fd)
425+
await _terminate_process_group_and_reap(
426+
keeper,
427+
process_group_id=keeper.pid,
428+
)
429+
raise asyncio.CancelledError()
430+
431+
try:
432+
proc, proc_cancelled = await _settle_subprocess_awaitable(
433+
asyncio.create_subprocess_exec(
434+
*exec_command,
435+
stdout=asyncio.subprocess.PIPE,
436+
stderr=asyncio.subprocess.PIPE,
437+
cwd=process_cwd,
438+
env=env,
439+
start_new_session=False,
440+
preexec_fn=lambda: os.setpgid(0, keeper.pid),
441+
)
442+
)
443+
except BaseException:
444+
_close_fd_quietly(control_write_fd)
445+
await _terminate_process_group_and_reap(
446+
keeper,
447+
process_group_id=keeper.pid,
448+
)
449+
raise
450+
451+
if proc_cancelled:
452+
_close_fd_quietly(control_write_fd)
453+
await _terminate_process_group_and_reap(
454+
proc,
455+
process_group_id=keeper.pid,
456+
keeper=keeper,
457+
)
458+
raise asyncio.CancelledError()
459+
395460
try:
396461
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
397462
except asyncio.TimeoutError as e:
398-
try:
399-
# process tree cleanup
400-
os.killpg(proc.pid, signal.SIGKILL)
401-
except Exception:
402-
pass
463+
await _terminate_process_group_and_reap(
464+
proc,
465+
process_group_id=keeper.pid,
466+
keeper=keeper,
467+
)
403468
raise ExecTimeoutError(command=command, timeout_s=timeout, cause=e) from e
404469
except asyncio.CancelledError:
405-
await _terminate_process_group_and_reap(proc)
470+
await _terminate_process_group_and_reap(
471+
proc,
472+
process_group_id=keeper.pid,
473+
keeper=keeper,
474+
)
406475
raise
476+
except Exception:
477+
await _terminate_process_group_and_reap(
478+
proc,
479+
process_group_id=keeper.pid,
480+
keeper=keeper,
481+
)
482+
raise
483+
484+
_close_fd_quietly(control_write_fd)
485+
_, keeper_cancelled = await _settle_subprocess_awaitable(keeper.wait())
486+
if keeper_cancelled:
487+
raise asyncio.CancelledError()
407488
except ExecTimeoutError:
408489
raise
409490
except Exception as e:
410491
raise ExecTransportError(command=command, cause=e) from e
411492
finally:
493+
_close_fd_quietly(control_read_fd)
412494
_close_fd_quietly(control_write_fd)
413495

414496
return ExecResult(

0 commit comments

Comments
 (0)