fix(agents): extend live CLI no-output watchdog to the blocked-tool floor

obviyus · obviyus · commit 4af2ca0a4d78 · 2026-07-09T14:26:05.000+05:30
diff --git a/src/agents/cli-runner.spawn.test.ts b/src/agents/cli-runner.spawn.test.ts
@@ -1349,6 +1349,90 @@ describe("runCliAgent spawn path", () => {
     }
   });
 
+  it("extends the live no-output watchdog to the blocked-tool floor while a tool is outstanding", async () => {
+    const toolErrorEvents: Array<Record<string, unknown>> = [];
+    const stopDiagnostics = onTrustedToolExecutionEvent((event) => {
+      if (event.type === "tool.execution.error") {
+        toolErrorEvents.push(event as unknown as Record<string, unknown>);
+      }
+    });
+    let stdoutListener: ((chunk: string) => void) | undefined;
+    const cancel = vi.fn();
+    const stdin = {
+      write: vi.fn((_data: string, callback?: (error?: Error | null) => void) => {
+        stdoutListener?.(
+          [
+            JSON.stringify({ type: "system", subtype: "init", session_id: "live-quiet-tool" }),
+            JSON.stringify({
+              type: "assistant",
+              message: {
+                content: [{ type: "tool_use", id: "tool-quiet-1", name: "Bash", input: {} }],
+              },
+            }),
+          ].join("\n") + "\n",
+        );
+        callback?.();
+      }),
+      end: vi.fn(),
+    };
+    supervisorSpawnMock.mockImplementationOnce(async (...args: unknown[]) => {
+      const input = (args[0] ?? {}) as { onStdout?: (chunk: string) => void };
+      stdoutListener = input.onStdout;
+      return {
+        runId: "live-quiet-tool-run",
+        pid: 2345,
+        startedAtMs: Date.now(),
+        stdin,
+        wait: vi.fn(() => new Promise(() => {})),
+        cancel,
+      };
+    });
+
+    const run = executePreparedCliRun(
+      buildPreparedCliRunContext({
+        provider: "claude-cli",
+        model: "sonnet",
+        runId: "run-live-quiet-tool",
+        backend: { liveSession: "claude-stdio" },
+        timeoutMs: 3_600_000,
+      }),
+    );
+    const rejection = expect(run).rejects.toThrow(/produced no output for 900s/);
+    await vi.waitFor(() => {
+      expect(stdin.write).toHaveBeenCalledOnce();
+    });
+
+    // Fake the clock only after the spawn path settled, then emit one more
+    // stdout line so the watchdog re-arms on the faked setTimeout/Date.
+    vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] });
+    stdoutListener?.(
+      `${JSON.stringify({
+        type: "stream_event",
+        event: { type: "content_block_delta", delta: { type: "text_delta", text: "running" } },
+      })}\n`,
+    );
+
+    // Base watchdog (600s cap for a 1h budget) must not kill the quiet tool.
+    await vi.advanceTimersByTimeAsync(650_000);
+    expect(cancel).not.toHaveBeenCalled();
+
+    // The blocked-tool floor (15min of quiet) still terminates a wedged tool.
+    try {
+      await vi.advanceTimersByTimeAsync(300_000);
+      expect(cancel).toHaveBeenCalledWith("manual-cancel");
+      await rejection;
+      // Watchdog-killed turns must keep timeout provenance for active tools.
+      expect(toolErrorEvents).toContainEqual(
+        expect.objectContaining({
+          toolCallId: "tool-quiet-1",
+          terminalReason: "timed_out",
+        }),
+      );
+    } finally {
+      stopDiagnostics();
+    }
+  });
+
   it("keeps non-capture live prepared backend cleanup with the whole-run owner", async () => {
     let stdoutListener: ((chunk: string) => void) | undefined;
     const stdin = {
@@ -2692,97 +2776,6 @@ ${JSON.stringify({
     },
   );
 
-  it("preserves no-output watchdog timeout provenance for active Claude live tools", async () => {
-    const diagnosticEvents: Array<Record<string, unknown>> = [];
-    const stopDiagnostics = onTrustedToolExecutionEvent((event) => {
-      if (event.type === "tool.execution.error") {
-        diagnosticEvents.push(event as unknown as Record<string, unknown>);
-      }
-    });
-    let stdoutListener: ((chunk: string) => void) | undefined;
-    const stdin = {
-      write: vi.fn((_data: string, cb?: (err?: Error | null) => void) => {
-        stdoutListener?.(
-          [
-            JSON.stringify({ type: "system", subtype: "init", session_id: "live-no-output" }),
-            JSON.stringify({
-              type: "assistant",
-              session_id: "live-no-output",
-              message: {
-                role: "assistant",
-                content: [
-                  {
-                    type: "tool_use",
-                    id: "tool-live-no-output",
-                    name: "Bash",
-                    input: { command: "sleep 10" },
-                  },
-                ],
-              },
-            }),
-          ].join("\n") + "\n",
-        );
-        cb?.();
-      }),
-      end: vi.fn(),
-    };
-    supervisorSpawnMock.mockImplementation(async (...args: unknown[]) => {
-      const input = (args[0] ?? {}) as { onStdout?: (chunk: string) => void };
-      stdoutListener = input.onStdout;
-      return {
-        runId: "live-run-no-output",
-        pid: 3062,
-        startedAtMs: Date.now(),
-        stdin,
-        wait: vi.fn(() => new Promise(() => {})),
-        cancel: vi.fn(),
-      };
-    });
-
-    try {
-      const context = buildPreparedCliRunContext({
-        provider: "claude-cli",
-        model: "sonnet",
-        runId: "run-live-no-output",
-        sessionId: "session-live-no-output",
-        sessionKey: "agent:main:no-output",
-        backend: { liveSession: "claude-stdio" },
-        timeoutMs: 120_000,
-      });
-      const resultPromise = runClaudeLiveSessionTurn({
-        context,
-        args: context.preparedBackend.backend.args ?? [],
-        env: {},
-        prompt: "hello",
-        useResume: false,
-        noOutputTimeoutMs: 25,
-        getProcessSupervisor: () => ({
-          spawn: (params: Parameters<SupervisorSpawnFn>[0]) =>
-            supervisorSpawnMock(params) as ReturnType<SupervisorSpawnFn>,
-          cancel: vi.fn(),
-          cancelScope: vi.fn(),
-          getRecord: vi.fn(),
-        }),
-        onAssistantDelta: () => {},
-        cleanup: async () => {},
-      });
-      const runExpectation = expectRejectsWithFields(resultPromise, {
-        name: "FailoverError",
-        message: "CLI produced no output for 0s and was terminated.",
-      });
-
-      await runExpectation;
-      expect(diagnosticEvents).toContainEqual(
-        expect.objectContaining({
-          toolCallId: "tool-live-no-output",
-          terminalReason: "timed_out",
-        }),
-      );
-    } finally {
-      stopDiagnostics();
-    }
-  });
-
   it("answers Claude live control_request can_use_tool with deny when exec policy is restrictive", async () => {
     const diagnosticEvents: Array<Record<string, unknown>> = [];
     const stopDiagnostics = onInternalDiagnosticEvent((event) => {
diff --git a/src/agents/cli-runner/claude-live-session.ts b/src/agents/cli-runner/claude-live-session.ts
@@ -22,6 +22,7 @@ import {
   type ExecAsk,
   type ExecSecurity,
 } from "../../infra/exec-approvals.js";
+import { BLOCKED_TOOL_CALL_ABORT_FLOOR_MS } from "../../logging/diagnostic-run-activity.js";
 import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
 import {
   CLI_STREAM_JSON_DEFAULT_MAX_TURN_RAW_CHARS,
@@ -57,6 +58,8 @@ type ClaudeLiveTurn = {
   rawChars: number;
   sessionId?: string;
   noOutputTimer: NodeJS.Timeout | null;
+  /** Last stdout/stderr time; null until the process emits anything this turn. */
+  lastOutputAtMs: number | null;
   timeoutTimer: NodeJS.Timeout | null;
   activeTools: Map<string, ClaudeLiveActiveTool>;
   observedStdout: boolean;
@@ -684,24 +687,44 @@ function noteClaudeLiveProgress(
   emitClaudeLiveProgress(turn, "cli_live:stream_progress");
 }
 
-function resetNoOutputTimer(session: ClaudeLiveSession): void {
-  const turn = session.currentTurn;
-  if (!turn) {
-    return;
-  }
+// The CLI emits a tool_use line, then nothing until the tool result, so a
+// quiet long-running tool is indistinguishable from a wedged process at the
+// stdout level. While observed tool calls are outstanding, extend the quiet
+// window to the blocked-tool floor instead of killing mid-tool.
+function armNoOutputTimer(session: ClaudeLiveSession, turn: ClaudeLiveTurn, delayMs: number): void {
   if (turn.noOutputTimer) {
     clearTimeout(turn.noOutputTimer);
   }
   turn.noOutputTimer = setTimeout(() => {
+    const quietSinceMs = turn.lastOutputAtMs ?? turn.startedAtMs;
+    if (turn.activeTools.size > 0) {
+      const quietBudgetMs = Math.max(session.noOutputTimeoutMs, BLOCKED_TOOL_CALL_ABORT_FLOOR_MS);
+      const remainingMs = quietSinceMs + quietBudgetMs - Date.now();
+      if (remainingMs > 0) {
+        armNoOutputTimer(session, turn, remainingMs);
+        return;
+      }
+    }
     closeLiveSession(
       session,
       "abort",
       createTimeoutError(
         session,
-        `CLI produced no output for ${Math.round(session.noOutputTimeoutMs / 1000)}s and was terminated.`,
+        `CLI produced no output for ${Math.round((Date.now() - quietSinceMs) / 1000)}s and was terminated.`,
+        // Retryable only when the process never produced any output this turn.
+        turn.lastOutputAtMs === null ? "cli_no_output_timeout" : undefined,
       ),
     );
-  }, session.noOutputTimeoutMs);
+  }, delayMs);
+}
+
+function resetNoOutputTimer(session: ClaudeLiveSession): void {
+  const turn = session.currentTurn;
+  if (!turn) {
+    return;
+  }
+  turn.lastOutputAtMs = Date.now();
+  armNoOutputTimer(session, turn, session.noOutputTimeoutMs);
 }
 
 function parseSessionId(parsed: Record<string, unknown>): string | undefined {
@@ -1136,6 +1159,7 @@ function createTurn(params: {
     rawLines: [],
     rawChars: 0,
     noOutputTimer: null,
+    lastOutputAtMs: null,
     timeoutTimer: null,
     activeTools: new Map(),
     observedStdout: false,
@@ -1161,17 +1185,7 @@ function createTurn(params: {
     resolve: params.resolve,
     reject: params.reject,
   };
-  turn.noOutputTimer = setTimeout(() => {
-    closeLiveSession(
-      params.session,
-      "abort",
-      createTimeoutError(
-        params.session,
-        `CLI produced no output for ${Math.round(params.noOutputTimeoutMs / 1000)}s and was terminated.`,
-        "cli_no_output_timeout",
-      ),
-    );
-  }, params.noOutputTimeoutMs);
+  armNoOutputTimer(params.session, turn, params.noOutputTimeoutMs);
   turn.timeoutTimer = setTimeout(() => {
     closeLiveSession(
       params.session,
