Fix heartbeat runner failure copy (#82848)

joshavant · web-flow · commit 39a9a3478f11 · 2026-05-16T22:23:22.000-05:00
* fix: scope heartbeat runner failures

* chore: add heartbeat failure changelog
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -118,6 +118,7 @@ Docs: https://docs.openclaw.ai
 - ACP/control plane: refresh cached runtime handles when agent config changes so ACP sessions stop using stale runtimes after `agents.defaults` edits. Fixes #82237. Thanks @giodl73-repo.
 - Gateway/sessions: scope session data lookups by agent id so multi-agent gateway state cannot cross-leak session records across configured agents. (#81386) Thanks @pgondhi987.
 - Gateway/restart: mark active main sessions as restart-aborted before forced restarts so startup recovery can resume interrupted turns instead of leaving them stranded as running. Fixes #82433. (#82772) Thanks @joshavant.
+- Gateway/heartbeat: report heartbeat runner failures with background-specific copy instead of foreground `/new` recovery guidance. Fixes #82708. (#82848) Thanks @joshavant.
 - Agents/media: require generated music/video completion agents to use the message tool for visible delivery and stop merging generated image attachments into message-tool-only source reply mirrors, avoiding direct fallback posts that can duplicate media the model already sent.
 - Agents/media: accept generated media attachments on internal completion events and report delivery-loss failures as errors, so completed background music/video tasks do not disappear after provider success.
 - Matrix/approvals: release in-flight reaction bindings when the channel approval handler stops mid-delivery, preventing stale approval targets after restart. Fixes #82485. (#82482) Thanks @Feelw00.
diff --git a/src/auto-reply/reply/agent-runner-execution.test.ts b/src/auto-reply/reply/agent-runner-execution.test.ts
@@ -12,6 +12,7 @@ import {
   MAX_LIVE_SWITCH_RETRIES,
   resolveRunAfterAutoFallbackPrimaryProbeRecheck,
 } from "./agent-runner-execution.js";
+import { HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT } from "./agent-runner-failure-copy.js";
 import type { FollowupRun } from "./queue.js";
 import type { ReplyOperation } from "./reply-run-registry.js";
 import type { TypingSignaler } from "./typing-mode.js";
@@ -3744,6 +3745,26 @@ describe("runAgentTurnWithFallback", () => {
     }
   });
 
+  it("uses heartbeat failure copy for raw external errors during heartbeat runs", async () => {
+    state.runEmbeddedPiAgentMock.mockRejectedValueOnce(
+      new Error('Command lane "main" task timed out after 120000ms'),
+    );
+
+    const runAgentTurnWithFallback = await getRunAgentTurnWithFallback();
+    const result = await runAgentTurnWithFallback({
+      ...createMinimalRunAgentTurnParams(),
+      isHeartbeat: true,
+    });
+
+    expect(result.kind).toBe("final");
+    if (result.kind !== "final") {
+      throw new Error("expected final reply");
+    }
+    expect(result.payload.text).toBe(HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT);
+    expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT);
+    expect(result.payload.text).not.toContain("/new");
+  });
+
   it.each([
     {
       rejection: new Error("CLI exceeded timeout (300s) and was terminated."),
diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts
@@ -87,6 +87,10 @@ import {
 } from "../tokens.js";
 import type { GetReplyOptions, ReplyPayload } from "../types.js";
 import { resolveRunAuthProfile } from "./agent-runner-auth-profile.js";
+import {
+  GENERIC_EXTERNAL_RUN_FAILURE_TEXT,
+  HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT,
+} from "./agent-runner-failure-copy.js";
 import {
   buildEmbeddedRunExecutionParams,
   resolveQueuedReplyRuntimeConfig,
@@ -480,8 +484,6 @@ function collapseRepeatedFailureDetail(message: string): string {
 const SAFE_MISSING_API_KEY_PROVIDERS = new Set(["anthropic", "google", "openai", "openai-codex"]);
 const EXTERNAL_RUN_FAILURE_DETAIL_MAX_CHARS = 900;
 const AGENT_FAILED_BEFORE_REPLY_TEXT = "Agent failed before reply:";
-const GENERIC_EXTERNAL_RUN_FAILURE_TEXT =
-  "⚠️ Something went wrong while processing your request. Please try again, or use /new to start a fresh session.";
 
 type ExternalRunFailureReply = {
   text: string;
@@ -581,15 +583,9 @@ function formatForwardedExternalRunFailureText(message: string): string {
 
 function buildExternalRunFailureReply(
   message: string,
-  options?: { includeDetails?: boolean },
+  options?: { includeDetails?: boolean; isHeartbeat?: boolean },
 ): ExternalRunFailureReply {
   const normalizedMessage = collapseRepeatedFailureDetail(message);
-  if (isToolResultTurnMismatchError(normalizedMessage)) {
-    return {
-      text: "⚠️ Session history got out of sync. Please try again, or use /new to start a fresh session.",
-      isGenericRunnerFailure: false,
-    };
-  }
   const missingApiKeyFailure = buildMissingApiKeyFailureText(normalizedMessage);
   if (missingApiKeyFailure) {
     return { text: missingApiKeyFailure, isGenericRunnerFailure: false };
@@ -608,6 +604,15 @@ function buildExternalRunFailureReply(
       isGenericRunnerFailure: false,
     };
   }
+  if (options?.isHeartbeat) {
+    return { text: HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT, isGenericRunnerFailure: false };
+  }
+  if (isToolResultTurnMismatchError(normalizedMessage)) {
+    return {
+      text: "⚠️ Session history got out of sync. Please try again, or use /new to start a fresh session.",
+      isGenericRunnerFailure: false,
+    };
+  }
   const cliBackendTimeoutFailure = buildCliBackendTimeoutFailureText(normalizedMessage);
   if (cliBackendTimeoutFailure) {
     return { text: cliBackendTimeoutFailure, isGenericRunnerFailure: false };
@@ -2590,8 +2595,12 @@ export async function runAgentTurnWithFallback(params: {
         !shouldSurfaceToControlUi
           ? buildExternalRunFailureReply(message, {
               includeDetails: isVerboseFailureDetailEnabled(params.resolvedVerboseLevel),
+              isHeartbeat: params.isHeartbeat,
             })
           : undefined;
+      const genericFallbackText = params.isHeartbeat
+        ? HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT
+        : GENERIC_EXTERNAL_RUN_FAILURE_TEXT;
       const fallbackText = isBilling
         ? BILLING_ERROR_USER_MESSAGE
         : isRateLimit && !isOverloadedErrorMessage(message)
@@ -2604,7 +2613,7 @@ export async function runAgentTurnWithFallback(params: {
                 ? "⚠️ Message ordering conflict - please try again. If this persists, use /new to start a fresh session."
                 : shouldSurfaceToControlUi
                   ? `⚠️ Agent failed before reply: ${trimmedMessage}.\nLogs: openclaw logs --follow`
-                  : (externalRunFailureReply?.text ?? GENERIC_EXTERNAL_RUN_FAILURE_TEXT);
+                  : (externalRunFailureReply?.text ?? genericFallbackText);
       const userVisibleFallbackText = resolveExternalRunFailureTextForConversation({
         text: fallbackText,
         sessionCtx: params.sessionCtx,
diff --git a/src/auto-reply/reply/agent-runner-failure-copy.ts b/src/auto-reply/reply/agent-runner-failure-copy.ts
@@ -0,0 +1,36 @@
+export const GENERIC_EXTERNAL_RUN_FAILURE_TEXT =
+  "⚠️ Something went wrong while processing your request. Please try again, or use /new to start a fresh session.";
+
+export const HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT =
+  "⚠️ Heartbeat check failed before it could produce an update. The main chat session remains available.";
+
+export function isGenericExternalRunFailureText(text: string | undefined): boolean {
+  return text?.trim() === GENERIC_EXTERNAL_RUN_FAILURE_TEXT;
+}
+
+export function replaceGenericExternalRunFailureText(text: string): {
+  text: string;
+  replaced: boolean;
+} {
+  if (isGenericExternalRunFailureText(text)) {
+    return { text: HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT, replaced: true };
+  }
+
+  const genericStart = text.indexOf(GENERIC_EXTERNAL_RUN_FAILURE_TEXT);
+  if (genericStart < 0) {
+    return { text, replaced: false };
+  }
+
+  const trailing = text.slice(genericStart + GENERIC_EXTERNAL_RUN_FAILURE_TEXT.length).trim();
+  if (trailing) {
+    return { text, replaced: false };
+  }
+
+  const prefix = text.slice(0, genericStart).trimEnd();
+  return {
+    text: prefix
+      ? `${prefix} ${HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT}`
+      : HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT,
+    replaced: true,
+  };
+}
diff --git a/src/infra/heartbeat-runner.tool-response.test.ts b/src/infra/heartbeat-runner.tool-response.test.ts
@@ -5,8 +5,13 @@ import {
   createHeartbeatToolResponsePayload,
   type HeartbeatToolResponse,
 } from "../auto-reply/heartbeat-tool-response.js";
+import {
+  GENERIC_EXTERNAL_RUN_FAILURE_TEXT,
+  HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT,
+} from "../auto-reply/reply/agent-runner-failure-copy.js";
 import { markReplyPayloadForSourceSuppressionDelivery } from "../auto-reply/types.js";
 import type { OpenClawConfig } from "../config/config.js";
+import { getLastHeartbeatEvent, resetHeartbeatEventsForTest } from "./heartbeat-events.js";
 import { runHeartbeatOnce, type HeartbeatDeps } from "./heartbeat-runner.js";
 import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js";
 import {
@@ -21,6 +26,7 @@ describe("runHeartbeatOnce heartbeat response tool", () => {
 
   afterEach(() => {
     vi.unstubAllEnvs();
+    resetHeartbeatEventsForTest();
   });
 
   function createConfig(params: {
@@ -264,6 +270,41 @@ describe("runHeartbeatOnce heartbeat response tool", () => {
     });
   });
 
+  it("rewrites foreground generic runner failure payloads before heartbeat delivery", async () => {
+    await withTempTelegramHeartbeatSandbox(async ({ tmpDir, storePath, replySpy }) => {
+      const cfg = createConfig({ tmpDir, storePath });
+      await seedMainSessionStore(storePath, cfg, {
+        lastChannel: "telegram",
+        lastProvider: "telegram",
+        lastTo: TELEGRAM_GROUP,
+      });
+      replySpy.mockResolvedValue(
+        markReplyPayloadForSourceSuppressionDelivery({
+          text: GENERIC_EXTERNAL_RUN_FAILURE_TEXT,
+        }),
+      );
+      const sendTelegram = vi.fn().mockResolvedValue({ messageId: "m1" });
+
+      const result = await runHeartbeatOnce({
+        cfg,
+        deps: createDeps({ sendTelegram, getReplyFromConfig: replySpy }),
+      });
+
+      expect(result.status).toBe("ran");
+      expectTelegramSend(sendTelegram, {
+        text: HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT,
+        cfg,
+      });
+      expect(HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT).not.toContain("/new");
+      expect(getLastHeartbeatEvent()).toMatchObject({
+        status: "failed",
+        reason: "agent-runner-failure",
+        preview: HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT,
+        channel: "telegram",
+      });
+    });
+  });
+
   it("uses the heartbeat response tool prompt for auto-selected Codex model sessions", async () => {
     const result = await runPromptScenario({
       config: {
diff --git a/src/infra/heartbeat-runner.ts b/src/infra/heartbeat-runner.ts
@@ -33,6 +33,7 @@ import {
   stripHeartbeatToken,
   type HeartbeatTask,
 } from "../auto-reply/heartbeat.js";
+import { replaceGenericExternalRunFailureText } from "../auto-reply/reply/agent-runner-failure-copy.js";
 import { resolveDefaultModel } from "../auto-reply/reply/directive-handling.defaults.js";
 import { replyRunRegistry } from "../auto-reply/reply/reply-run-registry.js";
 import { resolveResponsePrefixTemplate } from "../auto-reply/reply/response-prefix-template.js";
@@ -1822,6 +1823,14 @@ export async function runHeartbeatOnce(opts: {
       normalized.text = execFallbackText;
       normalized.shouldSkip = false;
     }
+    const replacement = !heartbeatToolResponse
+      ? replaceGenericExternalRunFailureText(normalized.text)
+      : { text: normalized.text, replaced: false };
+    const deliveredAgentRunFailure = replacement.replaced;
+    if (deliveredAgentRunFailure) {
+      normalized.text = replacement.text;
+      normalized.shouldSkip = false;
+    }
     const shouldSkipMain =
       normalized.shouldSkip && !normalized.hasMedia && !hasRelayableExecCompletion;
     if (shouldSkipMain && reasoningPayloads.length === 0) {
@@ -2012,15 +2021,17 @@ export async function runHeartbeatOnce(opts: {
       });
     }
 
+    const sentStatus = deliveredAgentRunFailure ? "failed" : "sent";
     emitHeartbeatEvent({
-      status: "sent",
+      status: sentStatus,
       to: delivery.to,
+      ...(deliveredAgentRunFailure ? { reason: "agent-runner-failure" } : {}),
       preview: previewText?.slice(0, 200),
       durationMs: Date.now() - startedAt,
       hasMedia: mediaUrls.length > 0,
       channel: delivery.channel,
       accountId: delivery.accountId,
-      indicatorType: visibility.useIndicator ? resolveIndicatorType("sent") : undefined,
+      indicatorType: visibility.useIndicator ? resolveIndicatorType(sentStatus) : undefined,
     });
     await updateTaskTimestamps();
     consumeInspectedSystemEvents();
