Warn on suppressed message-tool-only finals

OpenClaw Assistant · OpenClaw Assistant · commit 47056e8df729 · 2026-05-12T08:46:31.000Z
diff --git a/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts b/src/auto-reply/reply/dispatch-from-config.reply-dispatch.test.ts
@@ -1,5 +1,10 @@
 import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 import { clearAgentHarnesses } from "../../agents/harness/registry.js";
+import {
+  onInternalDiagnosticEvent,
+  resetDiagnosticEventsForTest,
+  type DiagnosticEventPayload,
+} from "../../infra/diagnostic-events.js";
 import type { PluginHookReplyDispatchResult } from "../../plugins/hooks.js";
 import { createInternalHookEventPayload } from "../../test-utils/internal-hook-event-payload.js";
 import {
@@ -33,6 +38,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
     clearAgentHarnesses();
     setDiscordTestRegistry();
     resetInboundDedupe();
+    resetDiagnosticEventsForTest();
     mocks.routeReply.mockReset().mockResolvedValue({ ok: true, messageId: "mock" });
     mocks.tryFastAbortFromMessage.mockReset().mockResolvedValue({
       handled: false,
@@ -200,6 +206,199 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
     expect(sessionStoreMocks.currentEntry?.pendingFinalDeliveryContext).toBeUndefined();
   });
 
+  it("audits substantive finals suppressed by message-tool-only delivery", async () => {
+    hookMocks.runner.hasHooks.mockReturnValue(false);
+    const diagnosticEvents: DiagnosticEventPayload[] = [];
+    const stopDiagnostics = onInternalDiagnosticEvent((event) => {
+      diagnosticEvents.push(event);
+    });
+
+    try {
+      const result = await dispatchReplyFromConfig({
+        ctx: {
+          ...createHookCtx(),
+          ChatType: "channel",
+          Surface: "discord",
+          Provider: "discord",
+        },
+        cfg: emptyConfig,
+        dispatcher: createDispatcher(),
+        replyResolver: async () => ({ text: "model reply" }),
+      });
+
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      const auditEvent = diagnosticEvents.find(
+        (event) =>
+          event.type === "log.record" &&
+          event.message.includes("Normal assistant final reply was suppressed"),
+      );
+
+      expect(result.queuedFinal).toBe(false);
+      expect(auditEvent?.type).toBe("log.record");
+      if (auditEvent?.type === "log.record") {
+        expect(auditEvent.level).toBe("warning");
+        expect(auditEvent.attributes).toMatchObject({
+          channel: "discord",
+          sessionKey: "agent:test:session",
+          sourceReplyDeliveryMode: "message_tool_only",
+          suppressedFinalCount: 1,
+          suppressedFinalTextChars: 11,
+          sendPolicyDenied: false,
+        });
+        expect(auditEvent.message).not.toContain("model reply");
+      }
+    } finally {
+      stopDiagnostics();
+    }
+  });
+
+  it("does not audit silent NO_REPLY finals suppressed by message-tool-only delivery", async () => {
+    hookMocks.runner.hasHooks.mockReturnValue(false);
+    const diagnosticEvents: DiagnosticEventPayload[] = [];
+    const stopDiagnostics = onInternalDiagnosticEvent((event) => {
+      diagnosticEvents.push(event);
+    });
+
+    try {
+      await dispatchReplyFromConfig({
+        ctx: {
+          ...createHookCtx(),
+          ChatType: "channel",
+          Surface: "discord",
+          Provider: "discord",
+        },
+        cfg: emptyConfig,
+        dispatcher: createDispatcher(),
+        replyResolver: async () => ({ text: "NO_REPLY" }),
+      });
+
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      expect(
+        diagnosticEvents.some(
+          (event) =>
+            event.type === "log.record" &&
+            event.message.includes("Normal assistant final reply was suppressed"),
+        ),
+      ).toBe(false);
+    } finally {
+      stopDiagnostics();
+    }
+  });
+
+  it("does not audit suppressed finals when send policy denies delivery", async () => {
+    hookMocks.runner.hasHooks.mockReturnValue(false);
+    const diagnosticEvents: DiagnosticEventPayload[] = [];
+    const stopDiagnostics = onInternalDiagnosticEvent((event) => {
+      diagnosticEvents.push(event);
+    });
+
+    try {
+      await dispatchReplyFromConfig({
+        ctx: {
+          ...createHookCtx(),
+          ChatType: "channel",
+          Surface: "discord",
+          Provider: "discord",
+        },
+        cfg: {
+          ...emptyConfig,
+          session: { sendPolicy: { default: "deny" } },
+        },
+        dispatcher: createDispatcher(),
+        replyResolver: async () => ({ text: "model reply" }),
+      });
+
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      expect(
+        diagnosticEvents.some(
+          (event) =>
+            event.type === "log.record" &&
+            event.message.includes("Normal assistant final reply was suppressed"),
+        ),
+      ).toBe(false);
+    } finally {
+      stopDiagnostics();
+    }
+  });
+
+  it("audits media-only finals suppressed by message-tool-only delivery", async () => {
+    hookMocks.runner.hasHooks.mockReturnValue(false);
+    const diagnosticEvents: DiagnosticEventPayload[] = [];
+    const stopDiagnostics = onInternalDiagnosticEvent((event) => {
+      diagnosticEvents.push(event);
+    });
+
+    try {
+      await dispatchReplyFromConfig({
+        ctx: {
+          ...createHookCtx(),
+          ChatType: "channel",
+          Surface: "discord",
+          Provider: "discord",
+        },
+        cfg: emptyConfig,
+        dispatcher: createDispatcher(),
+        replyResolver: async () => ({ mediaUrls: ["file:///tmp/result.png"] }),
+      });
+
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      const auditEvent = diagnosticEvents.find(
+        (event) =>
+          event.type === "log.record" &&
+          event.message.includes("Normal assistant final reply was suppressed"),
+      );
+      expect(auditEvent?.type).toBe("log.record");
+      if (auditEvent?.type === "log.record") {
+        expect(auditEvent.attributes).toMatchObject({
+          suppressedFinalCount: 1,
+          suppressedFinalTextChars: 0,
+        });
+      }
+    } finally {
+      stopDiagnostics();
+    }
+  });
+
+  it("audits aggregate suppressed final counts without recording text", async () => {
+    hookMocks.runner.hasHooks.mockReturnValue(false);
+    const diagnosticEvents: DiagnosticEventPayload[] = [];
+    const stopDiagnostics = onInternalDiagnosticEvent((event) => {
+      diagnosticEvents.push(event);
+    });
+
+    try {
+      await dispatchReplyFromConfig({
+        ctx: {
+          ...createHookCtx(),
+          ChatType: "channel",
+          Surface: "discord",
+          Provider: "discord",
+        },
+        cfg: emptyConfig,
+        dispatcher: createDispatcher(),
+        replyResolver: async () => [{ text: "first" }, { text: "second reply" }],
+      });
+
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      const auditEvent = diagnosticEvents.find(
+        (event) =>
+          event.type === "log.record" &&
+          event.message.includes("Normal assistant final reply was suppressed"),
+      );
+      expect(auditEvent?.type).toBe("log.record");
+      if (auditEvent?.type === "log.record") {
+        expect(auditEvent.attributes).toMatchObject({
+          suppressedFinalCount: 2,
+          suppressedFinalTextChars: 17,
+        });
+        expect(auditEvent.message).not.toContain("first");
+        expect(auditEvent.message).not.toContain("second reply");
+      }
+    } finally {
+      stopDiagnostics();
+    }
+  });
+
   it("preserves pending final delivery when final dispatch fails", async () => {
     hookMocks.runner.hasHooks.mockReturnValue(false);
     sessionStoreMocks.currentEntry = {
diff --git a/src/auto-reply/reply/dispatch-from-config.ts b/src/auto-reply/reply/dispatch-from-config.ts
@@ -38,7 +38,7 @@ import {
   toPluginMessageContext,
   toPluginMessageReceivedEvent,
 } from "../../hooks/message-hook-mappers.js";
-import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js";
+import { emitTrustedDiagnosticEvent, isDiagnosticsEnabled } from "../../infra/diagnostic-events.js";
 import { measureDiagnosticsTimelineSpan } from "../../infra/diagnostics-timeline.js";
 import { formatErrorMessage } from "../../infra/errors.js";
 import { getSessionBindingService } from "../../infra/outbound/session-binding-service.js";
@@ -78,6 +78,7 @@ import type { BlockReplyContext } from "../get-reply-options.types.js";
 import { getReplyPayloadMetadata, type ReplyPayload } from "../reply-payload.js";
 import type { FinalizedMsgContext } from "../templating.js";
 import { normalizeVerboseLevel } from "../thinking.js";
+import { isSilentReplyPayloadText } from "../tokens.js";
 import { resolveConversationBindingContextFromMessage } from "./conversation-binding-input.js";
 import {
   createInternalHookEvent,
@@ -160,6 +161,48 @@ const AUDIO_HEADER_RE = /^\[Audio\b/i;
 const normalizeMediaType = (value: string): string =>
   normalizeOptionalLowercaseString(value.split(";")[0]) ?? "";
 
+const isSubstantiveSuppressedFinalReply = (reply: ReplyPayload): boolean => {
+  if (reply.isReasoning === true) {
+    return false;
+  }
+  const parts = resolveSendableOutboundReplyParts(reply);
+  if (!parts.hasContent) {
+    return false;
+  }
+  if (!parts.hasMedia && isSilentReplyPayloadText(parts.text)) {
+    return false;
+  }
+  return true;
+};
+
+const emitSuppressedFinalReplyAudit = (params: {
+  channel?: string;
+  sessionKey?: string;
+  sourceReplyDeliveryMode: string;
+  suppressedFinalCount: number;
+  suppressedFinalTextChars: number;
+  sendPolicyDenied: boolean;
+}): void => {
+  if (params.suppressedFinalCount <= 0) {
+    return;
+  }
+  emitTrustedDiagnosticEvent({
+    type: "log.record",
+    level: "warning",
+    loggerName: "auto-reply.dispatch",
+    message:
+      "Normal assistant final reply was suppressed by message-tool-only source delivery; visible output must be sent with the message tool, then the normal final should be NO_REPLY.",
+    attributes: {
+      channel: params.channel ?? "unknown",
+      sessionKey: params.sessionKey ?? "unknown",
+      sourceReplyDeliveryMode: params.sourceReplyDeliveryMode,
+      suppressedFinalCount: params.suppressedFinalCount,
+      suppressedFinalTextChars: params.suppressedFinalTextChars,
+      sendPolicyDenied: params.sendPolicyDenied,
+    },
+  });
+};
+
 const isInboundAudioContext = (ctx: FinalizedMsgContext): boolean => {
   const rawTypes = [
     typeof ctx.MediaType === "string" ? ctx.MediaType : undefined,
@@ -1554,6 +1597,8 @@ export async function dispatchReplyFromConfig(
     let routedFinalCount = 0;
     let attemptedFinalDelivery = false;
     let finalDeliveryFailed = false;
+    let suppressedFinalCount = 0;
+    let suppressedFinalTextChars = 0;
     const shouldDeliverDespiteSourceReplySuppression = (reply: ReplyPayload) =>
       suppressAutomaticSourceDelivery &&
       !sendPolicyDenied &&
@@ -1564,7 +1609,16 @@ export async function dispatchReplyFromConfig(
       if (reply.isReasoning === true) {
         continue;
       }
-      if (suppressDelivery && !shouldDeliverDespiteSourceReplySuppression(reply)) {
+      const deliverDespiteSourceSuppression = shouldDeliverDespiteSourceReplySuppression(reply);
+      if (suppressDelivery && !deliverDespiteSourceSuppression) {
+        if (
+          suppressAutomaticSourceDelivery &&
+          !sendPolicyDenied &&
+          isSubstantiveSuppressedFinalReply(reply)
+        ) {
+          suppressedFinalCount += 1;
+          suppressedFinalTextChars += resolveSendableOutboundReplyParts(reply).trimmedText.length;
+        }
         continue;
       }
       attemptedFinalDelivery = true;
@@ -1583,6 +1637,17 @@ export async function dispatchReplyFromConfig(
       });
     }
 
+    if (suppressedFinalCount > 0) {
+      emitSuppressedFinalReplyAudit({
+        channel: deliveryChannel,
+        sessionKey: sessionStoreEntry.sessionKey ?? sessionKey,
+        sourceReplyDeliveryMode,
+        suppressedFinalCount,
+        suppressedFinalTextChars,
+        sendPolicyDenied,
+      });
+    }
+
     if (!suppressDelivery) {
       const ttsMode = resolveConfiguredTtsMode(cfg, {
         agentId: sessionAgentId,
