// Chat gateway methods implement chat.send/history/abort/inject/metadata and

// bridge UI RPCs to agent dispatch, transcripts, media, and streaming state.

import { createHash } from "node:crypto";

import fs from "node:fs";

import path from "node:path";

import { performance } from "node:perf_hooks";

import { fileURLToPath } from "node:url";

import { isAudioFileName } from "@openclaw/media-core/mime";

import { asOptionalRecord } from "@openclaw/normalization-core/record-coerce";

import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";

import {

buildTtsSupplementMediaPayload,

getReplyPayloadTtsSupplement,

isReplyPayloadTtsSupplement,

resolveSendableOutboundReplyParts,

} from "openclaw/plugin-sdk/reply-payload";

import {

GATEWAY_CLIENT_CAPS,

GATEWAY_CLIENT_MODES,

GATEWAY_CLIENT_NAMES,

hasGatewayClientCap,

} from "../../../packages/gateway-protocol/src/client-info.js";

import {

ErrorCodes,

errorShape,

formatValidationErrors,

validateChatAbortParams,

validateChatHistoryParams,

validateChatInjectParams,

validateChatMetadataParams,

validateChatMessageGetParams,

validateChatSendParams,

} from "../../../packages/gateway-protocol/src/index.js";

import { CHAT_SEND_SESSION_KEY_MAX_LENGTH } from "../../../packages/gateway-protocol/src/schema.js";

import {

listAgentIds,

resolveDefaultAgentId,

resolveAgentWorkspaceDir,

resolveSessionAgentId,

} from "../../agents/agent-scope.js";

import { rewriteTranscriptEntriesInRuntimeTranscript } from "../../agents/embedded-agent-runner/transcript-rewrite.js";

import { runAgentHarnessBeforeMessageWriteHook } from "../../agents/harness/hook-helpers.js";

import { modelCatalogBrowseRequiresFullDiscovery } from "../../agents/model-catalog-browse.js";

import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";

import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js";

import type { AgentMessage } from "../../agents/runtime/index.js";

import { ensureSandboxWorkspaceForSession } from "../../agents/sandbox/context.js";

import { resolveAgentTimeoutMs } from "../../agents/timeout.js";

import { dispatchInboundMessage } from "../../auto-reply/dispatch.js";

import {

getReplyPayloadMetadata,

isReplyPayloadStatusNotice,

type ReplyPayload,

} from "../../auto-reply/reply-payload.js";

import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.js";

import {

stageSandboxMedia,

type StageSandboxMediaResult,

} from "../../auto-reply/reply/stage-sandbox-media.js";

import type { MsgContext, TemplateContext } from "../../auto-reply/templating.js";

import { resolveSessionFilePath, updateSessionStoreEntry } from "../../config/sessions.js";

import { resolveMirroredTranscriptText } from "../../config/sessions/transcript-mirror.js";

import { CURRENT_SESSION_VERSION } from "../../config/sessions/version.js";

import type { OpenClawConfig } from "../../config/types.openclaw.js";

import {

claimAgentRunContext,

clearAgentRunContext,

getAgentEventLifecycleGeneration,

} from "../../infra/agent-events.js";

import {

emitDiagnosticsTimelineEvent,

measureDiagnosticsTimelineSpan,

measureDiagnosticsTimelineSpanSync,

} from "../../infra/diagnostics-timeline.js";

import { formatErrorMessage, formatUncaughtError } from "../../infra/errors.js";

import { jsonUtf8Bytes } from "../../infra/json-utf8-bytes.js";

import { normalizeReplyPayloadsForDelivery } from "../../infra/outbound/payloads.js";

import { getSessionBindingService } from "../../infra/outbound/session-binding-service.js";

import { logLargePayload } from "../../logging/diagnostic-payload.js";

import {

appendLocalMediaParentRoots,

getAgentScopedMediaLocalRoots,

} from "../../media/local-roots.js";

import { parseInboundMediaUri } from "../../media/media-reference.js";

import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";

import {

deleteMediaBuffer,

MEDIA_MAX_BYTES,

type SavedMedia,

saveMediaBuffer,

} from "../../media/store.js";

import { createChannelMessageReplyPipeline } from "../../plugin-sdk/channel-outbound.js";

import type { ChannelRouteRef } from "../../plugin-sdk/channel-route.js";

import { isPluginOwnedSessionBindingRecord } from "../../plugins/conversation-binding.js";

import { normalizeAgentId, scopeLegacySessionKeyToAgent } from "../../routing/session-key.js";

import { normalizeInputProvenance, type InputProvenance } from "../../sessions/input-provenance.js";

import { resolveSendPolicy } from "../../sessions/send-policy.js";

import { parseAgentSessionKey } from "../../sessions/session-key-utils.js";

import {

createUserTurnTranscriptRecorder,

type UserTurnInput,

type UserTurnTranscriptRecorder,

} from "../../sessions/user-turn-transcript.js";

import { deliveryContextFromSession } from "../../utils/delivery-context.shared.js";

import {

parseInlineDirectives,

stripInlineDirectiveTagsForDelivery,

stripInlineDirectiveTagsForDisplay,

sanitizeReplyDirectiveId,

} from "../../utils/directive-tags.js";

import {

INTERNAL_MESSAGE_CHANNEL,

isGatewayCliClient,

isOperatorUiClient,

isWebchatClient,

normalizeMessageChannel,

} from "../../utils/message-channel.js";

import {

abortChatRunById,

boundInFlightRunSnapshotForChatHistory,

type ChatAbortControllerEntry,

type ChatAbortOps,

isChatStopCommandText,

registerChatAbortController,

resolveInFlightRunSnapshot,

updateChatRunProvider,

} from "../chat-abort.js";

import {

type ChatImageContent,

MediaOffloadError,

type OffloadedRef,

parseMessageWithAttachments,

resolveChatAttachmentMaxBytes,

UnsupportedAttachmentError,

} from "../chat-attachments.js";

import {

augmentChatHistoryWithCanvasBlocks,

dropPreSessionStartAnnouncePairs,

projectChatDisplayMessage,

projectRecentChatDisplayMessages,

resolveEffectiveChatHistoryMaxChars,

} from "../chat-display-projection.js";

import { sanitizeChatSendMessageInput } from "../chat-input-sanitize.js";

import { stripEnvelopeFromMessage } from "../chat-sanitize.js";

import { augmentChatHistoryWithCliSessionImports } from "../cli-session-history.js";

import { isSuppressedControlReplyText } from "../control-reply-text.js";

import {

attachManagedOutgoingImagesToMessage,

cleanupManagedOutgoingImageRecords,

createManagedOutgoingImageBlocks,

} from "../managed-image-attachments.js";

import { ADMIN_SCOPE } from "../method-scopes.js";

import { chatAbortMarkerTimestampMs, type ChatRunTiming } from "../server-chat-state.js";

import { getMaxChatHistoryMessagesBytes, MAX_PAYLOAD_BYTES } from "../server-constants.js";

import { resolveSessionHistoryTailReadOptions } from "../session-history-state.js";

import { readSessionTranscriptIndex } from "../session-transcript-index.fs.js";

import {

capArrayByJsonBytes,

readSessionMessageByIdAsync,

readRecentSessionMessagesAsync,

readSessionMessagesAsync,

} from "../session-transcript-readers.js";

import {

buildGatewaySessionInfo,

getSessionDefaults,

loadSessionEntry,

listAgentsForGateway,

resolveGatewayModelSupportsImages,

resolveDeletedAgentIdFromSessionKey,

resolveSessionModelRef,

resolveSessionStoreKey,

} from "../session-utils.js";

import { formatForLog } from "../ws-log.js";

import { setGatewayDedupeEntry } from "./agent-wait-dedupe.js";

import { normalizeRpcAttachmentsToChatAttachments } from "./attachment-normalize.js";

import { normalizeWebchatReplyMediaPathsForDisplay } from "./chat-reply-media.js";

import {

appendInjectedAssistantMessageToTranscript,

type GatewayInjectedTtsSupplementMarker,

} from "./chat-transcript-inject.js";

import {

buildWebchatAssistantMessageFromReplyPayloads,

buildWebchatAudioContentBlocksFromReplyPayloads,

} from "./chat-webchat-media.js";

import {

loadOptionalServerMethodModelCatalog,

startOptionalServerMethodModelCatalogLoad,

} from "./optional-model-catalog.js";

import { hasTrackedActiveSessionRun } from "./session-active-runs.js";

import { emitSessionsChanged } from "./session-change-event.js";

import type {

GatewayClient,

GatewayRequestContext,

GatewayRequestHandlerOptions,

GatewayRequestHandlers,

} from "./types.js";

type TranscriptAppendResult = {

ok: boolean;

messageId?: string;

message?: Record<string, unknown>;

error?: string;

};

type AbortOrigin = "rpc" | "stop-command";

type AbortedPartialSnapshot = {

runId: string;

sessionId: string;

agentId?: string;

text: string;

abortOrigin: AbortOrigin;

};

type ChatAbortRequester = {

connId?: string;

deviceId?: string;

isAdmin: boolean;

};

type PreRegisteredAgentDedupePayload = {

agentId?: unknown;

controlUiVisible?: unknown;

dedupeKeys?: unknown;

ownerConnId?: unknown;

ownerDeviceId?: unknown;

runId?: unknown;

sessionKey?: unknown;

status?: unknown;

};

type PreRegisteredAgentRun = {

runId: string;

sessionKey: string;

payload: PreRegisteredAgentDedupePayload;

};

type ChatHistoryMethod = "chat.history" | "chat.startup";

type ChatMetadataResult = {

commands?: unknown[];

models?: unknown[];

};

type ChatSendAckServerTiming = {

receivedToAckMs: number;

loadSessionMs: number;

prepareAttachmentsMs?: number;

};

type ChatSendServerTimingPhase =

| "dispatch-started"

| "model-selected"

| "agent-run-started"

| "first-assistant-event"

| "dispatch-completed"

| "post-dispatch-completed";

function roundedChatSendTimingMs(value: number): number {

return Math.max(0, Math.round(value * 1000) / 1000);

}

function chatSendAckServerTimingAttributes(

timing: ChatSendAckServerTiming | undefined,

): Record<string, number> {

if (!timing) {

return {};

}

return {

serverReceivedToAckMs: timing.receivedToAckMs,

serverLoadSessionMs: timing.loadSessionMs,

...(timing.prepareAttachmentsMs !== undefined

? { serverPrepareAttachmentsMs: timing.prepareAttachmentsMs }

: {}),

};

}

function shouldIncludeChatSendAckServerTiming(client?: {

id?: string | null;

mode?: string | null;

}): boolean {

return isOperatorUiClient(client);

}

function emitOperatorChatSendServerTiming(params: {

context: Pick<GatewayRequestContext, "broadcastToConnIds">;

client?: GatewayClient | null;

phase: ChatSendServerTimingPhase;

runId: string;

sessionKey: string;

agentId?: string;

receivedAtMs: number;

ackedAtMs: number;

dispatchStartedAtMs?: number;

extra?: Record<string, string | number>;

}) {

const connId =

typeof params.client?.connId === "string" && params.client.connId.trim()

? params.client.connId.trim()

: undefined;

if (!connId || !isOperatorUiClient(params.client?.connect?.client)) {

return;

}

const nowMs = performance.now();

params.context.broadcastToConnIds(

"chat.send_timing",

{

phase: params.phase,

runId: params.runId,

sessionKey: params.sessionKey,

...(params.agentId ? { agentId: params.agentId } : {}),

ackToPhaseMs: roundedChatSendTimingMs(nowMs - params.ackedAtMs),

receivedToPhaseMs: roundedChatSendTimingMs(nowMs - params.receivedAtMs),

...(params.dispatchStartedAtMs !== undefined

? {

dispatchStartedToPhaseMs: roundedChatSendTimingMs(nowMs - params.dispatchStartedAtMs),

}

: {}),

...params.extra,

},

new Set([connId]),

{ dropIfSlow: true },

);

}

async function handleChatMetadataRequest({

params,

respond,

context,

}: GatewayRequestHandlerOptions): Promise<void> {

if (!validateChatMetadataParams(params)) {

respond(

false,

undefined,

errorShape(

ErrorCodes.INVALID_REQUEST,

`invalid chat.metadata params: ${formatValidationErrors(validateChatMetadataParams.errors)}`,

),

);

return;

}

const metadataParams = params;

const cfg = context.getRuntimeConfig();

const requestedAgentId =

typeof metadataParams.agentId === "string" && metadataParams.agentId.trim()

? normalizeAgentId(metadataParams.agentId)

: resolveDefaultAgentId(cfg);

if (!listAgentIds(cfg).includes(requestedAgentId)) {

respond(

false,

undefined,

errorShape(ErrorCodes.INVALID_REQUEST, `Unknown agent id "${metadataParams.agentId}"`),

);

return;

}

try {

respond(

true,

await buildChatMetadataResult({

cfg,

context,

agentId: requestedAgentId,

}),

);

} catch (err) {

respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, String(err)));

}

}

async function buildChatMetadataResult(params: {

cfg: OpenClawConfig;

context: GatewayRequestContext;

agentId: string;

preloadedModelCatalog?: ModelCatalogEntry[];

}): Promise<ChatMetadataResult> {

const [{ buildModelsListResult }, { buildCommandsListResult }] = await Promise.all([

import("./models-list-result.js"),

import("./commands-list-result.js"),

]);

const [models, commands] = await Promise.all([

buildModelsListResult({

context: params.context,

agentId: params.agentId,

params: { view: "configured" },

preloadedCatalog: params.preloadedModelCatalog,

}),

Promise.resolve(

buildCommandsListResult({

cfg: params.cfg,

agentId: params.agentId,

includeArgs: true,

scope: "text",

}),

),

]);

return { ...models, ...commands };

}

async function buildChatStartupMetadataResult(params: {

cfg: OpenClawConfig;

context: GatewayRequestContext;

agentId: string;

modelCatalog: ModelCatalogEntry[] | undefined;

}): Promise<ChatMetadataResult | undefined> {

if (!params.modelCatalog) {

return undefined;

}

if (modelCatalogBrowseRequiresFullDiscovery({ cfg: params.cfg, view: "configured" })) {

return undefined;

}

try {

const { buildModelsListResult } = await import("./models-list-result.js");

return await buildModelsListResult({

context: params.context,

agentId: params.agentId,

params: { view: "configured" },

preloadedCatalog: params.modelCatalog,

});

} catch (err) {

params.context.logGateway.debug(

`chat.startup continuing without metadata: ${formatErrorMessage(err)}`,

);

return undefined;

}

}

function normalizeUnknownText(value: unknown): string | undefined {

return typeof value === "string" ? normalizeOptionalText(value) : undefined;

}

/** True when a reply payload carries at least one media reference (mediaUrl or mediaUrls). */

function isMediaBearingPayload(payload: ReplyPayload): boolean {

if (payload.isReasoning === true) {

return false;

}

if (payload.mediaUrl?.trim()) {

return true;

}

if (payload.mediaUrls?.some((url) => url.trim())) {

return true;

}

return false;

}

function stripVisibleTextFromTtsSupplement(payload: ReplyPayload): ReplyPayload {

return isReplyPayloadTtsSupplement(payload) ? buildTtsSupplementMediaPayload(payload) : payload;

}

function resolveTtsSupplementMarkerText(text: string): string {

const trimmed = text.trim();

const projected = projectChatDisplayMessage(

{

role: "assistant",

content: [{ type: "text", text: trimmed }],

},

{ maxChars: Number.MAX_SAFE_INTEGER },

);

const projectedContent = Array.isArray(projected?.content)

? (projected.content as AssistantDisplayContentBlock[])

: undefined;

return (

extractAssistantDisplayTextFromContent(projectedContent) ??

(typeof projected?.text === "string" ? projected.text.trim() : undefined) ??

trimmed

);

}

function buildTtsSupplementTranscriptMarker(

payload: ReplyPayload,

): GatewayInjectedTtsSupplementMarker | undefined {

const supplement = getReplyPayloadTtsSupplement(payload);

if (!supplement) {

return undefined;

}

const visibleText = resolveTtsSupplementMarkerText(

payload.text?.trim() || supplement.spokenText.trim(),

);

return {

textSha256: createHash("sha256").update(visibleText).digest("hex"),

};

}

function buildMediaOnlyTtsSupplementTranscriptMarker(

payload: ReplyPayload,

): GatewayInjectedTtsSupplementMarker | undefined {

if (payload.text?.trim()) {

return undefined;

}

return buildTtsSupplementTranscriptMarker(payload);

}

function resolveWebchatPromptCacheKey(params: {

agentId: string;

model: string;

provider: string;

sessionKey: string;

}): string {

const digest = createHash("sha256")

.update(

[

"v1",

params.provider.trim().toLowerCase(),

params.model.trim(),

normalizeAgentId(params.agentId),

params.sessionKey,

].join("\0"),

"utf8",

)

.digest("hex")

.slice(0, 32);

return `openclaw-webchat-${digest}`;

}

async function buildWebchatAssistantMediaMessage(

payloads: ReplyPayload[],

options?: {

localRoots?: readonly string[];

onLocalAudioAccessDenied?: (message: string) => void;

},

): Promise<{ content: Array<Record<string, unknown>>; transcriptText: string } | null> {

return buildWebchatAssistantMessageFromReplyPayloads(payloads, {

localRoots: options?.localRoots,

onLocalAudioAccessDenied: (err) => {

options?.onLocalAudioAccessDenied?.(formatForLog(err));

},

});

}

export {

augmentChatHistoryWithCanvasBlocks,

DEFAULT_CHAT_HISTORY_TEXT_MAX_CHARS,

dropPreSessionStartAnnouncePairs,

resolveEffectiveChatHistoryMaxChars,

sanitizeChatHistoryMessages,

} from "../chat-display-projection.js";

export { sanitizeChatSendMessageInput } from "../chat-input-sanitize.js";

export const CHAT_HISTORY_MAX_SINGLE_MESSAGE_BYTES = 128 * 1024;

const CHAT_HISTORY_OVERSIZED_PLACEHOLDER = "[chat.history omitted: message too large]";

const CHAT_HISTORY_UNAVAILABLE_SENTINEL =

"[chat.history unavailable: transcript too large to display; the full history is preserved on disk]";

/**

* A minimal, metadata-free notice returned when even a single oversized

* placeholder cannot fit the chat-history byte budget. Returning this instead

* of an empty array guarantees the dashboard never renders a blank transcript,

* which otherwise reads to the operator as total history loss.

*/

function buildChatHistoryUnavailableSentinel(): Record<string, unknown> {

return {

role: "assistant",

timestamp: Date.now(),

content: [{ type: "text", text: CHAT_HISTORY_UNAVAILABLE_SENTINEL }],

};

}

const CHAT_STARTUP_OPTIONAL_MODEL_CATALOG_TIMEOUT_MS = 25;

const MANAGED_OUTGOING_IMAGE_PATH_PREFIX = "/api/chat/media/outgoing/";

let chatHistoryPlaceholderEmitCount = 0;

const chatHistoryManagedImageCleanupState = new Map<string, Promise<void>>();

const CHANNEL_AGNOSTIC_SESSION_SCOPES = new Set([

"main",

"direct",

"dm",

"group",

"channel",

"cron",

"run",

"subagent",

"acp",

"thread",

"topic",

]);

const CHANNEL_SCOPED_SESSION_SHAPES = new Set(["direct", "dm", "group", "channel"]);

type ChatSendDeliveryEntry = {

route?: ChannelRouteRef;

deliveryContext?: {

channel?: string;

to?: string;

accountId?: string;

threadId?: string | number;

};

origin?: {

provider?: string;

accountId?: string;

threadId?: string | number;

};

lastChannel?: string;

lastTo?: string;

lastAccountId?: string;

lastThreadId?: string | number;

};

type ChatSendOriginatingRoute = {

originatingChannel: string;

originatingTo?: string;

accountId?: string;

messageThreadId?: string | number;

explicitDeliverRoute: boolean;

};

const ACTIVE_CHAT_SEND_DEDUPE_PREFIX = "chat:active-send";

function resolveActiveChatSendRunId(value: unknown): string | null {

if (!value || typeof value !== "object" || Array.isArray(value)) {

return null;

}

const runId = (value as { runId?: unknown }).runId;

return typeof runId === "string" && runId.trim() ? runId : null;

}

function clearActiveChatSendDedupeRun(

dedupe: GatewayRequestContext["dedupe"],

key: string | null,

runId: string,

) {

if (!key || resolveActiveChatSendRunId(dedupe.get(key)?.payload) !== runId) {

return;

}

dedupe.delete(key);

}

function buildAbortedChatSendPayload(params: {

runId: string;

endedAt: number;

stopReason?: string;

}) {

return {

runId: params.runId,

status: "timeout" as const,

summary: "aborted",

...(params.stopReason ? { stopReason: params.stopReason } : {}),

endedAt: params.endedAt,

};

}

function buildActiveChatSendDedupeKey(params: {

attachmentCount: number;

explicitDeliverRoute: boolean;

message: string;

originatingChannel: string;

sessionKey: string;

systemScope?: string;

}): string | null {

const message = params.message.trim();

if (

!message ||

message.startsWith("/") ||

params.attachmentCount > 0 ||

params.explicitDeliverRoute ||

normalizeMessageChannel(params.originatingChannel) !== INTERNAL_MESSAGE_CHANNEL

) {

return null;

}

const dedupeParts = params.systemScope?.trim()

? [params.sessionKey, message, params.systemScope.trim()]

: [params.sessionKey, message];

const digest = createHash("sha256")

.update(JSON.stringify(dedupeParts))

.digest("hex")

.slice(0, 32);

return `${ACTIVE_CHAT_SEND_DEDUPE_PREFIX}:${digest}`;

}

function validateChatSelectedAgent(params: {

cfg: OpenClawConfig;

requestedSessionKey: string;

agentId?: string;

}): { ok: true; agentId?: string } | { ok: false; error: string } {

const agentId = params.agentId ? normalizeAgentId(params.agentId) : undefined;

if (!agentId) {

return { ok: true };

}

if (!listAgentIds(params.cfg).includes(agentId)) {

return { ok: false, error: `Unknown agent id "${params.agentId}"` };

}

const requestedSessionKey = params.requestedSessionKey.trim();

const parsed = parseAgentSessionKey(requestedSessionKey);

if (parsed && normalizeAgentId(parsed.agentId) !== agentId) {

return {

ok: false,

error: `agentId "${params.agentId}" does not match session key "${params.requestedSessionKey}"`,

};

}

if (requestedSessionKey.toLowerCase() === "global") {

return { ok: true, agentId };

}

if (resolveSessionStoreKey({ cfg: params.cfg, sessionKey: requestedSessionKey }) === "global") {

return { ok: true, agentId };

}

if (!parsed || normalizeAgentId(parsed.agentId) !== agentId) {

return {

ok: false,

error: `agentId "${params.agentId}" does not match session key "${params.requestedSessionKey}"`,

};

}

return { ok: true, agentId };

}

function resolveRequestedChatAgentId(params: {

cfg?: OpenClawConfig;

requestedSessionKey: string;

agentId?: string;

}): string | undefined {

const explicitAgentId = normalizeOptionalText(params.agentId);

if (explicitAgentId) {

return normalizeAgentId(explicitAgentId);

}

if (!params.cfg) {

return undefined;

}

const parsed = parseAgentSessionKey(params.requestedSessionKey.trim());

if (

!parsed?.agentId ||

resolveSessionStoreKey({ cfg: params.cfg, sessionKey: params.requestedSessionKey }) !== "global"

) {

return undefined;

}

return normalizeAgentId(parsed.agentId);

}

function resolveChatSendActiveScopeKey(params: {

sessionKey: string;

agentId?: string;

mainKey?: string;

}): string {

if (params.sessionKey !== "global" || !params.agentId) {

return params.sessionKey;

}

return (

scopeLegacySessionKeyToAgent({

agentId: params.agentId,

sessionKey: params.sessionKey,

mainKey: params.mainKey,

}) ?? params.sessionKey

);

}

type ChatSendExplicitOrigin = {

originatingChannel?: string;

originatingTo?: string;

accountId?: string;

messageThreadId?: string;

};

function formatAttachmentFailureForLog(err: unknown): string {

const primary = formatUncaughtError(err);

const cause = err instanceof Error ? err.cause : undefined;

if (cause === undefined) {

return primary;

}

const causeText = formatUncaughtError(cause);

if (!causeText || causeText === primary) {

return primary;

}

return `${primary}\nCaused by: ${causeText}`;

}

function logAttachmentFailure(

logGateway: Pick<GatewayRequestContext["logGateway"], "error">,

label: string,

err: unknown,

): void {

logGateway.error(label, {

error: formatAttachmentFailureForLog(err),

consoleMessage: `${label}: ${formatForLog(err)}`,

});

}

type SideResultPayload = {

kind: "btw";

runId: string;

sessionKey: string;

agentId?: string;

question: string;

text: string;

isError?: boolean;

ts: number;

};

function buildTranscriptReplyText(payloads: ReplyPayload[]): string {

const chunks = payloads

.map((payload) => {

if (payload.isReasoning === true) {

return "";

}

const parts = resolveSendableOutboundReplyParts(payload);

const lines: string[] = [];

const parsedText = payload.text?.includes("[[")

? parseInlineDirectives(payload.text)

: undefined;

const replyToId =

sanitizeReplyDirectiveId(payload.replyToId) ??

sanitizeReplyDirectiveId(parsedText?.replyToExplicitId);

if (replyToId) {

lines.push(`[[reply_to:${replyToId}]]`);

} else if (payload.replyToCurrent || parsedText?.replyToCurrent) {

lines.push("[[reply_to_current]]");

}

const text = payload.text

? stripInlineDirectiveTagsForDelivery(payload.text).text.trim()

: "";

if (text && !isSuppressedControlReplyText(text)) {

lines.push(text);

}

for (const mediaUrl of parts.mediaUrls) {

if (payload.sensitiveMedia === true) {

continue;

}

const trimmed = mediaUrl.trim();

if (trimmed) {

lines.push(`Attachment: ${trimmed}`);

}

}

if (

(payload.audioAsVoice || parsedText?.audioAsVoice) &&

parts.mediaUrls.some((mediaUrl) => isAudioFileName(mediaUrl))

) {

lines.push("[[audio_as_voice]]");

}

return lines.join("\n").trim();

})

.filter(Boolean);

return chunks.join("\n\n").trim();

}

function hasSensitiveMediaPayload(payloads: ReplyPayload[]): boolean {

return payloads.some(

(payload) => payload.sensitiveMedia === true && isMediaBearingPayload(payload),

);

}

type AssistantDisplayContentBlock = Record<string, unknown>;

function sanitizeAssistantDisplayText(value?: string | null): string | undefined {

if (!value) {

return undefined;

}

const withoutEnvelope = stripEnvelopeFromMessage(value);

const normalized = typeof withoutEnvelope === "string" ? withoutEnvelope : value;

const stripped = stripInlineDirectiveTagsForDisplay(normalized).text.trim();

return stripped || undefined;

}

function extractAssistantDisplayTextFromContent(

content?: readonly AssistantDisplayContentBlock[] | null,

): string | undefined {

if (!Array.isArray(content) || content.length === 0) {

return undefined;

}

const parts = content

.map((block) => {

if (block?.type !== "text" || typeof block.text !== "string") {

return "";

}

return block.text.trim();

})

.filter(Boolean);

return parts.length > 0 ? parts.join("\n\n") : undefined;

}

async function buildAssistantDisplayContentFromReplyPayloads(params: {

sessionKey: string;

agentId?: string;

payloads: ReplyPayload[];

managedImageLocalRoots?: Parameters<typeof createManagedOutgoingImageBlocks>[0]["localRoots"];

includeSensitiveMedia?: boolean;

onLocalAudioAccessDenied?: (message: string) => void;

onManagedImagePrepareError?: (message: string) => void;

}): Promise<AssistantDisplayContentBlock[] | undefined> {

const rawTextPayloadCount = params.payloads.filter(

(payload) =>

payload.isReasoning !== true &&

typeof payload.text === "string" &&

payload.text.trim().length > 0,

).length;

const normalized = normalizeReplyPayloadsForDelivery(params.payloads);

if (normalized.length === 0) {

return rawTextPayloadCount > 0 ? [{ type: "text", text: "" }] : undefined;

}

const content: AssistantDisplayContentBlock[] = [];

let strippedTextPayloadCount = 0;

for (const payload of normalized) {

const text = sanitizeAssistantDisplayText(payload.text);

if (text) {

content.push({ type: "text", text });

} else if (typeof payload.text === "string" && payload.text.trim().length > 0) {

strippedTextPayloadCount += 1;

}

if (params.includeSensitiveMedia === false && payload.sensitiveMedia === true) {

continue;

}

const audioBlocks = await buildWebchatAudioContentBlocksFromReplyPayloads([payload], {

localRoots: Array.isArray(params.managedImageLocalRoots)

? params.managedImageLocalRoots

: undefined,

onLocalAudioAccessDenied: (err) => {

params.onLocalAudioAccessDenied?.(formatForLog(err));

},

});

content.push(...audioBlocks);

const mediaUrls = Array.from(

new Set([

...(Array.isArray(payload.mediaUrls) ? payload.mediaUrls : []),

...(typeof payload.mediaUrl === "string" ? [payload.mediaUrl] : []),

]),

);

const imageBlocks = await createManagedOutgoingImageBlocks({

sessionKey: params.sessionKey,

...(params.sessionKey === "global" && params.agentId ? { agentId: params.agentId } : {}),

mediaUrls,

localRoots: params.managedImageLocalRoots,

continueOnPrepareError: true,

onPrepareError: (error) => {

params.onManagedImagePrepareError?.(error.message);

},

});

if (imageBlocks.length > 0) {

content.push(...imageBlocks);

}

}

if (content.length > 0) {

return content;

}

return strippedTextPayloadCount > 0 ? [{ type: "text", text: "" }] : undefined;

}

function replaceAssistantContentTextBlocks(

content: readonly AssistantDisplayContentBlock[] | undefined,

transcriptMediaMessage: { content: Array<Record<string, unknown>> } | null,

): AssistantDisplayContentBlock[] | undefined {

const transcriptTextBlocks = (transcriptMediaMessage?.content ?? []).filter(

(block): block is AssistantDisplayContentBlock =>

Boolean(block) &&

typeof block === "object" &&

block.type === "text" &&

typeof block.text === "string",

);

if (transcriptTextBlocks.length === 0) {

return content ? [...content] : undefined;

}

if (!content || content.length === 0) {

return [...transcriptTextBlocks];

}

const merged: AssistantDisplayContentBlock[] = [];

let transcriptTextIndex = 0;

for (const block of content) {

if (

block?.type === "text" &&

typeof block.text === "string" &&

transcriptTextIndex < transcriptTextBlocks.length

) {

merged.push(transcriptTextBlocks[transcriptTextIndex++]);

continue;

}

merged.push(block);

}

if (transcriptTextIndex < transcriptTextBlocks.length) {

merged.unshift(...transcriptTextBlocks.slice(transcriptTextIndex));

}

return merged;

}

function isManagedOutgoingImageUrl(value: unknown): boolean {

if (typeof value !== "string" || !value.trim()) {

return false;

}

try {

const parsed = new URL(value, "http://localhost");

return parsed.pathname.startsWith(MANAGED_OUTGOING_IMAGE_PATH_PREFIX);

} catch {

return false;

}

}

function stripManagedOutgoingAssistantContentBlocks(

content: readonly AssistantDisplayContentBlock[] | undefined,

): AssistantDisplayContentBlock[] | undefined {

if (!content || content.length === 0) {

return undefined;

}

const filtered = content.filter((block) => {

if (block?.type !== "image") {

return true;

}

return !(isManagedOutgoingImageUrl(block.url) || isManagedOutgoingImageUrl(block.openUrl));

});

return filtered.length > 0 ? filtered : undefined;

}

function extractAssistantDisplayText(

content: readonly AssistantDisplayContentBlock[] | undefined,

): string | undefined {

if (!content || content.length === 0) {

return undefined;

}

const text = content