Skip to content

Commit 83303f1

Browse files
steipetesteipete
authored
feat(channels): batch 1 producers drop media placeholder bodies (#111447)
* feat(channels): batch 1 producers drop media placeholder bodies Media-placeholder program batch 1: Google Chat, Zalo, LINE, and Mattermost stop minting <media:kind> placeholder bodies. Media-only messages carry an empty caption plus one structured fact per native attachment (type-only when a download fails or is rejected, so payload positions and kind signals stay aligned). The shared formatMediaPlaceholderText SDK formatter renders text-only carriers (Mattermost pending-room lines) from structured facts; per-channel placeholder builders and the expected-count side channel are deleted. * fix(mattermost): satisfy type, deadcode, and SDK manifest gates
1 parent 4c55a4f commit 83303f1

18 files changed

Lines changed: 543 additions & 135 deletions

‎docs/.generated/plugin-sdk-api-baseline.sha256‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ e6c20ca52397a90297c49928e9b8d98a48059ed107d21f8cec6a7ec9c9e2d1ab module/channel
4848
fbf353eb38ae68d8ded3f2a60b432c7bb2c245d2ec7e7c9f53c6da19a0db0938 module/channel-entry-contract
4949
264a002a036aa66f87008971130c0264b15e62b00a35f473149aa846e3bcddad module/channel-envelope
5050
982f29a18e07228e3da82cae67d06ff38249592a29c2fd28f01f0d2016ff80d9 module/channel-feedback
51-
49bc4ba7ac308e2ca926f1e5d5d7cdad7d022be102da9f58e11311032bbf0223 module/channel-inbound
51+
19d42de92c4f27a6d785638d6118a2e1302f0cb64cc71a12d6c1875e1e9e1e9a module/channel-inbound
5252
2940039d5ebdb16c1d745914390db01c0f5f1e9cff33f95ea36c9e77d97c2bae module/channel-inbound-debounce
5353
c84239139c3ce21cc8ca7c01ae9f2a653197afdc7be95a7598616e112e743f4f module/channel-inbound-roots
5454
91425ad75ddb58a3796f7bc88570cb2c81b3db0dd6521a7ce0e55f9148880a47 module/channel-ingress-runtime

‎docs/plugins/sdk-channel-inbound.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ import {
3838
- `dispatchChannelInboundReply(...)`: records and dispatches an already
3939
assembled inbound reply with a delivery adapter.
4040

41+
For media-only inbound events, keep the message body and command text empty and
42+
pass one `ChannelInboundMediaInput` fact per native attachment. When an ambient
43+
history line or another text-only carrier must describe those facts, use
44+
`formatMediaPlaceholderText(media)`. It classifies each fact from `kind`, MIME
45+
type, then path or URL extension; undownloaded native attachments should still
46+
contribute one type-only fact each. Do not use the formatter to synthesize the
47+
primary inbound body.
48+
4149
Bundled/native channels that already receive the injected plugin runtime
4250
object can call the same helpers under `runtime.channel.inbound.*` instead of
4351
importing this subpath directly:

‎extensions/googlechat/src/monitor.test.ts‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,18 @@ beforeEach(() => {
8383
function createInboundClassificationHarness() {
8484
const buildContext = vi.fn((payload: unknown) => payload);
8585
const runTurn = vi.fn();
86+
const saveMediaBuffer = vi.fn(async () => ({
87+
path: "/tmp/googlechat-first.png",
88+
contentType: "image/png",
89+
}));
8690
const core = {
8791
logging: { shouldLogVerbose: () => false },
8892
channel: {
8993
inbound: { buildContext, run: runTurn },
94+
media: { saveMediaBuffer },
9095
},
9196
} as unknown as GoogleChatCoreRuntime;
92-
return { buildContext, core, runTurn };
97+
return { buildContext, core, runTurn, saveMediaBuffer };
9398
}
9499

95100
async function processGoogleChatTestEvent(params: {
@@ -253,6 +258,79 @@ describe("googlechat monitor inbound space classification", () => {
253258
expect(runTurn).toHaveBeenCalledOnce();
254259
});
255260

261+
it("keeps media-only text empty and carries every native attachment fact", async () => {
262+
const { buildContext, core, runTurn, saveMediaBuffer } = createInboundClassificationHarness();
263+
apiMocks.downloadGoogleChatMedia.mockResolvedValue({
264+
buffer: Buffer.from("image"),
265+
contentType: "image/png",
266+
});
267+
accessMocks.applyGoogleChatInboundAccessPolicy.mockResolvedValue({
268+
ok: true,
269+
commandAuthorized: undefined,
270+
effectiveWasMentioned: undefined,
271+
groupBotLoopProtection: undefined,
272+
groupSystemPrompt: undefined,
273+
});
274+
275+
await processGoogleChatTestEvent({
276+
event: {
277+
type: "MESSAGE",
278+
space: { name: "spaces/MEDIA", type: "DM" },
279+
message: {
280+
name: "spaces/MEDIA/messages/1",
281+
sender: { name: "users/alice", displayName: "Alice", type: "HUMAN" },
282+
attachment: [
283+
{
284+
contentType: "image/png",
285+
contentName: "first.png",
286+
attachmentDataRef: { resourceName: "media/first" },
287+
},
288+
{ contentType: "application/pdf", contentName: "second.pdf" },
289+
],
290+
},
291+
},
292+
account: {
293+
accountId: "work",
294+
config: { typingIndicator: "none" },
295+
credentialSource: "inline",
296+
} as ResolvedGoogleChatAccount,
297+
config: {},
298+
runtime: { error: vi.fn(), log: vi.fn() },
299+
core,
300+
mediaMaxMb: 10,
301+
});
302+
303+
expect(accessMocks.applyGoogleChatInboundAccessPolicy).toHaveBeenCalledWith(
304+
expect.objectContaining({ rawBody: "" }),
305+
);
306+
expect(saveMediaBuffer).toHaveBeenCalledOnce();
307+
expect(buildContext).toHaveBeenCalledWith(
308+
expect.objectContaining({
309+
message: { body: "", bodyForAgent: "", rawBody: "", commandBody: "" },
310+
media: [
311+
expect.objectContaining({
312+
path: "/tmp/googlechat-first.png",
313+
url: "/tmp/googlechat-first.png",
314+
contentType: "image/png",
315+
}),
316+
expect.objectContaining({ contentType: "application/pdf" }),
317+
],
318+
}),
319+
);
320+
const runArg = runTurn.mock.calls[0]?.[0] as
321+
| {
322+
adapter?: {
323+
ingest?: () => { rawText: string; textForAgent: string; textForCommands: string };
324+
};
325+
}
326+
| undefined;
327+
expect(runArg?.adapter?.ingest?.()).toMatchObject({
328+
rawText: "",
329+
textForAgent: "",
330+
textForCommands: "",
331+
});
332+
});
333+
256334
it("passes durable ingress adoption ownership into the inbound turn", async () => {
257335
const { core, runTurn } = createInboundClassificationHarness();
258336
const turnAdoptionLifecycle = {

‎extensions/googlechat/src/monitor.ts‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
import {
33
recordChannelBotPairLoopAndCheckSuppression,
44
resolveChannelInboundRouteEnvelope,
5+
toInboundMediaFacts,
56
type ChannelBotLoopProtectionFacts,
7+
type ChannelInboundMediaInput,
68
} from "openclaw/plugin-sdk/channel-inbound";
79
import { mergePairLoopGuardConfig } from "openclaw/plugin-sdk/pair-loop-guard-runtime";
810
import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/string-coerce-runtime";
@@ -218,9 +220,8 @@ async function processMessageWithPipeline(params: {
218220

219221
const messageText = (message.argumentText ?? message.text ?? "").trim();
220222
const attachments = message.attachment ?? [];
221-
const hasMedia = attachments.length > 0;
222-
const rawBody = messageText || (hasMedia ? "<media:attachment>" : "");
223-
if (!rawBody) {
223+
const rawBody = messageText;
224+
if (!rawBody && attachments.length === 0) {
224225
return;
225226
}
226227

@@ -271,16 +272,21 @@ async function processMessageWithPipeline(params: {
271272
},
272273
});
273274

274-
let mediaPath: string | undefined;
275-
let mediaType: string | undefined;
275+
const mediaInputs: ChannelInboundMediaInput[] = attachments.map((attachment) => ({
276+
contentType: attachment.contentType,
277+
}));
276278
const first = attachments.at(0);
277279
if (first) {
278280
const attachmentData = await downloadAttachment(first, account, mediaMaxMb, core);
279281
if (attachmentData) {
280-
mediaPath = attachmentData.path;
281-
mediaType = attachmentData.contentType;
282+
mediaInputs[0] = {
283+
path: attachmentData.path,
284+
url: attachmentData.path,
285+
contentType: attachmentData.contentType ?? first.contentType,
286+
};
282287
}
283288
}
289+
const media = toInboundMediaFacts(mediaInputs);
284290

285291
const fromLabel = isGroup
286292
? space.displayName || `space:${spaceId}`
@@ -330,16 +336,7 @@ async function processMessageWithPipeline(params: {
330336
rawBody,
331337
commandBody: rawBody,
332338
},
333-
media:
334-
mediaPath || mediaType
335-
? [
336-
{
337-
path: mediaPath,
338-
url: mediaPath,
339-
contentType: mediaType,
340-
},
341-
]
342-
: undefined,
339+
media: media.length > 0 ? media : undefined,
343340
supplemental: {
344341
groupSystemPrompt: isGroup ? groupSystemPrompt : undefined,
345342
},

‎extensions/line/src/bot-message-context.test.ts‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ describe("buildLineMessageContext", () => {
185185
);
186186
});
187187

188-
it("replaces a failed media placeholder with an unavailable notice", async () => {
188+
it("keeps failed media-only command text empty and preserves its native media fact", async () => {
189189
const event = createMessageEvent({ type: "user", userId: "user-image" }, {
190190
message: {
191191
id: "image-1",
@@ -203,10 +203,35 @@ describe("buildLineMessageContext", () => {
203203
commandAuthorized: true,
204204
});
205205

206-
expect(context?.ctxPayload.RawBody).toBe("<media:image>");
207-
expect(context?.ctxPayload.CommandBody).toBe("<media:image>");
206+
expect(context?.ctxPayload.RawBody).toBe("");
207+
expect(context?.ctxPayload.CommandBody).toBe("");
208208
expect(context?.ctxPayload.BodyForAgent).toBe("[line attachment unavailable]");
209209
expect(context?.ctxPayload.MediaPath).toBeUndefined();
210+
expect(context?.ctxPayload.MediaType).toBe("image");
211+
});
212+
213+
it("keeps materialized media-only text empty and projects structured media facts", async () => {
214+
const event = createMessageEvent({ type: "user", userId: "user-image" }, {
215+
message: {
216+
id: "image-2",
217+
type: "image",
218+
contentProvider: { type: "line" },
219+
},
220+
} as Partial<MessageEvent>);
221+
222+
const context = await buildLineMessageContext({
223+
event,
224+
allMedia: [{ path: "/tmp/line-image.png", contentType: "image/png" }],
225+
cfg,
226+
account,
227+
commandAuthorized: false,
228+
});
229+
230+
expect(context?.ctxPayload.RawBody).toBe("");
231+
expect(context?.ctxPayload.CommandBody).toBe("");
232+
expect(context?.ctxPayload.BodyForAgent).toBe("");
233+
expect(context?.ctxPayload.MediaPath).toBe("/tmp/line-image.png");
234+
expect(context?.ctxPayload.MediaType).toBe("image/png");
210235
});
211236

212237
it("routes group postback replies to the group id", async () => {

‎extensions/line/src/bot-message-context.ts‎

Lines changed: 25 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@
22
import type { webhook } from "@line/bot-sdk";
33
import { recordChannelActivity } from "openclaw/plugin-sdk/channel-activity-runtime";
44
import {
5+
buildChannelInboundMediaPayload,
56
formatInboundMediaUnavailableText,
67
formatInboundEnvelope,
78
formatLocationText,
89
resolveInboundSessionEnvelopeContext,
10+
toInboundMediaFacts,
911
toLocationContext,
12+
type ChannelInboundMediaInput,
1013
} from "openclaw/plugin-sdk/channel-inbound";
1114
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
1215
import {
@@ -225,18 +228,20 @@ function extractMessageText(message: MessageEvent["message"]): string {
225228
return "";
226229
}
227230

228-
function extractMediaPlaceholder(message: MessageEvent["message"]): string {
231+
function extractNativeMediaKind(
232+
message: MessageEvent["message"],
233+
): ChannelInboundMediaInput["kind"] | undefined {
229234
switch (message.type) {
230235
case "image":
231-
return "<media:image>";
236+
return "image";
232237
case "video":
233-
return "<media:video>";
238+
return "video";
234239
case "audio":
235-
return "<media:audio>";
240+
return "audio";
236241
case "file":
237-
return "<media:document>";
242+
return "document";
238243
default:
239-
return "";
244+
return undefined;
240245
}
241246
}
242247

@@ -288,12 +293,7 @@ async function finalizeLineInboundContext(params: {
288293
timestamp: number;
289294
messageSid: string;
290295
commandAuthorized: boolean;
291-
media: {
292-
firstPath: string | undefined;
293-
firstContentType?: string;
294-
paths?: string[];
295-
types?: string[];
296-
};
296+
media: readonly ChannelInboundMediaInput[];
297297
locationContext?: ReturnType<typeof toLocationContext>;
298298
verboseLog: { kind: "inbound" | "postback"; mediaCount?: number };
299299
inboundHistory?: Pick<HistoryEntry, "sender" | "body" | "timestamp">[];
@@ -322,6 +322,7 @@ async function finalizeLineInboundContext(params: {
322322
});
323323

324324
const agentBody = params.agentBody ?? params.rawBody;
325+
const mediaPayload = buildChannelInboundMediaPayload(toInboundMediaFacts(params.media));
325326
const body = formatInboundEnvelope({
326327
channel: "LINE",
327328
from: conversationLabel,
@@ -355,12 +356,7 @@ async function finalizeLineInboundContext(params: {
355356
Surface: "line",
356357
MessageSid: params.messageSid,
357358
Timestamp: params.timestamp,
358-
MediaPath: params.media.firstPath,
359-
MediaType: params.media.firstContentType,
360-
MediaUrl: params.media.firstPath,
361-
MediaPaths: params.media.paths,
362-
MediaUrls: params.media.paths,
363-
MediaTypes: params.media.types,
359+
...mediaPayload,
364360
...params.locationContext,
365361
CommandAuthorized: params.commandAuthorized,
366362
OriginatingChannel: "line" as const,
@@ -457,21 +453,22 @@ export async function buildLineMessageContext(params: BuildLineMessageContextPar
457453
const timestamp = event.timestamp;
458454

459455
const textContent = extractMessageText(message);
460-
const placeholder = extractMediaPlaceholder(message);
461-
462-
let rawBody = textContent || placeholder;
463-
if (!rawBody && allMedia.length > 0) {
464-
rawBody = `<media:image>${allMedia.length > 1 ? ` (${allMedia.length} images)` : ""}`;
465-
}
456+
const nativeMediaKind = extractNativeMediaKind(message);
457+
const mediaFacts: ChannelInboundMediaInput[] =
458+
allMedia.length > 0
459+
? allMedia.map((media) => ({ ...media, kind: nativeMediaKind }))
460+
: nativeMediaKind
461+
? [{ kind: nativeMediaKind }]
462+
: [];
463+
const rawBody = textContent;
466464
const agentBody = mediaUnavailable
467465
? formatInboundMediaUnavailableText({
468466
body: rawBody,
469-
mediaPlaceholder: placeholder,
470467
notice: "[line attachment unavailable]",
471468
})
472469
: rawBody;
473470

474-
if (!agentBody && allMedia.length === 0) {
471+
if (!agentBody && mediaFacts.length === 0) {
475472
return null;
476473
}
477474

@@ -497,15 +494,7 @@ export async function buildLineMessageContext(params: BuildLineMessageContextPar
497494
timestamp,
498495
messageSid: messageId,
499496
commandAuthorized,
500-
media: {
501-
firstPath: allMedia[0]?.path,
502-
firstContentType: allMedia[0]?.contentType,
503-
paths: allMedia.length > 0 ? allMedia.map((m) => m.path) : undefined,
504-
types:
505-
allMedia.length > 0
506-
? (allMedia.map((m) => m.contentType).filter(Boolean) as string[])
507-
: undefined,
508-
},
497+
media: mediaFacts,
509498
locationContext,
510499
verboseLog: { kind: "inbound", mediaCount: allMedia.length },
511500
inboundHistory,
@@ -564,12 +553,7 @@ export async function buildLinePostbackContext(params: {
564553
timestamp,
565554
messageSid,
566555
commandAuthorized,
567-
media: {
568-
firstPath: "",
569-
firstContentType: undefined,
570-
paths: undefined,
571-
types: undefined,
572-
},
556+
media: [],
573557
verboseLog: { kind: "postback" },
574558
});
575559

0 commit comments

Comments
 (0)