Skip to content

Commit 0ac7b24

Browse files
steipetesteipete
authored
feat(channels): batch 3 producers drop media placeholder bodies (#111800)
* feat(channels): batch 3 producers drop media placeholder bodies Media-placeholder program batch 3: iMessage, Signal, and WhatsApp stop minting <media:kind> placeholder bodies. Primary bodies carry captions only with one aligned structured fact per native attachment (type-only for missing/rejected/oversized). Echo, quote, and pending-history records gain structured media facts so dedupe and previews work without rendered text. Signal media-only outbound sends drop the placeholder body entirely (signal-cli accepts attachment-only sends; recipients now see just the media). WhatsApp audio preflight de-stringed; qa-lab sticker recognizer moved to structured facts; WhatsApp gif maps to video (presentation mode, no behavioral consumer). * test(whatsapp): keep media classifier private
1 parent 7bac638 commit 0ac7b24

58 files changed

Lines changed: 888 additions & 466 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎extensions/imessage/src/monitor.media-policy.test.ts‎

Lines changed: 79 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -210,8 +210,13 @@ describe("iMessage monitor attachment policy", () => {
210210
missing: true,
211211
},
212212
],
213-
staged: { attachments: [], unavailableCount: 1 },
213+
staged: {
214+
attachments: [{ contentType: "image/heic", kind: "image" as const }],
215+
unavailableCount: 1,
216+
},
214217
expectedBody: "[imessage attachment unavailable]",
218+
expectedMediaTypes: ["image/heic"],
219+
expectedMediaUrls: undefined,
215220
},
216221
{
217222
name: "uses the first materialized attachment type when earlier media is unavailable",
@@ -229,76 +234,89 @@ describe("iMessage monitor attachment policy", () => {
229234
],
230235
staged: {
231236
attachments: [
237+
{ contentType: "image/heic", kind: "image" as const },
232238
{
233239
path: "/Users/openclaw/Library/Messages/Attachments/report.pdf",
234240
contentType: "application/pdf",
241+
kind: "document" as const,
235242
},
236243
],
237244
unavailableCount: 1,
238245
},
239-
expectedBody: "<media:document>\n\n[imessage attachment unavailable]",
246+
expectedBody: "[imessage attachment unavailable]",
247+
expectedMediaTypes: ["image/heic", "application/pdf"],
248+
expectedMediaUrls: ["", "/Users/openclaw/Library/Messages/Attachments/report.pdf"],
240249
},
241-
])("$name", async ({ name, attachments, staged, expectedBody }) => {
242-
stageIMessageAttachmentsMock.mockResolvedValue(staged);
243-
const runtime = { error: vi.fn(), exit: vi.fn(), log: vi.fn() };
244-
let onNotification:
245-
| ((message: { method: string; params: unknown }) => void | Promise<void>)
246-
| undefined;
247-
const client = {
248-
request: vi.fn(async () => ({ subscription: 1 })),
249-
waitForClose: vi.fn(async () => {
250-
await onNotification?.({
251-
method: "message",
252-
params: {
253-
message: {
254-
id: 1,
255-
guid: `inbound-media-guid-${name}-${Date.now()}`,
256-
chat_id: 123,
257-
chat_identifier: "+15550001111",
258-
sender: "+15550001111",
259-
is_from_me: false,
260-
is_group: false,
261-
text: "",
262-
created_at: new Date().toISOString(),
263-
attachments,
250+
])(
251+
"$name",
252+
async ({ name, attachments, staged, expectedBody, expectedMediaTypes, expectedMediaUrls }) => {
253+
stageIMessageAttachmentsMock.mockResolvedValue(staged);
254+
const runtime = { error: vi.fn(), exit: vi.fn(), log: vi.fn() };
255+
let onNotification:
256+
| ((message: { method: string; params: unknown }) => void | Promise<void>)
257+
| undefined;
258+
const client = {
259+
request: vi.fn(async () => ({ subscription: 1 })),
260+
waitForClose: vi.fn(async () => {
261+
await onNotification?.({
262+
method: "message",
263+
params: {
264+
message: {
265+
id: 1,
266+
guid: `inbound-media-guid-${name}-${Date.now()}`,
267+
chat_id: 123,
268+
chat_identifier: "+15550001111",
269+
sender: "+15550001111",
270+
is_from_me: false,
271+
is_group: false,
272+
text: "",
273+
created_at: new Date().toISOString(),
274+
attachments,
275+
},
264276
},
265-
},
266-
});
267-
await Promise.resolve();
268-
await Promise.resolve();
269-
}),
270-
stop: vi.fn(async () => {}),
271-
};
272-
createIMessageRpcClientMock.mockImplementation(async (params) => {
273-
onNotification = params?.onNotification;
274-
return client as never;
275-
});
277+
});
278+
await Promise.resolve();
279+
await Promise.resolve();
280+
}),
281+
stop: vi.fn(async () => {}),
282+
};
283+
createIMessageRpcClientMock.mockImplementation(async (params) => {
284+
onNotification = params?.onNotification;
285+
return client as never;
286+
});
276287

277-
await monitorIMessageProvider({
278-
includeAttachments: true,
279-
config: {
280-
channels: {
281-
imessage: {
282-
includeAttachments: true,
283-
attachmentRoots: ["/Users/openclaw/Library/Messages/Attachments"],
284-
dmPolicy: "allowlist",
285-
allowFrom: ["+15550001111"],
286-
groupPolicy: "open",
288+
await monitorIMessageProvider({
289+
includeAttachments: true,
290+
config: {
291+
channels: {
292+
imessage: {
293+
includeAttachments: true,
294+
attachmentRoots: ["/Users/openclaw/Library/Messages/Attachments"],
295+
dmPolicy: "allowlist",
296+
allowFrom: ["+15550001111"],
297+
groupPolicy: "open",
298+
},
287299
},
288-
},
289-
messages: { inbound: { debounceMs: 0 } },
290-
session: { mainKey: "main" },
291-
} as never,
292-
runtime,
293-
});
300+
messages: { inbound: { debounceMs: 0 } },
301+
session: { mainKey: "main" },
302+
} as never,
303+
runtime,
304+
});
294305

295-
expect(runtime.error).not.toHaveBeenCalled();
296-
await vi.waitFor(() => expect(stageIMessageAttachmentsMock).toHaveBeenCalledTimes(1));
297-
await vi.waitFor(() =>
298-
expect(dispatchReplyWithBufferedBlockDispatcherMock).toHaveBeenCalledTimes(1),
299-
);
300-
expect(dispatchReplyWithBufferedBlockDispatcherMock.mock.calls[0]?.[0].ctx.BodyForAgent).toBe(
301-
expectedBody,
302-
);
303-
});
306+
expect(runtime.error).not.toHaveBeenCalled();
307+
await vi.waitFor(() => expect(stageIMessageAttachmentsMock).toHaveBeenCalledTimes(1));
308+
await vi.waitFor(() =>
309+
expect(dispatchReplyWithBufferedBlockDispatcherMock).toHaveBeenCalledTimes(1),
310+
);
311+
expect(dispatchReplyWithBufferedBlockDispatcherMock.mock.calls[0]?.[0].ctx.BodyForAgent).toBe(
312+
expectedBody,
313+
);
314+
expect(
315+
dispatchReplyWithBufferedBlockDispatcherMock.mock.calls[0]?.[0].ctx.MediaTypes,
316+
).toEqual(expectedMediaTypes);
317+
expect(dispatchReplyWithBufferedBlockDispatcherMock.mock.calls[0]?.[0].ctx.MediaUrls).toEqual(
318+
expectedMediaUrls,
319+
);
320+
},
321+
);
304322
});

‎extensions/imessage/src/monitor/deliver.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ describe("deliverReplies", () => {
228228
sendMessageIMessageMock.mockResolvedValueOnce({
229229
messageId: "imsg-media-1",
230230
sentText: "",
231-
echoText: "<media:image>",
231+
echoMedia: { contentType: "image/jpeg", kind: "image" },
232232
});
233233

234234
await deliverReplies({
@@ -243,7 +243,7 @@ describe("deliverReplies", () => {
243243
});
244244

245245
expect(remember).toHaveBeenCalledWith("acct-4:chat_id:40", {
246-
text: "<media:image>",
246+
media: { contentType: "image/jpeg", kind: "image" },
247247
messageId: "imsg-media-1",
248248
});
249249
});

‎extensions/imessage/src/monitor/deliver.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,10 @@ export async function deliverReplies(params: {
5151
accountId,
5252
replyToId: payload.replyToId,
5353
});
54+
const echoText = sent.echoText ?? sent.sentText;
5455
sentMessageCache?.remember(scope, {
55-
text: sent.echoText ?? sent.sentText,
56+
...(echoText ? { text: echoText } : {}),
57+
...(sent.echoMedia ? { media: sent.echoMedia } : {}),
5658
messageId: sent.messageId,
5759
});
5860
},
@@ -64,8 +66,10 @@ export async function deliverReplies(params: {
6466
accountId,
6567
replyToId: payload.replyToId,
6668
});
69+
const echoText = sent.echoText ?? (sent.sentText || undefined);
6770
sentMessageCache?.remember(scope, {
68-
text: sent.echoText ?? (sent.sentText || undefined),
71+
...(echoText ? { text: echoText } : {}),
72+
...(sent.echoMedia ? { media: sent.echoMedia } : {}),
6973
messageId: sent.messageId,
7074
});
7175
},
@@ -84,8 +88,10 @@ export function createIMessageEchoCachingSend(params: {
8488
const sanitizedText = sanitizeOutboundText(text);
8589
const sent = await sendMessageIMessage(target, sanitizedText, opts);
8690
const scope = `${params.accountId ?? opts.accountId ?? ""}:${target}`;
91+
const echoText = sent.echoText ?? (sent.sentText || undefined);
8792
params.sentMessageCache?.remember(scope, {
88-
text: sent.echoText ?? (sent.sentText || undefined),
93+
...(echoText ? { text: echoText } : {}),
94+
...(sent.echoMedia ? { media: sent.echoMedia } : {}),
8995
messageId: sent.messageId,
9096
});
9197
return sent;

‎extensions/imessage/src/monitor/echo-cache.ts‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
import type { MediaPlaceholderTextFact } from "openclaw/plugin-sdk/channel-inbound";
12
// Imessage plugin module implements echo cache behavior.
23
import { stripLeadingEchoTextCorruptionMarkers } from "./echo-text-corruption.js";
3-
import { hasPersistedIMessageEcho } from "./persisted-echo-cache.js";
4+
import { hasPersistedIMessageEcho, resolveIMessageEchoMediaKey } from "./persisted-echo-cache.js";
45

56
type SentMessageLookup = {
67
text?: string;
8+
media?: MediaPlaceholderTextFact;
79
messageId?: string;
810
};
911

@@ -60,19 +62,28 @@ function normalizeEchoMessageIdKey(messageId: string | undefined): string | null
6062
class DefaultSentMessageCache implements SentMessageCache {
6163
private textCache = new Map<string, number>();
6264
private textBackedByIdCache = new Map<string, number>();
65+
private mediaCache = new Map<string, number>();
66+
private mediaBackedByIdCache = new Map<string, number>();
6367
private messageIdCache = new Map<string, number>();
6468

6569
remember(scope: string, lookup: SentMessageLookup): void {
6670
const textKey = normalizeEchoTextKey(lookup.text);
6771
if (textKey) {
6872
this.textCache.set(`${scope}:${textKey}`, Date.now());
6973
}
74+
const mediaKey = resolveIMessageEchoMediaKey(lookup.media);
75+
if (mediaKey) {
76+
this.mediaCache.set(`${scope}:${mediaKey}`, Date.now());
77+
}
7078
const messageIdKey = normalizeEchoMessageIdKey(lookup.messageId);
7179
if (messageIdKey) {
7280
this.messageIdCache.set(`${scope}:${messageIdKey}`, Date.now());
7381
if (textKey) {
7482
this.textBackedByIdCache.set(`${scope}:${textKey}`, Date.now());
7583
}
84+
if (mediaKey) {
85+
this.mediaBackedByIdCache.set(`${scope}:${mediaKey}`, Date.now());
86+
}
7687
}
7788
this.cleanup();
7889
}
@@ -88,14 +99,18 @@ class DefaultSentMessageCache implements SentMessageCache {
8899
if (
89100
hasPersistedIMessageEcho({
90101
scope,
91-
...lookup,
102+
text: lookup.text,
103+
media: lookup.media,
104+
messageId: lookup.messageId,
92105
includePendingText: resolvedOptions.includePendingText,
93106
})
94107
) {
95108
return true;
96109
}
97110
const textKey = normalizeEchoTextKey(lookup.text);
111+
const mediaKey = resolveIMessageEchoMediaKey(lookup.media);
98112
const messageIdKey = normalizeEchoMessageIdKey(lookup.messageId);
113+
let canUseMediaFallback = !messageIdKey;
99114
if (messageIdKey) {
100115
const idTimestamp = this.messageIdCache.get(`${scope}:${messageIdKey}`);
101116
if (idTimestamp && Date.now() - idTimestamp <= SENT_MESSAGE_ID_TTL_MS) {
@@ -108,7 +123,15 @@ class DefaultSentMessageCache implements SentMessageCache {
108123
const hasTextOnlyMatch =
109124
typeof textTimestamp === "number" &&
110125
(!textBackedByIdTimestamp || textTimestamp > textBackedByIdTimestamp);
111-
if (!resolvedOptions.skipIdShortCircuit && !hasTextOnlyMatch) {
126+
const mediaTimestamp = mediaKey ? this.mediaCache.get(`${scope}:${mediaKey}`) : undefined;
127+
const mediaBackedByIdTimestamp = mediaKey
128+
? this.mediaBackedByIdCache.get(`${scope}:${mediaKey}`)
129+
: undefined;
130+
const hasMediaOnlyMatch =
131+
typeof mediaTimestamp === "number" &&
132+
(!mediaBackedByIdTimestamp || mediaTimestamp > mediaBackedByIdTimestamp);
133+
canUseMediaFallback = hasMediaOnlyMatch;
134+
if (!resolvedOptions.skipIdShortCircuit && !hasTextOnlyMatch && !hasMediaOnlyMatch) {
112135
return false;
113136
}
114137
}
@@ -118,6 +141,12 @@ class DefaultSentMessageCache implements SentMessageCache {
118141
return true;
119142
}
120143
}
144+
if (mediaKey && canUseMediaFallback) {
145+
const mediaTimestamp = this.mediaCache.get(`${scope}:${mediaKey}`);
146+
if (mediaTimestamp && Date.now() - mediaTimestamp <= SENT_MESSAGE_TEXT_TTL_MS) {
147+
return true;
148+
}
149+
}
121150
return false;
122151
}
123152

@@ -133,6 +162,16 @@ class DefaultSentMessageCache implements SentMessageCache {
133162
this.textBackedByIdCache.delete(key);
134163
}
135164
}
165+
for (const [key, timestamp] of this.mediaCache.entries()) {
166+
if (now - timestamp > SENT_MESSAGE_TEXT_TTL_MS) {
167+
this.mediaCache.delete(key);
168+
}
169+
}
170+
for (const [key, timestamp] of this.mediaBackedByIdCache.entries()) {
171+
if (now - timestamp > SENT_MESSAGE_TEXT_TTL_MS) {
172+
this.mediaBackedByIdCache.delete(key);
173+
}
174+
}
136175
for (const [key, timestamp] of this.messageIdCache.entries()) {
137176
if (now - timestamp > SENT_MESSAGE_ID_TTL_MS) {
138177
this.messageIdCache.delete(key);

‎extensions/imessage/src/monitor/inbound-processing.test.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,18 +98,24 @@ describe("resolveIMessageInboundDecision echo detection", () => {
9898
expect(logVerbose).toHaveBeenCalledWith(expect.stringContaining("id=42"));
9999
});
100100

101-
it("matches attachment-only echoes by bodyText placeholder", async () => {
102-
const echoHas = vi.fn((_scope: string, lookup: { text?: string; messageId?: string }) => {
103-
return lookup.text === "<media:image>" && lookup.messageId === "42";
104-
});
101+
it("matches attachment-only echoes by structured media fact", async () => {
102+
const echoHas = vi.fn(
103+
(
104+
_scope: string,
105+
lookup: { text?: string; media?: { kind?: string | null }; messageId?: string },
106+
) => {
107+
return lookup.media?.kind === "image" && lookup.messageId === "42";
108+
},
109+
);
105110

106111
const decision = await resolveDecision({
107112
message: {
108113
id: 42,
109114
text: "",
110115
},
111116
messageText: "",
112-
bodyText: "<media:image>",
117+
bodyText: "",
118+
mediaFacts: [{ contentType: "image/png", kind: "image" }],
113119
echoCache: { has: echoHas },
114120
});
115121

@@ -121,7 +127,8 @@ describe("resolveIMessageInboundDecision echo detection", () => {
121127
2,
122128
"default:imessage:+15555550123",
123129
{
124-
text: "<media:image>",
130+
text: undefined,
131+
media: { contentType: "image/png", kind: "image" },
125132
messageId: "42",
126133
},
127134
{

0 commit comments

Comments
 (0)