Skip to content

Commit defde22

Browse files
vincentkocvincentkoc
committed
fix(telegram): preserve sticker media context
1 parent c2a70a5 commit defde22

17 files changed

Lines changed: 325 additions & 11 deletions

‎extensions/telegram/src/bot-message-context.body.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,73 @@ describe("resolveTelegramInboundBody", () => {
166166
expect(result?.bodyText).toBe("<media:document> (2 attachments)");
167167
});
168168

169+
it("preserves cached sticker descriptions when downloaded media exists", async () => {
170+
const result = await resolveTelegramBody({
171+
msg: {
172+
message_id: 6,
173+
date: 1_700_000_006,
174+
chat: { id: 42, type: "private", first_name: "Pat" },
175+
from: { id: 42, first_name: "Pat" },
176+
sticker: {
177+
file_id: "sticker-1",
178+
file_unique_id: "sticker-u1",
179+
type: "regular",
180+
width: 256,
181+
height: 256,
182+
is_animated: false,
183+
is_video: false,
184+
emoji: "ok",
185+
set_name: "test-set",
186+
},
187+
} as never,
188+
allMedia: [
189+
{
190+
path: "/tmp/sticker.webp",
191+
contentType: "image/webp",
192+
stickerMetadata: {
193+
emoji: "ok",
194+
setName: "test-set",
195+
cachedDescription: "Cached description",
196+
},
197+
},
198+
],
199+
});
200+
201+
expect(result?.bodyText).toBe('[Sticker ok from "test-set"] Cached description');
202+
expect(result?.stickerCacheHit).toBe(true);
203+
});
204+
205+
it("includes cached sticker descriptions with user captions", async () => {
206+
const result = await resolveTelegramBody({
207+
msg: {
208+
message_id: 7,
209+
date: 1_700_000_007,
210+
chat: { id: 42, type: "private", first_name: "Pat" },
211+
from: { id: 42, first_name: "Pat" },
212+
caption: "What is this?",
213+
sticker: {
214+
file_id: "sticker-2",
215+
file_unique_id: "sticker-u2",
216+
type: "regular",
217+
width: 256,
218+
height: 256,
219+
is_animated: false,
220+
is_video: false,
221+
},
222+
} as never,
223+
allMedia: [
224+
{
225+
path: "/tmp/sticker.webp",
226+
contentType: "image/webp",
227+
stickerMetadata: { cachedDescription: "Cached description" },
228+
},
229+
],
230+
});
231+
232+
expect(result?.bodyText).toBe("[Sticker] Cached description\nWhat is this?");
233+
expect(result?.stickerCacheHit).toBe(true);
234+
});
235+
169236
it("lets catch-all mention patterns activate captionless group photos", async () => {
170237
const logger = { info: vi.fn() };
171238

‎extensions/telegram/src/bot-message-context.body.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,9 @@ export async function resolveTelegramInboundBody(params: {
246246
}
247247

248248
let bodyText = rawBody;
249+
if (stickerCacheHit && placeholder && rawBody !== placeholder) {
250+
bodyText = `${placeholder}\n${bodyText}`.trim();
251+
}
249252
if (allMedia.length === 0 && placeholder && rawBody !== placeholder) {
250253
const mediaTag = primaryMedia?.fileRef.file_id
251254
? `${placeholder} [file_id:${primaryMedia.fileRef.file_id}]`
@@ -304,7 +307,13 @@ export async function resolveTelegramInboundBody(params: {
304307
}
305308

306309
const savedMediaPlaceholder = formatSavedMediaPlaceholder(allMedia);
307-
if (!hasAudio && savedMediaPlaceholder && placeholder && bodyText === placeholder) {
310+
if (
311+
!stickerCacheHit &&
312+
!hasAudio &&
313+
savedMediaPlaceholder &&
314+
placeholder &&
315+
bodyText === placeholder
316+
) {
308317
bodyText = savedMediaPlaceholder;
309318
}
310319
if (!bodyText && allMedia.length > 0) {

‎extensions/telegram/src/bot-message-context.session.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,7 @@ export async function buildTelegramInboundContextPayload(params: {
215215
topicConfig?: TelegramTopicConfig;
216216
effectiveWasMentioned: boolean;
217217
hasControlCommand: boolean;
218+
stickerCacheHit?: boolean;
218219
audioTranscribedMediaIndex?: number;
219220
commandAuthorized: boolean;
220221
locationData?: NormalizedLocation;
@@ -263,6 +264,7 @@ export async function buildTelegramInboundContextPayload(params: {
263264
topicConfig,
264265
effectiveWasMentioned,
265266
hasControlCommand,
267+
stickerCacheHit,
266268
audioTranscribedMediaIndex,
267269
commandAuthorized,
268270
locationData,
@@ -594,6 +596,7 @@ export async function buildTelegramInboundContextPayload(params: {
594596
WasMentioned: isGroup ? effectiveWasMentioned : undefined,
595597
Sticker: allMedia[0]?.stickerMetadata,
596598
StickerMediaIncluded: allMedia[0]?.stickerMetadata ? currentMediaFacts.length > 0 : undefined,
599+
SkipStickerMediaUnderstanding: stickerCacheHit ? true : undefined,
597600
...locationContext,
598601
IsForum: isForum,
599602
TopicName: isForum && topicName ? topicName : undefined,

‎extensions/telegram/src/bot-message-context.sticker-media.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ describe("buildTelegramMessageContext sticker media", () => {
6969
expect(ctx?.ctxPayload.MediaUrls).toEqual([stickerPath]);
7070
expect(ctx?.ctxPayload.MediaTypes).toEqual(["image/webp"]);
7171
expect(ctx?.ctxPayload.StickerMediaIncluded).toBe(true);
72+
expect(ctx?.ctxPayload.SkipStickerMediaUnderstanding).toBe(true);
7273
expect(ctx?.ctxPayload.Sticker).toMatchObject({
7374
fileId: "new_file_id",
7475
fileUniqueId: "sticker_unique_789",

‎extensions/telegram/src/bot-message-context.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,7 @@ export const buildTelegramMessageContext = async ({
521521
topicConfig,
522522
effectiveWasMentioned: bodyResult.effectiveWasMentioned,
523523
hasControlCommand: bodyResult.hasControlCommand,
524+
stickerCacheHit: bodyResult.stickerCacheHit,
524525
...(bodyResult.audioTranscribedMediaIndex !== undefined
525526
? { audioTranscribedMediaIndex: bodyResult.audioTranscribedMediaIndex }
526527
: {}),

‎extensions/telegram/src/bot-message-dispatch.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,61 @@ describe("dispatchTelegramMessage draft streaming", () => {
555555
});
556556
}
557557

558+
it("skips general understanding after describing a first-seen non-vision sticker", async () => {
559+
describeStickerImage.mockResolvedValueOnce("A curious sticker");
560+
const ctxPayload = {
561+
MediaPath: "/tmp/sticker.webp",
562+
Sticker: {
563+
fileId: "sticker-file",
564+
fileUniqueId: "sticker-unique",
565+
},
566+
StickerMediaIncluded: true,
567+
} as TelegramMessageContext["ctxPayload"];
568+
569+
await dispatchWithContext({
570+
context: createContext({ ctxPayload }),
571+
});
572+
573+
expect(describeStickerImage).toHaveBeenCalledOnce();
574+
expect(ctxPayload.BodyForAgent).toBe("[Sticker] A curious sticker");
575+
expect(ctxPayload.SkipStickerMediaUnderstanding).toBe(true);
576+
expectDispatchParams({
577+
ctx: expect.objectContaining({
578+
SkipStickerMediaUnderstanding: true,
579+
}),
580+
});
581+
});
582+
583+
it("preserves cached sticker descriptions with user text through dispatch", async () => {
584+
const body = "[Sticker] Cached description\nWhat is this?";
585+
const ctxPayload = {
586+
Body: body,
587+
BodyForAgent: body,
588+
MediaPath: "/tmp/sticker.webp",
589+
Sticker: {
590+
fileId: "sticker-file",
591+
fileUniqueId: "sticker-unique",
592+
cachedDescription: "Cached description",
593+
},
594+
StickerMediaIncluded: true,
595+
SkipStickerMediaUnderstanding: true,
596+
} as TelegramMessageContext["ctxPayload"];
597+
598+
await dispatchWithContext({
599+
context: createContext({ ctxPayload }),
600+
});
601+
602+
expect(describeStickerImage).not.toHaveBeenCalled();
603+
expect(ctxPayload.Body).toBe(body);
604+
expect(ctxPayload.BodyForAgent).toBe(body);
605+
expectDispatchParams({
606+
ctx: expect.objectContaining({
607+
BodyForAgent: body,
608+
SkipStickerMediaUnderstanding: true,
609+
}),
610+
});
611+
});
612+
558613
it("streams drafts in private threads and forwards thread id", async () => {
559614
const draftStream = createDraftStream();
560615
createTelegramDraftStream.mockReturnValue(draftStream);

‎extensions/telegram/src/bot-message-dispatch.ts‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,22 @@ async function resolveStickerVisionSupport(cfg: OpenClawConfig, agentId: string)
204204
}
205205
}
206206

207+
function includeStickerDescription(body: string | undefined, formattedDescription: string): string {
208+
if (!body) {
209+
return formattedDescription;
210+
}
211+
const current = body.trim();
212+
if (!current || current === "<media:image>") {
213+
return formattedDescription;
214+
}
215+
// Cached descriptions can already be present from inbound context construction.
216+
// Keep that body intact so captions, forwarded text, and supplemental context survive.
217+
if (body.includes(formattedDescription)) {
218+
return body;
219+
}
220+
return `${formattedDescription}\n${body}`;
221+
}
222+
207223
type DispatchTelegramMessageParams = {
208224
context: TelegramMessageContext;
209225
bot: Bot;
@@ -1522,8 +1538,12 @@ export const dispatchTelegramMessage = async ({
15221538

15231539
sticker.cachedDescription = description;
15241540
if (!stickerSupportsVision) {
1525-
ctxPayload.Body = formattedDesc;
1526-
ctxPayload.BodyForAgent = formattedDesc;
1541+
ctxPayload.Body = includeStickerDescription(ctxPayload.Body, formattedDesc);
1542+
ctxPayload.BodyForAgent = includeStickerDescription(
1543+
ctxPayload.BodyForAgent,
1544+
formattedDesc,
1545+
);
1546+
ctxPayload.SkipStickerMediaUnderstanding = true;
15271547
}
15281548
cacheSticker({
15291549
fileId: sticker.fileId,

‎src/auto-reply/reply/dispatch-acp.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -693,6 +693,34 @@ describe("tryDispatchAcpReply", () => {
693693
expect(mediaUnderstandingMocks.applyMediaUnderstanding).not.toHaveBeenCalled();
694694
});
695695

696+
it("skips media understanding for cached stickers while preserving their attachment", async () => {
697+
setReadyAcpResolution();
698+
mockVisibleTextTurn("cached sticker");
699+
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "dispatch-acp-"));
700+
const stickerPath = path.join(tempDir, "sticker.webp");
701+
try {
702+
await fs.writeFile(stickerPath, "image-bytes");
703+
704+
await runDispatch({
705+
bodyForAgent: "[Sticker] Cached description",
706+
ctxOverrides: {
707+
MediaPath: stickerPath,
708+
MediaPaths: [stickerPath],
709+
MediaType: "image/webp",
710+
MediaTypes: ["image/webp"],
711+
Sticker: { cachedDescription: "Cached description" },
712+
StickerMediaIncluded: true,
713+
SkipStickerMediaUnderstanding: true,
714+
},
715+
});
716+
717+
expect(mediaUnderstandingMocks.applyMediaUnderstanding).not.toHaveBeenCalled();
718+
expect(managerMocks.runTurn).toHaveBeenCalled();
719+
} finally {
720+
await fs.rm(tempDir, { recursive: true, force: true });
721+
}
722+
});
723+
696724
it("passes the ACP agent directory to media understanding", async () => {
697725
setReadyAcpResolution();
698726
mockVisibleTextTurn("image turn");

‎src/auto-reply/reply/dispatch-acp.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ import {
4242
type AcpDispatchDeliveryCoordinator,
4343
} from "./dispatch-acp-delivery.js";
4444
import { appendRecentHistoryImageContext } from "./history-media.js";
45-
import { hasInboundMedia } from "./inbound-media.js";
45+
import { hasInboundMediaForUnderstanding } from "./inbound-media.js";
4646
import type { ReplyDispatchKind, ReplyDispatcher } from "./reply-dispatcher.types.js";
4747

4848
const dispatchAcpManagerRuntimeLoader = createLazyImportLoader(
@@ -540,7 +540,7 @@ export async function tryDispatchAcpReply(params: {
540540
if (agentPolicyError) {
541541
throw agentPolicyError;
542542
}
543-
if (hasInboundMedia(params.ctx) && !params.ctx.MediaUnderstanding?.length) {
543+
if (hasInboundMediaForUnderstanding(params.ctx) && !params.ctx.MediaUnderstanding?.length) {
544544
try {
545545
const { applyMediaUnderstanding } = await loadAgentTurnMediaRuntime();
546546
await applyMediaUnderstanding({

‎src/auto-reply/reply/get-reply.message-hooks.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -433,6 +433,58 @@ describe("getReplyFromConfig message hooks", () => {
433433
expect(mocks.applyLinkUnderstanding).not.toHaveBeenCalled();
434434
});
435435

436+
it("keeps cached sticker media attached without repeating media understanding", async () => {
437+
const stickerPath = "/tmp/cached-sticker.webp";
438+
439+
await getReplyFromConfig(
440+
buildCtx({
441+
Body: "[Sticker] Cached description",
442+
BodyForAgent: "[Sticker] Cached description",
443+
RawBody: "[Sticker] Cached description",
444+
CommandBody: "[Sticker] Cached description",
445+
BodyForCommands: "[Sticker] Cached description",
446+
MediaPath: stickerPath,
447+
MediaUrl: stickerPath,
448+
MediaPaths: [stickerPath],
449+
MediaUrls: [stickerPath],
450+
MediaType: "image/webp",
451+
MediaTypes: ["image/webp"],
452+
Sticker: { cachedDescription: "Cached description" },
453+
StickerMediaIncluded: true,
454+
SkipStickerMediaUnderstanding: true,
455+
}),
456+
undefined,
457+
withFastReplyConfig({}),
458+
);
459+
460+
expect(mocks.applyMediaUnderstanding).not.toHaveBeenCalled();
461+
});
462+
463+
it("still understands supplemental media attached to a cached sticker reply", async () => {
464+
await getReplyFromConfig(
465+
buildCtx({
466+
Body: "[Sticker] Cached description",
467+
BodyForAgent: "[Sticker] Cached description",
468+
RawBody: "[Sticker] Cached description",
469+
CommandBody: "[Sticker] Cached description",
470+
BodyForCommands: "[Sticker] Cached description",
471+
MediaPath: "/tmp/cached-sticker.webp",
472+
MediaUrl: "/tmp/cached-sticker.webp",
473+
MediaPaths: ["/tmp/cached-sticker.webp", "/tmp/replied-audio.ogg"],
474+
MediaUrls: ["/tmp/cached-sticker.webp", "/tmp/replied-audio.ogg"],
475+
MediaType: "image/webp",
476+
MediaTypes: ["image/webp", "audio/ogg"],
477+
Sticker: { cachedDescription: "Cached description" },
478+
StickerMediaIncluded: true,
479+
SkipStickerMediaUnderstanding: true,
480+
}),
481+
undefined,
482+
withFastReplyConfig({}),
483+
);
484+
485+
expect(mocks.applyMediaUnderstanding).toHaveBeenCalledOnce();
486+
});
487+
436488
it("continues dispatching when media understanding fails before reply routing", async () => {
437489
mocks.applyMediaUnderstanding.mockRejectedValueOnce(
438490
new Error("Cannot find module '/tmp/openclaw/dist/media-understanding/apply.runtime-old.js'"),

0 commit comments

Comments
 (0)