Skip to content

Commit e20d4cd

Browse files
steipetesteipete
authored
fix(media): gate inbound audio on structured media facts (#111315)
hasInboundAudio previously accepted <media:audio> placeholder bodies, [Audio transcript headers, and filename sniffing alongside MediaTypes. It now consumes only the structured facts channels already carry (MediaType/MediaTypes, kind audio or audio/*), removing the string reverse-engineering ahead of the placeholder-elimination program. Discord — the one channel with a coverage hole — now carries its native audio classification (MIME, voice fields, constrained filename fallback) as structured kind: "audio", with definitive fetched/declared MIME precedence so generic container types cannot erase declared audio and filename hints cannot override definitive non-audio MIME.
1 parent c26d3cf commit e20d4cd

5 files changed

Lines changed: 302 additions & 55 deletions

File tree

‎extensions/discord/src/monitor/message-media.ts‎

Lines changed: 66 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Discord plugin module implements message media behavior.
22
import { StickerFormatType, type APIAttachment, type APIStickerItem } from "discord-api-types/v10";
3-
import { getFileExtension } from "openclaw/plugin-sdk/media-mime";
3+
import { getFileExtension, normalizeMimeType } from "openclaw/plugin-sdk/media-mime";
44
import { saveRemoteMedia, type FetchLike } from "openclaw/plugin-sdk/media-runtime";
55
import { buildMediaPayload } from "openclaw/plugin-sdk/reply-payload";
66
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
@@ -49,6 +49,7 @@ const DISCORD_STICKER_ASSET_BASE_URL = "https://media.discordapp.net/stickers";
4949
export type DiscordMediaInfo = {
5050
path: string;
5151
contentType?: string;
52+
kind?: "audio";
5253
placeholder: string;
5354
};
5455

@@ -74,6 +75,56 @@ function hasDiscordVoiceAttachmentFields(attachment: APIAttachment): boolean {
7475
return typeof attachment.duration_secs === "number" || typeof attachment.waveform === "string";
7576
}
7677

78+
const NON_DEFINITIVE_MEDIA_TYPES = new Set([
79+
"application/octet-stream",
80+
"binary/octet-stream",
81+
// Discord can report this container type without identifying whether it holds audio or video.
82+
"application/ogg",
83+
]);
84+
85+
function isDefinitiveMediaType(contentType: string | null | undefined): boolean {
86+
const normalized = normalizeMimeType(contentType);
87+
return Boolean(normalized && !NON_DEFINITIVE_MEDIA_TYPES.has(normalized));
88+
}
89+
90+
function resolveEffectiveMediaType(params: {
91+
declaredContentType?: string | null;
92+
fetchedContentType?: string | null;
93+
}): string | undefined {
94+
if (isDefinitiveMediaType(params.fetchedContentType)) {
95+
return params.fetchedContentType ?? undefined;
96+
}
97+
if (isDefinitiveMediaType(params.declaredContentType)) {
98+
return params.declaredContentType ?? undefined;
99+
}
100+
return params.fetchedContentType ?? params.declaredContentType ?? undefined;
101+
}
102+
103+
function resolveDiscordMediaClassification(params: {
104+
attachment: APIAttachment;
105+
fetchedContentType?: string | null;
106+
}): { contentType?: string; kind?: "audio" } {
107+
const contentType = resolveEffectiveMediaType({
108+
declaredContentType: params.attachment.content_type,
109+
fetchedContentType: params.fetchedContentType,
110+
});
111+
const mime = normalizeMimeType(contentType);
112+
const kind =
113+
mime?.startsWith("audio/") ||
114+
hasDiscordVoiceAttachmentFields(params.attachment) ||
115+
(isDiscordAudioAttachmentFileName(params.attachment.filename ?? params.attachment.url) &&
116+
!isDefinitiveMediaType(contentType))
117+
? "audio"
118+
: undefined;
119+
120+
return {
121+
// Inbound projection prefers MIME over kind. A native voice classification
122+
// must therefore replace a non-audio MIME rather than be masked by it.
123+
contentType: kind === "audio" && !mime?.startsWith("audio/") ? undefined : contentType,
124+
...(kind ? { kind } : {}),
125+
};
126+
}
127+
77128
function mergeHostnameList(...lists: Array<string[] | undefined>): string[] | undefined {
78129
const merged = lists
79130
.flatMap((list) => list ?? [])
@@ -331,18 +382,23 @@ async function appendResolvedMediaFromAttachments(params: {
331382
fallbackContentType: attachment.content_type,
332383
originalFilename: attachment.filename,
333384
});
385+
const classification = resolveDiscordMediaClassification({
386+
attachment,
387+
fetchedContentType: saved.contentType,
388+
});
334389
params.out.push({
335390
path: saved.path,
336-
contentType: saved.contentType,
337-
placeholder: inferPlaceholder(attachment),
391+
...classification,
392+
placeholder: inferPlaceholder(classification),
338393
});
339394
} catch (err) {
340395
const id = attachment.id ?? attachmentUrl;
341396
logVerbose(`${params.errorPrefix} ${id}: ${String(err)}`);
397+
const classification = resolveDiscordMediaClassification({ attachment });
342398
params.out.push({
343399
path: attachmentUrl,
344-
contentType: attachment.content_type,
345-
placeholder: inferPlaceholder(attachment),
400+
...classification,
401+
placeholder: inferPlaceholder(classification),
346402
});
347403
}
348404
}
@@ -457,8 +513,11 @@ async function appendResolvedMediaFromStickers(params: {
457513
}
458514
}
459515

460-
function inferPlaceholder(attachment: APIAttachment): string {
461-
const mime = attachment.content_type ?? "";
516+
function inferPlaceholder(media: Pick<DiscordMediaInfo, "contentType" | "kind">): string {
517+
if (media.kind === "audio") {
518+
return "<media:audio>";
519+
}
520+
const mime = normalizeMimeType(media.contentType) ?? "";
462521
if (mime.startsWith("image/")) {
463522
return "<media:image>";
464523
}
@@ -468,12 +527,6 @@ function inferPlaceholder(attachment: APIAttachment): string {
468527
if (mime.startsWith("audio/")) {
469528
return "<media:audio>";
470529
}
471-
if (hasDiscordVoiceAttachmentFields(attachment)) {
472-
return "<media:audio>";
473-
}
474-
if (isDiscordAudioAttachmentFileName(attachment.filename ?? attachment.url)) {
475-
return "<media:audio>";
476-
}
477530
return "<media:document>";
478531
}
479532

‎extensions/discord/src/monitor/message-utils.test.ts‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -656,6 +656,7 @@ describe("resolveMediaList", () => {
656656
{
657657
path: attachment.url,
658658
contentType: undefined,
659+
kind: "audio",
659660
placeholder: "<media:audio>",
660661
},
661662
]);
@@ -682,6 +683,205 @@ describe("resolveMediaList", () => {
682683
{
683684
path: attachment.url,
684685
contentType: undefined,
686+
kind: "audio",
687+
placeholder: "<media:audio>",
688+
},
689+
]);
690+
});
691+
692+
it.each(["application/octet-stream", "application/ogg"])(
693+
"prefers the structured audio kind over non-audio MIME %s",
694+
async (contentType) => {
695+
const attachment = {
696+
id: "att-audio-conflicting-mime",
697+
url: "https://cdn.discordapp.com/attachments/1/voice.ogg",
698+
filename: "voice.ogg",
699+
content_type: contentType,
700+
};
701+
readRemoteMediaBuffer.mockResolvedValueOnce({
702+
buffer: Buffer.from("audio"),
703+
contentType,
704+
});
705+
saveMediaBuffer.mockResolvedValueOnce({
706+
path: "/tmp/voice.ogg",
707+
contentType,
708+
});
709+
710+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
711+
712+
expect(result).toEqual([
713+
{
714+
path: "/tmp/voice.ogg",
715+
contentType: undefined,
716+
kind: "audio",
717+
placeholder: "<media:audio>",
718+
},
719+
]);
720+
},
721+
);
722+
723+
it("normalizes MIME case before classifying audio", async () => {
724+
const attachment = {
725+
id: "att-audio-mime-case",
726+
url: "https://cdn.discordapp.com/attachments/1/voice.bin",
727+
filename: "voice.bin",
728+
content_type: "Audio/OGG",
729+
};
730+
readRemoteMediaBuffer.mockRejectedValueOnce(new Error("blocked by ssrf guard"));
731+
732+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
733+
734+
expect(result).toEqual([
735+
{
736+
path: attachment.url,
737+
contentType: "Audio/OGG",
738+
kind: "audio",
739+
placeholder: "<media:audio>",
740+
},
741+
]);
742+
});
743+
744+
it("does not let an audio-looking filename override video MIME", async () => {
745+
const attachment = {
746+
id: "att-video-audio-extension",
747+
url: "https://cdn.discordapp.com/attachments/1/clip.ogg",
748+
filename: "clip.ogg",
749+
content_type: "video/ogg",
750+
};
751+
readRemoteMediaBuffer.mockRejectedValueOnce(new Error("blocked by ssrf guard"));
752+
753+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
754+
755+
expect(result).toEqual([
756+
{
757+
path: attachment.url,
758+
contentType: "video/ogg",
759+
placeholder: "<media:video>",
760+
},
761+
]);
762+
});
763+
764+
it("does not let an audio-looking filename override fetched image MIME", async () => {
765+
const attachment = {
766+
id: "att-image-audio-extension",
767+
url: "https://cdn.discordapp.com/attachments/1/image.ogg",
768+
filename: "image.ogg",
769+
};
770+
readRemoteMediaBuffer.mockResolvedValueOnce({
771+
buffer: Buffer.from("image"),
772+
contentType: "image/png",
773+
});
774+
saveMediaBuffer.mockResolvedValueOnce({
775+
path: "/tmp/image.png",
776+
contentType: "image/png",
777+
});
778+
779+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
780+
781+
expect(result).toEqual([
782+
{
783+
path: "/tmp/image.png",
784+
contentType: "image/png",
785+
placeholder: "<media:image>",
786+
},
787+
]);
788+
});
789+
790+
it("keeps declared audio when the fetched MIME is generic", async () => {
791+
const attachment = {
792+
id: "att-declared-audio-fetched-generic",
793+
url: "https://cdn.discordapp.com/attachments/1/voice",
794+
filename: "voice",
795+
content_type: "audio/ogg",
796+
};
797+
readRemoteMediaBuffer.mockResolvedValueOnce({
798+
buffer: Buffer.from("audio"),
799+
contentType: "application/octet-stream",
800+
});
801+
saveMediaBuffer.mockResolvedValueOnce({
802+
path: "/tmp/voice",
803+
contentType: "application/octet-stream",
804+
});
805+
806+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
807+
808+
expect(result).toEqual([
809+
{
810+
path: "/tmp/voice",
811+
contentType: "audio/ogg",
812+
kind: "audio",
813+
placeholder: "<media:audio>",
814+
},
815+
]);
816+
});
817+
818+
it.each(["application/pdf", "text/plain"])(
819+
"does not infer audio from an .ogg filename with definitive MIME %s",
820+
async (contentType) => {
821+
const attachment = {
822+
id: `att-definitive-${contentType}`,
823+
url: "https://cdn.discordapp.com/attachments/1/document.ogg",
824+
filename: "document.ogg",
825+
content_type: contentType,
826+
};
827+
readRemoteMediaBuffer.mockRejectedValueOnce(new Error("blocked by ssrf guard"));
828+
829+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
830+
831+
expect(result).toEqual([
832+
{
833+
path: attachment.url,
834+
contentType,
835+
placeholder: "<media:document>",
836+
},
837+
]);
838+
},
839+
);
840+
841+
it("uses fetched image MIME over declared audio", async () => {
842+
const attachment = {
843+
id: "att-declared-audio-fetched-image",
844+
url: "https://cdn.discordapp.com/attachments/1/voice.ogg",
845+
filename: "voice.ogg",
846+
content_type: "audio/ogg",
847+
};
848+
readRemoteMediaBuffer.mockResolvedValueOnce({
849+
buffer: Buffer.from("image"),
850+
contentType: "image/png",
851+
});
852+
saveMediaBuffer.mockResolvedValueOnce({
853+
path: "/tmp/image.png",
854+
contentType: "image/png",
855+
});
856+
857+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
858+
859+
expect(result).toEqual([
860+
{
861+
path: "/tmp/image.png",
862+
contentType: "image/png",
863+
placeholder: "<media:image>",
864+
},
865+
]);
866+
});
867+
868+
it("classifies extensionless Discord voice attachments from native fields", async () => {
869+
const attachment = {
870+
id: "att-voice-native-fields",
871+
url: "https://cdn.discordapp.com/attachments/1/voice",
872+
filename: "voice",
873+
duration_secs: 1.5,
874+
waveform: "AAAA",
875+
};
876+
readRemoteMediaBuffer.mockRejectedValueOnce(new Error("blocked by ssrf guard"));
877+
878+
const result = await resolveMediaList(asMessage({ attachments: [attachment] }), 512);
879+
880+
expect(result).toEqual([
881+
{
882+
path: attachment.url,
883+
contentType: undefined,
884+
kind: "audio",
685885
placeholder: "<media:audio>",
686886
},
687887
]);

‎src/auto-reply/reply/get-reply.message-hooks.test.ts‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@ describe("getReplyFromConfig message hooks", () => {
318318
);
319319
});
320320

321-
it("recognizes locked-harness audio from its filename when MIME metadata is missing", async () => {
321+
it("does not infer locked-harness audio from its filename when MIME metadata is missing", async () => {
322322
const sessionKey = "agent:main:harness:claude-cli:locked-audio-filename";
323323
mocks.resolveReplySessionPreprocessingState.mockReturnValueOnce({
324324
sessionEntry: {
@@ -340,15 +340,13 @@ describe("getReplyFromConfig message hooks", () => {
340340
RawBody: "<media:file>",
341341
CommandBody: "<media:file>",
342342
MediaType: undefined,
343+
MediaTypes: undefined,
343344
}),
344345
undefined,
345346
buildConfiguredAudioCfg(),
346347
);
347348

348-
expect(mocks.applyMediaUnderstanding).toHaveBeenCalledOnce();
349-
expect(mocks.applyMediaUnderstanding.mock.calls[0]?.[0]).toEqual(
350-
expect.objectContaining({ processingMode: "audio-only" }),
351-
);
349+
expect(mocks.applyMediaUnderstanding).not.toHaveBeenCalled();
352350
});
353351

354352
it("runs normal media understanding for an unlocked voice note", async () => {

‎src/auto-reply/reply/inbound-media.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { describe, expect, it } from "vitest";
2+
import { hasInboundAudio } from "./inbound-media.js";
3+
4+
describe("hasInboundAudio", () => {
5+
it("detects audio from the singular structured media type without a placeholder body", () => {
6+
expect(hasInboundAudio({ MediaType: " Audio/Ogg ; codecs=opus " })).toBe(true);
7+
});
8+
9+
it("detects audio in aligned structured media types", () => {
10+
expect(hasInboundAudio({ MediaTypes: ["image/png", "audio/mpeg"] })).toBe(true);
11+
});
12+
13+
it("accepts the structured audio kind when a MIME subtype is unavailable", () => {
14+
expect(hasInboundAudio({ MediaTypes: ["audio"] })).toBe(true);
15+
});
16+
17+
it("does not infer audio from placeholder or transcript text", () => {
18+
expect(hasInboundAudio({ Body: "<media:audio>" })).toBe(false);
19+
expect(hasInboundAudio({ Body: "[Audio]\nTranscript:\nhello" })).toBe(false);
20+
});
21+
22+
it("does not rederive audio from a media filename", () => {
23+
expect(hasInboundAudio({ MediaPath: "/tmp/voice.ogg" })).toBe(false);
24+
});
25+
26+
it("does not treat non-audio media as audio", () => {
27+
expect(hasInboundAudio({ MediaType: "image/png", MediaTypes: ["video/mp4"] })).toBe(false);
28+
});
29+
});

0 commit comments

Comments
 (0)