Skip to content

Commit c807a93

Browse files
committed
feat(opencode): optimize media attachments on submit with plugin extensibility
Add a media optimization layer that runs at submit time, right before sdk.client.session.prompt(), matching the message.parts.before pattern proposed in anomalyco#24125. Key design decisions: - Optimization runs at submit(), not paste() -- plugins see the final assembled parts array and can act on model context - registerOptimizer() lets plugins (like SHIFT) override per-modality optimization, replacing or augmenting built-in handlers - getMetrics() exposes before/after byte counts, tool name, and duration for measuring savings (payload size, token reduction) - OptimizeConfig provides tuning knobs: maxDimension, quality, minImageSize, maxKeyframes, audioSampleRate, disabled Built-in optimizers: - Image: resize + JPEG compress via sips (macOS) or ImageMagick - Audio: transcode to mono 16kHz WAV via ffmpeg - Video: extract keyframes + audio via ffmpeg/ffprobe - PDF: pass-through placeholder Also widens paste mime gate for audio/video file paths, adds modality-aware labels, and extends sniffAttachmentMime for audio/video. Closes anomalyco#24129
1 parent 63f3e84 commit c807a93

5 files changed

Lines changed: 930 additions & 15 deletions

File tree

‎packages/opencode/src/cli/cmd/tui/component/prompt/index.tsx‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import { useRenderer, useTerminalDimensions, type JSX } from "@opentui/solid"
3636
import * as Editor from "@tui/util/editor"
3737
import { useExit } from "../../context/exit"
3838
import * as Clipboard from "../../util/clipboard"
39+
import { optimizeParts, isAttachable } from "../../util/media-optimize"
3940
import type { AssistantMessage, FilePart, UserMessage } from "@opencode-ai/sdk/v2"
4041
import { TuiEvent } from "../../event"
4142
import { iife } from "@/util/iife"
@@ -1174,6 +1175,19 @@ export function Prompt(props: PromptProps) {
11741175
})),
11751176
})
11761177
} else {
1178+
const assembledParts = [
1179+
...editorParts,
1180+
{
1181+
id: PartID.ascending(),
1182+
type: "text" as const,
1183+
text: inputText,
1184+
},
1185+
...nonTextParts.map(assign),
1186+
]
1187+
// Optimize media parts before sending to server (image resize, audio
1188+
// transcode, video decomposition). Plugins can override per-modality
1189+
// via registerOptimizer(). See #24125.
1190+
const optimized = await optimizeParts(assembledParts)
11771191
sdk.client.session
11781192
.prompt({
11791193
sessionID,
@@ -1182,15 +1196,7 @@ export function Prompt(props: PromptProps) {
11821196
agent: agent.name,
11831197
model: selectedModel,
11841198
variant,
1185-
parts: [
1186-
...editorParts,
1187-
{
1188-
id: PartID.ascending(),
1189-
type: "text",
1190-
text: inputText,
1191-
},
1192-
...nonTextParts.map(assign),
1193-
],
1199+
parts: optimized.parts,
11941200
})
11951201
.catch(() => {})
11961202
if (editorParts.length > 0) editor.markSelectionSent()
@@ -1281,7 +1287,7 @@ export function Prompt(props: PromptProps) {
12811287
return
12821288
}
12831289
}
1284-
if (mime.startsWith("image/") || mime === "application/pdf") {
1290+
if (isAttachable(mime)) {
12851291
const content = await Filesystem.readArrayBuffer(filepath)
12861292
.then((buffer) => Buffer.from(buffer).toString("base64"))
12871293
.catch(() => {})
@@ -1319,13 +1325,21 @@ export function Prompt(props: PromptProps) {
13191325
async function pasteAttachment(file: { filename?: string; filepath?: string; content: string; mime: string }) {
13201326
const currentOffset = input.visualCursor.offset
13211327
const extmarkStart = currentOffset
1322-
const pdf = file.mime === "application/pdf"
1328+
const label = file.mime === "application/pdf"
1329+
? "PDF"
1330+
: file.mime.startsWith("audio/")
1331+
? "Audio"
1332+
: file.mime.startsWith("video/")
1333+
? "Video"
1334+
: "Image"
13231335
const count = store.prompt.parts.filter((x) => {
13241336
if (x.type !== "file") return false
1325-
if (pdf) return x.mime === "application/pdf"
1337+
if (file.mime === "application/pdf") return x.mime === "application/pdf"
1338+
if (file.mime.startsWith("audio/")) return x.mime.startsWith("audio/")
1339+
if (file.mime.startsWith("video/")) return x.mime.startsWith("video/")
13261340
return x.mime.startsWith("image/")
13271341
}).length
1328-
const virtualText = pdf ? `[PDF ${count + 1}]` : `[Image ${count + 1}]`
1342+
const virtualText = `[${label} ${count + 1}]`
13291343
const extmarkEnd = extmarkStart + virtualText.length
13301344
const textToInsert = virtualText + " "
13311345

0 commit comments

Comments
 (0)