Skip to content

Commit 472a0c9

Browse files
rekram1-noderekram1-node
committed
feat(core): normalize tool and attachment images at settlement
Image resizing previously ran only inside the read tool, so plugin, MCP, and codemode tools could persist unbounded inline base64 images that grow every provider request body (see #36552). - ToolRegistry.settleTool now normalizes image file content generically: a missing resizer keeps the original, an unresizable image is dropped and replaced with an omission note, mirroring V1 processor behavior. - SessionV2.prompt normalizes image attachments at admission, resolving the Location-scoped Image service lazily so text-only prompts do not boot location services. - read keeps its internal normalize call to bound the image persisted in its structured output.
1 parent 4678bd1 commit 472a0c9

10 files changed

Lines changed: 170 additions & 13 deletions

‎packages/core/src/session.ts‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import { Snapshot } from "./snapshot"
3737
import { SessionRevert } from "./session/revert"
3838
import { Session } from "@opencode-ai/schema/session"
3939
import { FSUtil } from "./fs-util"
40+
import { Image } from "./image"
4041
import { Mime } from "./mime"
4142
import type { EventLog } from "@opencode-ai/schema/event-log"
4243
import { SkillV2 } from "./skill"
@@ -531,9 +532,13 @@ const layer = Layer.effect(
531532
// continues from the reverted boundary rather than stale post-boundary history.
532533
if (session.revert)
533534
yield* SessionRevert.commit(session).pipe(Effect.provideService(EventV2.Service, events))
534-
const prompt = yield* resolvePrompt({ text: input.text, files: input.files, agents: input.agents }).pipe(
535-
Effect.provideService(FSUtil.Service, fs),
536-
)
535+
// Resolved lazily so prompt admission only boots location services when an
536+
// image attachment actually needs the resizer.
537+
const image = Image.Service.pipe(Effect.provide(locations.get(session.location)))
538+
const prompt = yield* resolvePrompt(
539+
{ text: input.text, files: input.files, agents: input.agents },
540+
image,
541+
).pipe(Effect.provideService(FSUtil.Service, fs))
537542
const messageID = input.id ?? SessionMessage.ID.create()
538543
const admittedInput = SessionPending.Message.make({
539544
type: "user",
@@ -859,10 +864,13 @@ function synthesizeTerminalShellInfo(started: ShellSchema.Info): ShellSchema.Inf
859864
}
860865
}
861866

862-
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (input: PromptInput.Prompt) {
867+
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (
868+
input: PromptInput.Prompt,
869+
image: Effect.Effect<Image.Interface>,
870+
) {
863871
const fs = yield* FSUtil.Service
864872
const files = input.files
865-
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file), { concurrency: 8 })
873+
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file, image), { concurrency: 8 })
866874
: undefined
867875
return Prompt.make({ text: input.text, agents: input.agents, files })
868876
})
@@ -872,6 +880,7 @@ const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024
872880
const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(function* (
873881
fs: FSUtil.Interface,
874882
input: PromptInput.FileAttachment,
883+
image: Effect.Effect<Image.Interface>,
875884
) {
876885
const resolved = input.uri.startsWith("data:")
877886
? {
@@ -900,16 +909,42 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
900909
.join("\n"),
901910
)
902911
: resolved.bytes
903-
return FileAttachment.create({
904-
data: Base64.make(Buffer.from(content).toString("base64")),
912+
const normalized = yield* normalizeImageAttachment(
913+
input,
914+
Base64.make(Buffer.from(content).toString("base64")),
905915
mime,
916+
image,
917+
)
918+
return FileAttachment.create({
919+
data: normalized.data,
920+
mime: normalized.mime,
906921
source: resolved.source,
907922
name: input.name ?? resolved.name,
908923
description: input.description,
909924
mention: input.mention,
910925
})
911926
})
912927

928+
// V1 parity: bound image attachments at prompt admission so oversized images never
929+
// reach persistence or providers. A missing resizer keeps the original image; an
930+
// image that cannot fit the configured limits fails the prompt.
931+
const normalizeImageAttachment = Effect.fn("V2Session.normalizeImageAttachment")(function* (
932+
input: PromptInput.FileAttachment,
933+
data: Base64,
934+
mime: string,
935+
image: Effect.Effect<Image.Interface>,
936+
) {
937+
if (!mime.startsWith("image/")) return { data, mime }
938+
const service = yield* image
939+
const label = input.name ?? (input.uri.startsWith("data:") ? "inline attachment" : input.uri)
940+
const content = { uri: label, content: data, encoding: "base64" as const, mime }
941+
const normalized = yield* service.normalize(label, content).pipe(
942+
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)),
943+
Effect.mapError((error) => new AttachmentError({ uri: label, message: error.message })),
944+
)
945+
return { data: Base64.make(normalized.content), mime: normalized.mime }
946+
})
947+
913948
const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (fs: FSUtil.Interface, uri: string) {
914949
const url = yield* Effect.try({
915950
try: () => new URL(uri),

‎packages/core/src/tool/registry.ts‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry"
33
import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai"
44
import { Context, Effect, Layer, Scope } from "effect"
55
import type { AgentV2 } from "../agent"
6+
import { Image } from "../image"
67
import { PermissionV2 } from "../permission"
78
import { SessionMessage } from "../session/message"
89
import { SessionSchema } from "../session/schema"
@@ -57,6 +58,44 @@ const registryLayer = Layer.effect(
5758
Effect.gen(function* () {
5859
const resources = yield* ToolOutputStore.Service
5960
const toolHooks = yield* ToolHooks.Service
61+
const image = yield* Image.Service
62+
63+
// Generic model-output image bounding: every tool's media content settles through
64+
// here, so individual tools do not add their own resize calls. A missing resizer
65+
// keeps the original image; an image that cannot fit the configured limits is
66+
// dropped and replaced with a note, mirroring V1 settlement behavior.
67+
const normalizeImages = Effect.fn("ToolRegistry.normalizeImages")(function* (output: ToolOutput) {
68+
const content = yield* Effect.forEach(output.content, (item) => {
69+
if (item.type !== "file" || !item.mime.startsWith("image/")) return Effect.succeed(item)
70+
const base64 = /^data:[^;,]*;base64,(.*)$/s.exec(item.uri)?.[1]
71+
if (base64 === undefined) return Effect.succeed(item)
72+
const resource = item.name ?? `${item.mime} tool output`
73+
return image
74+
.normalize(resource, { uri: resource, content: base64, encoding: "base64", mime: item.mime })
75+
.pipe(
76+
Effect.map((normalized) => ({
77+
...item,
78+
uri: `data:${normalized.mime};base64,${normalized.content}`,
79+
mime: normalized.mime,
80+
})),
81+
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(item)),
82+
Effect.catch(() => Effect.succeed(undefined)),
83+
)
84+
})
85+
const kept = content.filter((item) => item !== undefined)
86+
const omitted = content.length - kept.length
87+
if (omitted === 0) return { structured: output.structured, content: kept }
88+
return {
89+
structured: output.structured,
90+
content: [
91+
...kept,
92+
{
93+
type: "text" as const,
94+
text: `[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the image size limit.]`,
95+
},
96+
],
97+
}
98+
})
6099
type Registration = {
61100
readonly tool: AnyTool
62101
readonly name: string
@@ -115,7 +154,7 @@ const registryLayer = Layer.effect(
115154
const bounded = yield* resources.bound({
116155
sessionID: input.sessionID,
117156
callID: input.call.id,
118-
output: pending.output,
157+
output: yield* normalizeImages(pending.output),
119158
})
120159
const result = ToolOutput.toResultValue(bounded.output)
121160
settlement =
@@ -232,11 +271,11 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) {
232271
export const node = makeLocationNode({
233272
service: Service,
234273
layer,
235-
deps: [ToolOutputStore.node, ToolHooks.node],
274+
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
236275
})
237276

238277
export const toolsNode = makeLocationNode({
239278
service: Tools.Service,
240279
layer,
241-
deps: [ToolOutputStore.node, ToolHooks.node],
280+
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
242281
})

‎packages/core/test/lib/image.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { Image } from "@opencode-ai/core/image"
2+
import { Effect, Layer } from "effect"
3+
4+
/** Passthrough resizer for tests that build ToolRegistry.node without a Location. */
5+
export const imagePassthrough = Layer.mock(Image.Service, {
6+
normalize: (_resource, content) => Effect.succeed(content),
7+
})

‎packages/core/test/mcp.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ import { McpTool } from "@opencode-ai/core/tool/mcp"
2929
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
3030
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
3131
import { Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect"
32+
import { Image } from "@opencode-ai/core/image"
3233
import { testEffect } from "./lib/effect"
34+
import { imagePassthrough } from "./lib/image"
3335
import { location } from "./fixture/location"
3436
import { settleTool, toolDefinitions, toolIdentity, waitForTool } from "./lib/tool"
3537

@@ -250,6 +252,7 @@ const it = testEffect(
250252
[PermissionV2.node, permissions],
251253
[EventV2.node, events],
252254
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
255+
[Image.node, imagePassthrough],
253256
]),
254257
)
255258

‎packages/core/test/session-prompt.test.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect } from "bun:test"
2-
import { DateTime, Effect, Fiber, Layer, Schema, Stream } from "effect"
2+
import { DateTime, Effect, Fiber, Layer, LayerMap, Schema, Stream } from "effect"
33
import { mkdtemp, rm } from "fs/promises"
44
import { tmpdir } from "os"
55
import path from "path"
@@ -24,7 +24,10 @@ import { SessionExecution } from "@opencode-ai/core/session/execution"
2424
import { SessionPending } from "@opencode-ai/core/session/pending"
2525
import { SessionPendingTable, SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql"
2626
import { SessionStore } from "@opencode-ai/core/session/store"
27+
import { LocationServiceMap } from "@opencode-ai/core/location-service-map"
28+
import type { LocationServices } from "@opencode-ai/core/location-services"
2729
import { testEffect } from "./lib/effect"
30+
import { imagePassthrough } from "./lib/image"
2831

2932
const executionCalls: SessionV2.ID[] = []
3033
const interruptCalls: SessionV2.ID[] = []
@@ -49,10 +52,22 @@ const execution = Layer.succeed(
4952
awaitIdle: () => Effect.void,
5053
}),
5154
)
55+
const locations = Layer.effect(
56+
LocationServiceMap.Service,
57+
LayerMap.make(
58+
() =>
59+
// Attachment admission only needs the location-scoped Image service.
60+
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
61+
imagePassthrough as unknown as Layer.Layer<LocationServices>,
62+
),
63+
)
5264
const it = testEffect(
5365
AppNodeBuilder.build(
5466
LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node, SessionV2.node]),
55-
[[SessionExecution.node, execution]],
67+
[
68+
[SessionExecution.node, execution],
69+
[LocationServiceMap.node, locations],
70+
],
5671
),
5772
)
5873
const sessionID = SessionV2.ID.make("ses_prompt_test")

‎packages/core/test/session-runner-tool-registry.test.ts‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { Tool } from "@opencode-ai/core/tool/tool"
33
import { AgentV2 } from "@opencode-ai/core/agent"
44
import type { PermissionV2 } from "@opencode-ai/core/permission"
55
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
6+
import { Image } from "@opencode-ai/core/image"
67
import { SessionV2 } from "@opencode-ai/core/session"
78
import { SessionMessage } from "@opencode-ai/core/session/message"
89
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
@@ -28,7 +29,26 @@ const outputStore = Layer.mock(ToolOutputStore.Service, {
2829
)
2930
},
3031
})
31-
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [[ToolOutputStore.node, outputStore]])
32+
const imageStore = Layer.mock(Image.Service, {
33+
normalize: (resource, content) =>
34+
resource === "too-large.png"
35+
? Effect.fail(
36+
new Image.SizeError({
37+
resource,
38+
width: 9_000,
39+
height: 9_000,
40+
bytes: content.content.length,
41+
maxWidth: 2_000,
42+
maxHeight: 2_000,
43+
maxBytes: 5,
44+
}),
45+
)
46+
: Effect.succeed({ ...content, content: "bm9ybWFsaXplZA==", mime: "image/jpeg" }),
47+
})
48+
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [
49+
[ToolOutputStore.node, outputStore],
50+
[Image.node, imageStore],
51+
])
3252
const it = testEffect(registryLayer)
3353
const identity = {
3454
agent: AgentV2.ID.make("build"),
@@ -255,6 +275,32 @@ describe("ToolRegistry", () => {
255275
}),
256276
)
257277

278+
it.effect("normalizes image tool output at settlement and drops unresizable images", () =>
279+
Effect.gen(function* () {
280+
const service = yield* ToolRegistry.Service
281+
yield* service.register({
282+
snapshot: Tool.make({
283+
description: "Return images",
284+
input: Schema.Struct({ text: Schema.String }),
285+
output: Schema.Struct({ text: Schema.String }),
286+
execute: ({ text }) => Effect.succeed({ text }),
287+
toModelOutput: ({ output }) => [
288+
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" },
289+
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" },
290+
{ type: "text", text: output.text },
291+
],
292+
}),
293+
}, { codemode: false })
294+
295+
const settlement = yield* settleTool(service, call("snapshot"))
296+
expect(settlement.output?.content).toEqual([
297+
{ type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" },
298+
{ type: "text", text: "snapshot" },
299+
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
300+
])
301+
}),
302+
)
303+
258304
it.effect("enforces transformed codecs at execution and projection boundaries", () =>
259305
Effect.gen(function* () {
260306
const service = yield* ToolRegistry.Service

‎packages/core/test/tool-question.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ import { SessionV2 } from "@opencode-ai/core/session"
88
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
99
import { QuestionTool } from "@opencode-ai/core/tool/question"
1010
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
11+
import { Image } from "@opencode-ai/core/image"
1112
import { testEffect } from "./lib/effect"
13+
import { imagePassthrough } from "./lib/image"
1214
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
1315
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
1416

@@ -84,6 +86,7 @@ const it = testEffect(
8486
[PermissionV2.node, permission],
8587
[Form.node, form],
8688
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
89+
[Image.node, imagePassthrough],
8790
]),
8891
)
8992

‎packages/core/test/tool-skill.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import { SkillTool } from "@opencode-ai/core/tool/skill"
1212
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
1313
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
1414
import { tmpdir } from "./fixture/tmpdir"
15+
import { Image } from "@opencode-ai/core/image"
1516
import { it } from "./lib/effect"
17+
import { imagePassthrough } from "./lib/image"
1618
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
1719
import { FSUtil } from "@opencode-ai/core/fs-util"
1820
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
@@ -90,6 +92,7 @@ describe("SkillTool", () => {
9092
[PermissionV2.node, permission],
9193
[SkillV2.node, skills],
9294
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
95+
[Image.node, imagePassthrough],
9396
],
9497
)
9598

‎packages/core/test/tool-webfetch.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
1111
import { WebFetchTool } from "@opencode-ai/core/tool/webfetch"
1212
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
1313
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
14+
import { Image } from "@opencode-ai/core/image"
1415
import { testEffect } from "./lib/effect"
16+
import { imagePassthrough } from "./lib/image"
1517
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
1618

1719
const webFetchToolNode = makeLocationNode({
@@ -50,6 +52,7 @@ const toolLayer = (replacements: LayerNode.Replacements = []) =>
5052
AppNodeBuilder.build(LayerNode.group([ToolRegistry.node, ToolRegistry.toolsNode, webFetchToolNode]), [
5153
[PermissionV2.node, permission],
5254
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
55+
[Image.node, imagePassthrough],
5356
...replacements,
5457
])
5558
const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]]))

‎packages/core/test/tool-websearch.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
1010
import { WebSearchTool } from "@opencode-ai/core/tool/websearch"
1111
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
1212
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
13+
import { Image } from "@opencode-ai/core/image"
1314
import { testEffect } from "./lib/effect"
15+
import { imagePassthrough } from "./lib/image"
1416
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
1517

1618
const webSearchToolNode = makeLocationNode({
@@ -138,6 +140,7 @@ const it = testEffect(
138140
[LayerNodePlatform.httpClient, http],
139141
[WebSearchTool.configNode, websearchConfig],
140142
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
143+
[Image.node, imagePassthrough],
141144
],
142145
),
143146
)

0 commit comments

Comments
 (0)