Skip to content

Commit 15c1513

Browse files
t3dotggclaude
andcommitted
fix(server): images pasted mid-turn no longer break later Claude image reads
Claude Code stores a steer's images as sent. Once a request holds more than 20 images, Anthropic rejects any image over 2000px on a side, so one oversized paste broke every later request in the thread. Steers now scale PNG and JPEG attachments to 2000px on the long edge before they reach Claude. Fixes #13647 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 06a84de commit 15c1513

5 files changed

Lines changed: 294 additions & 1 deletion

File tree

‎apps/server/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,10 @@
5454
"@t3tools/source-control-gitlab": "workspace:*",
5555
"effect": "catalog:",
5656
"jose": "catalog:",
57+
"jpeg-js": "0.4.4",
5758
"node-pty": "^1.2.0-beta.15",
5859
"playwright-core": "1.60.0",
60+
"pngjs": "7.0.0",
5961
"proper-lockfile": "4.1.2",
6062
"stream-chain": "^4.2.5",
6163
"stream-json": "3.6.0",
@@ -73,6 +75,7 @@
7375
"@t3tools/tailscale": "workspace:*",
7476
"@t3tools/web": "workspace:*",
7577
"@types/node": "catalog:",
78+
"@types/pngjs": "6.0.5",
7679
"@types/proper-lockfile": "^4.1.4",
7780
"@types/yauzl": "^3.4.0",
7881
"effect-acp": "workspace:*",

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import type {
2222
TaskUpdateOutput,
2323
} from "@anthropic-ai/claude-agent-sdk/sdk-tools";
2424
import * as NodeServices from "@effect/platform-node/NodeServices";
25+
import * as JpegJs from "jpeg-js";
26+
import { PNG } from "pngjs";
2527
import {
2628
ChatAttachmentId,
2729
ChatFileAttachment,
@@ -1651,6 +1653,154 @@ describe("ClaudeAdapterV2 attachments", () => {
16511653
),
16521654
);
16531655

1656+
it.effect("scales images over 2000px on a side when they steer a running turn", () =>
1657+
Effect.scoped(
1658+
Effect.gen(function* () {
1659+
const fileSystem = yield* FileSystem.FileSystem;
1660+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
1661+
const path = yield* Path.Path;
1662+
const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({
1663+
prefix: "t3-claude-v2-oversized-attachments-",
1664+
});
1665+
const offeredMessages: Array<SDKUserMessage> = [];
1666+
const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({
1667+
instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID,
1668+
settings: DEFAULT_CLAUDE_SETTINGS,
1669+
environment: {},
1670+
attachmentsDir,
1671+
fileSystem,
1672+
path,
1673+
crypto: yield* Crypto.Crypto,
1674+
idAllocator,
1675+
queryRunner: {
1676+
allocateSessionId: Effect.succeed("native-thread-claude-oversized-attachments"),
1677+
open: () =>
1678+
Effect.succeed({
1679+
messages: Stream.never,
1680+
offer: (message) =>
1681+
Effect.sync(() => {
1682+
offeredMessages.push(message);
1683+
}),
1684+
setModel: () => Effect.void,
1685+
setPermissionMode: () => Effect.void,
1686+
stopTask: () => Effect.void,
1687+
interrupt: Effect.void,
1688+
close: Effect.void,
1689+
}),
1690+
forkSession: () => Effect.die("unused forkSession"),
1691+
subagentLaunchToolUseId: () => Effect.succeed(null),
1692+
assertComplete: Effect.void,
1693+
},
1694+
});
1695+
const threadId = ThreadId.make("thread-claude-oversized-attachments");
1696+
const runtime = yield* adapter.openSession({
1697+
threadId,
1698+
providerSessionId: ProviderSessionId.make(
1699+
"provider-session-claude-oversized-attachments",
1700+
),
1701+
modelSelection: CLAUDE_TEST_MODEL_SELECTION,
1702+
runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY,
1703+
});
1704+
const providerThread = yield* runtime.ensureThread({
1705+
threadId,
1706+
modelSelection: CLAUDE_TEST_MODEL_SELECTION,
1707+
runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY,
1708+
});
1709+
1710+
// A phone screenshot taller than 2000px, with a transparent left half.
1711+
const screenshot = new PNG({ width: 1080, height: 2424 });
1712+
for (let i = 0; i < screenshot.data.length; i += 4) {
1713+
const x = (i / 4) % screenshot.width;
1714+
screenshot.data.set(x < 540 ? [0, 0, 0, 0] : [200, 40, 90, 255], i);
1715+
}
1716+
const screenshotAttachment = ChatImageAttachment.make({
1717+
type: "image",
1718+
id: ChatAttachmentId.make("thread-claude-oversized-12345678-1234-1234-1234-123456789abc"),
1719+
name: "screenshot.png",
1720+
mimeType: "image/png",
1721+
sizeBytes: 1,
1722+
});
1723+
const photo = JpegJs.encode(
1724+
{ width: 3000, height: 1500, data: new Uint8Array(3000 * 1500 * 4).fill(128) },
1725+
90,
1726+
);
1727+
const photoAttachment = ChatImageAttachment.make({
1728+
type: "image",
1729+
id: ChatAttachmentId.make("thread-claude-oversized-abcdefab-1234-1234-1234-123456789abc"),
1730+
name: "photo.jpg",
1731+
mimeType: "image/jpeg",
1732+
sizeBytes: 1,
1733+
});
1734+
yield* fileSystem.writeFile(
1735+
path.join(attachmentsDir, attachmentRelativePath(screenshotAttachment)!),
1736+
PNG.sync.write(screenshot),
1737+
);
1738+
yield* fileSystem.writeFile(
1739+
path.join(attachmentsDir, attachmentRelativePath(photoAttachment)!),
1740+
photo.data,
1741+
);
1742+
const imageBytes = (message: SDKUserMessage | undefined) => {
1743+
const content = message?.message.content;
1744+
if (!Array.isArray(content)) return assert.fail("expected content blocks");
1745+
return content.flatMap((block) =>
1746+
block.type === "image" && block.source.type === "base64"
1747+
? [Buffer.from(block.source.data, "base64")]
1748+
: [],
1749+
);
1750+
};
1751+
const attemptId = RunAttemptId.make("attempt-claude-oversized-attachments");
1752+
1753+
yield* runtime.startTurn(
1754+
makeClaudeTestTurnInput({
1755+
threadId,
1756+
providerThread,
1757+
now: yield* DateTime.now,
1758+
attemptId,
1759+
text: "Compare these.",
1760+
attachments: [screenshotAttachment],
1761+
}),
1762+
);
1763+
// Claude Code scales the images of a message that starts a turn itself.
1764+
const [startedScreenshot] = imageBytes(offeredMessages[0]);
1765+
const decodedStarted = PNG.sync.read(startedScreenshot!);
1766+
assert.deepEqual([decodedStarted.width, decodedStarted.height], [1080, 2424]);
1767+
1768+
// Pasted while Claude is still working: the steer path.
1769+
yield* runtime.steerTurn({
1770+
threadId,
1771+
runId: RunId.make("run-claude-oversized-attachments"),
1772+
providerThread,
1773+
providerTurnId: idAllocator.derive.providerTurn({
1774+
driver: ClaudeAdapterV2.CLAUDE_PROVIDER,
1775+
nativeTurnId: `turn:${attemptId}`,
1776+
}),
1777+
message: {
1778+
createdBy: "user",
1779+
creationSource: "web",
1780+
messageId: MessageId.make("message-claude-oversized-attachments-steer"),
1781+
text: "Also these.",
1782+
attachments: [screenshotAttachment, photoAttachment],
1783+
},
1784+
});
1785+
const [sentScreenshot, sentPhoto] = imageBytes(offeredMessages[1]);
1786+
const decodedScreenshot = PNG.sync.read(sentScreenshot!);
1787+
assert.deepEqual([decodedScreenshot.width, decodedScreenshot.height], [891, 2000]);
1788+
assert.deepEqual([...decodedScreenshot.data.subarray(0, 4)], [0, 0, 0, 0]);
1789+
const lastPixel = decodedScreenshot.data.length - 4;
1790+
assert.deepEqual(
1791+
[...decodedScreenshot.data.subarray(lastPixel, lastPixel + 4)],
1792+
[200, 40, 90, 255],
1793+
);
1794+
const decodedPhoto = JpegJs.decode(sentPhoto!, { useTArray: true });
1795+
assert.deepEqual([decodedPhoto.width, decodedPhoto.height], [2000, 1000]);
1796+
}).pipe(
1797+
Effect.provide(
1798+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
1799+
),
1800+
),
1801+
),
1802+
);
1803+
16541804
it.effect("rejects unsupported image types before opening a provider query", () =>
16551805
Effect.scoped(
16561806
Effect.gen(function* () {

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";
2+
import { fitClaudeImage } from "./claudeImageFit.ts";
23
import {
34
dynamicToolTitle,
45
formatReadToolLabel,
@@ -1523,6 +1524,9 @@ const makeClaudeUserMessageWithAttachments = Effect.fnUntraced(function* (input:
15231524
readonly attachmentsDir: string;
15241525
readonly fileSystem: FileSystem.FileSystem;
15251526
readonly skillNames?: ReadonlySet<string>;
1527+
// Claude Code scales the images of a message that starts a turn, but stores
1528+
// a steer's images as sent. Only steers pay for scaling them here.
1529+
readonly fitImages?: boolean;
15261530
}) {
15271531
if (input.attachments.length === 0) {
15281532
return makeClaudeUserMessage({
@@ -1589,7 +1593,9 @@ const makeClaudeUserMessageWithAttachments = Effect.fnUntraced(function* (input:
15891593
source: {
15901594
type: "base64",
15911595
media_type: attachment.mimeType,
1592-
data: Buffer.from(bytes).toString("base64"),
1596+
data: Buffer.from(
1597+
input.fitImages === true ? yield* fitClaudeImage(bytes, attachment.mimeType) : bytes,
1598+
).toString("base64"),
15931599
},
15941600
});
15951601
}
@@ -8898,6 +8904,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
88988904
priority: "now",
88998905
attachmentsDir,
89008906
fileSystem,
8907+
fitImages: true,
89018908
skillNames: yield* userInvocableSkillNames(currentTurn.input.runtimePolicy.cwd),
89028909
});
89038910
yield* Ref.update(steeredTurns, (current) => {
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { readImageDimensions } from "@t3tools/shared/imageDimensions";
2+
import * as Effect from "effect/Effect";
3+
import * as JpegJs from "jpeg-js";
4+
import { PNG } from "pngjs";
5+
6+
/**
7+
* Anthropic rejects every image over 2000px on a side once a request holds
8+
* more than 20 images, and the whole conversation carries the bad image from
9+
* then on. Claude Code scales the images of a message that starts a turn, but
10+
* not those of a message that steers a running turn, so T3 fits those itself.
11+
* Decoding runs on the server's event loop: about 0.4s for a Retina
12+
* screenshot, up to about 1.7s for a 12MP photo.
13+
*/
14+
const CLAUDE_MAX_IMAGE_EDGE = 2000;
15+
16+
// About 100MB of decoded RGBA. Covers 6K screenshots and 24MP photos; bigger
17+
// images pass through unchanged rather than risk exhausting the server.
18+
const MAX_DECODE_PIXELS = 25_000_000;
19+
20+
const JPEG_QUALITY = 90;
21+
22+
/**
23+
* Returns image bytes Claude accepts in a many-image request: PNG and JPEG
24+
* over 2000px on a side are scaled down and re-encoded in the same format.
25+
* Everything else, including GIF, WebP, and images that fail to decode, is
26+
* returned unchanged.
27+
*/
28+
export const fitClaudeImage = (bytes: Uint8Array, mimeType: string) => {
29+
const dimensions = readImageDimensions(bytes);
30+
if (
31+
dimensions === null ||
32+
Math.max(dimensions.width, dimensions.height) <= CLAUDE_MAX_IMAGE_EDGE ||
33+
dimensions.width * dimensions.height > MAX_DECODE_PIXELS ||
34+
(mimeType !== "image/png" && mimeType !== "image/jpeg")
35+
) {
36+
return Effect.succeed(bytes);
37+
}
38+
return Effect.try(() => (mimeType === "image/png" ? fitPng(bytes) : fitJpeg(bytes))).pipe(
39+
Effect.catch((cause) =>
40+
Effect.logWarning("Could not scale a Claude image attachment; sending it unchanged", {
41+
mimeType,
42+
cause,
43+
}).pipe(Effect.as(bytes)),
44+
),
45+
);
46+
};
47+
48+
function fitPng(bytes: Uint8Array): Uint8Array {
49+
const source = PNG.sync.read(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
50+
const target = targetSize(source.width, source.height);
51+
const output = new PNG({ ...target, colorType: source.alpha ? 6 : 2 });
52+
downscale(source.data, source.width, source.height, output.data, target.width, target.height);
53+
return PNG.sync.write(output, { colorType: source.alpha ? 6 : 2 });
54+
}
55+
56+
function fitJpeg(bytes: Uint8Array): Uint8Array {
57+
// jpeg-js reads and writes the EXIF block at runtime, but its types omit it.
58+
// Keeping it preserves a phone photo's orientation.
59+
const source: JpegJs.RawImageData<Uint8Array> & { readonly exifBuffer?: Uint8Array } =
60+
JpegJs.decode(bytes, {
61+
useTArray: true,
62+
formatAsRGBA: true,
63+
maxResolutionInMP: MAX_DECODE_PIXELS / 1_000_000,
64+
});
65+
const target = targetSize(source.width, source.height);
66+
const data = new Uint8Array(target.width * target.height * 4);
67+
downscale(source.data, source.width, source.height, data, target.width, target.height);
68+
const raw = { ...target, data, exifBuffer: source.exifBuffer };
69+
return JpegJs.encode(raw, JPEG_QUALITY).data;
70+
}
71+
72+
function targetSize(width: number, height: number) {
73+
const scale = CLAUDE_MAX_IMAGE_EDGE / Math.max(width, height);
74+
return {
75+
width: Math.max(1, Math.round(width * scale)),
76+
height: Math.max(1, Math.round(height * scale)),
77+
};
78+
}
79+
80+
// Box filter over RGBA. Color is weighted by alpha so transparent pixels do
81+
// not darken the opaque pixels they are averaged with.
82+
function downscale(
83+
source: Uint8Array,
84+
sourceWidth: number,
85+
sourceHeight: number,
86+
target: Uint8Array,
87+
targetWidth: number,
88+
targetHeight: number,
89+
) {
90+
for (let ty = 0; ty < targetHeight; ty++) {
91+
const y0 = Math.floor((ty * sourceHeight) / targetHeight);
92+
const y1 = Math.max(y0 + 1, Math.floor(((ty + 1) * sourceHeight) / targetHeight));
93+
for (let tx = 0; tx < targetWidth; tx++) {
94+
const x0 = Math.floor((tx * sourceWidth) / targetWidth);
95+
const x1 = Math.max(x0 + 1, Math.floor(((tx + 1) * sourceWidth) / targetWidth));
96+
let r = 0;
97+
let g = 0;
98+
let b = 0;
99+
let a = 0;
100+
for (let y = y0; y < y1; y++) {
101+
for (let x = x0; x < x1; x++) {
102+
const i = (y * sourceWidth + x) * 4;
103+
const alpha = source[i + 3]!;
104+
r += source[i]! * alpha;
105+
g += source[i + 1]! * alpha;
106+
b += source[i + 2]! * alpha;
107+
a += alpha;
108+
}
109+
}
110+
const o = (ty * targetWidth + tx) * 4;
111+
if (a > 0) {
112+
target[o] = Math.round(r / a);
113+
target[o + 1] = Math.round(g / a);
114+
target[o + 2] = Math.round(b / a);
115+
}
116+
target[o + 3] = Math.round(a / ((y1 - y0) * (x1 - x0)));
117+
}
118+
}
119+
}

‎pnpm-lock.yaml‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)