Skip to content

Commit 9bec8e3

Browse files
committed
feat(server): send pdf attachments to claude and antigravity as documents
1 parent b727408 commit 9bec8e3

9 files changed

Lines changed: 306 additions & 5 deletions

File tree

‎apps/server/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
"effect": "catalog:",
5656
"jose": "catalog:",
5757
"node-pty": "^1.2.0-beta.15",
58+
"pdf-lib": "1.17.1",
5859
"playwright-core": "1.60.0",
5960
"proper-lockfile": "4.1.2",
6061
"stream-chain": "^4.2.5",

‎apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as NodeServices from "@effect/platform-node/NodeServices";
22
import { assert, describe, it } from "@effect/vitest";
33
import {
4+
ChatFileAttachment,
45
MessageId,
56
NodeId,
67
ProjectId,
@@ -13,6 +14,7 @@ import {
1314
import { resolveSelfInvocation } from "@t3tools/shared/nodeRuntime";
1415
import * as Crypto from "effect/Crypto";
1516
import * as DateTime from "effect/DateTime";
17+
import * as Deferred from "effect/Deferred";
1618
import * as Effect from "effect/Effect";
1719
import * as Exit from "effect/Exit";
1820
import * as FileSystem from "effect/FileSystem";
@@ -27,6 +29,7 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS
2729
import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost";
2830
import type * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime";
2931
import { makeAntigravityAcpRuntime } from "../../provider/acp/AntigravityAcpSupport.ts";
32+
import { resolveAttachmentPath } from "../../attachmentStore.ts";
3033
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
3134
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
3235
import {
@@ -289,6 +292,20 @@ describe("AntigravityAdapterV2 workspace changes", () => {
289292
new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
290293
);
291294
type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"];
295+
const promptReceived = yield* Deferred.make<Parameters<RuntimeService["prompt"]>[0]>();
296+
const host = yield* ProviderHost.ProviderHost;
297+
const document = ChatFileAttachment.make({
298+
type: "file",
299+
id: "native-pdf",
300+
name: "diagram.pdf",
301+
mimeType: "application/pdf",
302+
sizeBytes: 9,
303+
});
304+
const documentPath = resolveAttachmentPath({
305+
attachmentsDir: host.paths.attachmentsDir,
306+
attachment: document,
307+
})!;
308+
yield* fileSystem.writeFileString(documentPath, "%PDF-1.7");
292309
let readTextFile: Parameters<RuntimeService["handleReadTextFile"]>[0] | undefined;
293310
const instanceId = ProviderInstanceId.make("antigravity-workspace-change-test");
294311
const adapter = yield* makeAntigravityAdapterV2({
@@ -308,6 +325,23 @@ describe("AntigravityAdapterV2 workspace changes", () => {
308325
Effect.provideService(Crypto.Crypto, crypto),
309326
Effect.map((runtime): RuntimeService => ({
310327
...runtime,
328+
prompt: (request) =>
329+
Deferred.succeed(promptReceived, request).pipe(
330+
Effect.andThen(runtime.prompt(request)),
331+
),
332+
start: () =>
333+
runtime.start().pipe(
334+
Effect.map((started) => ({
335+
...started,
336+
initializeResult: {
337+
...started.initializeResult,
338+
agentCapabilities: {
339+
...started.initializeResult.agentCapabilities,
340+
promptCapabilities: { image: true, audio: true, embeddedContext: true },
341+
},
342+
},
343+
})),
344+
),
311345
handleReadTextFile: (handler) =>
312346
Effect.sync(() => {
313347
readTextFile = handler;
@@ -378,7 +412,7 @@ describe("AntigravityAdapterV2 workspace changes", () => {
378412
creationSource: "web",
379413
messageId: MessageId.make("message-antigravity-workspace-change"),
380414
text: "read b.txt",
381-
attachments: [],
415+
attachments: [document],
382416
},
383417
modelSelection,
384418
runtimePolicy: policyFor(workspaceB),
@@ -388,6 +422,19 @@ describe("AntigravityAdapterV2 workspace changes", () => {
388422
Stream.filter((event) => event.type === "provider_turn.updated"),
389423
Stream.runHead,
390424
);
425+
const sent = yield* Deferred.await(promptReceived);
426+
assert.deepEqual(
427+
sent.prompt.filter((block) => block.type === "resource_link"),
428+
[
429+
{
430+
type: "resource_link",
431+
uri: (yield* path.toFileUrl(documentPath)).href,
432+
name: "diagram.pdf",
433+
mimeType: "application/pdf",
434+
},
435+
],
436+
);
437+
assert.include(JSON.stringify(sent.prompt), host.resolveAttachmentPath(document)!);
391438
if (readTextFile === undefined) {
392439
return yield* Effect.die("Antigravity sessions must serve client file requests");
393440
}

‎apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import type * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSession
2020
import {
2121
antigravityPermissionMode,
2222
applyAntigravityAcpModelSelection,
23+
buildAntigravityPrompt,
2324
type AntigravityAcpRuntimeInput,
2425
} from "../../provider/acp/AntigravityAcpSupport.ts";
2526
import {
@@ -158,6 +159,16 @@ export function makeAntigravityAcpAdapterFlavor(
158159
runtimeHarness: "Antigravity",
159160
capabilities: AntigravityProviderCapabilitiesV2,
160161
makeRuntime,
162+
buildPrompt: ({ text, attachments, promptCapabilities }) =>
163+
buildAntigravityPrompt({
164+
input: text,
165+
attachments,
166+
attachmentsDir: options.host.paths.attachmentsDir,
167+
promptCapabilities,
168+
}).pipe(
169+
Effect.provideService(FileSystem.FileSystem, options.fileSystem),
170+
Effect.provideService(Path.Path, options.path),
171+
),
161172
// Loading history replays every tool call; resume restores the session
162173
// without the replay and is what the official client does.
163174
preferResumeSession: true,

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// @effect-diagnostics nodeBuiltinImport:off -- workflow fixtures use files and the fake Claude CLI uses Node streams
2+
import { PDFDocument } from "pdf-lib";
23
import * as NodeEvents from "node:events";
34
import * as NodeFS from "node:fs";
45
import * as NodeOS from "node:os";
@@ -24,6 +25,7 @@ import type {
2425
import * as NodeServices from "@effect/platform-node/NodeServices";
2526
import {
2627
ChatAttachmentId,
28+
type ChatAttachment,
2729
ChatFileAttachment,
2830
ChatImageAttachment,
2931
CheckpointId,
@@ -1601,7 +1603,7 @@ describe("ClaudeAdapterV2 resume compaction", () => {
16011603
});
16021604

16031605
describe("ClaudeAdapterV2 attachments", () => {
1604-
it.effect("forwards images and references generic files on sends and steering", () =>
1606+
it.effect("sends native PDFs and images with path fallbacks", () =>
16051607
Effect.scoped(
16061608
Effect.gen(function* () {
16071609
const fileSystem = yield* FileSystem.FileSystem;
@@ -1677,9 +1679,12 @@ describe("ClaudeAdapterV2 attachments", () => {
16771679
path.join(attachmentsDir, attachmentRelativePath(attachment)!),
16781680
Uint8Array.from([1, 2, 3, 4]),
16791681
);
1682+
const pdf = yield* Effect.promise(() => PDFDocument.create());
1683+
pdf.addPage();
1684+
const pdfBytes = yield* Effect.promise(() => pdf.save());
16801685
yield* fileSystem.writeFile(
16811686
path.join(attachmentsDir, attachmentRelativePath(document)!),
1682-
Uint8Array.from([5, 6, 7, 8]),
1687+
pdfBytes,
16831688
);
16841689
const attemptId = RunAttemptId.make("attempt-claude-attachments");
16851690
const now = yield* DateTime.now;
@@ -1703,13 +1708,22 @@ describe("ClaudeAdapterV2 attachments", () => {
17031708
data: "AQIDBA==",
17041709
},
17051710
} as const;
1711+
const expectedDocumentBlock = {
1712+
type: "document",
1713+
source: {
1714+
type: "base64",
1715+
media_type: "application/pdf",
1716+
data: Buffer.from(pdfBytes).toString("base64"),
1717+
},
1718+
} as const;
17061719
const expectedAttachmentPath = path.join(
17071720
attachmentsDir,
17081721
attachmentRelativePath(attachment)!,
17091722
);
17101723
const expectedDocumentPath = path.join(attachmentsDir, attachmentRelativePath(document)!);
17111724
assert.deepEqual(offeredMessages[0]?.message.content, [
17121725
expectedImageBlock,
1726+
expectedDocumentBlock,
17131727
{
17141728
type: "text",
17151729
text: `Ultrathink:\nWhat's in this image?\n\n[Attached image "diagram.png" is saved at: ${expectedAttachmentPath}]\n\n[Attached file "requirements.pdf" is saved at: ${expectedDocumentPath}]`,
@@ -1737,11 +1751,69 @@ describe("ClaudeAdapterV2 attachments", () => {
17371751
assert.equal(offeredMessages[1]?.priority, "now");
17381752
assert.deepEqual(offeredMessages[1]?.message.content, [
17391753
expectedImageBlock,
1754+
expectedDocumentBlock,
17401755
{
17411756
type: "text",
17421757
text: `Ultrathink:\nFocus on the diagram labels.\n\n[Attached image "diagram.png" is saved at: ${expectedAttachmentPath}]\n\n[Attached file "requirements.pdf" is saved at: ${expectedDocumentPath}]`,
17431758
},
17441759
]);
1760+
const documentPath = path.join(attachmentsDir, attachmentRelativePath(document)!);
1761+
for (const scenario of [
1762+
"pages",
1763+
"bytes",
1764+
"invalid",
1765+
"aggregate-pages",
1766+
"pasted-text",
1767+
"text-only",
1768+
]) {
1769+
let attachments: ReadonlyArray<ChatAttachment> = [document];
1770+
if (scenario === "pages" || scenario === "aggregate-pages") {
1771+
const large = yield* Effect.promise(() => PDFDocument.create());
1772+
for (let i = 0; i < (scenario === "pages" ? 101 : 60); i++) large.addPage();
1773+
yield* fileSystem.writeFile(documentPath, yield* Effect.promise(() => large.save()));
1774+
if (scenario === "aggregate-pages") {
1775+
const second = {
1776+
...document,
1777+
id: ChatAttachmentId.make("second-pdf"),
1778+
name: "second.pdf",
1779+
};
1780+
yield* fileSystem.copyFile(
1781+
documentPath,
1782+
path.join(attachmentsDir, attachmentRelativePath(second)!),
1783+
);
1784+
attachments = [document, second];
1785+
}
1786+
} else if (scenario === "bytes") {
1787+
yield* fileSystem.truncate(documentPath, 20 * 1024 * 1024 + 1);
1788+
} else if (scenario === "invalid") {
1789+
yield* fileSystem.writeFileString(documentPath, "This is not a PDF");
1790+
} else if (scenario === "pasted-text") {
1791+
yield* fileSystem.writeFile(documentPath, pdfBytes);
1792+
attachments = [{ ...document, source: { _tag: "pasted-text" } }];
1793+
} else {
1794+
attachments = [];
1795+
yield* fileSystem.remove(documentPath);
1796+
}
1797+
yield* runtime.steerTurn({
1798+
threadId,
1799+
runId: RunId.make("run-claude-attachments"),
1800+
providerThread,
1801+
providerTurnId,
1802+
message: {
1803+
createdBy: "user",
1804+
creationSource: "web",
1805+
messageId: MessageId.make(`message-${scenario}`),
1806+
text: "Continue.",
1807+
attachments,
1808+
},
1809+
});
1810+
const sent = offeredMessages.at(-1)!.message.content;
1811+
const documents =
1812+
typeof sent === "string" ? [] : sent.filter((block) => block.type === "document");
1813+
assert.lengthOf(documents, scenario === "aggregate-pages" ? 1 : 0, scenario);
1814+
if (scenario === "text-only") assert.equal(sent, "Ultrathink:\nContinue.");
1815+
else assert.include(JSON.stringify(sent), documentPath);
1816+
}
17451817
}).pipe(
17461818
Effect.provide(
17471819
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";
2+
import { PDFDocument } from "pdf-lib";
23
import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts";
34
import {
45
dynamicToolTitle,
@@ -59,6 +60,7 @@ import {
5960
} from "@t3tools/shared/claudeCompaction";
6061
import {
6162
type ChatAttachment,
63+
PROVIDER_SEND_TURN_MAX_FILE_BYTES,
6264
ClaudeSettings,
6365
defaultInstanceIdForDriver,
6466
type ModelSelection,
@@ -1652,12 +1654,61 @@ const makeClaudeUserMessageWithAttachments = Effect.fnUntraced(function* (input:
16521654
? undefined
16531655
: planClaudeSkillDispatch(textWithAttachmentPaths, input.skillNames);
16541656
const content: Array<ClaudeUserContentBlock> = [];
1657+
// Base64 adds a third to the payload. Leave room below Anthropic's 32 MB
1658+
// request limit for text, instructions, and SDK framing. The SDK does not
1659+
// expose document/context-window capabilities, so use the 100-page limit
1660+
// that also applies to models with less than a 1M-token context window.
1661+
let nativeBytes = Buffer.byteLength(textWithAttachmentPaths);
1662+
let pdfPages = 0;
1663+
const nativeByteBudget = Math.min(PROVIDER_SEND_TURN_MAX_FILE_BYTES, 20 * 1024 * 1024);
1664+
for (const attachment of input.attachments) {
1665+
if (attachment.type === "image") nativeBytes += attachment.sizeBytes;
1666+
}
16551667
if (dispatch?.leadingText !== undefined) {
16561668
content.push({ type: "text", text: dispatch.leadingText });
16571669
}
16581670

16591671
for (const attachment of input.attachments) {
16601672
if (attachment.type === "file") {
1673+
if (
1674+
attachment.mimeType.toLowerCase().split(";", 1)[0]?.trim() !== "application/pdf" ||
1675+
("source" in attachment && attachment.source?._tag === "pasted-text")
1676+
)
1677+
continue;
1678+
const attachmentPath = resolveAttachmentPath({
1679+
attachmentsDir: input.attachmentsDir,
1680+
attachment,
1681+
});
1682+
if (attachmentPath === null) continue;
1683+
const remainingBytes = nativeByteBudget - nativeBytes;
1684+
const info = yield* input.fileSystem
1685+
.stat(attachmentPath)
1686+
.pipe(Effect.orElseSucceed(() => null));
1687+
if (info?.type !== "File" || Number(info.size) > remainingBytes || remainingBytes <= 0)
1688+
continue;
1689+
const bytes = yield* input.fileSystem
1690+
.stream(attachmentPath, { bytesToRead: remainingBytes + 1 })
1691+
.pipe(
1692+
Stream.runCollect,
1693+
Effect.map((chunks) => Buffer.concat(chunks)),
1694+
Effect.orElseSucceed(() => null),
1695+
);
1696+
if (bytes === null || bytes.length > remainingBytes) continue;
1697+
const document = yield* Effect.tryPromise(() => PDFDocument.load(bytes)).pipe(
1698+
Effect.orElseSucceed(() => null),
1699+
);
1700+
if (
1701+
document === null ||
1702+
document.getPageCount() === 0 ||
1703+
pdfPages + document.getPageCount() > 100
1704+
)
1705+
continue;
1706+
nativeBytes += bytes.length;
1707+
pdfPages += document.getPageCount();
1708+
content.push({
1709+
type: "document",
1710+
source: { type: "base64", media_type: "application/pdf", data: bytes.toString("base64") },
1711+
});
16611712
continue;
16621713
}
16631714
if (!isSupportedClaudeImageMimeType(attachment.mimeType)) {

0 commit comments

Comments
 (0)