Skip to content

Commit 79ef593

Browse files
author
yida
committed
fix(tui): embed pasted images and carry path in description
Keep data: embedding at paste time so remote servers and ephemeral paths still work. Stash the absolute filesystem path in FileAttachment.description with the short basename in name, and have toLLMMessages emit Attached file text only for absolute path descriptions. Fixes anomalyco#41454
1 parent 9296e12 commit 79ef593

6 files changed

Lines changed: 106 additions & 14 deletions

File tree

‎packages/core/src/session/runner/to-llm-message.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,23 @@ const directoryAttachment = (file: FileAttachment): ContentPart => ({
7272
},
7373
})
7474

75+
// Local TUI path pastes embed bytes as data: and stash the absolute path in
76+
// description. Only treat absolute filesystem paths as locations — never
77+
// display names like "clipboard" or free-form descriptions.
78+
const absoluteFilesystemPath = (value: string) =>
79+
value.startsWith("/") || value.startsWith("\\\\") || /^[A-Za-z]:[\\/]/.test(value)
80+
81+
const inlineAttachmentPath = (file: FileAttachment) => {
82+
if (file.source.type !== "inline" || file.description === undefined) return undefined
83+
return absoluteFilesystemPath(file.description) ? file.description : undefined
84+
}
85+
7586
const attachmentContent = (file: FileAttachment): ContentPart[] => {
7687
if (file.mime === "text/plain") return [textAttachment(file)]
7788
if (file.mime === "application/x-directory") return [directoryAttachment(file)]
7889
if (imageMimes.has(file.mime) || file.mime === "application/pdf") {
79-
const location = attachmentLocation(file)
80-
return [...(location === undefined ? [] : [Message.text(`Attached file: ${location}`)]), media(file)]
90+
const origin = attachmentLocation(file) ?? inlineAttachmentPath(file)
91+
return [...(origin === undefined ? [] : [Message.text(`Attached file: ${origin}`)]), media(file)]
8192
}
8293
return []
8394
}

‎packages/core/test/session-runner-message.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,6 +529,62 @@ Earlier work
529529
])
530530
})
531531

532+
test("notes absolute paths carried in inline attachment descriptions", () => {
533+
const data = Base64.make("AAECAw==")
534+
const location = path.resolve("/project/IMG_3480.JPG")
535+
const messages = toLLMMessages(
536+
[
537+
SessionMessage.User.make({
538+
id: id("user-inline-image-path"),
539+
type: "user",
540+
text: "Inspect this image",
541+
files: [
542+
FileAttachment.make({
543+
data,
544+
mime: "image/png",
545+
source: { type: "inline" },
546+
name: "IMG_3480.JPG",
547+
description: location,
548+
}),
549+
FileAttachment.make({
550+
data,
551+
mime: "image/png",
552+
source: { type: "inline" },
553+
name: "clipboard",
554+
}),
555+
FileAttachment.make({
556+
data,
557+
mime: "image/png",
558+
source: { type: "inline" },
559+
name: "photo.png",
560+
description: "a screenshot from earlier",
561+
}),
562+
],
563+
time: { created },
564+
}),
565+
],
566+
model,
567+
)
568+
569+
expect(messages[0]?.content).toEqual([
570+
{ type: "text", text: "Inspect this image" },
571+
{ type: "text", text: `Attached file: ${location}` },
572+
{
573+
type: "media",
574+
media: Media.base64(data, "image/png"),
575+
filename: "IMG_3480.JPG",
576+
metadata: { description: location },
577+
},
578+
{ type: "media", media: Media.base64(data, "image/png"), filename: "clipboard" },
579+
{
580+
type: "media",
581+
media: Media.base64(data, "image/png"),
582+
filename: "photo.png",
583+
metadata: { description: "a screenshot from earlier" },
584+
},
585+
])
586+
})
587+
532588
test("exposes admitted local image source paths before provider media", () => {
533589
const data = Base64.make("AAECAw==")
534590
const location = path.resolve("/project/IMG_3480.JPG")

‎packages/tui/src/component/prompt/index.tsx‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1475,10 +1475,14 @@ export function Prompt(props: PromptProps) {
14751475
}, 0)
14761476
}
14771477

1478-
function pasteAttachment(file: { filename?: string; uri: string }) {
1478+
function pasteAttachment(file: { filename?: string; uri: string; description?: string }) {
14791479
const currentOffset = input.cursorOffset
14801480
const extmarkStart = currentOffset
1481-
const virtualText = promptAttachmentLabel(store.prompt.files, { uri: file.uri, name: file.filename })
1481+
const virtualText = promptAttachmentLabel(store.prompt.files, {
1482+
uri: file.uri,
1483+
name: file.filename,
1484+
description: file.description,
1485+
})
14821486
const extmarkEnd = extmarkStart + virtualText.length
14831487
const textToInsert = virtualText + " "
14841488

@@ -1495,6 +1499,7 @@ export function Prompt(props: PromptProps) {
14951499
const part: NonNullable<PromptInfo["files"]>[number] = {
14961500
uri: file.uri,
14971501
name: file.filename,
1502+
description: file.description,
14981503
mention: {
14991504
start: extmarkStart,
15001505
end: extmarkEnd,

‎packages/tui/src/component/prompt/local-attachment.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import path from "node:path"
2-
import { pathToFileURL } from "node:url"
32

43
// Bound filesystem work per terminal paste; the byte budget also bounds staged data.
54
const MAX_PASTED_FILEPATHS = 32
@@ -51,12 +50,13 @@ export async function resolvePastedAttachments(text: string, platform: string) {
5150
if (item.attachment.type === "text") {
5251
return { type: "text" as const, content: item.attachment.content, filename }
5352
}
54-
// Keep the short display name, and pass the path as a file:// URI so
55-
// admission retains source.uri and toLLMMessages can expose the location.
53+
// Embed bytes at paste time (remote servers / ephemeral paths) and carry
54+
// the absolute path in description for model-visible location text.
5655
return {
5756
type: "file" as const,
58-
uri: pathToFileURL(item.filepath).href,
57+
uri: `data:${item.attachment.mime};base64,${Buffer.from(item.attachment.content).toString("base64")}`,
5958
filename,
59+
description: item.filepath,
6060
}
6161
})
6262
}

‎packages/tui/src/mini/footer.prompt.tsx‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -840,10 +840,14 @@ export function createPromptState(input: PromptInput): PromptState {
840840
}
841841
}
842842

843-
const pasteAttachment = (file: { uri: string; filename?: string }) => {
843+
const pasteAttachment = (file: { uri: string; filename?: string; description?: string }) => {
844844
if (!area || area.isDestroyed) return
845845
syncDraft()
846-
const value = promptAttachmentLabel(attachments(), { uri: file.uri, name: file.filename })
846+
const value = promptAttachmentLabel(attachments(), {
847+
uri: file.uri,
848+
name: file.filename,
849+
description: file.description,
850+
})
847851
area.insertText(value + " ")
848852
const end = area.cursorOffset - 1
849853
const start = end - stringWidth(value)
@@ -853,6 +857,7 @@ export function createPromptState(input: PromptInput): PromptState {
853857
type: "file",
854858
url: file.uri,
855859
filename: file.filename,
860+
description: file.description,
856861
mime: file.uri.slice(5, file.uri.indexOf(";")),
857862
source: { type: "file", text: { start, end, value } },
858863
})

‎packages/tui/test/prompt/local-attachment.test.ts‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,12 @@ describe("prompt local attachments", () => {
9797

9898
for (const input of [file, `'${file}'`, pathToFileURL(file).href]) {
9999
expect(await resolvePastedAttachments(input, process.platform)).toEqual([
100-
{ type: "file", uri: pathToFileURL(file).href, filename: "one image.png" },
100+
{
101+
type: "file",
102+
uri: "data:image/png;base64,AQID",
103+
filename: "one image.png",
104+
description: file,
105+
},
101106
])
102107
}
103108
})
@@ -113,8 +118,18 @@ describe("prompt local attachments", () => {
113118
`# dropped files\r\n${pathToFileURL(image).href}\r\n${pathToFileURL(pdf).href}`,
114119
]) {
115120
expect(await resolvePastedAttachments(input, process.platform)).toEqual([
116-
{ type: "file", uri: pathToFileURL(image).href, filename: "one image.png" },
117-
{ type: "file", uri: pathToFileURL(pdf).href, filename: "two file.pdf" },
121+
{
122+
type: "file",
123+
uri: "data:image/png;base64,AQID",
124+
filename: "one image.png",
125+
description: image,
126+
},
127+
{
128+
type: "file",
129+
uri: "data:application/pdf;base64,BAUG",
130+
filename: "two file.pdf",
131+
description: pdf,
132+
},
118133
])
119134
}
120135
})
@@ -159,7 +174,7 @@ describe("prompt local attachments", () => {
159174
])
160175

161176
expect(await resolvePastedAttachments(`${image} ${svg}`, process.platform)).toMatchObject([
162-
{ type: "file", uri: pathToFileURL(image).href, filename: "image.png" },
177+
{ type: "file", filename: "image.png", description: image },
163178
{ type: "text", content, filename: "image.svg" },
164179
])
165180
await Bun.write(svg, content + " ")

0 commit comments

Comments
 (0)