Skip to content

Commit 8d199b4

Browse files
fix(mobile): image-only messages ask the agent to respond, like desktop (pingdotgg#16480)
Co-authored-by: Yash Singh <[email protected]>
1 parent 2025430 commit 8d199b4

7 files changed

Lines changed: 59 additions & 12 deletions

File tree

‎apps/mobile/src/lib/composerContext.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
} from "@t3tools/contracts";
99
import { collectComposerInlineTokens } from "@t3tools/shared/composerInlineTokens";
1010
import {
11+
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
1112
collectComposerContextReferences,
1213
formatComposerContextReference,
1314
projectComposerContextForProvider,
@@ -207,6 +208,20 @@ describe("mobile composer context", () => {
207208
});
208209

209210
describe("host context compatibility", () => {
211+
it("sends an image-only prompt as the attachment-only prompt and keeps the image", () => {
212+
const context: OrchestrationMessageContext = { version: 1, records: [image] };
213+
const reference = formatComposerContextReference(image);
214+
expect(serializeComposerMessageForServer(reference, context, true)).toEqual({
215+
text: ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
216+
context,
217+
});
218+
const withProse = `${reference} what is this?`;
219+
expect(serializeComposerMessageForServer(withProse, context, true)).toEqual({
220+
text: withProse,
221+
context,
222+
});
223+
});
224+
210225
it.each(["existing-thread", "new-task"])("serializes %s sends for an older host", (path) => {
211226
const pr = pullRequestComposerContext(
212227
{

‎apps/mobile/src/lib/composerContext.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ import {
1313
} from "@t3tools/contracts";
1414
import * as Schema from "effect/Schema";
1515
import {
16+
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
1617
collectComposerContextReferences,
1718
formatComposerContextReference,
19+
isImageReferencesOnly,
1820
replaceComposerContextReferences,
1921
sanitizeComposerContextLabel,
2022
} from "@t3tools/shared/composerContextReferences";
@@ -229,12 +231,19 @@ export function reidentifyComposerContext(
229231
};
230232
}
231233

232-
/** Keep queued records canonical; choose the wire format against the host at dispatch time. */
234+
/**
235+
* Keep queued records canonical; choose the wire format against the host at dispatch time.
236+
* An image-only prompt goes out as the attachment-only prompt web sends, so the agent is still
237+
* asked to respond. The images stay in the message's attachments and context records.
238+
*/
233239
export function serializeComposerMessageForServer(
234-
text: string,
240+
composerText: string,
235241
context: OrchestrationMessageContext | undefined,
236242
supportsInlineMessageContext: boolean,
237243
): { text: string; context?: OrchestrationMessageContext } {
244+
const text = isImageReferencesOnly(composerText)
245+
? ATTACHMENT_ONLY_BOOTSTRAP_PROMPT
246+
: composerText;
238247
return supportsInlineMessageContext
239248
? { text, ...(context ? { context } : {}) }
240249
: { text: serializeLegacyContextMessage({ text, records: context?.records ?? [] }) };

‎apps/web/src/components/ChatView.tsx‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -607,10 +607,8 @@ import {
607607
supportsServerUpdateThreadContinuation,
608608
} from "../versionSkew";
609609
import { useAssetUrls } from "../assets/assetUrls";
610-
import {
611-
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
612-
recallableComposerPrompt,
613-
} from "./chat/composerPromptHistory";
610+
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
611+
import { recallableComposerPrompt } from "./chat/composerPromptHistory";
614612
import { observeResize } from "~/lib/observeResize";
615613

616614
const EMPTY_PROVIDERS: ServerProvider[] = [];

‎apps/web/src/components/chat/composerPromptHistory.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { describe, expect, it } from "vite-plus/test";
22

3+
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
34
import { buildPlanImplementationPrompt } from "../../proposedPlan";
45
import {
5-
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
66
buildComposerPromptHistoryEntries,
77
recallableComposerPrompt,
88
stepComposerPromptHistory,

‎apps/web/src/components/chat/composerPromptHistory.ts‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
import { collectComposerContextReferences } from "@t3tools/shared/composerContextReferences";
1+
import {
2+
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
3+
collectComposerContextReferences,
4+
} from "@t3tools/shared/composerContextReferences";
25
import { PLAN_IMPLEMENTATION_PROMPT_PREFIX } from "../../proposedPlan";
36

47
/**
@@ -15,10 +18,6 @@ const REVIEW_COMMENT_BLOCK_PATTERN = /<review_comment\b[^>]*>[\s\S]*?<\/review_c
1518
const TRAILING_LEGACY_CONTEXT =
1619
/\n*<(terminal_context|element_context|preview_annotation)>\n([\s\S]*?)\n<\/\1>\s*$/;
1720

18-
/** Text sent in place of an empty prompt when a message is attachments only. */
19-
export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
20-
"[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
21-
2221
export interface ComposerPromptHistoryMessage {
2322
readonly id: string;
2423
readonly role: string;

‎packages/shared/src/composerContextReferences.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
formatComposerContextHref,
77
formatComposerContextProviderMarker,
88
formatComposerContextReference,
9+
isImageReferencesOnly,
910
parseComposerContextHref,
1011
projectComposerContextForProvider,
1112
replaceComposerContextReferences,
@@ -86,6 +87,17 @@ describe("labels and reference links", () => {
8687
expect(collectComposerContextReferences("[x](https://example.com)")).toEqual([]);
8788
});
8889

90+
it("recognizes a prompt made only of image references", () => {
91+
const image = "![a.png](t3-context://v1/image/ctx_1)";
92+
const file = "[notes.md](t3-context://v1/file/ctx_3)";
93+
expect(isImageReferencesOnly(image)).toBe(true);
94+
expect(isImageReferencesOnly(` ${image}\n![b.png](t3-context://v1/image/ctx_2) `)).toBe(true);
95+
expect(isImageReferencesOnly(`${image} what is this?`)).toBe(false);
96+
expect(isImageReferencesOnly(file)).toBe(false);
97+
expect(isImageReferencesOnly(`${image} ${file}`)).toBe(false);
98+
expect(isImageReferencesOnly(" ")).toBe(false);
99+
});
100+
89101
it("replaces occurrences in place", () => {
90102
const text = "a [x](t3-context://v1/skill/ctx_1) b [y](t3-context://v1/file/ctx_2) c";
91103
expect(replaceComposerContextReferences(text, (o) => `<${o.contextId}>`)).toBe(

‎packages/shared/src/composerContextReferences.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,20 @@ export function collectComposerContextReferences(
9292
return occurrences;
9393
}
9494

95+
/** Text sent in place of an empty prompt when a message is attachments only. */
96+
export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
97+
"[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
98+
99+
/** Whether `text` is image references and nothing else, which is how an image-only prompt reads. */
100+
export function isImageReferencesOnly(text: string): boolean {
101+
const references = collectComposerContextReferences(text);
102+
return (
103+
references.length > 0 &&
104+
references.every((reference) => reference.kind === "image") &&
105+
replaceComposerContextReferences(text, () => "").trim() === ""
106+
);
107+
}
108+
95109
export function replaceComposerContextReferences(
96110
text: string,
97111
replace: (occurrence: ComposerContextReferenceOccurrence) => string,

0 commit comments

Comments
 (0)