Skip to content

Commit 10d5dec

Browse files
committed
fix(mobile): image-only messages ask the agent to respond, like desktop
An image sent from the phone with no text reached the agent as only the image reference. Desktop sends the attachment-only prompt in that case. The constant moves to shared, and the mobile outbox sends it when the text is nothing but image references. The images stay attached. Closes #14049.
1 parent d15210c commit 10d5dec

8 files changed

Lines changed: 60 additions & 13 deletions

File tree

‎apps/mobile/src/lib/composerContext.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
} from "@t3tools/contracts";
99
import { collectComposerInlineTokens } from "@t3tools/shared/composerInlineTokens";
1010
import {
11+
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
1112
collectComposerContextReferences,
1213
formatComposerContextReference,
1314
projectComposerContextForProvider,
@@ -207,6 +208,20 @@ describe("mobile composer context", () => {
207208
});
208209

209210
describe("host context compatibility", () => {
211+
it("sends an image-only prompt as the attachment-only prompt and keeps the image", () => {
212+
const context: OrchestrationMessageContext = { version: 1, records: [image] };
213+
const reference = formatComposerContextReference(image);
214+
expect(serializeComposerMessageForServer(reference, context, true)).toEqual({
215+
text: ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
216+
context,
217+
});
218+
const withProse = `${reference} what is this?`;
219+
expect(serializeComposerMessageForServer(withProse, context, true)).toEqual({
220+
text: withProse,
221+
context,
222+
});
223+
});
224+
210225
it.each(["existing-thread", "new-task"])("serializes %s sends for an older host", (path) => {
211226
const pr = pullRequestComposerContext(
212227
{

‎apps/mobile/src/lib/composerContext.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@ import {
1111
} from "@t3tools/contracts";
1212
import * as Schema from "effect/Schema";
1313
import {
14+
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
1415
collectComposerContextReferences,
1516
formatComposerContextReference,
17+
isImageReferencesOnly,
1618
replaceComposerContextReferences,
1719
} from "@t3tools/shared/composerContextReferences";
1820
import {
@@ -212,12 +214,19 @@ export function reidentifyComposerContext(
212214
};
213215
}
214216

215-
/** Keep queued records canonical; choose the wire format against the host at dispatch time. */
217+
/**
218+
* Keep queued records canonical; choose the wire format against the host at dispatch time.
219+
* An image-only prompt goes out as the attachment-only prompt web sends, so the agent is still
220+
* asked to respond. The images stay in the message's attachments and context records.
221+
*/
216222
export function serializeComposerMessageForServer(
217-
text: string,
223+
composerText: string,
218224
context: OrchestrationMessageContext | undefined,
219225
supportsInlineMessageContext: boolean,
220226
): { text: string; context?: OrchestrationMessageContext } {
227+
const text = isImageReferencesOnly(composerText)
228+
? ATTACHMENT_ONLY_BOOTSTRAP_PROMPT
229+
: composerText;
221230
return supportsInlineMessageContext
222231
? { text, ...(context ? { context } : {}) }
223232
: { text: serializeLegacyContextMessage({ text, records: context?.records ?? [] }) };

‎apps/web/src/components/ChatView.tsx‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -528,10 +528,8 @@ import {
528528
supportsServerUpdateThreadContinuation,
529529
} from "../versionSkew";
530530
import { useAssetUrls } from "../assets/assetUrls";
531-
import {
532-
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
533-
recallableComposerPrompt,
534-
} from "./chat/composerPromptHistory";
531+
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
532+
import { recallableComposerPrompt } from "./chat/composerPromptHistory";
535533

536534
const EMPTY_ACTIVITIES: OrchestrationThreadActivity[] = [];
537535
const EMPTY_QUEUED_MESSAGES: QueuedComposerMessage[] = [];

‎apps/web/src/components/chat/composerPromptHistory.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { describe, expect, it } from "vite-plus/test";
22

3+
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
34
import { buildPlanImplementationPrompt } from "../../proposedPlan";
45
import {
5-
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
66
buildComposerPromptHistoryEntries,
77
recallableComposerPrompt,
88
stepComposerPromptHistory,

‎apps/web/src/components/chat/composerPromptHistory.ts‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
import { collectComposerContextReferences } from "@t3tools/shared/composerContextReferences";
1+
import {
2+
ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
3+
collectComposerContextReferences,
4+
} from "@t3tools/shared/composerContextReferences";
25
import { PLAN_IMPLEMENTATION_PROMPT_PREFIX } from "../../proposedPlan";
36

47
/**
@@ -15,10 +18,6 @@ const REVIEW_COMMENT_BLOCK_PATTERN = /<review_comment\b[^>]*>[\s\S]*?<\/review_c
1518
const TRAILING_LEGACY_CONTEXT =
1619
/\n*<(terminal_context|element_context|preview_annotation)>\n([\s\S]*?)\n<\/\1>\s*$/;
1720

18-
/** Text sent in place of an empty prompt when a message is attachments only. */
19-
export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
20-
"[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
21-
2221
export interface ComposerPromptHistoryMessage {
2322
readonly id: string;
2423
readonly role: string;

‎apps/web/src/components/chat/sendQueuedMessage.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
type AtomCommand,
66
} from "@t3tools/client-runtime/state/runtime";
77
import type { ScopedThreadRef } from "@t3tools/contracts";
8+
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
89
import { serializeLegacyContextMessage } from "@t3tools/shared/composerContextLegacySend";
910
import { applyClaudePromptEffortPrefix } from "@t3tools/shared/model";
1011

@@ -31,7 +32,6 @@ import {
3132
} from "../ChatView.logic";
3233
import { toastManager } from "../ui/toast";
3334
import { fileAttachmentCapabilityBlockReason } from "./composerAttachmentFiles";
34-
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "./composerPromptHistory";
3535

3636
async function run<W, A, E>(command: AtomCommand<W, A, E>, input: W): Promise<A> {
3737
const result = await runAtomCommand(appAtomRegistry, command, input, { reportFailure: false });

‎packages/shared/src/composerContextReferences.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
formatComposerContextHref,
77
formatComposerContextProviderMarker,
88
formatComposerContextReference,
9+
isImageReferencesOnly,
910
parseComposerContextHref,
1011
projectComposerContextForProvider,
1112
replaceComposerContextReferences,
@@ -86,6 +87,17 @@ describe("labels and reference links", () => {
8687
expect(collectComposerContextReferences("[x](https://example.com)")).toEqual([]);
8788
});
8889

90+
it("recognizes a prompt made only of image references", () => {
91+
const image = "![a.png](t3-context://v1/image/ctx_1)";
92+
const file = "[notes.md](t3-context://v1/file/ctx_3)";
93+
expect(isImageReferencesOnly(image)).toBe(true);
94+
expect(isImageReferencesOnly(` ${image}\n![b.png](t3-context://v1/image/ctx_2) `)).toBe(true);
95+
expect(isImageReferencesOnly(`${image} what is this?`)).toBe(false);
96+
expect(isImageReferencesOnly(file)).toBe(false);
97+
expect(isImageReferencesOnly(`${image} ${file}`)).toBe(false);
98+
expect(isImageReferencesOnly(" ")).toBe(false);
99+
});
100+
89101
it("replaces occurrences in place", () => {
90102
const text = "a [x](t3-context://v1/skill/ctx_1) b [y](t3-context://v1/file/ctx_2) c";
91103
expect(replaceComposerContextReferences(text, (o) => `<${o.contextId}>`)).toBe(

‎packages/shared/src/composerContextReferences.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,20 @@ export function collectComposerContextReferences(
9292
return occurrences;
9393
}
9494

95+
/** Text sent in place of an empty prompt when a message is attachments only. */
96+
export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
97+
"[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
98+
99+
/** Whether `text` is image references and nothing else, which is how an image-only prompt reads. */
100+
export function isImageReferencesOnly(text: string): boolean {
101+
const references = collectComposerContextReferences(text);
102+
return (
103+
references.length > 0 &&
104+
references.every((reference) => reference.kind === "image") &&
105+
replaceComposerContextReferences(text, () => "").trim() === ""
106+
);
107+
}
108+
95109
export function replaceComposerContextReferences(
96110
text: string,
97111
replace: (occurrence: ComposerContextReferenceOccurrence) => string,

0 commit comments

Comments
 (0)