Skip to content

Commit 0c4ec29

Browse files
Lucenx9Lucenx9
committed
fix(gateway): persist offloaded image refs in transcript for iOS share/node ingress
1 parent 71ab341 commit 0c4ec29

4 files changed

Lines changed: 154 additions & 7 deletions

File tree

‎src/gateway/server-methods/chat.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -698,12 +698,14 @@ function canInjectSystemProvenance(client: GatewayRequestHandlerOptions["client"
698698
return scopes.includes(ADMIN_SCOPE);
699699
}
700700

701-
async function persistChatSendImages(params: {
701+
export async function persistChatSendImages(params: {
702702
images: ChatImageContent[];
703703
imageOrder: PromptImageOrderEntry[];
704704
offloadedRefs: OffloadedRef[];
705705
client: GatewayRequestHandlerOptions["client"];
706-
logGateway: GatewayRequestContext["logGateway"];
706+
// Only `.warn` is consumed; widen so callers with a narrower logger
707+
// (e.g. server-node-events.ts) can reuse this helper without coupling.
708+
logGateway: Pick<GatewayRequestContext["logGateway"], "warn">;
707709
}): Promise<SavedMedia[]> {
708710
if (
709711
(params.images.length === 0 && params.offloadedRefs.length === 0) ||
@@ -949,7 +951,7 @@ function extractTranscriptUserText(content: unknown): string | undefined {
949951
return textBlocks.length > 0 ? textBlocks.join("") : undefined;
950952
}
951953

952-
async function rewriteChatSendUserTurnMediaPaths(params: {
954+
export async function rewriteChatSendUserTurnMediaPaths(params: {
953955
transcriptPath: string;
954956
sessionKey: string;
955957
message: string;
@@ -1234,7 +1236,7 @@ export function enforceChatHistoryFinalBudget(params: { messages: unknown[]; max
12341236
return { messages: [], placeholderCount: 0 };
12351237
}
12361238

1237-
function resolveTranscriptPath(params: {
1239+
export function resolveTranscriptPath(params: {
12381240
sessionId: string;
12391241
storePath: string | undefined;
12401242
sessionFile?: string;

‎src/gateway/server-node-events.runtime.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ export { deleteMediaBuffer } from "../media/store.js";
1616
export { normalizeMainKey, scopedHeartbeatWakeOptions } from "../routing/session-key.js";
1717
export { defaultRuntime } from "../runtime.js";
1818
export { parseMessageWithAttachments, resolveChatAttachmentMaxBytes } from "./chat-attachments.js";
19+
export {
20+
persistChatSendImages,
21+
resolveTranscriptPath,
22+
rewriteChatSendUserTurnMediaPaths,
23+
} from "./server-methods/chat.js";
1924
export { normalizeRpcAttachmentsToChatAttachments } from "./server-methods/attachment-normalize.js";
2025
export {
2126
loadSessionEntry,

‎src/gateway/server-node-events.test.ts‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { beforeEach, describe, expect, it, vi } from "vitest";
22
import type { OpenClawConfig } from "../config/config.js";
3+
import type { SavedMedia } from "../media/store.js";
34
import type { loadSessionEntry as loadSessionEntryType } from "./session-utils.js";
45

56
const buildSessionLookup = (
@@ -90,6 +91,7 @@ const runtimeMocks = vi.hoisted(() => ({
9091
normalizeMainKey: vi.fn((key?: string | null) => key?.trim() || "agent:main:main"),
9192
normalizeRpcAttachmentsToChatAttachments: vi.fn((attachments?: unknown[]) => attachments ?? []),
9293
parseMessageWithAttachments: parseMessageWithAttachmentsMock,
94+
persistChatSendImages: vi.fn(async (): Promise<SavedMedia[]> => []),
9395
registerApnsRegistration: registerApnsRegistrationMock,
9496
requestHeartbeatNow: vi.fn(),
9597
resolveChatAttachmentMaxBytes: vi.fn(() => 20 * 1024 * 1024),
@@ -123,6 +125,8 @@ const runtimeMocks = vi.hoisted(() => ({
123125
model: entry?.model ?? "default-model",
124126
}),
125127
),
128+
resolveTranscriptPath: vi.fn(() => null as string | null),
129+
rewriteChatSendUserTurnMediaPaths: vi.fn(async () => {}),
126130
sanitizeInboundSystemTags: sanitizeInboundSystemTagsMock,
127131
scopedHeartbeatWakeOptions: vi.fn((sessionKey?: string, opts?: { reason: string }) => {
128132
const wakeOptions = { reason: opts?.reason };
@@ -1015,6 +1019,85 @@ describe("agent request events", () => {
10151019
expect(warn).toHaveBeenCalledWith(expect.stringMatching(/attachment parse failed.*non-image/i));
10161020
});
10171021

1022+
it("persists offloaded image refs in transcript user turn for agent.request", async () => {
1023+
// Regression for #60339: iOS share / node ingress dispatches via
1024+
// agentCommandFromIngress and previously dropped parsed.offloadedRefs,
1025+
// leaving the transcript user turn without MediaPath/MediaPaths even
1026+
// though the media was already on disk. The fix mirrors chat.send's
1027+
// persistence + rewrite contract.
1028+
const persistChatSendImagesMock = runtimeMocks.persistChatSendImages;
1029+
const rewriteChatSendUserTurnMediaPathsMock = runtimeMocks.rewriteChatSendUserTurnMediaPaths;
1030+
const resolveTranscriptPathMock = runtimeMocks.resolveTranscriptPath;
1031+
persistChatSendImagesMock.mockClear();
1032+
rewriteChatSendUserTurnMediaPathsMock.mockClear();
1033+
resolveTranscriptPathMock.mockClear();
1034+
1035+
parseMessageWithAttachmentsMock.mockResolvedValueOnce({
1036+
message: "describe\n[media attached: media://inbound/img-1]",
1037+
images: [],
1038+
imageOrder: ["offloaded"],
1039+
offloadedRefs: [
1040+
{
1041+
id: "img-1",
1042+
path: "/media/inbound/img-1.jpg",
1043+
mimeType: "image/jpeg",
1044+
mediaRef: "media://inbound/img-1",
1045+
sizeBytes: 2_000_000,
1046+
label: "photo.jpg",
1047+
},
1048+
],
1049+
});
1050+
const savedImage: SavedMedia = {
1051+
id: "img-1",
1052+
path: "/media/inbound/img-1.jpg",
1053+
size: 0,
1054+
contentType: "image/jpeg",
1055+
};
1056+
persistChatSendImagesMock.mockResolvedValueOnce([savedImage]);
1057+
resolveTranscriptPathMock.mockReturnValueOnce("/sessions/sid-x/session.jsonl");
1058+
1059+
const ctx = buildCtx();
1060+
await handleNodeEvent(ctx, "node-ios-share", {
1061+
event: "agent.request",
1062+
payloadJSON: JSON.stringify({
1063+
sessionKey: "agent:main:main",
1064+
message: "describe",
1065+
attachments: [
1066+
{
1067+
type: "image",
1068+
mimeType: "image/jpeg",
1069+
fileName: "photo.jpg",
1070+
content: "BIG_BASE64_PAYLOAD",
1071+
},
1072+
],
1073+
}),
1074+
});
1075+
1076+
expect(persistChatSendImagesMock).toHaveBeenCalledTimes(1);
1077+
expect(persistChatSendImagesMock).toHaveBeenCalledWith(
1078+
expect.objectContaining({
1079+
images: [],
1080+
imageOrder: ["offloaded"],
1081+
offloadedRefs: [expect.objectContaining({ id: "img-1" })],
1082+
client: null,
1083+
}),
1084+
);
1085+
1086+
// Wait for the fire-and-forget agentCommandFromIngress chain's
1087+
// .finally rewrite to settle.
1088+
await vi.waitFor(() => {
1089+
expect(rewriteChatSendUserTurnMediaPathsMock).toHaveBeenCalledTimes(1);
1090+
});
1091+
expect(rewriteChatSendUserTurnMediaPathsMock).toHaveBeenCalledWith(
1092+
expect.objectContaining({
1093+
transcriptPath: "/sessions/sid-x/session.jsonl",
1094+
sessionKey: "agent:main:main",
1095+
message: "describe\n[media attached: media://inbound/img-1]",
1096+
savedImages: [savedImage],
1097+
}),
1098+
);
1099+
});
1100+
10181101
beforeEach(() => {
10191102
resetNodeEventDeduplicationForTests();
10201103
updatePairedDeviceMetadataMock.mockClear();

‎src/gateway/server-node-events.ts‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import {
1212
normalizeLowercaseStringOrEmpty,
1313
normalizeOptionalString,
1414
} from "../shared/string-coerce.js";
15+
import type { OffloadedRef } from "./chat-attachments.js";
1516
import type { NodeEvent, NodeEventContext } from "./server-node-events-types.js";
1617
import {
1718
agentCommandFromIngress,
@@ -30,13 +31,16 @@ import {
3031
normalizeMainKey,
3132
normalizeRpcAttachmentsToChatAttachments,
3233
parseMessageWithAttachments,
34+
persistChatSendImages,
3335
registerApnsRegistration,
3436
requestHeartbeatNow,
3537
resolveChatAttachmentMaxBytes,
3638
resolveGatewayModelSupportsImages,
3739
resolveOutboundTarget,
3840
resolveSessionAgentId,
3941
resolveSessionModelRef,
42+
resolveTranscriptPath,
43+
rewriteChatSendUserTurnMediaPaths,
4044
sanitizeInboundSystemTags,
4145
scopedHeartbeatWakeOptions,
4246
updateSessionStore,
@@ -470,6 +474,7 @@ export const handleNodeEvent = async (
470474
);
471475
let images: Array<{ type: "image"; data: string; mimeType: string }> = [];
472476
let imageOrder: PromptImageOrderEntry[] = [];
477+
let offloadedImageRefs: OffloadedRef[] = [];
473478
if (!message && normalizedAttachments.length === 0) {
474479
return undefined;
475480
}
@@ -497,6 +502,7 @@ export const handleNodeEvent = async (
497502
message = parsed.message.trim();
498503
images = parsed.images;
499504
imageOrder = parsed.imageOrder;
505+
offloadedImageRefs = parsed.offloadedRefs ?? [];
500506
if (message.length > 20_000) {
501507
ctx.logGateway.warn(
502508
`agent.request message exceeds limit after attachment parsing (length=${message.length})`,
@@ -576,6 +582,19 @@ export const handleNodeEvent = async (
576582
);
577583
}
578584

585+
// Mirror chat.send's media persistence contract: convert offloaded
586+
// image refs (and inline images) into SavedMedia entries so the user
587+
// turn in the session transcript carries MediaPath/MediaPaths. Without
588+
// this, history replay loses the offloaded attachment link even though
589+
// the media is already on disk.
590+
const persistedImagesPromise = persistChatSendImages({
591+
images,
592+
imageOrder,
593+
offloadedRefs: offloadedImageRefs,
594+
client: null,
595+
logGateway: ctx.logGateway,
596+
});
597+
579598
void agentCommandFromIngress(
580599
{
581600
runId: sessionId,
@@ -596,9 +615,47 @@ export const handleNodeEvent = async (
596615
},
597616
defaultRuntime,
598617
ctx.deps,
599-
).catch((err) => {
600-
ctx.logGateway.warn(`agent failed node=${nodeId}: ${formatForLog(err)}`);
601-
});
618+
)
619+
.catch((err) => {
620+
ctx.logGateway.warn(`agent failed node=${nodeId}: ${formatForLog(err)}`);
621+
})
622+
.finally(async () => {
623+
// Rewrite the user-turn entry once the agent run has completed so
624+
// the transcript reflects the persisted media. Mirrors chat.send's
625+
// rewriteChatSendUserTurnMediaPaths flow; safe no-op when no media
626+
// was persisted or the target entry was never written.
627+
try {
628+
const savedImages = await persistedImagesPromise;
629+
if (savedImages.length === 0) {
630+
return;
631+
}
632+
const { storePath: latestStorePath, entry: latestEntry } =
633+
loadSessionEntry(canonicalKey);
634+
const resolvedSessionId = latestEntry?.sessionId ?? sessionId;
635+
if (!resolvedSessionId) {
636+
return;
637+
}
638+
const transcriptPath = resolveTranscriptPath({
639+
sessionId: resolvedSessionId,
640+
storePath: latestStorePath,
641+
sessionFile: latestEntry?.sessionFile ?? entry?.sessionFile,
642+
agentId: resolveSessionAgentId({ sessionKey: canonicalKey, config: cfg }),
643+
});
644+
if (!transcriptPath) {
645+
return;
646+
}
647+
await rewriteChatSendUserTurnMediaPaths({
648+
transcriptPath,
649+
sessionKey: canonicalKey,
650+
message,
651+
savedImages,
652+
});
653+
} catch (err) {
654+
ctx.logGateway.warn(
655+
`agent.request transcript media rewrite failed node=${nodeId}: ${formatForLog(err)}`,
656+
);
657+
}
658+
});
602659
return undefined;
603660
}
604661
case "notifications.changed": {

0 commit comments

Comments
 (0)