Skip to content

Commit ae57eb6

Browse files
vincentkocvincentkoc
authored
fix(gateway): reduce session-store clone memory growth
## Summary - Addresses the remaining Gateway RSS/session-accumulation path tracked by #54155. - Narrows the fix to the structuredClone/session-store cache memory growth described in #45438. - Preserves prior report context from #57699, #62717, #66886, #69977, and #70717 as validation evidence. ## Validation - pnpm -s vitest run src/config/sessions/store.pruning.test.ts src/config/sessions/store.pruning.integration.test.ts src/gateway/sessions-resolve-store.test.ts - pnpm check:changed ## Credit Thanks @the-lobsternaut for #54155 and @markus-lassfolk plus the #45438 commenters for isolating the structuredClone/native-memory behavior. ProjectClownfish replacement details: - Cluster: ghcrawl-156648-autonomous-smoke - Source PRs: none - Credit: Credit #54155 reporter @the-lobsternaut for the multi-day Gateway RSS/session-accumulation report.; Credit #45438 reporter @markus-lassfolk and commenters for isolating the structuredClone/session-store native-memory path.; Preserve prior closed-report context from #57699, #62717, #66886, #69977, and #70717 in the PR body as reproduction evidence, not as new close targets. - Validation: pnpm -s vitest run src/config/sessions/store.pruning.test.ts src/config/sessions/store.pruning.integration.test.ts src/gateway/sessions-resolve-store.test.ts; pnpm check:changed
1 parent 07ca99d commit ae57eb6

5 files changed

Lines changed: 145 additions & 13 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ Docs: https://docs.openclaw.ai
199199

200200
- CLI/channel-setup: auto-skip the redundant "Install \<plugin\>?" confirmation when only one install source (npm or local) exists, show `download from <npm-spec>` hints for installable catalog channels in the picker, and suppress misleading npm hints for already-bundled channels. Fixes #73419. Thanks @sliverp.
201201
- BlueBubbles: tighten DM-vs-group routing across the outbound session route (`chat_guid:iMessage;-;...` DMs no longer classified as groups), reaction handling (drop group reactions that arrive without any chat identifier instead of synthesizing a `"group"` literal peerId), inbound `chatGuid` fallback (no longer fall back to the sender's DM chatGuid when resolving a group whose webhook omits chatGuid+chatId+chatIdentifier), and short message id resolution (carry caller chat context so a numeric short id reused after a long group conversation cannot silently resolve to a message in a different chat, with the same cross-chat guard applied to full GUIDs so retries cannot bypass it). Thanks @zqchris.
202+
- Gateway/sessions: clone cached session stores through the persisted JSON shape instead of `structuredClone`, reducing native-memory growth on the remaining #54155 Gateway RSS/session-accumulation path while keeping #54155 as the broader tracker and carrying forward the #45438 session-cache hypothesis. Thanks @vincentkoc and the #45438 reporters/commenters.
202203
- Agents/approvals: fail restart-interrupted sessions whose transcript tail is still `approval-pending` instead of replaying stale exec approval IDs into the new Gateway process after restart. Fixes #65486. Thanks @mjmai20682068-create.
203204
- CLI/Gateway: use method-specific least-privilege scopes for classified CLI Gateway calls while preserving legacy broad scopes for unclassified plugin methods, so read-only commands no longer create admin/write/pairing scope-upgrade prompts. Fixes #68634. Thanks @nightmusher.
204205
- Gateway/sessions: align `chat.history` and `sessions.list` thinking defaults with owning-agent and catalog-aware resolution so Control UI session defaults match backend runtime state. (#63418) Thanks @jpreagan.

‎src/config/sessions.cache.test.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import fs from "node:fs";
22
import path from "node:path";
33
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
44
import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js";
5+
import { readSessionStoreCache, writeSessionStoreCache } from "./sessions/store-cache.js";
56
import {
67
clearSessionStoreCacheForTest,
78
loadSessionStore,
@@ -107,6 +108,121 @@ describe("Session Store Cache", () => {
107108
expect(loaded2["session:1"].skillsSnapshot?.skills?.[0]?.name).toBe("alpha");
108109
});
109110

111+
it("does not cache pre-migration or pre-normalization disk JSON", () => {
112+
fs.writeFileSync(
113+
storePath,
114+
JSON.stringify({
115+
"session:1": {
116+
sessionId: "id-1",
117+
updatedAt: Date.now(),
118+
provider: "telegram",
119+
room: "room-1",
120+
modelProvider: " openai ",
121+
model: " gpt-5.4 ",
122+
},
123+
}),
124+
);
125+
126+
const loaded1 = loadSessionStore(storePath);
127+
const entry1 = loaded1["session:1"] as SessionEntry & { provider?: string; room?: string };
128+
expect(entry1.channel).toBe("telegram");
129+
expect(entry1.groupChannel).toBe("room-1");
130+
expect(entry1.provider).toBeUndefined();
131+
expect(entry1.room).toBeUndefined();
132+
expect(entry1.modelProvider).toBe("openai");
133+
expect(entry1.model).toBe("gpt-5.4");
134+
135+
const loaded2 = loadSessionStore(storePath);
136+
const entry2 = loaded2["session:1"] as SessionEntry & { provider?: string; room?: string };
137+
expect(entry2.channel).toBe("telegram");
138+
expect(entry2.groupChannel).toBe("room-1");
139+
expect(entry2.provider).toBeUndefined();
140+
expect(entry2.room).toBeUndefined();
141+
expect(entry2.modelProvider).toBe("openai");
142+
expect(entry2.model).toBe("gpt-5.4");
143+
});
144+
145+
it("isolates cached session stores without structuredClone", async () => {
146+
const structuredCloneSpy = vi.spyOn(globalThis, "structuredClone");
147+
const testStore = createSingleSessionStore(
148+
createSessionEntry({
149+
origin: { provider: "openai" },
150+
skillsSnapshot: {
151+
prompt: "skills",
152+
skills: [{ name: "alpha" }],
153+
},
154+
}),
155+
);
156+
157+
await saveSessionStore(storePath, testStore);
158+
159+
const loaded1 = loadSessionStore(storePath);
160+
loaded1["session:1"].origin = { provider: "mutated" };
161+
if (loaded1["session:1"].skillsSnapshot?.skills?.length) {
162+
loaded1["session:1"].skillsSnapshot.skills[0].name = "mutated";
163+
}
164+
165+
const loaded2 = loadSessionStore(storePath);
166+
expect(loaded2["session:1"].origin?.provider).toBe("openai");
167+
expect(loaded2["session:1"].skillsSnapshot?.skills?.[0]?.name).toBe("alpha");
168+
expect(structuredCloneSpy).not.toHaveBeenCalled();
169+
170+
structuredCloneSpy.mockRestore();
171+
});
172+
173+
it("does not parse serialized stores when writing the cache", () => {
174+
const testStore = createSingleSessionStore(
175+
createSessionEntry({
176+
origin: { provider: "openai" },
177+
}),
178+
);
179+
const serialized = JSON.stringify(testStore);
180+
const parseSpy = vi.spyOn(JSON, "parse");
181+
182+
writeSessionStoreCache({ storePath, store: testStore, serialized });
183+
184+
expect(parseSpy).not.toHaveBeenCalled();
185+
186+
testStore["session:1"].origin = { provider: "mutated" };
187+
const cached = readSessionStoreCache({ storePath });
188+
189+
expect(cached?.["session:1"].origin?.provider).toBe("openai");
190+
expect(parseSpy).toHaveBeenCalledTimes(1);
191+
192+
parseSpy.mockRestore();
193+
});
194+
195+
it("clones disk-loaded stores from the raw serialized JSON", () => {
196+
const testStore = createSingleSessionStore(
197+
createSessionEntry({
198+
origin: { provider: "openai" },
199+
skillsSnapshot: {
200+
prompt: "skills",
201+
skills: [{ name: "alpha" }],
202+
},
203+
}),
204+
);
205+
const serialized = JSON.stringify(testStore);
206+
fs.writeFileSync(storePath, serialized);
207+
208+
const stringifySpy = vi.spyOn(JSON, "stringify");
209+
const loaded = loadSessionStore(storePath, { skipCache: true });
210+
211+
expect(loaded).toEqual(testStore);
212+
expect(stringifySpy).not.toHaveBeenCalled();
213+
214+
loaded["session:1"].origin = { provider: "mutated" };
215+
if (loaded["session:1"].skillsSnapshot?.skills?.length) {
216+
loaded["session:1"].skillsSnapshot.skills[0].name = "mutated";
217+
}
218+
219+
const reloaded = loadSessionStore(storePath, { skipCache: true });
220+
expect(reloaded["session:1"].origin?.provider).toBe("openai");
221+
expect(reloaded["session:1"].skillsSnapshot?.skills?.[0]?.name).toBe("alpha");
222+
223+
stringifySpy.mockRestore();
224+
});
225+
110226
it("should refresh cache when store file changes on disk", async () => {
111227
const testStore = createSingleSessionStore();
112228

‎src/config/sessions/store-cache.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ const SESSION_STORE_CACHE = createExpiringMapCache<string, SessionStoreCacheEntr
1515
});
1616
const SESSION_STORE_SERIALIZED_CACHE = new Map<string, string>();
1717

18+
export function cloneSessionStoreRecord(
19+
store: Record<string, SessionEntry>,
20+
serialized?: string,
21+
): Record<string, SessionEntry> {
22+
return JSON.parse(serialized ?? JSON.stringify(store)) as Record<string, SessionEntry>;
23+
}
24+
1825
export function getSessionStoreTtl(): number {
1926
return resolveCacheTtlMs({
2027
envValue: process.env.OPENCLAW_SESSION_CACHE_TTL_MS,
@@ -65,7 +72,7 @@ export function readSessionStoreCache(params: {
6572
invalidateSessionStoreCache(params.storePath);
6673
return null;
6774
}
68-
return structuredClone(cached.store);
75+
return cloneSessionStoreRecord(cached.store, cached.serialized);
6976
}
7077

7178
export function writeSessionStoreCache(params: {
@@ -76,7 +83,7 @@ export function writeSessionStoreCache(params: {
7683
serialized?: string;
7784
}): void {
7885
SESSION_STORE_CACHE.set(params.storePath, {
79-
store: structuredClone(params.store),
86+
store: params.serialized === undefined ? cloneSessionStoreRecord(params.store) : params.store,
8087
mtimeMs: params.mtimeMs,
8188
sizeBytes: params.sizeBytes,
8289
serialized: params.serialized,

‎src/config/sessions/store-load.ts‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { createSubsystemLogger } from "../../logging/subsystem.js";
33
import { normalizeSessionDeliveryFields } from "../../utils/delivery-context.shared.js";
44
import { getFileStatSnapshot } from "../cache-utils.js";
55
import {
6+
cloneSessionStoreRecord,
67
isSessionStoreCacheEnabled,
78
readSessionStoreCache,
89
setSerializedSessionStore,
@@ -63,16 +64,19 @@ function normalizeSessionEntryDelivery(entry: SessionEntry): SessionEntry {
6364
};
6465
}
6566

66-
export function normalizeSessionStore(store: Record<string, SessionEntry>): void {
67+
export function normalizeSessionStore(store: Record<string, SessionEntry>): boolean {
68+
let changed = false;
6769
for (const [key, entry] of Object.entries(store)) {
6870
if (!entry) {
6971
continue;
7072
}
7173
const normalized = normalizeSessionEntryDelivery(normalizeSessionRuntimeModelFields(entry));
7274
if (normalized !== entry) {
7375
store[key] = normalized;
76+
changed = true;
7477
}
7578
}
79+
return changed;
7680
}
7781

7882
export function loadSessionStore(
@@ -122,14 +126,11 @@ export function loadSessionStore(
122126
}
123127
}
124128

125-
if (serializedFromDisk !== undefined) {
126-
setSerializedSessionStore(storePath, serializedFromDisk);
127-
} else {
128-
setSerializedSessionStore(storePath, undefined);
129+
const migrated = applySessionStoreMigrations(store);
130+
const normalized = normalizeSessionStore(store);
131+
if (migrated || normalized) {
132+
serializedFromDisk = undefined;
129133
}
130-
131-
applySessionStoreMigrations(store);
132-
normalizeSessionStore(store);
133134
const maintenance = opts.maintenanceConfig ?? resolveMaintenanceConfig();
134135
const beforeCount = Object.keys(store).length;
135136
if (maintenance.mode === "enforce" && beforeCount > maintenance.maxEntries) {
@@ -144,7 +145,6 @@ export function loadSessionStore(
144145
const afterCount = Object.keys(store).length;
145146
if (pruned > 0 || capped > 0) {
146147
serializedFromDisk = undefined;
147-
setSerializedSessionStore(storePath, undefined);
148148
log.info("applied load-time maintenance to oversized session store", {
149149
storePath,
150150
before: beforeCount,
@@ -156,6 +156,8 @@ export function loadSessionStore(
156156
}
157157
}
158158

159+
setSerializedSessionStore(storePath, serializedFromDisk);
160+
159161
if (!opts.skipCache && isSessionStoreCacheEnabled()) {
160162
writeSessionStoreCache({
161163
storePath,
@@ -166,5 +168,5 @@ export function loadSessionStore(
166168
});
167169
}
168170

169-
return opts.clone === false ? store : structuredClone(store);
171+
return opts.clone === false ? store : cloneSessionStoreRecord(store, serializedFromDisk);
170172
}

‎src/config/sessions/store-migrations.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { SessionEntry } from "./types.js";
22

3-
export function applySessionStoreMigrations(store: Record<string, SessionEntry>): void {
3+
export function applySessionStoreMigrations(store: Record<string, SessionEntry>): boolean {
4+
let changed = false;
45
// Best-effort migration: message provider → channel naming.
56
for (const entry of Object.values(store)) {
67
if (!entry || typeof entry !== "object") {
@@ -10,18 +11,23 @@ export function applySessionStoreMigrations(store: Record<string, SessionEntry>)
1011
if (typeof rec.channel !== "string" && typeof rec.provider === "string") {
1112
rec.channel = rec.provider;
1213
delete rec.provider;
14+
changed = true;
1315
}
1416
if (typeof rec.lastChannel !== "string" && typeof rec.lastProvider === "string") {
1517
rec.lastChannel = rec.lastProvider;
1618
delete rec.lastProvider;
19+
changed = true;
1720
}
1821

1922
// Best-effort migration: legacy `room` field → `groupChannel` (keep value, prune old key).
2023
if (typeof rec.groupChannel !== "string" && typeof rec.room === "string") {
2124
rec.groupChannel = rec.room;
2225
delete rec.room;
26+
changed = true;
2327
} else if ("room" in rec) {
2428
delete rec.room;
29+
changed = true;
2530
}
2631
}
32+
return changed;
2733
}

0 commit comments

Comments
 (0)