Skip to content

Commit f3cf75d

Browse files
jalehmanPatrick-Erichsen
jalehman
authored and
Patrick-Erichsen
committed
openclaw-cd9: filter archived, cron, and heartbeat transcript noise from dreaming corpus
1 parent ff99279 commit f3cf75d

3 files changed

Lines changed: 306 additions & 9 deletions

File tree

‎extensions/memory-core/src/dreaming-phases.test.ts‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1182,6 +1182,157 @@ describe("memory-core dreaming phases", () => {
11821182
expect(corpus).not.toContain("Handled internally.");
11831183
});
11841184

1185+
it("drops archive, cron, and heartbeat chatter from fresh session corpus output", async () => {
1186+
const workspaceDir = await createDreamingWorkspace();
1187+
vi.stubEnv("OPENCLAW_TEST_FAST", "1");
1188+
vi.stubEnv("OPENCLAW_STATE_DIR", path.join(workspaceDir, ".state"));
1189+
const sessionsDir = resolveSessionTranscriptsDirForAgent("main");
1190+
await fs.mkdir(sessionsDir, { recursive: true });
1191+
1192+
await fs.writeFile(
1193+
path.join(sessionsDir, "archived.jsonl.deleted.2026-04-16T18-06-16.529Z"),
1194+
[
1195+
JSON.stringify({
1196+
type: "message",
1197+
message: {
1198+
role: "user",
1199+
timestamp: "2026-04-16T18:01:00.000Z",
1200+
content: "[cron:job-1 Example] Run the nightly sync",
1201+
},
1202+
}),
1203+
JSON.stringify({
1204+
type: "message",
1205+
message: {
1206+
role: "assistant",
1207+
timestamp: "2026-04-16T18:02:00.000Z",
1208+
content: "Running the nightly sync now.",
1209+
},
1210+
}),
1211+
].join("\n") + "\n",
1212+
"utf-8",
1213+
);
1214+
await fs.writeFile(
1215+
path.join(sessionsDir, "ordinary.checkpoint.abc123.jsonl"),
1216+
JSON.stringify({
1217+
type: "message",
1218+
message: {
1219+
role: "user",
1220+
timestamp: "2026-04-16T18:03:00.000Z",
1221+
content: "Checkpoint chatter should stay out.",
1222+
},
1223+
}) + "\n",
1224+
"utf-8",
1225+
);
1226+
await fs.writeFile(
1227+
path.join(sessionsDir, "ordinary.jsonl"),
1228+
[
1229+
JSON.stringify({
1230+
type: "message",
1231+
message: {
1232+
role: "user",
1233+
timestamp: "2026-04-16T18:04:00.000Z",
1234+
content:
1235+
"Read HEARTBEAT.md if it exists (workspace context). Follow it strictly. Do not infer or repeat old tasks from prior chats. If nothing needs attention, reply HEARTBEAT_OK.",
1236+
},
1237+
}),
1238+
JSON.stringify({
1239+
type: "message",
1240+
message: {
1241+
role: "assistant",
1242+
timestamp: "2026-04-16T18:05:00.000Z",
1243+
content: "HEARTBEAT_OK",
1244+
},
1245+
}),
1246+
JSON.stringify({
1247+
type: "message",
1248+
message: {
1249+
role: "user",
1250+
timestamp: "2026-04-16T18:06:00.000Z",
1251+
content: "[cron:job-2 Example] Run the qmd sync",
1252+
},
1253+
}),
1254+
JSON.stringify({
1255+
type: "message",
1256+
message: {
1257+
role: "assistant",
1258+
timestamp: "2026-04-16T18:07:00.000Z",
1259+
content: "Running the qmd sync now.",
1260+
},
1261+
}),
1262+
JSON.stringify({
1263+
type: "message",
1264+
message: {
1265+
role: "user",
1266+
timestamp: "2026-04-16T18:08:00.000Z",
1267+
content: "Document the Ollama provider setup.",
1268+
},
1269+
}),
1270+
JSON.stringify({
1271+
type: "message",
1272+
message: {
1273+
role: "assistant",
1274+
timestamp: "2026-04-16T18:09:00.000Z",
1275+
content: "I documented the Ollama provider setup in the workspace notes.",
1276+
},
1277+
}),
1278+
].join("\n") + "\n",
1279+
"utf-8",
1280+
);
1281+
1282+
const { beforeAgentReply } = createHarness(
1283+
{
1284+
agents: {
1285+
defaults: {
1286+
workspace: workspaceDir,
1287+
},
1288+
list: [{ id: "main", workspace: workspaceDir }],
1289+
},
1290+
plugins: {
1291+
entries: {
1292+
"memory-core": {
1293+
config: {
1294+
dreaming: {
1295+
enabled: true,
1296+
phases: {
1297+
light: {
1298+
enabled: true,
1299+
limit: 20,
1300+
lookbackDays: 7,
1301+
},
1302+
},
1303+
},
1304+
},
1305+
},
1306+
},
1307+
},
1308+
},
1309+
workspaceDir,
1310+
);
1311+
1312+
try {
1313+
await beforeAgentReply(
1314+
{ cleanedBody: "__openclaw_memory_core_light_sleep__" },
1315+
{ trigger: "heartbeat", workspaceDir },
1316+
);
1317+
} finally {
1318+
vi.unstubAllEnvs();
1319+
}
1320+
1321+
const corpus = await fs.readFile(
1322+
path.join(workspaceDir, "memory", ".dreams", "session-corpus", "2026-04-16.txt"),
1323+
"utf-8",
1324+
);
1325+
expect(corpus).toContain("User: Document the Ollama provider setup.");
1326+
expect(corpus).toContain(
1327+
"Assistant: I documented the Ollama provider setup in the workspace notes.",
1328+
);
1329+
expect(corpus).not.toContain("Run the nightly sync");
1330+
expect(corpus).not.toContain("Checkpoint chatter should stay out.");
1331+
expect(corpus).not.toContain("Read HEARTBEAT.md");
1332+
expect(corpus).not.toContain("HEARTBEAT_OK");
1333+
expect(corpus).not.toContain("Run the qmd sync");
1334+
});
1335+
11851336
it("does not reread unchanged dreaming-generated transcripts after checkpointing skip state", async () => {
11861337
const workspaceDir = await createDreamingWorkspace();
11871338
vi.stubEnv("OPENCLAW_TEST_FAST", "1");

‎src/memory-host-sdk/host/session-files.test.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,108 @@ describe("buildSessionEntry", () => {
527527
expect(entry?.lineMap).toEqual([3, 4]);
528528
});
529529

530+
it("drops direct cron prompt chatter and the immediate assistant follow-up", async () => {
531+
const jsonlLines = [
532+
JSON.stringify({
533+
type: "message",
534+
message: {
535+
role: "user",
536+
content: "[cron:job-1 Example] Run the nightly sync",
537+
},
538+
}),
539+
JSON.stringify({
540+
type: "message",
541+
message: {
542+
role: "assistant",
543+
content: "Running the nightly sync now.",
544+
},
545+
}),
546+
JSON.stringify({
547+
type: "message",
548+
message: {
549+
role: "user",
550+
content: "Did the nightly sync actually change anything?",
551+
},
552+
}),
553+
JSON.stringify({
554+
type: "message",
555+
message: {
556+
role: "assistant",
557+
content: "No, everything was already current.",
558+
},
559+
}),
560+
];
561+
const filePath = path.join(tmpDir, "cron-prompt-session.jsonl");
562+
await fs.writeFile(filePath, jsonlLines.join("\n"));
563+
564+
const entry = await buildSessionEntry(filePath);
565+
566+
expect(entry).not.toBeNull();
567+
expect(entry?.content).toBe(
568+
[
569+
"User: Did the nightly sync actually change anything?",
570+
"Assistant: No, everything was already current.",
571+
].join("\n"),
572+
);
573+
expect(entry?.lineMap).toEqual([3, 4]);
574+
});
575+
576+
it("drops heartbeat prompt and ack chatter", async () => {
577+
const jsonlLines = [
578+
JSON.stringify({
579+
type: "message",
580+
message: {
581+
role: "user",
582+
content:
583+
"Read HEARTBEAT.md if it exists (workspace context). Follow it strictly. Do not infer or repeat old tasks from prior chats. If nothing needs attention, reply HEARTBEAT_OK.",
584+
},
585+
}),
586+
JSON.stringify({
587+
type: "message",
588+
message: {
589+
role: "assistant",
590+
content: "HEARTBEAT_OK",
591+
},
592+
}),
593+
JSON.stringify({
594+
type: "message",
595+
message: {
596+
role: "user",
597+
content: "Summarize what changed in the inbox today.",
598+
},
599+
}),
600+
];
601+
const filePath = path.join(tmpDir, "heartbeat-session.jsonl");
602+
await fs.writeFile(filePath, jsonlLines.join("\n"));
603+
604+
const entry = await buildSessionEntry(filePath);
605+
606+
expect(entry).not.toBeNull();
607+
expect(entry?.content).toBe("User: Summarize what changed in the inbox today.");
608+
expect(entry?.lineMap).toEqual([3]);
609+
});
610+
611+
it("skips deleted and checkpoint transcripts for dreaming ingestion", async () => {
612+
const deletedPath = path.join(tmpDir, "ordinary.jsonl.deleted.2026-02-16T22-27-33.000Z");
613+
const checkpointPath = path.join(tmpDir, "ordinary.checkpoint.abc123.jsonl");
614+
const content = JSON.stringify({
615+
type: "message",
616+
message: { role: "user", content: "This should never reach the dreaming corpus." },
617+
});
618+
await fs.writeFile(deletedPath, content);
619+
await fs.writeFile(checkpointPath, content);
620+
621+
const deletedEntry = await buildSessionEntry(deletedPath);
622+
const checkpointEntry = await buildSessionEntry(checkpointPath);
623+
624+
expect(deletedEntry).not.toBeNull();
625+
expect(deletedEntry?.content).toBe("");
626+
expect(deletedEntry?.lineMap).toEqual([]);
627+
expect(checkpointEntry).not.toBeNull();
628+
expect(checkpointEntry?.content).toBe("");
629+
expect(checkpointEntry?.lineMap).toEqual([]);
630+
});
631+
530632
it("strips internal runtime context blocks before flattening session text", async () => {
531633
const jsonlLines = [
532634
JSON.stringify({

‎src/memory-host-sdk/host/session-files.ts‎

Lines changed: 53 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
import fs from "node:fs/promises";
22
import path from "node:path";
33
import { stripInternalRuntimeContext } from "../../agents/internal-runtime-context.js";
4+
import { isHeartbeatUserMessage } from "../../auto-reply/heartbeat-filter.js";
5+
import { HEARTBEAT_PROMPT } from "../../auto-reply/heartbeat.js";
46
import { stripInboundMetadata } from "../../auto-reply/reply/strip-inbound-meta.js";
57
import { isSilentReplyPayloadText } from "../../auto-reply/tokens.js";
6-
import { isUsageCountedSessionTranscriptFileName } from "../../config/sessions/artifacts.js";
8+
import {
9+
isSessionArchiveArtifactName,
10+
isUsageCountedSessionTranscriptFileName,
11+
} from "../../config/sessions/artifacts.js";
712
import { resolveSessionTranscriptsDirForAgent } from "../../config/sessions/paths.js";
813
import { loadSessionStore } from "../../config/sessions/store-load.js";
914
import { isExecCompletionEvent } from "../../infra/heartbeat-events-filter.js";
@@ -19,6 +24,7 @@ const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-";
1924
// toxic line. Wrapped continuation lines still map back to the same JSONL line.
2025
// This limit applies to content only; the role label adds up to 11 chars.
2126
const SESSION_EXPORT_CONTENT_WRAP_CHARS = 800;
27+
const DIRECT_CRON_PROMPT_RE = /^\[cron:[^\]]+\]\s*/;
2228

2329
export type SessionFileEntry = {
2430
path: string;
@@ -49,6 +55,15 @@ export type SessionTranscriptClassification = {
4955
cronRunTranscriptPaths: ReadonlySet<string>;
5056
};
5157

58+
function isCheckpointTranscriptFileName(fileName: string): boolean {
59+
return fileName.endsWith(".jsonl") && fileName.includes(".checkpoint.");
60+
}
61+
62+
function shouldSkipTranscriptFileForDreaming(absPath: string): boolean {
63+
const fileName = path.basename(absPath);
64+
return isSessionArchiveArtifactName(fileName) || isCheckpointTranscriptFileName(fileName);
65+
}
66+
5267
function isDreamingNarrativeBootstrapRecord(record: unknown): boolean {
5368
if (!record || typeof record !== "object" || Array.isArray(record)) {
5469
return false;
@@ -336,7 +351,18 @@ function isGeneratedSystemWrapperMessage(text: string, role: "user" | "assistant
336351
return GENERATED_SYSTEM_MESSAGE_RE.test(text);
337352
}
338353

339-
function shouldSkipGeneratedSystemFollowup(rawText: string, role: "user" | "assistant"): boolean {
354+
function isGeneratedCronPromptMessage(text: string, role: "user" | "assistant"): boolean {
355+
if (role !== "user") {
356+
return false;
357+
}
358+
return DIRECT_CRON_PROMPT_RE.test(text);
359+
}
360+
361+
function isGeneratedHeartbeatPromptMessage(text: string, role: "user" | "assistant"): boolean {
362+
return role === "user" && isHeartbeatUserMessage({ role, content: text }, HEARTBEAT_PROMPT);
363+
}
364+
365+
function shouldSkipGeneratedPrompt(rawText: string, role: "user" | "assistant"): boolean {
340366
const strippedInbound = stripInboundMetadataForUserRole(rawText, role);
341367
const strippedInternal = stripInternalRuntimeContext(strippedInbound);
342368
const normalized = normalizeSessionText(strippedInternal);
@@ -345,6 +371,8 @@ function shouldSkipGeneratedSystemFollowup(rawText: string, role: "user" | "assi
345371
}
346372
return (
347373
isGeneratedSystemWrapperMessage(normalized, role) ||
374+
isGeneratedCronPromptMessage(normalized, role) ||
375+
isGeneratedHeartbeatPromptMessage(normalized, role) ||
348376
isExecCompletionEvent(normalized.replace(GENERATED_SYSTEM_MESSAGE_RE, "").trim())
349377
);
350378
}
@@ -359,6 +387,12 @@ function sanitizeSessionText(text: string, role: "user" | "assistant"): string |
359387
if (isGeneratedSystemWrapperMessage(normalized, role)) {
360388
return null;
361389
}
390+
if (isGeneratedCronPromptMessage(normalized, role)) {
391+
return null;
392+
}
393+
if (isGeneratedHeartbeatPromptMessage(normalized, role)) {
394+
return null;
395+
}
362396
if (isSilentReplyPayloadText(normalized)) {
363397
return null;
364398
}
@@ -408,6 +442,18 @@ export async function buildSessionEntry(
408442
): Promise<SessionFileEntry | null> {
409443
try {
410444
const stat = await fs.stat(absPath);
445+
if (shouldSkipTranscriptFileForDreaming(absPath)) {
446+
return {
447+
path: sessionPathForFile(absPath),
448+
absPath,
449+
mtimeMs: stat.mtimeMs,
450+
size: stat.size,
451+
hash: hashText("\n\n"),
452+
content: "",
453+
lineMap: [],
454+
messageTimestampsMs: [],
455+
};
456+
}
411457
const raw = await fs.readFile(absPath, "utf-8");
412458
const lines = raw.split("\n");
413459
const collected: string[] = [];
@@ -417,7 +463,7 @@ export async function buildSessionEntry(
417463
opts.generatedByDreamingNarrative ?? isDreamingNarrativeTranscriptFromSessionStore(absPath);
418464
const generatedByCronRun =
419465
opts.generatedByCronRun ?? isCronRunTranscriptFromSessionStore(absPath);
420-
let skippingGeneratedSystemFollowup = false;
466+
let skippingGeneratedFollowup = false;
421467
for (let jsonlIdx = 0; jsonlIdx < lines.length; jsonlIdx++) {
422468
const line = lines[jsonlIdx];
423469
if (!line.trim()) {
@@ -455,14 +501,12 @@ export async function buildSessionEntry(
455501
const text = sanitizeSessionText(rawText, message.role);
456502
if (message.role === "user") {
457503
if (!text) {
458-
skippingGeneratedSystemFollowup = shouldSkipGeneratedSystemFollowup(
459-
rawText,
460-
message.role,
461-
);
504+
skippingGeneratedFollowup = shouldSkipGeneratedPrompt(rawText, message.role);
462505
continue;
463506
}
464-
skippingGeneratedSystemFollowup = false;
465-
} else if (skippingGeneratedSystemFollowup) {
507+
skippingGeneratedFollowup = false;
508+
} else if (skippingGeneratedFollowup) {
509+
skippingGeneratedFollowup = false;
466510
continue;
467511
}
468512
if (!text) {

0 commit comments

Comments
 (0)