Skip to content

Commit 1e7676b

Browse files
chenyangjun-xychenyangjun-xy
committed
fix(telegram): strip Markdown formatting in conversation-context dedup
1 parent 3fd5251 commit 1e7676b

7 files changed

Lines changed: 552 additions & 5 deletions

‎extensions/telegram/src/bot-handlers.runtime.gateway-followup.test.ts‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
// P1 Real Telegram Follow-Up Turn Gateway Simulation
2+
// Exercises the exact production dedup pipeline from
3+
// bot-handlers.runtime.ts:1379-1412 — the same code path a real
4+
// OpenClaw Gateway runs when it builds prompt context for a Telegram
5+
// follow-up turn.
6+
import { describe, expect, it } from "vitest";
7+
import { resolvePromptContextTextDedupeKey } from "./bot-handlers.runtime.js";
8+
9+
const TS = 1_751_870_400_000;
10+
const TS_USER = 1_751_869_500_000;
11+
12+
// ─────────────────────────────────────────────────────────────────
13+
// Production dedup pipeline (bot-handlers.runtime.ts:1386-1397)
14+
//
15+
// The real gateway:
16+
// 1. Fetches session transcript messages (Markdown-formatted)
17+
// 2. Fetches cache messages (plain text, HTML stripped by Bot API)
18+
// 3. Builds Set<dedupKey> from cache messages
19+
// 4. Filters session messages, keeping those NOT in the cache set
20+
// 5. Merges + sorts chronologically
21+
// ─────────────────────────────────────────────────────────────────
22+
function runProductionDedupPipeline(
23+
sessionPromptMessages: { role: string; body: string; timestamp_ms: number }[],
24+
cachePromptMessages: { role: string; body: string; timestamp_ms: number }[],
25+
): { role: string; body: string; timestamp_ms: number }[] {
26+
const cacheTextKeys = new Set(
27+
cachePromptMessages
28+
.map((m) => resolvePromptContextTextDedupeKey(m))
29+
.filter((k) => k !== undefined),
30+
);
31+
const sessionOnlyPromptMessages = sessionPromptMessages.filter((m) => {
32+
const key = resolvePromptContextTextDedupeKey(m);
33+
return key === undefined || !cacheTextKeys.has(key);
34+
});
35+
return [...sessionOnlyPromptMessages, ...cachePromptMessages].toSorted(
36+
(left, right) => (left.timestamp_ms ?? 0) - (right.timestamp_ms ?? 0),
37+
);
38+
}
39+
40+
describe("[P1] Real Telegram follow-up turn — gateway simulation", () => {
41+
42+
it("dedup eliminates duplicate assistant row after Markdown normalization", () => {
43+
// Two-turn conversation:
44+
// User: "Write a function"
45+
// Asst: "**Here** is a function:\n\n```typescript\nfunction add(...){...}\n```"
46+
//
47+
// Session transcript keeps Markdown formatting (body).
48+
// Cache strips HTML tags via Bot API — plain text only.
49+
const sessionMessages = [
50+
{ role: "user", body: "Write a function", timestamp_ms: TS_USER },
51+
{
52+
role: "assistant",
53+
body: [
54+
"**Here** is a function:",
55+
"",
56+
"```typescript",
57+
"function add(a: number, b: number): number {",
58+
" return a + b;",
59+
"}",
60+
"```",
61+
"",
62+
"Call it with `add(1, 2)`.",
63+
].join("\n"),
64+
timestamp_ms: TS,
65+
},
66+
];
67+
68+
const cacheMessages = [
69+
{ role: "user", body: "Write a function", timestamp_ms: TS_USER },
70+
{
71+
role: "assistant",
72+
body: [
73+
"Here is a function:",
74+
"",
75+
"function add(a: number, b: number): number {",
76+
" return a + b;",
77+
"}",
78+
"",
79+
"Call it with add(1, 2).",
80+
].join("\n"),
81+
timestamp_ms: TS,
82+
},
83+
];
84+
85+
const promptMessages = runProductionDedupPipeline(sessionMessages, cacheMessages);
86+
87+
// After dedup: user message stays (session-only because no timestamp overlap),
88+
// one assistant row survives from the cache (session assistant is a dupe).
89+
expect(promptMessages.filter((m) => m.role === "assistant")).toHaveLength(1);
90+
expect(promptMessages.filter((m) => m.role === "user")).toHaveLength(1);
91+
// Chronological order: user first, then assistant
92+
expect(promptMessages.map((m) => m.role)).toEqual(["user", "assistant"]);
93+
});
94+
95+
it("mixed formatting — bold + inline code + tilde fence — produces 1 row", () => {
96+
const sessionMessages = [
97+
{
98+
role: "assistant",
99+
body: "**Note**: Use `const` for constants.\n\n~~~c++\nconst int MAX = 100;\n~~~",
100+
timestamp_ms: TS,
101+
},
102+
];
103+
const cacheMessages = [
104+
{
105+
role: "assistant",
106+
body: "Note: Use const for constants.\n\nconst int MAX = 100;",
107+
timestamp_ms: TS,
108+
},
109+
];
110+
111+
const promptMessages = runProductionDedupPipeline(sessionMessages, cacheMessages);
112+
113+
// Session message is a dedup key match — only cache row survives
114+
expect(promptMessages).toHaveLength(1);
115+
expect(promptMessages.filter((m) => m.role === "assistant")).toHaveLength(1);
116+
});
117+
118+
it("duplicate user message is also deduped by body + timestamp", () => {
119+
// When both session and cache have the same user message, dedup should
120+
// also eliminate the session duplicate.
121+
const sessionMessages = [
122+
{ role: "user", body: "Hello, world!", timestamp_ms: TS_USER },
123+
];
124+
const cacheMessages = [
125+
{ role: "user", body: "Hello, world!", timestamp_ms: TS_USER },
126+
];
127+
128+
const promptMessages = runProductionDedupPipeline(sessionMessages, cacheMessages);
129+
130+
expect(promptMessages).toHaveLength(1);
131+
expect(promptMessages[0].role).toBe("user");
132+
});
133+
134+
it("distinct timestamps preserve duplicates (no false dedup)", () => {
135+
// If timestamps differ, both rows survive even if text is identical.
136+
// This verifies the dedup does NOT collapse genuinely distinct turns.
137+
const sessionMessages = [
138+
{ role: "assistant", body: "Same text", timestamp_ms: TS },
139+
];
140+
const cacheMessages = [
141+
{ role: "assistant", body: "Same text", timestamp_ms: TS + 1 },
142+
];
143+
144+
const promptMessages = runProductionDedupPipeline(sessionMessages, cacheMessages);
145+
146+
expect(promptMessages).toHaveLength(2);
147+
});
148+
149+
it("plain text — no formatting — still dedups correctly (baseline)", () => {
150+
const sessionMessages = [
151+
{ role: "assistant", body: "Hello, this is a plain text response.", timestamp_ms: TS },
152+
];
153+
const cacheMessages = [
154+
{ role: "assistant", body: "Hello, this is a plain text response.", timestamp_ms: TS },
155+
];
156+
157+
const promptMessages = runProductionDedupPipeline(sessionMessages, cacheMessages);
158+
159+
expect(promptMessages).toHaveLength(1);
160+
expect(promptMessages.filter((m) => m.role === "assistant")).toHaveLength(1);
161+
});
162+
163+
it("multiple assistant rows — only Markdown variant is deduped", () => {
164+
// Two assistant rows: one with Markdown (duplicate of cache), one plain text
165+
// (genuinely new turn). Only the Markdown variant should be deduped.
166+
const sessionMessages = [
167+
{ role: "assistant", body: "**bold** message", timestamp_ms: TS },
168+
{ role: "assistant", body: "New text", timestamp_ms: TS + 1000 },
169+
];
170+
const cacheMessages = [
171+
{ role: "assistant", body: "bold message", timestamp_ms: TS },
172+
];
173+
174+
const promptMessages = runProductionDedupPipeline(sessionMessages, cacheMessages);
175+
176+
// 1 cache + 1 new session = 2 total assistant rows
177+
expect(promptMessages.filter((m) => m.role === "assistant")).toHaveLength(2);
178+
});
179+
});

‎extensions/telegram/src/bot-handlers.runtime.gateway-verification.test.ts‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// Production import paths match the actual module (bot-message-dispatch.ts:58-60)
2+
import {
3+
stripInlineDirectiveTagsForDelivery,
4+
stripMarkdown,
5+
} from "openclaw/plugin-sdk/text-chunking";
6+
// P1 Gateway Verification: production code path proof
7+
// Uses the actual resolvePromptContextTextDedupeKey from the production module
8+
import { describe, expect, it } from "vitest";
9+
import { resolvePromptContextTextDedupeKey } from "./bot-handlers.runtime.js";
10+
11+
describe("[P1] Gateway verification: single assistant row after Markdown normalization", () => {
12+
// PATH 1: resolvePromptContextTextDedupeKey - used in bot-handlers.runtime.ts:1388-1393
13+
describe("resolvePromptContextTextDedupeKey", () => {
14+
const ts = 1_751_870_400_000;
15+
16+
it("tilde fence + lang tag -> 1 row", () => {
17+
const sessionKey = resolvePromptContextTextDedupeKey({
18+
body: "Code:\n\n~~~typescript\nconst x = 42;\n~~~\n\nEnd.",
19+
timestamp_ms: ts,
20+
});
21+
const cacheKey = resolvePromptContextTextDedupeKey({
22+
body: "Code:\n\nconst x = 42;\n\nEnd.",
23+
timestamp_ms: ts,
24+
});
25+
expect(sessionKey).toBe(cacheKey);
26+
// Only ONE assistant row injected: dedup eliminates the cache duplicate
27+
});
28+
29+
it("backtick fence, non-word info (c++) -> 1 row", () => {
30+
const sessionKey = resolvePromptContextTextDedupeKey({
31+
body: "```c++\nstd::cout << 1;\n```",
32+
timestamp_ms: ts,
33+
});
34+
const cacheKey = resolvePromptContextTextDedupeKey({
35+
body: "std::cout << 1;",
36+
timestamp_ms: ts,
37+
});
38+
expect(sessionKey).toBe(cacheKey);
39+
});
40+
41+
it("mixed bold+code+fenced -> 1 row", () => {
42+
const sessionKey = resolvePromptContextTextDedupeKey({
43+
body: "**A**: `b`\n\n```\nx\n```\n\ndone",
44+
timestamp_ms: ts,
45+
});
46+
const cacheKey = resolvePromptContextTextDedupeKey({
47+
body: "A: b\n\nx\n\ndone",
48+
timestamp_ms: ts,
49+
});
50+
expect(sessionKey).toBe(cacheKey);
51+
});
52+
53+
it("multi-line ts function -> 1 row", () => {
54+
const sessionKey = resolvePromptContextTextDedupeKey({
55+
body: "Here:\n\n```typescript\nfunction add(a, b) {\n return a + b;\n}\n```\n\nDone.",
56+
timestamp_ms: ts,
57+
});
58+
const cacheKey = resolvePromptContextTextDedupeKey({
59+
body: "Here:\n\nfunction add(a, b) {\n return a + b;\n}\n\nDone.",
60+
timestamp_ms: ts,
61+
});
62+
expect(sessionKey).toBe(cacheKey);
63+
});
64+
65+
it("multiple code blocks -> 1 row", () => {
66+
const sessionKey = resolvePromptContextTextDedupeKey({
67+
body: "A:\n\n```js\na\n```\n\nB:\n\n```ts\nb\n```\n\nEnd",
68+
timestamp_ms: ts,
69+
});
70+
const cacheKey = resolvePromptContextTextDedupeKey({
71+
body: "A:\n\na\n\nB:\n\nb\n\nEnd",
72+
timestamp_ms: ts,
73+
});
74+
expect(sessionKey).toBe(cacheKey);
75+
});
76+
77+
it("tilde fence, non-word info (c#) -> 1 row", () => {
78+
const sessionKey = resolvePromptContextTextDedupeKey({
79+
body: "~~~c#\nConsole.Write(1);\n~~~",
80+
timestamp_ms: ts,
81+
});
82+
const cacheKey = resolvePromptContextTextDedupeKey({
83+
body: "Console.Write(1);",
84+
timestamp_ms: ts,
85+
});
86+
expect(sessionKey).toBe(cacheKey);
87+
});
88+
89+
it("backtick .tsx info string -> 1 row", () => {
90+
const sessionKey = resolvePromptContextTextDedupeKey({
91+
body: "```.tsx\nconst a = 1;\n```",
92+
timestamp_ms: ts,
93+
});
94+
const cacheKey = resolvePromptContextTextDedupeKey({
95+
body: "const a = 1;",
96+
timestamp_ms: ts,
97+
});
98+
expect(sessionKey).toBe(cacheKey);
99+
});
100+
101+
it("plain text baseline -> 1 row", () => {
102+
const sessionKey = resolvePromptContextTextDedupeKey({
103+
body: "Hello World",
104+
timestamp_ms: ts,
105+
});
106+
const cacheKey = resolvePromptContextTextDedupeKey({
107+
body: "Hello World",
108+
timestamp_ms: ts,
109+
});
110+
expect(sessionKey).toBe(cacheKey);
111+
});
112+
});
113+
114+
// PATH 2: normalizePromptContextTimestampText equivalent - used in bot-message-dispatch.ts:1588-1594
115+
// Replicates the production closure: `stripMarkdown(stripInlineDirectiveTagsForDelivery(text).text).trim()`
116+
describe("normalizePromptContextTimestampText (stripMarkdown + stripInlineDirectiveTagsForDelivery)", () => {
117+
// Same signature and behavior as the production normalizePromptContextTimestampText helper.
118+
const normalize = (text: string): string =>
119+
stripMarkdown(stripInlineDirectiveTagsForDelivery(text).text).trim();
120+
121+
it("tilde fence transcript matches cache", () => {
122+
expect(normalize("Code:\n\n~~~typescript\nconst x = 42;\n~~~\n\nEnd.")).toBe(
123+
normalize("Code:\n\nconst x = 42;\n\nEnd."),
124+
);
125+
});
126+
127+
it("bold+code transcript matches cache", () => {
128+
expect(normalize("**Important**: Use `const` not `var`.")).toBe(
129+
normalize("Important: Use const not var."),
130+
);
131+
});
132+
133+
it("fenced code transcript matches cache", () => {
134+
expect(normalize("Example:\n\n```javascript\nconst x = 42;\n```\n\nEnd.")).toBe(
135+
normalize("Example:\n\nconst x = 42;\n\nEnd."),
136+
);
137+
});
138+
139+
it("tilde c# matches cache", () => {
140+
expect(normalize("~~~c#\nConsole.Write(1);\n~~~")).toBe(normalize("Console.Write(1);"));
141+
});
142+
});
143+
});

0 commit comments

Comments
 (0)