Skip to content

Commit 5f46690

Browse files
vincentkocvincentkoc
committed
test(telegram): cover three chunk finals
1 parent a0d22e8 commit 5f46690

6 files changed

Lines changed: 183 additions & 3 deletions

File tree

‎docs/concepts/qa-e2e-automation.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime
233233
- `telegram-whoami-command`
234234
- `telegram-context-command`
235235
- `telegram-long-final-reuses-preview`
236+
- `telegram-long-final-three-chunks`
236237

237238
Output artifacts:
238239

‎extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ describe("telegram live qa runtime", () => {
334334
"telegram-current-session-status-tool",
335335
"telegram-mentioned-message-reply",
336336
"telegram-long-final-reuses-preview",
337+
"telegram-long-final-three-chunks",
337338
"telegram-mention-gating",
338339
]);
339340
expect(scenarios.map((scenario) => scenario.id)).toEqual([
@@ -345,6 +346,7 @@ describe("telegram live qa runtime", () => {
345346
"telegram-current-session-status-tool",
346347
"telegram-mentioned-message-reply",
347348
"telegram-long-final-reuses-preview",
349+
"telegram-long-final-three-chunks",
348350
"telegram-mention-gating",
349351
]);
350352
expect(
@@ -365,6 +367,17 @@ describe("telegram live qa runtime", () => {
365367
expectedJoinedSutTextIncludes: ["TELEGRAM-LONG-FINAL-BEGIN", "TELEGRAM-LONG-FINAL-END"],
366368
expectedSutMessageCount: 2,
367369
});
370+
expect(
371+
scenarios
372+
.find((scenario) => scenario.id === "telegram-long-final-three-chunks")
373+
?.buildRun("sut_bot"),
374+
).toMatchObject({
375+
expectedJoinedSutTextIncludes: [
376+
"TELEGRAM-LONG-FINAL-3CHUNK-BEGIN",
377+
"TELEGRAM-LONG-FINAL-3CHUNK-END",
378+
],
379+
expectedSutMessageCount: 3,
380+
});
368381
});
369382

370383
it("keeps bot-to-bot plain mentions out of the default Telegram live set", () => {
@@ -487,6 +500,65 @@ describe("telegram live qa runtime", () => {
487500
).toThrow("expected 2 SUT message(s), observed 3");
488501
});
489502

503+
it("accepts legitimate three-chunk Telegram final replies", () => {
504+
expect(() =>
505+
__testing.assertTelegramScenarioMessageSet({
506+
expectedJoinedSutTextIncludes: [
507+
"TELEGRAM-LONG-FINAL-3CHUNK-BEGIN",
508+
"TELEGRAM-LONG-FINAL-3CHUNK-END",
509+
],
510+
expectedSutMessageCount: 3,
511+
groupId: "-100123",
512+
scenarioId: "telegram-long-final-three-chunks",
513+
sutBotId: 99,
514+
observedMessages: [
515+
{
516+
updateId: 1,
517+
messageId: 10,
518+
chatId: -100123,
519+
senderId: 99,
520+
senderIsBot: true,
521+
scenarioId: "telegram-long-final-three-chunks",
522+
scenarioTitle: "Telegram three-chunk final keeps only final chunks",
523+
matchedScenario: true,
524+
text: "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN part one ",
525+
timestamp: 1_700_000_000_000,
526+
inlineButtons: [],
527+
mediaKinds: [],
528+
},
529+
{
530+
updateId: 2,
531+
messageId: 11,
532+
chatId: -100123,
533+
senderId: 99,
534+
senderIsBot: true,
535+
scenarioId: "telegram-long-final-three-chunks",
536+
scenarioTitle: "Telegram three-chunk final keeps only final chunks",
537+
matchedScenario: true,
538+
text: "part two ",
539+
timestamp: 1_700_000_001_000,
540+
inlineButtons: [],
541+
mediaKinds: [],
542+
},
543+
{
544+
updateId: 3,
545+
messageId: 12,
546+
chatId: -100123,
547+
senderId: 99,
548+
senderIsBot: true,
549+
scenarioId: "telegram-long-final-three-chunks",
550+
scenarioTitle: "Telegram three-chunk final keeps only final chunks",
551+
matchedScenario: true,
552+
text: "part three TELEGRAM-LONG-FINAL-3CHUNK-END",
553+
timestamp: 1_700_000_002_000,
554+
inlineButtons: [],
555+
mediaKinds: [],
556+
},
557+
],
558+
}),
559+
).not.toThrow();
560+
});
561+
490562
it("matches scenario replies by thread or exact marker", () => {
491563
expect(
492564
__testing.matchesTelegramScenarioReply({

‎extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ type TelegramQaScenarioId =
4848
| "telegram-whoami-command"
4949
| "telegram-context-command"
5050
| "telegram-current-session-status-tool"
51+
| "telegram-long-final-three-chunks"
5152
| "telegram-long-final-reuses-preview"
5253
| "telegram-mentioned-message-reply"
5354
| "telegram-mention-gating";
@@ -314,6 +315,24 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [
314315
settleMs: 4_000,
315316
}),
316317
},
318+
{
319+
id: "telegram-long-final-three-chunks",
320+
title: "Telegram three-chunk final keeps only final chunks",
321+
defaultEnabled: false,
322+
timeoutMs: 60_000,
323+
buildRun: (sutUsername) => ({
324+
allowAnySutReply: true,
325+
expectReply: true,
326+
input: `@${sutUsername} Telegram long final three chunk QA check. Use the scripted three chunk final response.`,
327+
expectedTextIncludes: ["TELEGRAM-LONG-FINAL-3CHUNK-BEGIN"],
328+
expectedJoinedSutTextIncludes: [
329+
"TELEGRAM-LONG-FINAL-3CHUNK-BEGIN",
330+
"TELEGRAM-LONG-FINAL-3CHUNK-END",
331+
],
332+
expectedSutMessageCount: 3,
333+
settleMs: 4_000,
334+
}),
335+
},
317336
{
318337
id: "telegram-mention-gating",
319338
standardId: "mention-gating",

‎extensions/qa-lab/src/providers/mock-openai/server.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,28 @@ describe("qa mock openai server", () => {
241241
expect(telegramLongBody).toContain("TELEGRAM-LONG-FINAL-END");
242242
expect(telegramLongBody.length).toBeGreaterThan(4_500);
243243

244+
const telegramThreeChunkLongResponse = await fetch(`${server.baseUrl}/v1/responses`, {
245+
method: "POST",
246+
headers: {
247+
"content-type": "application/json",
248+
},
249+
body: JSON.stringify({
250+
stream: true,
251+
input: [
252+
makeUserInput(
253+
"Telegram long final three chunk QA check. Use the scripted three chunk final response.",
254+
),
255+
],
256+
}),
257+
});
258+
expect(telegramThreeChunkLongResponse.status).toBe(200);
259+
const telegramThreeChunkLongBody = await telegramThreeChunkLongResponse.text();
260+
expect(telegramThreeChunkLongBody).toContain('"type":"response.output_text.delta"');
261+
expect(telegramThreeChunkLongBody).toContain('"phase":"final_answer"');
262+
expect(telegramThreeChunkLongBody).toContain("TELEGRAM-LONG-FINAL-3CHUNK-BEGIN");
263+
expect(telegramThreeChunkLongBody).toContain("TELEGRAM-LONG-FINAL-3CHUNK-END");
264+
expect(telegramThreeChunkLongBody.length).toBeGreaterThan(8_000);
265+
244266
const blockResponse = await fetch(`${server.baseUrl}/v1/responses`, {
245267
method: "POST",
246268
headers: {

‎extensions/qa-lab/src/providers/mock-openai/server.ts‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ const QA_GROUP_VISIBLE_REPLY_TOOL_PROMPT_RE = /qa group visible reply tool check
153153
const QA_GROUP_MESSAGE_UNAVAILABLE_FALLBACK_PROMPT_RE =
154154
/qa group message unavailable fallback check/i;
155155
const QA_TELEGRAM_CURRENT_SESSION_STATUS_PROMPT_RE = /telegram current session_status qa check/i;
156+
const QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE = /telegram long final three chunk qa check/i;
156157
const QA_TELEGRAM_LONG_FINAL_PROMPT_RE = /telegram long final qa check/i;
157158
const QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE = /subagent direct fallback qa check/i;
158159
const QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE = /subagent direct fallback worker/i;
@@ -1035,13 +1036,21 @@ function splitMockStreamingText(text: string, parts = 3) {
10351036
return chunks.length > 1 ? chunks : [text.slice(0, 1), text.slice(1)];
10361037
}
10371038

1038-
function buildTelegramLongFinalText() {
1039+
function buildTelegramLongFinalText({
1040+
endMarker = "TELEGRAM-LONG-FINAL-END",
1041+
segmentCount = 54,
1042+
startMarker = "TELEGRAM-LONG-FINAL-BEGIN",
1043+
}: {
1044+
endMarker?: string;
1045+
segmentCount?: number;
1046+
startMarker?: string;
1047+
} = {}) {
10391048
const body = Array.from(
1040-
{ length: 54 },
1049+
{ length: segmentCount },
10411050
(_, index) =>
10421051
`telegram-long-final-segment-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`,
10431052
).join("\n");
1044-
return `TELEGRAM-LONG-FINAL-BEGIN\n${body}\nTELEGRAM-LONG-FINAL-END`;
1053+
return `${startMarker}\n${body}\n${endMarker}`;
10451054
}
10461055

10471056
function buildAssistantOutputItem(spec: MockAssistantMessageSpec) {
@@ -1320,6 +1329,21 @@ async function buildResponsesPayload(
13201329
}
13211330
return buildAssistantEvents("");
13221331
}
1332+
if (QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE.test(allInputText)) {
1333+
const text = buildTelegramLongFinalText({
1334+
endMarker: "TELEGRAM-LONG-FINAL-3CHUNK-END",
1335+
segmentCount: 96,
1336+
startMarker: "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN",
1337+
});
1338+
return buildAssistantEvents([
1339+
{
1340+
id: "msg_mock_telegram_long_final_three_chunk",
1341+
phase: "final_answer",
1342+
streamDeltas: splitMockStreamingText(text),
1343+
text,
1344+
},
1345+
]);
1346+
}
13231347
if (QA_TELEGRAM_LONG_FINAL_PROMPT_RE.test(allInputText)) {
13241348
const text = buildTelegramLongFinalText();
13251349
return buildAssistantEvents([

‎extensions/telegram/src/bot-message-dispatch.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1618,6 +1618,48 @@ describe("dispatchTelegramMessage draft streaming", () => {
16181618
expect(answerDraftStream.clear).not.toHaveBeenCalled();
16191619
});
16201620

1621+
it("uses the active preview as the first chunk for three-chunk long text finals", async () => {
1622+
const answerDraftStream = createSequencedDraftStream(1001);
1623+
const reasoningDraftStream = createDraftStream();
1624+
createTelegramDraftStream
1625+
.mockImplementationOnce(() => answerDraftStream)
1626+
.mockImplementationOnce(() => reasoningDraftStream);
1627+
const finalText = `${"A".repeat(70)}${"B".repeat(70)}${"C".repeat(70)}`;
1628+
dispatchReplyWithBufferedBlockDispatcher.mockImplementation(
1629+
async ({ dispatcherOptions, replyOptions }) => {
1630+
await replyOptions?.onPartialReply?.({ text: "Working preview" });
1631+
await dispatcherOptions.deliver({ text: finalText, replyToId: "456" }, { kind: "final" });
1632+
return { queuedFinal: true };
1633+
},
1634+
);
1635+
deliverReplies.mockResolvedValue({ delivered: true });
1636+
editMessageTelegram.mockResolvedValue({ ok: true, chatId: "123", messageId: "1001" });
1637+
1638+
await dispatchWithContext({
1639+
context: createContext(),
1640+
streamMode: "partial",
1641+
textLimit: 80,
1642+
});
1643+
1644+
const editedText = editMessageTelegram.mock.calls[0]?.[2] as string;
1645+
const followUpReplies =
1646+
(deliverReplies.mock.calls[0]?.[0] as { replies?: Array<{ text?: string }> })?.replies ?? [];
1647+
const followUpText = followUpReplies.map((reply) => reply.text ?? "").join("");
1648+
1649+
expect(editMessageTelegram).toHaveBeenCalledTimes(1);
1650+
expect(editedText.length).toBeLessThanOrEqual(80);
1651+
expect(followUpReplies).toHaveLength(1);
1652+
expect(followUpText.length).toBeGreaterThan(80);
1653+
expect(`${editedText}${followUpText}`).toBe(finalText);
1654+
expect(deliverReplies).toHaveBeenCalledTimes(1);
1655+
expect(deliverReplies).toHaveBeenCalledWith(
1656+
expect.objectContaining({
1657+
replies: [expect.not.objectContaining({ replyToId: expect.any(String) })],
1658+
}),
1659+
);
1660+
expect(answerDraftStream.clear).not.toHaveBeenCalled();
1661+
});
1662+
16211663
it("does not force new message on first assistant message start", async () => {
16221664
const draftStream = createDraftStream(999);
16231665
createTelegramDraftStream.mockReturnValue(draftStream);

0 commit comments

Comments
 (0)