Skip to content

Commit 1affe4f

Browse files
RomneyDaRomneyDa
authored
Fold Telegram RTT sampling into live QA evidence (#92550)
* refactor(qa): fold telegram rtt into live evidence * test: default package telegram rtt samples * refactor(qa-lab): fold telegram rtt into live evidence * fix(qa-lab): keep package telegram rtt optional for focused runs * fix(qa-lab): avoid stale rtt evidence on failed samples * fix(qa-lab): pass telegram live env into credential leasing * fix(qa-lab): update telegram canary remediation artifacts * docs(qa): remove stale telegram observed artifact guidance * fix(qa-lab): clarify telegram empty-reply remediation * fix(qa-lab): honor telegram rtt timeout * ci(qa): drop stale telegram capture env * refactor: align telegram evidence coverage fields * fix: ignore stale telegram observed artifacts * fix: preserve telegram rtt coverage mapping * fix: omit unused telegram rtt catch binding * docs: document telegram rtt check selector
1 parent 439a9e9 commit 1affe4f

29 files changed

Lines changed: 729 additions & 4184 deletions

‎.github/workflows/mantis-telegram-live.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,6 @@ jobs:
379379
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
380380
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
381381
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
382-
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
383382
CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR }}
384383
CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN }}
385384
OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR: ${{ secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR }}

‎.github/workflows/npm-telegram-beta-e2e.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,6 @@ jobs:
220220
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
221221
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
222222
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
223-
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
224223
INPUT_SCENARIO: ${{ inputs.scenario }}
225224
PACKAGE_ARTIFACT_NAME: ${{ inputs.package_artifact_name || '' }}
226225
run: |

‎.github/workflows/openclaw-release-checks.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1412,7 +1412,6 @@ jobs:
14121412
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
14131413
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
14141414
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
1415-
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
14161415
run: |
14171416
set -euo pipefail
14181417

‎.github/workflows/qa-live-transports-convex.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -532,7 +532,6 @@ jobs:
532532
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
533533
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
534534
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
535-
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
536535
INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.scenario || '' }}
537536
run: |
538537
set -euo pipefail

‎docs/concepts/mantis.md‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -247,12 +247,13 @@ of only a bot-to-bot Slack transcript.
247247
evidence pipeline. It checks out the trusted candidate ref in a separate
248248
worktree, runs `pnpm openclaw qa telegram --credential-source convex
249249
--credential-role ci`, writes a `mantis-evidence.json` manifest from the
250-
Telegram QA summary and observed-message artifact, renders the redacted
251-
transcript HTML through a Crabbox desktop browser, generates a motion-trimmed GIF
252-
with `crabbox media preview`, and posts the inline PR evidence comment when a PR
253-
number is available. This lane is transcript-visual rather than logged-in
254-
Telegram Web proof: the Telegram Bot API gives stable live message evidence, but
255-
Telegram Web login state is not required for normal Mantis automation.
250+
Telegram QA summary, `qa-evidence.json`, and report artifacts, renders the
251+
redacted evidence HTML through a Crabbox desktop browser, generates a
252+
motion-trimmed GIF with `crabbox media preview`, and posts the inline PR
253+
evidence comment when a PR number is available. This lane is QA-evidence visual
254+
rather than logged-in Telegram Web proof: the Telegram Bot API gives stable live
255+
message evidence, but Telegram Web login state is not required for normal Mantis
256+
automation.
256257

257258
`Mantis Telegram Desktop Proof` is the agentic native Telegram Desktop
258259
before/after wrapper. A maintainer can trigger it from a PR comment with
@@ -494,8 +495,8 @@ zero:
494495

495496
- `pnpm openclaw qa discord` already runs a live Discord lane with driver and
496497
SUT bots.
497-
- The live transport runner already writes reports and observed-message
498-
artifacts under `.artifacts/qa-e2e/`.
498+
- The live transport runner already writes reports, QA evidence, and
499+
transport-specific artifacts under `.artifacts/qa-e2e/`.
499500
- Convex credential leases already provide exclusive access to shared live
500501
transport credentials.
501502
- The browser control service already supports screenshots, snapshots,

‎docs/concepts/qa-e2e-automation.md‎

Lines changed: 26 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -318,17 +318,17 @@ Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count
318318

319319
These lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags:
320320

321-
| Flag | Default | Description |
322-
| ------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
323-
| `--scenario <id>` | - | Run only this scenario. Repeatable. |
324-
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/<transport>-<timestamp>` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. |
325-
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral cwd. |
326-
| `--sut-account <id>` | `sut` | Temporary account id inside the QA gateway config. |
327-
| `--provider-mode <mode>` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). |
328-
| `--model <ref>` / `--alt-model <ref>` | provider default | Primary/alternate model refs. |
329-
| `--fast` | off | Provider fast mode where supported. |
330-
| `--credential-source <env\|convex>` | `env` | See [Convex credential pool](#convex-credential-pool). |
331-
| `--credential-role <maintainer\|ci>` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. |
321+
| Flag | Default | Description |
322+
| ------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
323+
| `--scenario <id>` | - | Run only this scenario. Repeatable. |
324+
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/<transport>-<timestamp>` | Where reports, summaries, evidence, transport-specific artifacts, and the output log are written. Relative paths resolve against `--repo-root`. |
325+
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral cwd. |
326+
| `--sut-account <id>` | `sut` | Temporary account id inside the QA gateway config. |
327+
| `--provider-mode <mode>` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). |
328+
| `--model <ref>` / `--alt-model <ref>` | provider default | Primary/alternate model refs. |
329+
| `--fast` | off | Provider fast mode where supported. |
330+
| `--credential-source <env\|convex>` | `env` | See [Convex credential pool](#convex-credential-pool). |
331+
| `--credential-role <maintainer\|ci>` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. |
332332

333333
Each lane exits non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code.
334334

@@ -346,10 +346,6 @@ Required env when `--credential-source env`:
346346
- `OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN`
347347
- `OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN`
348348

349-
Optional:
350-
351-
- `OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT=1` keeps message bodies in observed-message artifacts (default redacts).
352-
353349
Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts`):
354350

355351
- `telegram-canary`
@@ -375,26 +371,26 @@ Output artifacts:
375371

376372
- `telegram-qa-report.md`
377373
- `qa-evidence.json` - evidence entries for the live transport checks, including profile, coverage, provider, channel, artifacts, result, and RTT fields.
378-
- `telegram-qa-observed-messages.json` - bodies redacted unless `OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT=1`.
379374

380-
Package RTT comparison uses the same Telegram credential contract while keeping
381-
its RTT sample controls on the RTT harness path:
375+
Package Telegram runs use the same Telegram credential contract. Repeated RTT
376+
measurement is part of the normal package Telegram live lane; the RTT
377+
distribution is folded into `qa-evidence.json` under `result.timing` for the
378+
selected RTT check.
382379

383380
```bash
384-
pnpm rtt openclaw@beta \
385-
--credential-source convex \
386-
--credential-role maintainer \
387-
--samples 20 \
388-
--sample-timeout-ms 30000
381+
OPENCLAW_QA_CREDENTIAL_SOURCE=convex \
382+
pnpm test:docker:npm-telegram-live
389383
```
390384

391-
When `--credential-source convex` is set, the RTT Docker wrapper leases a
392-
`kind: "telegram"` credential, exports the leased group/driver/SUT bot env into
393-
the installed-package run, heartbeats the lease, and releases it on shutdown.
394-
`--samples` and `--sample-timeout-ms` still feed
395-
`OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES` and
396-
`OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS`, so `result.json` remains comparable
397-
across env-backed and Convex-backed RTT runs.
385+
When `OPENCLAW_QA_CREDENTIAL_SOURCE=convex` is set, the package live wrapper
386+
leases a `kind: "telegram"` credential, exports the leased group/driver/SUT bot
387+
env into the installed-package run, heartbeats the lease, and releases it on
388+
shutdown. The package wrapper defaults to 20 RTT checks of
389+
`telegram-mentioned-message-reply`, a 30s RTT timeout, and Convex role
390+
`maintainer` outside CI when Convex is selected. Override
391+
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`,
392+
or `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune RTT measurement without
393+
creating a separate RTT command or Telegram-specific summary format.
398394

399395
### Discord QA
400396

‎docs/help/testing.md‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -218,17 +218,27 @@ inside every shard.
218218
`OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ=/path/to/openclaw-current.tgz` or
219219
`OPENCLAW_CURRENT_PACKAGE_TGZ` to test a resolved local tarball instead of
220220
installing from the registry.
221+
- Emits repeated RTT timing in `qa-evidence.json` by default with
222+
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES=20`. Override
223+
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`,
224+
`OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, or
225+
`OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune the RTT run.
226+
`OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` accepts a comma-separated list of
227+
Telegram QA check IDs to sample; when unset, the default RTT-capable check
228+
is `telegram-mentioned-message-reply`.
221229
- Uses the same Telegram env credentials or Convex credential source as
222230
`pnpm openclaw qa telegram`. For CI/release automation, set
223231
`OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE=convex` plus
224-
`OPENCLAW_QA_CONVEX_SITE_URL` and the role secret. If
232+
`OPENCLAW_QA_CONVEX_SITE_URL` and a role secret. If
225233
`OPENCLAW_QA_CONVEX_SITE_URL` and a Convex role secret are present in CI,
226234
the Docker wrapper selects Convex automatically.
227235
- The wrapper validates Telegram or Convex credential env on the host before
228236
Docker build/install work. Set `OPENCLAW_NPM_TELEGRAM_SKIP_CREDENTIAL_PREFLIGHT=1`
229237
only when deliberately debugging pre-credential setup.
230238
- `OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE=ci|maintainer` overrides the shared
231-
`OPENCLAW_QA_CREDENTIAL_ROLE` for this lane only.
239+
`OPENCLAW_QA_CREDENTIAL_ROLE` for this lane only. When Convex credentials
240+
are selected and no role is set, the wrapper uses `ci` in CI and
241+
`maintainer` outside CI.
232242
- GitHub Actions exposes this lane as the manual maintainer workflow
233243
`NPM Telegram Beta E2E`. It does not run on merge. The workflow uses the
234244
`qa-live-shared` environment and Convex CI credential leases.
@@ -344,11 +354,11 @@ gh workflow run package-acceptance.yml --ref main \
344354
want artifacts without a failing exit code.
345355
- Requires two distinct bots in the same private group, with the SUT bot exposing a Telegram username.
346356
- For stable bot-to-bot observation, enable Bot-to-Bot Communication Mode in `@BotFather` for both bots and ensure the driver bot can observe group bot traffic.
347-
- Writes a Telegram QA report, summary, and observed-messages artifact under `.artifacts/qa-e2e/...`. Replying scenarios include RTT from driver send request to observed SUT reply.
357+
- Writes a Telegram QA report, summary, and `qa-evidence.json` under `.artifacts/qa-e2e/...`. Replying scenarios include RTT from driver send request to observed SUT reply.
348358

349359
`Mantis Telegram Live` is the PR-evidence wrapper around this lane. It runs the
350-
candidate ref with Convex-leased Telegram credentials, renders the redacted
351-
observed-message transcript in a Crabbox desktop browser, records MP4 evidence,
360+
candidate ref with Convex-leased Telegram credentials, renders the redacted QA
361+
report/evidence bundle in a Crabbox desktop browser, records MP4 evidence,
352362
generates a motion-trimmed GIF, uploads the artifact bundle, and posts inline PR
353363
evidence through the Mantis GitHub App when `pr_number` is set. Maintainers can
354364
start it from the Actions UI through `Mantis Scenario` (`scenario_id:

‎extensions/qa-lab/src/evidence-summary.test.ts‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,6 @@ describe("evidence summary", () => {
133133
artifactPaths: [
134134
{ kind: "summary", path: QA_EVIDENCE_FILENAME },
135135
{ kind: "report", path: "telegram-qa-report.md" },
136-
{ kind: "transport-observations", path: "telegram-qa-observed-messages.json" },
137136
],
138137
env: {
139138
OPENCLAW_QA_RUNNER: "crabbox",
@@ -206,11 +205,6 @@ describe("evidence summary", () => {
206205
path: "telegram-qa-report.md",
207206
source: "telegram-live-transport",
208207
},
209-
{
210-
kind: "transport-observations",
211-
path: "telegram-qa-observed-messages.json",
212-
source: "telegram-live-transport",
213-
},
214208
],
215209
}),
216210
result: {
@@ -226,6 +220,45 @@ describe("evidence summary", () => {
226220
]);
227221
});
228222

223+
it("preserves aggregate live transport timing", () => {
224+
const evidence = buildLiveTransportEvidenceSummary({
225+
artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
226+
generatedAt: "2026-06-07T12:05:00.000Z",
227+
primaryModel: "openai/gpt-5.5",
228+
providerMode: "live-frontier",
229+
checks: [
230+
{
231+
id: "telegram-mentioned-message-reply",
232+
coverageIds: ["channels.telegram.mention-gating"],
233+
title: "Telegram mentioned message gets a reply",
234+
status: "pass",
235+
details: "5 samples collected.",
236+
rttMs: 2000,
237+
timing: {
238+
rttMs: 1200,
239+
avgMs: 1300,
240+
p50Ms: 1200,
241+
p95Ms: 1800,
242+
maxMs: 2200,
243+
samples: 5,
244+
failedSamples: 1,
245+
},
246+
},
247+
],
248+
transportId: "telegram",
249+
});
250+
251+
expect(evidence.entries[0]?.result.timing).toEqual({
252+
rttMs: 1200,
253+
avgMs: 1300,
254+
p50Ms: 1200,
255+
p95Ms: 1800,
256+
maxMs: 2200,
257+
samples: 5,
258+
failedSamples: 1,
259+
});
260+
});
261+
229262
it("builds Vitest runner evidence entries", () => {
230263
const evidence = buildVitestEvidenceSummary({
231264
artifactPaths: [

‎extensions/qa-lab/src/evidence-summary.ts‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ type QaEvidenceScenarioResultInput = {
184184
name: string;
185185
status: QaEvidenceStatusInput;
186186
details?: string;
187+
timing?: QaEvidenceTiming;
187188
rttMs?: number;
188189
rttMeasurement?: {
189190
finalMatchedReplyRttMs?: number;
@@ -195,6 +196,7 @@ type QaEvidenceLiveTransportCheckInput = {
195196
title: string;
196197
status: QaEvidenceStatusInput;
197198
details: string;
199+
timing?: QaEvidenceTiming;
198200
rttMs?: number;
199201
rttMeasurement?: {
200202
finalMatchedReplyRttMs?: number;
@@ -203,7 +205,10 @@ type QaEvidenceLiveTransportCheckInput = {
203205
artifactPaths?: Readonly<Record<string, string>>;
204206
};
205207

206-
type QaEvidenceRttInput = Pick<QaEvidenceScenarioResultInput, "rttMeasurement" | "rttMs">;
208+
type QaEvidenceRttInput = Pick<
209+
QaEvidenceScenarioResultInput,
210+
"rttMeasurement" | "rttMs" | "timing"
211+
>;
207212

208213
type QaEvidenceTestTargetInput = {
209214
id: string;
@@ -423,8 +428,17 @@ function failureForResult(result: {
423428
}
424429

425430
function timingForRttResult(check: QaEvidenceRttInput) {
431+
const timing: QaEvidenceTiming = { ...check.timing };
426432
const rttMs = check.rttMeasurement?.finalMatchedReplyRttMs ?? check.rttMs;
427-
return typeof rttMs === "number" && Number.isFinite(rttMs) && rttMs > 0 ? { rttMs } : undefined;
433+
if (
434+
timing.rttMs === undefined &&
435+
typeof rttMs === "number" &&
436+
Number.isFinite(rttMs) &&
437+
rttMs > 0
438+
) {
439+
timing.rttMs = rttMs;
440+
}
441+
return Object.keys(timing).length > 0 ? timing : undefined;
428442
}
429443

430444
function timingForTestResult(result: QaEvidenceTestResultInput) {

‎extensions/qa-lab/src/live-transports/shared/live-transport-result.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Qa Lab plugin module implements shared live-transport result shapes.
2+
import type { QaEvidenceTiming } from "../../evidence-summary.js";
3+
4+
export type LiveTransportRttMeasurement = {
5+
finalMatchedReplyRttMs: number;
6+
requestStartedAt: string;
7+
responseObservedAt: string;
8+
source: "request-to-observed-message";
9+
};
10+
11+
export type LiveTransportCheckResult = {
12+
id: string;
13+
title: string;
14+
status: "pass" | "fail";
15+
details: string;
16+
coverageIds?: readonly string[];
17+
timing?: QaEvidenceTiming;
18+
rttMs?: number;
19+
requestStartedAt?: string;
20+
responseObservedAt?: string;
21+
rttMeasurement?: LiveTransportRttMeasurement;
22+
sentMessageId?: number;
23+
responseMessageId?: number;
24+
};

0 commit comments

Comments
 (0)