Skip to content

Commit d801d27

Browse files
vincentkocvincentkoc
committed
fix(qa-lab): add gateway log sentinels
1 parent ca236d0 commit d801d27

6 files changed

Lines changed: 591 additions & 9 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ Docs: https://docs.openclaw.ai
8383
- CLI/setup: collapse raw gateway config keys in existing-config summaries into friendly `Model` and `Gateway` rows.
8484
- CLI/config: show concise human config-write output with an indented backup path instead of printing checksum-heavy overwrite audit details by default.
8585
- CLI/docs: call the canonical lowercase docs MCP search tool and surface MCP errors instead of returning empty search results. Fixes #82702. (#82704) Thanks @hclsys.
86+
- QA-Lab: add gateway log sentinels for plugin hook failures, Codex app-server stalls/timeouts, cron allowlist drift, live quota blockers, and direct-reply self-message transcripts so harness proof fails on self-health regressions. (#80323) Thanks @100yenadmin.
8687
- QA-Lab: ignore heartbeat-only operational transcripts when capturing runtime parity cells so background checks cannot replace the scenario reply. (#80323) Thanks @100yenadmin.
8788
- QA-Lab: pin threaded-memory parity runs to `memory-core`, keep bundled plugin resolution enabled for QA commands, and retry transient session-store lock reads. (#72045) Thanks @WuKongAI-CMU.
8889
- QA-Lab/qa-channel: keep mock memory ranking, inbound media notes, and opened-file realpath checks stable for mock OpenAI qa-channel runs. (#66826) Thanks @gumadeiras.

‎extensions/qa-lab/src/gateway-log-sentinel.test.ts‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
import { describe, expect, it } from "vitest";
2+
import {
3+
assertNoGatewayLogSentinels,
4+
formatGatewayLogSentinelSummary,
5+
scanDirectReplyTranscriptSentinels,
6+
scanGatewayLogSentinels,
7+
} from "./gateway-log-sentinel.js";
8+
9+
describe("gateway log sentinels", () => {
10+
it("classifies May 13 beta.5 operational failure signatures", () => {
11+
const findings = scanGatewayLogSentinels(
12+
[
13+
"2026-05-13T00:00:01Z plugin before_prompt_build hook failed: TypeError: boom",
14+
"2026-05-13T00:00:02Z plugin before_tool_call crashed while evaluating policy",
15+
"2026-05-13T00:00:03Z plugin manifest invalid: missing contracts.tools registration",
16+
"2026-05-13T00:00:04Z codex app-server attempt timed out after 180000ms",
17+
"2026-05-13T00:00:05Z codex_app_server progress stalled for run abc123",
18+
"2026-05-13T00:00:06Z cron payload model openai/gpt-5.4 is not in model allowlist",
19+
"2026-05-13T00:00:07Z OpenAI quota exceeded for live-frontier request",
20+
].join("\n"),
21+
);
22+
23+
expect(findings.map((finding) => finding.kind)).toEqual([
24+
"plugin-hook-failure",
25+
"plugin-hook-failure",
26+
"plugin-contract-error",
27+
"codex-app-server-timeout",
28+
"stalled-agent-run",
29+
"cron-model-allowlist",
30+
"live-quota-or-subscription",
31+
]);
32+
expect(findings.find((finding) => finding.kind === "plugin-hook-failure")).toMatchObject({
33+
verdict: "qa-harness-bug",
34+
owner: "plugin",
35+
productImpact: "P1",
36+
});
37+
expect(findings.find((finding) => finding.kind === "live-quota-or-subscription")).toMatchObject(
38+
{
39+
verdict: "environment-blocked",
40+
owner: "environment",
41+
productImpact: "P4",
42+
},
43+
);
44+
});
45+
46+
it("honors log cursors while preserving absolute line numbers", () => {
47+
const prefix = "safe line\n";
48+
const findings = scanGatewayLogSentinels(`${prefix}codex app-server attempt timed out`, {
49+
since: prefix.length,
50+
});
51+
52+
expect(findings).toHaveLength(1);
53+
expect(findings[0]).toMatchObject({
54+
kind: "codex-app-server-timeout",
55+
line: 2,
56+
});
57+
});
58+
59+
it("throws actionable summaries unless only environment blockers are allowed", () => {
60+
expect(() => assertNoGatewayLogSentinels("codex_app_server progress stalled")).toThrow(
61+
"stalled-agent-run",
62+
);
63+
expect(() =>
64+
assertNoGatewayLogSentinels("OpenAI quota exceeded", { allowEnvironmentBlocked: true }),
65+
).not.toThrow();
66+
expect(formatGatewayLogSentinelSummary(scanGatewayLogSentinels("OpenAI quota exceeded"))).toBe(
67+
"live-quota-or-subscription@1 environment-blocked owner=environment: OpenAI quota exceeded",
68+
);
69+
});
70+
71+
it("detects direct reply self-message transcripts separately from gateway logs", () => {
72+
const findings = scanDirectReplyTranscriptSentinels(
73+
[
74+
JSON.stringify({
75+
message: {
76+
role: "assistant",
77+
content: [
78+
{
79+
type: "tool_use",
80+
name: "message",
81+
input: { action: "send", conversationId: "qa-operator", text: "hello" },
82+
},
83+
],
84+
},
85+
}),
86+
JSON.stringify({ message: { role: "assistant", content: "Sent." } }),
87+
].join("\n"),
88+
);
89+
90+
expect(findings).toHaveLength(1);
91+
expect(findings[0]).toMatchObject({
92+
kind: "direct-reply-self-message",
93+
verdict: "product-bug",
94+
owner: "openclaw-routing",
95+
});
96+
});
97+
98+
it("detects OpenAI function_call-shaped direct reply transcripts", () => {
99+
const findings = scanDirectReplyTranscriptSentinels(
100+
[
101+
JSON.stringify({
102+
message: {
103+
role: "assistant",
104+
content: [
105+
{
106+
type: "function_call",
107+
name: "message",
108+
arguments: JSON.stringify({
109+
action: "send",
110+
target: "current",
111+
text: "hello",
112+
}),
113+
},
114+
],
115+
},
116+
}),
117+
JSON.stringify({ message: { role: "assistant", content: "Sent." } }),
118+
].join("\n"),
119+
);
120+
121+
expect(findings.map((finding) => finding.kind)).toEqual(["direct-reply-self-message"]);
122+
});
123+
});

0 commit comments

Comments
 (0)