Skip to content

Commit ee9d9ea

Browse files
authored
fix(ci): verify live provider traffic in performance runs (#103073)
* fix(diagnostics): emit provider request timeline events * fix(ci): verify Kova live provider evidence
1 parent 1b03e51 commit ee9d9ea

6 files changed

Lines changed: 822 additions & 15 deletions

File tree

.github/workflows/openclaw-performance.yml

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,27 @@ jobs:
247247
chmod 0755 "$HOME/.local/bin/kova"
248248
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
249249
250+
- name: Allow live auth for OpenAI candidate state
251+
if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
252+
shell: bash
253+
run: |
254+
set -euo pipefail
255+
node - <<'NODE'
256+
const fs = require("node:fs");
257+
const path = require("node:path");
258+
const file = path.join(process.env.KOVA_SRC, "states/mock-openai-provider.json");
259+
const state = JSON.parse(fs.readFileSync(file, "utf8"));
260+
if (state.auth?.mode !== "mock") {
261+
throw new Error(`expected mock-openai-provider auth.mode=mock, got ${state.auth?.mode}`);
262+
}
263+
264+
// The release profile pairs the live agent scenario with this otherwise mock-only state.
265+
// This ephemeral checkout must honor the lane's explicit --auth live selection.
266+
state.auth.mode = "default";
267+
state.auth.reason = "Honor the workflow lane's explicit run-level auth selection.";
268+
fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
269+
NODE
270+
250271
- name: Configure OCM local workspace dependencies
251272
if: steps.lane.outputs.run == 'true'
252273
shell: bash
@@ -292,12 +313,24 @@ jobs:
292313
shell: bash
293314
run: |
294315
set -euo pipefail
316+
repeat="$REQUESTED_REPEAT"
317+
if [[ "$MATRIX_REPEAT" != "input" ]]; then
318+
repeat="$MATRIX_REPEAT"
319+
fi
320+
plan_dir="${RUNNER_TEMP}/kova-plans"
321+
plan_json="${plan_dir}/${LANE_ID}.json"
322+
mkdir -p "$plan_dir"
323+
295324
kova version --json
296325
kova matrix plan \
297326
--profile "$PROFILE" \
298327
--target "local-build:${GITHUB_WORKSPACE}" \
299-
--include scenario:fresh-install \
300-
--json >/tmp/kova-plan.json
328+
--include "$INCLUDE_FILTERS" \
329+
--parallel 1 \
330+
--repeat "$repeat" \
331+
--json >"$plan_json"
332+
echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"
333+
echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV"
301334
302335
- name: Configure live OpenAI auth
303336
if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
@@ -326,15 +359,13 @@ jobs:
326359
set -euo pipefail
327360
mkdir -p "$REPORT_DIR" "$BUNDLE_DIR" "$SUMMARY_DIR"
328361
329-
repeat="$REQUESTED_REPEAT"
330-
if [[ "$MATRIX_REPEAT" != "input" ]]; then
331-
repeat="$MATRIX_REPEAT"
332-
fi
362+
repeat="$KOVA_LANE_REPEAT"
333363
334364
args=(
335365
matrix run
336366
--profile "$PROFILE"
337367
--target "local-build:${GITHUB_WORKSPACE}"
368+
--include "$INCLUDE_FILTERS"
338369
--auth "$AUTH_MODE"
339370
--parallel 1
340371
--repeat "$repeat"
@@ -344,8 +375,6 @@ jobs:
344375
--json
345376
)
346377
347-
args+=(--include "$INCLUDE_FILTERS")
348-
349378
if [[ "$MATRIX_DEEP_PROFILE" == "true" ]]; then
350379
args+=(--deep-profile)
351380
fi
@@ -365,6 +394,15 @@ jobs:
365394
exit 1
366395
fi
367396
report_md="${report_json%.json}.md"
397+
node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \
398+
--plan "$KOVA_PLAN_JSON" \
399+
--report "$report_json" \
400+
--profile "$PROFILE" \
401+
--target "local-build:${GITHUB_WORKSPACE}" \
402+
--repeat "$repeat" \
403+
--include "$INCLUDE_FILTERS" \
404+
--auth "$AUTH_MODE"
405+
368406
effective_status="$status"
369407
if [[ "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
370408
if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json"
Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
import fs from "node:fs";
2+
import path from "node:path";
3+
import { fileURLToPath } from "node:url";
4+
5+
const AUTH_MODES = new Set(["live", "mock"]);
6+
const CLI_KEYS = new Set(["auth", "include", "plan", "profile", "repeat", "report", "target"]);
7+
8+
function check(condition, reason) {
9+
if (!condition) {
10+
throw new Error(reason);
11+
}
12+
}
13+
14+
function object(value, label) {
15+
check(value !== null && typeof value === "object" && !Array.isArray(value), `invalid ${label}`);
16+
return value;
17+
}
18+
19+
function array(value, label) {
20+
check(Array.isArray(value), `invalid ${label}`);
21+
return value;
22+
}
23+
24+
function text(value, label) {
25+
check(typeof value === "string" && value.trim().length > 0, `invalid ${label}`);
26+
return value;
27+
}
28+
29+
function positiveInteger(value, label) {
30+
check(Number.isSafeInteger(value) && value > 0, `invalid ${label}`);
31+
return value;
32+
}
33+
34+
function exactStrings(value, expected, label) {
35+
const actual = array(value, label);
36+
check(
37+
actual.length === expected.length && actual.every((item, index) => item === expected[index]),
38+
`${label} did not match`,
39+
);
40+
}
41+
42+
function pairKey(scenario, state, label) {
43+
return `${text(scenario, `${label} scenario`)}\u0000${text(state, `${label} state`)}`;
44+
}
45+
46+
function displayPair(key) {
47+
return key.replace("\u0000", "/");
48+
}
49+
50+
function profileId(value, label) {
51+
return text(object(value, label).id, `${label} id`);
52+
}
53+
54+
function repeatIndexesFor(selectedPairs) {
55+
return new Map([...selectedPairs].map((key) => [key, new Set()]));
56+
}
57+
58+
function validateLiveRecord(record, key) {
59+
const auth = object(record.auth, `record ${displayPair(key)} auth`);
60+
const provider = object(record.providerEvidence, `record ${displayPair(key)} provider evidence`);
61+
check(
62+
auth.environmentDependent === true,
63+
`live record ${displayPair(key)} auth was not environment-dependent`,
64+
);
65+
check(provider.authMode === "live", `live record ${displayPair(key)} provider auth was not live`);
66+
check(
67+
provider.environmentDependent === true,
68+
`live record ${displayPair(key)} provider evidence was not environment-dependent`,
69+
);
70+
check(
71+
provider.source === "openclaw-timeline",
72+
`live record ${displayPair(key)} provider source was ${provider.source}`,
73+
);
74+
check(
75+
provider.available === true,
76+
`live record ${displayPair(key)} provider evidence was unavailable`,
77+
);
78+
positiveInteger(provider.requestCount, `live record ${displayPair(key)} provider request count`);
79+
}
80+
81+
export function validateKovaWorkflowEvidence({
82+
plan,
83+
report,
84+
profile,
85+
target,
86+
repeat,
87+
includeFilters,
88+
authMode,
89+
}) {
90+
const expectedProfile = text(profile, "expected profile");
91+
const expectedTarget = text(target, "expected target");
92+
const expectedRepeat = positiveInteger(repeat, "expected repeat");
93+
const expectedFilters = array(includeFilters, "expected include filters").map((value, index) =>
94+
text(value, `expected include filter ${index}`),
95+
);
96+
const expectedAuth = text(authMode, "expected auth mode");
97+
check(AUTH_MODES.has(expectedAuth), `unsupported expected auth mode ${expectedAuth}`);
98+
check(expectedFilters.length > 0, "expected include filters were empty");
99+
100+
const lanePlan = object(plan, "lane plan");
101+
const laneReport = object(report, "lane report");
102+
check(lanePlan.schemaVersion === "kova.matrix.plan.v1", "unexpected lane plan schema");
103+
check(laneReport.schemaVersion === "kova.report.v1", "unexpected lane report schema");
104+
check(laneReport.mode === "execution", `lane report mode was ${laneReport.mode}`);
105+
check(
106+
profileId(lanePlan.profile, "lane plan profile") === expectedProfile,
107+
"lane plan profile did not match",
108+
);
109+
check(
110+
profileId(laneReport.profile, "lane report profile") === expectedProfile,
111+
"lane report profile did not match",
112+
);
113+
check(lanePlan.target === expectedTarget, "lane plan target did not match");
114+
check(laneReport.target === expectedTarget, "lane report target did not match");
115+
116+
const planControls = object(lanePlan.controls, "lane plan controls");
117+
const reportControls = object(laneReport.controls, "lane report controls");
118+
exactStrings(planControls.include, expectedFilters, "lane plan include filters");
119+
exactStrings(reportControls.include, expectedFilters, "lane report include filters");
120+
check(planControls.repeat === expectedRepeat, "lane plan repeat did not match");
121+
check(reportControls.repeat === expectedRepeat, "lane report repeat did not match");
122+
123+
const selectedPairs = new Set();
124+
for (const [index, value] of array(lanePlan.entries, "lane plan entries").entries()) {
125+
const entry = object(value, `lane plan entry ${index}`);
126+
const status = text(entry.status, `lane plan entry ${index} status`);
127+
check(status === "SELECTED" || status === "SKIPPED", `invalid lane plan entry ${index} status`);
128+
if (status === "SKIPPED") {
129+
continue;
130+
}
131+
const key = pairKey(
132+
object(entry.scenario, `lane plan entry ${index} scenario`).id,
133+
object(entry.state, `lane plan entry ${index} state`).id,
134+
`lane plan entry ${index}`,
135+
);
136+
check(!selectedPairs.has(key), `lane plan selected duplicate ${displayPair(key)}`);
137+
selectedPairs.add(key);
138+
}
139+
check(selectedPairs.size > 0, "lane plan selected no scenario/state pairs");
140+
141+
const reportAuth = object(laneReport.auth, "lane report auth");
142+
check(reportAuth.requestedMode === expectedAuth, "lane report requested auth did not match");
143+
if (expectedAuth === "live") {
144+
check(
145+
object(reportAuth.live, "lane report live auth").environmentDependent === true,
146+
"lane report live auth was not environment-dependent",
147+
);
148+
}
149+
150+
const records = array(laneReport.records, "lane report records");
151+
const repeatIndexes = repeatIndexesFor(selectedPairs);
152+
for (const [index, value] of records.entries()) {
153+
const record = object(value, `lane report record ${index}`);
154+
const state = object(record.state, `lane report record ${index} state`);
155+
const key = pairKey(record.scenario, state.id, `lane report record ${index}`);
156+
check(selectedPairs.has(key), `lane report contained unexpected ${displayPair(key)}`);
157+
const status = text(record.status, `record ${displayPair(key)} status`);
158+
check(
159+
["BLOCKED", "FAIL", "PASS"].includes(status),
160+
`invalid record ${displayPair(key)} status`,
161+
);
162+
163+
const recordRepeat = object(record.repeat, `record ${displayPair(key)} repeat`);
164+
check(
165+
recordRepeat.total === expectedRepeat,
166+
`record ${displayPair(key)} repeat total did not match`,
167+
);
168+
const repeatIndex = positiveInteger(
169+
recordRepeat.index,
170+
`record ${displayPair(key)} repeat index`,
171+
);
172+
check(
173+
repeatIndex <= expectedRepeat,
174+
`record ${displayPair(key)} repeat index exceeded ${expectedRepeat}`,
175+
);
176+
const indexes = repeatIndexes.get(key);
177+
check(!indexes.has(repeatIndex), `record ${displayPair(key)} repeated index ${repeatIndex}`);
178+
indexes.add(repeatIndex);
179+
180+
const recordAuth = object(record.auth, `record ${displayPair(key)} auth`);
181+
check(recordAuth.mode === expectedAuth, `record ${displayPair(key)} auth mode did not match`);
182+
if (expectedAuth === "live") {
183+
validateLiveRecord(record, key);
184+
}
185+
}
186+
187+
for (const [key, indexes] of repeatIndexes) {
188+
check(
189+
indexes.size === expectedRepeat,
190+
`lane report collapsed ${displayPair(key)} coverage to ${indexes.size}/${expectedRepeat} repeats`,
191+
);
192+
for (let index = 1; index <= expectedRepeat; index += 1) {
193+
check(indexes.has(index), `lane report missed ${displayPair(key)} repeat ${index}`);
194+
}
195+
}
196+
check(
197+
records.length === selectedPairs.size * expectedRepeat,
198+
"lane report record count did not match selected coverage",
199+
);
200+
201+
return {
202+
authMode: expectedAuth,
203+
pairCount: selectedPairs.size,
204+
recordCount: records.length,
205+
repeat: expectedRepeat,
206+
};
207+
}
208+
209+
function parseCliArgs(argv) {
210+
const flags = {};
211+
for (let index = 0; index < argv.length; index += 2) {
212+
const name = argv[index];
213+
const value = argv[index + 1];
214+
check(name?.startsWith("--") && value !== undefined, "invalid CLI arguments");
215+
const key = name.slice(2);
216+
check(CLI_KEYS.has(key), `unknown --${key}`);
217+
check(!Object.hasOwn(flags, key), `duplicate --${key}`);
218+
flags[key] = value;
219+
}
220+
for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth"]) {
221+
text(flags[key], `--${key}`);
222+
}
223+
return flags;
224+
}
225+
226+
function readJson(pathValue, label) {
227+
const file = text(pathValue, label);
228+
try {
229+
return JSON.parse(fs.readFileSync(file, "utf8"));
230+
} catch (error) {
231+
throw new Error(
232+
`${label} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
233+
{ cause: error },
234+
);
235+
}
236+
}
237+
238+
function runCli() {
239+
const flags = parseCliArgs(process.argv.slice(2));
240+
const result = validateKovaWorkflowEvidence({
241+
plan: readJson(flags.plan, "--plan"),
242+
report: readJson(flags.report, "--report"),
243+
profile: flags.profile,
244+
target: flags.target,
245+
repeat: Number(flags.repeat),
246+
includeFilters: flags.include
247+
.split(",")
248+
.map((value) => value.trim())
249+
.filter(Boolean),
250+
authMode: flags.auth,
251+
});
252+
console.log(
253+
`Kova plan/report evidence validated: ${result.pairCount} scenario/state pairs x ${result.repeat} repeats (${result.authMode})`,
254+
);
255+
}
256+
257+
const modulePath = fs.realpathSync.native(fileURLToPath(import.meta.url));
258+
const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(process.argv[1])) : "";
259+
260+
if (modulePath === invokedPath) {
261+
try {
262+
runCli();
263+
} catch (error) {
264+
console.error(
265+
`Kova evidence validation failed: ${error instanceof Error ? error.message : String(error)}`,
266+
);
267+
process.exit(1);
268+
}
269+
}

0 commit comments

Comments
 (0)