Skip to content

Commit 293348b

Browse files
vincentkocvincentkoc
committed
fix(plugins): prebuild private qa for gauntlet
1 parent 8e5fcff commit 293348b

3 files changed

Lines changed: 184 additions & 19 deletions

File tree

‎scripts/check-plugin-gateway-gauntlet.mjs‎

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import os from "node:os";
66
import path from "node:path";
77
import process from "node:process";
88
import {
9+
buildGauntletPrebuildEnv,
910
collectGatewayCpuObservations,
1011
collectMetricObservations,
12+
collectQaBaselineRegressionObservations,
1113
discoverBundledPluginManifests,
1214
selectPluginEntries,
1315
} from "./lib/plugin-gateway-gauntlet.mjs";
@@ -39,6 +41,7 @@ function parseArgs(argv) {
3941
skipPrebuild: false,
4042
skipLifecycle: false,
4143
skipQa: false,
44+
qaBaseline: false,
4245
skipSlashHelp: false,
4346
qaScenarios: [],
4447
qaPluginChunkSize: DEFAULT_QA_PLUGIN_CHUNK_SIZE,
@@ -47,6 +50,8 @@ function parseArgs(argv) {
4750
maxRssWarnMb: DEFAULT_MAX_RSS_WARN_MB,
4851
wallAnomalyMultiplier: 3,
4952
rssAnomalyMultiplier: 2.5,
53+
qaCpuRegressionMultiplier: 2,
54+
qaWallRegressionMultiplier: 2,
5055
commandTimeoutMs: 120_000,
5156
buildTimeoutMs: 600_000,
5257
qaTimeoutMs: 900_000,
@@ -90,6 +95,9 @@ function parseArgs(argv) {
9095
case "--qa-plugin-chunk-size":
9196
options.qaPluginChunkSize = parsePositiveInt(readValue(), "--qa-plugin-chunk-size");
9297
break;
98+
case "--qa-baseline":
99+
options.qaBaseline = true;
100+
break;
93101
case "--cpu-core-warn":
94102
options.cpuCoreWarn = parsePositiveNumber(readValue(), "--cpu-core-warn");
95103
break;
@@ -108,6 +116,18 @@ function parseArgs(argv) {
108116
case "--rss-anomaly-multiplier":
109117
options.rssAnomalyMultiplier = parsePositiveNumber(readValue(), "--rss-anomaly-multiplier");
110118
break;
119+
case "--qa-cpu-regression-multiplier":
120+
options.qaCpuRegressionMultiplier = parsePositiveNumber(
121+
readValue(),
122+
"--qa-cpu-regression-multiplier",
123+
);
124+
break;
125+
case "--qa-wall-regression-multiplier":
126+
options.qaWallRegressionMultiplier = parsePositiveNumber(
127+
readValue(),
128+
"--qa-wall-regression-multiplier",
129+
);
130+
break;
111131
case "--command-timeout-ms":
112132
options.commandTimeoutMs = parsePositiveInt(readValue(), "--command-timeout-ms");
113133
break;
@@ -156,6 +176,7 @@ Options:
156176
--output-dir <path> Artifact directory
157177
--qa-scenario <id> QA Lab scenario id, repeatable
158178
--qa-plugin-chunk-size <count> Plugins enabled per QA run (default: 12)
179+
--qa-baseline Run a no-extra-plugin QA baseline before plugin chunks
159180
--cpu-core-warn <ratio> Hot CPU threshold (default: 0.9)
160181
--hot-wall-warn-ms <ms> Minimum wall time for hot CPU observations (default: 30000)
161182
--max-rss-warn-mb <mb> Maximum RSS warning threshold (default: 1536)
@@ -416,19 +437,22 @@ function runSlashHelpProbes(params) {
416437
}
417438

418439
function runQaChunks(params) {
419-
const chunks = chunkArray(params.plugins, params.qaPluginChunkSize);
440+
const chunks = [
441+
...(params.qaBaseline ? [{ label: "baseline", plugins: [] }] : []),
442+
...chunkArray(params.plugins, params.qaPluginChunkSize).map((plugins, index) => ({
443+
label: `chunk-${String(index).padStart(2, "0")}`,
444+
plugins,
445+
})),
446+
];
420447
const summaries = [];
421448
for (let index = 0; index < chunks.length; index += 1) {
422449
const chunk = chunks[index];
423-
const outputDir = path.join(
424-
params.outputDir,
425-
"qa-suite",
426-
`chunk-${String(index).padStart(2, "0")}`,
427-
);
450+
const outputDir = path.join(params.outputDir, "qa-suite", chunk.label);
428451
const outputArg = toRepoRelativePath(params.repoRoot, outputDir);
429-
const pluginIds = chunk.map((plugin) => plugin.id);
452+
const pluginIds = chunk.plugins.map((plugin) => plugin.id);
453+
const pluginIdLabel = pluginIds.length > 0 ? pluginIds.join(",") : "<baseline>";
430454
process.stderr.write(
431-
`[plugin-gauntlet] qa chunk ${index + 1}/${chunks.length}: ${pluginIds.join(",")}\n`,
455+
`[plugin-gauntlet] qa chunk ${index + 1}/${chunks.length}: ${pluginIdLabel}\n`,
432456
);
433457
const row = runMeasuredCommand({
434458
cwd: params.repoRoot,
@@ -446,14 +470,21 @@ function runQaChunks(params) {
446470
...params.qaScenarios.flatMap((scenario) => ["--scenario", scenario]),
447471
...pluginIds.flatMap((pluginId) => ["--enable-plugin", pluginId]),
448472
]),
449-
label: `qa-chunk-${String(index).padStart(2, "0")}`,
473+
label: `qa-${chunk.label}`,
450474
phase: "qa:rpc",
451475
timeoutMs: params.qaTimeoutMs,
452476
});
453-
params.rows.push({ ...row, pluginId: pluginIds.join(",") });
454477
const summaryPath = path.join(outputDir, "qa-suite-summary.json");
478+
const qaSummary = fs.existsSync(summaryPath)
479+
? JSON.parse(fs.readFileSync(summaryPath, "utf8"))
480+
: null;
481+
params.rows.push({
482+
...row,
483+
pluginId: pluginIdLabel,
484+
...(qaSummary?.metrics ? { qaMetrics: qaSummary.metrics } : {}),
485+
});
455486
if (fs.existsSync(summaryPath)) {
456-
summaries.push(JSON.parse(fs.readFileSync(summaryPath, "utf8")));
487+
summaries.push(qaSummary);
457488
}
458489
}
459490
return summaries;
@@ -478,7 +509,7 @@ async function main() {
478509
rows.push(
479510
runMeasuredCommand({
480511
cwd: repoRoot,
481-
env,
512+
env: buildGauntletPrebuildEnv(env, { includePrivateQa: !options.skipQa }),
482513
logDir: path.join(options.outputDir, "logs", "prebuild"),
483514
command: pnpmCommand(),
484515
args: ["build"],
@@ -519,6 +550,7 @@ async function main() {
519550
outputDir: options.outputDir,
520551
env,
521552
plugins: selectedPlugins,
553+
qaBaseline: options.qaBaseline,
522554
rows,
523555
qaScenarios: options.qaScenarios,
524556
qaPluginChunkSize: options.qaPluginChunkSize,
@@ -531,6 +563,10 @@ async function main() {
531563
wallAnomalyMultiplier: options.wallAnomalyMultiplier,
532564
rssAnomalyMultiplier: options.rssAnomalyMultiplier,
533565
});
566+
const qaBaselineObservations = collectQaBaselineRegressionObservations(rows, {
567+
cpuRegressionMultiplier: options.qaCpuRegressionMultiplier,
568+
wallRegressionMultiplier: options.qaWallRegressionMultiplier,
569+
});
534570
const gatewayObservations = qaSummaries.flatMap((qa) =>
535571
collectGatewayCpuObservations({
536572
startup: null,
@@ -554,6 +590,7 @@ async function main() {
554590
limit: options.limit ?? null,
555591
qaScenarios: options.qaScenarios,
556592
qaPluginChunkSize: options.qaPluginChunkSize,
593+
qaBaseline: options.qaBaseline,
557594
skipLifecycle: options.skipLifecycle,
558595
skipQa: options.skipQa,
559596
skipSlashHelp: options.skipSlashHelp,
@@ -564,12 +601,14 @@ async function main() {
564601
maxRssWarnMb: options.maxRssWarnMb,
565602
wallAnomalyMultiplier: options.wallAnomalyMultiplier,
566603
rssAnomalyMultiplier: options.rssAnomalyMultiplier,
604+
qaCpuRegressionMultiplier: options.qaCpuRegressionMultiplier,
605+
qaWallRegressionMultiplier: options.qaWallRegressionMultiplier,
567606
},
568607
},
569608
matrix,
570609
selectedPlugins,
571610
rows,
572-
observations: [...metricObservations, ...gatewayObservations],
611+
observations: [...metricObservations, ...qaBaselineObservations, ...gatewayObservations],
573612
failures,
574613
};
575614
const summaryPath = path.join(options.outputDir, "plugin-gateway-gauntlet-summary.json");

‎scripts/lib/plugin-gateway-gauntlet.mjs‎

Lines changed: 74 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -216,18 +216,26 @@ function collectMetricObservations(rows, thresholds = {}) {
216216
const wallMedianMs = median(phaseRows.map((row) => row.wallMs));
217217
const rssMedianMb = median(phaseRows.map((row) => row.maxRssMb));
218218
for (const row of phaseRows) {
219+
const cpuCoreRatio =
220+
phase === "qa:rpc" && typeof row.qaMetrics?.gatewayCpuCoreRatio === "number"
221+
? row.qaMetrics.gatewayCpuCoreRatio
222+
: row.cpuCoreRatio;
223+
const wallMs =
224+
phase === "qa:rpc" && typeof row.qaMetrics?.wallMs === "number"
225+
? row.qaMetrics.wallMs
226+
: row.wallMs;
219227
if (
220-
typeof row.cpuCoreRatio === "number" &&
221-
typeof row.wallMs === "number" &&
222-
row.cpuCoreRatio >= cpuCoreWarn &&
223-
row.wallMs >= hotWallWarnMs
228+
typeof cpuCoreRatio === "number" &&
229+
typeof wallMs === "number" &&
230+
cpuCoreRatio >= cpuCoreWarn &&
231+
wallMs >= hotWallWarnMs
224232
) {
225233
observations.push({
226234
kind: "phase-cpu-hot",
227235
pluginId: row.pluginId ?? null,
228236
phase,
229-
cpuCoreRatio: row.cpuCoreRatio,
230-
wallMs: row.wallMs,
237+
cpuCoreRatio,
238+
wallMs,
231239
});
232240
}
233241
if (
@@ -279,6 +287,64 @@ function collectMetricObservations(rows, thresholds = {}) {
279287
return observations;
280288
}
281289

290+
function collectQaBaselineRegressionObservations(rows, thresholds = {}) {
291+
const baselinePluginId = thresholds.baselinePluginId ?? "<baseline>";
292+
const cpuRegressionMultiplier = thresholds.cpuRegressionMultiplier ?? 2;
293+
const wallRegressionMultiplier = thresholds.wallRegressionMultiplier ?? 2;
294+
const baseline = rows.find((row) => row.phase === "qa:rpc" && row.pluginId === baselinePluginId);
295+
const baselineMetrics = baseline?.qaMetrics;
296+
if (!baselineMetrics) {
297+
return [];
298+
}
299+
const observations = [];
300+
for (const row of rows) {
301+
if (row.phase !== "qa:rpc" || row.pluginId === baselinePluginId || !row.qaMetrics) {
302+
continue;
303+
}
304+
if (
305+
typeof baselineMetrics.gatewayCpuCoreRatio === "number" &&
306+
baselineMetrics.gatewayCpuCoreRatio > 0 &&
307+
typeof row.qaMetrics.gatewayCpuCoreRatio === "number" &&
308+
row.qaMetrics.gatewayCpuCoreRatio >=
309+
baselineMetrics.gatewayCpuCoreRatio * cpuRegressionMultiplier
310+
) {
311+
observations.push({
312+
kind: "qa-baseline-cpu-regression",
313+
pluginId: row.pluginId ?? null,
314+
cpuCoreRatio: row.qaMetrics.gatewayCpuCoreRatio,
315+
baselineCpuCoreRatio: baselineMetrics.gatewayCpuCoreRatio,
316+
multiplier: cpuRegressionMultiplier,
317+
});
318+
}
319+
if (
320+
typeof baselineMetrics.wallMs === "number" &&
321+
baselineMetrics.wallMs > 0 &&
322+
typeof row.qaMetrics.wallMs === "number" &&
323+
row.qaMetrics.wallMs >= baselineMetrics.wallMs * wallRegressionMultiplier
324+
) {
325+
observations.push({
326+
kind: "qa-baseline-wall-regression",
327+
pluginId: row.pluginId ?? null,
328+
wallMs: row.qaMetrics.wallMs,
329+
baselineWallMs: baselineMetrics.wallMs,
330+
multiplier: wallRegressionMultiplier,
331+
});
332+
}
333+
}
334+
return observations;
335+
}
336+
337+
function buildGauntletPrebuildEnv(env, options = {}) {
338+
if (!options.includePrivateQa) {
339+
return env;
340+
}
341+
return {
342+
...env,
343+
OPENCLAW_BUILD_PRIVATE_QA: "1",
344+
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1",
345+
};
346+
}
347+
282348
function collectGatewayCpuObservations(params) {
283349
const observations = [];
284350
for (const result of params.startup?.results ?? []) {
@@ -318,8 +384,10 @@ function collectGatewayCpuObservations(params) {
318384

319385
export {
320386
collectCommandAliasRecords,
387+
collectQaBaselineRegressionObservations,
321388
collectGatewayCpuObservations,
322389
collectMetricObservations,
390+
buildGauntletPrebuildEnv,
323391
discoverBundledPluginManifests,
324392
schemaHasRequiredFields,
325393
selectPluginEntries,

‎test/scripts/plugin-gateway-gauntlet.test.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ import os from "node:os";
33
import path from "node:path";
44
import { afterEach, beforeEach, describe, expect, it } from "vitest";
55
import {
6+
buildGauntletPrebuildEnv,
67
collectGatewayCpuObservations,
78
collectMetricObservations,
9+
collectQaBaselineRegressionObservations,
810
discoverBundledPluginManifests,
911
schemaHasRequiredFields,
1012
selectPluginEntries,
@@ -170,4 +172,60 @@ describe("plugin gateway gauntlet helpers", () => {
170172
"phase-rss-anomaly",
171173
]);
172174
});
175+
176+
it("uses QA gateway metrics instead of source CLI wrapper CPU for QA hot observations", () => {
177+
const observations = collectMetricObservations(
178+
[
179+
{
180+
pluginId: "browser,memory-core",
181+
phase: "qa:rpc",
182+
wallMs: 40_000,
183+
cpuCoreRatio: 1.2,
184+
qaMetrics: {
185+
wallMs: 25_000,
186+
gatewayCpuCoreRatio: 0.42,
187+
},
188+
},
189+
],
190+
{
191+
cpuCoreWarn: 0.9,
192+
hotWallWarnMs: 30_000,
193+
},
194+
);
195+
196+
expect(observations).toEqual([]);
197+
});
198+
199+
it("flags QA gateway regressions relative to an explicit baseline", () => {
200+
expect(
201+
collectQaBaselineRegressionObservations(
202+
[
203+
{
204+
pluginId: "<baseline>",
205+
phase: "qa:rpc",
206+
qaMetrics: { wallMs: 20_000, gatewayCpuCoreRatio: 0.25 },
207+
},
208+
{
209+
pluginId: "browser,memory-core",
210+
phase: "qa:rpc",
211+
qaMetrics: { wallMs: 45_000, gatewayCpuCoreRatio: 0.6 },
212+
},
213+
],
214+
{
215+
cpuRegressionMultiplier: 2,
216+
wallRegressionMultiplier: 2,
217+
},
218+
).map((observation) => observation.kind),
219+
).toEqual(["qa-baseline-cpu-regression", "qa-baseline-wall-regression"]);
220+
});
221+
222+
it("prebuilds private QA dist when QA chunks are enabled", () => {
223+
expect(buildGauntletPrebuildEnv({ EXISTING: "1" }, { includePrivateQa: true })).toEqual({
224+
EXISTING: "1",
225+
OPENCLAW_BUILD_PRIVATE_QA: "1",
226+
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1",
227+
});
228+
const env = { EXISTING: "1" };
229+
expect(buildGauntletPrebuildEnv(env, { includePrivateQa: false })).toBe(env);
230+
});
173231
});

0 commit comments

Comments
 (0)