Skip to content

Commit 036fb8a

Browse files
cliffhallclaude
andcommitted
fix: refuse masked intermediate hops and unusable eval counts
scriptChainRuns no longer follows an intermediate segment carrying a pipe or ||, so a target reached through a masked stage does not vouch as run. skills:eval rejects a non-positive-integer CHAIN_MAX_TURNS, RUNS or CONCURRENCY, which would otherwise remove Copilot's turn cap. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 6eb00c6 commit 036fb8a

4 files changed

Lines changed: 61 additions & 0 deletions

File tree

‎scripts/lib/npm-scripts.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,10 @@ export function scriptChainRuns(scripts, entry, target) {
153153
.map((part) => unwrapGateLease(part.trim()))) {
154154
if (segment === `npm run ${target}`) return true;
155155
const tokens = segment.split(/\s+/);
156+
// An intermediate hop must be as unmasked as the target itself: in
157+
// `npm run stage || true` (or `| tee`) the stage's failure is swallowed,
158+
// so a target reached through it gates nothing (Copilot, #4895).
159+
if (segment.includes("|")) continue;
156160
if (tokens[0] === "npm" && tokens[1] === "run" && tokens[2]) {
157161
queue.push(tokens[2]);
158162
}

‎scripts/skill-eval.mjs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -786,6 +786,15 @@ async function pool(items, n, fn) {
786786
return out;
787787
}
788788

789+
/**
790+
* Whether an env-supplied count is usable as a run, turn or worker count.
791+
*
792+
* @param {number} value
793+
*/
794+
export function isPositiveInteger(value) {
795+
return Number.isInteger(value) && value > 0;
796+
}
797+
789798
/**
790799
* Whether a measured rate clears its bar.
791800
*
@@ -906,6 +915,22 @@ async function main() {
906915
);
907916
process.exit(1);
908917
}
918+
// The turn budget is also Copilot's external stop condition, so a non-numeric
919+
// value would silently remove the cap on metered calls (`turnOffset >= NaN`
920+
// is never true); RUNS and CONCURRENCY fail the same quiet way (Copilot,
921+
// #4895).
922+
for (const [name, value] of [
923+
["CHAIN_MAX_TURNS", CHAIN_MAX_TURNS],
924+
["RUNS", RUNS],
925+
["CONCURRENCY", CONCURRENCY],
926+
]) {
927+
if (!isPositiveInteger(value)) {
928+
console.error(
929+
`skills:eval — ${name} must be a positive integer (got ${process.env[name] ?? value}).`,
930+
);
931+
process.exit(1);
932+
}
933+
}
909934
if (!AGENTS.includes(AGENT)) {
910935
console.error(
911936
`skills:eval — AGENT must be one of ${AGENTS.join(", ")} (got \`${AGENT}\`).`,

‎scripts/skill-eval.test.mjs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1053,3 +1053,11 @@ test("a rejected Copilot run leaves the in-flight set", async () => {
10531053
);
10541054
assert.equal(stopLiveCopilotRuns(), 0);
10551055
});
1056+
1057+
test("isPositiveInteger accepts only usable counts (#4895)", async () => {
1058+
const { isPositiveInteger } = await import("./skill-eval.mjs");
1059+
for (const ok of [1, 3, 14]) assert.equal(isPositiveInteger(ok), true, ok);
1060+
for (const bad of [0, -1, 1.5, Number.NaN, Number("abc"), Infinity]) {
1061+
assert.equal(isPositiveInteger(bad), false, String(bad));
1062+
}
1063+
});

‎scripts/verify-skills.main.test.mjs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -746,3 +746,27 @@ test("scriptChainRuns follows npm's implicit pre/post hooks", () => {
746746
true,
747747
);
748748
});
749+
750+
test("scriptChainRuns does not follow a masked intermediate hop (#4895)", () => {
751+
const T = "verify:skills:cli";
752+
for (const gate of ["npm run stage || true", "npm run stage | tee log"]) {
753+
assert.equal(
754+
scriptChainRuns(
755+
{ "local:gate": gate, stage: `npm run ${T}` },
756+
"local:gate",
757+
T,
758+
),
759+
false,
760+
gate,
761+
);
762+
}
763+
// The unmasked hop is still followed.
764+
assert.equal(
765+
scriptChainRuns(
766+
{ "local:gate": "npm run stage", stage: `npm run ${T}` },
767+
"local:gate",
768+
T,
769+
),
770+
true,
771+
);
772+
});

0 commit comments

Comments
 (0)