Skip to content

Commit e58e524

Browse files
committed
fix(ci): classify wrapped commands the same as bare ones (#1429)
`runsProjectCode` read only the first token, so `pnpm exec node scripts/x.ts` was dropped before the bypass check ran: `pnpm` is not a package script and not `pnpm gate`, and the `node` that actually executes is the third token. An unregistered gate could therefore be wired into a qualifying lane by wrapping it, while `check:gate-manifest` stayed green — contradicting the one invariant the design rests on. Reported against 67d535d. Reproduced, and three more spellings with it: `pnpm --silent exec`, `pnpm --dir X exec`, `pnpm dlx` and `npx` all had the same effect, and a runner reached without a scripts/ path (`pnpm exec vitest …`, `node --test <file>`) was never classified as project code at all. Wrappers are now stripped before classification, by scanning for the `exec`/`dlx` keyword rather than matching a flag grammar, so `pnpm --dir x exec y` unwraps the same as `pnpm exec y`. A test runner counts as project code whatever it points at. The paired regression asserts the property directly: the bare command is one bypass, and each of five wrapped spellings is also exactly one. Re-running a suite a registered check already owns stays allowed — that is the separate, deliberate escape the device lanes use, now pinned by its own test. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01SkS4S8XXrfkJ8TD1VBKkvJ
1 parent 67d535d commit e58e524

3 files changed

Lines changed: 85 additions & 7 deletions

File tree

‎docs/agents/testing.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,10 @@ strings, and four suites (`check:tmpdir-leaks`, its model tests, `test:fixture-c
216216

217217
`pnpm check:gate-manifest` (`scripts/gate/`) answers it with one rule and one primitive.
218218

219-
The rule: **CI may invoke a gate only through `pnpm gate <check-id>`.** Every gate is a `CheckId`
219+
The rule: **CI may invoke a gate only through `pnpm gate <check-id>`.** Command wrappers
220+
are normalized before that is decided — `pnpm exec`, `pnpm dlx` and `npx` in front of a
221+
command classify the same as the command alone, since otherwise wrapping one would be
222+
enough to slip an unregistered gate past the rule. Every gate is a `CheckId`
220223
in the registry (`scripts/check-affected/checks.ts`) — the same universe the affected-selector
221224
uses — so the workflow→check mapping is a token scan over `run:` blocks and action inputs rather
222225
than an interpretation of shell. That is what makes a new gate impossible to hide: an unregistered

‎scripts/gate/audit.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,17 @@ function mapLane(
3838
return model.lanes.map((lane) => (match(lane) ? change(lane) : lane));
3939
}
4040

41+
/** Plant one command as a new step in a qualifying lane. */
42+
function plantCommand(command: string): Model {
43+
return mutate((m) => ({
44+
lanes: mapLane(
45+
m,
46+
(lane) => lane.label === 'Layering Guard',
47+
(lane) => ({ ...lane, steps: [...lane.steps, { name: 'Planted', commands: [command] }] }),
48+
),
49+
}));
50+
}
51+
4152
test('the live tree is green — every planted failure below is a real difference', () => {
4253
assert.deepEqual(messages(base), []);
4354
});
@@ -131,6 +142,50 @@ test('running a registered gate script outside the runner fails as a bypass', ()
131142
assert.match(found[0] ?? '', /pnpm gate <id>/);
132143
});
133144

145+
test('wrapping a command cannot change how it classifies', () => {
146+
// Reported on #1714: `runsProjectCode` read only the first token, so `pnpm exec`
147+
// in front of an unregistered gate dropped it before the bypass check ever ran.
148+
// The paired form is the assertion — a wrapper is not a classification.
149+
const bare = 'node --experimental-strip-types scripts/gate/check.ts';
150+
const wrapped = [
151+
`pnpm exec ${bare}`,
152+
`pnpm --silent exec ${bare}`,
153+
`pnpm --dir . exec ${bare}`,
154+
`pnpm dlx ${bare}`,
155+
`npx ${bare}`,
156+
];
157+
const bypassesFor = (command: string) =>
158+
audit(plantCommand(command)).filter((failure) => failure.assertion === 'bypass');
159+
160+
assert.equal(bypassesFor(bare).length, 1, 'the bare command must be a bypass');
161+
for (const command of wrapped) {
162+
assert.equal(bypassesFor(command).length, 1, `wrapping must not hide it: ${command}`);
163+
}
164+
});
165+
166+
test('a test runner in a lane is project code, whatever it is pointed at', () => {
167+
// Reaching a suite no check owns must fail even without a scripts/ path in the
168+
// command — a `--test` file or a Vitest project is a gate by itself.
169+
for (const command of [
170+
'pnpm exec vitest run --project not-a-real-project',
171+
'node --test test/integration/does-not-exist.test.ts',
172+
]) {
173+
const found = audit(plantCommand(command)).filter((failure) => failure.assertion === 'bypass');
174+
assert.equal(found.length, 1, `must be a bypass: ${command}`);
175+
}
176+
});
177+
178+
test('re-running a suite a registered check already owns stays allowed', () => {
179+
// The deliberate escape the device lanes rely on: replaying an owned integration
180+
// file under a live emulator hides nothing, because the suite has an owner.
181+
for (const command of [
182+
'pnpm exec vitest run --project unit-core',
183+
'node --test test/integration/smoke-cli.test.ts',
184+
]) {
185+
assert.deepEqual(messages(plantCommand(command)), [], `must stay allowed: ${command}`);
186+
}
187+
});
188+
134189
test('`pnpm gate` naming an unregistered check fails rather than resolving to nothing', () => {
135190
const model = mutate((m) => ({
136191
lanes: mapLane(

‎scripts/gate/audit.ts‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,18 +55,38 @@ function isVerbatim(segment: string, lane: Lane, model: Model): boolean {
5555
return lane.verbatim.some((name) => model.scripts[name]?.replace(/\s+/g, ' ') === normalized);
5656
}
5757

58+
const PACKAGE_MANAGERS = new Set(['pnpm', 'npm', 'yarn']);
59+
60+
/**
61+
* Strip command wrappers before classifying. `pnpm exec node scripts/x.ts` runs
62+
* exactly what `node scripts/x.ts` runs, so the two must classify identically —
63+
* otherwise wrapping a command is by itself enough to slip an unregistered gate
64+
* past the no-bypass rule, which is the one invariant this file exists to hold.
65+
*/
66+
function unwrap(segment: string): string {
67+
const parts = segment.split(/\s+/).filter(Boolean);
68+
if (parts[0] === 'npx') return parts.slice(1).join(' ');
69+
if (!PACKAGE_MANAGERS.has(parts[0] ?? '')) return segment;
70+
// Scan for the keyword rather than matching a flag grammar, so `pnpm --dir x exec y`
71+
// and `pnpm --silent exec y` unwrap the same way.
72+
const at = parts.findIndex((part) => part === 'exec' || part === 'dlx');
73+
return at === -1 ? segment : parts.slice(at + 1).join(' ');
74+
}
75+
5876
/**
5977
* Whether a segment runs this project's own code, as opposed to shell, SDK and
60-
* CLI setup. Deliberately narrow: `cp scripts/size-report.mjs …` and
78+
* CLI setup. Deliberately narrow on paths: `cp scripts/size-report.mjs …` and
6179
* `git diff -- scripts/…` name a path under scripts/ without executing anything.
6280
*/
6381
function runsProjectCode(segment: string, model: Model): boolean {
64-
const [executable = ''] = segment.split(/\s+/);
65-
const executesFile =
66-
/^(?:node|sh|bash)$/.test(executable) &&
67-
/(?:^|\s)\.?\/?(?:scripts\/\S+|src\/bin\.ts)/.test(segment);
82+
if (isGateInvocation(segment) || invokedScript(segment, model.scripts) !== null) return true;
83+
const inner = unwrap(segment);
84+
const [executable = ''] = inner.split(/\s+/);
85+
// A test runner is project code whatever it is pointed at.
86+
if (executable === 'vitest' || /(?:^|\s)--test(?:\s|$)/.test(inner)) return true;
6887
return (
69-
isGateInvocation(segment) || invokedScript(segment, model.scripts) !== null || executesFile
88+
/^(?:node|sh|bash)$/.test(executable) &&
89+
/(?:^|\s)\.?\/?(?:scripts\/\S+|src\/bin\.ts)/.test(inner)
7090
);
7191
}
7292

0 commit comments

Comments
 (0)