Skip to content

Commit 159ca67

Browse files
committed
fix(tooling): pr:evidence measures pristine head/base worktrees from an os.tmpdir scratch; size --base gets a per-SHA lock, completion stamp, and non-destructive eviction
Review (three P1s): - pr:evidence created its scratch under an untracked .tmp/ that a fresh checkout lacks (ENOENT). Scratch now lives under os.tmpdir(), which exists by construction; a real entrypoint regression runs the whole pipeline with --base HEAD (no origin/main needed) and asserts JSON shape plus cleanup of both worktrees and the scratch. - Untracked or uncommitted production files could move the layering/depgraph numbers the block labels as HEAD's. Head is now measured from a pristine worktree of the head commit exactly like base, and the affected plan takes the head SHA (the literal HEAD folds the working tree in). The dirty flag now counts untracked files and says they are not in the block. - size --base force-pruned other cached bases without locking and trusted a dist/src that could be half-built. Per-SHA .lock (pid, O_EXCL) held from before the worktree exists until the base report is read; a live lock on the same base fails fast, a stale one is replaced; eviction skips worktrees whose lock owner is alive; dist/.size-base-complete marks a finished build. Orchestration tests run the real script against a throwaway git repo with pnpm/npm shimmed on PATH (build once, reuse, live lock, stale lock, interrupted build, guarded vs idle eviction). Also fixes the /tmp → /private/tmp realpath mismatch those tests surfaced (git lists worktrees by real path, so the registration check removed a live worktree).
1 parent ffb054e commit 159ca67

9 files changed

Lines changed: 372 additions & 57 deletions

File tree

‎docs/agents/testing.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,9 +225,15 @@ command, before the PR exists:
225225

226226
```sh
227227
pnpm size --base origin/main # first run: detached worktree + install + build of the base (~1-2 min)
228-
# later runs against the same base: ~3s (the worktree is kept under .tmp/size-base/)
228+
# later runs against the same base: ~3s (the worktree is kept under .tmp/size-base/<sha>)
229229
```
230230

231+
The cache is per SHA and never destructive toward a run in progress: a `.lock` (pid inside) is held
232+
from before the worktree exists until the base report is read, a concurrent run against the same
233+
base fails fast rather than reading a half-built `dist`, another base's run evicts only worktrees
234+
whose lock is absent or whose owner is dead, and a build that was interrupted before its
235+
`dist/.size-base-complete` stamp is rebuilt.
236+
231237
Requires a current `pnpm build` of your own tree. `JS raw`/`JS gzip` are the numbers to quote and to
232238
budget against (ADR 0019 §8 units state theirs before starting); the `npm tarball`/`npm unpacked`
233239
rows compare a fresh base checkout against your working tree, which may carry locally built helper

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@
118118
"maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts",
119119
"size": "node scripts/size-report.mjs",
120120
"pr:evidence": "node --experimental-strip-types scripts/pr-evidence/run.ts",
121-
"pr:evidence:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/pr-evidence/model.test.ts",
121+
"pr:evidence:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/pr-evidence/model.test.ts scripts/pr-evidence/run.test.ts",
122122
"perf": "node --experimental-strip-types scripts/perf/run.ts",
123123
"mutation:run": "node --experimental-strip-types scripts/mutation/run.ts",
124124
"mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run",
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
import assert from 'node:assert/strict';
2+
import fs from 'node:fs';
3+
import os from 'node:os';
4+
import path from 'node:path';
5+
import { afterAll, beforeAll, test } from 'vitest';
6+
import { runCmd, runCmdSync } from '../../src/utils/exec.ts';
7+
8+
// `pnpm size --base <ref>` orchestration against a throwaway git repository, with `pnpm` and
9+
// `npm` shimmed on PATH: the shim `pnpm build` writes dist/src and appends to a log, the shim
10+
// `npm pack` prints a fixed dry-run JSON. No install, no network; every run is git + node.
11+
12+
const ROOT = path.join(import.meta.dirname, '..', '..');
13+
const SCRIPT = path.join(ROOT, 'scripts', 'size-report.mjs');
14+
const NEVER_A_PID = 2_147_483_647; // outside every platform's pid range: dead by construction
15+
16+
let repo: string;
17+
let bin: string;
18+
let buildLog: string;
19+
let first: string;
20+
let second: string;
21+
22+
function git(args: string[], cwd = repo): string {
23+
return runCmdSync('git', args, { cwd }).stdout.trim();
24+
}
25+
26+
function writeExecutable(file: string, body: string): void {
27+
fs.writeFileSync(file, body);
28+
fs.chmodSync(file, 0o755);
29+
}
30+
31+
beforeAll(() => {
32+
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'size-report-base-'));
33+
repo = path.join(scratch, 'repo');
34+
bin = path.join(scratch, 'bin');
35+
buildLog = path.join(scratch, 'build.log');
36+
fs.mkdirSync(repo);
37+
fs.mkdirSync(bin);
38+
writeExecutable(
39+
path.join(bin, 'pnpm'),
40+
`#!/bin/sh
41+
echo "$PWD $*" >> "${buildLog}"
42+
if [ "$1" = "build" ]; then mkdir -p dist/src && printf 'export const built = 1;\\n' > dist/src/index.js; fi
43+
`,
44+
);
45+
writeExecutable(
46+
path.join(bin, 'npm'),
47+
`#!/bin/sh
48+
echo '[{"filename":"pkg.tgz","size":100,"unpackedSize":200,"entryCount":2}]'
49+
`,
50+
);
51+
git(['init', '-q', '-b', 'main']);
52+
git(['config', 'user.email', 'size@test']);
53+
git(['config', 'user.name', 'size test']);
54+
fs.writeFileSync(path.join(repo, 'package.json'), '{"name":"probe","version":"1.0.0"}\n');
55+
fs.writeFileSync(path.join(repo, '.gitignore'), '.tmp/\ndist/\n');
56+
git(['add', '.']);
57+
git(['commit', '-q', '-m', 'first']);
58+
first = git(['rev-parse', 'HEAD']);
59+
fs.writeFileSync(path.join(repo, 'README.md'), 'second\n');
60+
git(['add', '.']);
61+
git(['commit', '-q', '-m', 'second']);
62+
second = git(['rev-parse', 'HEAD']);
63+
// The head side of the comparison needs a dist too.
64+
fs.mkdirSync(path.join(repo, 'dist', 'src'), { recursive: true });
65+
fs.writeFileSync(path.join(repo, 'dist', 'src', 'index.js'), 'export const head = 1;\n');
66+
});
67+
68+
afterAll(() => {
69+
fs.rmSync(path.dirname(repo), { recursive: true, force: true });
70+
});
71+
72+
async function size(base: string) {
73+
return await runCmd(process.execPath, [SCRIPT, '--cwd', repo, '--base', base], {
74+
cwd: repo,
75+
env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ''}` },
76+
allowFailure: true,
77+
timeoutMs: 60_000,
78+
});
79+
}
80+
81+
const worktreeOf = (sha: string) => path.join(repo, '.tmp', 'size-base', sha.slice(0, 12));
82+
const lockOf = (sha: string) => `${worktreeOf(sha)}.lock`;
83+
const stampOf = (sha: string) => path.join(worktreeOf(sha), 'dist', '.size-base-complete');
84+
const builds = () =>
85+
fs
86+
.readFileSync(buildLog, 'utf8')
87+
.split('\n')
88+
.filter((l) => l.endsWith(' build'));
89+
90+
test('first run builds the base in a per-SHA worktree, stamps it, releases its lock; second run reuses it', async () => {
91+
const one = await size(first);
92+
assert.equal(one.exitCode, 0, one.stderr);
93+
assert.match(one.stdout, /\| JS raw \|/);
94+
assert.ok(fs.existsSync(stampOf(first)), 'completion stamp written after build');
95+
assert.equal(fs.existsSync(lockOf(first)), false, 'lock released after the report was read');
96+
assert.equal(builds().length, 1);
97+
98+
const two = await size(first);
99+
assert.equal(two.exitCode, 0, two.stderr);
100+
assert.equal(builds().length, 1, 'a stamped base is not rebuilt');
101+
});
102+
103+
test('a base whose lock is held by a live pid fails fast without touching its worktree', async () => {
104+
fs.writeFileSync(lockOf(first), `${process.pid}\n`); // this test process: alive
105+
const before = fs.statSync(stampOf(first)).mtimeMs;
106+
const result = await size(first);
107+
assert.notEqual(result.exitCode, 0);
108+
assert.match(
109+
result.stderr,
110+
new RegExp(`another \`size --base\` \\(pid ${process.pid}\\) is building`),
111+
);
112+
assert.equal(fs.statSync(stampOf(first)).mtimeMs, before);
113+
assert.equal(builds().length, 1);
114+
fs.rmSync(lockOf(first));
115+
});
116+
117+
test('a stale lock (dead pid) is replaced and the run proceeds', async () => {
118+
fs.writeFileSync(lockOf(first), `${NEVER_A_PID}\n`);
119+
const result = await size(first);
120+
assert.equal(result.exitCode, 0, result.stderr);
121+
assert.equal(fs.existsSync(lockOf(first)), false);
122+
});
123+
124+
test('an unstamped worktree (interrupted build) is rebuilt rather than trusted', async () => {
125+
fs.rmSync(stampOf(first));
126+
const result = await size(first);
127+
assert.equal(result.exitCode, 0, result.stderr);
128+
assert.equal(builds().length, 2, 'dist/src existing without the stamp is not enough');
129+
assert.ok(fs.existsSync(stampOf(first)));
130+
});
131+
132+
test('measuring another base evicts an idle cached base but never one whose lock is live', async () => {
133+
fs.writeFileSync(lockOf(first), `${process.pid}\n`); // in use by "another run"
134+
const guarded = await size(second);
135+
assert.equal(guarded.exitCode, 0, guarded.stderr);
136+
assert.ok(fs.existsSync(worktreeOf(first)), 'a live-locked worktree survives eviction');
137+
assert.ok(fs.existsSync(stampOf(second)));
138+
fs.rmSync(lockOf(first));
139+
140+
const evicting = await size(first);
141+
assert.equal(evicting.exitCode, 0, evicting.stderr);
142+
assert.equal(fs.existsSync(worktreeOf(second)), false, 'an idle other base is evicted');
143+
assert.equal(
144+
git(['worktree', 'list', '--porcelain']).includes(worktreeOf(second)),
145+
false,
146+
'and unregistered from git',
147+
);
148+
});

‎scripts/pr-evidence/model.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,10 @@ test('a dirty tree is called out and a fail-open plan is summarized, not enumera
141141
depgraph: { ...inputs().depgraph, base: undefined },
142142
}),
143143
);
144-
assert.match(block, /working tree dirty: describes the tree, not the head/);
144+
assert.match(
145+
block,
146+
/working tree has uncommitted\/untracked changes: none of them are in this block/,
147+
);
145148
assert.match(
146149
block,
147150
/fail-open \(workflow-tooling\): full set, 1 local \+ 1 GitHub-authoritative/,

‎scripts/pr-evidence/model.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export type GitFacts = Readonly<{
1010
base: string;
1111
baseRef: string;
1212
baseShort: string;
13-
/** Uncommitted changes exist: the block describes the tree, not the head. */
13+
/** Uncommitted or untracked changes exist; everything measured here is the head, not them. */
1414
dirty: boolean;
1515
changedFiles: readonly string[];
1616
}>;
@@ -125,7 +125,9 @@ export function renderEvidence(inputs: EvidenceInputs): string {
125125
const lines = [
126126
`<!-- pr-evidence base=${git.base} head=${git.head} -->`,
127127
`**Evidence** gathered ${inputs.generatedAt} at \`${git.headShort}\` (\`${git.branch}\`) against \`${git.baseRef}\` @ \`${git.baseShort}\`` +
128-
(git.dirty ? ' — **working tree dirty: describes the tree, not the head**' : ''),
128+
(git.dirty
129+
? ' — **working tree has uncommitted/untracked changes: none of them are in this block**'
130+
: ''),
129131
bullet(`Changed: ${git.changedFiles.length} files (${areas || 'none'})`),
130132
bullet(`Affected gates (\`check:affected\`): ${affectedLine}`),
131133
bullet(

‎scripts/pr-evidence/run.test.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import assert from 'node:assert/strict';
2+
import fs from 'node:fs';
3+
import os from 'node:os';
4+
import path from 'node:path';
5+
import { test } from 'node:test';
6+
import { runCmd, runCmdSync } from '../../src/utils/exec.ts';
7+
8+
const REPOSITORY_ROOT = path.resolve(import.meta.dirname, '..', '..');
9+
const RUN = path.join(REPOSITORY_ROOT, 'scripts', 'pr-evidence', 'run.ts');
10+
11+
function scratchDirectories(): string[] {
12+
return fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith('agent-device-pr-evidence-'));
13+
}
14+
15+
function evidenceWorktrees(): string[] {
16+
return runCmdSync('git', ['worktree', 'list', '--porcelain'], { cwd: REPOSITORY_ROOT })
17+
.stdout.split('\n')
18+
.filter((line) => line.startsWith('worktree ') && line.includes('agent-device-pr-evidence-'));
19+
}
20+
21+
// The real entrypoint, end to end, in this repository: `--base HEAD` makes the merge-base HEAD
22+
// itself, so it needs no origin/main and no network (a depth-1 CI checkout is enough) while
23+
// still creating both pristine worktrees, running the selector, the layering guard, and the
24+
// depgraph twice, and rendering the block. It is the regression for the fresh-checkout failure
25+
// (scratch used to be created under an untracked, possibly absent `.tmp/`) and for cleanup.
26+
test('pr:evidence runs end to end from a pristine head worktree and cleans up after itself', async () => {
27+
const before = { scratch: scratchDirectories(), worktrees: evidenceWorktrees() };
28+
const result = await runCmd(
29+
process.execPath,
30+
['--experimental-strip-types', RUN, '--base', 'HEAD', '--json'],
31+
{ cwd: REPOSITORY_ROOT, timeoutMs: 300_000 },
32+
);
33+
const inputs = JSON.parse(result.stdout) as {
34+
git: { head: string; base: string; changedFiles: string[]; dirty: boolean };
35+
affected: { checks: unknown[] };
36+
layering: { ok: boolean };
37+
depgraph: { head: { files: number; edges: number }; base: { files: number; edges: number } };
38+
coverage: { kind: string };
39+
size: { kind: string };
40+
};
41+
const head = runCmdSync('git', ['rev-parse', 'HEAD'], { cwd: REPOSITORY_ROOT }).stdout.trim();
42+
assert.equal(inputs.git.head, head);
43+
assert.equal(inputs.git.base, head, '--base HEAD makes the merge-base the head itself');
44+
assert.deepEqual(inputs.git.changedFiles, []);
45+
assert.ok(inputs.depgraph.head.files > 500, 'the head worktree was analyzed, not an empty tree');
46+
assert.deepEqual(inputs.depgraph.base, inputs.depgraph.head, 'same commit, same numbers');
47+
assert.equal(typeof inputs.layering.ok, 'boolean');
48+
assert.equal(inputs.coverage.kind, 'skipped');
49+
assert.equal(inputs.size.kind, 'skipped');
50+
// Both worktrees and the os.tmpdir() scratch are gone, whatever else was there before.
51+
assert.deepEqual(scratchDirectories(), before.scratch);
52+
assert.deepEqual(evidenceWorktrees(), before.worktrees);
53+
});

‎scripts/pr-evidence/run.ts‎

Lines changed: 39 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,22 @@
22
//
33
// One paste-ready evidence block for a PR body, stamped with the exact base and head, composed
44
// from the tools the repo already has: the affected selector (`check:affected --json`), the
5-
// layering guard, the depgraph report (head, and base through a throwaway worktree so the
6-
// dependency-edge delta is real), and — behind flags, because they need a build or a coverage
7-
// run — the changed-line coverage gate and `pnpm size --base`. It measures nothing itself and
8-
// claims nothing about CI: the last line is the link to the head's checks.
5+
// layering guard, the depgraph report, and — behind flags, because they need a build or a
6+
// coverage run — the changed-line coverage gate and `pnpm size --base`. It measures nothing
7+
// itself and claims nothing about CI: the last line is the link to the head's checks.
8+
//
9+
// Everything labelled "at <head>" is measured from a throwaway `git worktree` of that exact
10+
// commit, and the base likewise, so an untracked or uncommitted file in the working tree can
11+
// change nothing the block reports as HEAD's; the working tree only contributes the "dirty"
12+
// flag. The worktrees need no install: the scripts analyze whichever repository their cwd is
13+
// in, while their imports resolve from this checkout.
914
//
1015
// The default tier finishes in ~20s (the layering guard is most of it). `--coverage` reads the
1116
// existing coverage/lcov.info (run `pnpm test:coverage` first); `--size` runs the base worktree
1217
// build the first time (~1-2 min) and ~3s afterwards.
1318

1419
import fs from 'node:fs';
20+
import os from 'node:os';
1521
import path from 'node:path';
1622
import { pathToFileURL } from 'node:url';
1723
import { runCmd, runCmdSync } from '../../src/utils/exec.ts';
@@ -46,7 +52,8 @@ function collectGitFacts(baseRef: string): GitFacts {
4652
const head = git(['rev-parse', 'HEAD']);
4753
const base = git(['merge-base', baseRef, 'HEAD']);
4854
const branch = git(['rev-parse', '--abbrev-ref', 'HEAD']);
49-
const dirty = git(['status', '--porcelain', '--untracked-files=no']).length > 0;
55+
// Untracked files count: they are exactly what a pristine-worktree measurement leaves out.
56+
const dirty = git(['status', '--porcelain']).length > 0;
5057
const changedFiles = git(['diff', '--name-only', '--no-renames', `${base}..HEAD`])
5158
.split('\n')
5259
.filter(Boolean);
@@ -62,7 +69,9 @@ function collectGitFacts(baseRef: string): GitFacts {
6269
};
6370
}
6471

65-
async function collectAffected(base: string): Promise<AffectedPlan> {
72+
// The head SHA, not the literal `HEAD`: the selector folds working-tree changes into a plan for
73+
// `HEAD`, and this block describes the commit.
74+
async function collectAffected(base: string, head: string): Promise<AffectedPlan> {
6675
const result = await runCmd(
6776
process.execPath,
6877
[
@@ -71,7 +80,7 @@ async function collectAffected(base: string): Promise<AffectedPlan> {
7180
'--base',
7281
base,
7382
'--head',
74-
'HEAD',
83+
head,
7584
'--json',
7685
],
7786
{ cwd: repoRoot, timeoutMs: 120_000 },
@@ -84,18 +93,15 @@ async function collectAffected(base: string): Promise<AffectedPlan> {
8493
};
8594
}
8695

87-
async function collectLayering() {
96+
async function collectLayering(cwd: string) {
8897
const result = await runCmd(
8998
process.execPath,
9099
['--experimental-strip-types', path.join(scripts, 'layering', 'check.ts')],
91-
{ cwd: repoRoot, timeoutMs: 300_000, allowFailure: true },
100+
{ cwd, timeoutMs: 300_000, allowFailure: true },
92101
);
93102
return parseLayeringReport(`${result.stdout}\n${result.stderr}`, result.exitCode);
94103
}
95104

96-
// The depgraph script analyzes whichever repository its cwd is inside, while its imports resolve
97-
// from this checkout, so a bare `git worktree add` (no install) of the base is enough for the
98-
// base numbers to come from the same instrument as the head numbers.
99105
async function collectDepgraph(cwd: string, out: string): Promise<DepgraphFacts> {
100106
await runCmd(
101107
process.execPath,
@@ -105,14 +111,15 @@ async function collectDepgraph(cwd: string, out: string): Promise<DepgraphFacts>
105111
return depgraphFacts(JSON.parse(fs.readFileSync(out, 'utf8')));
106112
}
107113

108-
async function collectBaseDepgraph(base: string, scratch: string): Promise<DepgraphFacts> {
109-
const worktree = path.join(scratch, 'base');
110-
git(['worktree', 'add', '--detach', worktree, base]);
111-
try {
112-
return await collectDepgraph(worktree, path.join(scratch, 'depgraph-base.json'));
113-
} finally {
114-
git(['worktree', 'remove', '--force', worktree]);
115-
}
114+
/** A pristine checkout of one commit; removed by the caller's scratch cleanup. */
115+
function addWorktree(scratch: string, name: string, commit: string): string {
116+
const worktree = path.join(scratch, name);
117+
git(['worktree', 'add', '--detach', worktree, commit]);
118+
return worktree;
119+
}
120+
121+
function removeWorktree(worktree: string): void {
122+
git(['worktree', 'remove', '--force', worktree]);
116123
}
117124

118125
async function collectCoverage(base: string): Promise<EvidenceInputs['coverage']> {
@@ -162,13 +169,19 @@ async function main(argv: readonly string[]): Promise<number> {
162169
});
163170
const baseRef = values.base ?? 'origin/main';
164171
const gitFacts = collectGitFacts(baseRef);
165-
const scratch = fs.mkdtempSync(path.join(repoRoot, '.tmp', 'pr-evidence-'));
172+
// os.tmpdir() always exists; a repo-local scratch would have to be created first and is one
173+
// more thing a fresh checkout can lack.
174+
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-pr-evidence-'));
175+
const worktrees: string[] = [];
166176
try {
177+
const headTree = addWorktree(scratch, 'head', gitFacts.head);
178+
const baseTree = addWorktree(scratch, 'base', gitFacts.base);
179+
worktrees.push(headTree, baseTree);
167180
const [affected, layering, head, base] = await Promise.all([
168-
collectAffected(gitFacts.base),
169-
collectLayering(),
170-
collectDepgraph(repoRoot, path.join(scratch, 'depgraph-head.json')),
171-
collectBaseDepgraph(gitFacts.base, scratch),
181+
collectAffected(gitFacts.base, gitFacts.head),
182+
collectLayering(headTree),
183+
collectDepgraph(headTree, path.join(scratch, 'depgraph-head.json')),
184+
collectDepgraph(baseTree, path.join(scratch, 'depgraph-base.json')),
172185
]);
173186
const inputs: EvidenceInputs = {
174187
generatedAt: new Date().toISOString(),
@@ -185,6 +198,7 @@ async function main(argv: readonly string[]): Promise<number> {
185198
);
186199
return 0;
187200
} finally {
201+
for (const worktree of worktrees) removeWorktree(worktree);
188202
fs.rmSync(scratch, { recursive: true, force: true });
189203
}
190204
}

0 commit comments

Comments
 (0)