Skip to content

Commit 48761c7

Browse files
committed
fix(tooling): symlink-identity locks with compare-then-unlink; evict under the victim's lock; pr:evidence registers worktrees on add and cleans up exhaustively
Review (three P1s): - Lock creation/takeover races: the lock is now a symlink whose target is the owner identity (pid:nonce), created with its identity in one syscall (no empty-file window), taken over only by compare-then-unlink on the exact identity judged stale, and verified after creation; release unlinks only a link that still names this run. Real overlapping-process tests: two runs on one base (exactly one builds, the other fails fast), and a takeover race against a simulated other taker across delays straddling the acquire window (a live lock is never unlinked, both never proceed). - Cross-base eviction: a victim is removed only while holding its own lock, acquired through the same path, so a run wanting it after the check finds it locked rather than half-removed; a live-locked victim is skipped. - pr:evidence worktrees: withWorktrees registers each worktree the moment its add succeeds and sweeps every resource on the way out, collecting failures instead of stopping at the first; planted reds for both (second add fails → first removed; removal of the middle one throws → the others still go).
1 parent bd9b37e commit 48761c7

6 files changed

Lines changed: 406 additions & 73 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@
118118
"maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts",
119119
"size": "node scripts/size-report.mjs",
120120
"pr:evidence": "node --experimental-strip-types scripts/pr-evidence/run.ts",
121-
"pr:evidence:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/pr-evidence/model.test.ts scripts/pr-evidence/run.test.ts",
121+
"pr:evidence:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/pr-evidence/model.test.ts scripts/pr-evidence/worktrees.test.ts scripts/pr-evidence/run.test.ts",
122122
"perf": "node --experimental-strip-types scripts/perf/run.ts",
123123
"mutation:run": "node --experimental-strip-types scripts/mutation/run.ts",
124124
"mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run",

‎scripts/__tests__/size-report-base.test.ts‎

Lines changed: 101 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ async function size(base: string) {
8080

8181
const worktreeOf = (sha: string) => path.join(repo, '.tmp', 'size-base', sha.slice(0, 12));
8282
const lockOf = (sha: string) => `${worktreeOf(sha)}.lock`;
83+
const holdLock = (sha: string, pid: number) => fs.symlinkSync(`${pid}:test`, lockOf(sha));
8384
const stampOf = (sha: string) => path.join(worktreeOf(sha), 'dist', '.size-base-complete');
8485
const builds = () =>
8586
fs
@@ -101,21 +102,21 @@ test('first run builds the base in a per-SHA worktree, stamps it, releases its l
101102
});
102103

103104
test('a base whose lock is held by a live pid fails fast without touching its worktree', async () => {
104-
fs.writeFileSync(lockOf(first), `${process.pid}\n`); // this test process: alive
105+
holdLock(first, process.pid); // this test process: alive
105106
const before = fs.statSync(stampOf(first)).mtimeMs;
106107
const result = await size(first);
107108
assert.notEqual(result.exitCode, 0);
108109
assert.match(
109110
result.stderr,
110-
new RegExp(`another \`size --base\` \\(pid ${process.pid}\\) is building`),
111+
new RegExp(`another \`size --base\` \\(pid ${process.pid}\\) is using`),
111112
);
112113
assert.equal(fs.statSync(stampOf(first)).mtimeMs, before);
113114
assert.equal(builds().length, 1);
114115
fs.rmSync(lockOf(first));
115116
});
116117

117118
test('a stale lock (dead pid) is replaced and the run proceeds', async () => {
118-
fs.writeFileSync(lockOf(first), `${NEVER_A_PID}\n`);
119+
holdLock(first, NEVER_A_PID);
119120
const result = await size(first);
120121
assert.equal(result.exitCode, 0, result.stderr);
121122
assert.equal(fs.existsSync(lockOf(first)), false);
@@ -130,7 +131,7 @@ test('an unstamped worktree (interrupted build) is rebuilt rather than trusted',
130131
});
131132

132133
test('measuring another base evicts an idle cached base but never one whose lock is live', async () => {
133-
fs.writeFileSync(lockOf(first), `${process.pid}\n`); // in use by "another run"
134+
holdLock(first, process.pid); // in use by "another run"
134135
const guarded = await size(second);
135136
assert.equal(guarded.exitCode, 0, guarded.stderr);
136137
assert.ok(fs.existsSync(worktreeOf(first)), 'a live-locked worktree survives eviction');
@@ -146,3 +147,99 @@ test('measuring another base evicts an idle cached base but never one whose lock
146147
'and unregistered from git',
147148
);
148149
});
150+
151+
test('two overlapping runs on the same base: exactly one builds, the other fails fast on the live lock', async () => {
152+
// A slower shim build widens the overlap window: the second run must find the first run's
153+
// symlink lock (identity in place from its single creating syscall) and refuse.
154+
fs.rmSync(worktreeOf(first), { recursive: true, force: true });
155+
runCmdSync('git', ['worktree', 'prune'], { cwd: repo });
156+
fs.rmSync(stampOf(first), { force: true });
157+
const slowBin = path.join(path.dirname(bin), 'slow-bin');
158+
fs.mkdirSync(slowBin, { recursive: true });
159+
writeExecutable(
160+
path.join(slowBin, 'pnpm'),
161+
`#!/bin/sh
162+
echo "$PWD $*" >> "${buildLog}"
163+
if [ "$1" = "build" ]; then sleep 1; mkdir -p dist/src && printf 'export const built = 1;\\n' > dist/src/index.js; fi
164+
`,
165+
);
166+
fs.copyFileSync(path.join(bin, 'npm'), path.join(slowBin, 'npm'));
167+
fs.chmodSync(path.join(slowBin, 'npm'), 0o755);
168+
const buildsBefore = builds().length;
169+
const env = { ...process.env, PATH: `${slowBin}:${process.env.PATH ?? ''}` };
170+
const run = () =>
171+
runCmd(process.execPath, [SCRIPT, '--cwd', repo, '--base', first], {
172+
cwd: repo,
173+
env,
174+
allowFailure: true,
175+
timeoutMs: 60_000,
176+
});
177+
const [a, b] = await Promise.all([run(), run()]);
178+
const outcomes = [a, b].map((r) => r.exitCode === 0);
179+
assert.deepEqual(
180+
outcomes.sort(),
181+
[false, true],
182+
`one wins, one refuses: ${a.stderr} ${b.stderr}`,
183+
);
184+
const loser = a.exitCode === 0 ? b : a;
185+
assert.match(loser.stderr, /another `size --base` \(pid \d+\) is using base/);
186+
assert.equal(builds().length - buildsBefore, 1, 'exactly one build across the two runs');
187+
assert.ok(fs.existsSync(stampOf(first)));
188+
assert.equal(
189+
fs.existsSync(lockOf(first)),
190+
false,
191+
'the winner released; the loser removed nothing',
192+
);
193+
});
194+
195+
test('stale-lock takeover racing another taker: the live lock is never unlinked, and never both proceed', async () => {
196+
// Another process C also finds the stale lock and takes it over (compare-then-unlink, then
197+
// create), at a random moment while our run is doing the same. Whatever the interleaving:
198+
// if our run acquired, C's compare sees a foreign identity and skips; if C acquired first (or
199+
// in our run's window between judging stale and unlinking), our run finds a live lock and
200+
// refuses. It must never unlink C's live lock, and both must never proceed.
201+
const stale = `${NEVER_A_PID}:stale`;
202+
const live = `${process.pid}:taker-c`;
203+
const takeOver = (): 'took-over' | 'held-by-run' | 'absent' => {
204+
let current: string | undefined;
205+
try {
206+
current = fs.readlinkSync(lockOf(first));
207+
} catch {
208+
current = undefined;
209+
}
210+
if (current === undefined) return 'absent'; // our run already finished and released: no overlap
211+
if (current === stale) fs.unlinkSync(lockOf(first)); // compare-then-unlink, as the script does
212+
try {
213+
fs.symlinkSync(live, lockOf(first));
214+
return 'took-over';
215+
} catch {
216+
return 'held-by-run'; // our run holds it: C skips
217+
}
218+
};
219+
let ourWins = 0;
220+
let cWins = 0;
221+
// Delays straddle the run's time-to-acquire (node start + git rev-parse ≈ 300ms here) so both
222+
// orders occur in practice; the assertions hold under either, so the mix is reported, not required.
223+
const delays = [0, 200, 400, 650];
224+
for (const [round, delay] of delays.entries()) {
225+
fs.rmSync(lockOf(first), { force: true });
226+
fs.symlinkSync(stale, lockOf(first));
227+
const [result, c] = await Promise.all([
228+
size(first),
229+
new Promise<ReturnType<typeof takeOver>>((resolve) =>
230+
setTimeout(() => resolve(takeOver()), delay),
231+
),
232+
]);
233+
if (result.exitCode === 0) {
234+
ourWins += 1;
235+
assert.notEqual(c, 'took-over', `round ${round}: C must not acquire while our run holds it`);
236+
} else {
237+
cWins += 1;
238+
assert.match(result.stderr, /is using base/);
239+
assert.equal(fs.readlinkSync(lockOf(first)), live, `round ${round}: C's live lock intact`);
240+
}
241+
fs.rmSync(lockOf(first), { force: true });
242+
}
243+
assert.equal(ourWins + cWins, delays.length);
244+
process.stderr.write(`[size-report-base] takeover race: run won ${ourWins}, C won ${cWins}\n`);
245+
});

‎scripts/pr-evidence/run.ts‎

Lines changed: 33 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import {
3232
type EvidenceInputs,
3333
type GitFacts,
3434
} from './model.ts';
35+
import { withWorktrees } from './worktrees.ts';
3536

3637
const USAGE =
3738
'Usage: pnpm pr:evidence [--base <ref>] [--coverage] [--size] [--json]\n' +
@@ -111,17 +112,6 @@ async function collectDepgraph(cwd: string, out: string): Promise<DepgraphFacts>
111112
return depgraphFacts(JSON.parse(fs.readFileSync(out, 'utf8')));
112113
}
113114

114-
/** A pristine checkout of one commit; removed by the caller's scratch cleanup. */
115-
function addWorktree(scratch: string, name: string, commit: string): string {
116-
const worktree = path.join(scratch, name);
117-
git(['worktree', 'add', '--detach', worktree, commit]);
118-
return worktree;
119-
}
120-
121-
function removeWorktree(worktree: string): void {
122-
git(['worktree', 'remove', '--force', worktree]);
123-
}
124-
125115
async function collectCoverage(base: string): Promise<EvidenceInputs['coverage']> {
126116
if (!fs.existsSync(path.join(repoRoot, 'coverage', 'lcov.info'))) {
127117
return { kind: 'skipped', reason: 'no coverage/lcov.info — run pnpm test:coverage first' };
@@ -172,35 +162,38 @@ async function main(argv: readonly string[]): Promise<number> {
172162
// os.tmpdir() always exists; a repo-local scratch would have to be created first and is one
173163
// more thing a fresh checkout can lack.
174164
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-pr-evidence-'));
175-
const worktrees: string[] = [];
176-
try {
177-
const headTree = addWorktree(scratch, 'head', gitFacts.head);
178-
const baseTree = addWorktree(scratch, 'base', gitFacts.base);
179-
worktrees.push(headTree, baseTree);
180-
const [affected, layering, head, base] = await Promise.all([
181-
collectAffected(gitFacts.base, gitFacts.head),
182-
collectLayering(headTree),
183-
collectDepgraph(headTree, path.join(scratch, 'depgraph-head.json')),
184-
collectDepgraph(baseTree, path.join(scratch, 'depgraph-base.json')),
185-
]);
186-
const inputs: EvidenceInputs = {
187-
generatedAt: new Date().toISOString(),
188-
repository: REPOSITORY,
189-
git: gitFacts,
190-
affected,
191-
layering,
192-
depgraph: { head, base },
193-
coverage: await optional(values.coverage, '--coverage', () => collectCoverage(gitFacts.base)),
194-
size: await optional(values.size, '--size', () => collectSize(gitFacts.base)),
195-
};
196-
process.stdout.write(
197-
values.json ? `${JSON.stringify(inputs, null, 2)}\n` : renderEvidence(inputs),
198-
);
199-
return 0;
200-
} finally {
201-
for (const worktree of worktrees) removeWorktree(worktree);
202-
fs.rmSync(scratch, { recursive: true, force: true });
203-
}
165+
return await withWorktrees(
166+
repoRoot,
167+
scratch,
168+
[
169+
{ name: 'head', commit: gitFacts.head },
170+
{ name: 'base', commit: gitFacts.base },
171+
],
172+
async ([headTree, baseTree]) => {
173+
const [affected, layering, head, base] = await Promise.all([
174+
collectAffected(gitFacts.base, gitFacts.head),
175+
collectLayering(headTree),
176+
collectDepgraph(headTree, path.join(scratch, 'depgraph-head.json')),
177+
collectDepgraph(baseTree, path.join(scratch, 'depgraph-base.json')),
178+
]);
179+
const inputs: EvidenceInputs = {
180+
generatedAt: new Date().toISOString(),
181+
repository: REPOSITORY,
182+
git: gitFacts,
183+
affected,
184+
layering,
185+
depgraph: { head, base },
186+
coverage: await optional(values.coverage, '--coverage', () =>
187+
collectCoverage(gitFacts.base),
188+
),
189+
size: await optional(values.size, '--size', () => collectSize(gitFacts.base)),
190+
};
191+
process.stdout.write(
192+
values.json ? `${JSON.stringify(inputs, null, 2)}\n` : renderEvidence(inputs),
193+
);
194+
return 0;
195+
},
196+
);
204197
}
205198

206199
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import assert from 'node:assert/strict';
2+
import fs from 'node:fs';
3+
import os from 'node:os';
4+
import path from 'node:path';
5+
import { test } from 'node:test';
6+
import { runCmdSync } from '../../src/utils/exec.ts';
7+
import { withWorktrees } from './worktrees.ts';
8+
9+
// A throwaway repository with one commit; every case below plants a failure somewhere in the
10+
// add → run → cleanup sequence and asserts that nothing the helper created outlives it.
11+
12+
function makeRepo(): { repo: string; commit: string } {
13+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-evidence-worktrees-'));
14+
const repo = path.join(root, 'repo');
15+
fs.mkdirSync(repo);
16+
const git = (args: string[]) => runCmdSync('git', args, { cwd: repo }).stdout.trim();
17+
git(['init', '-q', '-b', 'main']);
18+
git(['config', 'user.email', 'wt@test']);
19+
git(['config', 'user.name', 'wt']);
20+
fs.writeFileSync(path.join(repo, 'file'), 'x\n');
21+
git(['add', '.']);
22+
git(['commit', '-q', '-m', 'one']);
23+
return { repo, commit: git(['rev-parse', 'HEAD']) };
24+
}
25+
26+
function registered(repo: string): string[] {
27+
return runCmdSync('git', ['worktree', 'list', '--porcelain'], { cwd: repo })
28+
.stdout.split('\n')
29+
.filter((line) => line.startsWith('worktree '))
30+
.map((line) => line.slice('worktree '.length))
31+
.filter((dir) => dir !== fs.realpathSync(repo));
32+
}
33+
34+
test('worktrees are created in order, handed over as a tuple, and removed with the scratch after success', async () => {
35+
const { repo, commit } = makeRepo();
36+
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-evidence-scratch-'));
37+
const seen = await withWorktrees(
38+
repo,
39+
scratch,
40+
[
41+
{ name: 'head', commit },
42+
{ name: 'base', commit },
43+
],
44+
async ([head, base]) => {
45+
assert.ok(fs.existsSync(path.join(head, 'file')));
46+
assert.ok(fs.existsSync(path.join(base, 'file')));
47+
assert.equal(registered(repo).length, 2);
48+
return [path.basename(head), path.basename(base)];
49+
},
50+
);
51+
assert.deepEqual(seen, ['head', 'base']);
52+
assert.deepEqual(registered(repo), []);
53+
assert.equal(fs.existsSync(scratch), false);
54+
});
55+
56+
test('a second add that fails leaks nothing: the first worktree is already registered and gets removed', async () => {
57+
const { repo, commit } = makeRepo();
58+
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-evidence-scratch-'));
59+
await assert.rejects(
60+
withWorktrees(
61+
repo,
62+
scratch,
63+
[
64+
{ name: 'head', commit },
65+
{ name: 'base', commit: 'not-a-commit' },
66+
],
67+
async () => {
68+
throw new Error('fn must not run when an add failed');
69+
},
70+
),
71+
/git exited with code 128/,
72+
);
73+
assert.deepEqual(registered(repo), [], 'the successful first add was cleaned up');
74+
assert.equal(fs.existsSync(scratch), false);
75+
});
76+
77+
test('a throwing fn still gets every worktree and the scratch removed', async () => {
78+
const { repo, commit } = makeRepo();
79+
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-evidence-scratch-'));
80+
await assert.rejects(
81+
withWorktrees(repo, scratch, [{ name: 'only', commit }], async () => {
82+
throw new Error('measurement failed');
83+
}),
84+
/measurement failed/,
85+
);
86+
assert.deepEqual(registered(repo), []);
87+
assert.equal(fs.existsSync(scratch), false);
88+
});
89+
90+
test('one cleanup failure never skips the remaining resources, and every failure is reported', async () => {
91+
const { repo, commit } = makeRepo();
92+
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-evidence-scratch-'));
93+
const removed: string[] = [];
94+
await assert.rejects(
95+
withWorktrees(
96+
repo,
97+
scratch,
98+
[
99+
{ name: 'first', commit },
100+
{ name: 'second', commit },
101+
{ name: 'third', commit },
102+
],
103+
async () => 'ok',
104+
(root, worktree) => {
105+
if (worktree.endsWith('second')) throw new Error('planted removal failure');
106+
runCmdSync('git', ['worktree', 'remove', '--force', worktree], { cwd: root });
107+
removed.push(path.basename(worktree));
108+
},
109+
),
110+
(error: unknown) => {
111+
assert.ok(error instanceof Error);
112+
assert.match(error.message, /cleanup left resources behind/);
113+
assert.match(error.message, /second: planted removal failure/);
114+
return true;
115+
},
116+
);
117+
assert.deepEqual(removed, ['first', 'third'], 'the failure in the middle skipped nothing');
118+
assert.equal(fs.existsSync(scratch), false, 'the scratch was still attempted');
119+
});

0 commit comments

Comments
 (0)