Skip to content

Commit ea5c623

Browse files
jakeleventhalclaude
andcommitted
fix(server): bulk worktree launches no longer fail on Git races
Launching many threads into new worktrees at once failed most of them: concurrent fetches of the base lost the ref lock, branch and path names collided, and concurrent writes to the shared Git config failed on its lock. Retry hit the same collisions again. Concurrent fetches of one ref now share a single running fetch. A taken branch or path gets the next `-1`, `-2` suffix, the same style renames use, while Git stays the arbiter. Config writes and branch renames take a per-repository lock, so checkouts stay concurrent and concurrent renames pick distinct names. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 701fc83 commit ea5c623

4 files changed

Lines changed: 538 additions & 58 deletions

File tree

‎apps/server/src/orchestration-v2/ThreadLaunchService.test.ts‎

Lines changed: 335 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,14 @@ import * as Effect from "effect/Effect";
3434
import * as Exit from "effect/Exit";
3535
import * as Layer from "effect/Layer";
3636
import * as Option from "effect/Option";
37+
import * as Path from "effect/Path";
3738
import * as Ref from "effect/Ref";
3839
import * as Stream from "effect/Stream";
3940
import * as Schema from "effect/Schema";
4041
import * as TestClock from "effect/testing/TestClock";
4142

4243
import * as GitWorkflow from "../git/GitWorkflowService.ts";
44+
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
4345
import * as SqlitePersistence from "../persistence/Sqlite.ts";
4446
import * as ProjectStore from "./ProjectStore.ts";
4547
import * as ProjectService from "../project/ProjectService.ts";
@@ -1242,6 +1244,289 @@ it.effect("keeps an explicit branch name instead of generating one", () =>
12421244
}),
12431245
);
12441246

1247+
it.effect.each([false, true])(
1248+
"bulk launches get independent real worktrees with existing branch %s",
1249+
(branchExists) =>
1250+
Effect.gen(function* () {
1251+
const driver = yield* GitVcsDriver.GitVcsDriver;
1252+
const fs = yield* FileSystem.FileSystem;
1253+
const path = yield* Path.Path;
1254+
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "launch-worktree-collisions-" });
1255+
const worktreesDirectory = yield* fs.makeTempDirectoryScoped({ prefix: "launch-worktrees-" });
1256+
const git = (args: ReadonlyArray<string>) =>
1257+
driver.execute({ operation: "ThreadLaunchService.test", cwd, args });
1258+
yield* git(["init", "--initial-branch=main"]);
1259+
yield* git([
1260+
"-c",
1261+
"user.name=Test",
1262+
"-c",
1263+
1264+
"commit",
1265+
"--allow-empty",
1266+
"-m",
1267+
"initial",
1268+
]);
1269+
const existingPath = path.join(worktreesDirectory, "existing");
1270+
if (branchExists) {
1271+
yield* driver.createWorktree({
1272+
cwd,
1273+
refName: "main",
1274+
newRefName: "feature/shared",
1275+
path: existingPath,
1276+
});
1277+
yield* fs.writeFileString(path.join(existingPath, "keep.txt"), "existing uncommitted work");
1278+
}
1279+
const harness = makeHarness({
1280+
createWorktree: (input, options) =>
1281+
driver.createWorktree({ ...input, cwd }, { ...options, worktreesDirectory }),
1282+
});
1283+
yield* Effect.gen(function* () {
1284+
const launches = yield* ThreadLaunch.ThreadLaunchService;
1285+
const threads = yield* ThreadManagement.ThreadManagementService;
1286+
const launched = yield* Effect.all(
1287+
Array.from({ length: 12 }, (_, index) => {
1288+
const command = `command:launch:bulk-${index}`;
1289+
return launches
1290+
.launch(
1291+
launchInput({
1292+
command,
1293+
thread: `thread:launch:bulk-${index}`,
1294+
message: "Independent work",
1295+
workspace: { type: "worktree", baseRef: "main", branch: "feature/shared" },
1296+
}),
1297+
)
1298+
.pipe(
1299+
Effect.flatMap((result) =>
1300+
threads.streamStoredEventsFrom({ threadId: result.threadId }).pipe(
1301+
Stream.filter(
1302+
(stored) =>
1303+
(stored.commandId === CommandId.make(`${command}:release`) ||
1304+
stored.commandId === CommandId.make(`${command}:fail`)) &&
1305+
stored.event.type === "run.updated",
1306+
),
1307+
Stream.runHead,
1308+
Effect.andThen(threads.getThreadProjection(result.threadId)),
1309+
),
1310+
),
1311+
);
1312+
}),
1313+
{ concurrency: "unbounded" },
1314+
);
1315+
assert.equal(new Set(launched.map((projection) => projection.thread.branch)).size, 12);
1316+
assert.equal(
1317+
new Set(launched.map((projection) => projection.thread.worktreePath)).size,
1318+
12,
1319+
);
1320+
for (const projection of launched) {
1321+
assert.equal(
1322+
projection.runs[0]?.status,
1323+
"starting",
1324+
projection.turnItems.find((item) => item.type === "error")?.failure.message,
1325+
);
1326+
assert.isTrue(yield* fs.exists(projection.thread.worktreePath!));
1327+
const base = yield* git(["config", `branch.${projection.thread.branch}.gh-merge-base`]);
1328+
assert.equal(base.stdout.trim(), "main");
1329+
}
1330+
assert.deepEqual(
1331+
launched.map((projection) => projection.thread.branch).toSorted(),
1332+
Array.from({ length: 12 }, (_, index) =>
1333+
branchExists || index > 0
1334+
? `feature/shared-${branchExists ? index + 1 : index}`
1335+
: "feature/shared",
1336+
).toSorted(),
1337+
);
1338+
if (branchExists) {
1339+
assert.equal(
1340+
yield* fs.readFileString(path.join(existingPath, "keep.txt")),
1341+
"existing uncommitted work",
1342+
);
1343+
}
1344+
assert.equal(harness.removeWorktree.mock.calls.length, 0);
1345+
}).pipe(Effect.provide(harness.layer));
1346+
}).pipe(
1347+
Effect.provide(
1348+
GitVcsDriver.layer.pipe(
1349+
Layer.provide(
1350+
ServerConfig.layerTest(process.cwd(), { prefix: "launch-collision-test-" }),
1351+
),
1352+
Layer.provideMerge(NodeServices.layer),
1353+
),
1354+
),
1355+
),
1356+
);
1357+
1358+
it.effect.each([
1359+
{ branch: "t3/abcd1234", flat: false },
1360+
{ branch: "t3-abcd1234", flat: true },
1361+
])(
1362+
"keeps temporary naming and background rename after a collision on $branch",
1363+
({ branch, flat }) => {
1364+
let attempts = 0;
1365+
const harness = makeHarness({
1366+
hasCommit: () => Effect.succeed(flat),
1367+
createWorktree: (input) =>
1368+
++attempts === 1
1369+
? Effect.fail(
1370+
new GitCommandError({
1371+
operation: "GitVcsDriver.createWorktree",
1372+
command: "git",
1373+
cwd: project.workspaceRoot,
1374+
detail: "git worktree add failed",
1375+
reason: "branch_already_exists",
1376+
exitCode: 128,
1377+
}),
1378+
)
1379+
: Effect.succeed({
1380+
worktree: { path: "/repo-worktrees/fresh", refName: input.newRefName! },
1381+
}),
1382+
});
1383+
return Effect.gen(function* () {
1384+
const launches = yield* ThreadLaunch.ThreadLaunchService;
1385+
const threads = yield* ThreadManagement.ThreadManagementService;
1386+
const command = `command:launch:temp-collision-${flat}`;
1387+
const launched = yield* launches.launch(
1388+
launchInput({
1389+
command,
1390+
thread: `thread:launch:temp-collision-${flat}`,
1391+
message: "Name this worktree",
1392+
workspace: { type: "worktree", baseRef: "main", branch },
1393+
}),
1394+
);
1395+
yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
1396+
Stream.filter((stored) => stored.commandId === CommandId.make(`${command}:branch-rename`)),
1397+
Stream.runHead,
1398+
);
1399+
assert.equal(harness.createWorktree.mock.calls.length, 2);
1400+
const replacement = harness.createWorktree.mock.calls[1]![0].newRefName!;
1401+
assert.notEqual(replacement, branch);
1402+
assert.match(replacement, flat ? /^t3-[0-9a-f]{8}$/u : /^t3\/[0-9a-f]{8}$/u);
1403+
assert.equal(harness.renameBranch.mock.calls[0]![0].oldBranch, replacement);
1404+
assert.equal(
1405+
(yield* threads.getThreadProjection(launched.threadId)).thread.branch,
1406+
"generated-branch",
1407+
);
1408+
assert.equal(harness.removeWorktree.mock.calls.length, 0);
1409+
}).pipe(Effect.provide(harness.layer));
1410+
},
1411+
);
1412+
1413+
it.effect.each(["path_already_exists", "authentication_failed", "branch_already_exists"] as const)(
1414+
"handles %s without unbounded attempts",
1415+
(reason) => {
1416+
let attempts = 0;
1417+
const harness = makeHarness({
1418+
createWorktree: (input) => {
1419+
attempts += 1;
1420+
return reason === "path_already_exists" && attempts > 1
1421+
? Effect.succeed({
1422+
worktree: { path: "/repo-worktrees/fresh", refName: input.newRefName! },
1423+
})
1424+
: Effect.fail(
1425+
new GitCommandError({
1426+
operation: "GitVcsDriver.createWorktree",
1427+
command: "git",
1428+
cwd: project.workspaceRoot,
1429+
detail: "git worktree add failed",
1430+
reason,
1431+
exitCode: 128,
1432+
}),
1433+
);
1434+
},
1435+
});
1436+
return Effect.gen(function* () {
1437+
const launches = yield* ThreadLaunch.ThreadLaunchService;
1438+
const threads = yield* ThreadManagement.ThreadManagementService;
1439+
const command = `command:launch:collision-limit-${reason}`;
1440+
const launched = yield* launches.launch(
1441+
launchInput({
1442+
command,
1443+
thread: `thread:launch:collision-limit-${reason}`,
1444+
message: "Prepare worktree",
1445+
workspace: { type: "worktree", baseRef: "main", branch: "my-feature" },
1446+
}),
1447+
);
1448+
const succeeds = reason === "path_already_exists";
1449+
yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
1450+
Stream.filter(
1451+
(stored) =>
1452+
stored.commandId === CommandId.make(`${command}:${succeeds ? "release" : "fail"}`) &&
1453+
stored.event.type === "run.updated",
1454+
),
1455+
Stream.runHead,
1456+
);
1457+
assert.equal(attempts, succeeds ? 2 : reason === "authentication_failed" ? 1 : 101);
1458+
const projection = yield* threads.getThreadProjection(launched.threadId);
1459+
assert.equal(projection.runs[0]?.status, succeeds ? "starting" : "failed");
1460+
assert.equal(harness.runSetup.mock.calls.length, succeeds ? 1 : 0);
1461+
assert.equal(harness.removeWorktree.mock.calls.length, 0);
1462+
}).pipe(Effect.provide(harness.layer));
1463+
},
1464+
);
1465+
1466+
it.effect("Retry recovers an existing failed run whose requested branch is taken", () => {
1467+
let fails = true;
1468+
const harness = makeHarness({
1469+
createWorktree: (input) =>
1470+
fails || input.newRefName === "my-feature"
1471+
? Effect.fail(
1472+
new GitCommandError({
1473+
operation: "GitVcsDriver.createWorktree",
1474+
command: "git",
1475+
cwd: project.workspaceRoot,
1476+
detail: "git worktree add failed",
1477+
reason: "branch_already_exists",
1478+
exitCode: 128,
1479+
}),
1480+
)
1481+
: Effect.succeed({
1482+
worktree: { path: "/repo-worktrees/recovered", refName: input.newRefName! },
1483+
}),
1484+
});
1485+
return Effect.gen(function* () {
1486+
const launches = yield* ThreadLaunch.ThreadLaunchService;
1487+
const threads = yield* ThreadManagement.ThreadManagementService;
1488+
const launched = yield* launches.launch(
1489+
launchInput({
1490+
command: "command:launch:taken-retry",
1491+
thread: "thread:launch:taken-retry",
1492+
message: "Retry me",
1493+
workspace: { type: "worktree", baseRef: "main", branch: "my-feature" },
1494+
}),
1495+
);
1496+
yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
1497+
Stream.filter(
1498+
(stored) =>
1499+
stored.commandId === CommandId.make("command:launch:taken-retry:fail") &&
1500+
stored.event.type === "run.updated",
1501+
),
1502+
Stream.runHead,
1503+
);
1504+
const failed = yield* threads.getThreadProjection(launched.threadId);
1505+
fails = false;
1506+
yield* launches.retryPreparation({
1507+
commandId: CommandId.make("command:launch:taken-retry:retry"),
1508+
threadId: launched.threadId,
1509+
runId: failed.runs[0]!.id,
1510+
});
1511+
yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
1512+
Stream.filter(
1513+
(stored) =>
1514+
stored.commandId === CommandId.make("command:launch:taken-retry:retry:release") &&
1515+
stored.event.type === "run.updated",
1516+
),
1517+
Stream.runHead,
1518+
);
1519+
const recovered = yield* threads.getThreadProjection(launched.threadId);
1520+
assert.equal(recovered.runs.length, 1);
1521+
assert.equal(recovered.runs[0]!.id, failed.runs[0]!.id);
1522+
assert.equal(recovered.messages.length, 1);
1523+
assert.equal(recovered.runs[0]?.status, "starting");
1524+
assert.equal(recovered.thread.branch, "my-feature-1");
1525+
assert.equal(recovered.thread.worktreePath, "/repo-worktrees/recovered");
1526+
assert.equal(recovered.turnItems.find((item) => item.type === "error")?.status, "cancelled");
1527+
}).pipe(Effect.provide(harness.layer));
1528+
});
1529+
12451530
it.effect("keeps the temporary branch when branch generation fails", () =>
12461531
Effect.gen(function* () {
12471532
const harness = makeHarness({
@@ -1647,39 +1932,56 @@ it.effect("a retry reuses a recorded worktree without undoing its branch rename"
16471932
}).pipe(Effect.provide(harness.layer));
16481933
});
16491934

1650-
it.effect("removes a worktree that failed before the thread recorded it", () => {
1651-
const harness = makeHarness({
1652-
// A checkout that dies after claiming its directory.
1653-
createWorktree: (_input, options) =>
1654-
(options?.progress?.onWorktreeClaimed?.("/repo-worktrees/partial") ?? Effect.void).pipe(
1655-
Effect.andThen(Effect.fail(new Error("checkout failed") as never)),
1656-
),
1657-
});
1658-
return Effect.gen(function* () {
1659-
const launches = yield* ThreadLaunch.ThreadLaunchService;
1660-
const threads = yield* ThreadManagement.ThreadManagementService;
1661-
const launched = yield* launches.launch(
1662-
launchInput({
1663-
command: "command:launch:partial-worktree",
1664-
thread: "thread:launch:partial-worktree",
1665-
message: "Partial checkout",
1666-
workspace: { type: "worktree", baseRef: "main" },
1667-
}),
1668-
);
1669-
yield* waitUntil(() =>
1670-
threads
1671-
.getThreadProjection(launched.threadId)
1672-
.pipe(Effect.map((projection) => projection.runs[0]?.status === "failed")),
1673-
);
1674-
const projection = yield* threads.getThreadProjection(launched.threadId);
1675-
// Unrecorded, so a retry would create a second checkout beside it.
1676-
assert.equal(projection.thread.worktreePath, null);
1677-
assert.deepEqual(
1678-
harness.removeWorktree.mock.calls.map(([input]) => input.path),
1679-
["/repo-worktrees/partial"],
1680-
);
1681-
}).pipe(Effect.provide(harness.layer));
1682-
});
1935+
it.effect.each([undefined, "branch_already_exists"] as const)(
1936+
"removes a claimed worktree after failure %s without creating another",
1937+
(reason) => {
1938+
const harness = makeHarness({
1939+
// A checkout that dies after claiming its directory.
1940+
createWorktree: (_input, options) =>
1941+
(options?.progress?.onWorktreeClaimed?.("/repo-worktrees/partial") ?? Effect.void).pipe(
1942+
Effect.andThen(
1943+
Effect.fail(
1944+
new GitCommandError({
1945+
operation: "GitVcsDriver.createWorktree.configureBaseRef",
1946+
command: "git",
1947+
cwd: project.workspaceRoot,
1948+
detail: "checkout failed",
1949+
...(reason === undefined ? {} : { reason }),
1950+
}),
1951+
),
1952+
),
1953+
),
1954+
});
1955+
return Effect.gen(function* () {
1956+
const launches = yield* ThreadLaunch.ThreadLaunchService;
1957+
const threads = yield* ThreadManagement.ThreadManagementService;
1958+
const launched = yield* launches.launch(
1959+
launchInput({
1960+
command: "command:launch:partial-worktree",
1961+
thread: "thread:launch:partial-worktree",
1962+
message: "Partial checkout",
1963+
workspace: { type: "worktree", baseRef: "main" },
1964+
}),
1965+
);
1966+
yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
1967+
Stream.filter(
1968+
(stored) =>
1969+
stored.commandId === CommandId.make("command:launch:partial-worktree:fail") &&
1970+
stored.event.type === "run.updated",
1971+
),
1972+
Stream.runHead,
1973+
);
1974+
const projection = yield* threads.getThreadProjection(launched.threadId);
1975+
// Unrecorded, so a retry would create a second checkout beside it.
1976+
assert.equal(projection.thread.worktreePath, null);
1977+
assert.equal(harness.createWorktree.mock.calls.length, 1);
1978+
assert.deepEqual(
1979+
harness.removeWorktree.mock.calls.map(([input]) => input.path),
1980+
["/repo-worktrees/partial"],
1981+
);
1982+
}).pipe(Effect.provide(harness.layer));
1983+
},
1984+
);
16831985

16841986
it.effect.each(["worktree", "setup"] as const)(
16851987
"%s failure keeps the thread and message visible and emits failure items",

0 commit comments

Comments
 (0)