Skip to content

Commit 180745f

Browse files
donjor-agent[bot]donjor
authored andcommitted
fix(server): a server-started turn recovers its removed worktree after the root goes idle (#1601)
## Problem A server-started turn (MCP, queue, schedule) can land on a thread whose worktree and branch were removed. If another thread is running in the project root at that moment, recovery yields and records a failed move. The turn-start gate retries under the same run's recovery id, and the retry never moves the thread. Reproduced in VM UAT on the V2 port: 1. Codex thread on worktree `feat-b` 2. Worktree and branch removed 3. Claude thread in the root calls `t3_thread_send` 4. First attempt refused: root busy 5. Retry logs "moved", persists nothing 6. Turn fails: workspace folder no longer exists ## Change Recovery dispatched its refused write and its committed write under one command id. The orchestrator replays an existing receipt as success, so the commit became a no-op. The refused write now carries its own command id (`<recoveryId>:refused`). The committed write keeps the recovery id, so its timeline notice id stays stable across retries. ## Scope and approval Fork-only fix to the V2 port's removed-worktree recovery, which gates the first 0.0.46 fork nightly. Browser turns were unaffected: the client path mints a fresh recovery id per attempt. ## Verification A new integration test covers a recovery refused while the root is busy, then retried after the root settles. Without the fix it fails: `worktreePath` stays on the removed path. | Check | Result | | --- | --- | | `vp test run` on the gate integration test | exit 0, 9 passed | | `vp run typecheck` in `apps/server` | exit 0 | | `vp run fork:ci` | exit 0 | The VM rerun of the MCP recovery scenario follows on the rebuilt server. Claude Opus 5.5 in Claude Code, running inside T3 Code. ## Fork trailers Fork-Domain: zmux-estate Fork-Tier: bugfix Fork-Upstreamable: no Co-authored-by: donjor <[email protected]>
1 parent d68a7a5 commit 180745f

2 files changed

Lines changed: 120 additions & 21 deletions

File tree

‎apps/server/src/git/checkoutRecovery.fork.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,9 @@ export const makeCheckoutRecoveryFork = Effect.gen(function* () {
106106
const owner = yield* activeOnCheckout(thread.id, [root.checkoutRoot]);
107107
yield* threads.dispatch({
108108
type: "thread.metadata.update",
109-
commandId: requestId,
109+
// The turn-start gate retries a run under one recovery id; a refusal
110+
// sharing the commit's command id would replay the commit as a no-op.
111+
commandId: owner === null ? requestId : CommandId.make(`${recoveryId}:refused`),
110112
threadId: thread.id,
111113
expectedWorktreePath: deadPath as OrchestrationV2ThreadShell["worktreePath"],
112114
...(owner === null

‎apps/server/src/orchestration-v2/ProviderTurnStartCheckoutGate.fork.integration.test.ts‎

Lines changed: 117 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,29 @@ const noticeInRecoveredTurn: RecoveryCase["timeline"] = [
262262
["checkpoint", 2],
263263
];
264264

265+
/** Runs the worker until the thread's newest run settles, then drains what follows. */
266+
const settleRunOn = (threadId: ThreadId, afterSequence: number) =>
267+
Effect.gen(function* () {
268+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
269+
const settled = yield* (yield* EventSink.EventSinkV2)
270+
.stream({ threadId, afterSequence, eventType: "run.updated" })
271+
.pipe(
272+
Stream.filter(
273+
({ event }) =>
274+
event.type === "run.updated" && terminalRunStatuses.has(event.payload.status),
275+
),
276+
Stream.take(1),
277+
Stream.runDrain,
278+
Effect.forkScoped,
279+
);
280+
while (true) {
281+
yield* worker.drain();
282+
if (settled.pollUnsafe() !== undefined) break;
283+
yield* Effect.raceFirst(worker.awaitWork, Fiber.await(settled));
284+
}
285+
yield* worker.drain();
286+
});
287+
265288
const recoveries: ReadonlyArray<RecoveryCase> = [
266289
// An MCP, scheduled or queued turn finds the worktree gone at its start.
267290
{
@@ -387,27 +410,8 @@ it.live.each(recoveries)(
387410
),
388411
);
389412

390-
/** Runs the worker until the thread's newest run settles, then drains what follows. */
391413
const settleRun = (afterSequence: number, runThreadId = threadId) =>
392-
Effect.gen(function* () {
393-
const settled = yield* eventSink
394-
.stream({ threadId: runThreadId, afterSequence, eventType: "run.updated" })
395-
.pipe(
396-
Stream.filter(
397-
({ event }) =>
398-
event.type === "run.updated" && terminalRunStatuses.has(event.payload.status),
399-
),
400-
Stream.take(1),
401-
Stream.runDrain,
402-
Effect.forkScoped,
403-
);
404-
while (true) {
405-
yield* worker.drain();
406-
if (settled.pollUnsafe() !== undefined) break;
407-
yield* Effect.raceFirst(worker.awaitWork, Fiber.await(settled));
408-
}
409-
yield* worker.drain();
410-
});
414+
settleRunOn(runThreadId, afterSequence);
411415
const send = (text: string, sendThreadId = threadId) =>
412416
orchestrator.dispatch({
413417
type: "message.dispatch",
@@ -566,3 +570,96 @@ it.live.each(recoveries)(
566570
}).pipe(Effect.provide(NodeServices.layer)),
567571
),
568572
);
573+
574+
// The turn-start gate retries a run under one recovery id. A recovery refused
575+
// because another thread runs in the project root must not swallow the retry
576+
// that finds the root idle.
577+
it.live("commits a recovery retried after the project root went idle", () =>
578+
Effect.scoped(
579+
Effect.gen(function* () {
580+
const fileSystem = yield* FileSystem.FileSystem;
581+
const root = yield* checkpointWorkspace("worktree-recovery-retry-root");
582+
const base = yield* fileSystem.realPath(
583+
yield* fileSystem.makeTempDirectoryScoped({ prefix: "worktree-recovery-retry-" }),
584+
);
585+
const feature = `${base}/feature`;
586+
yield* git(root, ["worktree", "add", "-b", "feature", feature]);
587+
const threadId = ThreadId.make("thread:worktree-recovery-retry");
588+
const rootThreadId = ThreadId.make("thread:worktree-recovery-retry:root");
589+
const log = yield* Ref.make<SessionLog>({ opened: [], turns: [], closed: 0, unloaded: 0 });
590+
const registry = ProviderAdapterRegistry.makeSingleLayer(
591+
makeFixedCwdAdapter(log, root, ClaudeProviderCapabilitiesV2),
592+
);
593+
594+
yield* Effect.gen(function* () {
595+
const orchestrator = yield* Orchestrator.OrchestratorV2;
596+
const eventSink = yield* EventSink.EventSinkV2;
597+
const recovery = yield* makeCheckoutRecoveryFork.pipe(
598+
Effect.provide(Layer.mergeAll(threadManagementFromOrchestrator, checkoutServices(root))),
599+
);
600+
for (const [id, worktreePath] of [
601+
[threadId, feature],
602+
[rootThreadId, null],
603+
] as const) {
604+
yield* orchestrator.dispatch({
605+
type: "thread.create",
606+
createdBy: "user",
607+
creationSource: "web",
608+
commandId: CommandId.make(`command:${id}:create`),
609+
threadId: id,
610+
projectId,
611+
title: "Worktree recovery retry",
612+
modelSelection: selection,
613+
runtimeMode: "full-access",
614+
interactionMode: "default",
615+
branch: null,
616+
worktreePath,
617+
});
618+
}
619+
const rootTurn = yield* eventSink.latestSequence();
620+
yield* orchestrator.dispatch({
621+
type: "message.dispatch",
622+
createdBy: "user",
623+
creationSource: "web",
624+
commandId: CommandId.make("command:worktree-recovery-retry:root-turn"),
625+
threadId: rootThreadId,
626+
messageId: MessageId.make("message:worktree-recovery-retry:root-turn"),
627+
text: "busy in the root",
628+
attachments: [],
629+
modelSelection: selection,
630+
dispatchMode: { type: "start_immediately" },
631+
});
632+
yield* fileSystem.remove(feature, { recursive: true });
633+
const rootThread = yield* orchestrator.getThreadShell(rootThreadId);
634+
assert.isNotNull(rootThread?.activeRunId ?? null);
635+
636+
const recover = Effect.gen(function* () {
637+
const thread = yield* orchestrator.getThreadShell(threadId);
638+
assert.isNotNull(thread);
639+
return yield* recovery.recoverRemovedWorktree(thread!, {
640+
recoveryId: "server:worktree-checkout-recovery:retried-run",
641+
duringRun: true,
642+
});
643+
});
644+
assert.isFalse((yield* recover).moved);
645+
assert.equal(
646+
(yield* orchestrator.getThreadShell(threadId))?.checkoutMove?.status,
647+
"failed",
648+
);
649+
650+
yield* settleRunOn(rootThreadId, rootTurn);
651+
assert.isTrue((yield* recover).moved);
652+
const recovered = yield* orchestrator.getThreadShell(threadId);
653+
assert.isNull(recovered?.worktreePath);
654+
assert.equal(recovered?.checkoutMove?.status, "committed");
655+
}).pipe(
656+
Effect.provide(
657+
makeOrchestratorV2ReplayLayerWithRegistry({ name: "worktree-recovery-retry" }, registry, {
658+
runEffectWorker: false,
659+
wrapTurnStartFork: withCheckoutGate(root),
660+
}),
661+
),
662+
);
663+
}).pipe(Effect.provide(NodeServices.layer)),
664+
),
665+
);

0 commit comments

Comments
 (0)