Skip to content

Commit 0c4b1d8

Browse files
fix(server): new turns wait for overlapping checkpoint file restores (#23671)
1 parent 97cfc07 commit 0c4b1d8

5 files changed

Lines changed: 343 additions & 9 deletions

File tree

‎apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts‎

Lines changed: 131 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import * as SqlitePersistence from "../persistence/Sqlite.ts";
3232
import * as ServerConfig from "../config.ts";
3333
import * as ServerSettings from "../serverSettings.ts";
3434
import * as GitWorkflow from "../git/GitWorkflowService.ts";
35-
import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
35+
import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts";
3636
import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
3737
import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts";
3838
import * as CheckpointService from "./CheckpointService.ts";
@@ -394,6 +394,136 @@ it.effect.each([
394394
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
395395
);
396396

397+
it.effect.each([{ restoreFiles: true }, { restoreFiles: false }])(
398+
"holds turn starts inside the worktree during rollback with restoreFiles $restoreFiles",
399+
({ restoreFiles }) =>
400+
Effect.gen(function* () {
401+
const fileSystem = yield* FileSystem.FileSystem;
402+
const path = yield* Path.Path;
403+
const workspaceRoot = yield* fileSystem.realPath(
404+
yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-rollback-reservation-" }),
405+
);
406+
const worktreesDir = path.join(workspaceRoot, "worktrees");
407+
const worktreePath = path.join(worktreesDir, "repo", "feature");
408+
yield* fileSystem.makeDirectory(path.join(worktreePath, ".git"), { recursive: true });
409+
const threadId = ThreadId.make("rollback-reservation");
410+
const providerThreadId = ProviderThreadId.make("rollback-reservation-provider-thread");
411+
const providerInstanceId = ProviderInstanceId.make("rollback-reservation-provider");
412+
const checkpointId = CheckpointId.make("rollback-reservation-checkpoint");
413+
const scopeId = CheckpointScopeId.make("rollback-reservation-scope");
414+
const release = Deferred.makeUnsafe<void>();
415+
const openEntered = Deferred.makeUnsafe<void>();
416+
const restoreEntered = Deferred.makeUnsafe<void>();
417+
const releaseRestore = Deferred.makeUnsafe<void>();
418+
const calls: string[] = [];
419+
const projection = {
420+
thread: {
421+
id: threadId,
422+
projectId: ProjectId.make("rollback-reservation-project"),
423+
worktreePath,
424+
branch: "feature/rollback-reservation",
425+
activeProviderThreadId: providerThreadId,
426+
modelSelection: { instanceId: providerInstanceId, model: "test-model" },
427+
},
428+
providerThread: {
429+
id: providerThreadId,
430+
providerSessionId: ProviderSessionId.make("rollback-reservation-session"),
431+
providerInstanceId,
432+
},
433+
providerSession: undefined,
434+
checkpoint: { id: checkpointId, scopeId, status: "ready", appRunOrdinal: 0 },
435+
scope: { id: scopeId, cwd: worktreePath },
436+
runsToRollback: [],
437+
targetAttempt: undefined,
438+
rolledBackAttemptIds: [],
439+
staleCheckpoints: [],
440+
rootNodes: [],
441+
providerTurns: [],
442+
} as unknown as ProjectionStore.ProjectionCheckpointRollbackContext;
443+
const testLayer = CheckpointRollbackService.layer.pipe(
444+
Layer.provide(
445+
Layer.mergeAll(
446+
ThreadCommandExecutor.layer,
447+
NodeServices.layer,
448+
Layer.mock(ServerConfig.ServerConfig)({ worktreesDir } as never),
449+
ServerSettings.layerTest(),
450+
Layer.mock(ProjectStore.ProjectStoreV2)({
451+
get: () => Effect.succeedSome({ workspaceRoot } as never),
452+
}),
453+
Layer.mock(GitWorkflow.GitWorkflowService)({}),
454+
Layer.mock(CheckpointService.CheckpointServiceV2)({
455+
restore: () =>
456+
Deferred.succeed(restoreEntered, undefined).pipe(
457+
Effect.andThen(Deferred.await(releaseRestore)),
458+
Effect.andThen(Effect.sync(() => calls.push("files"))),
459+
),
460+
}),
461+
Layer.mock(EventSink.EventSinkV2)({ write: () => Effect.succeed([]) }),
462+
IdAllocator.layer,
463+
Layer.mock(ProjectionStore.ProjectionStoreV2)({
464+
getThread: () => Effect.succeed(projection.thread),
465+
getCheckpointRollbackContext: () => Effect.succeed(projection),
466+
getShellSnapshot: () =>
467+
Effect.succeed({
468+
schemaVersion: 1,
469+
snapshotSequence: 0,
470+
threads: [],
471+
archivedThreads: [],
472+
}),
473+
}),
474+
Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
475+
open: () =>
476+
Deferred.succeed(openEntered, undefined).pipe(
477+
Effect.andThen(Deferred.await(release)),
478+
Effect.as({} as never),
479+
),
480+
}),
481+
Layer.mock(RuntimePolicy.RuntimePolicyV2)({
482+
resolve: () => Effect.succeed({ cwd: worktreePath } as never),
483+
}),
484+
),
485+
),
486+
);
487+
yield* Effect.gen(function* () {
488+
const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
489+
const rollback = yield* service
490+
.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles })
491+
.pipe(Effect.forkChild({ startImmediately: true }));
492+
yield* Deferred.await(openEntered);
493+
// A synchronous realPath lets the forked start reach the reservation
494+
// before the next assertion, so "not entered" means "waiting".
495+
const startTurn = withTurnStartReservation(
496+
FileSystem.makeNoop({ realPath: (target) => Effect.succeed(target) }),
497+
path,
498+
path.join(worktreePath, "packages", "app"),
499+
Effect.sync(() => calls.push("turn start")),
500+
).pipe(Effect.forkChild({ startImmediately: true }));
501+
const turnStart = yield* startTurn;
502+
if (restoreFiles) {
503+
assert.deepEqual(calls, []);
504+
yield* Deferred.succeed(release, undefined);
505+
// The reservation must still be held while files are being restored,
506+
// not only while the provider rollback is in flight.
507+
yield* Deferred.await(restoreEntered);
508+
const duringRestore = yield* startTurn;
509+
assert.deepEqual(calls, []);
510+
yield* Deferred.succeed(releaseRestore, undefined);
511+
yield* Fiber.join(duringRestore);
512+
} else {
513+
// Nothing is rewritten, so the turn start must not wait for the provider.
514+
yield* Fiber.join(turnStart);
515+
yield* Deferred.succeed(release, undefined);
516+
}
517+
yield* Fiber.join(rollback);
518+
yield* Fiber.join(turnStart);
519+
assert.deepEqual(
520+
calls,
521+
restoreFiles ? ["files", "turn start", "turn start"] : ["turn start"],
522+
);
523+
}).pipe(Effect.provide(testLayer));
524+
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
525+
);
526+
397527
const seedRollbackHistory = Effect.gen(function* () {
398528
const store = yield* ProjectionStore.ProjectionStoreV2;
399529
const now = yield* DateTime.now;

‎apps/server/src/orchestration-v2/CheckpointRollbackService.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@ import * as GitWorkflowService from "../git/GitWorkflowService.ts";
2121
import type * as ServerConfig from "../config.ts";
2222
import type * as ServerSettings from "../serverSettings.ts";
2323
import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts";
24-
import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
24+
import {
25+
withCheckpointRestoreReservation,
26+
withWorkspaceLease,
27+
} from "../workspace/workspaceLease.ts";
2528
import {
2629
isCheckpointRestoreIsolated,
2730
SHARED_WORKSPACE_RESTORE_MESSAGE,
@@ -422,9 +425,15 @@ export const layer: Layer.Layer<
422425
function* (input) {
423426
// Storage cleanup removes worktrees under the same lease.
424427
const thread = yield* projections.getThread(input.threadId);
425-
yield* thread.worktreePath === null
426-
? execute(input)
427-
: withWorkspaceLease(path.resolve(thread.worktreePath), execute(input));
428+
if (thread.worktreePath === null) return yield* execute(input);
429+
const worktreePath = path.resolve(thread.worktreePath);
430+
const rollback = withWorkspaceLease(worktreePath, execute(input));
431+
// Shared-workspace restores are rejected and conversation-only rewinds
432+
// touch no files; reserving for them would stall turn starts on the
433+
// slow provider rollback for nothing.
434+
yield* input.restoreFiles === false
435+
? rollback
436+
: withCheckpointRestoreReservation(fileSystem, path, worktreePath, rollback);
428437
},
429438
(effect, input) =>
430439
effect.pipe(

‎apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ import * as PlatformError from "effect/PlatformError";
3535
import * as Schema from "effect/Schema";
3636

3737
import * as ServerConfig from "../config.ts";
38-
import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
38+
import {
39+
withCheckpointRestoreReservation,
40+
withWorkspaceLease,
41+
} from "../workspace/workspaceLease.ts";
3942
import * as GitWorkflow from "../git/GitWorkflowService.ts";
4043
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
4144
import { layerRealGit } from "./ProviderTurnStartService.testkit.ts";
@@ -1565,6 +1568,92 @@ effectIt.effect.each(["cleanup-first", "startup-first"])(
15651568
}).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
15661569
);
15671570

1571+
// regression: #12341 a turn start overlapping a checkpoint restore wrote into the checkout
1572+
// The filesystem is synchronous so the turn start either reaches the provider or parks on the
1573+
// lease before the restore is released; realPath models the symlink alias and missing worktrees.
1574+
effectIt.effect.each([
1575+
{
1576+
overlap: "a project-root thread during a restore of the project checkout",
1577+
restorePath: "/tmp/native-account-project",
1578+
worktreePath: undefined,
1579+
realPaths: {},
1580+
missing: [],
1581+
},
1582+
{
1583+
overlap: "a symlink alias of the restoring checkout",
1584+
restorePath: "/tmp/restore-overlap/link",
1585+
worktreePath: "/tmp/restore-overlap/checkout",
1586+
realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
1587+
missing: [],
1588+
},
1589+
{
1590+
overlap: "a directory inside the restoring checkout",
1591+
restorePath: "/tmp/restore-overlap/checkout",
1592+
worktreePath: "/tmp/restore-overlap/checkout/packages/app",
1593+
realPaths: {},
1594+
missing: [],
1595+
},
1596+
{
1597+
overlap: "a not-yet-recreated worktree under a symlink alias of the restoring one",
1598+
restorePath: "/tmp/restore-overlap/link/missing-child",
1599+
worktreePath: "/tmp/restore-overlap/checkout/missing-child",
1600+
realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
1601+
missing: [
1602+
"/tmp/restore-overlap/link/missing-child",
1603+
"/tmp/restore-overlap/checkout/missing-child",
1604+
],
1605+
},
1606+
] as const)(
1607+
"does not start provider work in $overlap until the restore finishes",
1608+
({ restorePath, worktreePath, realPaths, missing }) =>
1609+
Effect.gen(function* () {
1610+
const order: Array<string> = [];
1611+
const releaseRestore = yield* Deferred.make<void>();
1612+
const restoreEntered = yield* Deferred.make<void>();
1613+
const fileSystem = FileSystem.makeNoop({
1614+
exists: () => Effect.succeed(true),
1615+
realPath: (candidate) =>
1616+
(missing as ReadonlyArray<string>).includes(candidate)
1617+
? Effect.fail(
1618+
PlatformError.systemError({
1619+
_tag: "NotFound",
1620+
module: "FileSystem",
1621+
method: "realPath",
1622+
pathOrDescriptor: candidate,
1623+
}),
1624+
)
1625+
: Effect.succeed((realPaths as Record<string, string>)[candidate] ?? candidate),
1626+
});
1627+
const harness = makeLocalCommandHarness({
1628+
text: "Continue",
1629+
...(worktreePath === undefined ? {} : { worktreePath }),
1630+
fileSystem,
1631+
failReadsAfterRunning: true,
1632+
});
1633+
const open = harness.open.getMockImplementation()!;
1634+
harness.open.mockImplementation(() =>
1635+
Effect.sync(() => order.push("provider session opened")).pipe(Effect.andThen(open())),
1636+
);
1637+
// The reservation CheckpointRollbackServiceV2.execute holds around its
1638+
// isolation check, provider rollback, and git restore of `restorePath`.
1639+
const restore = yield* withCheckpointRestoreReservation(
1640+
fileSystem,
1641+
yield* Path.Path,
1642+
restorePath,
1643+
Deferred.succeed(restoreEntered, undefined).pipe(
1644+
Effect.andThen(Deferred.await(releaseRestore)),
1645+
Effect.andThen(Effect.sync(() => order.push("checkpoint files restored"))),
1646+
),
1647+
).pipe(Effect.forkChild({ startImmediately: true }));
1648+
yield* Deferred.await(restoreEntered);
1649+
const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
1650+
yield* Deferred.succeed(releaseRestore, undefined);
1651+
yield* Fiber.join(restore);
1652+
yield* Fiber.join(startup);
1653+
expect(order).toEqual(["checkpoint files restored", "provider session opened"]);
1654+
}).pipe(Effect.provide(Path.layer)),
1655+
);
1656+
15681657
effectIt.effect("reads current startup state after waiting for the workspace lease", () =>
15691658
Effect.gen(function* () {
15701659
const worktreePath = "/tmp/provider-start-fresh-state";

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import {
3030
CLAUDE_PROVIDER,
3131
CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL,
3232
} from "./Adapters/ClaudeAdapterV2.ts";
33-
import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
33+
import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts";
3434
import * as GitWorkflowService from "../git/GitWorkflowService.ts";
3535
import * as ProjectService from "../project/ProjectService.ts";
3636
import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
@@ -154,7 +154,9 @@ export const layer: Layer.Layer<
154154
const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics;
155155
const contextHandoffService = yield* ContextHandoffService.ContextHandoffServiceV2;
156156
const idAllocator = yield* IdAllocator.IdAllocatorV2;
157+
const fileSystem = yield* FileSystem.FileSystem;
157158
const path = yield* Path.Path;
159+
const projects = yield* ProjectService.ProjectService;
158160
const providerAuth = yield* ProviderAuthService.ProviderAuthService;
159161
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
160162
const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
@@ -1763,9 +1765,21 @@ export const layer: Layer.Layer<
17631765
// Storage cleanup removes worktrees under the same lease. Only startup
17641766
// holds it; the provider turn runs after it is released.
17651767
const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1766-
const providerTurn = yield* thread.worktreePath === null
1768+
const cwd =
1769+
thread.worktreePath ??
1770+
(yield* projects
1771+
.getById(thread.projectId, { includeDeleted: true })
1772+
.pipe(Effect.map((project) => Option.getOrUndefined(project)?.workspaceRoot)));
1773+
const providerTurn = yield* cwd === undefined
17671774
? start(input)
1768-
: withWorkspaceLease(path.resolve(thread.worktreePath), start(input));
1775+
: withTurnStartReservation(
1776+
fileSystem,
1777+
path,
1778+
path.resolve(cwd),
1779+
thread.worktreePath === null
1780+
? start(input)
1781+
: withWorkspaceLease(path.resolve(thread.worktreePath), start(input)),
1782+
);
17691783
if (providerTurn !== undefined) yield* providerTurn;
17701784
},
17711785
(effect, input) =>

0 commit comments

Comments
 (0)