Skip to content

Commit 5b28630

Browse files
committed
fix(server): guard provider startup against worktree cleanup
1 parent 3694acf commit 5b28630

5 files changed

Lines changed: 208 additions & 28 deletions

File tree

‎apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,16 @@ import {
1010
ThreadId,
1111
} from "@t3tools/contracts";
1212
import * as Effect from "effect/Effect";
13+
import * as Deferred from "effect/Deferred";
14+
import * as Fiber from "effect/Fiber";
1315
import * as FileSystem from "effect/FileSystem";
1416
import * as Path from "effect/Path";
1517
import * as Option from "effect/Option";
1618
import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
1719
import * as Layer from "effect/Layer";
1820

21+
import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
22+
import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
1923
import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts";
2024
import { isCheckpointRestoreIsolated } from "./CheckpointRestoreSafety.ts";
2125
import * as CheckpointService from "./CheckpointService.ts";
@@ -263,7 +267,7 @@ it.effect("rejects a rollback when provider selection changed before execution",
263267
}).pipe(Effect.provide(testLayer));
264268
});
265269

266-
it.effect("reports a missing provider turn as a structured rollback failure", () => {
270+
it.effect.each(["normal", "cleanup-first", "rollback-first"])("locks rollback with %s", (order) => {
267271
const threadId = ThreadId.make("thread:rollback-provider-turn-unavailable");
268272
const providerThreadId = ProviderThreadId.make(
269273
"provider-thread:rollback-provider-turn-unavailable",
@@ -275,6 +279,19 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
275279
const scopeId = CheckpointScopeId.make("checkpoint-scope:rollback-provider-turn-unavailable");
276280
const providerInstanceId = ProviderInstanceId.make("provider_rollback_provider_turn_unavailable");
277281
const restore = vi.fn(() => Effect.die("checkpoint restore must not run"));
282+
const release = Deferred.makeUnsafe<void>();
283+
const openEntered = Deferred.makeUnsafe<void>();
284+
let checkoutExists = true;
285+
let cleanupEntered = false;
286+
const open = vi.fn(() =>
287+
Effect.gen(function* () {
288+
if (!checkoutExists)
289+
return yield* new ProviderWorkspaceMissingError({ threadId, cwd: process.cwd() });
290+
yield* Deferred.succeed(openEntered, undefined);
291+
if (order === "rollback-first") yield* Deferred.await(release);
292+
return {} as never;
293+
}),
294+
);
278295
const projection = {
279296
thread: {
280297
worktreePath: process.cwd(),
@@ -306,7 +323,7 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
306323
}),
307324
}),
308325
Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
309-
open: () => Effect.succeed({} as never),
326+
open,
310327
}),
311328
Layer.mock(RuntimePolicy.RuntimePolicyV2)({
312329
resolve: () => Effect.succeed({} as never),
@@ -317,14 +334,48 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
317334

318335
return Effect.gen(function* () {
319336
const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
320-
const error = yield* service
337+
const execute = service
321338
.execute({
322339
threadId,
323340
providerThreadId,
324341
checkpointId,
325342
scopeId,
343+
restoreFiles: false,
326344
})
327345
.pipe(Effect.flip);
346+
const cleanupEffect = withWorkspaceLease(
347+
process.cwd(),
348+
Effect.gen(function* () {
349+
cleanupEntered = true;
350+
if (order === "cleanup-first") yield* Deferred.await(release);
351+
checkoutExists = false;
352+
}),
353+
);
354+
const cleanup =
355+
order === "cleanup-first"
356+
? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
357+
: undefined;
358+
const rollback = yield* execute.pipe(Effect.forkChild({ startImmediately: true }));
359+
if (order === "cleanup-first") assert.equal(open.mock.calls.length, 0);
360+
if (order === "rollback-first") yield* Deferred.await(openEntered);
361+
const laterCleanup =
362+
order === "rollback-first"
363+
? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
364+
: undefined;
365+
if (order === "rollback-first") assert.isFalse(cleanupEntered);
366+
yield* Deferred.succeed(release, undefined);
367+
if (cleanup !== undefined) yield* Fiber.join(cleanup);
368+
const error = yield* Fiber.join(rollback);
369+
if (laterCleanup !== undefined) {
370+
yield* Fiber.join(laterCleanup);
371+
assert.isTrue(cleanupEntered);
372+
}
373+
if (order === "cleanup-first") {
374+
assert.equal(error.reason, "unexpected-failure");
375+
assert.instanceOf(error.cause, ProviderWorkspaceMissingError);
376+
assert.equal(restore.mock.calls.length, 0);
377+
return;
378+
}
328379

329380
assert.equal(error.reason, "provider-turn-unavailable");
330381
assert.equal(

‎apps/server/src/orchestration-v2/CheckpointRollbackService.ts‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@ import * as Layer from "effect/Layer";
1313
import * as Path from "effect/Path";
1414
import * as Schema from "effect/Schema";
1515

16+
import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
1617
import {
1718
isCheckpointRestoreIsolated,
1819
SHARED_WORKSPACE_RESTORE_MESSAGE,
1920
} from "./CheckpointRestoreSafety.ts";
2021
import { CheckpointServiceV2 } from "./CheckpointService.ts";
2122
import { EventSinkV2 } from "./EventSink.ts";
2223
import { IdAllocatorV2 } from "./IdAllocator.ts";
23-
import { ProjectionStoreV2 } from "./ProjectionStore.ts";
24+
import * as ProjectionStore from "./ProjectionStore.ts";
2425
import * as ProjectStore from "./ProjectStore.ts";
2526
import type { ProviderAdapterV2RollbackTarget } from "./ProviderAdapter.ts";
2627
import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts";
@@ -84,7 +85,7 @@ export const layer: Layer.Layer<
8485
| CheckpointServiceV2
8586
| EventSinkV2
8687
| IdAllocatorV2
87-
| ProjectionStoreV2
88+
| ProjectionStore.ProjectionStoreV2
8889
| ProviderSessionManagerV2
8990
| RuntimePolicyV2
9091
| FileSystem.FileSystem
@@ -96,21 +97,15 @@ export const layer: Layer.Layer<
9697
const checkpoints = yield* CheckpointServiceV2;
9798
const eventSink = yield* EventSinkV2;
9899
const ids = yield* IdAllocatorV2;
99-
const projections = yield* ProjectionStoreV2;
100+
const projections = yield* ProjectionStore.ProjectionStoreV2;
100101
const sessions = yield* ProviderSessionManagerV2;
101102
const runtimePolicy = yield* RuntimePolicyV2;
102103
const fileSystem = yield* FileSystem.FileSystem;
103104
const projects = yield* ProjectStore.ProjectStoreV2;
104105
const path = yield* Path.Path;
105106

106-
const execute = Effect.fn("orchestrationV2.checkpointRollback.execute")(function* (input: {
107-
readonly threadId: ThreadId;
108-
readonly providerThreadId: ProviderThreadId;
109-
readonly checkpointId: CheckpointId;
110-
readonly scopeId: CheckpointScopeId;
111-
readonly restoreFiles?: boolean;
112-
}) {
113-
const projection = yield* projections.getThreadRecords(input.threadId, [
107+
const readProjection = (threadId: ThreadId) =>
108+
projections.getThreadRecords(threadId, [
114109
"providerThreads",
115110
"providerSessions",
116111
"checkpoints",
@@ -120,6 +115,17 @@ export const layer: Layer.Layer<
120115
"nodes",
121116
"providerTurns",
122117
]);
118+
119+
const execute = Effect.fn("orchestrationV2.checkpointRollback.execute")(function* (
120+
input: {
121+
readonly threadId: ThreadId;
122+
readonly providerThreadId: ProviderThreadId;
123+
readonly checkpointId: CheckpointId;
124+
readonly scopeId: CheckpointScopeId;
125+
readonly restoreFiles?: boolean;
126+
},
127+
projection: Effect.Success<ReturnType<typeof readProjection>>,
128+
) {
123129
const providerThread = projection.providerThreads.find(
124130
(candidate) => candidate.id === input.providerThreadId,
125131
);
@@ -344,7 +350,13 @@ export const layer: Layer.Layer<
344350

345351
return CheckpointRollbackServiceV2.of({
346352
execute: (input) =>
347-
execute(input).pipe(
353+
Effect.gen(function* () {
354+
const projection = yield* readProjection(input.threadId);
355+
const effect = execute(input, projection);
356+
return yield* projection.thread.worktreePath == null
357+
? effect
358+
: withWorkspaceLease(path.resolve(projection.thread.worktreePath), effect);
359+
}).pipe(
348360
Effect.mapError((cause) =>
349361
isCheckpointRollbackExecutionError(cause)
350362
? cause

‎apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts‎

Lines changed: 103 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,16 @@ import {
1818
} from "@t3tools/contracts";
1919
import * as Cause from "effect/Cause";
2020
import * as DateTime from "effect/DateTime";
21+
import * as Deferred from "effect/Deferred";
2122
import * as Effect from "effect/Effect";
2223
import * as FileSystem from "effect/FileSystem";
24+
import * as Fiber from "effect/Fiber";
2325
import * as Layer from "effect/Layer";
2426
import * as Option from "effect/Option";
27+
import * as Path from "effect/Path";
2528
import * as Schema from "effect/Schema";
2629

30+
import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
2731
import * as GitWorkflow from "../git/GitWorkflowService.ts";
2832
import * as ProjectService from "../project/ProjectService.ts";
2933
import * as ProviderAuthService from "../provider/Services/ProviderAuthService.ts";
@@ -95,6 +99,7 @@ it("does not commit running state when inherited background routing cannot be re
9599
Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({}),
96100
Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),
97101
IdAllocator.layer,
102+
Path.layer,
98103
Layer.succeed(FileSystem.FileSystem, { exists: () => Effect.succeed(false) } as never),
99104
Layer.mock(GitWorkflow.GitWorkflowService)({ pruneWorktrees, createWorktree }),
100105
Layer.mock(ProjectService.ProjectService)({
@@ -155,6 +160,7 @@ it("does not commit running state when inherited background routing cannot be re
155160

156161
function makeLocalCommandHarness(input: {
157162
readonly text: string;
163+
readonly worktreePath?: string;
158164
readonly previousNativeSession?: boolean;
159165
readonly previousMessages?: ReadonlyArray<string>;
160166
readonly logoutFailure?: string;
@@ -236,8 +242,9 @@ function makeLocalCommandHarness(input: {
236242
thread: {
237243
id: threadId,
238244
activeProviderThreadId: providerThreadId,
239-
branch: null,
240-
worktreePath: null,
245+
projectId: ProjectId.make("project-native-account-command"),
246+
branch: input.worktreePath === undefined ? null : "feature/restore",
247+
worktreePath: input.worktreePath ?? null,
241248
} as OrchestrationV2ThreadProjection["thread"],
242249
runs: [
243250
...(input.previousNativeSession
@@ -483,6 +490,8 @@ function makeLocalCommandHarness(input: {
483490
return { committed, storedEvents: [] };
484491
}),
485492
);
493+
const exists = vi.fn(() => Effect.succeed(true));
494+
const createWorktree = vi.fn(() => Effect.succeed({} as never));
486495
const layer = ProviderTurnStart.layer.pipe(
487496
Layer.provide(
488497
Layer.mergeAll(
@@ -491,9 +500,16 @@ function makeLocalCommandHarness(input: {
491500
}),
492501
Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),
493502
IdAllocator.layer,
494-
FileSystem.layerNoop({}),
495-
Layer.mock(GitWorkflow.GitWorkflowService)({}),
496-
Layer.mock(ProjectService.ProjectService)({}),
503+
Path.layer,
504+
FileSystem.layerNoop({ exists }),
505+
Layer.mock(GitWorkflow.GitWorkflowService)({
506+
pruneWorktrees: () => Effect.void,
507+
createWorktree,
508+
}),
509+
Layer.mock(ProjectService.ProjectService)({
510+
getById: () =>
511+
Effect.succeed(Option.some({ workspaceRoot: "/tmp/native-account-project" } as never)),
512+
}),
497513
Layer.mock(ProjectionStore.ProjectionStoreV2)({
498514
getTurnStartContext: () =>
499515
Effect.succeed({
@@ -531,6 +547,8 @@ function makeLocalCommandHarness(input: {
531547
),
532548
);
533549
return {
550+
exists,
551+
createWorktree,
534552
open,
535553
writeIfRunCurrent,
536554
startRootRun,
@@ -553,6 +571,86 @@ function makeLocalCommandHarness(input: {
553571
};
554572
}
555573

574+
effectIt.effect("waits for cleanup and repairs its checkout before opening a provider", () =>
575+
Effect.gen(function* () {
576+
const worktreePath = "/tmp/provider-start-cleanup-first";
577+
const releaseCleanup = yield* Deferred.make<void>();
578+
let checkoutExists = true;
579+
const harness = makeLocalCommandHarness({
580+
text: "Continue",
581+
worktreePath,
582+
failReadsAfterRunning: true,
583+
});
584+
harness.exists.mockImplementation(() => Effect.sync(() => checkoutExists));
585+
harness.createWorktree.mockImplementation(() =>
586+
Effect.sync(() => {
587+
checkoutExists = true;
588+
return {} as never;
589+
}),
590+
);
591+
const open = harness.open.getMockImplementation()!;
592+
harness.open.mockImplementation(() => {
593+
expect(checkoutExists).toBe(true);
594+
return open();
595+
});
596+
const cleanup = yield* withWorkspaceLease(
597+
worktreePath,
598+
Deferred.await(releaseCleanup).pipe(
599+
Effect.andThen(
600+
Effect.sync(() => {
601+
checkoutExists = false;
602+
}),
603+
),
604+
),
605+
).pipe(Effect.forkChild({ startImmediately: true }));
606+
const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
607+
expect(harness.open).not.toHaveBeenCalled();
608+
expect(harness.exists).not.toHaveBeenCalled();
609+
yield* Deferred.succeed(releaseCleanup, undefined);
610+
yield* Fiber.join(cleanup);
611+
yield* Fiber.join(startup);
612+
expect(harness.createWorktree).toHaveBeenCalledWith({
613+
cwd: "/tmp/native-account-project",
614+
refName: "feature/restore",
615+
path: worktreePath,
616+
});
617+
expect(harness.open).toHaveBeenCalledOnce();
618+
expect(harness.startRootRun).toHaveBeenCalledOnce();
619+
}),
620+
);
621+
622+
effectIt.effect("keeps cleanup out until provider turn startup accepts the turn", () =>
623+
Effect.gen(function* () {
624+
const worktreePath = "/tmp/provider-start-before-cleanup";
625+
const startupEntered = yield* Deferred.make<void>();
626+
const releaseStartup = yield* Deferred.make<void>();
627+
const harness = makeLocalCommandHarness({
628+
text: "Continue",
629+
worktreePath,
630+
failReadsAfterRunning: true,
631+
});
632+
harness.startRootRun.mockImplementation(() =>
633+
Deferred.succeed(startupEntered, undefined).pipe(
634+
Effect.andThen(Deferred.await(releaseStartup)),
635+
),
636+
);
637+
const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
638+
yield* Deferred.await(startupEntered);
639+
let cleanupEntered = false;
640+
const cleanup = yield* withWorkspaceLease(
641+
worktreePath,
642+
Effect.sync(() => {
643+
cleanupEntered = true;
644+
}),
645+
).pipe(Effect.forkChild({ startImmediately: true }));
646+
expect(cleanupEntered).toBe(false);
647+
yield* Deferred.succeed(releaseStartup, undefined);
648+
yield* Fiber.join(startup);
649+
yield* Fiber.join(cleanup);
650+
expect(cleanupEntered).toBe(true);
651+
}),
652+
);
653+
556654
effectIt.effect("terminalizes a starting run when its provider session cannot open", () =>
557655
Effect.gen(function* () {
558656
const harness = makeLocalCommandHarness({

0 commit comments

Comments
 (0)