Skip to content

Commit ebe2236

Browse files
PR Batch Testerclaude
andcommitted
fix(server): concurrent thread deletions no longer orphan shared pages
Deleting a thread now reads whether other threads in its fork family still show its pages, but each deletion planned under only its own thread lock. Two deletions in the same family could each see the other as live before either committed, so neither cleaned up the shared pages. Thread deletion now plans and commits under one deletion lock, taken inside the thread lock by both Orchestrator thread.delete and project deletion. Nothing inside it takes a thread lock, so it adds no lock order. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 458dbc5 commit ebe2236

4 files changed

Lines changed: 165 additions & 19 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12364,7 +12364,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1236412364
const dispatchWithReceipt = Effect.fnUntraced(function* (command: OrchestrationV2ServerCommand) {
1236512365
const dispatch = threadDispatch.withLock(
1236612366
commandThreadId(command),
12367-
dispatchWithReceiptEffect(command),
12367+
command.type === "thread.delete"
12368+
? threadDispatch.withDeletionLock(dispatchWithReceiptEffect(command))
12369+
: dispatchWithReceiptEffect(command),
1236812370
);
1236912371
if (command.type !== "message.dispatch" || !mayStartSubagentFollowup(command))
1237012372
return yield* dispatch;
Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,27 @@
11
import type { ThreadId } from "@t3tools/contracts";
22
import * as KeyedLock from "@t3tools/shared/KeyedLock";
33
import * as Context from "effect/Context";
4+
import * as Effect from "effect/Effect";
45
import * as Layer from "effect/Layer";
6+
import * as Semaphore from "effect/Semaphore";
57

68
export class ThreadCommandExecutor extends Context.Service<
79
ThreadCommandExecutor,
8-
KeyedLock.KeyedLock<ThreadId>
10+
KeyedLock.KeyedLock<ThreadId> & {
11+
/**
12+
* Serializes thread deletion planning through its commit, since deleting one thread
13+
* reads whether other threads still show its pages. Take it inside the thread lock and
14+
* never take a thread lock while holding it.
15+
*/
16+
readonly withDeletionLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
17+
}
918
>()("t3/orchestration-v2/ThreadCommandExecutor") {}
1019

11-
export const layer = Layer.effect(ThreadCommandExecutor, KeyedLock.make<ThreadId>());
20+
export const layer = Layer.effect(
21+
ThreadCommandExecutor,
22+
Effect.gen(function* () {
23+
const threads = yield* KeyedLock.make<ThreadId>();
24+
const deletion = yield* Semaphore.make(1);
25+
return { ...threads, withDeletionLock: deletion.withPermit };
26+
}),
27+
);

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,7 @@ const layerProjectDeletionTest = Layer.mergeAll(
331331
RuntimeLayer.layer.pipe(Layer.provide(RuntimeLayer.layerProjectService)),
332332
RuntimeLayer.layerProjectService,
333333
RuntimeLayer.layerEventSink,
334+
ProjectionStore.layer,
334335
ThreadCommandExecutor.layer,
335336
).pipe(
336337
Layer.provide(
@@ -592,6 +593,129 @@ it.layer(layerProjectDeletionTest)("project deletion during thread commands", (i
592593
);
593594
}),
594595
);
596+
597+
// Deleting a thread reads whether other threads still show its pages, so a second
598+
// deletion must not plan against a first one that has not committed yet.
599+
it.effect("plans a thread deletion only after an in-flight deletion commits", () =>
600+
Effect.gen(function* () {
601+
const projects = yield* ProjectService.ProjectService;
602+
const orchestrator = yield* Orchestrator.OrchestratorV2;
603+
const eventSink = yield* EventSink.EventSinkV2;
604+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
605+
const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
606+
const threadProjectId = ProjectId.make("runtime-delete-serial-thread-project");
607+
const deletedProjectId = ProjectId.make("runtime-delete-serial-project");
608+
const threadId = ThreadId.make("runtime-delete-serial-thread");
609+
const projectThreadId = ThreadId.make("runtime-delete-serial-project-thread");
610+
const threadDeleteId = CommandId.make("runtime-delete-serial-thread-delete");
611+
for (const [projectId, threadIdInProject] of [
612+
[threadProjectId, threadId],
613+
[deletedProjectId, projectThreadId],
614+
] as const) {
615+
yield* projects.create({
616+
commandId: CommandId.make(`runtime-delete-serial-create:${projectId}`),
617+
projectId,
618+
title: "Serial deletion",
619+
workspaceRoot: `/work/serial-deletion/${projectId}`,
620+
});
621+
yield* orchestrator.dispatch({
622+
type: "thread.create",
623+
commandId: CommandId.make(`runtime-delete-serial-create:${threadIdInProject}`),
624+
createdBy: "user",
625+
creationSource: "web",
626+
threadId: threadIdInProject,
627+
projectId,
628+
title: "Serial deletion",
629+
modelSelection,
630+
runtimeMode: "full-access",
631+
interactionMode: "default",
632+
branch: null,
633+
worktreePath: null,
634+
});
635+
}
636+
637+
const threadDeleteReady = yield* Deferred.make<void>();
638+
const releaseThreadDelete = yield* Deferred.make<void>();
639+
const projectDeleteQueued = yield* Deferred.make<void>();
640+
const projectDeletePlanning = yield* Deferred.make<void>();
641+
let threadDeleteCommitted = false;
642+
const projectDeletePlannedAfterCommit: Array<boolean> = [];
643+
const commitCommand = eventSink.commitCommand;
644+
const commitSpy = vi.spyOn(eventSink, "commitCommand").mockImplementation((input) =>
645+
input.commandId === threadDeleteId
646+
? Deferred.succeed(threadDeleteReady, undefined).pipe(
647+
Effect.andThen(Deferred.await(releaseThreadDelete)),
648+
Effect.andThen(commitCommand(input)),
649+
Effect.tap(() =>
650+
Effect.sync(() => {
651+
threadDeleteCommitted = true;
652+
}),
653+
),
654+
)
655+
: commitCommand(input),
656+
);
657+
const getThreadAttachmentIds = projectionStore.getThreadAttachmentIds;
658+
const readSpy = vi
659+
.spyOn(projectionStore, "getThreadAttachmentIds")
660+
.mockImplementation((id) =>
661+
id === projectThreadId
662+
? Effect.sync(() => projectDeletePlannedAfterCommit.push(threadDeleteCommitted)).pipe(
663+
Effect.andThen(Deferred.succeed(projectDeletePlanning, undefined)),
664+
Effect.andThen(getThreadAttachmentIds(id)),
665+
)
666+
: getThreadAttachmentIds(id),
667+
);
668+
const withDeletionLock = executor.withDeletionLock;
669+
let deletionLockRequests = 0;
670+
const observeDeletionLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withDeletionLock"] =
671+
(effect) =>
672+
++deletionLockRequests === 2
673+
? Deferred.succeed(projectDeleteQueued, undefined).pipe(
674+
Effect.andThen(withDeletionLock(effect)),
675+
)
676+
: withDeletionLock(effect);
677+
const lockSpy = vi
678+
.spyOn(executor, "withDeletionLock")
679+
.mockImplementation(observeDeletionLock);
680+
yield* Effect.gen(function* () {
681+
const threadDeleteFiber = yield* orchestrator
682+
.dispatch({ type: "thread.delete", commandId: threadDeleteId, threadId })
683+
.pipe(Effect.forkChild({ startImmediately: true }));
684+
yield* Effect.raceFirst(
685+
Deferred.await(threadDeleteReady),
686+
Fiber.join(threadDeleteFiber).pipe(
687+
Effect.andThen(Effect.die("The thread deletion completed before its commit barrier.")),
688+
),
689+
);
690+
const projectDeleteFiber = yield* projects
691+
.delete({
692+
commandId: CommandId.make("runtime-delete-serial-project-delete"),
693+
projectId: deletedProjectId,
694+
force: true,
695+
})
696+
.pipe(Effect.forkChild({ startImmediately: true }));
697+
yield* Effect.raceFirst(
698+
Deferred.await(projectDeleteQueued),
699+
Deferred.await(projectDeletePlanning),
700+
);
701+
yield* Deferred.succeed(releaseThreadDelete, undefined);
702+
yield* Fiber.join(threadDeleteFiber);
703+
yield* Fiber.join(projectDeleteFiber);
704+
assert.deepEqual(projectDeletePlannedAfterCommit, [true]);
705+
assert.isNotNull(
706+
(yield* orchestrator.getThreadProjection(projectThreadId)).thread.deletedAt,
707+
);
708+
}).pipe(
709+
Effect.ensuring(
710+
Effect.sync(() => {
711+
commitSpy.mockRestore();
712+
readSpy.mockRestore();
713+
lockSpy.mockRestore();
714+
}),
715+
),
716+
);
717+
}),
718+
);
595719
});
596720

597721
const layerSharedApplicationDataPlaneTest = Layer.mergeAll(

‎apps/server/src/project/ProjectService.ts‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -412,22 +412,26 @@ export const make = Effect.gen(function* () {
412412
commandId: CommandId.make(`${input.commandId}:delete-thread:${threadId}`),
413413
threadId,
414414
};
415-
const now = yield* DateTime.now;
416-
const plan = yield* planThreadDeletion({
417-
command,
418-
projection,
419-
attachmentIds: yield* threadProjections.getThreadAttachmentIds(threadId),
420-
now,
421-
idAllocator,
422-
});
423-
const committed = yield* eventSink.commitCommand({
424-
commandId: command.commandId,
425-
commandType: command.type,
426-
threadId,
427-
acceptedAt: now,
428-
events: plan.events,
429-
effects: plan.effects,
430-
});
415+
const committed = yield* threadCommands.withDeletionLock(
416+
Effect.gen(function* () {
417+
const now = yield* DateTime.now;
418+
const plan = yield* planThreadDeletion({
419+
command,
420+
projection,
421+
attachmentIds: yield* threadProjections.getThreadAttachmentIds(threadId),
422+
now,
423+
idAllocator,
424+
});
425+
return yield* eventSink.commitCommand({
426+
commandId: command.commandId,
427+
commandType: command.type,
428+
threadId,
429+
acceptedAt: now,
430+
events: plan.events,
431+
effects: plan.effects,
432+
});
433+
}),
434+
);
431435
if (
432436
committed.receipt.threadId !== command.threadId ||
433437
committed.receipt.commandType !== command.type

0 commit comments

Comments
 (0)