Skip to content

Commit d96b8ce

Browse files
committed
fix(server): read thread families without lock cycles
1 parent cc1edc5 commit d96b8ce

3 files changed

Lines changed: 114 additions & 15 deletions

File tree

‎apps/server/src/orchestration-v2/TemporaryThreadStorage.ts‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
OrchestrationV2ProviderSessionJson,
77
} from "@t3tools/contracts";
88
import * as Context from "effect/Context";
9+
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
910
import * as DateTime from "effect/DateTime";
1011
import * as Deferred from "effect/Deferred";
1112
import * as Effect from "effect/Effect";
@@ -109,6 +110,7 @@ export class TemporaryThreadStores extends Context.Service<
109110
{
110111
readonly durable: RawStores;
111112
readonly volatile: RawStores;
113+
readonly getShellSnapshot: ProjectionStore.ProjectionStoreV2["Service"]["getShellSnapshot"];
112114
}
113115
>()("t3/orchestration-v2/TemporaryThreadStorage/TemporaryThreadStores") {}
114116

@@ -223,6 +225,45 @@ const make = Effect.gen(function* () {
223225
return ids.length === 0 ? durable : target(ids[0]);
224226
});
225227
const locked = <A, E, R>(effect: Effect.Effect<A, E, R>) => gate.withPermit(effect);
228+
const getShellSnapshot = Effect.fn("TemporaryThreadStorage.getShellSnapshot")(
229+
function* (options?: ProjectionStore.ShellSnapshotOptions) {
230+
const read = yield* durable.sql
231+
.withTransaction(
232+
Effect.all({
233+
disk: durable.projectionStore.readShellSnapshot(options),
234+
memory: volatile.projectionStore.readShellSnapshot(options),
235+
snapshotSequence: durable.applicationEvents.latestApplicationSequence,
236+
}),
237+
)
238+
.pipe(NodeSqliteClient.readOnly);
239+
const disk = yield* read.disk;
240+
const memory = yield* read.memory;
241+
return {
242+
...disk,
243+
snapshotSequence: read.snapshotSequence,
244+
threads: [
245+
...disk.threads,
246+
...memory.threads
247+
.filter((thread) => temporary(thread.id))
248+
.map((thread) => ({ ...thread, temporary: true })),
249+
],
250+
archivedThreads: [
251+
...disk.archivedThreads,
252+
...memory.archivedThreads
253+
.filter((thread) => temporary(thread.id))
254+
.map((thread) => ({ ...thread, temporary: true })),
255+
],
256+
};
257+
},
258+
locked,
259+
Effect.mapError(
260+
(cause) =>
261+
new ProjectionStore.ProjectionStoreReadError({
262+
threadId: ThreadId.make("thread:shell"),
263+
cause,
264+
}),
265+
),
266+
);
226267
const withThread = <A, E, R>(id: ThreadId, use: (stores: RawStores) => Effect.Effect<A, E, R>) =>
227268
locked(Effect.suspend(() => use(target(id))));
228269
const withWrite = <A, E, R>(
@@ -1195,7 +1236,7 @@ const make = Effect.gen(function* () {
11951236
});
11961237
return Context.empty().pipe(
11971238
Context.add(TemporaryThreadStorage, storage),
1198-
Context.add(TemporaryThreadStores, { durable, volatile }),
1239+
Context.add(TemporaryThreadStores, { durable, volatile, getShellSnapshot }),
11991240
Context.add(ProjectStore.ProjectStoreV2, durable.projectStore),
12001241
Context.add(EventSink.EventSinkV2, eventSink),
12011242
Context.add(EventStore.EventStoreV2, eventStore),

‎apps/server/src/ws.test.ts‎

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import * as NodeServices from "@effect/platform-node/NodeServices";
22
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
3-
import { assert, it } from "@effect/vitest";
3+
import { assert, it, vi } from "@effect/vitest";
44
import {
55
AuthSessionId,
66
CommandId,
@@ -42,6 +42,7 @@ import * as Stream from "effect/Stream";
4242
import * as TestClock from "effect/testing/TestClock";
4343
import * as Scope from "effect/Scope";
4444
import * as Exit from "effect/Exit";
45+
import * as Fiber from "effect/Fiber";
4546
import { Headers } from "effect/http";
4647
import { Rpc, RpcGroup, RpcMessage, RpcServer } from "effect/rpc";
4748
import * as SqlClient from "effect/sql/SqlClient";
@@ -59,6 +60,7 @@ import * as CheckpointDiffQuery from "./checkpointing/CheckpointDiffQuery.ts";
5960
import * as CheckpointStore from "./checkpointing/CheckpointStore.ts";
6061
import { CheckpointTurnRangeUnavailableError } from "./checkpointing/Errors.ts";
6162
import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts";
63+
import * as CommandReceiptStore from "./orchestration-v2/CommandReceiptStore.ts";
6264
import * as CodexInstallation from "./provider/CodexInstallation.ts";
6365
import * as DeviceService from "./device/DeviceService.ts";
6466
import * as DirectEndpoints from "./environment/DirectEndpoints.ts";
@@ -621,6 +623,10 @@ it.effect.each(["family", "worktree"] as const)(
621623
yield* projection.apply(create(id, parent));
622624
}
623625
let keepDuringRead = false;
626+
let checkLockOrder = false;
627+
const readerHeld = yield* Deferred.make<void>();
628+
const receiptHasStorage = yield* Deferred.make<void>();
629+
const lockCycle = yield* Deferred.make<void>();
624630
const responses =
625631
yield* Queue.unbounded<
626632
Exit.Exit<
@@ -679,9 +685,40 @@ it.effect.each(["family", "worktree"] as const)(
679685
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
680686
namedProjectsRoot: process.cwd(),
681687
}),
688+
Layer.succeed(TemporaryThreadStorage.TemporaryThreadStores, {
689+
...stores,
690+
getShellSnapshot: (options) =>
691+
stores.getShellSnapshot(options).pipe(
692+
Effect.tap(() => {
693+
if (!keepDuringRead) return Effect.void;
694+
keepDuringRead = false;
695+
return storage
696+
.keep({
697+
threadId: ThreadId.make("racing"),
698+
commandId: CommandId.make("keep-racing"),
699+
})
700+
.pipe(
701+
Effect.mapError(
702+
(cause) =>
703+
new ProjectionStore.ProjectionStoreReadError({
704+
threadId: ThreadId.make("racing"),
705+
cause,
706+
}),
707+
),
708+
);
709+
}),
710+
),
711+
}),
682712
Layer.mock(ThreadManagement.ThreadManagementService)({
683713
readShellSnapshot: (options) =>
684-
projection.readShellSnapshot(options).pipe(
714+
Effect.gen(function* () {
715+
if (checkLockOrder) {
716+
yield* Deferred.succeed(readerHeld, undefined);
717+
yield* Deferred.await(receiptHasStorage);
718+
yield* Deferred.succeed(lockCycle, undefined);
719+
}
720+
return yield* projection.readShellSnapshot(options);
721+
}).pipe(
685722
Effect.map((decode) =>
686723
decode.pipe(
687724
Effect.tap(() => {
@@ -784,6 +821,30 @@ it.effect.each(["family", "worktree"] as const)(
784821
? ["racing-child"]
785822
: ["private", "private-child", "racing", "racing-child", "saved", "saved-child"],
786823
);
824+
checkLockOrder = true;
825+
const familyRead = yield* read(true).pipe(Effect.forkChild);
826+
const acquiredReader = yield* Effect.raceFirst(
827+
Deferred.await(readerHeld).pipe(Effect.as(true)),
828+
Fiber.join(familyRead).pipe(Effect.as(false)),
829+
);
830+
const getReceipt = stores.volatile.applicationReceipts.getByCommandId;
831+
using _receipt = vi
832+
.spyOn(stores.volatile.applicationReceipts, "getByCommandId")
833+
.mockImplementation((input) =>
834+
Deferred.succeed(receiptHasStorage, undefined).pipe(Effect.andThen(getReceipt(input))),
835+
);
836+
const receiptRead = yield* Context.get(services, CommandReceiptStore.CommandReceiptStoreV2)
837+
.getByCommandId(CommandId.make("lock-order-receipt"))
838+
.pipe(Effect.forkChild);
839+
if (acquiredReader) yield* Deferred.await(receiptHasStorage);
840+
const completed = yield* Effect.raceFirst(
841+
Effect.all([Fiber.join(familyRead), Fiber.join(receiptRead)]).pipe(Effect.as(true)),
842+
Deferred.await(lockCycle).pipe(Effect.as(false)),
843+
);
844+
assert.isTrue(
845+
completed,
846+
"Shell and receipt reads must finish without a storage/SQL lock cycle",
847+
);
787848
}).pipe(Effect.scoped),
788849
);
789850

‎apps/server/src/ws.ts‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2055,20 +2055,17 @@ export const layerWsRpc = (
20552055
acceptTemporaryThreads: boolean | undefined,
20562056
message: string,
20572057
) {
2058-
const { threads, snapshotSequence } = yield* loadShellSnapshotParts({
2059-
sql,
2060-
readThreads: threadManagement.readShellSnapshot({
2058+
const { threads, snapshotSequence } = yield* rawStores
2059+
.getShellSnapshot({
20612060
location: "active",
20622061
...options,
2063-
}),
2064-
listProjects: Effect.succeed([]),
2065-
latestSequence: applicationEvents.latestApplicationSequence,
2066-
}).pipe(
2067-
Effect.mapError(
2068-
(cause) => new OrchestrationV2GetShellSnapshotError({ message, cause }),
2069-
),
2070-
);
2071-
const readable = yield* Effect.filter(threads.threads, (thread) =>
2062+
})
2063+
.pipe(
2064+
Effect.mapError(
2065+
(cause) => new OrchestrationV2GetShellSnapshotError({ message, cause }),
2066+
),
2067+
);
2068+
const readable = yield* Effect.filter(threads, (thread) =>
20722069
TemporaryThreadTransport.getReadState(thread.id, acceptTemporaryThreads).pipe(
20732070
Effect.map(
20742071
(state) => state.allowed && (thread.temporary !== true || state.temporary),

0 commit comments

Comments
 (0)