Skip to content

Commit 0fa225d

Browse files
committed
fix(server): fence session commits and protect live worktrees
1 parent d51bc78 commit 0fa225d

6 files changed

Lines changed: 303 additions & 55 deletions

File tree

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ export type EventSinkV2Error = typeof EventSinkV2Error.Type;
6868

6969
export interface EventSinkV2Shape {
7070
readonly write: (input: {
71+
/** Synchronous in-memory handoff after commit, before publication or pending cancellation. */
72+
readonly onCommitted?: (events: ReadonlyArray<OrchestrationV2StoredEvent>) => void;
7173
/** Return no events if another thread owns this native session at commit time. */
7274
readonly unlessNativeThreadOwned?: Pick<
7375
OrchestrationV2ProviderThread,
@@ -409,6 +411,7 @@ const layerBase: Layer.Layer<
409411

410412
const writeEffect = Effect.fn("orchestrationV2.EventSink.write")(function* (
411413
input: Parameters<EventSinkV2Shape["writeWithEffects"]>[0],
414+
onCommitted?: Parameters<EventSinkV2Shape["write"]>[0]["onCommitted"],
412415
) {
413416
yield* Effect.annotateCurrentSpan({
414417
"orchestration_v2.command_id": input.commandId ?? null,
@@ -452,6 +455,7 @@ const layerBase: Layer.Layer<
452455
}).pipe(Effect.withTracerEnabled(false)),
453456
({ committed, outcomes }) =>
454457
Effect.gen(function* () {
458+
yield* Effect.sync(() => onCommitted?.(committed));
455459
if (input.effects.length > 0) {
456460
yield* effectOutbox.notifyAvailable(input.effects.length);
457461
}
@@ -879,7 +883,7 @@ const layerBase: Layer.Layer<
879883

880884
return EventSinkV2.of({
881885
write: (input) =>
882-
writeEffect({ ...input, effects: [] }).pipe(
886+
writeEffect({ ...input, effects: [] }, input.onCommitted).pipe(
883887
Effect.mapError(
884888
(cause) =>
885889
new EventSinkWriteError({

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 98 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -763,7 +763,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
763763
}),
764764
);
765765

766-
it.effect("replays shared provider-session payloads across every bound thread", () =>
766+
it.effect("replays provider-session payloads independently for each bound thread", () =>
767767
Effect.gen(function* () {
768768
const eventSink = yield* EventSink.EventSinkV2;
769769
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
@@ -819,12 +819,20 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
819819
],
820820
});
821821

822-
assert.equal(
823-
(yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions[0]?.cwd,
824-
secondSession.cwd,
825-
);
822+
const assertBindings = Effect.gen(function* () {
823+
assert.deepEqual(
824+
(yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions,
825+
[firstSession],
826+
);
827+
assert.deepEqual(
828+
(yield* projectionStore.getThreadProjection(secondThreadId)).providerSessions,
829+
[secondSession],
830+
);
831+
});
832+
yield* assertBindings;
826833
assert.isTrue((yield* maintenance.verify).valid);
827834
assert.isTrue((yield* maintenance.rebuild).valid);
835+
yield* assertBindings;
828836
}),
829837
);
830838

@@ -3666,3 +3674,88 @@ it.effect.each(["cancel transaction", "fail transaction", "cancel publication"]
36663674
);
36673675
}).pipe(Effect.provide(layerDatabase)),
36683676
);
3677+
3678+
it.effect.each(["cancel transaction", "fail transaction", "cancel publication"] as const)(
3679+
"hands off committed ownership only after a successful transaction during %s",
3680+
(scenario) =>
3681+
Effect.gen(function* () {
3682+
const paused = yield* Deferred.make<void>();
3683+
const resume = yield* Deferred.make<void>();
3684+
const pause = Deferred.succeed(paused, undefined).pipe(
3685+
Effect.andThen(Deferred.await(resume)),
3686+
);
3687+
const committed = scenario === "cancel publication";
3688+
const stores = Layer.mergeAll(
3689+
layerStoresProvided,
3690+
Layer.effect(
3691+
EventStore.EventStoreV2,
3692+
Effect.gen(function* () {
3693+
const delegate = yield* EventStore.EventStoreV2;
3694+
return EventStore.EventStoreV2.of({
3695+
...delegate,
3696+
publishCommitted: (events) =>
3697+
(committed ? pause : Effect.void).pipe(
3698+
Effect.andThen(delegate.publishCommitted(events)),
3699+
),
3700+
});
3701+
}),
3702+
).pipe(Layer.provide(layerEventStoreProvided)),
3703+
Layer.effect(
3704+
EffectOutbox.EffectOutboxV2,
3705+
Effect.gen(function* () {
3706+
const delegate = yield* EffectOutbox.EffectOutboxV2;
3707+
return EffectOutbox.EffectOutboxV2.of({
3708+
...delegate,
3709+
enqueue: (effects) =>
3710+
delegate
3711+
.enqueue(effects)
3712+
.pipe(
3713+
Effect.andThen(committed ? Effect.void : pause),
3714+
Effect.andThen(
3715+
scenario === "fail transaction"
3716+
? Effect.die("Injected rollback")
3717+
: Effect.void,
3718+
),
3719+
),
3720+
});
3721+
}),
3722+
).pipe(Layer.provide(layerEffectOutboxProvided)),
3723+
layerCommandReceiptStoreProvided,
3724+
ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
3725+
TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
3726+
);
3727+
yield* Effect.gen(function* () {
3728+
const sink = yield* EventSink.EventSinkV2;
3729+
const projections = yield* ProjectionStore.ProjectionStoreV2;
3730+
const eventStore = yield* EventStore.EventStoreV2;
3731+
const now = yield* DateTime.now;
3732+
const thread = makeThread(ThreadId.make("thread:commit-handoff"), now);
3733+
const events = [threadCreatedEvent({ id: "event:commit-handoff", thread, now })];
3734+
const handedOff: Array<OrchestrationV2DomainEvent> = [];
3735+
const writer = yield* sink
3736+
.write({
3737+
events,
3738+
onCommitted: (stored) => {
3739+
handedOff.push(...stored.map((entry) => entry.event));
3740+
},
3741+
})
3742+
.pipe(Effect.forkScoped);
3743+
yield* Deferred.await(paused);
3744+
// The handoff precedes publication, and never sees transaction work that can roll back.
3745+
assert.deepEqual(handedOff, committed ? events : []);
3746+
if (scenario !== "fail transaction") writer.interruptUnsafe();
3747+
if (scenario !== "cancel transaction") yield* Deferred.succeed(resume, undefined);
3748+
assert.equal((yield* Fiber.await(writer))._tag, "Failure");
3749+
assert.deepEqual(handedOff, committed ? events : []);
3750+
assert.equal(
3751+
(yield* projections.getThreadShell(thread.id))?.id ?? null,
3752+
committed ? thread.id : null,
3753+
);
3754+
const stored = yield* eventStore.read({ threadId: thread.id }).pipe(Stream.runCollect);
3755+
assert.deepEqual(
3756+
Array.from(stored, (entry) => entry.event),
3757+
committed ? events : [],
3758+
);
3759+
}).pipe(Effect.provide(EventSink.layerFromStores.pipe(Layer.provideMerge(stores))));
3760+
}).pipe(Effect.provide(layerDatabase)),
3761+
);

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ const layerFlakyReleaseEventSink = (flaky: FlakyReleaseWrites) =>
122122
).pipe(Layer.provide(layerTestEventSink));
123123

124124
const layerPausingAttachEventSink = (pause: {
125+
readonly afterCommitResume?: Deferred.Deferred<void>;
125126
readonly armed: Ref.Ref<boolean>;
126127
readonly paused: Deferred.Deferred<void>;
127128
}) =>
@@ -135,6 +136,14 @@ const layerPausingAttachEventSink = (pause: {
135136
Effect.gen(function* () {
136137
const attach = input.events.some((event) => event.type === "provider-session.attached");
137138
if (attach && (yield* Ref.getAndSet(pause.armed, false))) {
139+
const resume = pause.afterCommitResume;
140+
if (resume !== undefined) {
141+
return yield* delegate.write(input).pipe(
142+
Effect.tap(() => Deferred.succeed(pause.paused, undefined)),
143+
Effect.tap(() => Deferred.await(resume)),
144+
Effect.uninterruptible,
145+
);
146+
}
138147
yield* Deferred.succeed(pause.paused, undefined);
139148
return yield* Effect.never;
140149
}
@@ -3927,3 +3936,85 @@ it.effect.each([false, true])(
39273936
);
39283937
}),
39293938
);
3939+
3940+
it.effect.each([false, true])(
3941+
"ProviderSessionManagerV2 preserves release ownership when attachment cancellation follows commit=%s",
3942+
(committed) =>
3943+
Effect.gen(function* () {
3944+
const state = yield* Ref.make(emptyState);
3945+
const closing = yield* Deferred.make<void>();
3946+
const resumeOld = yield* Deferred.make<void>();
3947+
const first = yield* Ref.make(true);
3948+
const armed = yield* Ref.make(false);
3949+
const paused = yield* Deferred.make<void>();
3950+
const resumeAttach = yield* Deferred.make<void>();
3951+
yield* Effect.gen(function* () {
3952+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
3953+
const sink = yield* EventSink.EventSinkV2;
3954+
const ids = yield* IdAllocator.IdAllocatorV2;
3955+
const store = yield* ProjectionStore.ProjectionStoreV2;
3956+
const threadId = ThreadId.make("thread:attachment-commit-interrupt");
3957+
const providerSessionId = yield* ids.allocate.providerSession({
3958+
providerInstanceId: modelSelection.instanceId,
3959+
threadId,
3960+
});
3961+
yield* sink.write({
3962+
events: [
3963+
yield* makeThreadCreatedEvent({
3964+
idAllocator: ids,
3965+
threadId,
3966+
now: yield* DateTime.now,
3967+
}),
3968+
],
3969+
});
3970+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3971+
const oldClose = yield* manager
3972+
.release({ providerSessionId, reason: "runtime_error", detail: "old runtime failure" })
3973+
.pipe(Effect.forkScoped);
3974+
yield* Deferred.await(closing);
3975+
yield* Ref.set(armed, true);
3976+
const replacementOpen = yield* manager
3977+
.open({
3978+
threadId,
3979+
providerSessionId,
3980+
modelSelection,
3981+
runtimePolicy,
3982+
})
3983+
.pipe(Effect.forkScoped);
3984+
yield* Deferred.await(paused);
3985+
// Post-commit cancellation waits on masked completion; pre-commit cancellation must abort.
3986+
replacementOpen.interruptUnsafe();
3987+
yield* Deferred.succeed(resumeAttach, undefined);
3988+
const openExit = yield* Fiber.await(replacementOpen);
3989+
assert.isTrue(Exit.isFailure(openExit));
3990+
const replacement = (yield* store.getThreadProjection(threadId)).providerSessions[0];
3991+
assert.equal(replacement?.status, "stopped");
3992+
yield* Deferred.succeed(resumeOld, undefined);
3993+
yield* Fiber.join(oldClose);
3994+
const after = (yield* store.getThreadProjection(threadId)).providerSessions[0];
3995+
assert.equal(after?.status, committed ? replacement?.status : "error");
3996+
assert.equal(after?.lastError, committed ? replacement?.lastError : "old runtime failure");
3997+
}).pipe(
3998+
Effect.provide(
3999+
layerTest({
4000+
state,
4001+
idleTimeoutMs: 60_000,
4002+
pauseAttachWrite: {
4003+
armed,
4004+
paused,
4005+
...(committed ? { afterCommitResume: resumeAttach } : {}),
4006+
},
4007+
beforeClose: Ref.getAndSet(first, false).pipe(
4008+
Effect.flatMap((isFirst) =>
4009+
isFirst
4010+
? Deferred.succeed(closing, undefined).pipe(
4011+
Effect.andThen(Deferred.await(resumeOld)),
4012+
)
4013+
: Effect.void,
4014+
),
4015+
),
4016+
}),
4017+
),
4018+
);
4019+
}),
4020+
);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -617,20 +617,22 @@ export const layerWithOptions = (
617617
}),
618618
);
619619
if (events.length > 0) {
620-
yield* Effect.uninterruptibleMask((restore) =>
621-
restore(eventSink.write({ events })).pipe(
622-
Effect.tap(() =>
623-
Effect.sync(() => {
624-
if (input.type !== "provider-session.attached") return;
625-
for (const release of pendingReleaseThreads.values()) {
626-
if (release.providerSessionId === input.runtime.providerSessionId) {
627-
for (const event of events) release.threadIds.delete(event.threadId);
628-
}
629-
}
630-
}),
631-
),
632-
),
633-
);
620+
yield* eventSink.write({
621+
events,
622+
// EventSink runs this inside its masked post-commit boundary. A pending
623+
// interrupt must not leave the old release owning a durable attachment.
624+
onCommitted: (committed) => {
625+
if (input.type !== "provider-session.attached") return;
626+
for (const [runtime, release] of pendingReleaseThreads) {
627+
if (
628+
runtime !== input.runtime &&
629+
release.providerSessionId === input.runtime.providerSessionId
630+
) {
631+
for (const event of committed) release.threadIds.delete(event.event.threadId);
632+
}
633+
}
634+
},
635+
});
634636
}
635637
},
636638
(effect, input) =>

0 commit comments

Comments
 (0)