Skip to content

Commit a6b1996

Browse files
t3dotggclaude
andcommitted
fix(server): release cleanup no longer overwrites a provider update
The roster cleanup read the provider thread and wrote it back in two steps, so a provider update landing in between was overwritten. It now commits through a compare-and-set on the row's updatedAt and retries. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent bd435e1 commit a6b1996

3 files changed

Lines changed: 144 additions & 30 deletions

File tree

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,22 @@ export interface EventSinkV2Shape {
119119
},
120120
EventSinkV2Error
121121
>;
122+
/**
123+
* Atomically commit only while the provider thread row still has the
124+
* `updatedAt` its writer read, so a read-modify-write of the row cannot
125+
* overwrite an update a provider published in between.
126+
*/
127+
readonly writeIfProviderThreadUnchanged: (input: {
128+
readonly providerThreadId: ProviderThreadId;
129+
readonly expectedUpdatedAt: DateTime.Utc;
130+
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
131+
}) => Effect.Effect<
132+
{
133+
readonly committed: boolean;
134+
readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
135+
},
136+
EventSinkV2Error
137+
>;
122138
readonly commitCommand: (input: {
123139
readonly commandId: CommandId;
124140
readonly threadId: ThreadId;
@@ -509,6 +525,38 @@ const layerBase: Layer.Layer<
509525
);
510526
});
511527

528+
const writeIfProviderThreadUnchangedEffect = Effect.fn(
529+
"orchestrationV2.EventSink.writeIfProviderThreadUnchanged",
530+
)(function* (input: Parameters<EventSinkV2Shape["writeIfProviderThreadUnchanged"]>[0]) {
531+
yield* Effect.annotateCurrentSpan({
532+
"orchestration_v2.event_count": input.events.length,
533+
"orchestration_v2.provider_thread_id": input.providerThreadId,
534+
});
535+
536+
return yield* commitThenPublish(
537+
Effect.gen(function* () {
538+
const rows = yield* sql<{ readonly updated_at: string | null }>`
539+
SELECT updated_at
540+
FROM orchestration_v2_projection_provider_threads
541+
WHERE provider_thread_id = ${input.providerThreadId}
542+
LIMIT 1
543+
`;
544+
if (rows[0]?.updated_at !== DateTime.formatIso(input.expectedUpdatedAt)) {
545+
return {
546+
committed: false as const,
547+
storedEvents: [] as ReadonlyArray<OrchestrationV2StoredEvent>,
548+
};
549+
}
550+
551+
const normalized = yield* normalizeEvents(input.events);
552+
const storedEvents = yield* eventStore.append({ events: normalized });
553+
yield* applyStoredEvents(storedEvents);
554+
return { committed: true as const, storedEvents };
555+
}),
556+
(result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
557+
);
558+
});
559+
512560
const existingCommandResult = (commandId: CommandId) =>
513561
Effect.gen(function* () {
514562
const existing = yield* commandReceipts.getByCommandId(commandId);
@@ -804,6 +852,12 @@ const layerBase: Layer.Layer<
804852
}),
805853
),
806854
),
855+
writeIfProviderThreadUnchanged: (input) =>
856+
writeIfProviderThreadUnchangedEffect(input).pipe(
857+
Effect.mapError(
858+
(cause) => new EventSinkWriteError({ eventCount: input.events.length, cause }),
859+
),
860+
),
807861
commitCommand: (input) =>
808862
commitCommandEffect(input).pipe(
809863
Effect.mapError(

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,22 @@ const layerPausingAttachEventSink = (pause: {
147147
}),
148148
).pipe(Layer.provide(layerTestEventSink));
149149

150+
const layerBeforeUnchangedWriteEventSink = (before: Ref.Ref<Effect.Effect<void>>) =>
151+
Layer.effect(
152+
EventSink.EventSinkV2,
153+
Effect.gen(function* () {
154+
const delegate = yield* EventSink.EventSinkV2;
155+
return EventSink.EventSinkV2.of({
156+
...delegate,
157+
writeIfProviderThreadUnchanged: (input) =>
158+
Ref.getAndSet(before, Effect.void).pipe(
159+
Effect.flatten,
160+
Effect.andThen(delegate.writeIfProviderThreadUnchanged(input)),
161+
),
162+
});
163+
}),
164+
).pipe(Layer.provide(layerTestEventSink));
165+
150166
const CodexCapabilities: OrchestrationV2ProviderCapabilities = CodexProviderCapabilitiesV2;
151167
const ExclusiveCapabilities: OrchestrationV2ProviderCapabilities = {
152168
...CodexCapabilities,
@@ -458,6 +474,8 @@ function layerTest(input: {
458474
readonly failReleaseEventWrites?: boolean;
459475
readonly flakyReleaseWrites?: FlakyReleaseWrites;
460476
readonly pauseAttachWrite?: Parameters<typeof layerPausingAttachEventSink>[0];
477+
/** Runs once, right before the next provider-thread compare-and-set write. */
478+
readonly beforeUnchangedWrite?: Ref.Ref<Effect.Effect<void>>;
461479
/** Once armed, holds the next credential lookup until it is interrupted. */
462480
readonly pauseResolve?: {
463481
readonly armed: Ref.Ref<boolean>;
@@ -478,9 +496,11 @@ function layerTest(input: {
478496
? layerFlakyReleaseEventSink(input.flakyReleaseWrites)
479497
: input.pauseAttachWrite !== undefined
480498
? layerPausingAttachEventSink(input.pauseAttachWrite)
481-
: input.failReleaseEventWrites
482-
? layerFailingReleaseEventSink
483-
: layerTestEventSink;
499+
: input.beforeUnchangedWrite !== undefined
500+
? layerBeforeUnchangedWriteEventSink(input.beforeUnchangedWrite)
501+
: input.failReleaseEventWrites
502+
? layerFailingReleaseEventSink
503+
: layerTestEventSink;
484504
const layerRegistry = Layer.unwrap(
485505
makeProviderAdapter(input.state, {
486506
failEventStream: input.failEventStream ?? false,
@@ -2932,6 +2952,39 @@ it.effect(
29322952
}),
29332953
);
29342954

2955+
it.effect("ProviderSessionManagerV2 keeps a provider update that lands while release clears", () =>
2956+
Effect.gen(function* () {
2957+
const state = yield* Ref.make(emptyState);
2958+
const beforeUnchangedWrite = yield* Ref.make<Effect.Effect<void>>(Effect.void);
2959+
const effect = Effect.gen(function* () {
2960+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2961+
const { providerSessionId, providerThreads, writeProviderThread, rosters } =
2962+
yield* openSessionWithBackgroundCommand("provider-session-manager-raced-roster");
2963+
// A replacement reports a task after cleanup read the row, before it writes.
2964+
const seeded = providerThreads[0]!;
2965+
yield* Ref.set(
2966+
beforeUnchangedWrite,
2967+
writeProviderThread({
2968+
...seeded,
2969+
pendingBackgroundTasks: [
2970+
...seeded.pendingBackgroundTasks,
2971+
{ taskId: "replacement-task", kind: "command" },
2972+
],
2973+
updatedAt: DateTime.add(seeded.updatedAt, { seconds: 1 }),
2974+
}).pipe(Effect.orDie),
2975+
);
2976+
2977+
yield* manager.close(providerSessionId);
2978+
2979+
assert.deepStrictEqual(yield* rosters, [["replacement-task"], ["dev-server"]]);
2980+
});
2981+
2982+
yield* effect.pipe(
2983+
Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, beforeUnchangedWrite })),
2984+
);
2985+
}),
2986+
);
2987+
29352988
it.effect("ProviderSessionManagerV2 defers idle release while background work is pending", () =>
29362989
Effect.gen(function* () {
29372990
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -826,41 +826,48 @@ export const layerWithOptions = (
826826
// Removes the tasks `readReleasedBackgroundTasks` found from their
827827
// rosters. Their process is gone, so nothing will report them ending,
828828
// and a leftover roster keeps the thread Waiting (#15239). Tasks a
829-
// replacement session reported since are left in place.
829+
// replacement session reported since are left in place. The write
830+
// commits only if the row did not change after it was read, and is
831+
// retried a few times when a provider update lands in between.
830832
const writeReleasedBackgroundWorkEvents = (
831833
released: ReadonlyArray<ReleasedBackgroundTasks>,
832834
) =>
833835
Effect.forEach(
834836
released,
835837
({ threadId, providerSessionId, providerThreadId, taskIds }) =>
836838
Effect.gen(function* () {
837-
const projection = yield* projectionStore.getThreadRecords(threadId, [
838-
"providerThreads",
839-
]);
840-
const providerThread = projection.providerThreads.find(
841-
(candidate) => candidate.id === providerThreadId,
842-
);
843-
const tasks = providerThread?.pendingBackgroundTasks ?? [];
844-
const remaining = tasks.filter((task) => !taskIds.has(task.taskId));
845-
if (providerThread === undefined || remaining.length === tasks.length) return;
846-
const now = yield* DateTime.now;
847-
yield* eventSink.write({
848-
events: [
849-
{
850-
id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
851-
type: "provider-thread.updated",
852-
threadId,
853-
driver: providerThread.driver,
854-
providerInstanceId: providerThread.providerInstanceId,
855-
occurredAt: now,
856-
payload: {
857-
...providerThread,
858-
pendingBackgroundTasks: remaining,
859-
updatedAt: now,
839+
for (let attempt = 0; attempt < 3; attempt++) {
840+
const projection = yield* projectionStore.getThreadRecords(threadId, [
841+
"providerThreads",
842+
]);
843+
const providerThread = projection.providerThreads.find(
844+
(candidate) => candidate.id === providerThreadId,
845+
);
846+
const tasks = providerThread?.pendingBackgroundTasks ?? [];
847+
const remaining = tasks.filter((task) => !taskIds.has(task.taskId));
848+
if (providerThread === undefined || remaining.length === tasks.length) return;
849+
const now = yield* DateTime.now;
850+
const { committed } = yield* eventSink.writeIfProviderThreadUnchanged({
851+
providerThreadId,
852+
expectedUpdatedAt: providerThread.updatedAt,
853+
events: [
854+
{
855+
id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
856+
type: "provider-thread.updated",
857+
threadId,
858+
driver: providerThread.driver,
859+
providerInstanceId: providerThread.providerInstanceId,
860+
occurredAt: now,
861+
payload: {
862+
...providerThread,
863+
pendingBackgroundTasks: remaining,
864+
updatedAt: now,
865+
},
860866
},
861-
},
862-
],
863-
});
867+
],
868+
});
869+
if (committed) return;
870+
}
864871
}),
865872
{ discard: true },
866873
);

0 commit comments

Comments
 (0)