Skip to content

Commit b39ea65

Browse files
fix(server): contain background artifact persistence failures
1 parent 7ac31ee commit b39ea65

2 files changed

Lines changed: 222 additions & 13 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 195 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -471,6 +471,7 @@ function layerTest(input: {
471471
readonly providerSessionId: ProviderSessionId;
472472
readonly initialProviderItemIdentityVersion?: 2;
473473
}) => Effect.Effect<void>;
474+
readonly eventSinkLayer?: typeof layerTestEventSink;
474475
readonly failReleaseEventWrites?: boolean;
475476
readonly flakyReleaseWrites?: FlakyReleaseWrites;
476477
readonly pauseAttachWrite?: Parameters<typeof layerPausingAttachEventSink>[0];
@@ -491,15 +492,16 @@ function layerTest(input: {
491492
readonly projectServiceLayer?: Layer.Layer<ProjectService.ProjectService>;
492493
}) {
493494
const layerConfiguredEventSink =
494-
input.flakyReleaseWrites !== undefined
495+
input.eventSinkLayer ??
496+
(input.flakyReleaseWrites !== undefined
495497
? layerFlakyReleaseEventSink(input.flakyReleaseWrites)
496498
: input.pauseAttachWrite !== undefined
497499
? layerPausingAttachEventSink(input.pauseAttachWrite)
498500
: input.beforeUnchangedWrite !== undefined
499501
? layerBeforeUnchangedWriteEventSink(input.beforeUnchangedWrite)
500502
: input.failReleaseEventWrites
501503
? layerFailingReleaseEventSink
502-
: layerTestEventSink;
504+
: layerTestEventSink);
503505
const layerRegistry = Layer.unwrap(
504506
makeProviderAdapter(input.state, {
505507
failEventStream: input.failEventStream ?? false,
@@ -4913,10 +4915,32 @@ it.effect(
49134915
);
49144916

49154917
it.effect(
4916-
"persists session-owned command settlement once, forwards it and releases its live output slot",
4918+
"persists session-owned command settlement and keeps processing after an artifact write fails",
49174919
() =>
49184920
Effect.gen(function* () {
49194921
const state = yield* Ref.make(emptyState);
4922+
const failNextArtifact = yield* Ref.make(false);
4923+
const eventSinkLayer = Layer.effect(
4924+
EventSink.EventSinkV2,
4925+
Effect.gen(function* () {
4926+
const delegate = yield* EventSink.EventSinkV2;
4927+
return EventSink.EventSinkV2.of({
4928+
...delegate,
4929+
write: (input) =>
4930+
Effect.gen(function* () {
4931+
if (
4932+
input.events.some((event) => event.type === "turn-item.updated") &&
4933+
(yield* Ref.getAndSet(failNextArtifact, false))
4934+
) {
4935+
return yield* new EventSink.EventSinkWriteError({
4936+
eventCount: input.events.length,
4937+
});
4938+
}
4939+
return yield* delegate.write(input);
4940+
}),
4941+
});
4942+
}),
4943+
).pipe(Layer.provide(layerTestEventSink));
49204944
yield* Effect.scoped(
49214945
Effect.gen(function* () {
49224946
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
@@ -4954,7 +4978,7 @@ it.effect(
49544978
itemId,
49554979
chunk: "Child output before settlement.",
49564980
});
4957-
yield* Queue.offer(queue, {
4981+
const settlement = {
49584982
type: "turn_item.updated",
49594983
driver: CODEX_DRIVER,
49604984
sessionThreadId: threadId,
@@ -4976,7 +5000,8 @@ it.effect(
49765000
type: "command_execution",
49775001
input: "run child work",
49785002
},
4979-
});
5003+
} satisfies ProviderAdapter.ProviderAdapterV2Event;
5004+
yield* Queue.offer(queue, settlement);
49805005
yield* Queue.offer(queue, {
49815006
type: "provider_session.updated",
49825007
driver: CODEX_DRIVER,
@@ -5000,7 +5025,171 @@ it.effect(
50005025
assert.equal(final.value.text, "Child output before settlement.");
50015026
assert.isFalse(final.value.running);
50025027
}
5028+
5029+
const laterItemId = TurnItemId.make("command:later-session-owned");
5030+
const recoverySubscription = yield* runtime.subscribeEvents!;
5031+
const continued = yield* recoverySubscription.events.pipe(
5032+
Stream.filter(
5033+
(event) =>
5034+
event.type === "provider_session.updated" ||
5035+
(event.type === "turn_item.updated" && event.turnItem.id === laterItemId),
5036+
),
5037+
Stream.take(2),
5038+
Stream.runCollect,
5039+
Effect.exit,
5040+
Effect.forkScoped,
5041+
);
5042+
yield* Ref.set(failNextArtifact, true);
5043+
yield* Queue.offerAll(queue, [
5044+
{
5045+
...settlement,
5046+
turnItem: {
5047+
...settlement.turnItem,
5048+
id: TurnItemId.make("command:failed-session-owned"),
5049+
},
5050+
},
5051+
{
5052+
type: "provider_session.updated",
5053+
driver: CODEX_DRIVER,
5054+
providerSession: runtime.providerSession,
5055+
},
5056+
{ ...settlement, turnItem: { ...settlement.turnItem, id: laterItemId } },
5057+
]);
5058+
const continuedExit = yield* Fiber.join(continued);
5059+
assert.isFalse(yield* Ref.get(failNextArtifact));
5060+
assert.isTrue(
5061+
Exit.isSuccess(continuedExit),
5062+
Exit.isFailure(continuedExit) ? Cause.pretty(continuedExit.cause) : undefined,
5063+
);
5064+
if (Exit.isSuccess(continuedExit)) {
5065+
assert.deepEqual(
5066+
continuedExit.value.map((event) => event.type),
5067+
["provider_session.updated", "turn_item.updated"],
5068+
);
5069+
}
5070+
assert.isTrue(Option.isSome(yield* manager.get(providerSessionId)));
5071+
assert.equal((yield* Ref.get(state)).closeCount, 0);
5072+
const artifacts = yield* store
5073+
.read({ threadId, eventType: "turn-item.updated" })
5074+
.pipe(Stream.runCollect);
5075+
assert.deepEqual(
5076+
artifacts.map((stored) =>
5077+
stored.event.type === "turn-item.updated" ? stored.event.payload.id : null,
5078+
),
5079+
[itemId, laterItemId],
5080+
);
50035081
}),
5004-
).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })));
5082+
).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, eventSinkLayer })));
5083+
}),
5084+
);
5085+
5086+
it.effect.each(["runless request", "session-tagged request", "session artifact defect"] as const)(
5087+
"ProviderSessionManagerV2 releases the session on fatal persistence failures (%s)",
5088+
(scenario) =>
5089+
Effect.gen(function* () {
5090+
const state = yield* Ref.make(emptyState);
5091+
const eventSinkLayer = Layer.effect(
5092+
EventSink.EventSinkV2,
5093+
Effect.gen(function* () {
5094+
const delegate = yield* EventSink.EventSinkV2;
5095+
return EventSink.EventSinkV2.of({
5096+
...delegate,
5097+
write: (input) =>
5098+
input.events.some(
5099+
(event) =>
5100+
event.type === "runtime-request.updated" || event.type === "node.updated",
5101+
)
5102+
? scenario === "session artifact defect"
5103+
? Effect.die(new Error("Unexpected artifact persistence defect"))
5104+
: Effect.fail(
5105+
new EventSink.EventSinkWriteError({ eventCount: input.events.length }),
5106+
)
5107+
: delegate.write(input),
5108+
});
5109+
}),
5110+
).pipe(Layer.provide(layerTestEventSink));
5111+
yield* Effect.gen(function* () {
5112+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
5113+
const sink = yield* EventSink.EventSinkV2;
5114+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
5115+
const now = yield* DateTime.now;
5116+
const threadId = ThreadId.make("thread:request-persistence-failure");
5117+
yield* sink.write({
5118+
events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
5119+
});
5120+
const providerSessionId = yield* idAllocator.allocate.providerSession({
5121+
providerInstanceId: modelSelection.instanceId,
5122+
threadId,
5123+
});
5124+
const runtime = yield* manager.open({
5125+
threadId,
5126+
providerSessionId,
5127+
modelSelection,
5128+
runtimePolicy,
5129+
});
5130+
const pendingRequest = yield* makePendingRuntimeRequestEvents({
5131+
idAllocator,
5132+
threadId,
5133+
providerSessionId,
5134+
providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }),
5135+
now,
5136+
});
5137+
const subscription = yield* runtime.subscribeEvents!;
5138+
const forwarded = yield* subscription.events.pipe(
5139+
Stream.runHead,
5140+
Effect.exit,
5141+
Effect.forkScoped,
5142+
);
5143+
const released = yield* sink
5144+
.stream({
5145+
threadId,
5146+
afterSequence: yield* sink.latestSequence({ threadId }),
5147+
})
5148+
.pipe(
5149+
Stream.filter(
5150+
(stored) =>
5151+
stored.event.type === "provider-session.updated" &&
5152+
stored.event.payload.status === "error",
5153+
),
5154+
Stream.runHead,
5155+
Effect.forkScoped,
5156+
);
5157+
const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!;
5158+
const requestNodeEvent = pendingRequest.providerEvents.find(
5159+
(event) => event.type === "node.updated",
5160+
)!;
5161+
yield* Queue.offerAll(queue, [
5162+
scenario === "runless request"
5163+
? pendingRequest.providerEvents[0]!
5164+
: {
5165+
...requestNodeEvent,
5166+
sessionThreadId: threadId,
5167+
node: {
5168+
...requestNodeEvent.node,
5169+
...(scenario === "session artifact defect"
5170+
? { kind: "subagent" as const, runtimeRequestId: null }
5171+
: {}),
5172+
},
5173+
},
5174+
{
5175+
type: "provider_session.updated",
5176+
driver: CODEX_DRIVER,
5177+
providerSession: runtime.providerSession,
5178+
},
5179+
]);
5180+
const forwardedExit = yield* Fiber.join(forwarded);
5181+
assert.isTrue(Exit.isFailure(forwardedExit));
5182+
if (Exit.isFailure(forwardedExit)) {
5183+
assert.include(
5184+
Cause.pretty(forwardedExit.cause),
5185+
scenario === "session artifact defect"
5186+
? "Unexpected artifact persistence defect"
5187+
: "ProviderEventPublishError",
5188+
);
5189+
}
5190+
yield* Fiber.join(released);
5191+
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
5192+
assert.equal((yield* Ref.get(state)).closeCount, 1);
5193+
}).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, eventSinkLayer })));
50055194
}),
50065195
);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -248,21 +248,26 @@ function sessionKey(providerSessionId: ProviderSessionId): string {
248248
return String(providerSessionId);
249249
}
250250

251+
function isRuntimeRequestEvent(event: ProviderAdapter.ProviderAdapterV2Event) {
252+
return (
253+
event.type === "runtime_request.updated" ||
254+
(event.type === "node.updated" && event.node.runtimeRequestId !== null) ||
255+
(event.type === "turn_item.updated" &&
256+
(event.turnItem.type === "approval_request" || event.turnItem.type === "user_input_request"))
257+
);
258+
}
259+
251260
function sessionScopedRuntimeRequestThreadId(
252261
event: ProviderAdapter.ProviderAdapterV2Event,
253262
): ThreadId | undefined {
263+
if (!isRuntimeRequestEvent(event)) return undefined;
254264
switch (event.type) {
255265
case "runtime_request.updated":
256266
return event.runtimeRequest.providerTurnId === null ? event.threadId : undefined;
257267
case "node.updated":
258-
return event.node.runId === null && event.node.runtimeRequestId !== null
259-
? event.node.threadId
260-
: undefined;
268+
return event.node.runId === null ? event.node.threadId : undefined;
261269
case "turn_item.updated":
262-
return event.turnItem.runId === null &&
263-
(event.turnItem.type === "approval_request" || event.turnItem.type === "user_input_request")
264-
? event.turnItem.threadId
265-
: undefined;
270+
return event.turnItem.runId === null ? event.turnItem.threadId : undefined;
266271
default:
267272
return undefined;
268273
}
@@ -1971,6 +1976,21 @@ export const layerWithOptions = (
19711976
event,
19721977
})
19731978
.pipe(
1979+
// Requests need persisted state before the provider can be answered;
1980+
// other artifact write failures must not stop shared background work.
1981+
Effect.catchTags({
1982+
ProviderEventPublishError: (cause) =>
1983+
event.sessionThreadId !== undefined && !isRuntimeRequestEvent(event)
1984+
? Effect.logWarning(
1985+
"orchestration-v2.driver-session.session-artifact-persist-failed",
1986+
{
1987+
providerSessionId: entry.runtime.providerSessionId,
1988+
threadId,
1989+
cause,
1990+
},
1991+
)
1992+
: Effect.fail(cause),
1993+
}),
19741994
Effect.mapError(
19751995
(cause) =>
19761996
new ProviderAdapter.ProviderAdapterEventStreamError({

0 commit comments

Comments
 (0)