Skip to content

Commit d155032

Browse files
SunkenInTimeclaude
authored andcommitted
fix(server): stop the root turn even when saving fails before it is reported
If the first failed write came before the provider reported its root turn, recovery had no turn id and could not stop the agent. The consumer now records the failure, stops writing, and keeps routing until the root turn is known, then recovers as before. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent f8ae16f commit d155032

2 files changed

Lines changed: 118 additions & 33 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3253,6 +3253,62 @@ it.effect("stops the agent and records the run failed when its events cannot be
32533253
}),
32543254
);
32553255

3256+
it.effect("stops the root turn even when saving fails before the turn is reported", () =>
3257+
Effect.gen(function* () {
3258+
const ids = backgroundScenarioIds("lost-ingestion-early");
3259+
const now = yield* DateTime.now;
3260+
const lateTurnId = ProviderTurnId.make("provider-turn:lost-ingestion-early:native");
3261+
const firstWriteFailed = yield* Deferred.make<void>();
3262+
const providerThreadEvent = {
3263+
type: "provider_thread.updated",
3264+
driver,
3265+
providerThread: { id: ids.providerThreadId } as OrchestrationV2ProviderThread,
3266+
} as ProviderAdapterV2Event;
3267+
const rootTurnEvent: ProviderAdapterV2Event = {
3268+
type: "provider_turn.updated",
3269+
driver,
3270+
threadId: ids.threadId,
3271+
providerTurn: {
3272+
id: lateTurnId,
3273+
providerThreadId: ids.providerThreadId,
3274+
nodeId: ids.rootNodeId,
3275+
runAttemptId: ids.attemptId,
3276+
nativeTurnRef: null,
3277+
ordinal: 1,
3278+
status: "running",
3279+
startedAt: now,
3280+
completedAt: null,
3281+
},
3282+
};
3283+
const { observed, turnInterrupted } = yield* captureRootRunTermination({
3284+
key: "lost-ingestion-early",
3285+
shouldFinalizeRun: () => Effect.succeed(true),
3286+
attemptProviderTurnId: null,
3287+
// The provider reports its turn only after the first write has failed.
3288+
events: () =>
3289+
Stream.make(providerThreadEvent).pipe(
3290+
Stream.concat(
3291+
Stream.fromEffect(Deferred.await(firstWriteFailed).pipe(Effect.as(rootTurnEvent))),
3292+
),
3293+
),
3294+
ingestNormalized: (event) =>
3295+
Deferred.succeed(firstWriteFailed, undefined).pipe(
3296+
Effect.andThen(
3297+
Effect.fail(
3298+
new ProviderEventIngestor.ProviderEventPublishError({
3299+
providerSessionId: event.providerSessionId,
3300+
eventCount: 1,
3301+
cause: "database is locked",
3302+
}),
3303+
),
3304+
),
3305+
),
3306+
});
3307+
assert.equal(yield* Deferred.await(turnInterrupted), lateTurnId);
3308+
assert.include(observed, "run:failed");
3309+
}),
3310+
);
3311+
32563312
it.effect("does not record a lost run as failed after the run moved on", () =>
32573313
Effect.gen(function* () {
32583314
const { observed, written } = yield* captureRootRunTermination({
@@ -3332,6 +3388,8 @@ function captureRootRunTermination(input: {
33323388
readonly onFinalizationWriteFailed?: Effect.Effect<void>;
33333389
/** False when another attempt or a Stop already moved the run on. */
33343390
readonly runIsCurrent?: boolean;
3391+
/** Null when the provider has not reported the root turn yet. */
3392+
readonly attemptProviderTurnId?: ProviderTurnId | null;
33353393
}) {
33363394
return Effect.gen(function* () {
33373395
const ids = backgroundScenarioIds(input.key);
@@ -3471,7 +3529,10 @@ function captureRootRunTermination(input: {
34713529
} as OrchestrationV2ProviderThread,
34723530
attempt: {
34733531
id: ids.attemptId,
3474-
providerTurnId: ids.rootProviderTurnId,
3532+
providerTurnId:
3533+
input.attemptProviderTurnId === undefined
3534+
? ids.rootProviderTurnId
3535+
: input.attemptProviderTurnId,
34753536
} as OrchestrationV2RunAttempt,
34763537
attemptId: ids.attemptId,
34773538
providerTurnOrdinal: 1,

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 56 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -948,6 +948,10 @@ export const layer: Layer.Layer<
948948
);
949949
const rootTerminalSeen = yield* Ref.make(false);
950950
const rootRunFinalized = yield* Ref.make(false);
951+
// Set when an event write fails. The stream then stops writing but
952+
// keeps routing until it has seen the root provider turn, so
953+
// recovery always knows which turn to stop.
954+
const lostIngestion = yield* Ref.make<Cause.Cause<unknown> | null>(null);
951955
const providerThreadOwnerLost = yield* Ref.make(false);
952956
const activeChildProviderTurns = yield* Ref.make<ReadonlySet<ProviderTurnId>>(new Set());
953957
const activeChildSubagents = yield* Ref.make<ReadonlySet<NodeId>>(new Set());
@@ -1271,6 +1275,7 @@ export const layer: Layer.Layer<
12711275
),
12721276
Stream.tap((event) =>
12731277
Effect.gen(function* () {
1278+
if ((yield* Ref.get(lostIngestion)) !== null) return;
12741279
let storedEventCount = 0;
12751280
const deliveredEvent = filterAssistantEvent(
12761281
event,
@@ -1286,37 +1291,45 @@ export const layer: Layer.Layer<
12861291
const isRootProviderThreadUpdate =
12871292
event.type === "provider_thread.updated" &&
12881293
event.providerThread.id === input.providerThread.id;
1289-
const storedEvents = yield* providerEventIngestor.ingestNormalized({
1290-
analyticsContext: {
1291-
modelSelection: input.modelSelection,
1292-
runtimeMode: input.runtimePolicy.runtimeMode,
1293-
interactionMode: input.runtimePolicy.interactionMode,
1294-
},
1295-
providerSessionId: input.providerSessionId,
1296-
providerInstanceId: input.run.providerInstanceId,
1297-
threadId: input.run.threadId,
1298-
runId: input.run.id,
1299-
nodeId: input.rootNode.id,
1300-
event: deliveredEvent,
1301-
...(isRootProviderThreadUpdate
1302-
? rootTerminalAlreadySeen
1303-
? {
1304-
writeIfProviderThreadOwner: {
1305-
providerThreadId: input.providerThread.id,
1306-
runId: input.run.id,
1307-
activeAttemptId: input.attempt.id,
1308-
expectedLastRunOrdinal: input.run.ordinal,
1309-
},
1310-
}
1311-
: {
1312-
writeIfRunCurrent: {
1313-
runId: input.run.id,
1314-
activeAttemptId: input.attempt.id,
1315-
expectedStatus: "running" as const,
1316-
},
1317-
}
1318-
: {}),
1319-
});
1294+
const storedEvents = yield* providerEventIngestor
1295+
.ingestNormalized({
1296+
analyticsContext: {
1297+
modelSelection: input.modelSelection,
1298+
runtimeMode: input.runtimePolicy.runtimeMode,
1299+
interactionMode: input.runtimePolicy.interactionMode,
1300+
},
1301+
providerSessionId: input.providerSessionId,
1302+
providerInstanceId: input.run.providerInstanceId,
1303+
threadId: input.run.threadId,
1304+
runId: input.run.id,
1305+
nodeId: input.rootNode.id,
1306+
event: deliveredEvent,
1307+
...(isRootProviderThreadUpdate
1308+
? rootTerminalAlreadySeen
1309+
? {
1310+
writeIfProviderThreadOwner: {
1311+
providerThreadId: input.providerThread.id,
1312+
runId: input.run.id,
1313+
activeAttemptId: input.attempt.id,
1314+
expectedLastRunOrdinal: input.run.ordinal,
1315+
},
1316+
}
1317+
: {
1318+
writeIfRunCurrent: {
1319+
runId: input.run.id,
1320+
activeAttemptId: input.attempt.id,
1321+
expectedStatus: "running" as const,
1322+
},
1323+
}
1324+
: {}),
1325+
})
1326+
.pipe(
1327+
Effect.catchCauseIf(
1328+
(cause) => !Cause.hasInterruptsOnly(cause),
1329+
(cause) => Ref.set(lostIngestion, cause).pipe(Effect.as(null)),
1330+
),
1331+
);
1332+
if (storedEvents === null) return;
13201333
storedEventCount = storedEvents.length;
13211334
if (
13221335
isRootProviderThreadUpdate &&
@@ -1350,11 +1363,22 @@ export const layer: Layer.Layer<
13501363
yield* trackChildLifecycle(event, deliveredEvent !== null);
13511364
}),
13521365
),
1353-
Stream.takeUntilEffect(() => shouldStopProviderEventIngestion),
1366+
Stream.takeUntilEffect(() =>
1367+
Effect.gen(function* () {
1368+
if ((yield* Ref.get(lostIngestion)) !== null) {
1369+
return (yield* Ref.get(eventRouting)).rootProviderTurnId !== null;
1370+
}
1371+
return yield* shouldStopProviderEventIngestion;
1372+
}),
1373+
),
13541374
Stream.runDrain,
13551375
Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })),
13561376
Effect.flatMap(() =>
13571377
Effect.gen(function* () {
1378+
const lost = yield* Ref.get(lostIngestion);
1379+
if (lost !== null) {
1380+
return yield* new RunExecutionIngestError({ runId: input.run.id, cause: lost });
1381+
}
13581382
const terminal = yield* Ref.get(terminalEvent);
13591383
if (terminal === null) {
13601384
return;

0 commit comments

Comments
 (0)