Skip to content

Commit d3e6416

Browse files
fix(server): only provider thread-load failures fail a starting run
The last-attempt settle wrapped the whole thread-load block, so a projection, id or handoff failure around a successful provider call was also recorded as a provider_error and the run was failed. Settle only when the provider's forkThread, ensureThread or fallback ensureThread call itself fails; other failures keep their typed error so the effect can retry. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent e6e7ce3 commit d3e6416

2 files changed

Lines changed: 134 additions & 54 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts‎

Lines changed: 79 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,11 @@ function makeLocalCommandHarness(input: {
158158
readonly openFailure?: unknown;
159159
/** Opens the session, then fails loading its provider thread. */
160160
readonly ensureThreadFailure?: unknown;
161+
/**
162+
* Resumes a thread that has a native ref: resume fails, the fresh-thread
163+
* fallback succeeds, then reading history for its handoff fails.
164+
*/
165+
readonly historyReadFailureAfterFallback?: unknown;
161166
readonly interruptOpen?: boolean;
162167
readonly interruptRunBeforeOpenFailure?: boolean;
163168
readonly writeFailure?: unknown;
@@ -332,6 +337,19 @@ function makeLocalCommandHarness(input: {
332337
checkpoints: [],
333338
updatedAt: now,
334339
};
340+
if ("historyReadFailureAfterFallback" in input) {
341+
const nativeThreadRef = {
342+
driver: providerThread.driver,
343+
nativeId: "native-resume-thread",
344+
strength: "strong" as const,
345+
};
346+
projection = {
347+
...projection,
348+
providerThreads: projection.providerThreads.map((candidate) =>
349+
candidate.id === providerThreadId ? { ...candidate, nativeThreadRef } : candidate,
350+
),
351+
};
352+
}
335353
const events: Array<OrchestrationV2DomainEvent> = [];
336354
const interruptRun = () => {
337355
projection = {
@@ -358,26 +376,40 @@ function makeLocalCommandHarness(input: {
358376
),
359377
),
360378
);
379+
const resumeFallbackSession = {
380+
driver: providerThread.driver,
381+
resumeThread: () =>
382+
Effect.fail(
383+
new ProviderAdapterEventStreamError({
384+
driver: providerThread.driver,
385+
providerSessionId,
386+
cause: "native thread is gone",
387+
}),
388+
),
389+
ensureThread: () => Effect.succeed(providerThread),
390+
};
361391
const open = vi.fn(() =>
362392
input.interruptOpen === true
363393
? Effect.interrupt
364-
: "ensureThreadFailure" in input
365-
? Effect.succeed({ driver: providerThread.driver, ensureThread } as never)
366-
: "openFailure" in input
367-
? Effect.sync(() => {
368-
if (input.interruptRunBeforeOpenFailure === true) interruptRun();
369-
}).pipe(
370-
Effect.andThen(
371-
Effect.fail(
372-
new ProviderSessionManager.ProviderSessionOpenError({
373-
instanceId: newInstanceId,
374-
providerSessionId,
375-
cause: input.openFailure,
376-
}),
394+
: "historyReadFailureAfterFallback" in input
395+
? Effect.succeed(resumeFallbackSession as never)
396+
: "ensureThreadFailure" in input
397+
? Effect.succeed({ driver: providerThread.driver, ensureThread } as never)
398+
: "openFailure" in input
399+
? Effect.sync(() => {
400+
if (input.interruptRunBeforeOpenFailure === true) interruptRun();
401+
}).pipe(
402+
Effect.andThen(
403+
Effect.fail(
404+
new ProviderSessionManager.ProviderSessionOpenError({
405+
instanceId: newInstanceId,
406+
providerSessionId,
407+
cause: input.openFailure,
408+
}),
409+
),
377410
),
378-
),
379-
)
380-
: Effect.die("A local command must not open a native session."),
411+
)
412+
: Effect.die("A local command must not open a native session."),
381413
);
382414
const startRootRun = vi.fn(() => Effect.die("A local command must not start a native turn."));
383415
const tryHandlePromptCommand = vi.fn(() =>
@@ -418,7 +450,9 @@ function makeLocalCommandHarness(input: {
418450
const layer = ProviderTurnStart.layer.pipe(
419451
Layer.provide(
420452
Layer.mergeAll(
421-
Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({}),
453+
Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({
454+
prepareProviderHandoff: () => Effect.die("history read must fail first"),
455+
}),
422456
Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),
423457
IdAllocator.layer,
424458
FileSystem.layerNoop({}),
@@ -443,6 +477,13 @@ function makeLocalCommandHarness(input: {
443477
(m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
444478
),
445479
}),
480+
getTurnStartHistory: () =>
481+
Effect.fail(
482+
new ProjectionStore.ProjectionStoreReadError({
483+
threadId,
484+
cause: input.historyReadFailureAfterFallback,
485+
}),
486+
),
446487
}),
447488
Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open }),
448489
Layer.mock(ProviderAuthService)({ tryHandlePromptCommand }),
@@ -639,6 +680,27 @@ effectIt.effect("keeps a thread-load failure retryable when terminal persistence
639680
}),
640681
);
641682

683+
effectIt.effect(
684+
"keeps a store failure after the provider loaded the thread typed and retryable",
685+
() =>
686+
Effect.gen(function* () {
687+
const harness = makeLocalCommandHarness({
688+
text: "Continue",
689+
historyReadFailureAfterFallback: new Error("database unavailable"),
690+
});
691+
692+
const error = yield* harness.start.pipe(Effect.flip);
693+
694+
// The provider succeeded; the failing stage is the projection read, so
695+
// the run is not failed as a provider error on the last attempt.
696+
expect(error._tag).toBe("ProviderTurnStartError");
697+
expect((error.cause as { _tag?: string } | undefined)?._tag).toBe("ProjectionStoreReadError");
698+
expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
699+
expect(harness.projection().runs.at(-1)?.status).toBe("starting");
700+
expect(harness.events).toEqual([]);
701+
}),
702+
);
703+
642704
effectIt.effect("does not overwrite a run interrupted while its thread loads", () =>
643705
Effect.gen(function* () {
644706
const harness = makeLocalCommandHarness({

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 55 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import {
3737
import { deliverContextHandoffs } from "./ContextHandoffDelivery.ts";
3838
import {
3939
ProviderAdapterTurnStartError,
40+
type ProviderAdapterV2Error,
4041
type ProviderAdapterV2HistoricalContext,
4142
type ProviderAdapterV2SessionRuntime,
4243
} from "./ProviderAdapter.ts";
@@ -586,8 +587,24 @@ export const layer: Layer.Layer<
586587
return;
587588
}
588589
const session = sessionResult.success;
590+
// Only the provider's own thread load fails the run on the last attempt;
591+
// store, id and handoff failures around it keep their typed errors.
592+
const loadFromProvider = (
593+
load: Effect.Effect<OrchestrationV2ProviderThread, ProviderAdapterV2Error>,
594+
) =>
595+
Effect.gen(function* () {
596+
const loaded = yield* Effect.result(load);
597+
if (loaded._tag === "Success") return loaded.success;
598+
if (input.willRetry === true) return yield* loaded.failure;
599+
yield* settleStartFailure({
600+
signal: "provider-thread-load-failure",
601+
title: "Provider turn failed to start",
602+
error: loaded.failure,
603+
});
604+
return undefined;
605+
});
589606
let effectiveHandoffs = handoffs;
590-
const loadResult = yield* Effect.gen(function* () {
607+
const loadedProviderThread = yield* Effect.gen(function* () {
591608
if (nativeForkTransfer !== undefined) {
592609
const sourceProjection = yield* projectionStore.getThreadRecords(
593610
nativeForkTransfer.sourceThreadId,
@@ -613,27 +630,33 @@ export const layer: Layer.Layer<
613630
cause: `Native fork transfer ${nativeForkTransfer.id} has no source provider execution.`,
614631
});
615632
}
616-
return yield* session.forkThread({
617-
sourceProviderThread,
618-
sourceProviderTurns: sourceProjection.providerTurns,
619-
targetThreadId: projection.thread.id,
620-
modelSelection: run.modelSelection,
621-
runtimePolicy: resolvedRuntimePolicy,
622-
...(sourceProviderTurn === undefined ? {} : { providerTurnId: sourceProviderTurn.id }),
623-
});
633+
return yield* loadFromProvider(
634+
session.forkThread({
635+
sourceProviderThread,
636+
sourceProviderTurns: sourceProjection.providerTurns,
637+
targetThreadId: projection.thread.id,
638+
modelSelection: run.modelSelection,
639+
runtimePolicy: resolvedRuntimePolicy,
640+
...(sourceProviderTurn === undefined
641+
? {}
642+
: { providerTurnId: sourceProviderTurn.id }),
643+
}),
644+
);
624645
}
625646
if (providerThread.nativeThreadRef === null) {
626647
// Hand the run's provider thread to the adapter so it adopts this
627648
// row's identity when attaching native state. An adapter that mints
628649
// its own row instead leaves two live rows per app thread, and
629650
// `activeProviderThreadId` then flaps between them on every update.
630-
return yield* session.ensureThread({
631-
threadId: projection.thread.id,
632-
modelSelection: run.modelSelection,
633-
runtimePolicy: resolvedRuntimePolicy,
634-
providerSessionId,
635-
existingProviderThread: providerThread,
636-
});
651+
return yield* loadFromProvider(
652+
session.ensureThread({
653+
threadId: projection.thread.id,
654+
modelSelection: run.modelSelection,
655+
runtimePolicy: resolvedRuntimePolicy,
656+
providerSessionId,
657+
existingProviderThread: providerThread,
658+
}),
659+
);
637660
}
638661
const uncertainDelivery = projection.contextHandoffs.some(
639662
(handoff) =>
@@ -670,16 +693,19 @@ export const layer: Layer.Layer<
670693
reason: uncertainDelivery ? "uncertain_history_delivery" : "resume_failed",
671694
errorTag: resumed.failure._tag,
672695
});
673-
const replacement = yield* session.ensureThread({
674-
threadId: projection.thread.id,
675-
modelSelection: run.modelSelection,
676-
runtimePolicy: resolvedRuntimePolicy,
677-
providerSessionId,
678-
// The native ref is dropped so the adapter binds a fresh native
679-
// session instead of retrying the resume that just failed, while
680-
// still adopting this row's identity.
681-
existingProviderThread: { ...providerThread, nativeThreadRef: null },
682-
});
696+
const replacement = yield* loadFromProvider(
697+
session.ensureThread({
698+
threadId: projection.thread.id,
699+
modelSelection: run.modelSelection,
700+
runtimePolicy: resolvedRuntimePolicy,
701+
providerSessionId,
702+
// The native ref is dropped so the adapter binds a fresh native
703+
// session instead of retrying the resume that just failed, while
704+
// still adopting this row's identity.
705+
existingProviderThread: { ...providerThread, nativeThreadRef: null },
706+
}),
707+
);
708+
if (replacement === undefined) return undefined;
683709
const transferId = yield* idAllocator.allocate.contextTransfer({
684710
sourceThreadId: projection.thread.id,
685711
targetThreadId: projection.thread.id,
@@ -747,17 +773,9 @@ export const layer: Layer.Layer<
747773
],
748774
});
749775
return replacement;
750-
}).pipe(Effect.result);
751-
if (loadResult._tag === "Failure") {
752-
if (input.willRetry === true) return yield* loadResult.failure;
753-
yield* settleStartFailure({
754-
signal: "provider-thread-load-failure",
755-
title: "Provider turn failed to start",
756-
error: loadResult.failure,
757-
});
758-
return;
759-
}
760-
const loadedProviderThread = loadResult.success;
776+
});
777+
// The last attempt already failed the run.
778+
if (loadedProviderThread === undefined) return;
761779
if (!(yield* isCurrentAttemptInStatus("starting"))) {
762780
return;
763781
}

0 commit comments

Comments
 (0)