Skip to content

Commit 6f6f68a

Browse files
authored
fix(server): stop abandoned turns before continuing threads (pingdotgg#23169)
1 parent 49e97d4 commit 6f6f68a

6 files changed

Lines changed: 489 additions & 29 deletions

File tree

‎apps/mobile/src/features/threads/worktree-setup-card.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ export function WorktreeSetupCard(props: WorktreeSetupCardProps) {
8484
accessibilityRole="button"
8585
accessibilityLabel={
8686
backgroundSetup
87-
? `${scriptName} is still running. Show setup progress.`
87+
? `Background setup is still running: ${scriptName}. Show setup progress.`
8888
: "Worktree setup details"
8989
}
9090
accessibilityState={{ expanded: detailsOpen }}
@@ -116,7 +116,7 @@ export function WorktreeSetupCard(props: WorktreeSetupCardProps) {
116116
/>
117117
) : null}
118118
<Text numberOfLines={1} className="shrink text-2xs text-foreground-secondary">
119-
{backgroundSetup ? scriptName : "Details"}
119+
{backgroundSetup ? "Background setup" : "Details"}
120120
</Text>
121121
{!backgroundSetup ? (
122122
<SymbolView name="chevron.right" size={10} tintColorClassName="accent-icon-muted" />

‎apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts‎

Lines changed: 197 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
NodeId,
88
ProviderSessionId,
99
ProviderThreadId,
10+
ProviderTurnId,
1011
ProviderDriverKind,
1112
ProviderInstanceId,
1213
ProviderSetupError,
@@ -271,6 +272,14 @@ function makeLocalCommandHarness(input: {
271272
readonly interruptRunBeforeOpenFailure?: boolean;
272273
readonly writeFailure?: unknown;
273274
readonly failReadsAfterRunning?: boolean;
275+
readonly previousTurn?: {
276+
readonly runStatus: "failed" | "completed";
277+
readonly turnStatus: "running" | "completed" | "interrupted";
278+
readonly differentThread?: boolean;
279+
readonly liveSession?: boolean;
280+
readonly interruptFailure?: boolean;
281+
readonly interruptCurrentRun?: boolean;
282+
};
274283
}) {
275284
const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z");
276285
const threadId = ThreadId.make("thread-native-account-command");
@@ -456,6 +465,74 @@ function makeLocalCommandHarness(input: {
456465
),
457466
};
458467
}
468+
const previousTurnId = ProviderTurnId.make("previous-provider-turn");
469+
if (input.previousTurn !== undefined) {
470+
const previousRunId = RunId.make("previous-failed-run");
471+
const previousAttemptId = RunAttemptId.make("previous-failed-attempt");
472+
const previousProviderThreadId = input.previousTurn.differentThread
473+
? oldProviderThreadId
474+
: providerThreadId;
475+
projection = {
476+
...projection,
477+
runs: [
478+
{
479+
...run,
480+
id: previousRunId,
481+
ordinal: 1,
482+
status: input.previousTurn.runStatus,
483+
rootNodeId: NodeId.make("previous-root"),
484+
activeAttemptId: previousAttemptId,
485+
providerThreadId: previousProviderThreadId,
486+
completedAt: now,
487+
},
488+
run,
489+
],
490+
providerThreads: [
491+
{
492+
...providerThread,
493+
nativeThreadRef: {
494+
driver: providerThread.driver,
495+
nativeId: "existing-native-conversation",
496+
strength: "strong",
497+
},
498+
},
499+
],
500+
providerTurns: [
501+
{
502+
id: previousTurnId,
503+
providerThreadId: previousProviderThreadId,
504+
nodeId: NodeId.make("previous-root"),
505+
runAttemptId: previousAttemptId,
506+
ordinal: 1,
507+
nativeTurnRef: null,
508+
status: input.previousTurn.turnStatus,
509+
startedAt: now,
510+
completedAt: input.previousTurn.turnStatus === "running" ? null : now,
511+
},
512+
],
513+
};
514+
}
515+
let providerBusy =
516+
input.previousTurn?.runStatus === "failed" &&
517+
input.previousTurn.turnStatus === "running" &&
518+
!input.previousTurn.differentThread &&
519+
input.previousTurn.liveSession !== false;
520+
const interruptTurn = vi.fn((request: ProviderAdapter.ProviderAdapterV2InterruptInput) =>
521+
Effect.gen(function* () {
522+
expect(request.providerTurnId).toBe(previousTurnId);
523+
expect(request.providerThread.id).toBe(providerThreadId);
524+
if (input.previousTurn?.interruptFailure) {
525+
return yield* new ProviderAdapter.ProviderAdapterInterruptError({
526+
driver: providerThread.driver,
527+
providerThreadId,
528+
providerTurnId: request.providerTurnId,
529+
cause: "provider interrupt failed",
530+
});
531+
}
532+
providerBusy = false;
533+
if (input.previousTurn?.interruptCurrentRun) interruptRun();
534+
}),
535+
);
459536
const events: Array<OrchestrationV2DomainEvent> = [];
460537
const interruptRun = () => {
461538
projection = {
@@ -515,7 +592,7 @@ function makeLocalCommandHarness(input: {
515592
),
516593
),
517594
)
518-
: input.failReadsAfterRunning === true
595+
: input.failReadsAfterRunning === true || input.previousTurn !== undefined
519596
? Effect.succeed({
520597
driver: providerThread.driver,
521598
providerSession: {
@@ -531,14 +608,17 @@ function makeLocalCommandHarness(input: {
531608
lastError: null,
532609
},
533610
ensureThread: () => Effect.succeed(providerThread),
611+
resumeThread: () => Effect.succeed(projection.providerThreads[0]),
534612
} as never)
535613
: Effect.die("A local command must not open a native session."),
536614
);
537615
const startRootRun = vi.fn<
538616
(input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect<void>
539617
>(() =>
540-
input.failReadsAfterRunning === true
541-
? Effect.void
618+
input.failReadsAfterRunning === true || input.previousTurn !== undefined
619+
? Effect.suspend(() =>
620+
providerBusy ? Effect.die("The previous provider turn is still running.") : Effect.void,
621+
)
542622
: Effect.die("A local command must not start a native turn."),
543623
);
544624
const failReadIfRunning = Effect.suspend(() =>
@@ -642,7 +722,19 @@ function makeLocalCommandHarness(input: {
642722
}),
643723
),
644724
}),
645-
Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open }),
725+
Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
726+
open,
727+
get: () =>
728+
Effect.succeed(
729+
input.previousTurn?.liveSession === false
730+
? Option.none()
731+
: Option.some({
732+
driver: providerThread.driver,
733+
providerSessionId,
734+
interruptTurn,
735+
} as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime),
736+
),
737+
}),
646738
Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),
647739
Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),
648740
Layer.mock(RuntimePolicy.RuntimePolicyV2)({
@@ -662,6 +754,7 @@ function makeLocalCommandHarness(input: {
662754
open,
663755
writeIfRunCurrent,
664756
startRootRun,
757+
interruptTurn,
665758
tryHandlePromptCommand,
666759
events,
667760
oldInstanceId,
@@ -681,6 +774,106 @@ function makeLocalCommandHarness(input: {
681774
};
682775
}
683776

777+
effectIt.effect("recovers an abandoned failed turn before continuing its native conversation", () =>
778+
Effect.gen(function* () {
779+
const harness = makeLocalCommandHarness({
780+
text: "Continue",
781+
previousTurn: { runStatus: "failed", turnStatus: "running" },
782+
});
783+
784+
yield* harness.start;
785+
786+
expect(harness.interruptTurn).toHaveBeenCalledOnce();
787+
expect(harness.projection().runs.at(-1)?.status).toBe("running");
788+
expect(harness.projection().providerTurns[0]).toMatchObject({
789+
status: "interrupted",
790+
completedAt: expect.anything(),
791+
});
792+
expect(harness.startRootRun).toHaveBeenCalledOnce();
793+
expect(harness.startRootRun.mock.calls[0]?.[0].providerThread.nativeThreadRef?.nativeId).toBe(
794+
"existing-native-conversation",
795+
);
796+
}),
797+
);
798+
799+
effectIt.effect.each([
800+
{ runStatus: "failed", turnStatus: "completed" },
801+
{ runStatus: "completed", turnStatus: "completed" },
802+
{ runStatus: "failed", turnStatus: "running", differentThread: true },
803+
{ runStatus: "failed", turnStatus: "running", liveSession: false },
804+
{ runStatus: "failed", turnStatus: "interrupted", interruptFailure: true },
805+
] as const)("continues without interrupting when recovery does not apply: %j", (previousTurn) =>
806+
Effect.gen(function* () {
807+
const harness = makeLocalCommandHarness({ text: "Continue", previousTurn });
808+
809+
yield* harness.start;
810+
811+
expect(harness.interruptTurn).not.toHaveBeenCalled();
812+
expect(harness.projection().runs.at(-1)?.status).toBe("running");
813+
}),
814+
);
815+
816+
effectIt.effect("does not start another turn when abandoned-turn cleanup fails", () =>
817+
Effect.gen(function* () {
818+
const harness = makeLocalCommandHarness({
819+
text: "Continue",
820+
previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
821+
});
822+
823+
yield* harness.start;
824+
825+
expect(harness.interruptTurn).toHaveBeenCalledOnce();
826+
expect(harness.startRootRun).not.toHaveBeenCalled();
827+
expect(harness.projection().runs.at(-1)?.status).toBe("failed");
828+
}),
829+
);
830+
831+
effectIt.effect("retries abandoned-turn cleanup without failing the continuation early", () =>
832+
Effect.gen(function* () {
833+
const harness = makeLocalCommandHarness({
834+
text: "Continue",
835+
previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
836+
});
837+
838+
const error = yield* harness.startWithRetry.pipe(Effect.flip);
839+
840+
expect(error._tag).toBe("ProviderTurnStartError");
841+
expect(harness.projection().runs.at(-1)?.status).toBe("starting");
842+
expect(harness.open).not.toHaveBeenCalled();
843+
}),
844+
);
845+
846+
effectIt.effect("does not start the continuation if recording recovery fails", () =>
847+
Effect.gen(function* () {
848+
const harness = makeLocalCommandHarness({
849+
text: "Continue",
850+
previousTurn: { runStatus: "failed", turnStatus: "running" },
851+
writeFailure: new Error("database unavailable"),
852+
});
853+
854+
const error = yield* harness.start.pipe(Effect.flip);
855+
856+
expect(error._tag).toBe("ProviderTurnStartError");
857+
expect(harness.projection().runs.at(-1)?.status).toBe("starting");
858+
expect(harness.projection().providerTurns[0]?.status).toBe("running");
859+
expect(harness.open).not.toHaveBeenCalled();
860+
}),
861+
);
862+
863+
effectIt.effect("does not resume a continuation stopped while recovery was in flight", () =>
864+
Effect.gen(function* () {
865+
const harness = makeLocalCommandHarness({
866+
text: "Continue",
867+
previousTurn: { runStatus: "failed", turnStatus: "running", interruptCurrentRun: true },
868+
});
869+
870+
yield* harness.start;
871+
872+
expect(harness.projection().runs.at(-1)?.status).toBe("interrupted");
873+
expect(harness.open).not.toHaveBeenCalled();
874+
}),
875+
);
876+
684877
effectIt.effect.each(["cleanup-first", "startup-first"])(
685878
"coordinates provider startup with %s",
686879
(order) =>

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 85 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -541,26 +541,6 @@ export const layer: Layer.Layer<
541541
const existingSessionProjection = projection.providerSessions.find(
542542
(candidate) => candidate.id === providerSessionId,
543543
);
544-
const sessionResult = yield* Effect.result(
545-
providerSessions.open({
546-
threadId: projection.thread.id,
547-
providerSessionId,
548-
modelSelection: run.modelSelection,
549-
runtimePolicy: resolvedRuntimePolicy,
550-
...(existingSessionProjection === undefined
551-
? {}
552-
: { resumeFromSession: existingSessionProjection }),
553-
...(providerThread.nativeThreadRef?.nativeId == null
554-
? {}
555-
: { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }),
556-
...(providerThread.nativeMetadata?.itemIdentityVersion === undefined
557-
? {}
558-
: {
559-
initialProviderItemIdentityVersion:
560-
providerThread.nativeMetadata.itemIdentityVersion,
561-
}),
562-
}),
563-
);
564544
const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
565545
function* (failed: {
566546
readonly signal: string;
@@ -592,6 +572,91 @@ export const layer: Layer.Layer<
592572
});
593573
},
594574
);
575+
const failedRuns = new Map(
576+
projection.runs
577+
.filter(
578+
(previous) =>
579+
previous.ordinal < run.ordinal &&
580+
previous.providerThreadId === providerThread.id &&
581+
previous.status === "failed",
582+
)
583+
.map((previous) => [previous.activeAttemptId, previous] as const),
584+
);
585+
const abandonedTurns = projection.providerTurns.flatMap((turn) => {
586+
const previous = failedRuns.get(turn.runAttemptId);
587+
return turn.providerThreadId === providerThread.id &&
588+
turn.status === "running" &&
589+
previous !== undefined &&
590+
turn.nodeId === previous.rootNodeId
591+
? [{ turn, run: previous }]
592+
: [];
593+
});
594+
const abandonedTurn = abandonedTurns.at(-1)?.turn;
595+
if (abandonedTurn !== undefined) {
596+
const liveSession = yield* providerSessions.get(providerSessionId);
597+
if (Option.isSome(liveSession)) {
598+
// Older versions could fail the run without stopping its provider.
599+
// Recover that exact turn before reusing its native conversation.
600+
const recovered = yield* Effect.result(
601+
liveSession.value.interruptTurn({
602+
providerThread,
603+
providerTurnId: abandonedTurn.id,
604+
requestRuntimeRestart: true,
605+
}),
606+
);
607+
if (recovered._tag === "Failure") {
608+
if (input.willRetry === true) return yield* recovered.failure;
609+
yield* settleStartFailure({
610+
signal: "provider-turn-recovery-failure",
611+
title: "Provider turn recovery failed",
612+
error: recovered.failure,
613+
});
614+
return;
615+
}
616+
}
617+
const completedAt = yield* DateTime.now;
618+
const repaired = yield* eventSink.writeIfRunCurrent({
619+
threadId: projection.thread.id,
620+
runId: run.id,
621+
activeAttemptId: attempt.id,
622+
expectedStatus: "starting",
623+
events: yield* Effect.forEach(abandonedTurns, ({ turn, run: failedRun }) =>
624+
Effect.gen(function* () {
625+
return {
626+
id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
627+
type: "provider-turn.updated" as const,
628+
threadId: projection.thread.id,
629+
runId: failedRun.id,
630+
nodeId: turn.nodeId,
631+
providerInstanceId: failedRun.providerInstanceId,
632+
occurredAt: completedAt,
633+
payload: { ...turn, status: "interrupted" as const, completedAt },
634+
};
635+
}),
636+
),
637+
});
638+
if (!repaired.committed) return;
639+
}
640+
const sessionResult = yield* Effect.result(
641+
providerSessions.open({
642+
threadId: projection.thread.id,
643+
providerSessionId,
644+
modelSelection: run.modelSelection,
645+
runtimePolicy: resolvedRuntimePolicy,
646+
...(existingSessionProjection === undefined
647+
? {}
648+
: { resumeFromSession: existingSessionProjection }),
649+
...(providerThread.nativeThreadRef?.nativeId == null
650+
? {}
651+
: { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }),
652+
...(providerThread.nativeMetadata?.itemIdentityVersion === undefined
653+
? {}
654+
: {
655+
initialProviderItemIdentityVersion:
656+
providerThread.nativeMetadata.itemIdentityVersion,
657+
}),
658+
}),
659+
);
595660
if (sessionResult._tag === "Failure") {
596661
if (input.willRetry === true) return yield* sessionResult.failure;
597662
yield* settleStartFailure({

0 commit comments

Comments
 (0)