Skip to content

Commit f7aed24

Browse files
fix(server): Stop always completes, and an agent that ignores it can be force stopped
A provider starved of memory never answered Stop: Claude's interrupt request had no timeout, so the turn stayed running for hours and the user had no way out. Every provider interrupt now goes through one 15 s bound in the session manager. If the provider has not confirmed by then, the Stop is marked unresponsive and the thread offers Force stop, which kills the thread's own agent scope with cgroup.kill (or tears down its session elsewhere) and ends the turn. A process shared with other working threads is never killed. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent dd14848 commit f7aed24

36 files changed

Lines changed: 1481 additions & 96 deletions

‎apps/mobile/src/features/threads/ThreadDetailScreen.tsx‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import type {
4646
ScopedThreadRef,
4747
ServerConfig as T3ServerConfig,
4848
UsageLimitsReport,
49+
RunId,
4950
} from "@t3tools/contracts";
5051
import {
5152
appendCodexArtifactTemplateUsePrompt,
@@ -156,8 +157,10 @@ import {
156157
import { appAtomRegistry } from "../../state/atom-registry";
157158
import { threadStreamHaptics } from "./threadStreamHaptics";
158159
import { useAtomCommand } from "../../state/use-atom-command";
160+
import { orchestrationEnvironment } from "../../state/orchestration";
159161
import { useDelayedStatus } from "../../lib/useDelayedStatus";
160162
import type { PendingUserInputDraftAnswer } from "@t3tools/client-runtime/state/pending-user-input";
163+
import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution";
161164
import type {
162165
PendingApproval,
163166
PendingUserInput,
@@ -339,6 +342,9 @@ const threadVisitTracker = createThreadVisitTracker({
339342

340343
// Past the composer's 180 ms height animation.
341344
const COMPOSER_HEIGHT_SETTLE_MS = 220;
345+
const NO_RUN_STOP_ATOM = Atom.make<RunStopPresentation | null>(null).pipe(
346+
Atom.withLabel("mobile-thread-run-stop:empty"),
347+
);
342348

343349
// Anchors, the submitted message, follow state and pending scrolls belong to
344350
// one thread, so the body remounts when the thread changes.
@@ -528,6 +534,50 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) {
528534
const unsettleThread = useAtomCommand(threadEnvironment.unsettle, { reportFailure: false });
529535
const updateMetadata = useAtomCommand(threadEnvironment.updateMetadata, { reportFailure: false });
530536
const visitThread = useAtomCommand(threadEnvironment.visit, { reportFailure: false });
537+
const anyRunStop = useAtomValue(
538+
props.selectedThreadDetailRef === null
539+
? NO_RUN_STOP_ATOM
540+
: environmentThreadDetails.runStopAtom(props.selectedThreadDetailRef),
541+
);
542+
// A background Stop matters only while that work is still pending.
543+
const runStop =
544+
anyRunStop?.background && props.selectedThread.pendingBackgroundTasks.length === 0
545+
? null
546+
: anyRunStop;
547+
const forceStopThread = useAtomCommand(orchestrationEnvironment.forceStopThread, {
548+
reportFailure: false,
549+
});
550+
const canForceStop =
551+
useAtomValue(orchestrationEnvironment.forceStopThread.permissionAtom(props.environmentId)) &&
552+
props.serverConfig?.environment.capabilities.threadForceStop === true;
553+
const [forceStoppingKey, setForceStoppingKey] = useState<string | null>(null);
554+
// The run that ignored Stop, so a confirmation left open cannot hit a newer run.
555+
const forceStop = async (runId: RunId | undefined) => {
556+
const threadKey = selectedThreadKey;
557+
setForceStoppingKey(threadKey);
558+
const result = await forceStopThread({
559+
environmentId: props.environmentId,
560+
input:
561+
runId === undefined
562+
? { threadId: props.selectedThread.id }
563+
: { threadId: props.selectedThread.id, runId },
564+
});
565+
setForceStoppingKey((current) => (current === threadKey ? null : current));
566+
if (result._tag === "Failure") {
567+
if (!isAtomCommandInterrupted(result)) Dialog.alert("Could not force stop the agent");
568+
return;
569+
}
570+
const others = result.value.sharedWithThreadIds.length;
571+
if (result.value.outcome === "releasing") {
572+
Dialog.alert("Turn stopped; the agent is still shutting down");
573+
}
574+
if (result.value.outcome === "shared") {
575+
Dialog.alert(
576+
"Turn stopped; the agent process kept running",
577+
`It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`,
578+
);
579+
}
580+
};
531581
const [statusRequest, setStatusRequest] = useState<{ threadKey: string } | null>(null);
532582
const statusPending = statusRequest?.threadKey === selectedThreadKey;
533583
const [lifecycleNow, setLifecycleNow] = useState(() => new Date().toISOString());
@@ -619,6 +669,32 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) {
619669
if (connectionStatus !== null && !props.draftMode) {
620670
return connectionStatus;
621671
}
672+
// Before the request check: an agent stuck on an approval may also ignore Stop.
673+
if (runStop !== null && contentPresentationKind === "ready") {
674+
const unresponsive = runStop.state === "unresponsive";
675+
return {
676+
kind: "stop",
677+
unresponsive,
678+
label: unresponsive ? "Not responding to Stop" : "Stopping…",
679+
action:
680+
unresponsive && canForceStop
681+
? {
682+
label: forceStoppingKey === selectedThreadKey ? "Force stopping..." : "Force stop",
683+
accessibilityLabel: "Force stop the agent",
684+
disabled: forceStoppingKey === selectedThreadKey,
685+
onPress: () =>
686+
Dialog.confirm({
687+
title: "Force stop the agent?",
688+
message:
689+
"It has not answered Stop. Force stop ends the turn and kills the agent's processes, unless other working threads share them.",
690+
confirmText: "Force stop",
691+
destructive: true,
692+
onConfirm: () => void forceStop(runStop.runId),
693+
}),
694+
}
695+
: null,
696+
};
697+
}
622698
if (props.activePendingApproval !== null || props.activePendingUserInput !== null) {
623699
return null;
624700
}

‎apps/mobile/src/features/threads/floating-working-control.tsx‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,9 @@ export function FloatingWorkingControl(props: {
8383
const [queueWidth, setQueueWidth] = useState(0);
8484
const [actionWidth, setActionWidth] = useState(0);
8585
const action =
86-
props.status?.kind === "background" || props.status?.kind === "lifecycle"
86+
props.status?.kind === "background" ||
87+
props.status?.kind === "lifecycle" ||
88+
props.status?.kind === "stop"
8789
? props.status.action
8890
: null;
8991
const [agentsWidth, setAgentsWidth] = useState(0);
@@ -471,6 +473,30 @@ function FloatingStatusLabel(props: {
471473
</StatusLabelRow>
472474
);
473475
}
476+
if (props.status.kind === "stop") {
477+
return (
478+
<StatusLabelRow
479+
key="stop"
480+
accessibilityLabel={props.status.label}
481+
className="gap-2"
482+
onLayout={props.onLayout}
483+
>
484+
{props.status.unresponsive ? (
485+
<SymbolView
486+
name="exclamationmark.triangle"
487+
size={13}
488+
tintColorClassName="accent-warning-foreground"
489+
type="monochrome"
490+
/>
491+
) : (
492+
<AppActivityIndicator size="small" colorClassName="accent-icon-muted" />
493+
)}
494+
<Text className="shrink font-t3-medium text-xs text-foreground" numberOfLines={1}>
495+
{props.status.label}
496+
</Text>
497+
</StatusLabelRow>
498+
);
499+
}
474500
if (props.status.kind === "goal") {
475501
return (
476502
<StatusLabelRow

‎apps/mobile/src/features/threads/floating-working-status.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ export type FloatingWorkingStatus =
3131
readonly label: string;
3232
readonly action: FloatingStatusAction;
3333
}
34+
| {
35+
readonly kind: "stop";
36+
readonly label: string;
37+
readonly unresponsive: boolean;
38+
readonly action: FloatingStatusAction | null;
39+
}
3440
| { readonly kind: "goal"; readonly label: string; readonly accessibilityLabel: string }
3541
| { readonly kind: "preparing"; readonly label: string }
3642
| {

‎apps/server/src/auth/RpcAuthorization.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ export const RPC_REQUIRED_SCOPES = {
4242
[ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: AuthOrchestrationOperateScope,
4343
[ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: AuthOrchestrationReadScope,
4444
[ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope,
45+
[ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope,
4546
[ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: AuthOrchestrationReadScope,
4647
[ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: AuthOrchestrationReadScope,
4748
[ORCHESTRATION_V2_WS_METHODS.searchThreads]: AuthOrchestrationReadScope,

‎apps/server/src/environment/ServerEnvironment.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ export const make = Effect.gen(function* () {
240240
projectSettingsOverrides: true,
241241
projectProviderInstanceOverrides: true,
242242
threadSnooze: true,
243+
threadForceStop: true,
243244
environmentThemes: true,
244245
usageLimitSources: true,
245246
providerWorkspaceSnapshotsOnDemand: true,

‎apps/server/src/observability/RpcInstrumentation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ const RPC_AGGREGATES = {
1414
[ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: "orchestrationV2",
1515
[ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: "orchestration",
1616
[ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: "orchestration",
17+
[ORCHESTRATION_V2_WS_METHODS.forceStopThread]: "orchestration",
1718
[ORCHESTRATION_V2_WS_METHODS.getTurnItem]: "orchestration",
1819
[ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: "orchestration",
1920
[ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: "orchestration",

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 20 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,8 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500;
322322
// A steer waits at most this long, from its request, for its foreground
323323
// subagents to move to the background before it is sent.
324324
const BACKGROUND_MOVE_TIMEOUT_MS = 5_000;
325+
// How long a Stop waits for the CLI to exit before the user is offered Force stop.
326+
const CLAUDE_STOP_CONFIRM_TIMEOUT_MS = 10_000;
325327

326328
const CLAUDE_GROUP_WATCH_SCRIPT = [
327329
"{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } </dev/null >/dev/null 2>&1 &",
@@ -10433,44 +10435,36 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
1043310435
yield* existing.query.interrupt;
1043410436
yield* existing.query.close.pipe(Effect.ignore);
1043510437
const closed = yield* Deferred.await(existing.closed).pipe(
10436-
Effect.timeoutOption("10 seconds"),
10438+
Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS),
1043710439
);
1043810440
if (Option.isSome(closed)) {
1043910441
return;
1044010442
}
10441-
10442-
const completedAt = yield* DateTime.now;
10443+
// The turn stays open: the CLI may still be working, and the user decides
10444+
// whether to Force stop it.
1044310445
yield* Effect.logWarning("orchestration-v2.claude-query-interrupt-timeout", {
1044410446
providerSessionId: input.providerSessionId,
1044510447
providerThreadId: turnInput.providerThread.id,
1044610448
providerTurnId: turnInput.providerTurnId,
1044710449
});
10448-
const ownedLiveQuery = yield* Ref.modify(queryContext, (current) =>
10449-
current?.query === existing.query ? [true, null] : [false, current],
10450-
);
10451-
if (ownedLiveQuery) {
10452-
existing.stopping = true;
10453-
yield* finalizeActiveTurnAfterQueryExit(existing);
10454-
yield* endSubagentCallsOfClosedQuery(existing.nativeThreadId, "interrupted");
10455-
} else {
10456-
yield* finalizeActiveTurn({
10457-
context: currentTurn,
10458-
status: "interrupted",
10459-
completedAt,
10460-
});
10461-
}
10462-
yield* Deferred.succeed(existing.closed, undefined);
10450+
return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
10451+
driver: CLAUDE_PROVIDER,
10452+
providerThreadId: turnInput.providerThread.id,
10453+
providerTurnId: turnInput.providerTurnId,
10454+
timeoutMs: CLAUDE_STOP_CONFIRM_TIMEOUT_MS,
10455+
});
1046310456
},
1046410457
(effect, turnInput) =>
1046510458
effect.pipe(
10466-
Effect.mapError(
10467-
(cause) =>
10468-
new ProviderAdapter.ProviderAdapterInterruptError({
10469-
driver: CLAUDE_PROVIDER,
10470-
providerThreadId: turnInput.providerThread.id,
10471-
providerTurnId: turnInput.providerTurnId,
10472-
cause,
10473-
}),
10459+
Effect.mapError((cause) =>
10460+
cause._tag === "ProviderAdapterInterruptTimeoutError"
10461+
? cause
10462+
: new ProviderAdapter.ProviderAdapterInterruptError({
10463+
driver: CLAUDE_PROVIDER,
10464+
providerThreadId: turnInput.providerThread.id,
10465+
providerTurnId: turnInput.providerTurnId,
10466+
cause,
10467+
}),
1047410468
),
1047510469
),
1047610470
);

‎apps/server/src/orchestration-v2/EffectWorker.test.ts‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ function layerExecutorFor(input: {
111111
closeInstance: () => Effect.void,
112112
release: () => record("release"),
113113
detach: () => record("detach"),
114+
otherWorkingThreads: () => Effect.succeed([]),
115+
releaseUnlessBusy: () => Effect.succeed(true),
114116
}),
115117
),
116118
Layer.succeed(
@@ -167,7 +169,18 @@ function layerExecutorFor(input: {
167169
);
168170
}
169171

170-
it.effect("settles a stopped run when its adapter has already lost the native turn", () =>
172+
it.effect.each([
173+
{
174+
name: "settles a stopped run when its adapter has already lost the native turn",
175+
unresponsive: false,
176+
dispatched: ["thread.background-work.settle"],
177+
},
178+
{
179+
name: "leaves the run running and flags the Stop when the provider never confirms it",
180+
unresponsive: true,
181+
dispatched: ["thread.interrupt.unresponsive"],
182+
},
183+
])("$name", ({ unresponsive, dispatched }) =>
171184
Effect.gen(function* () {
172185
const now = yield* DateTime.now;
173186
const events = yield* Ref.make<ReadonlyArray<string>>([]);
@@ -178,7 +191,8 @@ it.effect("settles a stopped run when its adapter has already lost the native tu
178191
threadId,
179192
operation: "interrupt",
180193
providerTurnId,
181-
cause: "Provider turn is not active.",
194+
...(unresponsive ? { unresponsive } : {}),
195+
cause: unresponsive ? "Timed out." : "Provider turn is not active.",
182196
}),
183197
threads: {
184198
dispatch: (command) =>
@@ -199,7 +213,7 @@ it.effect("settles a stopped run when its adapter has already lost the native tu
199213
},
200214
});
201215
}).pipe(Effect.provide(layer));
202-
assert.deepEqual(yield* Ref.get(events), ["thread.background-work.settle"]);
216+
assert.deepEqual(yield* Ref.get(events), dispatched);
203217
}),
204218
);
205219

‎apps/server/src/orchestration-v2/EffectWorker.ts‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -177,34 +177,49 @@ export const layerExecutor: Layer.Layer<
177177
cause,
178178
}),
179179
),
180+
Effect.map((outcome) => ({ unresponsive: false, outcome }) as const),
181+
// Settling here would end the run while the agent may still be running; the
182+
// user decides with Force stop instead.
183+
Effect.catchIf(
184+
(cause) => request.subagent === undefined && cause.unresponsive === true,
185+
() => Effect.succeed({ unresponsive: true, outcome: undefined } as const),
186+
),
180187
Effect.catchIf(
181188
(cause) =>
182189
request.subagent === undefined &&
183190
isNonRetryableProviderTurnControlFailure(
184191
request.type,
185192
Cause.pretty(Cause.fail(cause)),
186193
),
187-
() => Effect.void,
194+
() => Effect.succeed({ unresponsive: false, outcome: undefined } as const),
188195
),
189-
Effect.andThen((outcome) =>
190-
request.subagent !== undefined && outcome === undefined
191-
? Effect.void
192-
: threads.dispatch({
193-
type: "thread.background-work.settle",
194-
commandId: CommandId.make(`${effect.id}:background-work-settled`),
195-
...(request.subagent === undefined || outcome === undefined
196-
? {}
197-
: {
198-
subagent: {
199-
id: request.subagent.id,
200-
stoppedAt: DateTime.makeUnsafe(effect.createdAt),
201-
outcome,
202-
},
203-
}),
196+
Effect.andThen(({ unresponsive, outcome }) =>
197+
unresponsive
198+
? threads.dispatch({
199+
type: "thread.interrupt.unresponsive",
200+
commandId: CommandId.make(`${effect.id}:unresponsive`),
204201
threadId: effect.threadId,
205202
providerThreadId: request.providerThreadId,
206203
providerTurnId: request.providerTurnId,
207-
}),
204+
})
205+
: request.subagent !== undefined && outcome === undefined
206+
? Effect.void
207+
: threads.dispatch({
208+
type: "thread.background-work.settle",
209+
commandId: CommandId.make(`${effect.id}:background-work-settled`),
210+
...(request.subagent === undefined || outcome === undefined
211+
? {}
212+
: {
213+
subagent: {
214+
id: request.subagent.id,
215+
stoppedAt: DateTime.makeUnsafe(effect.createdAt),
216+
outcome,
217+
},
218+
}),
219+
threadId: effect.threadId,
220+
providerThreadId: request.providerThreadId,
221+
providerTurnId: request.providerTurnId,
222+
}),
208223
),
209224
Effect.mapError(
210225
(cause) =>

0 commit comments

Comments
 (0)