Skip to content

Commit 0f120c0

Browse files
steipetesteipete
committed
fix(agents): bound subagent orphan recovery
1 parent f3145f6 commit 0f120c0

12 files changed

Lines changed: 495 additions & 7 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ Docs: https://docs.openclaw.ai
66

77
### Fixes
88

9+
- Agents/subagents: bound automatic orphan recovery with persisted recovery attempts and a wedged-session tombstone, and teach task maintenance/doctor to reconcile those sessions so restart loops no longer require manual `sessions.json` surgery. Fixes #74864. Thanks @solosage1.
910
- CLI/progress: suppress nested progress spinners and line clears while TUI input owns raw stdin, so Crestodian `/status` no longer disturbs the active input row. (#75003) Thanks @velvet-shark.
1011
- Telegram: use durable message edits for streaming previews instead of native draft state, so generated replies no longer flicker through draft-to-message transitions that look like duplicates. (#75073) Thanks @obviyus.
1112

‎docs/automation/tasks.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ openclaw tasks notify <lookup> state_changes
247247
Reconciliation is runtime-aware:
248248

249249
- ACP/subagent tasks check their backing child session.
250+
- Subagent tasks whose child session has a restart-recovery tombstone are marked lost instead of being treated as recoverable backing sessions.
250251
- Cron tasks check whether the cron runtime still owns the job, then recover terminal status from persisted cron run logs/job state before falling back to `lost`. Only the Gateway process is authoritative for the in-memory cron active-job set; offline CLI audit uses durable history but does not mark a cron task lost solely because that local Set is empty.
251252
- Chat-backed CLI tasks check the owning live run context, not just the chat session row.
252253

‎docs/gateway/doctor.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ cat ~/.openclaw/openclaw.json
9393
<Accordion title="State and integrity">
9494
- Session lock file inspection and stale lock cleanup.
9595
- Session transcript repair for duplicated prompt-rewrite branches created by affected 2026.4.24 builds.
96+
- Wedged subagent restart-recovery tombstone detection, with `--fix` support for clearing stale aborted recovery flags so startup does not keep treating the child as restart-aborted.
9697
- State integrity and permissions checks (sessions, transcripts, state dir).
9798
- Config file permission checks (chmod 600) when running locally.
9899
- Model auth health: checks OAuth expiry, can refresh expiring tokens, and reports auth-profile cooldown/disabled states.

‎docs/tools/subagents.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,6 +512,14 @@ restart-aborted child sessions remain recoverable through the sub-agent
512512
orphan recovery flow, which sends a synthetic resume message before
513513
clearing the aborted marker.
514514

515+
Automatic restart recovery is bounded per child session. If the same
516+
sub-agent child is accepted for orphan recovery repeatedly inside the
517+
rapid re-wedge window, OpenClaw persists a recovery tombstone on that
518+
session and stops auto-resuming it on later restarts. Run
519+
`openclaw tasks maintenance --apply` to reconcile the task record, or
520+
`openclaw doctor --fix` to clear stale aborted recovery flags on
521+
tombstoned sessions.
522+
515523
<Note>
516524
If a sub-agent spawn fails with Gateway `PAIRING_REQUIRED` /
517525
`scope-upgrade`, check the RPC caller before editing pairing state.

‎src/agents/subagent-orphan-recovery.test.ts‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,110 @@ describe("subagent-orphan-recovery", () => {
342342
expect(mockStore["agent:main:subagent:test-session-1"]?.abortedLastRun).toBe(false);
343343
});
344344

345+
it("persists accepted recovery attempts after successful resume", async () => {
346+
vi.mocked(gateway.callGateway).mockResolvedValue({ runId: "resumed-run" } as never);
347+
mockSingleAbortedSession();
348+
349+
await recoverOrphanedSubagentSessions({
350+
getActiveRuns: () => createActiveRuns(createTestRunRecord()),
351+
});
352+
353+
const [, updater] = vi.mocked(sessions.updateSessionStore).mock.calls[0];
354+
const mockStore: ReturnType<typeof sessions.loadSessionStore> = {
355+
"agent:main:subagent:test-session-1": {
356+
sessionId: "session-abc",
357+
updatedAt: 0,
358+
abortedLastRun: true,
359+
},
360+
};
361+
await updater(mockStore);
362+
expect(mockStore["agent:main:subagent:test-session-1"]).toMatchObject({
363+
abortedLastRun: false,
364+
subagentRecovery: {
365+
automaticAttempts: 1,
366+
lastRunId: "run-1",
367+
lastAttemptAt: expect.any(Number),
368+
},
369+
});
370+
});
371+
372+
it("tombstones rapid repeated accepted recovery before resuming again", async () => {
373+
const now = Date.now();
374+
mockSingleAbortedSession({
375+
subagentRecovery: {
376+
automaticAttempts: 2,
377+
lastAttemptAt: now - 30_000,
378+
lastRunId: "previous-run",
379+
},
380+
});
381+
382+
const result = await recoverOrphanedSubagentSessions({
383+
getActiveRuns: () => createActiveRuns(createTestRunRecord()),
384+
});
385+
386+
expect(result).toMatchObject({
387+
recovered: 0,
388+
failed: 0,
389+
skipped: 1,
390+
failedRuns: [
391+
expect.objectContaining({
392+
runId: "run-1",
393+
childSessionKey: "agent:main:subagent:test-session-1",
394+
error: expect.stringContaining("recovery blocked after 2 rapid accepted resume attempts"),
395+
}),
396+
],
397+
});
398+
expect(gateway.callGateway).not.toHaveBeenCalled();
399+
expect(sessions.updateSessionStore).toHaveBeenCalledOnce();
400+
401+
const [, updater] = vi.mocked(sessions.updateSessionStore).mock.calls[0];
402+
const mockStore: ReturnType<typeof sessions.loadSessionStore> = {
403+
"agent:main:subagent:test-session-1": {
404+
sessionId: "session-abc",
405+
updatedAt: 0,
406+
abortedLastRun: true,
407+
subagentRecovery: {
408+
automaticAttempts: 2,
409+
lastAttemptAt: now - 30_000,
410+
lastRunId: "previous-run",
411+
},
412+
},
413+
};
414+
await updater(mockStore);
415+
expect(mockStore["agent:main:subagent:test-session-1"]).toMatchObject({
416+
abortedLastRun: false,
417+
subagentRecovery: {
418+
automaticAttempts: 2,
419+
lastRunId: "run-1",
420+
wedgedAt: expect.any(Number),
421+
wedgedReason: expect.stringContaining("recovery blocked"),
422+
},
423+
});
424+
});
425+
426+
it("skips already tombstoned wedged sessions without rewriting them", async () => {
427+
mockSingleAbortedSession({
428+
subagentRecovery: {
429+
automaticAttempts: 2,
430+
lastAttemptAt: Date.now() - 20_000,
431+
lastRunId: "previous-run",
432+
wedgedAt: Date.now() - 10_000,
433+
wedgedReason: "subagent orphan recovery blocked after 2 rapid accepted resume attempts",
434+
},
435+
});
436+
437+
const result = await recoverOrphanedSubagentSessions({
438+
getActiveRuns: () => createActiveRuns(createTestRunRecord()),
439+
});
440+
441+
expect(result.recovered).toBe(0);
442+
expect(result.failed).toBe(0);
443+
expect(result.skipped).toBe(1);
444+
expect(result.failedRuns).toHaveLength(1);
445+
expect(gateway.callGateway).not.toHaveBeenCalled();
446+
expect(sessions.updateSessionStore).not.toHaveBeenCalled();
447+
});
448+
345449
it("truncates long task descriptions in resume message", async () => {
346450
mockSingleAbortedSession();
347451

‎src/agents/subagent-orphan-recovery.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ import {
2929
loadRequesterSessionEntry,
3030
} from "./subagent-announce-delivery.js";
3131
import { resolveAnnounceOrigin } from "./subagent-announce-origin.js";
32+
import {
33+
evaluateSubagentRecoveryGate,
34+
markSubagentRecoveryAttempt,
35+
markSubagentRecoveryWedged,
36+
} from "./subagent-recovery-state.js";
3237
import {
3338
finalizeInterruptedSubagentRun,
3439
replaceSubagentRunAfterSteer,
@@ -266,6 +271,7 @@ export async function recoverOrphanedSubagentSessions(params: {
266271
if (!childSessionKey) {
267272
continue;
268273
}
274+
const now = Date.now();
269275
if (resumedSessionKeys.has(childSessionKey)) {
270276
result.skipped++;
271277
continue;
@@ -304,6 +310,44 @@ export async function recoverOrphanedSubagentSessions(params: {
304310
continue;
305311
}
306312

313+
const recoveryGate = evaluateSubagentRecoveryGate(entry, now);
314+
if (!recoveryGate.allowed) {
315+
if (recoveryGate.shouldMarkWedged) {
316+
try {
317+
await updateSessionStore(storePath, (currentStore) => {
318+
const current = currentStore[childSessionKey];
319+
if (current) {
320+
markSubagentRecoveryWedged({
321+
entry: current,
322+
now,
323+
runId,
324+
reason: recoveryGate.reason,
325+
});
326+
currentStore[childSessionKey] = current;
327+
}
328+
});
329+
markSubagentRecoveryWedged({
330+
entry,
331+
now,
332+
runId,
333+
reason: recoveryGate.reason,
334+
});
335+
} catch (err) {
336+
log.warn(
337+
`failed to persist wedged subagent recovery marker for ${childSessionKey}: ${String(err)}`,
338+
);
339+
}
340+
}
341+
log.warn(`skipping orphan recovery for ${childSessionKey}: ${recoveryGate.reason}`);
342+
result.skipped++;
343+
result.failedRuns.push({
344+
runId,
345+
childSessionKey,
346+
error: recoveryGate.reason,
347+
});
348+
continue;
349+
}
350+
307351
log.info(`found orphaned subagent session: ${childSessionKey} (run=${runId})`);
308352

309353
const messages = readSessionMessages(entry.sessionId, storePath, entry.sessionFile);
@@ -352,6 +396,12 @@ export async function recoverOrphanedSubagentSessions(params: {
352396
const current = currentStore[childSessionKey];
353397
if (current) {
354398
current.abortedLastRun = false;
399+
markSubagentRecoveryAttempt({
400+
entry: current,
401+
now: Date.now(),
402+
runId,
403+
attempt: recoveryGate.nextAttempt,
404+
});
355405
current.updatedAt = Date.now();
356406
currentStore[childSessionKey] = current;
357407
}

‎src/agents/subagent-recovery-state.ts‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import type { SessionEntry } from "../config/sessions.js";
2+
3+
export const SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS = 2;
4+
export const SUBAGENT_RECOVERY_REWEDGE_WINDOW_MS = 2 * 60_000;
5+
6+
export type SubagentRecoveryGate =
7+
| {
8+
allowed: true;
9+
nextAttempt: number;
10+
}
11+
| {
12+
allowed: false;
13+
reason: string;
14+
shouldMarkWedged: boolean;
15+
};
16+
17+
function isRecentRecoveryAttempt(entry: SessionEntry, now: number): boolean {
18+
const lastAttemptAt = entry.subagentRecovery?.lastAttemptAt;
19+
return (
20+
typeof lastAttemptAt === "number" &&
21+
Number.isFinite(lastAttemptAt) &&
22+
now - lastAttemptAt <= SUBAGENT_RECOVERY_REWEDGE_WINDOW_MS
23+
);
24+
}
25+
26+
export function isSubagentRecoveryWedgedEntry(entry: unknown): boolean {
27+
if (!entry || typeof entry !== "object") {
28+
return false;
29+
}
30+
const recovery = (entry as SessionEntry).subagentRecovery;
31+
return (
32+
typeof recovery?.wedgedAt === "number" &&
33+
Number.isFinite(recovery.wedgedAt) &&
34+
recovery.wedgedAt > 0
35+
);
36+
}
37+
38+
export function formatSubagentRecoveryWedgedReason(entry: SessionEntry): string {
39+
return (
40+
entry.subagentRecovery?.wedgedReason?.trim() ||
41+
"subagent orphan recovery is tombstoned for this session"
42+
);
43+
}
44+
45+
export function evaluateSubagentRecoveryGate(
46+
entry: SessionEntry,
47+
now: number,
48+
): SubagentRecoveryGate {
49+
if (isSubagentRecoveryWedgedEntry(entry)) {
50+
return {
51+
allowed: false,
52+
reason: formatSubagentRecoveryWedgedReason(entry),
53+
shouldMarkWedged: false,
54+
};
55+
}
56+
57+
const previousAttempts = isRecentRecoveryAttempt(entry, now)
58+
? Math.max(0, entry.subagentRecovery?.automaticAttempts ?? 0)
59+
: 0;
60+
if (previousAttempts >= SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS) {
61+
return {
62+
allowed: false,
63+
reason:
64+
`subagent orphan recovery blocked after ${previousAttempts} rapid accepted resume attempts; ` +
65+
`run "openclaw tasks maintenance --apply" or "openclaw doctor --fix" to reconcile it`,
66+
shouldMarkWedged: true,
67+
};
68+
}
69+
70+
return {
71+
allowed: true,
72+
nextAttempt: previousAttempts + 1,
73+
};
74+
}
75+
76+
export function markSubagentRecoveryAttempt(params: {
77+
entry: SessionEntry;
78+
now: number;
79+
runId: string;
80+
attempt: number;
81+
}): void {
82+
params.entry.subagentRecovery = {
83+
automaticAttempts: Math.max(1, params.attempt),
84+
lastAttemptAt: params.now,
85+
lastRunId: params.runId,
86+
};
87+
}
88+
89+
export function markSubagentRecoveryWedged(params: {
90+
entry: SessionEntry;
91+
now: number;
92+
runId?: string;
93+
reason: string;
94+
}): void {
95+
params.entry.abortedLastRun = false;
96+
params.entry.subagentRecovery = {
97+
...params.entry.subagentRecovery,
98+
automaticAttempts: Math.max(
99+
params.entry.subagentRecovery?.automaticAttempts ?? 0,
100+
SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS,
101+
),
102+
lastAttemptAt: params.entry.subagentRecovery?.lastAttemptAt ?? params.now,
103+
...(params.runId ? { lastRunId: params.runId } : {}),
104+
wedgedAt: params.now,
105+
wedgedReason: params.reason,
106+
};
107+
params.entry.updatedAt = params.now;
108+
}
109+
110+
export function clearWedgedSubagentRecoveryAbort(entry: SessionEntry, now: number): boolean {
111+
if (!isSubagentRecoveryWedgedEntry(entry) || entry.abortedLastRun !== true) {
112+
return false;
113+
}
114+
entry.abortedLastRun = false;
115+
entry.updatedAt = now;
116+
return true;
117+
}

0 commit comments

Comments
 (0)