Skip to content

Commit 61dce7c

Browse files
Pick-catclaudesteipete
authored
fix: forward pending timeout snapshot in waitForAgentJob fallback timer (#89367)
* fix(agents): export isHardAgentRunTimeoutPhase for wait-layer reuse * fix(gateway): forward pending hard-timeout snapshot in waitForAgentJob fallback timer (fixes #89095) * test(gateway): add e2e proof for subagent hard-timeout parent notification (#89095) * fix(gateway): match fallback hard-timeout gate to terminal-outcome contract Also treat provider-started timeout snapshots as hard timeouts in the waitForAgentJob fallback, mirroring buildAgentRunTerminalOutcome. Co-Authored-By: Claude Opus 4.7 <[email protected]> * fix(gateway): use shared hard-timeout classifier in waitForAgentJob fallback * chore: amend author email * chore: rebase on main, resolve conflicts * test(gateway): focus timeout fallback coverage * docs(changelog): note agent wait timeout fix * fix(gateway): isolate fresh wait terminal state --------- Co-authored-by: Pick-cat <[email protected]> Co-authored-by: Claude Opus 4.7 <[email protected]> Co-authored-by: Peter Steinberger <[email protected]>
1 parent a00e9fc commit 61dce7c

4 files changed

Lines changed: 159 additions & 2 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ Docs: https://docs.openclaw.ai
2323

2424
### Fixes
2525

26+
- **Agent wait hard-timeout snapshots:** preserve canonical hard-timeout phase and timestamps when the outer `agent.wait` timer wins the retry-grace race, while leaving queue, draining, and restart-cancelled waits correctable. (#89367) Thanks @Pick-cat.
2627
- **Control UI typed approvals:** send `/approve` commands immediately through the authorized Gateway command path while an agent run is blocked instead of queueing the command behind that run. (#77672) Thanks @vincentkoc.
2728
- **Microsoft Teams Graph response bounds:** cap successful file-upload and chat JSON reads so oversized Microsoft Graph responses cannot be buffered without limit. (#97784) Thanks @Alix-007.
2829
- **Packaged speech runtime:** stop treating package-backed `speech-core` as a bundled plugin sidecar, restoring TTS startup in npm installs while release checks keep true activation-bypassing facades package-complete. (#89899, #89425) Thanks @zhangguiping-xydt.

‎src/gateway/server-methods/agent-job.timeout-fallback.test.ts‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// Focused wait-layer coverage for the outer-timer race behind #89095.
2+
// Parent announce delivery is a separate contract and remains outside this test.
3+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
4+
import { AGENT_RUN_RESTART_ABORT_STOP_REASON } from "../../agents/run-termination.js";
5+
import { emitAgentEvent } from "../../infra/agent-events.js";
6+
import { waitForAgentJob } from "./agent-job.js";
7+
8+
const HARD_TIMEOUT_PHASES = ["preflight", "provider", "post_turn"] as const;
9+
const NON_HARD_TIMEOUTS = [
10+
{
11+
label: "soft queue timeout",
12+
data: { timeoutPhase: "queue" },
13+
},
14+
] as const;
15+
16+
let runSequence = 0;
17+
18+
async function resolveOuterTimeoutRace(
19+
data: Readonly<Record<string, unknown>>,
20+
options?: { ignoreCachedSnapshot?: boolean },
21+
) {
22+
const runId = `run-timeout-fallback-${runSequence++}`;
23+
const waitPromise = waitForAgentJob({
24+
runId,
25+
timeoutMs: 5_000,
26+
ignoreCachedSnapshot: options?.ignoreCachedSnapshot,
27+
});
28+
29+
emitAgentEvent({
30+
runId,
31+
stream: "lifecycle",
32+
data: { phase: "start", startedAt: 1_000 },
33+
});
34+
emitAgentEvent({
35+
runId,
36+
stream: "lifecycle",
37+
data: {
38+
phase: "end",
39+
startedAt: 1_000,
40+
endedAt: 1_100,
41+
aborted: true,
42+
...data,
43+
},
44+
});
45+
46+
// Fire the outer wait before the 15-second terminal retry grace publishes.
47+
await vi.advanceTimersByTimeAsync(6_000);
48+
return await waitPromise;
49+
}
50+
51+
describe("waitForAgentJob timeout fallback", () => {
52+
beforeEach(() => {
53+
vi.useFakeTimers();
54+
});
55+
56+
afterEach(() => {
57+
vi.clearAllTimers();
58+
vi.useRealTimers();
59+
});
60+
61+
for (const phase of HARD_TIMEOUT_PHASES) {
62+
it(`forwards a pending ${phase} hard timeout`, async () => {
63+
await expect(resolveOuterTimeoutRace({ timeoutPhase: phase })).resolves.toMatchObject({
64+
status: "timeout",
65+
timeoutPhase: phase,
66+
startedAt: 1_000,
67+
endedAt: 1_100,
68+
});
69+
});
70+
}
71+
72+
for (const scenario of NON_HARD_TIMEOUTS) {
73+
it(`does not forward a ${scenario.label}`, async () => {
74+
await expect(resolveOuterTimeoutRace(scenario.data)).resolves.toBeNull();
75+
});
76+
}
77+
78+
it("keeps restart cancellation as an error instead of a hard timeout", async () => {
79+
await expect(
80+
resolveOuterTimeoutRace({
81+
providerStarted: true,
82+
stopReason: AGENT_RUN_RESTART_ABORT_STOP_REASON,
83+
}),
84+
).resolves.toMatchObject({
85+
status: "error",
86+
stopReason: AGENT_RUN_RESTART_ABORT_STOP_REASON,
87+
startedAt: 1_000,
88+
endedAt: 1_100,
89+
});
90+
});
91+
92+
it("ignores a hard timeout that predates a fresh wait", async () => {
93+
const runId = `run-timeout-fallback-${runSequence++}`;
94+
emitAgentEvent({
95+
runId,
96+
stream: "lifecycle",
97+
data: { phase: "start", startedAt: 1_000 },
98+
});
99+
emitAgentEvent({
100+
runId,
101+
stream: "lifecycle",
102+
data: {
103+
phase: "end",
104+
startedAt: 1_000,
105+
endedAt: 1_100,
106+
aborted: true,
107+
timeoutPhase: "provider",
108+
},
109+
});
110+
111+
const waitPromise = waitForAgentJob({
112+
runId,
113+
timeoutMs: 5_000,
114+
ignoreCachedSnapshot: true,
115+
});
116+
await vi.advanceTimersByTimeAsync(6_000);
117+
118+
await expect(waitPromise).resolves.toBeNull();
119+
});
120+
121+
it("lets a fresh wait consume a hard timeout it observes", async () => {
122+
await expect(
123+
resolveOuterTimeoutRace({ timeoutPhase: "provider" }, { ignoreCachedSnapshot: true }),
124+
).resolves.toMatchObject({
125+
status: "timeout",
126+
timeoutPhase: "provider",
127+
endedAt: 1_100,
128+
});
129+
});
130+
});

‎src/gateway/server-methods/agent-job.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,7 @@ export async function waitForAgentJob(params: {
331331
return await new Promise((resolve) => {
332332
let settled = false;
333333
let pendingErrorTimer: NodeJS.Timeout | undefined;
334+
let pendingErrorSnapshot: AgentRunSnapshot | undefined;
334335
let pendingTimeoutTimer: NodeJS.Timeout | undefined;
335336
let pendingTimeoutSnapshot: AgentRunSnapshot | undefined;
336337
let removeWaiter = () => {};
@@ -341,6 +342,7 @@ export async function waitForAgentJob(params: {
341342
}
342343
clearTimeout(pendingErrorTimer);
343344
pendingErrorTimer = undefined;
345+
pendingErrorSnapshot = undefined;
344346
};
345347

346348
const clearPendingTimeoutTimer = () => {
@@ -395,6 +397,7 @@ export async function waitForAgentJob(params: {
395397
timerRef.unref?.();
396398
if (kind === "error") {
397399
pendingErrorTimer = timerRef;
400+
pendingErrorSnapshot = snapshot;
398401
} else {
399402
pendingTimeoutTimer = timerRef;
400403
pendingTimeoutSnapshot = snapshot;
@@ -478,8 +481,30 @@ export async function waitForAgentJob(params: {
478481
removeWaiter = addAgentRunWaiter(runId);
479482

480483
const timer = setSafeTimeout(() => {
481-
const pendingError = getPendingAgentRunError(runId);
482-
finish(pendingError ? createPendingErrorTimeoutSnapshot(pendingError.snapshot) : null);
484+
// Fresh waits can only consume terminal state observed by this waiter;
485+
// the shared maps may still hold a previous attempt inside retry grace.
486+
const pendingErrorCandidate = ignoreCachedSnapshot
487+
? pendingErrorSnapshot
488+
: getPendingAgentRunError(runId)?.snapshot;
489+
if (pendingErrorCandidate) {
490+
finish(createPendingErrorTimeoutSnapshot(pendingErrorCandidate));
491+
return;
492+
}
493+
const pendingTimeoutCandidate = ignoreCachedSnapshot
494+
? pendingTimeoutSnapshot
495+
: getPendingAgentRunTimeout(runId)?.snapshot;
496+
// Forward only canonical hard timeouts. Reuse the shared terminal-outcome
497+
// classifier (which excludes restart-cancelled and soft queue/draining
498+
// snapshots) instead of rederiving the hard/soft gate here, so those stay
499+
// correctable via retry grace.
500+
if (
501+
pendingTimeoutCandidate &&
502+
terminalOutcomeFromSnapshot(pendingTimeoutCandidate)?.reason === "hard_timeout"
503+
) {
504+
finish(pendingTimeoutCandidate);
505+
return;
506+
}
507+
finish(null);
483508
}, timeoutMs);
484509
const onAbort: (() => void) | undefined = () => finish(null);
485510
signal?.addEventListener("abort", onAbort, { once: true });

‎src/gateway/server-methods/server-methods.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -660,6 +660,7 @@ describe("waitForAgentJob", () => {
660660
expect(agentJobTesting.getAgentRunCacheSize()).toBe(max);
661661
agentJobTesting.resetAgentRunCache();
662662
});
663+
663664
});
664665

665666
describe("augmentChatHistoryWithCanvasBlocks", () => {

0 commit comments

Comments
 (0)