Skip to content

Commit 2605490

Browse files
simonusaclaude
authored and
obviyus
committed
fix(agents): classify tool-execution timeouts
Detect run-level timeouts that fire while a tool call is still active and keep them out of assistant model fallback. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent afbc395 commit 2605490

17 files changed

Lines changed: 122 additions & 1 deletion

‎extensions/codex/src/app-server/event-projector.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@ export class CodexAppServerEventProjector {
220220
timedOut: false,
221221
idleTimedOut: false,
222222
timedOutDuringCompaction: false,
223+
timedOutDuringToolExecution: false,
223224
promptError,
224225
promptErrorSource: promptError ? this.promptErrorSource || "prompt" : null,
225226
sessionIdUsed: this.params.sessionId,

‎src/agents/harness/selection.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ function createAttemptResult(sessionIdUsed: string): EmbeddedRunAttemptResult {
6868
timedOut: false,
6969
idleTimedOut: false,
7070
timedOutDuringCompaction: false,
71+
timedOutDuringToolExecution: false,
7172
promptError: null,
7273
promptErrorSource: null,
7374
sessionIdUsed,

‎src/agents/harness/v2.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ function createAttemptResult(): EmbeddedRunAttemptResult {
4747
timedOut: false,
4848
idleTimedOut: false,
4949
timedOutDuringCompaction: false,
50+
timedOutDuringToolExecution: false,
5051
promptError: null,
5152
promptErrorSource: null,
5253
sessionIdUsed: "session-1",

‎src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.e2e.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ const makeAttempt = (overrides: Partial<EmbeddedRunAttemptResult>): EmbeddedRunA
168168
timedOut: false,
169169
idleTimedOut: false,
170170
timedOutDuringCompaction: false,
171+
timedOutDuringToolExecution: false,
171172
promptError: null,
172173
promptErrorSource: null,
173174
sessionIdUsed: "session:test",

‎src/agents/pi-embedded-runner/run.overflow-compaction.fixture.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ export function makeAttemptResult(
4545
timedOut: false,
4646
idleTimedOut: false,
4747
timedOutDuringCompaction: false,
48+
timedOutDuringToolExecution: false,
4849
promptError: null,
4950
promptErrorSource: null,
5051
sessionIdUsed: "test-session",

‎src/agents/pi-embedded-runner/run.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1133,6 +1133,7 @@ export async function runEmbeddedPiAgent(
11331133
timedOut,
11341134
idleTimedOut,
11351135
timedOutDuringCompaction,
1136+
timedOutDuringToolExecution,
11361137
sessionIdUsed,
11371138
sessionFileUsed,
11381139
lastAssistant: sessionLastAssistant,
@@ -1932,6 +1933,7 @@ export async function runEmbeddedPiAgent(
19321933
failoverReason: assistantFailoverReason,
19331934
timedOut,
19341935
timedOutDuringCompaction,
1936+
timedOutDuringToolExecution,
19351937
profileRotated: false,
19361938
});
19371939
const assistantFailoverOutcome = await handleAssistantFailover({
@@ -1944,6 +1946,7 @@ export async function runEmbeddedPiAgent(
19441946
timedOut,
19451947
idleTimedOut,
19461948
timedOutDuringCompaction,
1949+
timedOutDuringToolExecution,
19471950
allowSameModelIdleTimeoutRetry:
19481951
timedOut &&
19491952
idleTimedOut &&

‎src/agents/pi-embedded-runner/run/assistant-failover.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ function makeParams(overrides: Partial<Params> = {}): Params {
1919
timedOut: false,
2020
idleTimedOut: false,
2121
timedOutDuringCompaction: false,
22+
timedOutDuringToolExecution: false,
2223
allowSameModelIdleTimeoutRetry: false,
2324
assistantProfileFailureReason: null,
2425
lastProfileId: undefined,

‎src/agents/pi-embedded-runner/run/assistant-failover.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ export async function handleAssistantFailover(params: {
4242
timedOut: boolean;
4343
idleTimedOut: boolean;
4444
timedOutDuringCompaction: boolean;
45+
timedOutDuringToolExecution: boolean;
4546
allowSameModelIdleTimeoutRetry: boolean;
4647
assistantProfileFailureReason: AuthProfileFailureReason | null;
4748
lastProfileId?: string;
@@ -177,6 +178,7 @@ export async function handleAssistantFailover(params: {
177178
failoverReason: params.failoverReason,
178179
timedOut: params.timedOut,
179180
timedOutDuringCompaction: params.timedOutDuringCompaction,
181+
timedOutDuringToolExecution: params.timedOutDuringToolExecution,
180182
profileRotated: true,
181183
});
182184
}

‎src/agents/pi-embedded-runner/run/attempt.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ import {
101101
resolveBootstrapPromptTruncationWarningMode,
102102
resolveBootstrapTotalMaxChars,
103103
} from "../../pi-embedded-helpers.js";
104+
import { countActiveToolExecutions } from "../../pi-embedded-subscribe.handlers.tools.js";
104105
import { subscribeEmbeddedPiSession } from "../../pi-embedded-subscribe.js";
105106
import { createPreparedEmbeddedPiSettingsManager } from "../../pi-project-settings.js";
106107
import {
@@ -782,6 +783,7 @@ export async function runEmbeddedAttempt(
782783
let timedOut = false;
783784
let idleTimedOut = false;
784785
let timedOutDuringCompaction = false;
786+
let timedOutDuringToolExecution = false;
785787
let promptError: unknown = null;
786788
let emitDiagnosticRunCompleted:
787789
| ((outcome: "completed" | "aborted" | "error", err?: unknown) => void)
@@ -2250,6 +2252,14 @@ export async function runEmbeddedAttempt(
22502252
aborted = true;
22512253
if (isTimeout) {
22522254
timedOut = true;
2255+
// Distinguish run-timer fires that occur while tool execution is in
2256+
// flight (LLM already responded; primary model is not at fault) from
2257+
// LLM-phase timeouts. Mirrors the `timedOutDuringCompaction` precedent
2258+
// (#46889) so the failover policy can skip pointless model fallback.
2259+
// Closes #52147.
2260+
if (!timedOutDuringCompaction && countActiveToolExecutions(params.runId) > 0) {
2261+
timedOutDuringToolExecution = true;
2262+
}
22532263
}
22542264
if (isTimeout) {
22552265
runAbortController.abort(reason ?? makeTimeoutAbortReason());
@@ -3456,6 +3466,7 @@ export async function runEmbeddedAttempt(
34563466
timedOut,
34573467
idleTimedOut,
34583468
timedOutDuringCompaction,
3469+
timedOutDuringToolExecution,
34593470
promptError: promptError ? formatErrorMessage(promptError) : undefined,
34603471
promptErrorSource,
34613472
usage: attemptUsage,
@@ -3474,6 +3485,7 @@ export async function runEmbeddedAttempt(
34743485
timedOut,
34753486
idleTimedOut,
34763487
timedOutDuringCompaction,
3488+
timedOutDuringToolExecution,
34773489
promptError: promptError ? formatErrorMessage(promptError) : undefined,
34783490
promptErrorSource,
34793491
usage: attemptUsage,
@@ -3498,6 +3510,7 @@ export async function runEmbeddedAttempt(
34983510
timedOut,
34993511
idleTimedOut,
35003512
timedOutDuringCompaction,
3513+
timedOutDuringToolExecution,
35013514
promptError: promptError ? formatErrorMessage(promptError) : undefined,
35023515
});
35033516
trajectoryEndRecorded = true;
@@ -3511,6 +3524,7 @@ export async function runEmbeddedAttempt(
35113524
timedOut,
35123525
idleTimedOut,
35133526
timedOutDuringCompaction,
3527+
timedOutDuringToolExecution,
35143528
promptError,
35153529
promptErrorSource,
35163530
preflightRecovery,
@@ -3555,6 +3569,7 @@ export async function runEmbeddedAttempt(
35553569
timedOut,
35563570
idleTimedOut,
35573571
timedOutDuringCompaction,
3572+
timedOutDuringToolExecution,
35583573
promptError: promptError ? formatErrorMessage(promptError) : undefined,
35593574
});
35603575
}

‎src/agents/pi-embedded-runner/run/failover-policy.test.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ describe("resolveRunFailoverDecision", () => {
7272
failoverReason: "rate_limit",
7373
timedOut: false,
7474
timedOutDuringCompaction: false,
75+
timedOutDuringToolExecution: false,
7576
profileRotated: false,
7677
}),
7778
).toEqual({
@@ -91,6 +92,7 @@ describe("resolveRunFailoverDecision", () => {
9192
failoverReason: "rate_limit",
9293
timedOut: false,
9394
timedOutDuringCompaction: false,
95+
timedOutDuringToolExecution: false,
9496
profileRotated: true,
9597
}),
9698
).toEqual({
@@ -110,6 +112,7 @@ describe("resolveRunFailoverDecision", () => {
110112
failoverReason: null,
111113
timedOut: false,
112114
timedOutDuringCompaction: false,
115+
timedOutDuringToolExecution: false,
113116
profileRotated: false,
114117
}),
115118
).toEqual({
@@ -134,6 +137,64 @@ describe("resolveRunFailoverDecision", () => {
134137
});
135138
});
136139

140+
it("does not rotate or fallback assistant timeouts that fired during tool execution (#52147)", () => {
141+
expect(
142+
resolveRunFailoverDecision({
143+
stage: "assistant",
144+
aborted: true,
145+
externalAbort: false,
146+
fallbackConfigured: true,
147+
failoverFailure: false,
148+
failoverReason: null,
149+
timedOut: true,
150+
timedOutDuringCompaction: false,
151+
timedOutDuringToolExecution: true,
152+
profileRotated: false,
153+
}),
154+
).toEqual({
155+
action: "continue_normal",
156+
});
157+
});
158+
159+
it("does not fallback assistant tool-execution timeouts even after profile rotation exhausted (#52147)", () => {
160+
expect(
161+
resolveRunFailoverDecision({
162+
stage: "assistant",
163+
aborted: true,
164+
externalAbort: false,
165+
fallbackConfigured: true,
166+
failoverFailure: false,
167+
failoverReason: null,
168+
timedOut: true,
169+
timedOutDuringCompaction: false,
170+
timedOutDuringToolExecution: true,
171+
profileRotated: true,
172+
}),
173+
).toEqual({
174+
action: "continue_normal",
175+
});
176+
});
177+
178+
it("still rotates assistant timeouts that fired during LLM phase (no active tool execution)", () => {
179+
expect(
180+
resolveRunFailoverDecision({
181+
stage: "assistant",
182+
aborted: true,
183+
externalAbort: false,
184+
fallbackConfigured: true,
185+
failoverFailure: false,
186+
failoverReason: null,
187+
timedOut: true,
188+
timedOutDuringCompaction: false,
189+
timedOutDuringToolExecution: false,
190+
profileRotated: false,
191+
}),
192+
).toEqual({
193+
action: "rotate_profile",
194+
reason: null,
195+
});
196+
});
197+
137198
it("does not rotate or fallback assistant timeouts after an external abort", () => {
138199
expect(
139200
resolveRunFailoverDecision({
@@ -145,6 +206,7 @@ describe("resolveRunFailoverDecision", () => {
145206
failoverReason: null,
146207
timedOut: true,
147208
timedOutDuringCompaction: false,
209+
timedOutDuringToolExecution: false,
148210
profileRotated: false,
149211
}),
150212
).toEqual({

0 commit comments

Comments
 (0)