Skip to content

Commit 26fdb9a

Browse files
altaywtfaltaywtf
committed
fix: preserve failover error attribution
1 parent 1de4b36 commit 26fdb9a

12 files changed

Lines changed: 164 additions & 6 deletions

File tree

‎src/agents/cli-runner.reliability.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ function buildPreparedContext(params?: {
9494
sessionKey?: string;
9595
cliSessionId?: string;
9696
runId?: string;
97+
lane?: string;
9798
openClawHistoryPrompt?: string;
9899
}): PreparedCliRunContext {
99100
const backend = {
@@ -117,6 +118,7 @@ function buildPreparedContext(params?: {
117118
thinkLevel: "low",
118119
timeoutMs: 1_000,
119120
runId: params?.runId ?? "run-2",
121+
lane: params?.lane,
120122
},
121123
started: Date.now(),
122124
workspaceDir: "/tmp",
@@ -173,6 +175,36 @@ describe("runCliAgent reliability", () => {
173175
).rejects.toThrow("produced no output");
174176
});
175177

178+
it("adds request attribution to CLI watchdog failover errors", async () => {
179+
supervisorSpawnMock.mockResolvedValueOnce(
180+
createManagedRun({
181+
reason: "no-output-timeout",
182+
exitCode: null,
183+
exitSignal: "SIGKILL",
184+
durationMs: 200,
185+
stdout: "",
186+
stderr: "",
187+
timedOut: true,
188+
noOutputTimedOut: true,
189+
}),
190+
);
191+
192+
await expect(
193+
executePreparedCliRun(
194+
buildPreparedContext({
195+
cliSessionId: "thread-123",
196+
lane: "custom-lane",
197+
runId: "run-attribution",
198+
}),
199+
"thread-123",
200+
),
201+
).rejects.toMatchObject({
202+
name: "FailoverError",
203+
sessionId: "s1",
204+
lane: "custom-lane",
205+
});
206+
});
207+
176208
it("enqueues a system event and heartbeat wake on no-output watchdog timeout for session runs", async () => {
177209
supervisorSpawnMock.mockResolvedValueOnce(
178210
createManagedRun({

‎src/agents/cli-runner.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,8 @@ export async function runPreparedCliAgent(
188188
reason,
189189
provider: params.provider,
190190
model: context.modelId,
191+
sessionId: params.sessionId,
192+
lane: params.lane,
191193
status,
192194
});
193195
}

‎src/agents/cli-runner/execute.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,8 @@ export async function executePreparedCliRun(
553553
reason: "timeout",
554554
provider: params.provider,
555555
model: context.modelId,
556+
sessionId: params.sessionId,
557+
lane: params.lane,
556558
status: resolveFailoverStatus("timeout"),
557559
});
558560
}
@@ -562,6 +564,8 @@ export async function executePreparedCliRun(
562564
reason: "timeout",
563565
provider: params.provider,
564566
model: context.modelId,
567+
sessionId: params.sessionId,
568+
lane: params.lane,
565569
status: resolveFailoverStatus("timeout"),
566570
});
567571
}
@@ -576,6 +580,8 @@ export async function executePreparedCliRun(
576580
reason,
577581
provider: params.provider,
578582
model: context.modelId,
583+
sessionId: params.sessionId,
584+
lane: params.lane,
579585
status,
580586
});
581587
}

‎src/agents/cli-runner/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export type RunCliAgentParams = {
2929
thinkLevel?: ThinkLevel;
3030
timeoutMs: number;
3131
runId: string;
32+
lane?: string;
3233
jobId?: string;
3334
extraSystemPrompt?: string;
3435
sourceReplyDeliveryMode?: SourceReplyDeliveryMode;

‎src/agents/model-fallback-observation.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ export type ModelFallbackDecisionParams = {
4646
| "candidate_failed"
4747
| "candidate_succeeded";
4848
runId?: string;
49+
sessionId?: string;
50+
lane?: string;
4951
requestedProvider: string;
5052
requestedModel: string;
5153
candidate: ModelCandidate;
@@ -145,6 +147,8 @@ export function logModelFallbackDecision(
145147
event: "model_fallback_decision",
146148
tags: ["error_handling", "model_fallback", params.decision],
147149
runId: params.runId,
150+
sessionId: params.sessionId,
151+
lane: params.lane,
148152
decision: params.decision,
149153
requestedProvider: params.requestedProvider,
150154
requestedModel: params.requestedModel,

‎src/agents/model-fallback.test.ts‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
88
import type { AuthProfileStore } from "./auth-profiles/types.js";
99
import { FailoverError } from "./failover-error.js";
1010
import { LiveSessionModelSwitchError } from "./live-model-switch-error.js";
11-
import { runWithImageModelFallback, runWithModelFallback } from "./model-fallback.js";
11+
import {
12+
FallbackSummaryError,
13+
runWithImageModelFallback,
14+
runWithModelFallback,
15+
} from "./model-fallback.js";
1216
import { classifyEmbeddedPiRunResultForModelFallback } from "./pi-embedded-runner/result-fallback-classifier.js";
1317
import type { EmbeddedPiRunResult } from "./pi-embedded-runner/types.js";
1418
import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js";
@@ -474,6 +478,51 @@ describe("runWithModelFallback", () => {
474478
});
475479
});
476480

481+
it("carries request attribution through exhausted fallback summaries", async () => {
482+
const cfg = makeCfg({
483+
agents: {
484+
defaults: {
485+
model: {
486+
primary: "openai/gpt-5.4",
487+
fallbacks: ["anthropic/claude-opus-4-6"],
488+
},
489+
},
490+
},
491+
});
492+
const run = vi
493+
.fn()
494+
.mockRejectedValueOnce(Object.assign(new Error("rate limit exceeded"), { status: 429 }))
495+
.mockRejectedValueOnce(Object.assign(new Error("overloaded"), { status: 503 }));
496+
497+
try {
498+
await runWithModelFallback({
499+
cfg,
500+
provider: "openai",
501+
model: "gpt-5.4",
502+
runId: "run-42713",
503+
sessionId: "session:browser-42713",
504+
lane: "answer",
505+
run,
506+
});
507+
throw new Error("expected fallback summary");
508+
} catch (err) {
509+
expect(err).toBeInstanceOf(FallbackSummaryError);
510+
if (!(err instanceof FallbackSummaryError)) {
511+
throw err;
512+
}
513+
expect(err).toMatchObject({
514+
name: "FallbackSummaryError",
515+
sessionId: "session:browser-42713",
516+
lane: "answer",
517+
});
518+
expect(err.cause).toMatchObject({
519+
name: "FailoverError",
520+
sessionId: "session:browser-42713",
521+
lane: "answer",
522+
});
523+
}
524+
});
525+
477526
it("uses optional result classification to continue to configured fallbacks", async () => {
478527
const cfg = makeCfg({
479528
agents: {

0 commit comments

Comments
 (0)