Skip to content

Commit 9df0ae6

Browse files
wenxu007altaywtf
wenxu007
and
altaywtf
authored
fix(agents,failover): propagate sessionId/lane/provider attribution through FailoverError (#73506)
* fix(agents,failover): propagate sessionId/lane/provider attribution through FailoverError Adds optional `sessionId` and `lane` fields to `FailoverError` and threads them — together with the existing `provider`, `model`, `profileId` — through `describeFailoverError` and `coerceToFailoverError` context, so structured error log ingestion can attribute exhausted-fallback wrapper errors back to the originating request instead of dropping the per-profile metadata when the final wrapper is built. Fixes #42713. * fix: preserve failover error attribution --------- Co-authored-by: Altay <[email protected]>
1 parent 29ed526 commit 9df0ae6

15 files changed

Lines changed: 227 additions & 7 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ Docs: https://docs.openclaw.ai
8787
- fix(logging): add redaction patterns for Tencent Cloud, Alibaba Cloud, HuggingFace and Replicate API keys (#58162). Thanks @gavyngong
8888
- Pairing: surface unexpected allowlist filesystem stat errors instead of treating the allowlist as missing, so permission and I/O failures are visible during pairing authorization checks. (#63324) Thanks @franciscomaestre.
8989
- macOS app: reserve layout space for exec approval command details so the allow dialog no longer overlaps the command, context, and action buttons. (#75470) Thanks @ngutman.
90+
- Agents/failover: carry `sessionId`, `lane`, `provider`, `model`, and `profileId` attribution through `FailoverError` and `describeFailoverError`/`coerceToFailoverError` so structured error logs (e.g. `gateway.err.log` ingestion) can attribute exhausted-fallback wrapper errors to the originating session and last-attempted provider instead of dropping the metadata after the per-profile errors. Fixes #42713. (#73506) Thanks @wenxu007.
9091

9192
## 2026.4.29
9293

‎src/agents/cli-runner.reliability.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ function buildPreparedContext(params?: {
9494
sessionKey?: string;
9595
cliSessionId?: string;
9696
runId?: string;
97+
lane?: string;
9798
openClawHistoryPrompt?: string;
9899
}): PreparedCliRunContext {
99100
const backend = {
@@ -117,6 +118,7 @@ function buildPreparedContext(params?: {
117118
thinkLevel: "low",
118119
timeoutMs: 1_000,
119120
runId: params?.runId ?? "run-2",
121+
lane: params?.lane,
120122
},
121123
started: Date.now(),
122124
workspaceDir: "/tmp",
@@ -173,6 +175,36 @@ describe("runCliAgent reliability", () => {
173175
).rejects.toThrow("produced no output");
174176
});
175177

178+
it("adds request attribution to CLI watchdog failover errors", async () => {
179+
supervisorSpawnMock.mockResolvedValueOnce(
180+
createManagedRun({
181+
reason: "no-output-timeout",
182+
exitCode: null,
183+
exitSignal: "SIGKILL",
184+
durationMs: 200,
185+
stdout: "",
186+
stderr: "",
187+
timedOut: true,
188+
noOutputTimedOut: true,
189+
}),
190+
);
191+
192+
await expect(
193+
executePreparedCliRun(
194+
buildPreparedContext({
195+
cliSessionId: "thread-123",
196+
lane: "custom-lane",
197+
runId: "run-attribution",
198+
}),
199+
"thread-123",
200+
),
201+
).rejects.toMatchObject({
202+
name: "FailoverError",
203+
sessionId: "s1",
204+
lane: "custom-lane",
205+
});
206+
});
207+
176208
it("enqueues a system event and heartbeat wake on no-output watchdog timeout for session runs", async () => {
177209
supervisorSpawnMock.mockResolvedValueOnce(
178210
createManagedRun({

‎src/agents/cli-runner.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,8 @@ export async function runPreparedCliAgent(
188188
reason,
189189
provider: params.provider,
190190
model: context.modelId,
191+
sessionId: params.sessionId,
192+
lane: params.lane,
191193
status,
192194
});
193195
}

‎src/agents/cli-runner/execute.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,8 @@ export async function executePreparedCliRun(
553553
reason: "timeout",
554554
provider: params.provider,
555555
model: context.modelId,
556+
sessionId: params.sessionId,
557+
lane: params.lane,
556558
status: resolveFailoverStatus("timeout"),
557559
});
558560
}
@@ -562,6 +564,8 @@ export async function executePreparedCliRun(
562564
reason: "timeout",
563565
provider: params.provider,
564566
model: context.modelId,
567+
sessionId: params.sessionId,
568+
lane: params.lane,
565569
status: resolveFailoverStatus("timeout"),
566570
});
567571
}
@@ -576,6 +580,8 @@ export async function executePreparedCliRun(
576580
reason,
577581
provider: params.provider,
578582
model: context.modelId,
583+
sessionId: params.sessionId,
584+
lane: params.lane,
579585
status,
580586
});
581587
}

‎src/agents/cli-runner/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export type RunCliAgentParams = {
2929
thinkLevel?: ThinkLevel;
3030
timeoutMs: number;
3131
runId: string;
32+
lane?: string;
3233
jobId?: string;
3334
extraSystemPrompt?: string;
3435
sourceReplyDeliveryMode?: SourceReplyDeliveryMode;

‎src/agents/failover-error.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -972,4 +972,40 @@ describe("failover-error", () => {
972972
expect(described.message).toBe("123");
973973
expect(described.reason).toBeUndefined();
974974
});
975+
976+
it("propagates sessionId/lane/provider attribution through FailoverError (#42713)", () => {
977+
const err = new FailoverError("all fallbacks exhausted", {
978+
reason: "rate_limit",
979+
provider: "anthropic",
980+
model: "claude-opus-4-6",
981+
profileId: "profile-2",
982+
sessionId: "session:browser-abcd",
983+
lane: "answer",
984+
status: 429,
985+
});
986+
expect(err.sessionId).toBe("session:browser-abcd");
987+
expect(err.lane).toBe("answer");
988+
expect(describeFailoverError(err)).toMatchObject({
989+
provider: "anthropic",
990+
model: "claude-opus-4-6",
991+
profileId: "profile-2",
992+
sessionId: "session:browser-abcd",
993+
lane: "answer",
994+
reason: "rate_limit",
995+
status: 429,
996+
});
997+
});
998+
999+
it("coerceToFailoverError carries sessionId/lane from context (#42713)", () => {
1000+
const err = coerceToFailoverError("rate limit exceeded", {
1001+
provider: "openai",
1002+
model: "gpt-5",
1003+
profileId: "p1",
1004+
sessionId: "session:browser-1234",
1005+
lane: "draft",
1006+
});
1007+
expect(err?.sessionId).toBe("session:browser-1234");
1008+
expect(err?.lane).toBe("draft");
1009+
expect(err?.provider).toBe("openai");
1010+
});
9751011
});

‎src/agents/failover-error.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ export class FailoverError extends Error {
2121
readonly status?: number;
2222
readonly code?: string;
2323
readonly rawError?: string;
24+
// Originating request attribution propagated through wrapper errors so
25+
// structured log ingestion (e.g. api_health_log) can attribute exhausted
26+
// failover failures back to a session/lane and the last attempted provider.
27+
// See #42713.
28+
readonly sessionId?: string;
29+
readonly lane?: string;
2430

2531
constructor(
2632
message: string,
@@ -32,6 +38,8 @@ export class FailoverError extends Error {
3238
status?: number;
3339
code?: string;
3440
rawError?: string;
41+
sessionId?: string;
42+
lane?: string;
3543
cause?: unknown;
3644
},
3745
) {
@@ -44,6 +52,8 @@ export class FailoverError extends Error {
4452
this.status = params.status;
4553
this.code = params.code;
4654
this.rawError = params.rawError;
55+
this.sessionId = params.sessionId;
56+
this.lane = params.lane;
4757
}
4858
}
4959

@@ -422,6 +432,11 @@ export function describeFailoverError(err: unknown): {
422432
reason?: FailoverReason;
423433
status?: number;
424434
code?: string;
435+
provider?: string;
436+
model?: string;
437+
profileId?: string;
438+
sessionId?: string;
439+
lane?: string;
425440
} {
426441
if (isFailoverError(err)) {
427442
return {
@@ -430,6 +445,11 @@ export function describeFailoverError(err: unknown): {
430445
reason: err.reason,
431446
status: err.status,
432447
code: err.code,
448+
provider: err.provider,
449+
model: err.model,
450+
profileId: err.profileId,
451+
sessionId: err.sessionId,
452+
lane: err.lane,
433453
};
434454
}
435455
const signal = normalizeErrorSignal(err);
@@ -439,6 +459,7 @@ export function describeFailoverError(err: unknown): {
439459
reason: resolveFailoverReasonFromError(err) ?? undefined,
440460
status: signal.status,
441461
code: signal.code,
462+
provider: signal.provider,
442463
};
443464
}
444465

@@ -448,6 +469,8 @@ export function coerceToFailoverError(
448469
provider?: string;
449470
model?: string;
450471
profileId?: string;
472+
sessionId?: string;
473+
lane?: string;
451474
},
452475
): FailoverError | null {
453476
if (isFailoverError(err)) {
@@ -465,9 +488,11 @@ export function coerceToFailoverError(
465488

466489
return new FailoverError(message, {
467490
reason,
468-
provider: context?.provider,
491+
provider: context?.provider ?? signal.provider,
469492
model: context?.model,
470493
profileId: context?.profileId,
494+
sessionId: context?.sessionId,
495+
lane: context?.lane,
471496
status,
472497
code,
473498
rawError: message,

‎src/agents/model-fallback-observation.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ export type ModelFallbackDecisionParams = {
4646
| "candidate_failed"
4747
| "candidate_succeeded";
4848
runId?: string;
49+
sessionId?: string;
50+
lane?: string;
4951
requestedProvider: string;
5052
requestedModel: string;
5153
candidate: ModelCandidate;
@@ -145,6 +147,8 @@ export function logModelFallbackDecision(
145147
event: "model_fallback_decision",
146148
tags: ["error_handling", "model_fallback", params.decision],
147149
runId: params.runId,
150+
sessionId: params.sessionId,
151+
lane: params.lane,
148152
decision: params.decision,
149153
requestedProvider: params.requestedProvider,
150154
requestedModel: params.requestedModel,

‎src/agents/model-fallback.test.ts‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
88
import type { AuthProfileStore } from "./auth-profiles/types.js";
99
import { FailoverError } from "./failover-error.js";
1010
import { LiveSessionModelSwitchError } from "./live-model-switch-error.js";
11-
import { runWithImageModelFallback, runWithModelFallback } from "./model-fallback.js";
11+
import {
12+
FallbackSummaryError,
13+
runWithImageModelFallback,
14+
runWithModelFallback,
15+
} from "./model-fallback.js";
1216
import { classifyEmbeddedPiRunResultForModelFallback } from "./pi-embedded-runner/result-fallback-classifier.js";
1317
import type { EmbeddedPiRunResult } from "./pi-embedded-runner/types.js";
1418
import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js";
@@ -474,6 +478,51 @@ describe("runWithModelFallback", () => {
474478
});
475479
});
476480

481+
it("carries request attribution through exhausted fallback summaries", async () => {
482+
const cfg = makeCfg({
483+
agents: {
484+
defaults: {
485+
model: {
486+
primary: "openai/gpt-5.4",
487+
fallbacks: ["anthropic/claude-opus-4-6"],
488+
},
489+
},
490+
},
491+
});
492+
const run = vi
493+
.fn()
494+
.mockRejectedValueOnce(Object.assign(new Error("rate limit exceeded"), { status: 429 }))
495+
.mockRejectedValueOnce(Object.assign(new Error("overloaded"), { status: 503 }));
496+
497+
try {
498+
await runWithModelFallback({
499+
cfg,
500+
provider: "openai",
501+
model: "gpt-5.4",
502+
runId: "run-42713",
503+
sessionId: "session:browser-42713",
504+
lane: "answer",
505+
run,
506+
});
507+
throw new Error("expected fallback summary");
508+
} catch (err) {
509+
expect(err).toBeInstanceOf(FallbackSummaryError);
510+
if (!(err instanceof FallbackSummaryError)) {
511+
throw err;
512+
}
513+
expect(err).toMatchObject({
514+
name: "FallbackSummaryError",
515+
sessionId: "session:browser-42713",
516+
lane: "answer",
517+
});
518+
expect(err.cause).toMatchObject({
519+
name: "FailoverError",
520+
sessionId: "session:browser-42713",
521+
lane: "answer",
522+
});
523+
}
524+
});
525+
477526
it("uses optional result classification to continue to configured fallbacks", async () => {
478527
const cfg = makeCfg({
479528
agents: {

0 commit comments

Comments
 (0)