- [Production AI governance](/architecture/production-ai-governance) — lightweight adoption checklist for evaluation, replay, tracing, cost, and agent context governance

Structured `model_fallback_decision` logs also include flat `fallbackStep*` fields when a candidate fails, is skipped, or a later fallback succeeds. These fields make the attempted transition explicit (`fallbackStepFromModel`, `fallbackStepToModel`, `fallbackStepFromFailureReason`, `fallbackStepFromFailureDetail`, `fallbackStepFinalOutcome`) and include aggregate queue pressure (`fallbackStepQueueActive`, `fallbackStepQueueQueued`, `fallbackStepQueueDraining`) so log and diagnostic exporters can reconstruct the primary failure even when the terminal fallback also fails.

Isolated cron agent runs mirror the same primary/fallback/queue chain into the cron task `progressSummary`, so `openclaw tasks show <task>` can identify the primary model, fallback target, failure reason, and queue/drain state without replaying gateway logs.

"architecture/production-ai-governance",

For high-risk AI pipeline changes, pair the regular test selection below with

the lightweight [Production AI governance](/architecture/production-ai-governance)

contract. The minimum golden coverage is:

- memory retrieval preserves source, scope, and degraded-mode behavior

- provider and tool replay preserves transcript bytes unless a documented

repair path intentionally rewrites them

- channel routing names every configured inbound surface separately, including

DMs, groups, channels, threads, mentions, slash commands, webhooks, and native

command delivery when they exist

- agent context enters through scoped `AGENTS.md`, workspace files, skills, and

the documented system prompt path

- token, cache, and cost changes can be attributed to a session, provider, model,

or pipeline stage

When existing tests already cover a behavior, link to or run those tests instead

of adding a parallel harness only to satisfy a checklist.

#!/usr/bin/env tsx

import { mkdtempSync } from "node:fs";

import { tmpdir } from "node:os";

import path from "node:path";

const stateDir = mkdtempSync(path.join(tmpdir(), "openclaw-pr-87343-proof-"));

process.env.OPENCLAW_STATE_DIR = stateDir;

const {

GatewayDrainingError,

enqueueCommand,

enqueueCommandInLane,

getCommandLaneSnapshot,

markGatewayDraining,

resetCommandQueueStateForTest,

} = await import("../../src/process/command-queue.js");

const {

createRunningTaskRun,

recordTaskRunProgressByRunId,

resetDetachedTaskLifecycleRuntimeForTests,

} = await import("../../src/tasks/detached-task-runtime.js");

const { findTaskByRunId, resetTaskRegistryForTests } =

await import("../../src/tasks/task-registry.js");

const { resetTaskFlowRegistryForTests } = await import("../../src/tasks/task-flow-registry.js");

const { FailoverError } = await import("../../src/agents/failover-error.js");

const { runWithModelFallback } = await import("../../src/agents/model-fallback.js");

const runId = "pr-87343-cron-proof";

const sessionLane = "session:pr-87343-cron-proof";

const globalLane = "agent:cron:pr-87343-cron-proof";

const primary = "openai/gpt-4.1-mini";

const fallback = "anthropic/claude-haiku-3-5";

const cfg = {

agents: {

defaults: {

model: {

primary,

fallbacks: [fallback],

},

},

},

function formatProgressSummary(step?: {

fallbackStepFromModel: string;

fallbackStepToModel?: string;

fallbackStepFromFailureReason?: string;

fallbackStepFinalOutcome: string;

fallbackStepFromFailureDetail?: string;

fallbackStepQueueActive?: number;

fallbackStepQueueQueued?: number;

fallbackStepQueueDraining?: boolean;

}): string {

if (!step) {

return `model primary=${primary}; queue active=0 queued=0 draining=no`;

}

const target = step.fallbackStepToModel ? ` -> ${step.fallbackStepToModel}` : "";

const reason = step.fallbackStepFromFailureReason

? ` reason=${step.fallbackStepFromFailureReason}`

: "";

const detail = step.fallbackStepFromFailureDetail

? ` detail=${step.fallbackStepFromFailureDetail}`

: "";

const active = step.fallbackStepQueueActive ?? 0;

const queued = step.fallbackStepQueueQueued ?? 0;

const draining = step.fallbackStepQueueDraining === true ? "yes" : "no";

return `model fallback: ${step.fallbackStepFromModel}${target}${reason} outcome=${step.fallbackStepFinalOutcome}${detail}; queue active=${active} queued=${queued} draining=${draining}`;

resetTaskRegistryForTests({ persist: false });

resetTaskFlowRegistryForTests({ persist: false });

const task = createRunningTaskRun({

runtime: "cron",

ownerKey: "agent:main:main",

scopeKind: "session",

runId,

task: "PR #87343 cron fallback progress proof",

startedAt: Date.now(),

progressSummary: formatProgressSummary(),

if (!task) {

throw new Error("failed to create cron proof task run");

const activeTask = enqueueCommand(async () => {

console.log(`[proof] active queue before fallback=${JSON.stringify(getCommandLaneSnapshot())}`);

let embeddedFallbackContinuationObserved = false;

const result = await runWithModelFallback({

cfg,

provider: "openai",

model: "gpt-4.1-mini",

manifestPlugins: [],

skipAuthProfileRuntime: true,

allowGatewayDrainingContinuation: true,

onFallbackStep: (step) => {

const progressSummary = formatProgressSummary(step);

const updated = recordTaskRunProgressByRunId({

runId,

runtime: "cron",

lastEventAt: Date.now(),

progressSummary,

});

console.log(`[proof] fallback step=${JSON.stringify(step)}`);

console.log(`[proof] recorded progressSummary=${updated.at(-1)?.progressSummary}`);

},

run: async (provider, model, options) => {

console.log(

`[proof] attempt provider=${provider} model=${model} allowGatewayDrainingContinuation=${options?.allowGatewayDrainingContinuation === true}`,

);

return enqueueCommandInLane(

sessionLane,

() =>

enqueueCommandInLane(

globalLane,

async () => {

if (provider === "openai") {

markGatewayDraining();

console.log("[proof] gateway drain marked while embedded queue work is active");

throw new FailoverError("primary rate limited", {

reason: "rate_limit",

provider,

model,

status: 429,

code: "RESOURCE_EXHAUSTED",

});

}

embeddedFallbackContinuationObserved =

options?.allowGatewayDrainingContinuation === true;

return "fallback ok";

},

{

allowGatewayDrainingContinuation: options?.allowGatewayDrainingContinuation === true,

},

),

{

allowGatewayDrainingContinuation: options?.allowGatewayDrainingContinuation === true,

},

);

},

});

if (!embeddedFallbackContinuationObserved) {

throw new Error("fallback attempt did not run as an embedded queue continuation");

}

console.log(`[proof] active task result=${result.provider}/${result.model}:${result.result}`);

return result;

await activeTask;

try {

await enqueueCommand(async () => "should not enqueue during drain");

throw new Error("enqueue unexpectedly succeeded during gateway drain");

} catch (error) {

if (!(error instanceof GatewayDrainingError)) {

throw error;

}

console.log("[proof] new enqueue rejected with GatewayDrainingError");

const finalTask = findTaskByRunId(runId);

"[proof] PASS embedded queue-shaped fallback continued during drain; new enqueue still rejected",

allowGatewayDrainingContinuation: true,

allowGatewayDrainingContinuation:

runOptions?.allowGatewayDrainingContinuation === true,