fix(cron): preflight model fallbacks before skip (#82887)

chen-zhang-cs-code · osolmaz · web-flow · commit 7a381b807e23 · 2026-05-29T14:29:26.000+08:00
Fix cron local-model preflight fallback handling so scheduled runs try configured fallback candidates before skipping when the local primary is unavailable. Verification: - GitHub CI on PR head fe884da: passing required CI checks. - Local focused cron/model fallback tests passed earlier for the touched surface. - Local merge-wrapper build and check passed on the prepared candidate. - Local full pnpm test reported unrelated failures outside this PR's touched files; touched files are limited to cron docs, src/agents/model-fallback.ts, and src/cron/isolated-agent/*. Co-authored-by: chen-zhang-cs-code <chenzhangcode@163.com> Co-authored-by: Onur Solmaz <2453968+osolmaz@users.noreply.github.com>
diff --git a/docs/automation/cron-jobs.md b/docs/automation/cron-jobs.md
@@ -145,6 +145,8 @@ This fires ~5–6 times per month instead of 0–1 times per month. OpenClaw use
 
 Cron jobs can also carry payload-level `fallbacks`. When present, that list replaces the configured fallback chain for the job. Use `fallbacks: []` in the job payload/API when you want a strict cron run that tries only the selected model. If a job has `--model` but neither payload nor configured fallbacks, OpenClaw passes an explicit empty fallback override so the agent primary is not appended as a hidden extra retry target.
 
+Local-provider preflight checks walk configured fallbacks before marking a cron run `skipped`; `fallbacks: []` keeps that preflight path strict.
+
 Model-selection precedence for isolated jobs is:
 
 1. Gmail hook model override (when the run came from Gmail and that override is allowed)
diff --git a/docs/cli/cron.md b/docs/cli/cron.md
@@ -155,6 +155,7 @@ Cron `--model` is a **job primary**, not a chat-session `/model` override. That
 - Per-job payload `fallbacks` replaces the configured fallback list when present.
 - An empty per-job fallback list (`fallbacks: []` in the job payload/API) makes the cron run strict.
 - When a job has `--model` but no fallback list is configured, OpenClaw passes an explicit empty fallback override so the agent primary is not appended as a hidden retry target.
+- Local-provider preflight checks walk configured fallbacks before marking a cron run `skipped`.
 
 `openclaw doctor` reports jobs that already have `payload.model` set, including provider namespace counts and mismatches against `agents.defaults.model`. Use that check when auth, provider, or billing behavior looks different between live chat and scheduled jobs.
 
diff --git a/src/agents/model-fallback.ts b/src/agents/model-fallback.ts
@@ -690,13 +690,13 @@ export function resolveImageFallbackDefaultProvider(cfg: OpenClawConfig | undefi
 }
 
 export const testing = {
-  resolveFallbackCandidates,
+  resolveFallbackCandidates: resolveModelCandidateChain,
   resolveImageFallbackCandidates,
   resolveCooldownDecision,
   resolveSessionSuspensionReason,
 } as const;
 
-function resolveFallbackCandidates(
+export function resolveModelCandidateChain(
   params: {
     cfg: OpenClawConfig | undefined;
     provider: string;
@@ -1142,7 +1142,7 @@ export async function runWithModelFallback<T>(
     abortSignal?: AbortSignal;
   } & ModelManifestNormalizationContext,
 ): Promise<ModelFallbackRunResult<T>> {
-  const candidates = resolveFallbackCandidates({
+  const candidates = resolveModelCandidateChain({
     cfg: params.cfg,
     provider: params.provider,
     model: params.model,
diff --git a/src/cron/isolated-agent.model-preflight.test.ts b/src/cron/isolated-agent.model-preflight.test.ts
@@ -1,12 +1,15 @@
 import { beforeEach, describe, expect, it } from "vitest";
 import {
   loadRunCronIsolatedAgentTurn,
+  logWarnMock,
   makeCronSession,
+  mockRunCronFallbackPassthrough,
   preflightCronModelProviderMock,
   resolveConfiguredModelRefMock,
   resolveCronSessionMock,
   resetRunCronIsolatedAgentTurnHarness,
   runEmbeddedAgentMock,
+  runWithModelFallbackMock,
 } from "./isolated-agent/run.test-harness.js";
 
 const runCronIsolatedAgentTurn = await loadRunCronIsolatedAgentTurn();
@@ -43,6 +46,14 @@ describe("runCronIsolatedAgentTurn model provider preflight", () => {
 
     const result = await runCronIsolatedAgentTurn({
       cfg: {
+        agents: {
+          defaults: {
+            model: {
+              primary: "ollama/qwen3:32b",
+              fallbacks: [],
+            },
+          },
+        },
         models: {
           providers: {
             ollama: {
@@ -79,4 +90,135 @@ describe("runCronIsolatedAgentTurn model provider preflight", () => {
     expect(result.error).toContain("local provider endpoint is not reachable");
     expect(runEmbeddedAgentMock).not.toHaveBeenCalled();
   });
+
+  it("continues with configured fallback when the local primary preflight is unavailable", async () => {
+    mockRunCronFallbackPassthrough();
+    preflightCronModelProviderMock.mockResolvedValueOnce({
+      status: "unavailable",
+      reason:
+        "Agent cron job uses ollama/qwen3:32b but the local provider endpoint is not reachable at http://127.0.0.1:11434.",
+      provider: "ollama",
+      model: "qwen3:32b",
+      baseUrl: "http://127.0.0.1:11434",
+      retryAfterMs: 300000,
+    });
+
+    const result = await runCronIsolatedAgentTurn({
+      cfg: {
+        agents: {
+          defaults: {
+            model: {
+              primary: "ollama/qwen3:32b",
+              fallbacks: ["openrouter/nvidia/nemotron-3-super-120b-a12b:free", "openai/gpt-5.4"],
+            },
+          },
+        },
+        models: {
+          providers: {
+            ollama: {
+              api: "ollama",
+              baseUrl: "http://127.0.0.1:11434",
+              models: [],
+            },
+            openrouter: {
+              api: "openai-completions",
+              baseUrl: "https://openrouter.ai/api/v1",
+              models: [],
+            },
+          },
+        },
+      },
+      deps: {} as never,
+      job: {
+        id: "fallback-from-dead-ollama",
+        name: "Fallback From Dead Ollama",
+        enabled: true,
+        createdAtMs: 0,
+        updatedAtMs: 0,
+        schedule: { kind: "cron", expr: "*/5 * * * *", tz: "UTC" },
+        sessionTarget: "isolated",
+        state: {},
+        wakeMode: "next-heartbeat",
+        payload: { kind: "agentTurn", message: "summarize" },
+        delivery: { mode: "none" },
+      },
+      message: "summarize",
+      sessionKey: "cron:fallback-from-dead-ollama",
+      lane: "cron",
+    });
+
+    expect(result.status).toBe("ok");
+    expect(result.provider).toBe("openrouter");
+    expect(result.model).toBe("nvidia/nemotron-3-super-120b-a12b:free");
+    expect(preflightCronModelProviderMock.mock.calls.map((call) => call[0])).toMatchObject([
+      { provider: "ollama", model: "qwen3:32b" },
+      { provider: "openrouter", model: "nvidia/nemotron-3-super-120b-a12b:free" },
+    ]);
+    expect(runEmbeddedAgentMock.mock.calls[0]?.[0]).toMatchObject({
+      provider: "openrouter",
+      model: "nvidia/nemotron-3-super-120b-a12b:free",
+    });
+    expect(runWithModelFallbackMock.mock.calls[0]?.[0]).toMatchObject({
+      fallbacksOverride: ["openai/gpt-5.4"],
+    });
+    expect(String(logWarnMock.mock.calls[0]?.[0] ?? "")).toContain(
+      "continuing with fallback openrouter/nvidia/nemotron-3-super-120b-a12b:free",
+    );
+    expect(String(logWarnMock.mock.calls[0]?.[0] ?? "")).not.toContain("Skipping this cron run");
+  });
+
+  it("keeps explicit empty payload fallbacks strict when local primary preflight fails", async () => {
+    preflightCronModelProviderMock.mockResolvedValueOnce({
+      status: "unavailable",
+      reason:
+        "Agent cron job uses ollama/qwen3:32b but the local provider endpoint is not reachable at http://127.0.0.1:11434.",
+      provider: "ollama",
+      model: "qwen3:32b",
+      baseUrl: "http://127.0.0.1:11434",
+      retryAfterMs: 300000,
+    });
+
+    const result = await runCronIsolatedAgentTurn({
+      cfg: {
+        agents: {
+          defaults: {
+            model: {
+              primary: "ollama/qwen3:32b",
+              fallbacks: ["openrouter/nvidia/nemotron-3-super-120b-a12b:free"],
+            },
+          },
+        },
+        models: {
+          providers: {
+            ollama: {
+              api: "ollama",
+              baseUrl: "http://127.0.0.1:11434",
+              models: [],
+            },
+          },
+        },
+      },
+      deps: {} as never,
+      job: {
+        id: "strict-dead-ollama",
+        name: "Strict Dead Ollama",
+        enabled: true,
+        createdAtMs: 0,
+        updatedAtMs: 0,
+        schedule: { kind: "cron", expr: "*/5 * * * *", tz: "UTC" },
+        sessionTarget: "isolated",
+        state: {},
+        wakeMode: "next-heartbeat",
+        payload: { kind: "agentTurn", message: "summarize", fallbacks: [] },
+        delivery: { mode: "none" },
+      },
+      message: "summarize",
+      sessionKey: "cron:strict-dead-ollama",
+      lane: "cron",
+    });
+
+    expect(result.status).toBe("skipped");
+    expect(preflightCronModelProviderMock).toHaveBeenCalledOnce();
+    expect(runEmbeddedAgentMock).not.toHaveBeenCalled();
+  });
 });
diff --git a/src/cron/isolated-agent/run-executor.ts b/src/cron/isolated-agent/run-executor.ts
@@ -139,6 +139,7 @@ export function createCronPromptExecutor(params: {
   skillsSnapshot: SkillSnapshot;
   agentPayload: AgentTurnPayload;
   useSubagentFallbacks: boolean;
+  modelFallbacksOverride?: string[];
   liveSelection: CronLiveSelection;
   cronSession: MutableCronSession;
   abortSignal?: AbortSignal;
@@ -156,12 +157,14 @@ export function createCronPromptExecutor(params: {
   if (!params.cronSession.sessionEntry.sessionFile?.trim()) {
     params.cronSession.sessionEntry.sessionFile = sessionFile;
   }
-  const cronFallbacksOverride = resolveCronFallbacksOverride({
-    cfg: params.cfg,
-    job: params.job,
-    agentId: params.agentId,
-    useSubagentFallbacks: params.useSubagentFallbacks,
-  });
+  const cronFallbacksOverride =
+    params.modelFallbacksOverride ??
+    resolveCronFallbacksOverride({
+      cfg: params.cfg,
+      job: params.job,
+      agentId: params.agentId,
+      useSubagentFallbacks: params.useSubagentFallbacks,
+    });
   let runResult: CronPromptRunResult | undefined;
   let fallbackProvider = params.liveSelection.provider;
   let fallbackModel = params.liveSelection.model;
@@ -366,6 +369,7 @@ export async function executeCronRun(params: {
   skillsSnapshot: SkillSnapshot;
   agentPayload: AgentTurnPayload;
   useSubagentFallbacks: boolean;
+  modelFallbacksOverride?: string[];
   agentVerboseDefault: AgentDefaultsConfig["verboseDefault"];
   liveSelection: CronLiveSelection;
   cronSession: MutableCronSession;
@@ -414,6 +418,7 @@ export async function executeCronRun(params: {
     skillsSnapshot: params.skillsSnapshot,
     agentPayload: params.agentPayload,
     useSubagentFallbacks: params.useSubagentFallbacks,
+    modelFallbacksOverride: params.modelFallbacksOverride,
     liveSelection: params.liveSelection,
     cronSession: params.cronSession,
     abortSignal: params.abortSignal,
diff --git a/src/cron/isolated-agent/run-fallback-policy.test.ts b/src/cron/isolated-agent/run-fallback-policy.test.ts
@@ -1,7 +1,11 @@
+import { readFileSync } from "node:fs";
 import { describe, expect, it } from "vitest";
 import type { OpenClawConfig } from "../../config/types.openclaw.js";
 import type { CronJob } from "../types.js";
-import { resolveCronFallbacksOverride } from "./run-fallback-policy.js";
+import {
+  resolveCronFallbacksOverride,
+  resolveCronPreflightCandidates,
+} from "./run-fallback-policy.js";
 
 function makeJob(payload: CronJob["payload"]): CronJob {
   return {
@@ -257,4 +261,56 @@ describe("resolveCronFallbacksOverride", () => {
       }),
     ).toBeUndefined();
   });
+
+  it("plans the full configured candidate chain for cron preflight", () => {
+    expect(
+      resolveCronPreflightCandidates({
+        cfg: {
+          agents: {
+            defaults: {
+              model: {
+                primary: "ollama/qwen3:32b",
+                fallbacks: ["openrouter/nvidia/nemotron-3-super-120b-a12b:free", "openai/gpt-5.4"],
+              },
+            },
+          },
+        },
+        agentId: "main",
+        provider: "ollama",
+        model: "qwen3:32b",
+        job: makeJob({
+          kind: "agentTurn",
+          message: "summarize",
+        }),
+      }),
+    ).toEqual([
+      { provider: "ollama", model: "qwen3:32b" },
+      { provider: "openrouter", model: "nvidia/nemotron-3-super-120b-a12b:free" },
+      { provider: "openai", model: "gpt-5.4" },
+    ]);
+  });
+
+  it("keeps cron preflight strict when payload fallbacks are explicitly empty", () => {
+    expect(
+      resolveCronPreflightCandidates({
+        cfg: makeConfig(["openai/gpt-5.4"]),
+        agentId: "main",
+        provider: "ollama",
+        model: "qwen3:32b",
+        job: makeJob({
+          kind: "agentTurn",
+          message: "summarize",
+          fallbacks: [],
+        }),
+      }),
+    ).toStrictEqual([{ provider: "ollama", model: "qwen3:32b" }]);
+  });
+
+  it("documents that cron preflight walks fallbacks before skipping", () => {
+    const cliDocs = readFileSync("docs/cli/cron.md", "utf8");
+    const automationDocs = readFileSync("docs/automation/cron-jobs.md", "utf8");
+
+    expect(cliDocs).toContain("Local-provider preflight checks walk configured fallbacks");
+    expect(automationDocs).toContain("Local-provider preflight checks walk configured fallbacks");
+  });
 });
diff --git a/src/cron/isolated-agent/run-fallback-policy.ts b/src/cron/isolated-agent/run-fallback-policy.ts
@@ -1,3 +1,5 @@
+import { resolveModelCandidateChain } from "../../agents/model-fallback.js";
+import type { ModelCandidate } from "../../agents/model-fallback.types.js";
 import type { OpenClawConfig } from "../../config/types.openclaw.js";
 import type { CronJob } from "../types.js";
 import {
@@ -34,3 +36,25 @@ export function resolveCronFallbacksOverride(params: {
     modelOverrideSource: hasCronPayloadModelOverride ? "auto" : undefined,
   });
 }
+
+export function resolveCronPreflightCandidates(params: {
+  cfg: OpenClawConfig;
+  job: CronJob;
+  agentId: string;
+  provider: string;
+  model: string;
+  useSubagentFallbacks?: boolean;
+}): ModelCandidate[] {
+  const fallbacksOverride = resolveCronFallbacksOverride({
+    cfg: params.cfg,
+    job: params.job,
+    agentId: params.agentId,
+    useSubagentFallbacks: params.useSubagentFallbacks,
+  });
+  return resolveModelCandidateChain({
+    cfg: params.cfg,
+    provider: params.provider,
+    model: params.model,
+    fallbacksOverride,
+  });
+}
diff --git a/src/cron/isolated-agent/run.ts b/src/cron/isolated-agent/run.ts
