Skip to content

Commit e864cf5

Browse files
committed
fix(cron): allow cold isolated runner setup
1 parent 4dd3ba1 commit e864cf5

2 files changed

Lines changed: 76 additions & 2 deletions

File tree

src/cron/service/timer.regression.test.ts

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1573,6 +1573,72 @@ describe("cron service timer regressions", () => {
15731573
}
15741574
});
15751575

1576+
it("allows long isolated cron jobs more setup time for cold runner startup", async () => {
1577+
vi.useFakeTimers();
1578+
try {
1579+
const store = timerRegressionFixtures.makeStorePath();
1580+
const scheduledAt = Date.parse("2026-05-10T09:02:00.000Z");
1581+
const cronJob = createIsolatedRegressionJob({
1582+
id: "isolated-cold-runner-setup",
1583+
name: "cold runner setup",
1584+
scheduledAt,
1585+
schedule: { kind: "at", at: new Date(scheduledAt).toISOString() },
1586+
payload: { kind: "agentTurn", message: "work", timeoutSeconds: 1080 },
1587+
state: { nextRunAtMs: scheduledAt },
1588+
});
1589+
await writeCronJobs(store.storePath, [cronJob]);
1590+
1591+
vi.setSystemTime(scheduledAt);
1592+
let now = scheduledAt;
1593+
const started = createDeferred<void>();
1594+
let settled = false;
1595+
let abortObserved = false;
1596+
const cleanupTimedOutAgentRun = vi.fn(async () => {});
1597+
const state = createCronServiceState({
1598+
cronEnabled: true,
1599+
storePath: store.storePath,
1600+
log: noopLogger,
1601+
nowMs: () => now,
1602+
enqueueSystemEvent: vi.fn(),
1603+
requestHeartbeat: vi.fn(),
1604+
cleanupTimedOutAgentRun,
1605+
runIsolatedAgentJob: vi.fn(async ({ abortSignal }: { abortSignal?: AbortSignal }) => {
1606+
started.resolve();
1607+
abortSignal?.addEventListener(
1608+
"abort",
1609+
() => {
1610+
abortObserved = true;
1611+
},
1612+
{ once: true },
1613+
);
1614+
return await new Promise<never>(() => {});
1615+
}),
1616+
});
1617+
1618+
const timerPromise = onTimer(state).finally(() => {
1619+
settled = true;
1620+
});
1621+
await started.promise;
1622+
await vi.advanceTimersByTimeAsync(60_100);
1623+
now += 60_100;
1624+
expect(settled).toBe(false);
1625+
expect(abortObserved).toBe(false);
1626+
expect(cleanupTimedOutAgentRun).not.toHaveBeenCalled();
1627+
1628+
await vi.advanceTimersByTimeAsync(240_000);
1629+
now += 240_000;
1630+
await timerPromise;
1631+
1632+
const job = requireJob(state, "isolated-cold-runner-setup");
1633+
expect(abortObserved).toBe(true);
1634+
expect(job.state.lastStatus).toBe("error");
1635+
expect(job.state.lastError).toContain("setup timed out before runner start");
1636+
expect(cleanupTimedOutAgentRun).toHaveBeenCalledTimes(1);
1637+
} finally {
1638+
vi.useRealTimers();
1639+
}
1640+
});
1641+
15761642
it("times out isolated agent runs that stall before execution starts (#74803)", async () => {
15771643
vi.useFakeTimers();
15781644
try {

src/cron/service/timer.ts

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ export { DEFAULT_JOB_TIMEOUT_MS } from "./timeout-policy.js";
6969

7070
const MAX_TIMER_DELAY_MS = 60_000;
7171
const CRON_TIMEOUT_CLEANUP_GUARD_MS = 20_000;
72-
const CRON_AGENT_SETUP_WATCHDOG_MS = 60_000;
72+
const CRON_AGENT_SETUP_MIN_WATCHDOG_MS = 60_000;
73+
const CRON_AGENT_SETUP_MAX_WATCHDOG_MS = 5 * 60_000;
7374
const CRON_AGENT_PRE_EXECUTION_WATCHDOG_MS = 60_000;
7475
const CRON_AGENT_PRE_EXECUTION_MIN_WATCHDOG_MS = 1_000;
7576

@@ -302,7 +303,7 @@ function createCronAgentWatchdog(params: {
302303
if (state === "waiting_for_runner") {
303304
setTimedOut(setupTimeoutErrorMessage(activeExecution));
304305
}
305-
}, CRON_AGENT_SETUP_WATCHDOG_MS);
306+
}, resolveCronAgentSetupWatchdogMs(params.jobTimeoutMs));
306307
return;
307308
}
308309
startTimeout();
@@ -400,6 +401,13 @@ function formatCronAgentExecutionPhase(execution?: CronAgentExecutionStarted): s
400401
return formatEmbeddedAgentExecutionPhase(execution?.phase);
401402
}
402403

404+
function resolveCronAgentSetupWatchdogMs(jobTimeoutMs: number): number {
405+
return Math.max(
406+
CRON_AGENT_SETUP_MIN_WATCHDOG_MS,
407+
Math.min(CRON_AGENT_SETUP_MAX_WATCHDOG_MS, Math.floor(jobTimeoutMs / 2)),
408+
);
409+
}
410+
403411
function resolveCronAgentPreExecutionWatchdogMs(jobTimeoutMs: number): number {
404412
return Math.max(
405413
CRON_AGENT_PRE_EXECUTION_MIN_WATCHDOG_MS,

0 commit comments

Comments
 (0)