Skip to content

Commit ea14857

Browse files
committed
fix: let isolated cron setup use job timeout budget
1 parent 9512294 commit ea14857

3 files changed

Lines changed: 31 additions & 7 deletions

File tree

src/cron/service/agent-watchdog.test.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,23 @@ describe("cron agent setup watchdog", () => {
66
vi.useRealTimers();
77
});
88

9+
it("allows setup to continue past 60 seconds when the job timeout budget allows it", async () => {
10+
vi.useFakeTimers();
11+
const triggerTimeout = vi.fn();
12+
const watchdog = createCronAgentWatchdog({
13+
deferUntilRunner: true,
14+
jobTimeoutMs: CRON_AGENT_SETUP_WATCHDOG_MS * 5,
15+
triggerTimeout,
16+
});
17+
18+
watchdog.start();
19+
20+
await vi.advanceTimersByTimeAsync(CRON_AGENT_SETUP_WATCHDOG_MS + 10_000);
21+
watchdog.noteRunnerStarted();
22+
23+
expect(triggerTimeout).not.toHaveBeenCalled();
24+
});
25+
926
it("does not keep lane-wait suppression after lane admission", async () => {
1027
vi.useFakeTimers();
1128
const triggerTimeout = vi.fn();

src/cron/service/agent-watchdog.ts

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import type { CronServiceState } from "./state.js";
1414

1515
const CRON_TIMEOUT_CLEANUP_GUARD_MS = 20_000;
1616
export const CRON_AGENT_SETUP_WATCHDOG_MS = 60_000;
17+
const CRON_AGENT_SETUP_WATCHDOG_MAX_MS = 10 * 60_000;
1718
const CRON_AGENT_PRE_EXECUTION_WATCHDOG_MS = 60_000;
1819
const CRON_AGENT_PRE_EXECUTION_MIN_WATCHDOG_MS = 1_000;
1920

@@ -48,6 +49,7 @@ const CRON_AGENT_PHASE_WATCHDOG_STAGE = {
4849
/** Handle for feeding isolated-agent progress into cron timeout watchdogs. */
4950
export type CronAgentWatchdog = {
5051
start: () => void;
52+
setupTimeoutMs: () => number;
5153
noteLaneWait: () => void;
5254
noteLaneAdmitted: () => void;
5355
noteRunnerStarted: (info?: CronAgentExecutionStarted) => void;
@@ -69,6 +71,7 @@ export function createCronAgentWatchdog(params: {
6971
let preExecutionTimeoutId: NodeJS.Timeout | undefined;
7072
let activeExecution: CronAgentExecutionStarted | undefined;
7173
let observedLaneWait = false;
74+
const setupTimeoutMs = resolveCronAgentSetupWatchdogMs(params.jobTimeoutMs);
7275

7376
const setTimedOut = (reason: string) => {
7477
if (state === "timed_out" || state === "disposed") {
@@ -142,11 +145,12 @@ export function createCronAgentWatchdog(params: {
142145
if (state === "waiting_for_runner") {
143146
setTimedOut(setupTimeoutErrorMessage(activeExecution));
144147
}
145-
}, CRON_AGENT_SETUP_WATCHDOG_MS);
148+
}, setupTimeoutMs);
146149
return;
147150
}
148151
startTimeout();
149152
},
153+
setupTimeoutMs: () => setupTimeoutMs,
150154
noteLaneWait: () => {
151155
if (state === "waiting_for_runner") {
152156
observedLaneWait = true;
@@ -223,3 +227,10 @@ function resolveCronAgentPreExecutionWatchdogMs(jobTimeoutMs: number): number {
223227
Math.min(CRON_AGENT_PRE_EXECUTION_WATCHDOG_MS, Math.floor(jobTimeoutMs / 2)),
224228
);
225229
}
230+
231+
export function resolveCronAgentSetupWatchdogMs(jobTimeoutMs: number): number {
232+
return Math.max(
233+
CRON_AGENT_SETUP_WATCHDOG_MS,
234+
Math.min(CRON_AGENT_SETUP_WATCHDOG_MAX_MS, Math.floor(jobTimeoutMs / 2)),
235+
);
236+
}

src/cron/service/timer.ts

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,7 @@ import type {
4646
CronRunStatus,
4747
CronRunTelemetry,
4848
} from "../types.js";
49-
import {
50-
cleanupTimedOutCronAgentRun,
51-
createCronAgentWatchdog,
52-
CRON_AGENT_SETUP_WATCHDOG_MS,
53-
} from "./agent-watchdog.js";
49+
import { cleanupTimedOutCronAgentRun, createCronAgentWatchdog } from "./agent-watchdog.js";
5450
import {
5551
abortErrorMessage,
5652
isSetupTimeoutErrorText,
@@ -278,7 +274,7 @@ export async function executeJobCoreWithTimeout(
278274
job.sessionTarget === "isolated" && isSetupTimeoutErrorText(error) && !observedLaneWait
279275
? {
280276
error,
281-
timeoutMs: CRON_AGENT_SETUP_WATCHDOG_MS,
277+
timeoutMs: watchdog.setupTimeoutMs(),
282278
otherCronJobsActiveAtTimeout: false,
283279
}
284280
: undefined;

0 commit comments

Comments
 (0)