Skip to content

Commit 57d9452

Browse files
author
Cadbury
committed
fix(cron): treat isolated setup phases as progress
1 parent ee2d4e1 commit 57d9452

2 files changed

Lines changed: 95 additions & 39 deletions

File tree

src/cron/service/agent-watchdog.ts

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -113,23 +113,9 @@ export function createCronAgentWatchdog(params: {
113113
if (!info) {
114114
return;
115115
}
116-
const previousPhase = activeExecution?.phase;
117116
activeExecution = { ...activeExecution, ...info };
118117
const stage = info.phase ? CRON_AGENT_PHASE_WATCHDOG_STAGE[info.phase] : undefined;
119-
// A fallback attempt can return to setup-like phases after execution began;
120-
// re-arm pre-execution timing so the fallback path cannot stall silently.
121-
if (
122-
state === "executing" &&
123-
previousPhase === "before_agent_reply" &&
124-
stage === "pre_execution"
125-
) {
126-
// Model fallback can move from an execution phase back into setup-like
127-
// phases; restart the pre-execution watchdog so fallback stalls are seen.
128-
state = "waiting_for_execution";
129-
startPreExecutionTimeout();
130-
return;
131-
}
132-
if (stage === "execution" || info.firstModelCallStarted) {
118+
if (stage !== undefined || info.firstModelCallStarted) {
133119
state = "executing";
134120
clearPreExecutionTimeout();
135121
}

src/cron/service/timer.regression.test.ts

Lines changed: 94 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2860,7 +2860,7 @@ describe("cron service timer regressions", () => {
28602860
}
28612861
});
28622862

2863-
it("times out isolated agent runs that stall before execution starts (#74803)", async () => {
2863+
it("times out isolated agent runs when the runner emits no phase progress (#74803)", async () => {
28642864
vi.useFakeTimers();
28652865
try {
28662866
const store = timerRegressionFixtures.makeStorePath();
@@ -2895,26 +2895,11 @@ describe("cron service timer regressions", () => {
28952895
async ({
28962896
abortSignal,
28972897
onExecutionStarted,
2898-
onExecutionPhase,
28992898
}: {
29002899
abortSignal?: AbortSignal;
29012900
onExecutionStarted?: (info?: CronAgentExecutionStarted) => void;
2902-
onExecutionPhase?: (info: CronAgentExecutionPhaseUpdate) => void;
29032901
}) => {
2904-
onExecutionStarted?.({
2905-
jobId: "isolated-pre-model-timeout-74803",
2906-
agentId: "main",
2907-
sessionId: "cron-run-session",
2908-
sessionKey: "agent:main:cron:isolated-pre-model-timeout-74803:run:cron-run-session",
2909-
phase: "runner_entered",
2910-
});
2911-
onExecutionPhase?.({
2912-
jobId: "isolated-pre-model-timeout-74803",
2913-
agentId: "main",
2914-
sessionId: "cron-run-session",
2915-
sessionKey: "agent:main:cron:isolated-pre-model-timeout-74803:run:cron-run-session",
2916-
phase: "context_engine",
2917-
});
2902+
onExecutionStarted?.();
29182903
started.resolve();
29192904
abortSignal?.addEventListener(
29202905
"abort",
@@ -2939,24 +2924,104 @@ describe("cron service timer regressions", () => {
29392924
expect(abortObserved).toBe(true);
29402925
expect(job.state.lastStatus).toBe("error");
29412926
expect(job.state.lastError).toContain("stalled before execution start");
2942-
expect(job.state.lastError).toContain("context-engine");
29432927
expect(abortReason).toMatchObject({
29442928
name: "TimeoutError",
2945-
message: expect.stringContaining("context-engine"),
2929+
message: expect.stringContaining("stalled before execution start"),
29462930
});
29472931
expect(cleanupTimedOutAgentRun).toHaveBeenCalledTimes(1);
29482932
const cleanupArgs = requireRecord(firstMockArg(cleanupTimedOutAgentRun));
29492933
expect(requireRecord(cleanupArgs.job).id).toBe("isolated-pre-model-timeout-74803");
29502934
expect(cleanupArgs.timeoutMs).toBe(1_200_000);
2951-
const execution = requireRecord(cleanupArgs.execution);
2952-
expect(execution.jobId).toBe("isolated-pre-model-timeout-74803");
2953-
expect(execution.phase).toBe("context_engine");
2935+
expect(cleanupArgs.execution).toBeUndefined();
29542936
expect(onIsolatedAgentSetupTimeout).not.toHaveBeenCalled();
29552937
} finally {
29562938
vi.useRealTimers();
29572939
}
29582940
});
29592941

2942+
it("keeps isolated runs alive when setup phases keep progressing (#93530)", async () => {
2943+
vi.useFakeTimers();
2944+
try {
2945+
const store = timerRegressionFixtures.makeStorePath();
2946+
const scheduledAt = Date.parse("2026-05-10T09:07:00.000Z");
2947+
const cronJob = createIsolatedRegressionJob({
2948+
id: "isolated-setup-progress-93530",
2949+
name: "setup progress regression",
2950+
scheduledAt,
2951+
schedule: { kind: "at", at: new Date(scheduledAt).toISOString() },
2952+
payload: { kind: "agentTurn", message: "work", timeoutSeconds: 1_200 },
2953+
state: { nextRunAtMs: scheduledAt },
2954+
});
2955+
await saveCronStore(store.storePath, { version: 1, jobs: [cronJob] });
2956+
2957+
vi.setSystemTime(scheduledAt);
2958+
let now = scheduledAt;
2959+
const started = createDeferred<void>();
2960+
let abortObserved = false;
2961+
const cleanupTimedOutAgentRun = vi.fn(async () => {});
2962+
const state = createCronServiceState({
2963+
cronEnabled: true,
2964+
storePath: store.storePath,
2965+
log: noopLogger,
2966+
nowMs: () => now,
2967+
enqueueSystemEvent: vi.fn(),
2968+
requestHeartbeat: vi.fn(),
2969+
cleanupTimedOutAgentRun,
2970+
runIsolatedAgentJob: vi.fn(
2971+
async ({
2972+
abortSignal,
2973+
onExecutionStarted,
2974+
onExecutionPhase,
2975+
}: {
2976+
abortSignal?: AbortSignal;
2977+
onExecutionStarted?: (info?: CronAgentExecutionStarted) => void;
2978+
onExecutionPhase?: (info: CronAgentExecutionPhaseUpdate) => void;
2979+
}) => {
2980+
onExecutionStarted?.();
2981+
for (const phase of ["model_resolution", "auth", "context_engine"] as const) {
2982+
onExecutionPhase?.({
2983+
jobId: "isolated-setup-progress-93530",
2984+
agentId: "main",
2985+
sessionId: "cron-run-session",
2986+
sessionKey: "agent:main:cron:isolated-setup-progress-93530:run:cron-run-session",
2987+
phase,
2988+
});
2989+
}
2990+
started.resolve();
2991+
abortSignal?.addEventListener(
2992+
"abort",
2993+
() => {
2994+
abortObserved = true;
2995+
},
2996+
{ once: true },
2997+
);
2998+
return await new Promise<never>(() => {});
2999+
},
3000+
),
3001+
});
3002+
3003+
const timerPromise = onTimer(state);
3004+
await started.promise;
3005+
await vi.advanceTimersByTimeAsync(60_100);
3006+
now += 60_100;
3007+
expect(abortObserved).toBe(false);
3008+
expect(cleanupTimedOutAgentRun).not.toHaveBeenCalled();
3009+
3010+
await vi.advanceTimersByTimeAsync(1_139_900);
3011+
now += 1_139_900;
3012+
await timerPromise;
3013+
3014+
const job = requireJob(state, "isolated-setup-progress-93530");
3015+
expect(abortObserved).toBe(true);
3016+
expect(job.state.lastStatus).toBe("error");
3017+
expect(job.state.lastError).toContain("job execution timed out");
3018+
expect(job.state.lastError).toContain("context-engine");
3019+
expect(cleanupTimedOutAgentRun).toHaveBeenCalledTimes(1);
3020+
} finally {
3021+
vi.useRealTimers();
3022+
}
3023+
});
3024+
29603025
it("clears the pre-execution watchdog on explicit execution milestones (#80283)", async () => {
29613026
vi.useFakeTimers();
29623027
try {
@@ -3148,7 +3213,7 @@ describe("cron service timer regressions", () => {
31483213
},
31493214
);
31503215

3151-
it("re-arms the pre-execution watchdog when before_agent_reply does not claim (#82811)", async () => {
3216+
it("does not re-arm the pre-execution watchdog after fallback setup progress (#93530)", async () => {
31523217
vi.useFakeTimers();
31533218
try {
31543219
const store = timerRegressionFixtures.makeStorePath();
@@ -3215,12 +3280,17 @@ describe("cron service timer regressions", () => {
32153280
await started.promise;
32163281
await vi.advanceTimersByTimeAsync(60_100);
32173282
now += 60_100;
3283+
expect(abortObserved).toBe(false);
3284+
expect(cleanupTimedOutAgentRun).not.toHaveBeenCalled();
3285+
3286+
await vi.advanceTimersByTimeAsync(1_139_900);
3287+
now += 1_139_900;
32183288
await timerPromise;
32193289

32203290
const job = requireJob(state, "isolated-before-agent-reply-unhandled-82811");
32213291
expect(abortObserved).toBe(true);
32223292
expect(job.state.lastStatus).toBe("error");
3223-
expect(job.state.lastError).toContain("stalled before execution start");
3293+
expect(job.state.lastError).toContain("job execution timed out");
32243294
expect(job.state.lastError).toContain("runtime-plugins");
32253295
expect(cleanupTimedOutAgentRun).toHaveBeenCalledTimes(1);
32263296
} finally {

0 commit comments

Comments
 (0)