Skip to content

Commit b807d83

Browse files
committed
fix(server): attribute queued-start failures and expire abandoned drains
Pass the selected queued run id into start failure handling, and release a desktop prepare drain if commit never arrives within the watchdog window.
1 parent 25daf50 commit b807d83

3 files changed

Lines changed: 78 additions & 41 deletions

File tree

‎apps/server/src/cloud/selfUpdate.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,28 @@ it.layer(NodeServices.layer)("server self update", (it) => {
326326
}),
327327
);
328328

329+
it.effect("releases an abandoned desktop prepare drain after the watchdog timeout", () =>
330+
Effect.gen(function* () {
331+
const { selfUpdate, completedShutdownIds } = yield* makeHarness({
332+
mode: "desktop",
333+
desktopAppUpdate: {
334+
available: true,
335+
run: () =>
336+
Effect.succeed({
337+
targetVersion: "1.2.0",
338+
method: "desktop-app" as const,
339+
desktopUpdateToken: "desktop-token",
340+
}),
341+
commit: () => Effect.never,
342+
},
343+
});
344+
yield* selfUpdate.update({ targetVersion: "1.1.0" });
345+
expect(yield* Ref.get(completedShutdownIds)).toEqual([]);
346+
yield* TestClock.adjust("20 minutes");
347+
expect(yield* Ref.get(completedShutdownIds)).toEqual([TEST_DESKTOP_SHUTDOWN_ID]);
348+
}),
349+
);
350+
329351
it.effect("allows only one update at a time", () =>
330352
Effect.gen(function* () {
331353
const requested = yield* Deferred.make<void>();

‎apps/server/src/cloud/selfUpdate.ts‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ import { isExactServiceVersion, SERVICE_LAUNCHER_PROTOCOL } from "./serviceProto
5151

5252
// A staged runtime's cold start can take over a minute on a loaded, swapping host.
5353
const PREFLIGHT_TIMEOUT = Duration.minutes(3);
54+
/** Prepared desktop installs keep the drain until commit; abandon releases it. */
55+
const DESKTOP_ABANDONED_PREPARE_DRAIN_TIMEOUT = Duration.minutes(20);
5456

5557
export function resolveServerSelfUpdateCapability(input: {
5658
readonly desktopManaged: boolean;
@@ -216,9 +218,27 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () {
216218
: { channel: input.channel, targetVersion: input.targetVersion },
217219
)
218220
.pipe(
219-
Effect.tap((result) =>
220-
result.desktopUpdateToken === undefined ? releaseOwnedDesktopDrain() : Effect.void,
221-
),
221+
Effect.tap((result) => {
222+
if (result.desktopUpdateToken === undefined) {
223+
return releaseOwnedDesktopDrain();
224+
}
225+
// Commit is a separate RPC; if it never arrives, release this owned drain only.
226+
const shutdownId = drain.shutdownId;
227+
return Effect.sleep(DESKTOP_ABANDONED_PREPARE_DRAIN_TIMEOUT).pipe(
228+
Effect.andThen(
229+
Effect.gen(function* () {
230+
if ((yield* Ref.get(ownedDesktopDrainId)) !== shutdownId) return;
231+
yield* Effect.logWarning(
232+
"Releasing desktop update drain after abandoned prepare.",
233+
{ shutdownId },
234+
);
235+
yield* releaseOwnedDesktopDrain();
236+
}),
237+
),
238+
Effect.forkDetach,
239+
Effect.asVoid,
240+
);
241+
}),
222242
Effect.onError(() => releaseOwnedDesktopDrain()),
223243
);
224244
}

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 33 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,40 +1744,39 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17441744
]);
17451745
});
17461746

1747-
const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")(
1748-
function* (
1749-
threadId: ThreadId,
1750-
options?: {
1751-
readonly failedRunId?: RunId;
1752-
readonly endedRunId?: RunId;
1753-
readonly queuedRunId?: RunId;
1754-
},
1747+
const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")(function* (
1748+
threadId: ThreadId,
1749+
options?: {
1750+
readonly failedRunId?: RunId;
1751+
readonly endedRunId?: RunId;
1752+
readonly queuedRunId?: RunId;
1753+
},
1754+
) {
1755+
if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
1756+
const projection = yield* readCommandProjection(threadId);
1757+
if (
1758+
projection.thread.archivedAt !== null ||
1759+
projection.thread.deletedAt !== null ||
1760+
projection.runs.some(isActiveRun)
17551761
) {
1756-
if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
1757-
const projection = yield* readCommandProjection(threadId);
1758-
if (
1759-
projection.thread.archivedAt !== null ||
1760-
projection.thread.deletedAt !== null ||
1761-
projection.runs.some(isActiveRun)
1762-
) {
1763-
return;
1764-
}
1762+
return;
1763+
}
17651764

1766-
const selectedQueuedRun =
1767-
options?.queuedRunId === undefined
1768-
? undefined
1769-
: projection.runs.find(
1770-
(run) => run.id === options.queuedRunId && run.status === "queued",
1771-
);
1772-
const queuedCandidates =
1773-
selectedQueuedRun === undefined
1774-
? queuedRunsInDeliveryOrder(projection).filter((run) => run.queueHeld !== true)
1775-
: [selectedQueuedRun];
1776-
const selected = yield* selectStartableQueuedRun(threadId, projection, queuedCandidates);
1777-
if (selected === undefined) {
1778-
return;
1779-
}
1780-
const { storedQueuedRun, modelSelection } = selected;
1765+
const selectedQueuedRun =
1766+
options?.queuedRunId === undefined
1767+
? undefined
1768+
: projection.runs.find((run) => run.id === options.queuedRunId && run.status === "queued");
1769+
const queuedCandidates =
1770+
selectedQueuedRun === undefined
1771+
? queuedRunsInDeliveryOrder(projection).filter((run) => run.queueHeld !== true)
1772+
: [selectedQueuedRun];
1773+
const selected = yield* selectStartableQueuedRun(threadId, projection, queuedCandidates);
1774+
if (selected === undefined) {
1775+
return;
1776+
}
1777+
const { storedQueuedRun, modelSelection } = selected;
1778+
// Capture the selected id for failure attribution; the gate may clear before catch runs.
1779+
yield* Effect.gen(function* () {
17811780
const queuedMessage = projection.messages.find(
17821781
(candidate) => candidate.id === storedQueuedRun.userMessageId,
17831782
);
@@ -2530,12 +2529,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
25302529
},
25312530
],
25322531
);
2533-
},
2534-
(effect, threadId, options?) =>
2535-
effect.pipe(
2536-
Effect.catch((cause) => failQueuedRunStart(threadId, cause, options?.queuedRunId)),
2537-
),
2538-
);
2532+
}).pipe(Effect.catch((cause) => failQueuedRunStart(threadId, cause, storedQueuedRun.id)));
2533+
});
25392534

25402535
// Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
25412536
const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (

0 commit comments

Comments
 (0)