Skip to content

Commit 36f4af9

Browse files
committed
fix(server): restarts no longer drop prompts the provider never accepted
1 parent 7504744 commit 36f4af9

4 files changed

Lines changed: 255 additions & 54 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4627,14 +4627,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
46274627
WHERE latest.thread_id = ${threadId}
46284628
ORDER BY latest.ordinal DESC LIMIT 1
46294629
)
4630-
-- Retain the latest run for each provider thread with lost
4631-
-- background work, including owners used before a handoff.
4630+
-- Retain the latest ended run for each provider thread with lost
4631+
-- background work, including owners used before a handoff. An
4632+
-- unfinished run is read above and may prove never started.
46324633
OR run.run_id IN (
46334634
SELECT (
46344635
SELECT ended.run_id FROM orchestration_v2_projection_runs AS ended
46354636
WHERE ended.thread_id = ${threadId}
46364637
AND ended.provider_thread_id = roster.provider_thread_id
4637-
AND ended.status NOT IN ('queued', 'rolled_back')
4638+
AND ended.status NOT IN (
4639+
'queued', 'preparing', 'starting', 'running', 'waiting', 'rolled_back'
4640+
)
46384641
-- A run that never started (see neverStarted) owns no work.
46394642
AND NOT (
46404643
json_type(ended.payload_json, '$.startedAt') = 'null'

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 184 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
isProviderNativeSubagentThread,
66
type OrchestrationV2DomainEvent,
77
type ProviderThreadId,
8+
type RunAttemptId,
89
type OrchestrationV2RestartCancelledBackgroundWork,
910
type OrchestrationV2Subagent,
1011
type OrchestrationV2ThreadProjection,
@@ -23,7 +24,11 @@ import * as EventSink from "./EventSink.ts";
2324
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
2425
import * as ProjectionStore from "./ProjectionStore.ts";
2526
import * as ServerSettings from "../serverSettings.ts";
26-
import { restartContinuationRun } from "./RestartContinuation.ts";
27+
import {
28+
restartContinuationBlocked,
29+
restartContinuationRun,
30+
restartStopRequested,
31+
} from "./RestartContinuation.ts";
2732
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
2833
import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
2934
import {
@@ -53,6 +58,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
5358

5459
const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);
5560

61+
const MAX_RESENDS = 3;
62+
5663
export interface ProviderRuntimeRecoverySummary {
5764
readonly terminalizedRuns: number;
5865
readonly stoppedSessions: number;
@@ -205,14 +212,71 @@ export const make = Effect.gen(function* () {
205212
return [...sessions.values()];
206213
};
207214

215+
/**
216+
* A run is marked running before its provider accepts the prompt, so only a provider turn of
217+
* one of its attempts proves the prompt arrived. Recovery treats the others as never started.
218+
*/
219+
const rewindUnacceptedRuns = Effect.fn("ProviderRuntimeRecoveryService.rewindUnacceptedRuns")(
220+
function* (projection: ProjectionStore.ProjectionRuntimeRecoveryState) {
221+
const runIdByAttempt = new Map(
222+
projection.runs.flatMap((run) =>
223+
(run.status === "running" || (run.status === "starting" && run.startedAt !== null)) &&
224+
run.activeAttemptId !== null
225+
? [[run.activeAttemptId, run.id] as const]
226+
: [],
227+
),
228+
);
229+
if (runIdByAttempt.size === 0) return projection;
230+
const unaccepted = new Set(runIdByAttempt.values());
231+
const accept = (turns: ReadonlyArray<{ readonly runAttemptId: RunAttemptId | null }>) => {
232+
for (const turn of turns) {
233+
const runId =
234+
turn.runAttemptId === null ? undefined : runIdByAttempt.get(turn.runAttemptId);
235+
if (runId !== undefined) unaccepted.delete(runId);
236+
}
237+
};
238+
accept(projection.providerTurns);
239+
if (unaccepted.size === 0) return projection;
240+
for (const attempt of projection.attempts) {
241+
if (unaccepted.has(attempt.runId)) runIdByAttempt.set(attempt.id, attempt.runId);
242+
}
243+
// The recovery projection omits ended provider turns, such as one a steer replaced.
244+
accept(
245+
(yield* projections
246+
.getThreadRecords(projection.thread.id, ["providerTurns"], {
247+
providerTurnAttemptIds: [...runIdByAttempt].flatMap(([attemptId, runId]) =>
248+
unaccepted.has(runId) ? [attemptId] : [],
249+
),
250+
})
251+
.pipe(
252+
Effect.mapError(
253+
(cause) =>
254+
new ProviderRuntimeRecoveryError({
255+
operation: "read-projections",
256+
threadId: projection.thread.id,
257+
cause,
258+
}),
259+
),
260+
)).providerTurns,
261+
);
262+
return {
263+
...projection,
264+
runs: projection.runs.map((run) =>
265+
unaccepted.has(run.id) ? { ...run, status: "starting" as const, startedAt: null } : run,
266+
),
267+
};
268+
},
269+
);
270+
208271
const reconcileProjection = Effect.fn("ProviderRuntimeRecoveryService.reconcileProjection")(
209272
function* (
210-
projection: ProjectionStore.ProjectionRuntimeRecoveryState,
273+
stored: ProjectionStore.ProjectionRuntimeRecoveryState,
211274
trigger: "startup" | "shutdown",
212275
continueAfterRestart: boolean,
213276
stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
214277
) {
215278
const now = yield* DateTime.now;
279+
const projection = yield* rewindUnacceptedRuns(stored);
216280
let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
217281
for (const run of nonterminalRuns(projection)) {
218282
if (run.status === "waiting") {
@@ -379,7 +443,105 @@ export const make = Effect.gen(function* () {
379443
const retryableRuns = unstartedRuns.filter((run) =>
380444
retryableMessageIds.has(run.userMessageId),
381445
);
382-
const retryableRunIds = new Set(retryableRuns.map((run) => run.id));
446+
// Nobody reviews a delegated, agent-sent or scheduled prompt, so it is sent again instead.
447+
const unattendedRuns = continueAfterRestart
448+
? unstartedRuns.filter((run) =>
449+
queuedMessages.some(
450+
(message) =>
451+
message.id === run.userMessageId &&
452+
!retryableMessageIds.has(message.id) &&
453+
message.notification === undefined &&
454+
message.delegatedCompletion === undefined &&
455+
(message.scheduledTaskId !== undefined || message.creationSource === "mcp"),
456+
),
457+
)
458+
: [];
459+
const resentRuns: Array<OrchestrationV2ThreadProjection["runs"][number]> = [];
460+
const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
461+
for (const run of unattendedRuns) {
462+
if (
463+
restartContinuationBlocked(projection, run, now, false) ||
464+
(yield* restartStopRequested(projections.getThreadRecords, projection.thread, run).pipe(
465+
Effect.mapError(
466+
(cause) =>
467+
new ProviderRuntimeRecoveryError({
468+
operation: "read-projections",
469+
threadId: projection.thread.id,
470+
cause,
471+
}),
472+
),
473+
))
474+
)
475+
continue;
476+
const resendCommandId = CommandId.make(`command:restart-resend:${run.id}`);
477+
const earlierResends = yield* outbox.listByCommandId(resendCommandId).pipe(
478+
Effect.mapError(
479+
(cause) =>
480+
new ProviderRuntimeRecoveryError({
481+
operation: "reconcile",
482+
threadId: projection.thread.id,
483+
cause,
484+
}),
485+
),
486+
);
487+
// A prompt whose start keeps crashing the server would otherwise be sent forever.
488+
if (earlierResends.filter((effect) => effect.attemptCount > 0).length >= MAX_RESENDS) {
489+
yield* Effect.logWarning("Not resending a prompt that restarts cut too many times", {
490+
threadId: projection.thread.id,
491+
runId: run.id,
492+
});
493+
continue;
494+
}
495+
resentRuns.push(run);
496+
resends.push({
497+
id: `effect:restart-resend:${run.id}:${commandId}`,
498+
commandId: resendCommandId,
499+
threadId: projection.thread.id,
500+
request: { type: "provider-turn.start", runId: run.id },
501+
});
502+
}
503+
for (const run of resentRuns) {
504+
events.push({
505+
id: yield* allocateEventId(),
506+
type: "run.updated",
507+
threadId: projection.thread.id,
508+
runId: run.id,
509+
providerInstanceId: run.providerInstanceId,
510+
occurredAt: now,
511+
payload: run,
512+
});
513+
}
514+
const keptRuns = [...retryableRuns, ...resentRuns];
515+
for (const run of keptRuns) {
516+
for (const attempt of projection.attempts.filter(
517+
(candidate) => candidate.id === run.activeAttemptId && candidate.status === "running",
518+
)) {
519+
events.push({
520+
id: yield* allocateEventId(),
521+
type: "run-attempt.updated",
522+
threadId: projection.thread.id,
523+
runId: run.id,
524+
nodeId: attempt.rootNodeId,
525+
providerInstanceId: run.providerInstanceId,
526+
occurredAt: now,
527+
payload: { ...attempt, status: "pending", startedAt: null },
528+
});
529+
}
530+
for (const node of projection.nodes.filter(
531+
(candidate) => candidate.id === run.rootNodeId && candidate.status === "running",
532+
)) {
533+
events.push({
534+
id: yield* allocateEventId(),
535+
type: "node.updated",
536+
threadId: projection.thread.id,
537+
runId: run.id,
538+
nodeId: node.id,
539+
providerInstanceId: run.providerInstanceId,
540+
occurredAt: now,
541+
payload: { ...node, status: "pending", startedAt: null },
542+
});
543+
}
544+
}
383545
const retryableItems =
384546
retryableRuns.length === 0
385547
? []
@@ -416,7 +578,7 @@ export const make = Effect.gen(function* () {
416578
},
417579
});
418580
}
419-
runs = runs.filter((run) => !retryableRunIds.has(run.id));
581+
runs = runs.filter((run) => !keptRuns.some((kept) => kept.id === run.id));
420582
const heldRuns = [
421583
...retryableRuns,
422584
...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
@@ -875,6 +1037,7 @@ export const make = Effect.gen(function* () {
8751037
closedRequests: requests.length,
8761038
retiredEffects,
8771039
cancelledEffectIds,
1040+
resends,
8781041
};
8791042
},
8801043
);
@@ -894,6 +1057,7 @@ export const make = Effect.gen(function* () {
8941057
let stoppedSessions = 0;
8951058
let closedRequests = 0;
8961059
let retiredEffects = 0;
1060+
const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
8971061
const stoppedThisPass = new Map<string, RecoverySession>();
8981062
const grouped = trigger === "shutdown" && threadIds.length > 1;
8991063
const groupSize = grouped ? 32 : 1;
@@ -923,6 +1087,7 @@ export const make = Effect.gen(function* () {
9231087
}
9241088
}
9251089
cancelledEffectIds.push(...result.cancelledEffectIds);
1090+
resends.push(...result.resends);
9261091
terminalizedRuns += result.terminalizedRuns;
9271092
stoppedSessions += result.stoppedSessions;
9281093
closedRequests += result.closedRequests;
@@ -944,6 +1109,17 @@ export const make = Effect.gen(function* () {
9441109
(cause) => new ProviderRuntimeRecoveryError({ operation: "drain-outbox", cause }),
9451110
),
9461111
);
1112+
// After the drain, which cancels every unsettled provider start. A crash before this write
1113+
// leaves the run unstarted, so the next recovery sends it again.
1114+
if (resends.length > 0) {
1115+
yield* eventSink
1116+
.writeWithEffects({ events: [], effects: resends })
1117+
.pipe(
1118+
Effect.mapError(
1119+
(cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
1120+
),
1121+
);
1122+
}
9471123
return {
9481124
terminalizedRuns,
9491125
stoppedSessions,
@@ -979,7 +1155,10 @@ export const make = Effect.gen(function* () {
9791155
.continueThreadsAfterServerUpdate
9801156
)
9811157
return;
982-
const run = restartContinuationRun(projection, yield* DateTime.now);
1158+
const run = restartContinuationRun(
1159+
yield* rewindUnacceptedRuns(projection),
1160+
yield* DateTime.now,
1161+
);
9831162
if (!run) return;
9841163
const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
9851164
yield* eventSink.writeWithEffects({

0 commit comments

Comments
 (0)