Skip to content

Commit b906ce2

Browse files
authored
fix(server): recover opted-in threads after machine restarts (pingdotgg#9803)
1 parent 5a2f3eb commit b906ce2

11 files changed

Lines changed: 708 additions & 183 deletions

File tree

‎apps/desktop/src/settings/DesktopClientSettings.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ const clientSettings: ClientSettings = {
2626
confirmThreadArchive: true,
2727
confirmThreadDelete: false,
2828
confirmThreadUnpin: false,
29-
continueThreadsAfterServerUpdate: true,
3029
contextWindowMeterEnabled: false,
3130
composerCollapseOnBlur: false,
3231
composerCollapseOnScroll: true,

‎apps/server/src/provider/Layers/ProviderService.test.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,125 @@ function makeProviderServiceLayer(
452452
};
453453
}
454454

455+
for (const [enabled, completed] of [
456+
[false, false],
457+
[true, false],
458+
[true, true],
459+
] as const) {
460+
it.effect(
461+
`persists shutdown recovery before stopping providers when enabled=${enabled}, completed=${completed}`,
462+
() =>
463+
Effect.gen(function* () {
464+
const codex = makeFakeCodexAdapter();
465+
const persistence = yield* Layer.build(
466+
ProviderSessionDirectoryLive.pipe(
467+
Layer.provide(
468+
ProviderSessionRuntime.layer.pipe(Layer.provide(SqlitePersistenceMemory)),
469+
),
470+
),
471+
);
472+
const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory.pipe(
473+
Effect.provide(persistence),
474+
);
475+
const threadId = asThreadId("shutdown-recovery");
476+
const turnId = asTurnId("shutdown-recovery-turn");
477+
const scope = yield* Scope.make();
478+
const services = yield* Layer.build(
479+
makeProviderServiceLive().pipe(
480+
Layer.provide(
481+
Layer.succeed(ProviderSessionDirectory.ProviderSessionDirectory, directory),
482+
),
483+
Layer.provide(
484+
Layer.succeed(
485+
ProviderAdapterRegistry.ProviderAdapterRegistry,
486+
makeStaticInstanceRegistry([[codexInstanceId, codex.adapter]]),
487+
),
488+
),
489+
Layer.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: enabled })),
490+
Layer.provide(serverConfigTestLayer),
491+
Layer.provide(AnalyticsService.layerTest),
492+
Layer.provide(
493+
Layer.succeed(
494+
ProviderEventLoggers.ProviderEventLoggers,
495+
ProviderEventLoggers.NoOpProviderEventLoggers,
496+
),
497+
),
498+
),
499+
).pipe(Scope.provide(scope));
500+
const provider = yield* ProviderService.ProviderService.pipe(Effect.provide(services));
501+
const session = yield* provider.startSession(threadId, {
502+
provider: CODEX_DRIVER,
503+
providerInstanceId: codexInstanceId,
504+
threadId,
505+
runtimeMode: "full-access",
506+
});
507+
codex.listSessions.mockReturnValue(
508+
Effect.succeed([
509+
{
510+
...session,
511+
status: completed ? "ready" : "running",
512+
activeTurnId: completed ? undefined : turnId,
513+
},
514+
]),
515+
);
516+
const pending = yield* directory.getBinding(threadId);
517+
assert(Option.isSome(pending));
518+
yield* directory.upsert({
519+
...pending.value,
520+
runtimePayload: { activeTurnId: null, continueAfterServerUpdate: turnId },
521+
});
522+
const accepted = yield* provider.sendTurn({ threadId, continuation: true });
523+
const admitted = yield* directory.getBinding(threadId);
524+
assert(Option.isSome(admitted));
525+
assert.propertyVal(admitted.value.runtimePayload, "activeTurnId", accepted.turnId);
526+
assert.propertyVal(admitted.value.runtimePayload, "continueAfterServerUpdate", null);
527+
if (completed) {
528+
// Updates can mark an already-admitted turn immediately before it finishes.
529+
yield* directory.upsert({
530+
...admitted.value,
531+
runtimePayload: {
532+
continueAfterServerUpdate: accepted.turnId,
533+
continueAfterServerUpdatePrepared: null,
534+
},
535+
});
536+
}
537+
const markers: unknown[] = [];
538+
codex.stopAll.mockImplementation(() =>
539+
Effect.gen(function* () {
540+
const binding = yield* directory.getBinding(threadId);
541+
assert(Option.isSome(binding));
542+
markers.push(binding.value.runtimePayload);
543+
}).pipe(Effect.orDie),
544+
);
545+
yield* Scope.close(scope, Exit.void);
546+
const binding = yield* directory.getBinding(threadId);
547+
assert(Option.isSome(binding));
548+
assert.equal(codex.stopAll.mock.calls.length, 1);
549+
assert.deepStrictEqual(binding.value.resumeCursor, session.resumeCursor);
550+
assert.equal(binding.value.status, "stopped");
551+
assert.propertyVal(markers[0], "activeTurnId", completed ? null : turnId);
552+
if (enabled && !completed) {
553+
assert.propertyVal(markers[0], "continueAfterServerUpdate", turnId);
554+
assert.propertyVal(binding.value.runtimePayload, "continueAfterServerUpdate", turnId);
555+
} else if (completed) {
556+
assert.propertyVal(
557+
binding.value.runtimePayload,
558+
"continueAfterServerUpdate",
559+
accepted.turnId,
560+
);
561+
assert.propertyVal(
562+
binding.value.runtimePayload,
563+
"continueAfterServerUpdatePrepared",
564+
null,
565+
);
566+
} else {
567+
assert.propertyVal(markers[0], "continueAfterServerUpdate", null);
568+
assert.propertyVal(binding.value.runtimePayload, "continueAfterServerUpdate", null);
569+
}
570+
}).pipe(Effect.provide(NodeServices.layer)),
571+
);
572+
}
573+
455574
it.effect("ProviderServiceLive catches stopAll failures during shutdown", () =>
456575
Effect.gen(function* () {
457576
const codex = makeFakeCodexAdapter();

‎apps/server/src/provider/Layers/ProviderService.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ function toRuntimePayloadFromSession(
223223
session: ProviderSession,
224224
extra?: {
225225
readonly modelSelection?: unknown;
226+
readonly continueAfterServerUpdate?: TurnId;
226227
readonly lastRuntimeEvent?: string;
227228
readonly lastRuntimeEventAt?: string;
228229
},
@@ -232,6 +233,9 @@ function toRuntimePayloadFromSession(
232233
model: session.model ?? null,
233234
activeTurnId: session.activeTurnId ?? null,
234235
lastError: session.lastError ?? null,
236+
...(extra?.continueAfterServerUpdate !== undefined
237+
? { continueAfterServerUpdate: extra.continueAfterServerUpdate }
238+
: {}),
235239
...(extra?.modelSelection !== undefined ? { modelSelection: extra.modelSelection } : {}),
236240
...(extra?.lastRuntimeEvent !== undefined ? { lastRuntimeEvent: extra.lastRuntimeEvent } : {}),
237241
...(extra?.lastRuntimeEventAt !== undefined
@@ -831,6 +835,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
831835
threadId: ThreadId,
832836
extra?: {
833837
readonly modelSelection?: unknown;
838+
readonly continueAfterServerUpdate?: TurnId;
834839
readonly lastRuntimeEvent?: string;
835840
readonly lastRuntimeEventAt?: string;
836841
},
@@ -1433,6 +1438,9 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
14331438
runtimePayload: {
14341439
...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),
14351440
activeTurnId: turn.turnId,
1441+
// Admission and marker consumption must survive the same restart.
1442+
continueAfterServerUpdate: null,
1443+
continueAfterServerUpdatePrepared: null,
14361444
lastRuntimeEvent: "provider.sendTurn",
14371445
lastRuntimeEventAt: yield* nowIso,
14381446
},
@@ -1733,6 +1741,8 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
17331741
status: "stopped",
17341742
runtimePayload: {
17351743
activeTurnId: null,
1744+
continueAfterServerUpdate: null,
1745+
continueAfterServerUpdatePrepared: null,
17361746
},
17371747
});
17381748
yield* analytics.record("provider.session.stopped", {
@@ -1942,6 +1952,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
19421952
);
19431953

19441954
const runStopAll = Effect.fn("runStopAll")(function* () {
1955+
const continueAfterRestart = yield* serverSettings.getSettings.pipe(
1956+
Effect.map((settings) => settings.continueThreadsAfterServerUpdate),
1957+
Effect.orElseSucceed(() => false),
1958+
);
19451959
const properties = yield* Ref.modify(turnAnalytics, (state) => {
19461960
const completed: Array<Readonly<Record<string, unknown>>> = [];
19471961
for (const [sessionKey, session] of state.sessions) {
@@ -1969,6 +1983,9 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
19691983
yield* Effect.forEach(activeSessions, (session) =>
19701984
Effect.flatMap(nowIso, (lastRuntimeEventAt) =>
19711985
upsertSessionBinding(session, session.threadId, {
1986+
...(continueAfterRestart && session.status === "running" && session.activeTurnId
1987+
? { continueAfterServerUpdate: session.activeTurnId }
1988+
: {}),
19721989
lastRuntimeEvent: "provider.stopAll",
19731990
lastRuntimeEventAt,
19741991
}),

0 commit comments

Comments
 (0)