Skip to content

Commit dac971b

Browse files
fix(server): harden provider resume and rollback state
Surface resume cursor persistence failures as typed runtime errors instead of publishing unsafe completions. Treat anchorless rollback targets as exact full-history snapshots so concurrent turns invalidate them. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
1 parent f3504eb commit dac971b

3 files changed

Lines changed: 152 additions & 16 deletions

File tree

‎apps/server/src/provider/Layers/ProviderService.test.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";
4242
import {
4343
ProviderAdapterRequestError,
4444
ProviderAdapterSessionNotFoundError,
45+
ProviderSessionDirectoryPersistenceError,
4546
ProviderUnsupportedError,
4647
ProviderValidationError,
4748
type ProviderAdapterError,
@@ -1068,6 +1069,46 @@ routing.layer("ProviderServiceLive routing", (it) => {
10681069
}),
10691070
);
10701071

1072+
it.effect("rejects a full-history rollback target after another turn is appended", () =>
1073+
Effect.gen(function* () {
1074+
const provider = yield* ProviderService.ProviderService;
1075+
const session = yield* provider.startSession(asThreadId("thread-rollback-stale-full"), {
1076+
provider: ProviderDriverKind.make("codex"),
1077+
providerInstanceId: codexInstanceId,
1078+
threadId: asThreadId("thread-rollback-stale-full"),
1079+
runtimeMode: "full-access",
1080+
});
1081+
const retainedTurn = { id: asTurnId("turn-retained"), items: [] as const };
1082+
routing.codex.readThread
1083+
.mockReturnValueOnce(
1084+
Effect.succeed({
1085+
threadId: session.threadId,
1086+
turns: [retainedTurn],
1087+
}),
1088+
)
1089+
.mockReturnValueOnce(
1090+
Effect.succeed({
1091+
threadId: session.threadId,
1092+
turns: [retainedTurn, { id: asTurnId("turn-appended"), items: [] as const }],
1093+
}),
1094+
);
1095+
routing.codex.rollbackThread.mockClear();
1096+
1097+
const target = yield* provider.prepareConversationRollback({
1098+
threadId: session.threadId,
1099+
retainedThroughTurnId: retainedTurn.id,
1100+
});
1101+
const error = yield* provider.rollbackConversation(target).pipe(Effect.flip);
1102+
1103+
assert.deepStrictEqual(target, {
1104+
threadId: session.threadId,
1105+
turnIds: [retainedTurn.id],
1106+
});
1107+
assert.equal(error._tag, "ProviderValidationError");
1108+
assert.equal(routing.codex.rollbackThread.mock.calls.length, 0);
1109+
}),
1110+
);
1111+
10711112
it.effect("derives rollback targets from complete provider history", () =>
10721113
Effect.gen(function* () {
10731114
const provider = yield* ProviderService.ProviderService;
@@ -1951,6 +1992,84 @@ fanout.layer("ProviderServiceLive fanout", (it) => {
19511992
}),
19521993
);
19531994

1995+
it.effect("surfaces a failed resume cursor write without stopping adapter event delivery", () =>
1996+
Effect.gen(function* () {
1997+
const provider = yield* ProviderService.ProviderService;
1998+
const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
1999+
const session = yield* provider.startSession(asThreadId("thread-cursor-write-failure"), {
2000+
provider: CODEX_DRIVER,
2001+
providerInstanceId: codexInstanceId,
2002+
threadId: asThreadId("thread-cursor-write-failure"),
2003+
runtimeMode: "full-access",
2004+
});
2005+
const originalUpsert = directory.upsert;
2006+
const upsertSpy = vi.spyOn(directory, "upsert").mockImplementation((binding) =>
2007+
binding.resumeCursor !== undefined
2008+
? Effect.fail(
2009+
new ProviderSessionDirectoryPersistenceError({
2010+
operation: "ProviderSessionDirectory.upsert",
2011+
detail: "Injected resume cursor write failure.",
2012+
}),
2013+
)
2014+
: originalUpsert(binding),
2015+
);
2016+
yield* Effect.addFinalizer(() => Effect.sync(() => upsertSpy.mockRestore()));
2017+
2018+
const eventsRef = yield* Ref.make<Array<ProviderRuntimeEvent>>([]);
2019+
const persistenceFailureObserved = yield* Deferred.make<void>();
2020+
const followUpObserved = yield* Deferred.make<void>();
2021+
const consumer = yield* Stream.runForEach(provider.streamEvents, (event) =>
2022+
Ref.update(eventsRef, (current) => [...current, event]).pipe(
2023+
Effect.andThen(
2024+
event.eventId === asEventId("evt-cursor-write-failure")
2025+
? Deferred.succeed(persistenceFailureObserved, undefined)
2026+
: event.eventId === asEventId("evt-after-cursor-write-failure")
2027+
? Deferred.succeed(followUpObserved, undefined)
2028+
: Effect.void,
2029+
),
2030+
),
2031+
).pipe(Effect.forkChild({ startImmediately: true }));
2032+
2033+
yield* fanout.codex.streamSubscribed;
2034+
fanout.codex.emit({
2035+
type: "turn.completed",
2036+
eventId: asEventId("evt-cursor-write-failure"),
2037+
provider: CODEX_DRIVER,
2038+
createdAt: "2026-01-01T00:00:00.000Z",
2039+
threadId: session.threadId,
2040+
turnId: asTurnId("turn-cursor-write-failure"),
2041+
payload: {
2042+
state: "completed",
2043+
resumeCursor: { threadId: "provider-thread-after-fork" },
2044+
},
2045+
});
2046+
yield* Deferred.await(persistenceFailureObserved);
2047+
2048+
upsertSpy.mockRestore();
2049+
fanout.codex.emit({
2050+
type: "tool.completed",
2051+
eventId: asEventId("evt-after-cursor-write-failure"),
2052+
provider: CODEX_DRIVER,
2053+
createdAt: "2026-01-01T00:00:01.000Z",
2054+
threadId: session.threadId,
2055+
turnId: asTurnId("turn-cursor-write-failure"),
2056+
toolKind: "command",
2057+
title: "Ran command",
2058+
});
2059+
yield* Deferred.await(followUpObserved);
2060+
2061+
const events = yield* Ref.get(eventsRef);
2062+
yield* Fiber.interrupt(consumer);
2063+
assert.deepEqual(
2064+
events.map((event) => [event.eventId, event.type]),
2065+
[
2066+
[asEventId("evt-cursor-write-failure"), "runtime.error"],
2067+
[asEventId("evt-after-cursor-write-failure"), "tool.completed"],
2068+
],
2069+
);
2070+
}),
2071+
);
2072+
19542073
it.effect("fans out canonical runtime events in emission order", () =>
19552074
Effect.gen(function* () {
19562075
const provider = yield* ProviderService.ProviderService;

‎apps/server/src/provider/Layers/ProviderService.ts‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -350,34 +350,51 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
350350
): Effect.Effect<void> =>
351351
Effect.sync(() => correlateRuntimeEventWithInstance(source, event)).pipe(
352352
Effect.flatMap((canonicalEvent) =>
353-
(canonicalEvent.type === "turn.completed" &&
354-
canonicalEvent.payload?.resumeCursor !== undefined
355-
? directory
353+
Effect.gen(function* () {
354+
let eventToPublish = canonicalEvent;
355+
if (
356+
canonicalEvent.type === "turn.completed" &&
357+
canonicalEvent.payload?.resumeCursor !== undefined
358+
) {
359+
const cursorPersisted = yield* directory
356360
.upsert({
357361
threadId: canonicalEvent.threadId,
358362
provider: canonicalEvent.provider,
359363
providerInstanceId: source.instanceId,
360364
resumeCursor: canonicalEvent.payload.resumeCursor,
361365
})
362366
.pipe(
367+
Effect.as(true),
363368
Effect.catch((error) =>
364369
Effect.logWarning("Failed to persist provider resume cursor.", {
365370
threadId: canonicalEvent.threadId,
366371
provider: canonicalEvent.provider,
367372
cause: error,
368-
}),
373+
}).pipe(Effect.as(false)),
369374
),
370-
)
371-
: Effect.void
372-
).pipe(
373-
Effect.andThen(
374-
increment(providerRuntimeEventsTotal, {
375-
provider: canonicalEvent.provider,
376-
eventType: canonicalEvent.type,
377-
}),
378-
),
379-
Effect.andThen(publishRuntimeEvent(canonicalEvent)),
380-
),
375+
);
376+
if (!cursorPersisted) {
377+
eventToPublish = {
378+
type: "runtime.error",
379+
eventId: canonicalEvent.eventId,
380+
provider: canonicalEvent.provider,
381+
providerInstanceId: source.instanceId,
382+
threadId: canonicalEvent.threadId,
383+
createdAt: canonicalEvent.createdAt,
384+
payload: {
385+
message:
386+
"Failed to persist provider resume state. Restarting this thread may resume stale provider history.",
387+
class: "unknown",
388+
},
389+
};
390+
}
391+
}
392+
yield* increment(providerRuntimeEventsTotal, {
393+
provider: eventToPublish.provider,
394+
eventType: eventToPublish.type,
395+
});
396+
yield* publishRuntimeEvent(eventToPublish);
397+
}),
381398
),
382399
);
383400

‎apps/server/src/provider/Services/ProviderAdapter.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ export function rollbackTargetMatchesKnownHistory(
5858
return false;
5959
}
6060
if (target.anchorTurnId === undefined) {
61-
return true;
61+
return turns.length === target.turnIds.length;
6262
}
6363
return turns[target.turnIds.length]?.id === target.anchorTurnId;
6464
}

0 commit comments

Comments
 (0)