Skip to content

Commit 39da423

Browse files
voltcrashclaude
andcommitted
fix(server): managed Codex conversations recover after ChatGPT token renewal
Managed ChatGPT sign-in passes the access token to Codex app-server through its launch environment, and the session manager kept reusing that process after the token was renewed, so every turn failed with 401 token_expired until T3 restarted. Codex sessions now report when the token they launched with no longer matches the current one (reading it refreshes a token near expiry). Before reusing an idle session for a new turn, the session manager releases a stale one, the same way idle release does, and the turn reopens app-server with the renewed token and resumes the thread. Fixes #16132 Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 611132c commit 39da423

7 files changed

Lines changed: 196 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
ProjectId,
2222
ProviderInstanceId,
2323
ProviderSessionId,
24+
ProviderSetupError,
2425
ProviderThreadId,
2526
ProviderTurnId,
2627
RunAttemptId,
@@ -1680,6 +1681,10 @@ describe("CodexAdapterV2 session initialize", () => {
16801681
const openReplaySession = (
16811682
transcript: CodexReplay.CodexAppServerReplayTranscript,
16821683
beforeEmitInbound?: CodexReplay.CodexAppServerReplayDriver["beforeEmitInbound"],
1684+
hooks: Pick<
1685+
CodexAdapterV2.CodexAdapterV2Options,
1686+
"resolveRuntime" | "currentRuntimeRevision"
1687+
> = {},
16831688
) =>
16841689
Effect.gen(function* () {
16851690
const driver = yield* CodexReplay.makeReplayDriver(
@@ -1721,6 +1726,7 @@ describe("CodexAdapterV2 session initialize", () => {
17211726
fileSystem: yield* FileSystem.FileSystem,
17221727
idAllocator: yield* IdAllocator.IdAllocatorV2,
17231728
serverConfig: yield* makeReplayServerConfig(transcript.scenario).pipe(Effect.orDie),
1729+
...hooks,
17241730
});
17251731
const runtime = yield* adapter.openSession({
17261732
threadId: ThreadId.make(`thread-${transcript.scenario}`),
@@ -1736,6 +1742,7 @@ describe("CodexAdapterV2 session initialize", () => {
17361742
runtimePolicy: CODEX_TEST_RUNTIME_POLICY,
17371743
}),
17381744
initializeRequests: () => initializeRequests,
1745+
hasStaleCredentials: runtime.hasStaleCredentials,
17391746
};
17401747
});
17411748

@@ -1853,6 +1860,42 @@ describe("CodexAdapterV2 session initialize", () => {
18531860
assert.equal(session.initializeRequests(), 2);
18541861
}).pipe(Effect.scoped, Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer))),
18551862
);
1863+
1864+
it.effect("reports stale credentials once the token it launched with is renewed", () =>
1865+
Effect.gen(function* () {
1866+
const currentToken = yield* Ref.make<string | null>("token-1");
1867+
const session = yield* openReplaySession(
1868+
makeCodexReplayTranscript({ scenario: "stale-credentials", entries: [] }),
1869+
undefined,
1870+
{
1871+
resolveRuntime: Effect.succeed({
1872+
config: DEFAULT_CODEX_SETTINGS,
1873+
environment: { ACCESS_TOKEN: "token-1" },
1874+
revision: "token-1",
1875+
}),
1876+
currentRuntimeRevision: Ref.get(currentToken).pipe(
1877+
Effect.flatMap((token) =>
1878+
token === null
1879+
? Effect.fail(
1880+
new ProviderSetupError({
1881+
instanceId: CodexAdapterV2.CODEX_DEFAULT_INSTANCE_ID,
1882+
operation: "refresh",
1883+
detail: "Could not renew the ChatGPT connection.",
1884+
}),
1885+
)
1886+
: Effect.succeed(token),
1887+
),
1888+
),
1889+
},
1890+
);
1891+
assert.isFalse(yield* session.hasStaleCredentials!);
1892+
yield* Ref.set(currentToken, "token-2");
1893+
assert.isTrue(yield* session.hasStaleCredentials!);
1894+
// A failed renewal keeps the session rather than replacing it blindly.
1895+
yield* Ref.set(currentToken, null);
1896+
assert.isFalse(yield* session.hasStaleCredentials!);
1897+
}).pipe(Effect.scoped, Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer))),
1898+
);
18561899
});
18571900

18581901
describe("CodexAdapterV2 post-settle continuation", () => {

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1534,7 +1534,10 @@ export type CodexAdapterV2DriverEnv =
15341534

15351535
export const createCodexAdapterV2 = (
15361536
{ instanceId, environment, enabled, config }: ProviderAdapterDriverCreateInput<CodexSettings>,
1537-
hooks: Pick<CodexAdapterV2Options, "onUsageLimits" | "resolveRuntime"> = {},
1537+
hooks: Pick<
1538+
CodexAdapterV2Options,
1539+
"onUsageLimits" | "resolveRuntime" | "currentRuntimeRevision"
1540+
> = {},
15381541
) =>
15391542
Effect.gen(function* () {
15401543
const clientFactory = yield* CodexAppServerClientFactory;
@@ -1627,6 +1630,11 @@ export interface CodexAdapterV2Options {
16271630
* Codex with a current access token.
16281631
*/
16291632
readonly resolveRuntime?: Effect.Effect<CodexEffectiveRuntime, ProviderSetupError, Scope.Scope>;
1633+
/**
1634+
* The revision `resolveRuntime` would launch with now. A session whose
1635+
* process launched with an older revision reports stale credentials.
1636+
*/
1637+
readonly currentRuntimeRevision?: Effect.Effect<string, ProviderSetupError>;
16301638
readonly crypto: Crypto.Crypto;
16311639
readonly fileSystem: FileSystem.FileSystem;
16321640
readonly idAllocator: IdAllocatorV2Shape;
@@ -6029,6 +6037,15 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi
60296037
providerSession: session,
60306038
events: Stream.fromEffectRepeat(Queue.take(events)),
60316039
canReuseContextUsage: canReuseCodexContextUsage,
6040+
// App-server reads its access token from the environment only at launch.
6041+
...(resolvedRuntime === undefined || adapterOptions.currentRuntimeRevision === undefined
6042+
? {}
6043+
: {
6044+
hasStaleCredentials: adapterOptions.currentRuntimeRevision.pipe(
6045+
Effect.map((revision) => revision !== resolvedRuntime.revision),
6046+
Effect.orElseSucceed(() => false),
6047+
),
6048+
}),
60326049
// Known gap: a subagent that Codex resumes later reads as completed
60336050
// (not pending) between turns, so idle release can win the race
60346051
// against a long-delayed resume. Codex emits no resume-expected

‎apps/server/src/orchestration-v2/ProviderAdapter.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,12 @@ export interface ProviderAdapterV2SessionRuntime {
501501
* here so the session manager defers idle release while it is pending.
502502
*/
503503
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
504+
/**
505+
* Adapters whose native process captured credentials at launch report when
506+
* those have since been renewed. The session manager then replaces the idle
507+
* session on its next open instead of reusing a process that fails auth.
508+
*/
509+
readonly hasStaleCredentials?: Effect.Effect<boolean>;
504510
/**
505511
* Per-provider-thread pending work for root-run ingestion stop gates. When
506512
* present, RunExecutionService uses only this probe (never the session-wide

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,8 @@ function makeProviderAdapter(
319319
readonly initialProviderItemIdentityVersion?: 2;
320320
}) => Effect.Effect<void>;
321321
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
322+
/** Each session launches with the current value and is stale once it changes. */
323+
readonly credentialRevision?: Ref.Ref<number>;
322324
readonly hangSessionScopeClose?: boolean;
323325
readonly startTurn?: Effect.Effect<void>;
324326
readonly beforeUnload?: Effect.Effect<void>;
@@ -389,6 +391,9 @@ function makeProviderAdapter(
389391
yield* countClose;
390392
if (options.hangSessionScopeClose === true) yield* hangClose;
391393
}
394+
const credentialRevision = options.credentialRevision;
395+
const launchedRevision =
396+
credentialRevision === undefined ? undefined : yield* Ref.get(credentialRevision);
392397

393398
return {
394399
instanceId: ProviderInstanceId.make("codex"),
@@ -407,6 +412,13 @@ function makeProviderAdapter(
407412
...(options.hasPendingBackgroundWork === undefined
408413
? {}
409414
: { hasPendingBackgroundWork: options.hasPendingBackgroundWork }),
415+
...(credentialRevision === undefined
416+
? {}
417+
: {
418+
hasStaleCredentials: Ref.get(credentialRevision).pipe(
419+
Effect.map((revision) => revision !== launchedRevision),
420+
),
421+
}),
410422
ensureThread: () => unimplemented("ensureThread unused in test"),
411423
resumeThread: (threadInput) =>
412424
Ref.update(state, (current) => ({
@@ -463,6 +475,7 @@ function layerTest(input: {
463475
readonly paused: Deferred.Deferred<void>;
464476
};
465477
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
478+
readonly credentialRevision?: Ref.Ref<number>;
466479
readonly hangSessionScopeClose?: boolean;
467480
readonly startTurn?: Effect.Effect<void>;
468481
readonly beforeUnload?: Effect.Effect<void>;
@@ -488,6 +501,9 @@ function layerTest(input: {
488501
...(input.hasPendingBackgroundWork === undefined
489502
? {}
490503
: { hasPendingBackgroundWork: input.hasPendingBackgroundWork }),
504+
...(input.credentialRevision === undefined
505+
? {}
506+
: { credentialRevision: input.credentialRevision }),
491507
...(input.hangSessionScopeClose === undefined
492508
? {}
493509
: { hangSessionScopeClose: input.hangSessionScopeClose }),
@@ -2716,6 +2732,82 @@ it.effect("ProviderSessionManagerV2 defers idle release while background work is
27162732
}),
27172733
);
27182734

2735+
it.effect("ProviderSessionManagerV2 replaces an idle session whose credentials were renewed", () =>
2736+
Effect.gen(function* () {
2737+
const state = yield* Ref.make(emptyState);
2738+
const credentialRevision = yield* Ref.make(1);
2739+
const startTurnReached = yield* Deferred.make<void>();
2740+
yield* Effect.gen(function* () {
2741+
const eventSink = yield* EventSink.EventSinkV2;
2742+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2743+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2744+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2745+
const now = yield* DateTime.now;
2746+
const threadId = ThreadId.make("thread-provider-session-manager-renewed-credentials");
2747+
const providerSessionId = yield* idAllocator.allocate.providerSession({
2748+
providerInstanceId: modelSelection.instanceId,
2749+
threadId,
2750+
});
2751+
yield* eventSink.write({
2752+
events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
2753+
});
2754+
const open = manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
2755+
const first = yield* open;
2756+
assert.strictEqual(yield* open, first);
2757+
2758+
const runId = idAllocator.derive.run({ threadId, ordinal: 1 });
2759+
const running = yield* first
2760+
.startTurn({
2761+
appThread: (yield* projectionStore.getThreadProjection(threadId)).thread,
2762+
threadId,
2763+
runId,
2764+
runOrdinal: 1,
2765+
providerTurnOrdinal: 1,
2766+
attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }),
2767+
rootNodeId: idAllocator.derive.rootNode({ runId }),
2768+
providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }),
2769+
message: {
2770+
createdBy: "user",
2771+
creationSource: "web",
2772+
messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }),
2773+
text: "running while the token renews",
2774+
attachments: [],
2775+
},
2776+
modelSelection,
2777+
runtimePolicy,
2778+
})
2779+
.pipe(Effect.forkChild);
2780+
yield* Deferred.await(startTurnReached);
2781+
yield* Ref.set(credentialRevision, 2);
2782+
2783+
// A running turn keeps the process it started on.
2784+
assert.strictEqual(yield* open, first);
2785+
assert.equal((yield* Ref.get(state)).closeCount, 0);
2786+
2787+
yield* Fiber.interrupt(running);
2788+
const replacement = yield* open;
2789+
assert.notStrictEqual(replacement, first);
2790+
assert.equal((yield* Ref.get(state)).openCount, 2);
2791+
assert.equal((yield* Ref.get(state)).closeCount, 1);
2792+
assert.strictEqual(yield* open, replacement);
2793+
const session = (yield* projectionStore.getThreadRecords(threadId, ["providerSessions"]))
2794+
.providerSessions[0];
2795+
assert.equal(session?.status, "ready");
2796+
}).pipe(
2797+
Effect.provide(
2798+
layerTest({
2799+
state,
2800+
idleTimeoutMs: 60_000,
2801+
credentialRevision,
2802+
startTurn: Deferred.succeed(startTurnReached, undefined).pipe(
2803+
Effect.andThen(Effect.never),
2804+
),
2805+
}),
2806+
),
2807+
);
2808+
}),
2809+
);
2810+
27192811
it.effect("ProviderSessionManagerV2 releases pinned idle sessions once the pin cap expires", () =>
27202812
Effect.gen(function* () {
27212813
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1114,6 +1114,35 @@ export const layerWithOptions = (
11141114
);
11151115
});
11161116

1117+
// A process launched with since-renewed credentials keeps failing auth
1118+
// until it restarts. Release it only when idle release could: a running
1119+
// turn or pending background work keeps the current process.
1120+
const releaseIfStaleCredentials = (entry: LiveSessionEntry) =>
1121+
Effect.gen(function* () {
1122+
const runtime = entry.runtime;
1123+
if (entry.busyTurns.size > 0 || runtime.hasStaleCredentials === undefined) return;
1124+
if (!(yield* runtime.hasStaleCredentials)) return;
1125+
if (
1126+
runtime.hasPendingBackgroundWork !== undefined &&
1127+
(yield* runtime.hasPendingBackgroundWork)
1128+
) {
1129+
return;
1130+
}
1131+
yield* releaseEntry({
1132+
providerSessionId: runtime.providerSessionId,
1133+
reason: "manual_shutdown",
1134+
detail: "Provider credentials were renewed.",
1135+
onlyIfIdleGeneration: entry.idleGeneration,
1136+
});
1137+
}).pipe(
1138+
Effect.catchCause((cause) =>
1139+
Effect.logWarning("orchestration-v2.driver-session.stale-credentials-release-failed", {
1140+
providerSessionId: entry.runtime.providerSessionId,
1141+
cause,
1142+
}),
1143+
),
1144+
);
1145+
11171146
const withActivityError = <A, E, R>(
11181147
providerSessionId: ProviderSessionId,
11191148
effect: Effect.Effect<A, E, R>,
@@ -1834,6 +1863,8 @@ export const layerWithOptions = (
18341863
}
18351864
}
18361865
const key = sessionKey(input.providerSessionId);
1866+
const live = (yield* Ref.get(sessions)).get(key);
1867+
if (live !== undefined) yield* releaseIfStaleCredentials(live);
18371868
const existing = (yield* Ref.get(sessions)).get(key);
18381869
if (existing !== undefined) {
18391870
if (

‎apps/server/src/provider/CodexManagedRuntime.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,5 +108,7 @@ export const makeCodexManagedRuntime = Effect.fn("makeCodexManagedRuntime")(func
108108
revision: credentials.accessToken,
109109
} satisfies CodexEffectiveRuntime;
110110
});
111-
return { auth, resolve, installation, homePath, homeLayout };
111+
// Refreshes like `resolve`, so an expiring token reads as a new revision.
112+
const revision = auth.access.pipe(Effect.map((credentials) => credentials.accessToken));
113+
return { auth, resolve, revision, installation, homePath, homeLayout };
112114
});

‎apps/server/src/provider/Drivers/CodexManagedProvider.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,9 @@ export const makeManagedCodexProvider = Effect.fn("makeManagedCodexProvider")(fu
220220
const orchestrationAdapter = yield* createCodexAdapterV2(input, {
221221
onUsageLimits: (update) => snapshot.applyUsageLimits(update),
222222
resolveRuntime: runtime.resolve,
223+
currentRuntimeRevision: runtime.auth.controller.withAccess!(runtime.revision).pipe(
224+
Effect.scoped,
225+
),
223226
}).pipe(
224227
Effect.mapError(
225228
(cause) =>

0 commit comments

Comments
 (0)