Skip to content

Commit 8bd44f2

Browse files
fix(server): keep provider sessions alive during background agent work
The provider session reaper stops sessions after 30 min of lastSeenAt inactivity. Background agent work (dynamic workflows, subagents) keeps running after the foreground turn settles, but the adapter session goes "ready" with no active turn and nothing refreshes lastSeenAt — so the reaper tears the session down mid-workflow, and the in-flight background orchestration is lost. Refresh lastSeenAt from runtime activity: ProviderService now bumps the binding's last_seen_at when the adapter emits runtime events (e.g. task.progress from background subagents), throttled per thread so an event burst is at most one lightweight write per window. A new targeted touchLastSeen on the runtime repository / session directory updates only last_seen_at, and only for non-stopped rows so a reaped session is never resurrected. Co-Authored-By: Claude Opus 4.8 <[email protected]>
1 parent 5d34f9f commit 8bd44f2

8 files changed

Lines changed: 236 additions & 1 deletion

‎apps/server/src/persistence/ProviderSessionRuntime.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ export type GetProviderSessionRuntimeInput = typeof GetProviderSessionRuntimeInp
5858
export const DeleteProviderSessionRuntimeInput = Schema.Struct({ threadId: ThreadId });
5959
export type DeleteProviderSessionRuntimeInput = typeof DeleteProviderSessionRuntimeInput.Type;
6060

61+
export const TouchLastSeenInput = Schema.Struct({
62+
threadId: ThreadId,
63+
lastSeenAt: IsoDateTime,
64+
});
65+
export type TouchLastSeenInput = typeof TouchLastSeenInput.Type;
66+
6167
/**
6268
* ProviderSessionRuntimeRepository - Service tag for provider runtime persistence.
6369
*/
@@ -93,6 +99,18 @@ export class ProviderSessionRuntimeRepository extends Context.Service<
9399
ProviderSessionRuntimeRepositoryError
94100
>;
95101

102+
/**
103+
* Bump only `last_seen_at` for an existing, non-stopped row.
104+
*
105+
* A targeted update used to keep a session's inactivity clock fresh from
106+
* background runtime activity (e.g. a running dynamic workflow) without
107+
* rewriting the full runtime payload. Rows in `stopped` status are left
108+
* untouched so a reaped session is never resurrected.
109+
*/
110+
readonly touchLastSeen: (
111+
input: TouchLastSeenInput,
112+
) => Effect.Effect<void, ProviderSessionRuntimeRepositoryError>;
113+
96114
/**
97115
* Delete provider runtime state by canonical thread id.
98116
*/
@@ -235,6 +253,17 @@ export const make = Effect.gen(function* () {
235253
`,
236254
});
237255

256+
const touchLastSeenRow = SqlSchema.void({
257+
Request: TouchLastSeenInput,
258+
execute: ({ threadId, lastSeenAt }) =>
259+
sql`
260+
UPDATE provider_session_runtime
261+
SET last_seen_at = ${lastSeenAt}
262+
WHERE thread_id = ${threadId}
263+
AND status != 'stopped'
264+
`,
265+
});
266+
238267
const upsert: ProviderSessionRuntimeRepository["Service"]["upsert"] = (runtime) =>
239268
upsertRuntimeRow(runtime).pipe(
240269
Effect.mapError(
@@ -308,6 +337,17 @@ export const make = Effect.gen(function* () {
308337
),
309338
);
310339

340+
const touchLastSeen: ProviderSessionRuntimeRepository["Service"]["touchLastSeen"] = (input) =>
341+
touchLastSeenRow(input).pipe(
342+
Effect.mapError(
343+
toPersistenceSqlOrDecodeError(
344+
"ProviderSessionRuntimeRepository.touchLastSeen:query",
345+
"ProviderSessionRuntimeRepository.touchLastSeen:encodeRequest",
346+
{ threadId: input.threadId },
347+
),
348+
),
349+
);
350+
311351
const deleteByThreadId: ProviderSessionRuntimeRepository["Service"]["deleteByThreadId"] = (
312352
input,
313353
) =>
@@ -326,6 +366,7 @@ export const make = Effect.gen(function* () {
326366
upsert,
327367
getByThreadId,
328368
list,
369+
touchLastSeen,
329370
deleteByThreadId,
330371
} satisfies ProviderSessionRuntimeRepository["Service"];
331372
});

‎apps/server/src/provider/Layers/CodexAdapter.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ const providerSessionDirectoryTestLayer = Layer.succeed(ProviderSessionDirectory
218218
getBinding: () => Effect.succeed(Option.none()),
219219
listThreadIds: () => Effect.succeed([]),
220220
listBindings: () => Effect.succeed([]),
221+
touchLastSeen: () => Effect.void,
221222
});
222223

223224
const validationRuntimeFactory = makeRuntimeFactory();

‎apps/server/src/provider/Layers/OpenCodeAdapter.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ const providerSessionDirectoryTestLayer = Layer.succeed(ProviderSessionDirectory
238238
getBinding: () => Effect.succeed(Option.none()),
239239
listThreadIds: () => Effect.succeed([]),
240240
listBindings: () => Effect.succeed([]),
241+
touchLastSeen: () => Effect.void,
241242
});
242243

243244
// The adapter now receives its settings as a plain argument (the old design

‎apps/server/src/provider/Layers/ProviderService.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,81 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se
642642
}).pipe(Effect.provide(NodeServices.layer)),
643643
);
644644

645+
it.effect("ProviderServiceLive touches lastSeenAt on background runtime activity, throttled", () =>
646+
Effect.gen(function* () {
647+
const codex = makeFakeCodexAdapter();
648+
const registry = makeAdapterRegistryMock({
649+
[ProviderDriverKind.make("codex")]: codex.adapter,
650+
});
651+
652+
// Spy directory: records every touchLastSeen call so we assert the wiring
653+
// directly (a runtime event triggers a touch) rather than inferring it from
654+
// DB timestamp noise, and confirms throttling collapses a burst.
655+
const touched: string[] = [];
656+
const spyDirectoryLayer = Layer.succeed(ProviderSessionDirectory.ProviderSessionDirectory, {
657+
upsert: () => Effect.void,
658+
getProvider: () => Effect.die(new Error("getProvider unused in test")),
659+
getBinding: () => Effect.succeed(Option.none()),
660+
listThreadIds: () => Effect.succeed([]),
661+
listBindings: () => Effect.succeed([]),
662+
touchLastSeen: (threadId) =>
663+
Effect.sync(() => {
664+
touched.push(threadId);
665+
}),
666+
});
667+
668+
const threadId = asThreadId("thread-runtime-activity-touch");
669+
670+
const providerLayer = makeProviderServiceLive({
671+
// Tight throttle window so the test can prove both "touches" and
672+
// "throttles a rapid burst" deterministically with the test clock.
673+
runtimeActivityTouchThrottleMs: 1_000,
674+
}).pipe(
675+
Layer.provide(Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistry, registry)),
676+
Layer.provide(spyDirectoryLayer),
677+
Layer.provide(defaultServerSettingsLayer),
678+
Layer.provide(AnalyticsService.layerTest),
679+
Layer.provide(
680+
Layer.succeed(
681+
ProviderEventLoggers.ProviderEventLoggers,
682+
ProviderEventLoggers.NoOpProviderEventLoggers,
683+
),
684+
),
685+
);
686+
687+
const emitTaskProgress = (id: string) =>
688+
codex.emit({
689+
eventId: asEventId(id),
690+
provider: ProviderDriverKind.make("codex"),
691+
threadId,
692+
createdAt: "2026-01-01T00:05:00.000Z",
693+
type: "task.progress",
694+
payload: {},
695+
});
696+
697+
yield* Effect.gen(function* () {
698+
yield* ProviderService.ProviderService;
699+
yield* advanceTestClock(10);
700+
701+
// First background event → one touch.
702+
emitTaskProgress("evt-task-progress-1");
703+
yield* advanceTestClock(20);
704+
assert.deepEqual(touched, [threadId]);
705+
706+
// Rapid second event within the throttle window → collapsed (no new touch).
707+
emitTaskProgress("evt-task-progress-2");
708+
yield* advanceTestClock(20);
709+
assert.deepEqual(touched, [threadId]);
710+
711+
// After the throttle window elapses, a further event touches again.
712+
yield* advanceTestClock(1_000);
713+
emitTaskProgress("evt-task-progress-3");
714+
yield* advanceTestClock(20);
715+
assert.deepEqual(touched, [threadId, threadId]);
716+
}).pipe(Effect.provide(providerLayer));
717+
}).pipe(Effect.provide(NodeServices.layer)),
718+
);
719+
645720
it.effect("ProviderServiceLive keeps persisted resumable sessions on startup", () =>
646721
Effect.gen(function* () {
647722
const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-service-"));

‎apps/server/src/provider/Layers/ProviderService.ts‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import {
2525
type ProviderSession,
2626
} from "@t3tools/contracts";
2727
import { causeErrorTag } from "@t3tools/shared/observability";
28+
import * as Clock from "effect/Clock";
2829
import * as DateTime from "effect/DateTime";
2930
import * as Effect from "effect/Effect";
3031
import * as Layer from "effect/Layer";
@@ -64,6 +65,11 @@ const isModelSelection = Schema.is(ModelSelection);
6465
*/
6566
export interface ProviderServiceLiveOptions {
6667
readonly canonicalEventLogger?: EventNdjsonLogger;
68+
/**
69+
* Minimum interval between `lastSeenAt` refreshes triggered by runtime
70+
* activity, per thread. Defaults to 60s. Exposed for tests.
71+
*/
72+
readonly runtimeActivityTouchThrottleMs?: number;
6773
}
6874

6975
type ProviderServiceMethod<Name extends keyof ProviderService.ProviderService["Service"]> =
@@ -213,6 +219,13 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
213219
const registry = yield* ProviderAdapterRegistry.ProviderAdapterRegistry;
214220
const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
215221
const runtimeEventPubSub = yield* PubSub.unbounded<ProviderRuntimeEvent>();
222+
// Throttle `lastSeenAt` refreshes keyed by thread. A running dynamic workflow
223+
// emits many runtime events per second (`task.progress`, token-usage updates,
224+
// …); we only need to bump the inactivity clock often enough that the session
225+
// reaper never mistakes an active background session for an idle one. One
226+
// write per thread per window is plenty and keeps the DB churn negligible.
227+
const lastSeenTouchMs = options?.runtimeActivityTouchThrottleMs ?? 60_000;
228+
const lastSeenTouchByThread = new Map<ThreadId, number>();
216229
const nowIso = Effect.map(DateTime.now, DateTime.formatIso);
217230
const prepareMcpSession = (threadId: ThreadId, providerInstanceId: ProviderInstanceId) =>
218231
McpSessionRegistry.issueActiveMcpCredential({ threadId, providerInstanceId }).pipe(
@@ -281,6 +294,27 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
281294
});
282295
});
283296

297+
// Keep a live session's inactivity clock fresh from runtime activity. This
298+
// matters for work that runs *after* the foreground turn settles — a
299+
// background dynamic workflow / subagents keep emitting runtime events (e.g.
300+
// `task.progress`) while the adapter session has already gone `ready` with no
301+
// active turn. Without this, the session reaper sees a stale `lastSeenAt` and
302+
// tears the session down mid-workflow. Throttled per thread so a burst of
303+
// events is at most one lightweight `last_seen_at` write per window.
304+
const refreshLastSeenForActivity = (event: ProviderRuntimeEvent): Effect.Effect<void> =>
305+
Effect.gen(function* () {
306+
const threadId = event.threadId;
307+
if (threadId === undefined) return;
308+
const now = yield* Clock.currentTimeMillis;
309+
const previous = lastSeenTouchByThread.get(threadId);
310+
if (previous !== undefined && now - previous < lastSeenTouchMs) return;
311+
lastSeenTouchByThread.set(threadId, now);
312+
// Best-effort: a failed touch must never break event processing. The row
313+
// may be absent/stopped (touch is a no-op) or the write may transiently
314+
// fail; the next event refreshes it.
315+
yield* directory.touchLastSeen(threadId).pipe(Effect.catchCause(() => Effect.void));
316+
});
317+
284318
const processRuntimeEvent = (
285319
source: {
286320
readonly instanceId: ProviderInstanceId;
@@ -293,7 +327,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
293327
increment(providerRuntimeEventsTotal, {
294328
provider: canonicalEvent.provider,
295329
eventType: canonicalEvent.type,
296-
}).pipe(Effect.andThen(publishRuntimeEvent(canonicalEvent))),
330+
}).pipe(
331+
Effect.andThen(refreshLastSeenForActivity(canonicalEvent)),
332+
Effect.andThen(publishRuntimeEvent(canonicalEvent)),
333+
),
297334
),
298335
);
299336

‎apps/server/src/provider/Layers/ProviderSessionDirectory.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,66 @@ it.layer(makeDirectoryLayer(SqlitePersistenceMemory))("ProviderSessionDirectoryL
196196
]);
197197
}));
198198

199+
it("touchLastSeen bumps last_seen_at for a live binding", () =>
200+
Effect.gen(function* () {
201+
const directory = yield* ProviderSessionDirectory;
202+
const runtimeRepository = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository;
203+
const threadId = ThreadId.make("thread-touch-live");
204+
205+
yield* runtimeRepository.upsert({
206+
threadId,
207+
providerName: "claudeAgent",
208+
providerInstanceId: null,
209+
adapterKey: "claudeAgent",
210+
runtimeMode: "full-access",
211+
status: "running",
212+
lastSeenAt: "2026-01-01T00:00:00.000Z",
213+
resumeCursor: null,
214+
runtimePayload: null,
215+
});
216+
217+
yield* directory.touchLastSeen(threadId);
218+
219+
const runtime = yield* runtimeRepository.getByThreadId({ threadId });
220+
assert.equal(Option.isSome(runtime), true);
221+
if (Option.isSome(runtime)) {
222+
// A fresh timestamp replaces the stale one; other fields are preserved.
223+
assert.notEqual(runtime.value.lastSeenAt, "2026-01-01T00:00:00.000Z");
224+
assert.equal(runtime.value.status, "running");
225+
assert.equal(runtime.value.providerName, "claudeAgent");
226+
}
227+
}));
228+
229+
it("touchLastSeen does not resurrect a stopped binding", () =>
230+
Effect.gen(function* () {
231+
const directory = yield* ProviderSessionDirectory;
232+
const runtimeRepository = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository;
233+
const threadId = ThreadId.make("thread-touch-stopped");
234+
const stoppedAt = "2026-01-01T00:00:00.000Z";
235+
236+
yield* runtimeRepository.upsert({
237+
threadId,
238+
providerName: "claudeAgent",
239+
providerInstanceId: null,
240+
adapterKey: "claudeAgent",
241+
runtimeMode: "full-access",
242+
status: "stopped",
243+
lastSeenAt: stoppedAt,
244+
resumeCursor: null,
245+
runtimePayload: null,
246+
});
247+
248+
yield* directory.touchLastSeen(threadId);
249+
250+
const runtime = yield* runtimeRepository.getByThreadId({ threadId });
251+
assert.equal(Option.isSome(runtime), true);
252+
if (Option.isSome(runtime)) {
253+
// Stopped rows are left untouched — the reaper already killed them.
254+
assert.equal(runtime.value.lastSeenAt, stoppedAt);
255+
assert.equal(runtime.value.status, "stopped");
256+
}
257+
}));
258+
199259
it("resets adapterKey to the new provider when provider changes without an explicit adapter key", () =>
200260
Effect.gen(function* () {
201261
const directory = yield* ProviderSessionDirectory;

‎apps/server/src/provider/Layers/ProviderSessionDirectory.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,12 +182,21 @@ const makeProviderSessionDirectory = Effect.gen(function* () {
182182
),
183183
);
184184

185+
const touchLastSeen: ProviderSessionDirectoryShape["touchLastSeen"] = (threadId) =>
186+
Effect.gen(function* () {
187+
const now = DateTime.formatIso(yield* DateTime.now);
188+
yield* repository
189+
.touchLastSeen({ threadId, lastSeenAt: now })
190+
.pipe(Effect.mapError(toPersistenceError("ProviderSessionDirectory.touchLastSeen")));
191+
});
192+
185193
return {
186194
upsert,
187195
getProvider,
188196
getBinding,
189197
listThreadIds,
190198
listBindings,
199+
touchLastSeen,
191200
} satisfies ProviderSessionDirectoryShape;
192201
});
193202

‎apps/server/src/provider/Services/ProviderSessionDirectory.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,17 @@ export interface ProviderSessionDirectoryShape {
6262
ReadonlyArray<ProviderRuntimeBindingWithMetadata>,
6363
ProviderSessionDirectoryPersistenceError
6464
>;
65+
66+
/**
67+
* Bump only `last_seen_at` for a live (non-stopped) binding.
68+
*
69+
* Used to keep a session's inactivity clock fresh from background runtime
70+
* activity without rewriting the full binding. No-op if the row is absent
71+
* or already stopped.
72+
*/
73+
readonly touchLastSeen: (
74+
threadId: ThreadId,
75+
) => Effect.Effect<void, ProviderSessionDirectoryWriteError>;
6576
}
6677

6778
export class ProviderSessionDirectory extends Context.Service<

0 commit comments

Comments
 (0)