Skip to content

Commit e578131

Browse files
fix(server): idle reaper no longer kills in-flight background agent work
ProviderSessionReaper stopped any provider session after 30 min of last_seen_at inactivity, with session.activeTurnId as its only busy guard. Background work (dynamic workflows, subagents, background shells) outlives the foreground turn, and nothing refreshed last_seen_at from runtime activity - not even later foreground or synthetic follow-up turns - so live sessions were torn down mid-run and their background orchestration silently lost. - Refresh last_seen_at from provider runtime events via a targeted touchLastSeen (throttled per thread, armed only on a successful write of a live row, never resurrecting stopped rows, bounded by a timeout so a stalled write cannot block the event pump). - Track started-but-not-terminal background tasks in memory and have the reaper skip threads with outstanding tasks, even if a task is temporarily quiet; a staleness ceiling (3x the reap threshold of total task-event silence) settles presumed-dead entries so a stranded entry cannot pin a thread forever. - On stopSession/runStopAll/session.exited, publish synthetic terminal task.completed(status: "stopped") events for whatever is still outstanding, so a torn-down task records a terminal activity instead of leaving an orphaned spinner. The activeTurnId guard is preserved unchanged. Regression tests cover the reaper guard (not reaped while a task is outstanding, reapable after it settles), touch semantics, throttling, and the backstop. Closes #4198 Co-authored-by: Chamaru Amasara <[email protected]>
1 parent 9dd425b commit e578131

14 files changed

Lines changed: 1030 additions & 8 deletions

‎apps/server/src/orchestration/Layers/CheckpointReactor.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ function createProviderServiceHarness(
123123
},
124124
}),
125125
rollbackConversation,
126+
hasOutstandingBackgroundTasks: () => Effect.succeed(false),
126127
get streamEvents() {
127128
return Stream.fromPubSub(runtimeEventPubSub);
128129
},

‎apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,7 @@ describe("ProviderCommandReactor", () => {
338338
});
339339
},
340340
rollbackConversation: () => unsupported(),
341+
hasOutstandingBackgroundTasks: () => Effect.succeed(false),
341342
get streamEvents() {
342343
return Stream.fromPubSub(runtimeEventPubSub);
343344
},

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ function createProviderServiceHarness() {
121121
});
122122
},
123123
rollbackConversation: () => unsupported(),
124+
hasOutstandingBackgroundTasks: () => Effect.succeed(false),
124125
get streamEvents() {
125126
return Stream.fromPubSub(runtimeEventPubSub);
126127
},

‎apps/server/src/persistence/ProviderSessionRuntime.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ export type GetProviderSessionRuntimeInput = typeof GetProviderSessionRuntimeInp
5858
export const DeleteProviderSessionRuntimeInput = Schema.Struct({ threadId: ThreadId });
5959
export type DeleteProviderSessionRuntimeInput = typeof DeleteProviderSessionRuntimeInput.Type;
6060

61+
export const TouchLastSeenInput = Schema.Struct({
62+
threadId: ThreadId,
63+
lastSeenAt: IsoDateTime,
64+
});
65+
export type TouchLastSeenInput = typeof TouchLastSeenInput.Type;
66+
6167
/**
6268
* ProviderSessionRuntimeRepository - Service tag for provider runtime persistence.
6369
*/
@@ -93,6 +99,21 @@ export class ProviderSessionRuntimeRepository extends Context.Service<
9399
ProviderSessionRuntimeRepositoryError
94100
>;
95101

102+
/**
103+
* Bump only `last_seen_at` for an existing, non-stopped row.
104+
*
105+
* A targeted update used to keep a session's inactivity clock fresh from
106+
* background runtime activity (e.g. a running dynamic workflow) without
107+
* rewriting the full runtime payload. Rows in `stopped` status are left
108+
* untouched so a reaped session is never resurrected.
109+
*
110+
* Returns whether a live row was actually updated, so callers can tell a
111+
* real refresh from a no-op against an absent or stopped row.
112+
*/
113+
readonly touchLastSeen: (
114+
input: TouchLastSeenInput,
115+
) => Effect.Effect<boolean, ProviderSessionRuntimeRepositoryError>;
116+
96117
/**
97118
* Delete provider runtime state by canonical thread id.
98119
*/
@@ -226,6 +247,22 @@ export const make = Effect.gen(function* () {
226247
`,
227248
});
228249

250+
// `RETURNING` reports which rows the update actually matched: an absent or
251+
// already-stopped row yields none, and the caller needs that distinction to
252+
// avoid treating a no-op as a successful refresh.
253+
const touchLastSeenRow = SqlSchema.findAll({
254+
Request: TouchLastSeenInput,
255+
Result: Schema.Struct({ threadId: Schema.String }),
256+
execute: ({ threadId, lastSeenAt }) =>
257+
sql`
258+
UPDATE provider_session_runtime
259+
SET last_seen_at = ${lastSeenAt}
260+
WHERE thread_id = ${threadId}
261+
AND status != 'stopped'
262+
RETURNING thread_id AS "threadId"
263+
`,
264+
});
265+
229266
const deleteRuntimeByThreadId = SqlSchema.void({
230267
Request: DeleteRuntimeRequestSchema,
231268
execute: ({ threadId }) =>
@@ -308,6 +345,18 @@ export const make = Effect.gen(function* () {
308345
),
309346
);
310347

348+
const touchLastSeen: ProviderSessionRuntimeRepository["Service"]["touchLastSeen"] = (input) =>
349+
touchLastSeenRow(input).pipe(
350+
Effect.mapError(
351+
toPersistenceSqlOrDecodeError(
352+
"ProviderSessionRuntimeRepository.touchLastSeen:query",
353+
"ProviderSessionRuntimeRepository.touchLastSeen:encodeRequest",
354+
{ threadId: input.threadId },
355+
),
356+
),
357+
Effect.map((rows) => rows.length > 0),
358+
);
359+
311360
const deleteByThreadId: ProviderSessionRuntimeRepository["Service"]["deleteByThreadId"] = (
312361
input,
313362
) =>
@@ -326,6 +375,7 @@ export const make = Effect.gen(function* () {
326375
upsert,
327376
getByThreadId,
328377
list,
378+
touchLastSeen,
329379
deleteByThreadId,
330380
} satisfies ProviderSessionRuntimeRepository["Service"];
331381
});

‎apps/server/src/provider/Layers/CodexAdapter.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ const providerSessionDirectoryTestLayer = Layer.succeed(ProviderSessionDirectory
218218
getBinding: () => Effect.succeed(Option.none()),
219219
listThreadIds: () => Effect.succeed([]),
220220
listBindings: () => Effect.succeed([]),
221+
touchLastSeen: () => Effect.succeed(true),
221222
});
222223

223224
const validationRuntimeFactory = makeRuntimeFactory();

‎apps/server/src/provider/Layers/OpenCodeAdapter.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ const providerSessionDirectoryTestLayer = Layer.succeed(ProviderSessionDirectory
238238
getBinding: () => Effect.succeed(Option.none()),
239239
listThreadIds: () => Effect.succeed([]),
240240
listBindings: () => Effect.succeed([]),
241+
touchLastSeen: () => Effect.succeed(true),
241242
});
242243

243244
// The adapter now receives its settings as a plain argument (the old design

0 commit comments

Comments
 (0)