Skip to content

Commit 34ac145

Browse files
committed
fix(server): keep unanswered questions available after restart
Preserve saved native questions across server shutdown and crash recovery by accepting their answers through the existing message path. Keep blocking questions out of automatic restart continuation and support multiple selections. Implemented and verified with GPT-6.1-Sol and GPT-6-Astra through the Codex harness.
1 parent cfafb4b commit 34ac145

11 files changed

Lines changed: 561 additions & 310 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7403,15 +7403,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
74037403
const replies: string[] = [];
74047404
for (const question of approvalTurnItem.questions) {
74057405
const answer = command.answers?.[question.id];
7406-
if (typeof answer !== "string" || answer.trim().length === 0) {
7406+
const text =
7407+
typeof answer === "string"
7408+
? answer.trim()
7409+
: question.multiSelect &&
7410+
Array.isArray(answer) &&
7411+
answer.every((value) => typeof value === "string" && value.trim().length > 0)
7412+
? answer.join(", ")
7413+
: "";
7414+
if (text.length === 0) {
74077415
if (question.required === false) continue;
74087416
return yield* new OrchestratorDispatchError({
74097417
commandId: command.commandId,
74107418
commandType: command.type,
74117419
cause: "Answer each question before sending.",
74127420
});
74137421
}
7414-
replies.push(`${question.question}\n${answer.trim()}`);
7422+
replies.push(`${question.question}\n${text}`);
74157423
}
74167424
if (replies.length === 0) {
74177425
return yield* new OrchestratorDispatchError({

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4135,6 +4135,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
41354135
WHERE thread_id = ${threadId} AND origin = 'provider_native'
41364136
AND status IN ('pending', 'running', 'waiting')
41374137
)
4138+
OR node.node_id IN (
4139+
SELECT node_id FROM orchestration_v2_projection_runtime_requests
4140+
WHERE thread_id = ${threadId} AND kind = 'user_input' AND status = 'pending'
4141+
)
41384142
)
41394143
ORDER BY COALESCE(node.started_at, ''), node.node_id ASC
41404144
`,
@@ -4270,6 +4274,12 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
42704274
AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
42714275
)
42724276
OR item.type IN ('command_execution', 'dynamic_tool', 'subagent')
4277+
OR (
4278+
item.type = 'user_input_request' AND item.node_id IN (
4279+
SELECT node_id FROM orchestration_v2_projection_runtime_requests
4280+
WHERE thread_id = ${threadId} AND kind = 'user_input' AND status = 'pending'
4281+
)
4282+
)
42734283
OR (
42744284
item.run_id IS NULL
42754285
AND item.node_id IN (

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,17 @@ export const make = Effect.gen(function* () {
214214
const requests = projection.runtimeRequests.filter(
215215
(request) => request.status === "pending" && request.responseCapability.type !== "message",
216216
);
217+
const savedQuestionIds = new Set(
218+
(projection.turnItems ?? []).flatMap((item) =>
219+
item.type === "user_input_request" && item.questions.length > 0 ? [item.requestId] : [],
220+
),
221+
);
222+
// The native reply channel is gone, but an answer can resume the conversation as a message.
223+
for (const request of requests) {
224+
if (request.kind === "user_input" && savedQuestionIds.has(request.id)) {
225+
messageRequestNodeIds.add(request.nodeId);
226+
}
227+
}
217228
// Delegated task rows, items and nodes stay open: the child settles them.
218229
const delegatedTaskNodeIds = new Set<string>([
219230
...(projection.subagents ?? []).filter(isAppOwnedDelegation).map((subagent) => subagent.id),
@@ -304,6 +315,7 @@ export const make = Effect.gen(function* () {
304315
});
305316
}
306317
for (const request of requests) {
318+
const preserveQuestion = request.kind === "user_input" && savedQuestionIds.has(request.id);
307319
events.push({
308320
id: yield* allocateEventId(),
309321
type: "runtime-request.updated",
@@ -312,12 +324,14 @@ export const make = Effect.gen(function* () {
312324
occurredAt: now,
313325
payload: {
314326
...request,
315-
status: trigger === "startup" ? "expired" : "cancelled",
316-
responseCapability: {
317-
type: "not_resumable",
318-
reason: `The server ${trigger === "startup" ? "restarted" : "shut down"} before this runtime request was resolved.`,
319-
},
320-
resolvedAt: now,
327+
status: preserveQuestion ? "pending" : trigger === "startup" ? "expired" : "cancelled",
328+
responseCapability: preserveQuestion
329+
? { type: "message" }
330+
: {
331+
type: "not_resumable",
332+
reason: `The server ${trigger === "startup" ? "restarted" : "shut down"} before this runtime request was resolved.`,
333+
},
334+
resolvedAt: preserveQuestion ? null : now,
321335
},
322336
});
323337
}
@@ -646,7 +660,9 @@ export const make = Effect.gen(function* () {
646660
return {
647661
terminalizedRuns: runs.length,
648662
stoppedSessions,
649-
closedRequests: requests.length,
663+
closedRequests: requests.filter(
664+
(request) => request.kind !== "user_input" || !savedQuestionIds.has(request.id),
665+
).length,
650666
retiredEffects,
651667
};
652668
},

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 102 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -3896,77 +3896,112 @@ it.effect("ProviderSessionManagerV2 settles a request the event pump persists du
38963896
}),
38973897
);
38983898

3899-
it.effect("ProviderSessionManagerV2 terminalizes a pending input transcript item on release", () =>
3900-
Effect.gen(function* () {
3901-
const state = yield* Ref.make(emptyState);
3902-
const effect = Effect.gen(function* () {
3903-
const eventSink = yield* EventSink.EventSinkV2;
3904-
const idAllocator = yield* IdAllocator.IdAllocatorV2;
3905-
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
3906-
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
3907-
const now = yield* DateTime.now;
3908-
const projectId = yield* idAllocator.allocate.project({
3909-
fixtureName: "provider-session-manager-request-expire",
3910-
});
3911-
const threadId = yield* idAllocator.allocate.thread({
3912-
fixtureName: "provider-session-manager-request-expire",
3913-
projectId,
3914-
});
3915-
const providerSessionId = yield* idAllocator.allocate.providerSession({
3916-
providerInstanceId: modelSelection.instanceId,
3917-
threadId,
3918-
});
3919-
const providerThread = makeProviderThread({
3920-
idAllocator,
3921-
threadId,
3922-
providerSessionId,
3923-
now,
3924-
});
3899+
it.effect.each(["runtime_error", "manual_shutdown", "server_shutdown"] as const)(
3900+
"ProviderSessionManagerV2 applies question recovery policy during %s release",
3901+
(reason) =>
3902+
Effect.gen(function* () {
3903+
const state = yield* Ref.make(emptyState);
3904+
const effect = Effect.gen(function* () {
3905+
const eventSink = yield* EventSink.EventSinkV2;
3906+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
3907+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
3908+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
3909+
const now = yield* DateTime.now;
3910+
const projectId = yield* idAllocator.allocate.project({
3911+
fixtureName: "provider-session-manager-request-expire",
3912+
});
3913+
const threadId = yield* idAllocator.allocate.thread({
3914+
fixtureName: "provider-session-manager-request-expire",
3915+
projectId,
3916+
});
3917+
const providerSessionId = yield* idAllocator.allocate.providerSession({
3918+
providerInstanceId: modelSelection.instanceId,
3919+
threadId,
3920+
});
3921+
const providerThread = makeProviderThread({
3922+
idAllocator,
3923+
threadId,
3924+
providerSessionId,
3925+
now,
3926+
});
39253927

3926-
yield* eventSink.write({
3927-
events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
3928-
});
3929-
const pendingRequest = yield* makePendingRuntimeRequestEvents({
3930-
idAllocator,
3931-
threadId,
3932-
providerSessionId,
3933-
providerThread,
3934-
now,
3935-
});
3936-
yield* eventSink.write({
3937-
events: pendingRequest.events.map((event) =>
3938-
event.type === "turn-item.updated"
3939-
? { ...event, payload: { ...event.payload, type: "user_input_request", questions: [] } }
3940-
: event,
3941-
),
3942-
});
3943-
yield* manager.open({
3944-
threadId,
3945-
providerSessionId,
3946-
modelSelection,
3947-
runtimePolicy,
3948-
});
3949-
yield* manager.release({
3950-
providerSessionId,
3951-
reason: "runtime_error",
3952-
detail: "process exited",
3953-
});
3928+
yield* eventSink.write({
3929+
events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
3930+
});
3931+
const pendingRequest = yield* makePendingRuntimeRequestEvents({
3932+
idAllocator,
3933+
threadId,
3934+
providerSessionId,
3935+
providerThread,
3936+
now,
3937+
});
3938+
yield* eventSink.write({
3939+
events: pendingRequest.events.map((event) =>
3940+
event.type === "turn-item.updated"
3941+
? {
3942+
...event,
3943+
payload: {
3944+
...event.payload,
3945+
type: "user_input_request",
3946+
questions: [
3947+
{
3948+
id: "lookback",
3949+
header: "Lookback",
3950+
question: "How far back?",
3951+
options: [],
3952+
},
3953+
],
3954+
},
3955+
}
3956+
: event.type === "runtime-request.updated"
3957+
? { ...event, payload: { ...event.payload, kind: "user_input" } }
3958+
: event.type === "node.updated" &&
3959+
event.payload.runtimeRequestId === pendingRequest.requestId
3960+
? { ...event, payload: { ...event.payload, kind: "user_input_request" } }
3961+
: event,
3962+
),
3963+
});
3964+
yield* manager.open({
3965+
threadId,
3966+
providerSessionId,
3967+
modelSelection,
3968+
runtimePolicy,
3969+
});
3970+
yield* manager.release({
3971+
providerSessionId,
3972+
reason,
3973+
detail: "process exited",
3974+
});
39543975

3955-
const projection = yield* projectionStore.getThreadProjection(threadId);
3956-
const request = projection.runtimeRequests.at(-1);
3957-
const requestNode = projection.nodes.find((node) => node.id === request?.nodeId);
3958-
const requestTurnItem = projection.turnItems.find(
3959-
(item) => item.type === "user_input_request" && item.requestId === request?.id,
3960-
);
3976+
const projection = yield* projectionStore.getThreadProjection(threadId);
3977+
const request = projection.runtimeRequests.at(-1);
3978+
const requestNode = projection.nodes.find((node) => node.id === request?.nodeId);
3979+
const requestTurnItem = projection.turnItems.find(
3980+
(item) => item.type === "user_input_request" && item.requestId === request?.id,
3981+
);
39613982

3962-
assert.equal(request?.status, "expired");
3963-
assert.equal(request?.responseCapability.type, "not_resumable");
3964-
assert.equal(requestNode?.status, "failed");
3965-
assert.equal(requestTurnItem?.status, "failed");
3966-
});
3983+
if (reason === "server_shutdown") {
3984+
assert.equal(request?.status, "pending");
3985+
assert.equal(request?.responseCapability.type, "message");
3986+
assert.equal(requestNode?.status, "waiting");
3987+
assert.equal(requestTurnItem?.status, "waiting");
3988+
assert.equal(requestTurnItem?.type, "user_input_request");
3989+
if (requestTurnItem?.type === "user_input_request") {
3990+
assert.isUndefined(requestTurnItem.responseMode);
3991+
}
3992+
} else {
3993+
assert.equal(request?.status, reason === "runtime_error" ? "expired" : "cancelled");
3994+
assert.equal(request?.responseCapability.type, "not_resumable");
3995+
assert.equal(requestNode?.status, reason === "runtime_error" ? "failed" : "cancelled");
3996+
assert.equal(
3997+
requestTurnItem?.status,
3998+
reason === "runtime_error" ? "failed" : "cancelled",
3999+
);
4000+
}
4001+
});
39674002

3968-
yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
3969-
}),
4003+
yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
4004+
}),
39704005
);
39714006

39724007
it.effect("ProviderSessionManagerV2 persists session-scoped runtime requests without a run", () =>

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,16 @@ export const layerWithOptions = (
653653
);
654654

655655
for (const request of releasedRequests) {
656+
const turnItem = projection.turnItems.find(
657+
(item) =>
658+
(item.type === "approval_request" || item.type === "user_input_request") &&
659+
item.requestId === request.id,
660+
);
661+
const preserveQuestion =
662+
input.reason === "server_shutdown" &&
663+
request.kind === "user_input" &&
664+
turnItem?.type === "user_input_request" &&
665+
turnItem.questions.length > 0;
656666
events.push({
657667
id: yield* idAllocator.allocate.event({
658668
threadId,
@@ -665,17 +675,16 @@ export const layerWithOptions = (
665675
occurredAt: now,
666676
payload: {
667677
...request,
668-
status,
669-
responseCapability: {
670-
type: "not_resumable",
671-
reason,
672-
},
673-
resolvedAt: now,
678+
status: preserveQuestion ? "pending" : status,
679+
responseCapability: preserveQuestion
680+
? { type: "message" }
681+
: { type: "not_resumable", reason },
682+
resolvedAt: preserveQuestion ? null : now,
674683
},
675684
});
676685

677686
const requestNode = projection.nodes.find((node) => node.id === request.nodeId);
678-
if (requestNode !== undefined) {
687+
if (requestNode !== undefined && !preserveQuestion) {
679688
events.push({
680689
id: yield* idAllocator.allocate.event({
681690
threadId,
@@ -695,12 +704,7 @@ export const layerWithOptions = (
695704
});
696705
}
697706

698-
const turnItem = projection.turnItems.find(
699-
(item) =>
700-
(item.type === "approval_request" || item.type === "user_input_request") &&
701-
item.requestId === request.id,
702-
);
703-
if (turnItem !== undefined) {
707+
if (turnItem !== undefined && !preserveQuestion) {
704708
events.push({
705709
id: yield* idAllocator.allocate.event({
706710
threadId,

0 commit comments

Comments
 (0)