Skip to content

Commit cd2b681

Browse files
fix(server): surface failed runtime request answer delivery (pingdotgg#24616)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 14b828e commit cd2b681

5 files changed

Lines changed: 266 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/EffectWorker.test.ts‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
ProviderTurnId,
1010
RunAttemptId,
1111
RunId,
12+
RuntimeRequestId,
1213
ThreadId,
1314
VcsProcessExitError,
1415
} from "@t3tools/contracts";
@@ -83,6 +84,7 @@ function layerExecutorFor(input: {
8384
readonly continueAfterRestart?: boolean;
8485
readonly interrupt?: ProviderTurnControlService.ProviderTurnControlServiceV2Shape["interrupt"];
8586
readonly rollback?: CheckpointRollbackService.CheckpointRollbackServiceV2Shape["execute"];
87+
readonly respond?: RuntimeRequestService.RuntimeRequestServiceV2["Service"]["respond"];
8688
}) {
8789
const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
8890
const layerDependencies = Layer.mergeAll(
@@ -142,7 +144,9 @@ function layerExecutorFor(input: {
142144
),
143145
Layer.succeed(
144146
RuntimeRequestService.RuntimeRequestServiceV2,
145-
RuntimeRequestService.RuntimeRequestServiceV2.of({ respond: () => Effect.void }),
147+
RuntimeRequestService.RuntimeRequestServiceV2.of({
148+
respond: input.respond ?? (() => Effect.void),
149+
}),
146150
),
147151
Layer.succeed(
148152
ThreadTitleRegenerationService.ThreadTitleRegenerationService,
@@ -269,6 +273,42 @@ it.effect.each([
269273
}),
270274
);
271275

276+
it.effect("does not report an answer whose last delivery attempt is interrupted", () =>
277+
Effect.gen(function* () {
278+
const now = yield* DateTime.now;
279+
const reported = yield* Ref.make(0);
280+
const layer = layerExecutorFor({
281+
events: yield* Ref.make<ReadonlyArray<string>>([]),
282+
respond: () => Effect.interrupt,
283+
threads: {
284+
dispatch: (command) =>
285+
Ref.update(reported, (count) =>
286+
command.type === "runtime-request.delivery.fail" ? count + 1 : count,
287+
).pipe(Effect.as({ sequence: 1, storedEvents: [] })),
288+
},
289+
});
290+
const exit = yield* Effect.gen(function* () {
291+
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
292+
return yield* Effect.exit(
293+
executor.execute(
294+
{
295+
...restartEffect(now, { type: "detach" }),
296+
request: {
297+
type: "runtime-request.respond",
298+
providerSessionId: oldSessionId,
299+
requestId: RuntimeRequestId.make("runtime-request:effect-worker-interrupted"),
300+
answers: { next: "Ship it" },
301+
},
302+
},
303+
{ willRetry: false },
304+
),
305+
);
306+
}).pipe(Effect.provide(layer));
307+
assert.isTrue(Exit.hasInterrupts(exit));
308+
assert.equal(yield* Ref.get(reported), 0);
309+
}),
310+
);
311+
272312
it.effect("requeues a claim when a pre-execution worker check fails", () =>
273313
Effect.gen(function* () {
274314
const now = DateTime.formatIso(yield* DateTime.now);

‎apps/server/src/orchestration-v2/EffectWorker.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,8 @@ export const layerExecutor: Layer.Layer<
375375
}),
376376
),
377377
);
378-
case "runtime-request.respond":
378+
case "runtime-request.respond": {
379+
const requestId = effect.request.requestId;
379380
return runtimeRequests
380381
.respond({
381382
threadId: effect.threadId,
@@ -392,6 +393,25 @@ export const layerExecutor: Layer.Layer<
392393
: { answers: effect.request.answers }),
393394
})
394395
.pipe(
396+
Effect.tapCause((cause) =>
397+
willRetry || Cause.hasInterruptsOnly(cause)
398+
? Effect.void
399+
: threads
400+
.dispatch({
401+
type: "runtime-request.delivery.fail",
402+
commandId: CommandId.make(`${effect.commandId}:delivery-failed`),
403+
threadId: effect.threadId,
404+
requestId,
405+
})
406+
.pipe(
407+
Effect.catchCause((recordCause) =>
408+
Effect.logWarning("Failed to record answer delivery failure", {
409+
effectId: effect.id,
410+
cause: recordCause,
411+
}),
412+
),
413+
),
414+
),
395415
Effect.mapError(
396416
(cause) =>
397417
new OrchestrationEffectExecutionError({
@@ -401,6 +421,7 @@ export const layerExecutor: Layer.Layer<
401421
}),
402422
),
403423
);
424+
}
404425
case "provider-thread.rollback":
405426
return checkpointRollback
406427
.execute({

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ import {
6161
RuntimeMode,
6262
ThreadLinkedPullRequest,
6363
ThreadId,
64-
type TurnItemId,
64+
TurnItemId,
6565
} from "@t3tools/contracts";
6666
import { modelSelectionsEqual } from "@t3tools/shared/model";
6767
import { recordThreadOpen } from "@t3tools/shared/paletteSearch";
@@ -490,6 +490,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId {
490490
case "queued-run.cancel":
491491
case "queued-run.edit":
492492
case "runtime-request.respond":
493+
case "runtime-request.delivery.fail":
493494
case "thread.user-input.dismiss":
494495
case "checkpoint.rollback":
495496
case "checkpoint.rollback.fail":
@@ -555,6 +556,8 @@ function delegatedTasksStopEffect(command: {
555556
}
556557

557558
const WORKSPACE_PREPARATION_INPUT = "Preparing workspace";
559+
const RUNTIME_REQUEST_DELIVERY_FAILED_MESSAGE =
560+
"T3 Code could not confirm that your answer reached the agent. If the run stays stuck, stop it and try again.";
558561

559562
function withoutPreparationResult(
560563
item: Extract<OrchestrationV2TurnItem, { readonly type: "command_execution" }>,
@@ -8005,6 +8008,65 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
80058008
},
80068009
);
80078010

8011+
/**
8012+
* Respond resolves the request before delivery, and a failed delivery cannot
8013+
* tell whether the provider is still waiting, so the answer stays recorded and
8014+
* the timeline shows the failure instead of a run spinning with no question.
8015+
*/
8016+
const dispatchRuntimeRequestDeliveryFail = Effect.fn(
8017+
"Orchestrator.dispatchRuntimeRequestDeliveryFail",
8018+
)(function* (
8019+
command: Extract<
8020+
OrchestrationV2InternalCommand,
8021+
{ readonly type: "runtime-request.delivery.fail" }
8022+
>,
8023+
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8024+
) {
8025+
const thread = yield* projectionStore
8026+
.getThread(command.threadId)
8027+
.pipe(mapDispatchError(command));
8028+
if (thread.deletedAt !== null) return;
8029+
const node = (yield* projectionStore
8030+
.getRuntimeResponseContext(command.threadId, command.requestId)
8031+
.pipe(mapDispatchError(command))).node;
8032+
if (node === undefined) return;
8033+
const now = yield* DateTime.now;
8034+
yield* emit(
8035+
events,
8036+
command,
8037+
)({
8038+
type: "turn-item.updated",
8039+
threadId: command.threadId,
8040+
...(node.runId === null ? {} : { runId: node.runId }),
8041+
nodeId: node.id,
8042+
providerInstanceId: thread.providerInstanceId,
8043+
occurredAt: now,
8044+
payload: {
8045+
id: TurnItemId.make(`turn-item:runtime-request-delivery-failure:${command.requestId}`),
8046+
threadId: command.threadId,
8047+
runId: node.runId,
8048+
nodeId: node.id,
8049+
providerThreadId: node.providerThreadId,
8050+
providerTurnId: node.providerTurnId,
8051+
nativeItemRef: null,
8052+
parentItemId: null,
8053+
ordinal: yield* nextTurnItemOrdinal({ thread }),
8054+
status: "failed",
8055+
title: "Answer delivery not confirmed",
8056+
startedAt: now,
8057+
completedAt: now,
8058+
updatedAt: now,
8059+
type: "error",
8060+
failure: {
8061+
class: "transport_error",
8062+
message: RUNTIME_REQUEST_DELIVERY_FAILED_MESSAGE,
8063+
code: null,
8064+
retryable: null,
8065+
},
8066+
},
8067+
});
8068+
});
8069+
80088070
const dispatchThreadUserInputDismiss = Effect.fn("Orchestrator.dispatchThreadUserInputDismiss")(
80098071
function* (
80108072
command: Extract<OrchestrationV2Command, { readonly type: "thread.user-input.dismiss" }>,
@@ -11697,6 +11759,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1169711759
case "checkpoint.rollback.fail":
1169811760
yield* dispatchCheckpointRollbackFail(command, events);
1169911761
break;
11762+
case "runtime-request.delivery.fail":
11763+
yield* dispatchRuntimeRequestDeliveryFail(command, events);
11764+
break;
1170011765
case "thread.background-work.settle":
1170111766
yield* dispatchBackgroundWorkSettle(command, events, effects);
1170211767
break;

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1231,6 +1231,136 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
12311231
}),
12321232
);
12331233

1234+
it.effect("reports an answer that never reaches its provider without reopening it", () =>
1235+
Effect.gen(function* () {
1236+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1237+
const eventSink = yield* EventSink.EventSinkV2;
1238+
const outbox = yield* EffectOutbox.EffectOutboxV2;
1239+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1240+
const now = yield* DateTime.now;
1241+
const threadId = ThreadId.make("runtime-lost-answer");
1242+
// Projected as attached, but no live session holds it, so every delivery fails.
1243+
const providerSessionId = ProviderSessionId.make("runtime-lost-answer-session");
1244+
const requestId = RuntimeRequestId.make("runtime-lost-answer-request");
1245+
const nodeId = NodeId.make("runtime-lost-answer-node");
1246+
yield* orchestrator.dispatch({
1247+
type: "thread.create",
1248+
commandId: CommandId.make("runtime-lost-answer-create"),
1249+
createdBy: "user",
1250+
creationSource: "web",
1251+
threadId,
1252+
projectId: ProjectId.make("runtime-lost-answer-project"),
1253+
title: "Lost answer",
1254+
modelSelection,
1255+
runtimeMode: "full-access",
1256+
interactionMode: "default",
1257+
branch: null,
1258+
worktreePath: process.cwd(),
1259+
});
1260+
yield* eventSink.write({
1261+
commandId: CommandId.make("runtime-lost-answer-seed"),
1262+
events: [
1263+
{
1264+
id: EventId.make("runtime-lost-answer-session-event"),
1265+
type: "provider-session.attached",
1266+
threadId,
1267+
driver,
1268+
occurredAt: now,
1269+
payload: {
1270+
id: providerSessionId,
1271+
driver,
1272+
providerInstanceId: modelSelection.instanceId,
1273+
status: "running",
1274+
cwd: process.cwd(),
1275+
model: modelSelection.model,
1276+
capabilities: CodexProviderCapabilitiesV2,
1277+
createdAt: now,
1278+
updatedAt: now,
1279+
lastError: null,
1280+
},
1281+
},
1282+
{
1283+
id: EventId.make("runtime-lost-answer-node-event"),
1284+
type: "node.updated",
1285+
threadId,
1286+
nodeId,
1287+
occurredAt: now,
1288+
payload: {
1289+
id: nodeId,
1290+
threadId,
1291+
runId: null,
1292+
parentNodeId: null,
1293+
rootNodeId: nodeId,
1294+
kind: "user_input_request",
1295+
status: "waiting",
1296+
countsForRun: false,
1297+
providerThreadId: null,
1298+
providerTurnId: null,
1299+
nativeItemRef: null,
1300+
runtimeRequestId: requestId,
1301+
checkpointScopeId: null,
1302+
startedAt: now,
1303+
completedAt: null,
1304+
},
1305+
},
1306+
{
1307+
id: EventId.make("runtime-lost-answer-request-event"),
1308+
type: "runtime-request.updated",
1309+
threadId,
1310+
nodeId,
1311+
occurredAt: now,
1312+
payload: {
1313+
id: requestId,
1314+
nodeId,
1315+
providerTurnId: null,
1316+
nativeRequestRef: null,
1317+
kind: "user_input",
1318+
status: "pending",
1319+
responseCapability: { type: "live", providerSessionId },
1320+
createdAt: now,
1321+
resolvedAt: null,
1322+
},
1323+
},
1324+
],
1325+
});
1326+
const commandId = CommandId.make("runtime-lost-answer-answer");
1327+
const answers = { color: "Blue" };
1328+
yield* orchestrator.dispatch({
1329+
type: "runtime-request.respond",
1330+
commandId,
1331+
threadId,
1332+
requestId,
1333+
answers,
1334+
});
1335+
const errors = orchestrator
1336+
.getThreadProjection(threadId)
1337+
.pipe(
1338+
Effect.map((projection) => projection.turnItems.filter((item) => item.type === "error")),
1339+
);
1340+
1341+
// A delivery that can still be retried reports nothing.
1342+
yield* worker.drain();
1343+
assert.deepEqual(yield* errors, []);
1344+
for (let attempt = 1; attempt < 5; attempt++) {
1345+
yield* TestClock.adjust("30 seconds");
1346+
yield* worker.drain();
1347+
}
1348+
1349+
const [deliveryEffect] = yield* outbox.listByCommandId(commandId);
1350+
assert.equal(deliveryEffect?.request.type, "runtime-request.respond");
1351+
assert.equal(deliveryEffect?.status, "failed");
1352+
const failed = yield* orchestrator.getThreadProjection(threadId);
1353+
const reported = failed.turnItems.filter((item) => item.type === "error");
1354+
assert.equal(reported.length, 1);
1355+
assert.equal(reported[0]?.status, "failed");
1356+
assert.equal(reported[0]?.nodeId, nodeId);
1357+
// The answer stays recorded; the user is not asked again.
1358+
const request = failed.runtimeRequests.find((row) => row.id === requestId);
1359+
assert.equal(request?.status, "resolved");
1360+
assert.deepEqual(request?.answers, answers);
1361+
}),
1362+
);
1363+
12341364
it.effect("dismisses message-capable questions directly and while settling", () =>
12351365
Effect.gen(function* () {
12361366
const orchestrator = yield* Orchestrator.OrchestratorV2;

‎packages/contracts/src/orchestrationV2.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3136,6 +3136,13 @@ const OrchestrationV2InternalCommand = Schema.Union([
31363136
requestId: CommandId,
31373137
message: TrimmedNonEmptyString,
31383138
}),
3139+
/** Delivering the answer to request `requestId` failed on its last attempt. */
3140+
Schema.Struct({
3141+
type: Schema.Literal("runtime-request.delivery.fail"),
3142+
commandId: CommandId,
3143+
threadId: ThreadId,
3144+
requestId: RuntimeRequestId,
3145+
}),
31393146
Schema.Struct({
31403147
type: Schema.Literal("thread.background-work.settle"),
31413148
commandId: CommandId,

0 commit comments

Comments
 (0)