Skip to content

Commit 7111501

Browse files
authored
fix(server): wake threads whose subagents' background work a restart cut (#19750)
1 parent 11ce406 commit 7111501

14 files changed

Lines changed: 640 additions & 32 deletions

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4694,6 +4694,33 @@ describe("ClaudeAdapterV2 background wake turns", () => {
46944694
WAKE_TASK_ID,
46954695
);
46964696

4697+
// A subagent's own background shell keeps the subagent waiting on it.
4698+
yield* Queue.offer(
4699+
harness.sdkMessages,
4700+
claudeSdkFrame({
4701+
type: "system",
4702+
subtype: "background_tasks_changed",
4703+
tasks: [
4704+
{
4705+
task_id: "subagent-shell",
4706+
task_type: "local_bash",
4707+
description: "sleep 150",
4708+
parent_task_id: "agent-1",
4709+
},
4710+
],
4711+
uuid: "00000000-0000-4000-8000-000000000203",
4712+
session_id: WAKE_NATIVE_SESSION,
4713+
}),
4714+
);
4715+
yield* awaitUntil(
4716+
() =>
4717+
providerThreadRosterEvents(harness.events).some(
4718+
(event) =>
4719+
event.providerThread.pendingBackgroundTasks?.[0]?.parentTaskId === "agent-1",
4720+
),
4721+
"subagent shell roster",
4722+
);
4723+
46974724
yield* Queue.offer(harness.sdkMessages, emptyRoster);
46984725
yield* awaitUntil(
46994726
() =>

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2043,6 +2043,8 @@ function claudePendingBackgroundTask(input: {
20432043
// that started it tells it apart from a background Bash command.
20442044
readonly startedByMonitor: boolean;
20452045
readonly description: string | undefined;
2046+
// Undeclared SDK field: the subagent whose own background work this is.
2047+
readonly parentTaskId: unknown;
20462048
}): OrchestrationV2PendingBackgroundTask {
20472049
return {
20482050
taskId: input.taskId,
@@ -2054,6 +2056,9 @@ function claudePendingBackgroundTask(input: {
20542056
...(input.description !== undefined && input.description.trim().length > 0
20552057
? { description: input.description }
20562058
: {}),
2059+
...(typeof input.parentTaskId === "string" && input.parentTaskId.trim().length > 0
2060+
? { parentTaskId: input.parentTaskId }
2061+
: {}),
20572062
};
20582063
}
20592064

@@ -2105,6 +2110,7 @@ function parseClaudeBackgroundTaskEntry(
21052110
taskType,
21062111
startedByMonitor: monitorTasks.has(taskId),
21072112
description: typeof description === "string" ? description : undefined,
2113+
parentTaskId: Reflect.get(entry, "parent_task_id"),
21082114
});
21092115
}
21102116

@@ -6285,6 +6291,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
62856291
}),
62866292
description:
62876293
typeof message.description === "string" ? message.description : undefined,
6294+
parentTaskId: Reflect.get(message, "parent_task_id"),
62886295
}),
62896296
);
62906297
rosterChanged = true;

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,11 @@ import {
9292
import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
9393
import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
9494
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
95-
import { isRestartContinuationSource, runOutcome } from "./RestartBackgroundNote.ts";
95+
import {
96+
cancelledBeforeStart,
97+
isRestartContinuationSource,
98+
runOutcome,
99+
} from "./RestartBackgroundNote.ts";
96100
import {
97101
isBufferedProviderNotification,
98102
isUndeliveredMailboxSteer,
@@ -4569,7 +4573,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
45694573
projection.thread.deletedAt !== null ||
45704574
projection.thread.providerInstanceId !== source.providerInstanceId ||
45714575
projection.runs.some(
4572-
(run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source),
4576+
(run) =>
4577+
run.id !== source.id &&
4578+
run.status !== "queued" &&
4579+
!cancelledBeforeStart(run) &&
4580+
runRanAfter(run, source),
45734581
) ||
45744582
(yield* stopReachedRun(command, command.threadId, source.id))
45754583
) {

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -723,6 +723,221 @@ it.effect("reads the run that owns a background roster, not a queued or resumed
723723
}).pipe(Effect.provide(layerTest)),
724724
);
725725

726+
it.effect("wakes the run whose subagent lost its shell, past a wake cut before it started", () =>
727+
Effect.gen(function* () {
728+
const projections = yield* ProjectionStore.ProjectionStoreV2;
729+
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
730+
const sql = yield* SqlClient.SqlClient;
731+
const now = yield* DateTime.now;
732+
const threadId = yield* createThread("subagent-shell");
733+
const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
734+
const providerSessionId = ProviderSessionId.make(`session:${threadId}`);
735+
yield* projections.apply({
736+
id: EventId.make(`event:${threadId}:session`),
737+
type: "provider-session.attached",
738+
threadId,
739+
driver,
740+
providerInstanceId,
741+
occurredAt: now,
742+
payload: {
743+
id: providerSessionId,
744+
driver,
745+
providerInstanceId,
746+
status: "ready",
747+
cwd: "/workspace",
748+
model: modelSelection.model,
749+
capabilities: CodexProviderCapabilitiesV2,
750+
createdAt: now,
751+
updatedAt: now,
752+
lastError: null,
753+
},
754+
});
755+
const settledRun = yield* createRun(threadId, "completed", { providerThreadId });
756+
yield* createRun(threadId, "starting", {
757+
ordinal: 2,
758+
providerThreadId,
759+
startedAt: null,
760+
completedAt: null,
761+
});
762+
yield* projections.apply({
763+
id: EventId.make(`event:${threadId}:roster`),
764+
type: "provider-thread.updated",
765+
threadId,
766+
driver,
767+
providerInstanceId,
768+
occurredAt: now,
769+
payload: {
770+
id: providerThreadId,
771+
appThreadId: threadId,
772+
ownerNodeId: null,
773+
driver,
774+
providerInstanceId,
775+
providerSessionId,
776+
nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
777+
nativeConversationHeadRef: null,
778+
status: "idle",
779+
firstRunOrdinal: 1,
780+
lastRunOrdinal: 2,
781+
handoffIds: [],
782+
forkedFrom: null,
783+
createdAt: now,
784+
updatedAt: now,
785+
pendingBackgroundTasks: [
786+
{ taskId: "shell", description: "sleep 150", kind: "command", parentTaskId: "agent" },
787+
],
788+
},
789+
});
790+
791+
yield* recovery.reconcile("startup");
792+
793+
const effects = yield* sql<{ readonly effect_id: string }>`
794+
SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${threadId}
795+
`;
796+
assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${settledRun}` }]);
797+
const projection = yield* projections.getThreadProjection(threadId);
798+
assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork, [
799+
{ kind: "subagent", label: "sleep 150 (subagent agent)", id: "agent" },
800+
]);
801+
}).pipe(Effect.provide(layerRecoveryTest)),
802+
);
803+
804+
it.effect("wakes a settled run whose native subagent ran on its own provider thread", () =>
805+
Effect.gen(function* () {
806+
const projections = yield* ProjectionStore.ProjectionStoreV2;
807+
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
808+
const sql = yield* SqlClient.SqlClient;
809+
const now = yield* DateTime.now;
810+
const threadId = yield* createThread("native-subagent");
811+
const parentProviderThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
812+
const childProviderThreadId = ProviderThreadId.make(`provider-thread:${threadId}:child`);
813+
const providerSessionId = ProviderSessionId.make(`session:${threadId}`);
814+
const subagentId = NodeId.make(`node:${threadId}:subagent`);
815+
// An older thread on the same Codex session is recovered first and stops it.
816+
const sibling = yield* createThread("a-shared-session");
817+
yield* createRun(sibling, "completed");
818+
for (const boundThreadId of [sibling, threadId]) {
819+
yield* projections.apply({
820+
id: EventId.make(`event:${boundThreadId}:session`),
821+
type: "provider-session.attached",
822+
threadId: boundThreadId,
823+
driver,
824+
providerInstanceId,
825+
occurredAt: now,
826+
payload: {
827+
id: providerSessionId,
828+
driver,
829+
providerInstanceId,
830+
status: "ready",
831+
cwd: "/workspace",
832+
model: modelSelection.model,
833+
capabilities: CodexProviderCapabilitiesV2,
834+
createdAt: now,
835+
updatedAt: now,
836+
lastError: null,
837+
},
838+
});
839+
}
840+
const settledRun = yield* createRun(threadId, "completed", {
841+
providerThreadId: parentProviderThreadId,
842+
});
843+
for (const [id, nativeId] of [
844+
[parentProviderThreadId, "native-thread"],
845+
[childProviderThreadId, "agent-thread"],
846+
] as const) {
847+
yield* projections.apply({
848+
id: EventId.make(`event:${id}`),
849+
type: "provider-thread.updated",
850+
threadId,
851+
driver,
852+
providerInstanceId,
853+
occurredAt: now,
854+
payload: {
855+
id,
856+
appThreadId:
857+
id === parentProviderThreadId ? threadId : ThreadId.make(`${threadId}:child`),
858+
ownerNodeId: null,
859+
driver,
860+
providerInstanceId,
861+
providerSessionId,
862+
nativeThreadRef: { driver, nativeId, strength: "strong" },
863+
nativeConversationHeadRef: null,
864+
status: "idle",
865+
firstRunOrdinal: id === parentProviderThreadId ? 1 : null,
866+
lastRunOrdinal: id === parentProviderThreadId ? 1 : null,
867+
handoffIds: [],
868+
forkedFrom: null,
869+
createdAt: now,
870+
updatedAt: now,
871+
},
872+
});
873+
}
874+
const subagent = {
875+
threadId,
876+
runId: settledRun,
877+
origin: "provider_native",
878+
driver,
879+
providerInstanceId,
880+
childThreadId: null,
881+
prompt: "audit src",
882+
status: "running",
883+
result: null,
884+
startedAt: now,
885+
completedAt: null,
886+
updatedAt: now,
887+
} as const;
888+
yield* projections.apply({
889+
id: EventId.make(`event:${threadId}:subagent`),
890+
type: "subagent.updated",
891+
threadId,
892+
occurredAt: now,
893+
payload: {
894+
...subagent,
895+
id: subagentId,
896+
parentNodeId: NodeId.make(`node:${threadId}:root`),
897+
createdBy: "agent",
898+
providerThreadId: childProviderThreadId,
899+
nativeTaskRef: { driver, nativeId: "call_1:agent-thread", strength: "strong" },
900+
title: null,
901+
model: null,
902+
},
903+
});
904+
yield* projections.apply({
905+
id: EventId.make(`event:${threadId}:subagent-item`),
906+
type: "turn-item.updated",
907+
threadId,
908+
occurredAt: now,
909+
payload: {
910+
...subagent,
911+
id: TurnItemId.make(`item:${threadId}:subagent`),
912+
nodeId: subagentId,
913+
providerThreadId: childProviderThreadId,
914+
providerTurnId: null,
915+
nativeItemRef: { driver, nativeId: "call_1:agent-thread", strength: "strong" },
916+
parentItemId: null,
917+
ordinal: 1,
918+
type: "subagent",
919+
title: "audit",
920+
subagentId,
921+
},
922+
});
923+
924+
yield* recovery.reconcile("startup");
925+
926+
const effects = yield* sql<{ readonly effect_id: string }>`
927+
SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${threadId}
928+
`;
929+
assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${settledRun}` }]);
930+
const projection = yield* projections.getThreadProjection(threadId);
931+
assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork, [
932+
{
933+
kind: "subagent",
934+
label: "audit (id agent-thread)",
935+
id: TurnItemId.make(`item:${threadId}:subagent`),
936+
},
937+
]);
938+
}).pipe(Effect.provide(layerRecoveryTest)),
939+
);
940+
726941
it.effect("marks fork descendants unreadable when their source is missing or corrupt", () =>
727942
Effect.gen(function* () {
728943
const projections = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4103,6 +4103,11 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
41034103
WHERE ended.thread_id = ${threadId}
41044104
AND ended.provider_thread_id = roster.provider_thread_id
41054105
AND ended.status NOT IN ('queued', 'rolled_back')
4106+
-- A run that never started (see neverStarted) owns no work.
4107+
AND NOT (
4108+
json_type(ended.payload_json, '$.startedAt') = 'null'
4109+
AND json_type(ended.payload_json, '$.restartContinuationOfRunId') IS NULL
4110+
)
41064111
ORDER BY ended.completed_at IS NULL DESC, ended.completed_at DESC,
41074112
ended.ordinal DESC
41084113
LIMIT 1
@@ -4252,6 +4257,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
42524257
WHERE item.thread_id = ${threadId}
42534258
AND item.status IN ('pending', 'running', 'waiting')
42544259
)
4260+
-- A native subagent's item names its own provider thread; the
4261+
-- run that waits on it (see awaitingProviderThreadId) is woken.
4262+
OR provider_thread.provider_thread_id IN (
4263+
SELECT run.provider_thread_id
4264+
FROM orchestration_v2_projection_turn_items AS item
4265+
JOIN orchestration_v2_projection_runs AS run ON run.run_id = item.run_id
4266+
WHERE item.thread_id = ${threadId}
4267+
AND item.status IN ('pending', 'running', 'waiting')
4268+
)
42554269
)
42564270
ORDER BY COALESCE(provider_thread.first_run_ordinal, 0), provider_thread.provider_thread_id ASC
42574271
`,

0 commit comments

Comments
 (0)