Skip to content

Commit 5258d9f

Browse files
fix(server): remote following keeps background work pending and ends on older peers
A wait bounded at a later message's run kept dropping provider-native background work, which names no run, so it could settle while the selected run's background subagent still ran. That work and run-less tasks now still hold the wait. An older peer ignores afterRunId and answers at once with the run already followed. The follower now ends following there instead of retrying forever, which could later record a scheduled or monitor run there as a follow-up. AI-assisted; verified with the tests and tools listed above.
1 parent 63ddfe3 commit 5258d9f

6 files changed

Lines changed: 73 additions & 13 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8900,8 +8900,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
89008900
task.status === "cancelled" ||
89018901
task.status === "interrupted" ||
89028902
task.remoteChild?.stoppedThere === false;
8903-
// A stop there also ends following the thread for later runs.
8904-
const unfollows = followsAfterRunId !== undefined && command.stoppedThere === "stopped";
8903+
// A stop there ends following the thread for later runs, as does a run-out report naming
8904+
// no last run: a peer that cannot report later runs.
8905+
const unfollows =
8906+
followsAfterRunId !== undefined &&
8907+
(command.stoppedThere === "stopped" || command.lastRunId === undefined);
89058908
if (
89068909
!unfollows &&
89078910
(task.remoteChild?.stoppedThere === true ||

‎apps/server/src/orchestration-v2/ThreadManagementService.test.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,7 @@ const settledRunsService = (
424424
threadId: ThreadId,
425425
runs: () => ReadonlyArray<OrchestrationV2Run>,
426426
messages: ReadonlyArray<object> = [],
427+
providerThreads: ReadonlyArray<object> = [],
427428
) =>
428429
ThreadManagementService.ThreadManagementService.pipe(
429430
Effect.provide(
@@ -443,7 +444,7 @@ const settledRunsService = (
443444
runs: runs(),
444445
messages,
445446
subagents: [],
446-
providerThreads: [],
447+
providerThreads,
447448
}) as unknown as OrchestrationV2ThreadProjection,
448449
),
449450
}),
@@ -693,6 +694,20 @@ it.effect("waitForThread on a run answers with its own work, not a later message
693694
expect(yield* resultOf("run:r1")).toMatchObject({ resultRun: { id: "run:r1" } });
694695
expect(yield* resultOf("run:r2")).toMatchObject({ resultRun: { id: "run:r2-wake" } });
695696
expect(yield* resultOf("run:r3")).toMatchObject({ resultRun: { id: "run:r3" } });
697+
698+
// Background work names no run, so a later message's run does not settle it for run 1.
699+
const pending = yield* settledRunsService(projectId, threadId, () => runs, messages, [
700+
{ pendingBackgroundTasks: [{ taskId: "subagent", kind: "subagent" }] },
701+
]);
702+
expect(
703+
yield* pending.waitForThread({
704+
projectId,
705+
threadId,
706+
runId: RunId.make("run:r1"),
707+
timeoutMs: 1,
708+
untilDelegatedWorkSettled: true,
709+
}),
710+
).toMatchObject({ timedOut: true });
696711
}),
697712
);
698713

‎apps/server/src/orchestration-v2/ThreadManagementService.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1121,16 +1121,16 @@ const make = Effect.gen(function* () {
11211121
const runs = yield* ownWork(records.runs, selected);
11221122
const bounded = runs.length < records.runs.length;
11231123
const own = new Set(runs.map((run) => run.id));
1124-
// Past new work, only the tasks its own runs started count; background work names no run.
1124+
// Past new work, a later run's tasks no longer count. Background work names no run, so it
1125+
// may be this run's own and still holds the wait.
11251126
const progress = delegatedTaskProgress(
11261127
bounded
11271128
? {
11281129
...records,
11291130
runs,
11301131
subagents: records.subagents.filter(
1131-
(task) => task.runId !== null && own.has(task.runId),
1132+
(task) => task.runId === null || own.has(task.runId),
11321133
),
1133-
providerThreads: [],
11341134
}
11351135
: records,
11361136
);

‎apps/server/src/peer/RemoteDelegation.test.ts‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ interface BoxThread {
118118
readonly nestedSettled: Deferred.Deferred<void>;
119119
/** Waiting there fails with this, as a thread deleted there does. */
120120
readonly gone: Ref.Ref<boolean>;
121+
/** The box runs an older version, whose wait ignores `afterRunId`. */
122+
readonly ignoresAfterRunId: Ref.Ref<boolean>;
121123
readonly interrupted: Ref.Ref<number>;
122124
/** What each interrupt or stop there did, once per command id. */
123125
readonly stops: Ref.Ref<ReadonlyArray<"interrupt" | "thread.stop" | "delegated-tasks">>;
@@ -164,6 +166,7 @@ const makeBoxThread = Effect.gen(function* () {
164166
nestedWaiting: yield* Deferred.make<void>(),
165167
nestedSettled: yield* Deferred.make<void>(),
166168
gone: yield* Ref.make(false),
169+
ignoresAfterRunId: yield* Ref.make(false),
167170
interrupted: yield* Ref.make(0),
168171
stops: yield* Ref.make<ReadonlyArray<"interrupt" | "thread.stop" | "delegated-tasks">>([]),
169172
runless: yield* Ref.make(false),
@@ -293,7 +296,7 @@ const boxToolkitLayer = (thread: BoxThread) => {
293296
waitForThread: (input) =>
294297
Effect.gen(function* () {
295298
// The later run starts after the task's own; none starts after it.
296-
if (input.afterRunId !== undefined) {
299+
if (input.afterRunId !== undefined && !(yield* Ref.get(thread.ignoresAfterRunId))) {
297300
const started = yield* Deferred.await(
298301
input.afterRunId === boxRunId ? thread.laterStarted : Deferred.makeUnsafe<void>(),
299302
).pipe(Effect.timeoutOption(input.timeoutMs));
@@ -1397,6 +1400,37 @@ it.effect("a task's result is its own run's whole last reply, not a later run's"
13971400
).pipe(Effect.provide(NodeHttpServer.layerTest)),
13981401
);
13991402

1403+
it.effect("a finished task stops following its thread on a box that cannot report later runs", () =>
1404+
Effect.scoped(
1405+
Effect.gen(function* () {
1406+
const thread = yield* makeBoxThread;
1407+
const b = yield* serveBox(thread);
1408+
const offers = yield* SubscriptionRef.make<
1409+
ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1410+
>([]);
1411+
const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1412+
const a = yield* makeLaptop(database, offers);
1413+
yield* linkTo(a.links, b, "full-access");
1414+
yield* seedParent(a);
1415+
yield* Ref.set(thread.ignoresAfterRunId, true);
1416+
const delegated = yield* delegateToBox(a);
1417+
yield* Ref.set(thread.status, "completed");
1418+
yield* Deferred.succeed(thread.finished, undefined);
1419+
1420+
// Its wait answers at once with the task's own run, so following ends instead of retrying.
1421+
const unfollowed = yield* taskUpdate(
1422+
a,
1423+
(task) =>
1424+
task.id === delegated.taskId &&
1425+
task.result !== null &&
1426+
task.remoteChild?.followsAfterRunId === undefined,
1427+
);
1428+
expect(unfollowed?.remoteChild).toMatchObject({ stoppedThere: true });
1429+
expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]);
1430+
}),
1431+
).pipe(Effect.provide(NodeHttpServer.layerTest)),
1432+
);
1433+
14001434
it.effect("a later run on a finished task's thread there is a new task that wakes the parent", () =>
14011435
Effect.scoped(
14021436
Effect.gen(function* () {

‎apps/server/src/peer/RemoteDelegation.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -744,13 +744,20 @@ const make = Effect.gen(function* () {
744744
.pipe(Effect.raceFirst(changed));
745745
// Stopped or taken over here meanwhile: the next look sees it.
746746
if ("endedHere" in waited || waited.timedOut) return false;
747-
// An older peer ignores `afterRunId` and reports the run already followed at once,
748-
// so it is retried on the back-off schedule instead of asked again straight away.
747+
// An older peer ignores `afterRunId` and reports the run already followed at once. Asked
748+
// again it would answer for whatever runs there next, say a scheduled task's, so following ends.
749749
if (waited.runId === null || waited.runId === afterRunId) {
750-
return yield* new OrchestratorMcpFailure({
751-
code: "orchestration_error",
752-
message: `${remote.label} cannot report a later run on a thread yet.`,
750+
yield* Effect.logWarning("A linked environment cannot report later runs on a thread", {
751+
taskId: task.id,
753752
});
753+
yield* complete(
754+
parentThreadId,
755+
task.id,
756+
terminalStatus(task.status),
757+
task.result ?? noReply,
758+
"ran-out",
759+
);
760+
return yield* settled(parentThreadId, task.id);
754761
}
755762
const laterRunId = waited.runId;
756763
const stillFollows = readTask(parentThreadId, task.id).pipe(

‎docs/internals/remote.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ it starts and takes over following. Following belongs to the parent, environment
141141
together: a Stop or cancel here of any task on that thread ends it for all of
142142
them under the parent's lock, before the stop is sent there, and a handoff that
143143
loses that race is refused. That is one forwarded wait per followed thread
144-
until then, or until the thread, parent or link is gone.
144+
until then, or until the thread, parent or link is gone. An older peer that ignores
145+
`afterRunId` answers at once with that run, which ends following there.
145146

146147
The link is routing, not isolation: an agent the link starts runs as the
147148
receiving environment's user. **A full-access link is therefore not fenced**:

0 commit comments

Comments
 (0)