Skip to content

Commit e62b133

Browse files
felixleopoldclaude
andcommitted
fix(server): settling a parent settles its tasks on other machines
A task delegated to another environment stays unsettled after its parent settles. Clients hide it under the parent only while they hold the parent, and a settled parent leaves their snapshots once 25 newer threads settle, so the task reappeared as an active thread. RemoteDelegation now settles each remote task's thread there when its parent settles, retrying for up to a day while that machine is asleep or the task is still running, and only while the parent stays settled. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5fe18d7 commit e62b133

2 files changed

Lines changed: 126 additions & 3 deletions

File tree

‎apps/server/src/peer/RemoteDelegation.test.ts‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import { CodexProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/CodexA
4141
import * as EventSink from "../orchestration-v2/EventSink.ts";
4242
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
4343
import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
44+
import * as ThreadSearch from "../orchestration-v2/ThreadSearch.ts";
4445
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
4546
import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";
4647
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
@@ -128,6 +129,9 @@ interface BoxThread {
128129
readonly sent: Ref.Ref<ReadonlyArray<string>>;
129130
/** Runs as the launch there is answered, before the laptop hears back. */
130131
readonly afterLaunch: Ref.Ref<Effect.Effect<void>>;
132+
/** Threads settled there, in order; `settledOnce` completes on the first. */
133+
readonly settled: Ref.Ref<ReadonlyArray<ThreadId>>;
134+
readonly settledOnce: Deferred.Deferred<void>;
131135
}
132136

133137
const makeBoxThread = Effect.gen(function* () {
@@ -162,6 +166,8 @@ const makeBoxThread = Effect.gen(function* () {
162166
releaseStop: yield* Deferred.make<void>(),
163167
sent: yield* Ref.make<ReadonlyArray<string>>([]),
164168
afterLaunch: yield* Ref.make<Effect.Effect<void>>(Effect.void),
169+
settled: yield* Ref.make<ReadonlyArray<ThreadId>>([]),
170+
settledOnce: yield* Deferred.make<void>(),
165171
} satisfies BoxThread;
166172
});
167173

@@ -410,6 +416,11 @@ const boxToolkitLayer = (thread: BoxThread) => {
410416
)
411417
: Effect.die(`The box does not expect ${command.type}`),
412418
stopDelegatedTasks: (input) => recordStop(input.commandId, "delegated-tasks"),
419+
settleThread: (input) =>
420+
Ref.update(thread.settled, (seen) => [...seen, input.threadId]).pipe(
421+
Effect.andThen(Deferred.succeed(thread.settledOnce, undefined)),
422+
Effect.as({ sequence: 0 }),
423+
),
413424
sendToThread: (input) =>
414425
Ref.update(thread.sent, (seen) => [...seen, input.text]).pipe(
415426
Effect.as({
@@ -460,15 +471,21 @@ const boxToolkitLayer = (thread: BoxThread) => {
460471
),
461472
),
462473
});
463-
return Layer.merge(
474+
return Layer.mergeAll(
464475
McpHttpServer.layerOrchestratorToolkit,
465476
McpHttpServer.layerProjectRegistration,
477+
McpHttpServer.layerThreadToolkit,
466478
).pipe(
467479
Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
468480
Layer.provide(NodeCrypto.layer),
469481
Layer.provide(layerBoxThreads),
470482
Layer.provide(layerBoxLaunches),
471-
Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})),
483+
Layer.provide(
484+
Layer.mergeAll(
485+
Layer.mock(RemoteDelegation.RemoteDelegation)({}),
486+
Layer.mock(ThreadSearch.ThreadSearch)({}),
487+
),
488+
),
472489
Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})),
473490
Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})),
474491
Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})),
@@ -1450,6 +1467,56 @@ it.effect("stopping the parent stops its task there", () =>
14501467
).pipe(Effect.provide(NodeHttpServer.layerTest)),
14511468
);
14521469

1470+
it.effect("settling the parent settles its task there", () =>
1471+
Effect.scoped(
1472+
Effect.gen(function* () {
1473+
const thread = yield* makeBoxThread;
1474+
const b = yield* serveBox(thread);
1475+
const offers = yield* SubscriptionRef.make<
1476+
ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1477+
>([]);
1478+
const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1479+
const a = yield* makeLaptop(database, offers);
1480+
yield* linkTo(a.links, b, "full-access");
1481+
yield* seedParent(a);
1482+
yield* a.remote.start();
1483+
1484+
const delegated = yield* delegateToBox(a);
1485+
yield* Ref.set(thread.status, "completed");
1486+
yield* Deferred.succeed(thread.finished, undefined);
1487+
yield* taskResult(a, delegated.taskId);
1488+
const now = yield* DateTime.now;
1489+
const run = (yield* a.orchestrator.getThreadProjection(parentThreadId)).runs.find(
1490+
(candidate) => candidate.id === parentRunId,
1491+
)!;
1492+
yield* a.sink.write({
1493+
commandId: CommandId.make("command:complete:laptop-parent"),
1494+
events: [
1495+
{
1496+
id: EventId.make("event:complete-run:laptop-parent"),
1497+
type: "run.updated",
1498+
threadId: parentThreadId,
1499+
runId: parentRunId,
1500+
nodeId: parentRootNode,
1501+
providerInstanceId: instanceId,
1502+
occurredAt: now,
1503+
payload: { ...run, status: "completed", completedAt: now },
1504+
},
1505+
],
1506+
});
1507+
expect(yield* Ref.get(thread.settled)).toEqual([]);
1508+
1509+
yield* a.orchestrator.dispatch({
1510+
type: "thread.settle",
1511+
commandId: CommandId.make("command:settle:laptop-parent"),
1512+
threadId: parentThreadId,
1513+
});
1514+
yield* Deferred.await(thread.settledOnce);
1515+
expect(yield* Ref.get(thread.settled)).toEqual([delegated.remoteChild.threadId]);
1516+
}),
1517+
).pipe(Effect.provide(NodeHttpServer.layerTest)),
1518+
);
1519+
14531520
/** A follower retry schedule the test steps: each failed attempt is offered, then waits for `retry`. */
14541521
const steppedRetries = Effect.gen(function* () {
14551522
const failed = yield* Queue.unbounded<void>();

‎apps/server/src/peer/RemoteDelegation.ts‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ import * as Effect from "effect/Effect";
2121
import * as Layer from "effect/Layer";
2222
import * as Option from "effect/Option";
2323
import * as Schedule from "effect/Schedule";
24+
import * as Schema from "effect/Schema";
2425
import * as Scope from "effect/Scope";
2526
import * as Stream from "effect/Stream";
2627
import type * as Tool from "effect/ai/Tool";
2728

2829
import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
2930
import { ProjectToolkit } from "../mcp/toolkits/project/tools.ts";
31+
import { ThreadToolkit } from "../mcp/toolkits/thread/tools.ts";
3032
import {
3133
resolveInteractionMode,
3234
resolveRuntimeMode,
@@ -45,6 +47,8 @@ import * as PeerLinks from "./PeerLinks.ts";
4547
const FOLLOW_WAIT_MS = 10 * 60 * 1_000;
4648
/** A linked environment that stops answering is retried, backing off to this. */
4749
const FOLLOW_MAX_BACKOFF = Duration.minutes(5);
50+
/** How long settling a task there keeps retrying, say while that machine sleeps. */
51+
const SETTLE_RETRY_WINDOW = Duration.days(1);
4852
/** How a follower retries; tests step it by hand. */
4953
export const FollowRetrySchedule = Context.Reference<Schedule.Schedule<unknown>>(
5054
"t3/peer/RemoteDelegation/FollowRetrySchedule",
@@ -98,6 +102,8 @@ export class RemoteDelegation extends Context.Service<
98102
}
99103
>()("t3/peer/RemoteDelegation") {}
100104

105+
const isMcpFailure = Schema.is(OrchestratorMcpFailure);
106+
101107
/** The other side refused because the thread there runs above this caller's modes. */
102108
const isModeRefusal = (error: OrchestratorMcpFailure) =>
103109
error.code === "runtime_mode_escalation_denied" ||
@@ -759,6 +765,54 @@ const make = Effect.gen(function* () {
759765
},
760766
);
761767

768+
/**
769+
* Settles the task's thread there while its parent here stays settled. Clients list that thread
770+
* under its parent only while they hold the parent, and a settled parent drops out of their
771+
* snapshots once newer threads settle, so an unsettled task would resurface as active there.
772+
*/
773+
const settleThere = Effect.fn("RemoteDelegation.settleThere")(
774+
function* (parentThreadId: ThreadId, taskId: NodeId) {
775+
const { thread, task } = yield* readTask(parentThreadId, taskId);
776+
const remote = task?.remoteChild;
777+
if (thread.settledOverride !== "settled" || remote === undefined) return;
778+
const scope = followScope(
779+
{ threadId: parentThreadId, providerInstanceId: thread.providerInstanceId },
780+
remote,
781+
);
782+
yield* forwarding.call(scope, ThreadToolkit.tools.t3_thread_organize, remote.environmentId, {
783+
threadId: remote.threadId,
784+
action: "settle",
785+
});
786+
},
787+
// A task still running there refuses until it ends; a machine asleep, until it wakes.
788+
Effect.retry({
789+
schedule: followRetry.pipe(Schedule.upTo({ duration: SETTLE_RETRY_WINDOW })),
790+
while: (error) =>
791+
isMcpFailure(error) &&
792+
error.code !== "invalid_request" &&
793+
!(isFinal(error) || isModeRefusal(error)),
794+
}),
795+
Effect.ignoreCause({ log: true }),
796+
);
797+
798+
// Each retrying settle re-reads its parent, so a repeated settle here needs no second one.
799+
const settling = new Set<NodeId>();
800+
801+
const settleAllThere = Effect.fn("RemoteDelegation.settleAllThere")(
802+
function* (parentThreadId: ThreadId) {
803+
const { subagents } = yield* threads.getThreadRecords(parentThreadId, ["subagents"]);
804+
for (const task of subagents) {
805+
if (task.remoteChild === undefined || settling.has(task.id)) continue;
806+
settling.add(task.id);
807+
yield* settleThere(parentThreadId, task.id).pipe(
808+
Effect.ensuring(Effect.sync(() => settling.delete(task.id))),
809+
Effect.forkIn(followers),
810+
);
811+
}
812+
},
813+
Effect.ignoreCause({ log: true }),
814+
);
815+
762816
const start: RemoteDelegation["Service"]["start"] = () =>
763817
forkParked(
764818
Effect.gen(function* () {
@@ -768,7 +822,9 @@ const make = Effect.gen(function* () {
768822
yield* Stream.runForEach(threads.streamDomainEvents, (event) =>
769823
event.type === "subagent.updated" && event.payload.remoteChild?.stoppedThere === false
770824
? followOne(event.threadId, event.payload.id).pipe(Effect.forkIn(followers))
771-
: Effect.void,
825+
: event.type === "thread.settled"
826+
? settleAllThere(event.threadId)
827+
: Effect.void,
772828
).pipe(
773829
Effect.ignoreCause({ log: true }),
774830
Effect.forkIn(followers, { startImmediately: true }),

0 commit comments

Comments
 (0)