Skip to content

Commit 2e051a5

Browse files
fix(server): device releases no longer stall other threads' settle cleanup
Releasing a settled thread's devices waits on the device lifecycle lock, which a starting SSH host can hold for minutes. It ran inline in the serial domain-event loop, so every later thread.settled waited behind it. Releases now run in a scoped FiberSet beside the event loop, and drain awaits them. The settle script no longer waits on device releases. AI-assisted; verified with the tests and tools listed above.
1 parent 120c6e8 commit 2e051a5

2 files changed

Lines changed: 74 additions & 8 deletions

File tree

‎apps/server/src/orchestration-v2/ThreadSettlementService.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1542,6 +1542,7 @@ describe("ThreadSettlementServiceV2 terminals", () => {
15421542
yield* fixture.publishEvent(settledEvent(marker));
15431543
assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: settled.id });
15441544
assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: marker.id });
1545+
yield* service.drain;
15451546
assert.deepStrictEqual(
15461547
(yield* Ref.get(fixture.deviceCloses)).filter((close) => close.threadId === settled.id),
15471548
[{ threadId: settled.id, hostId: "slow-ssh", deviceId: "pixel" }],
@@ -1551,6 +1552,42 @@ describe("ThreadSettlementServiceV2 terminals", () => {
15511552
),
15521553
);
15531554

1555+
it.effect("closes another thread's terminals while a device release waits on the lock", () =>
1556+
Effect.scoped(
1557+
Effect.gen(function* () {
1558+
const releaseStarted = yield* Deferred.make<void>();
1559+
const lockFreed = yield* Deferred.make<void>();
1560+
const waiting = appThread("waiting-device-thread", "settled");
1561+
const other = appThread("other-thread", "settled");
1562+
const fixture = yield* makeHarness({
1563+
snapshot: makeSnapshot([]),
1564+
currentThreads: [waiting, other],
1565+
deviceSessions: [{ hostId: "starting-ssh", deviceId: "pixel" }],
1566+
onDeviceClose: () =>
1567+
Deferred.succeed(releaseStarted, undefined).pipe(
1568+
Effect.andThen(Deferred.await(lockFreed)),
1569+
),
1570+
});
1571+
1572+
yield* Effect.gen(function* () {
1573+
const service = yield* ThreadSettlementService.ThreadSettlementServiceV2;
1574+
yield* startHarness(service, fixture.activation, fixture.snapshotReads);
1575+
yield* fixture.publishEvent(settledEvent(waiting));
1576+
assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: waiting.id });
1577+
yield* Deferred.await(releaseStarted);
1578+
yield* fixture.publishEvent(settledEvent(other));
1579+
assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: other.id });
1580+
yield* Deferred.succeed(lockFreed, undefined);
1581+
yield* service.drain;
1582+
assert.deepStrictEqual(
1583+
(yield* Ref.get(fixture.deviceCloses)).map((close) => close.threadId),
1584+
[waiting.id, other.id],
1585+
);
1586+
}).pipe(Effect.provide(fixture.layer));
1587+
}),
1588+
),
1589+
);
1590+
15541591
it.effect("releases a settled thread's devices without shutting them down", () =>
15551592
Effect.scoped(
15561593
Effect.gen(function* () {
@@ -1568,6 +1605,7 @@ describe("ThreadSettlementServiceV2 terminals", () => {
15681605
yield* fixture.publishEvent(settledEvent(marker));
15691606
assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: settled.id });
15701607
assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: marker.id });
1608+
yield* service.drain;
15711609
const closes = yield* Ref.get(fixture.deviceCloses);
15721610
assert.deepStrictEqual(
15731611
closes.filter((close) => close.threadId === settled.id),
@@ -1603,6 +1641,7 @@ describe("ThreadSettlementServiceV2 terminals", () => {
16031641
assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: settled.id });
16041642
const run = yield* Queue.take(fixture.scriptRuns);
16051643
assert.strictEqual(run.threadId, settled.id);
1644+
yield* service.drain;
16061645
assert.deepStrictEqual(yield* Ref.get(fixture.deviceCloses), [
16071646
{ threadId: settled.id, hostId: "ssh-host", deviceId: "remote-pixel" },
16081647
{ threadId: settled.id, hostId: "local", deviceId: "emulator-5554" },
@@ -1628,6 +1667,7 @@ describe("ThreadSettlementServiceV2 terminals", () => {
16281667
yield* startHarness(service, fixture.activation, fixture.snapshotReads);
16291668
yield* fixture.publishEvent(settledEvent(settled));
16301669
assert.strictEqual((yield* Queue.take(fixture.scriptRuns)).threadId, settled.id);
1670+
yield* service.drain;
16311671
assert.deepStrictEqual(yield* Ref.get(fixture.deviceCloses), []);
16321672
}).pipe(Effect.provide(fixture.layer));
16331673
}),
@@ -1683,6 +1723,7 @@ describe("ThreadSettlementServiceV2 terminals", () => {
16831723
yield* fixture.publishEvent(settledEvent(settled));
16841724
yield* fixture.publishEvent(settledEvent(marker));
16851725
assert.strictEqual((yield* Queue.take(fixture.scriptRuns)).threadId, marker.id);
1726+
yield* service.drain;
16861727
assert.deepStrictEqual(
16871728
(yield* Ref.get(fixture.deviceCloses)).filter((close) => close.threadId === settled.id),
16881729
[],

‎apps/server/src/orchestration-v2/ThreadSettlementService.ts‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -598,14 +598,12 @@ export const make = Effect.gen(function* () {
598598
return sweepThread(threadId).pipe(FiberSet.run(threadSweeps), Effect.asVoid);
599599
});
600600

601-
// Skip settle scripts in shared checkouts where other threads may still work.
602-
const cleanUpSettledThread = Effect.fn("ThreadSettlementServiceV2.cleanUpSettledThread")(
601+
// Per device: a thread without devices skips the lock a starting host holds, one unreachable
602+
// host leaves no other device claimed, and a thread re-engaged meanwhile keeps the rest.
603+
const releaseSettledThreadDevices = Effect.fn(
604+
"ThreadSettlementServiceV2.releaseSettledThreadDevices",
605+
)(
603606
function* (threadId: ThreadId) {
604-
const settled = yield* projections.getThread(threadId);
605-
if (settled.settledOverride !== "settled") return;
606-
yield* terminals.closeOnSettle({ threadId });
607-
// Per device: a thread without devices skips the lock a starting host holds, one unreachable
608-
// host leaves no other device claimed, and a thread re-engaged meanwhile keeps the rest.
609607
yield* Effect.forEach(
610608
yield* devices.sessionsForThread(threadId),
611609
({ hostId, deviceId }) =>
@@ -624,6 +622,30 @@ export const make = Effect.gen(function* () {
624622
}),
625623
{ discard: true },
626624
);
625+
},
626+
(effect, threadId) =>
627+
effect.pipe(
628+
Effect.catchCauseIf(
629+
(cause) => !Cause.hasInterruptsOnly(cause),
630+
(cause) =>
631+
Effect.logWarning("releasing a settled thread's devices failed", {
632+
threadId,
633+
cause: Cause.pretty(cause),
634+
}),
635+
),
636+
),
637+
);
638+
// Releases wait on the device lock a starting host can hold for minutes, so they run beside
639+
// the event loop instead of stalling every other thread's settle cleanup.
640+
const deviceReleases = yield* FiberSet.make<void>();
641+
642+
// Skip settle scripts in shared checkouts where other threads may still work.
643+
const cleanUpSettledThread = Effect.fn("ThreadSettlementServiceV2.cleanUpSettledThread")(
644+
function* (threadId: ThreadId) {
645+
const settled = yield* projections.getThread(threadId);
646+
if (settled.settledOverride !== "settled") return;
647+
yield* terminals.closeOnSettle({ threadId });
648+
yield* releaseSettledThreadDevices(threadId).pipe(FiberSet.run(deviceReleases));
627649
const worktreePath = settled.worktreePath;
628650
if (worktreePath === null || !(yield* fileSystem.exists(worktreePath))) return;
629651
// Recheck after closing and worktree I/O; re-engaged threads must not be cleaned up.
@@ -740,6 +762,9 @@ export const make = Effect.gen(function* () {
740762

741763
return {
742764
start,
743-
drain: Effect.andThen(fullSweeps.drain, FiberSet.awaitEmpty(threadSweeps)),
765+
drain: fullSweeps.drain.pipe(
766+
Effect.andThen(FiberSet.awaitEmpty(threadSweeps)),
767+
Effect.andThen(FiberSet.awaitEmpty(deviceReleases)),
768+
),
744769
} satisfies ThreadSettlementServiceV2["Service"];
745770
});

0 commit comments

Comments
 (0)