Skip to content

Commit 7798267

Browse files
committed
fix(storage): leave settled cleanup to regular sweeps and trust live sessions
Event-driven 0-day cleanup kept missing settlements: a settle after the session already stopped requested nothing, and recording stops for detached threads surfaced old session errors on them and could mark a reopened session stopped. Revert the session manager to main. Settled worktrees go at the next regular sweep, and cleanup counts a stored session as open only while the session manager still runs it.
1 parent 27ef12b commit 7798267

6 files changed

Lines changed: 39 additions & 75 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2889,13 +2889,6 @@ it.effect(
28892889

28902890
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
28912891
assert.deepStrictEqual(yield* rosters, [[], ["dev-server"]]);
2892-
// Storage cleanup waits for the stored session to stop.
2893-
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2894-
const { providerSessions } = yield* projectionStore.getThreadProjection(threadId);
2895-
assert.deepStrictEqual(
2896-
providerSessions.map((session) => session.status),
2897-
["stopped"],
2898-
);
28992892
});
29002893

29012894
yield* effect.pipe(
@@ -4527,14 +4520,6 @@ it.effect(
45274520
yield* TestClock.adjust("1 second");
45284521
yield* Effect.yieldNow;
45294522
assert.equal((yield* Ref.get(state)).closeCount, 1);
4530-
// Every thread detached before the idle release; the stored session
4531-
// still ends, so storage cleanup stops waiting for it.
4532-
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
4533-
const { providerSessions } = yield* projectionStore.getThreadProjection(secondThreadId);
4534-
assert.deepStrictEqual(
4535-
providerSessions.map((session) => session.status),
4536-
["stopped"],
4537-
);
45384523
});
45394524

45404525
yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,6 @@ export class ProviderSessionManagerV2 extends Context.Service<
201201

202202
interface LiveSessionEntry {
203203
readonly attachedThreadIds: ReadonlySet<ThreadId>;
204-
/** Threads detached while the session stayed up; its release still records their session ended. */
205-
readonly detachedThreadIds: ReadonlySet<ThreadId>;
206204
readonly loadedProviderThreadKeyByThread: ReadonlyMap<ThreadId, string>;
207205
/**
208206
* MCP credential session id issued for each attached thread. Revocation on
@@ -664,10 +662,7 @@ export const layerWithOptions = (
664662
};
665663
yield* writeProviderSessionEvents({
666664
runtime: input.entry.runtime,
667-
threadIds: new Set([
668-
...input.entry.attachedThreadIds,
669-
...input.entry.detachedThreadIds,
670-
]),
665+
threadIds: input.entry.attachedThreadIds,
671666
type: "provider-session.updated",
672667
payload,
673668
});
@@ -2291,7 +2286,6 @@ export const layerWithOptions = (
22912286
const now = yield* Clock.currentTimeMillis;
22922287
const entry: LiveSessionEntry = {
22932288
attachedThreadIds: new Set([input.threadId]),
2294-
detachedThreadIds: new Set(),
22952289
loadedProviderThreadKeyByThread: new Map(),
22962290
mcpCredentialIdByThread:
22972291
mcpCredentialId === undefined
@@ -2493,7 +2487,6 @@ export const layerWithOptions = (
24932487
const updatedEntry = {
24942488
...entry,
24952489
attachedThreadIds,
2496-
detachedThreadIds: new Set([...entry.detachedThreadIds, input.threadId]),
24972490
loadedProviderThreadKeyByThread,
24982491
mcpCredentialIdByThread,
24992492
idleThreadUnloads,

‎apps/server/src/storageCleanup.test.ts‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import * as Deferred from "effect/Deferred";
55
import * as Fiber from "effect/Fiber";
66
import * as FileSystem from "effect/FileSystem";
77
import * as Layer from "effect/Layer";
8+
import * as Option from "effect/Option";
89
import type * as Path from "effect/Path";
910
import type * as Scope from "effect/Scope";
1011
import * as Stream from "effect/Stream";
@@ -24,6 +25,7 @@ import * as GitVcsDriver from "./vcs/GitVcsDriver.ts";
2425
import * as GitManager from "./git/GitManager.ts";
2526
import * as Settings from "./serverSettings.ts";
2627
import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts";
28+
import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
2729
import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
2830
import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";
2931
import * as TerminalManager from "./terminal/Manager.ts";
@@ -266,6 +268,7 @@ const cleanupFixture = (
266268
let threads = [shell({ branch: "feature", worktreePath: worktree })];
267269
let editBeforeRemoval = false;
268270
let openSessionCwd: string | null = null;
271+
let sessionLive = true;
269272
let fullStatusCalls = 0;
270273
let snapshotReads = 0;
271274
let events: Stream.Stream<OrchestrationV2DomainEvent> = Stream.empty;
@@ -309,6 +312,9 @@ const cleanupFixture = (
309312
getThreadShell: (threadId: ThreadId) =>
310313
Effect.sync(() => threads.find((thread) => thread.id === threadId) ?? null),
311314
} as unknown as ProjectionStore.ProjectionStoreV2["Service"]),
315+
Effect.provideService(ProviderSessionManager.ProviderSessionManagerV2, {
316+
get: () => Effect.sync(() => (sessionLive ? Option.some({}) : Option.none())),
317+
} as unknown as ProviderSessionManager.ProviderSessionManagerV2["Service"]),
312318
Effect.provideService(Orchestrator.OrchestratorV2, {
313319
streamDomainEvents: Stream.suspend(() => events),
314320
} as unknown as Orchestrator.OrchestratorV2["Service"]),
@@ -383,6 +389,9 @@ const cleanupFixture = (
383389
setThreads: (next: typeof threads) => {
384390
threads = next;
385391
},
392+
setSessionLive: (live: boolean) => {
393+
sessionLive = live;
394+
},
386395
setOpenSessionCwd: (cwd: string | null) => {
387396
openSessionCwd = cwd;
388397
},
@@ -593,7 +602,8 @@ describe("storage cleanup reports and local file policies", () => {
593602
bytes: null,
594603
});
595604
expect(walked.filter((directory) => directory.startsWith(root))).toEqual([]);
596-
fixture.setOpenSessionCwd(null);
605+
// The stored session stays ready after its runtime was released.
606+
fixture.setSessionLive(false);
597607
const removed = yield* fixture.service.runNow;
598608
expect(removed.entries[0]).toMatchObject({ outcome: "removed" });
599609
expect(removed.entries[0]?.bytes).toBeGreaterThan(0);
@@ -967,13 +977,6 @@ describe("settled worktree retention", () => {
967977
: `Settled for ${scenario === "expired" ? 20 : scenario === "boundary" ? 8 : 0} days`;
968978
expect(report.entries[0]).toMatchObject({ outcome: removed ? "removed" : "kept", reason });
969979
}
970-
if (scenario === "recent-update" || scenario === "inactive-settling") {
971-
// Automatic settlement backdates settledAt; the grace runs from the
972-
// settle itself, and another sweep follows once it ends.
973-
yield* TestClock.adjust("1 minute");
974-
yield* fixture.service.drain;
975-
expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false);
976-
}
977980
expect(thread.branch).toBe("feature");
978981
expect(
979982
(yield* fixture.command(fixture.repo, ["rev-parse", "--verify", "refs/heads/feature"]))
@@ -1037,7 +1040,7 @@ it.live("keeps saved usage data when removing a settled checkout", () =>
10371040
);
10381041

10391042
it.live.each(["thread.settled", "stopped", "ready"] as const)(
1040-
"sweeps for settled cleanup after %s only when a session ended",
1043+
"leaves settled cleanup to the regular sweeps after %s",
10411044
(trigger) =>
10421045
runCleanupTest(
10431046
Effect.gen(function* () {
@@ -1080,7 +1083,7 @@ it.live.each(["thread.settled", "stopped", "ready"] as const)(
10801083
yield* Deferred.await(handled);
10811084
yield* Deferred.succeed(fixture.resume, undefined);
10821085
yield* fixture.service.drain;
1083-
expect(fixture.snapshotReads()).toBe(trigger === "stopped" ? 2 : 1);
1086+
expect(fixture.snapshotReads()).toBe(1);
10841087
}),
10851088
),
10861089
);

‎apps/server/src/storageCleanup.ts‎

Lines changed: 22 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import * as Deferred from "effect/Deferred";
2222
import * as Layer from "effect/Layer";
2323
import * as Clock from "effect/Clock";
2424
import * as DateTime from "effect/DateTime";
25-
import * as Duration from "effect/Duration";
2625
import * as Effect from "effect/Effect";
2726
import * as Equal from "effect/Equal";
2827
import * as FileSystem from "effect/FileSystem";
@@ -39,6 +38,7 @@ import * as GitManager from "./git/GitManager.ts";
3938
import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
4039
import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";
4140
import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts";
41+
import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
4242
import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts";
4343
import { forkParked } from "./serverActivation.ts";
4444
import * as Settings from "./serverSettings.ts";
@@ -177,6 +177,7 @@ const make = Effect.gen(function* () {
177177
const projectStore = yield* ProjectStore.ProjectStoreV2;
178178
const engine = yield* Orchestrator.OrchestratorV2;
179179
const projections = yield* ProjectionStore.ProjectionStoreV2;
180+
const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
180181
const sql = yield* SqlClient.SqlClient;
181182
const git = yield* GitVcsDriver.GitVcsDriver;
182183
const gitManager = yield* GitManager.GitManager;
@@ -403,10 +404,6 @@ const make = Effect.gen(function* () {
403404
);
404405
});
405406

406-
// Set when a sweep kept a worktree for the settle-script grace.
407-
let settleGraceRetry = false;
408-
// Typed separately: the sweep that schedules it is defined before it.
409-
let requestGraceSweep: Effect.Effect<void> = Effect.void;
410407
const cleanWorktrees = Effect.fn("StorageCleanup.cleanWorktrees")(function* (
411408
serverSettings: ServerSettings,
412409
now: number,
@@ -499,10 +496,8 @@ const make = Effect.gen(function* () {
499496
!deleted &&
500497
thread.settledAt !== null &&
501498
now - DateTime.toEpochMillis(thread.updatedAt) < SETTLE_SCRIPT_GRACE_MS
502-
) {
503-
settleGraceRetry = true;
499+
)
504500
return keep("settled moments ago");
505-
}
506501
const head = yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" });
507502
const old =
508503
!deleted &&
@@ -625,13 +620,16 @@ const make = Effect.gen(function* () {
625620
const sessions = yield* Effect.forEach(sessionRows, (row) =>
626621
decodeCleanupSession(row.payload_json),
627622
);
628-
if (
629-
sessions.some((session) => {
630-
const cwd = path.resolve(session.cwd);
631-
return cwd === worktreePath || inside(worktreePath, cwd);
632-
})
633-
)
634-
return "provider session is still open";
623+
// A stored session can stay ready after its runtime was released,
624+
// for example once every thread detached; only a live one holds it.
625+
for (const session of sessions) {
626+
const cwd = path.resolve(session.cwd);
627+
if (
628+
(cwd === worktreePath || inside(worktreePath, cwd)) &&
629+
Option.isSome(yield* providerSessions.get(session.id))
630+
)
631+
return "provider session is still open";
632+
}
635633
return null;
636634
});
637635
// Most cancelled removals are caught here, before the worktree is walked.
@@ -820,14 +818,6 @@ const make = Effect.gen(function* () {
820818
bytesFreed,
821819
omittedCount: Math.max(0, entries.length - REPORT_ENTRY_LIMIT),
822820
};
823-
if (settleGraceRetry) {
824-
settleGraceRetry = false;
825-
// Worktrees kept for the settle-script grace get another sweep once it ends.
826-
yield* Effect.suspend(() => requestGraceSweep).pipe(
827-
Effect.delay(Duration.millis(SETTLE_SCRIPT_GRACE_MS)),
828-
Effect.forkDetach,
829-
);
830-
}
831821
yield* SubscriptionRef.set(reportRef, latestReport);
832822
return latestReport;
833823
});
@@ -855,7 +845,6 @@ const make = Effect.gen(function* () {
855845
const requestSweep = Ref.getAndSet(automaticSweepQueued, true).pipe(
856846
Effect.flatMap((queued) => (queued ? Effect.void : worker.enqueue(undefined))),
857847
);
858-
requestGraceSweep = requestSweep;
859848
const runNow = Effect.gen(function* () {
860849
const completion = yield* Deferred.make<StorageCleanupReport, ServerSettingsError>();
861850
yield* worker.enqueue(completion);
@@ -901,21 +890,16 @@ const make = Effect.gen(function* () {
901890
}),
902891
);
903892
yield* forkParked(
904-
// Deleted threads and immediately removable settled threads wait for
905-
// their sessions to end. Moving between live statuses, such as running
906-
// to ready, cannot free one. Settling itself does not sweep: the
907-
// project's settle script runs in the worktree right after.
908-
Stream.runForEach(events, (event) => {
909-
const sessionEnded =
910-
event.type === "provider-session.updated" &&
911-
(event.payload.status === "stopped" || event.payload.status === "error");
912-
return ((event.type === "thread.deleted" || sessionEnded) &&
913-
anyWorktreePolicy(lastSettings, (rules) => rules.worktreeOnDelete)) ||
914-
(sessionEnded &&
915-
anyWorktreePolicy(lastSettings, (rules) => rules.worktreeSettledAfterDays === 0))
893+
// A deleted thread's worktree waits for its sessions to end. Moving
894+
// between live statuses, such as running to ready, cannot free one.
895+
Stream.runForEach(events, (event) =>
896+
(event.type === "thread.deleted" ||
897+
(event.type === "provider-session.updated" &&
898+
(event.payload.status === "stopped" || event.payload.status === "error"))) &&
899+
anyWorktreePolicy(lastSettings, (rules) => rules.worktreeOnDelete)
916900
? requestSweep
917-
: Effect.void;
918-
}).pipe(
901+
: Effect.void,
902+
).pipe(
919903
Effect.catchCause((cause) =>
920904
Effect.logWarning("Storage cleanup event stream failed", { cause }),
921905
),

‎apps/web/src/components/settings/StorageSettings.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -557,7 +557,7 @@ export function StorageSettingsPanel() {
557557
]
558558
.filter(Boolean)
559559
.join(". ")}
560-
description="Remove worktrees this many days after their thread settles. Set 0 days to remove them as soon as they are safe to remove. Branches and thread history are kept."
560+
description="Remove worktrees this many days after their thread settles. Set 0 days to remove them at the next cleanup. Branches and thread history are kept."
561561
serverScoped={!isProjectScope}
562562
control={
563563
<RetentionControl

‎docs/user/project-settings.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,7 @@ are, and cleanup covers both the default folder and the custom one.
142142

143143
Open **Settings → Storage** to enable automatic cleanup on one machine or all connected
144144
environments. Policies are off by default and run on the server at startup, when changed, and
145-
hourly. With 0 settled days, a settled thread's worktree is removed once its session ends.
146-
Offline machines keep their existing policies.
145+
hourly. Offline machines keep their existing policies.
147146

148147
Select a project to set **Automatic worktree cleanup** to **Inherit**, **Off**, or **Custom**.
149148
Inherit follows each machine's rules; Off keeps that project's worktrees until you remove them
@@ -152,7 +151,7 @@ captures and log retention remain machine-wide.
152151

153152
Worktrees can be removed after a chosen number of inactive days, after a thread settles, after
154153
merging, or when they have no commits beyond the default branch. Set settled retention to 0 days
155-
to remove a worktree as soon as it is safe after settlement. Only T3-managed worktrees are eligible.
154+
to remove a worktree at the next cleanup after settlement. Only T3-managed worktrees are eligible.
156155
Active sessions and shared worktrees prevent removal. **Keep worktrees with local changes** defaults
157156
to **Uncommitted changes**, which protects tracked edits and untracked files but deletes ignored
158157
files such as `.env` and build output. **Any local files** also protects ignored files other than

0 commit comments

Comments
 (0)