Skip to content

Commit 29b0303

Browse files
committed
fix(server): background server restarts wait for review guides, clones, and git refreshes
Every pull request review guide generation is tracked through fetch, checkout, generation, storage and cleanup, including superseded generations and their finalizers, and holds back a server restart until it ends. Project clones, workspace preparation and background git refreshes do the same. The counts show in the update window's blocker log.
1 parent eb22e76 commit 29b0303

11 files changed

Lines changed: 499 additions & 12 deletions

‎apps/server/src/orchestration-v2/ThreadLaunchService.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import * as DateTime from "effect/DateTime";
3232
import * as Duration from "effect/Duration";
3333
import * as Effect from "effect/Effect";
3434
import * as Exit from "effect/Exit";
35+
import * as Fiber from "effect/Fiber";
3536
import * as Layer from "effect/Layer";
3637
import * as Option from "effect/Option";
3738
import * as Ref from "effect/Ref";
@@ -401,6 +402,40 @@ it.effect("retains automation and sender attribution while a message waits in th
401402
}).pipe(Effect.provide(harness.layer));
402403
});
403404

405+
it.effect("tracks workspace preparation for an empty thread with no active run", () =>
406+
Effect.gen(function* () {
407+
const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
408+
const release = yield* Deferred.make<void>();
409+
const harness = makeHarness({
410+
createWorktree: (input) =>
411+
Effect.gen(function* () {
412+
yield* Deferred.succeed(entered, yield* Effect.fiber);
413+
yield* Deferred.await(release);
414+
return {
415+
worktree: { path: "/repo-worktrees/empty", refName: input.newRefName, headSha: "abc" },
416+
} as never;
417+
}),
418+
});
419+
yield* Effect.gen(function* () {
420+
const launches = yield* ThreadLaunch.ThreadLaunchService;
421+
const threads = yield* ThreadManagement.ThreadManagementService;
422+
const result = yield* launches.launch(
423+
launchInput({
424+
command: "command:launch:empty-preparation",
425+
thread: "thread:launch:empty-preparation",
426+
workspace: { type: "worktree", baseRef: "main", branch: "empty" },
427+
}),
428+
);
429+
const fiber = yield* Deferred.await(entered);
430+
assert.equal((yield* threads.getThreadProjection(result.threadId)).runs.length, 0);
431+
assert.equal(yield* launches.activePreparationCount, 1);
432+
yield* Deferred.succeed(release, undefined);
433+
assert.isTrue(Exit.isSuccess(yield* Fiber.await(fiber)));
434+
assert.equal(yield* launches.activePreparationCount, 0);
435+
}).pipe(Effect.provide(harness.layer));
436+
}),
437+
);
438+
404439
it.effect("returns a visible preparing message while provisioning is still blocked", () =>
405440
Effect.gen(function* () {
406441
const worktreeEntered = yield* Deferred.make<void>();
@@ -2263,6 +2298,7 @@ it.effect("shared intake preserves durable attachment bytes after a lost launch
22632298
),
22642299
checkWorktreeBase: launches.checkWorktreeBase,
22652300
retryPreparation: launches.retryPreparation,
2301+
activePreparationCount: launches.activePreparationCount,
22662302
}),
22672303
Effect.flip,
22682304
);

‎apps/server/src/orchestration-v2/ThreadLaunchService.ts‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import * as Cause from "effect/Cause";
2323
import * as Context from "effect/Context";
2424
import * as Effect from "effect/Effect";
2525
import * as Exit from "effect/Exit";
26+
import * as FiberSet from "effect/FiberSet";
2627
import * as Layer from "effect/Layer";
2728
import * as Option from "effect/Option";
2829
import * as Ref from "effect/Ref";
@@ -47,6 +48,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
4748
import type * as Orchestrator from "./Orchestrator.ts";
4849
import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
4950
import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid";
51+
import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance";
5052
import * as ThreadManagement from "./ThreadManagementService.ts";
5153

5254
export type ThreadLaunchWorkspaceStrategy =
@@ -187,6 +189,8 @@ export class ThreadLaunchService extends Context.Service<
187189
| Orchestrator.OrchestratorV2Error
188190
| ThreadManagement.ThreadManagementProviderInstanceDisabledError
189191
>;
192+
/** Workspace preparation and branch naming can outlive a thread's run. */
193+
readonly activePreparationCount: Effect.Effect<number>;
190194
}
191195
>()("t3/orchestration-v2/ThreadLaunchService") {}
192196

@@ -215,9 +219,13 @@ const make = Effect.gen(function* () {
215219
const ids = yield* IdAllocator.IdAllocatorV2;
216220
const threads = yield* ThreadManagement.ThreadManagementService;
217221
const managedFolders = yield* ManagedProjectFolders.ManagedProjectFolders;
222+
const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission;
218223
const preparationScope = yield* Scope.make("sequential");
219224
const scheduledLaunches = yield* Ref.make<ReadonlySet<CommandId>>(new Set());
220225
yield* Effect.addFinalizer(() => Scope.close(preparationScope, Exit.void));
226+
const preparations = yield* FiberSet.make().pipe(
227+
Effect.provideService(Scope.Scope, preparationScope),
228+
);
221229

222230
const mapError =
223231
(input: PreparationInput, operation: ThreadLaunchError["operation"], threadId?: ThreadId) =>
@@ -531,7 +539,7 @@ const make = Effect.gen(function* () {
531539
cause,
532540
}),
533541
),
534-
Effect.forkIn(preparationScope),
542+
FiberSet.run(preparations, { startImmediately: false }),
535543
);
536544
}
537545

@@ -745,7 +753,8 @@ const make = Effect.gen(function* () {
745753
),
746754
Effect.ignoreCause,
747755
Effect.ensuring(releasePreparation(input.commandId)),
748-
Effect.forkIn(preparationScope),
756+
FiberSet.run(preparations, { startImmediately: false }),
757+
admission.withLaunchPermit,
749758
);
750759
});
751760

@@ -1050,7 +1059,12 @@ const make = Effect.gen(function* () {
10501059
);
10511060
};
10521061

1053-
return ThreadLaunchService.of({ launch, checkWorktreeBase, retryPreparation });
1062+
return ThreadLaunchService.of({
1063+
launch,
1064+
checkWorktreeBase,
1065+
retryPreparation,
1066+
activePreparationCount: FiberSet.size(preparations),
1067+
});
10541068
});
10551069

10561070
export const layer = Layer.effect(ThreadLaunchService, make);

‎apps/server/src/project/ProjectCloneTracker.test.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
import * as Deferred from "effect/Deferred";
99
import * as Effect from "effect/Effect";
1010
import * as Fiber from "effect/Fiber";
11+
import * as Exit from "effect/Exit";
1112
import * as Layer from "effect/Layer";
1213
import * as Stream from "effect/Stream";
1314
import * as TestClock from "effect/testing/TestClock";
@@ -162,20 +163,28 @@ describe("ProjectCloneTracker", () => {
162163

163164
it.effect("a cancel that lands after git finished keeps the checkout", () => {
164165
const gate = Deferred.makeUnsafe<void>();
166+
const entered = Deferred.makeUnsafe<Fiber.Fiber<unknown, unknown>>();
165167
const harness = makeHarness();
166168
// The clone itself completes instantly; the post-clone hook is what hangs.
167169
const hooks: ProjectCloneTracker.ProjectCloneHooks = {
168170
...harness.hooks,
169-
onCloned: () => Deferred.await(gate),
171+
onCloned: () =>
172+
Effect.fiber.pipe(
173+
Effect.flatMap((fiber) => Deferred.succeed(entered, fiber)),
174+
Effect.andThen(Deferred.await(gate)),
175+
),
170176
};
171177
return Effect.gen(function* () {
172178
const tracker = yield* ProjectCloneTracker.ProjectCloneTracker;
173179
yield* tracker.start(startInput, hooks);
174-
yield* Effect.yieldNow;
180+
const fiber = yield* Deferred.await(entered);
175181
expect((yield* tracker.get(projectId))?.phase).toBe("done");
182+
expect(yield* tracker.activeCloneCount).toBe(1);
176183
expect(yield* tracker.cancel(projectId)).toBe(false);
177184
expect(harness.discarded).toEqual([]);
178185
yield* Deferred.succeed(gate, undefined);
186+
expect(Exit.isSuccess(yield* Fiber.await(fiber))).toBe(true);
187+
expect(yield* tracker.activeCloneCount).toBe(0);
179188
}).pipe(Effect.provide(harness.layer));
180189
});
181190

‎apps/server/src/project/ProjectCloneTracker.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import * as DateTime from "effect/DateTime";
1919
import * as Effect from "effect/Effect";
2020
import * as Exit from "effect/Exit";
2121
import * as Fiber from "effect/Fiber";
22+
import * as FiberSet from "effect/FiberSet";
2223
import * as Layer from "effect/Layer";
2324
import * as PubSub from "effect/PubSub";
2425
import * as Queue from "effect/Queue";
@@ -29,6 +30,7 @@ import * as Scope from "effect/Scope";
2930
import * as Stream from "effect/Stream";
3031

3132
import * as SourceControlRepositoryService from "../sourceControl/SourceControlRepositoryService.ts";
33+
import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance";
3234

3335
/**
3436
* Runs repository clones that back newly added projects and tracks their
@@ -69,6 +71,8 @@ export class ProjectCloneTracker extends Context.Service<
6971
readonly get: (projectId: ProjectId) => Effect.Effect<ProjectCloneSnapshot | null>;
7072
/** Emits every tracked clone first, then the full list after each change. */
7173
readonly stream: Stream.Stream<ReadonlyArray<ProjectCloneSnapshot>>;
74+
/** Includes post-clone hooks after the progress snapshot has settled. */
75+
readonly activeCloneCount: Effect.Effect<number>;
7276
}
7377
>()("t3/project/ProjectCloneTracker") {}
7478

@@ -115,13 +119,15 @@ interface TrackedClone {
115119
/** @public Service construction is part of the canonical Effect module API. */
116120
export const make = Effect.gen(function* () {
117121
const repositories = yield* SourceControlRepositoryService.SourceControlRepositoryService;
122+
const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission;
118123
const clones = yield* Ref.make(new Map<ProjectId, TrackedClone>());
119124
const changes = yield* PubSub.unbounded<ReadonlyArray<ProjectCloneSnapshot>>();
120125
const retentionFibers = new Map<ProjectId, Fiber.Fiber<unknown, never>>();
121126
let sequence = 0;
122127
// Clone fibers outlive the RPC that started them but not the server.
123128
const cloneScope = yield* Scope.make("parallel");
124129
yield* Effect.addFinalizer(() => Scope.close(cloneScope, Exit.void));
130+
const cloneFibers = yield* FiberSet.make().pipe(Effect.provideService(Scope.Scope, cloneScope));
125131
// start/cancel/retry/discard mutate the same entry and the same directory;
126132
// one at a time keeps a double-clicked Retry from racing two clones into it.
127133
const actionLock = yield* Semaphore.make(1);
@@ -249,7 +255,10 @@ export const make = Effect.gen(function* () {
249255
const current = yield* Ref.get(clones);
250256
const tracked = current.get(projectId);
251257
if (!tracked) return;
252-
const fiber = yield* runClone(projectId, tracked).pipe(Effect.forkIn(cloneScope));
258+
const fiber = yield* runClone(projectId, tracked).pipe(
259+
FiberSet.run(cloneFibers, { startImmediately: false }),
260+
admission.withLaunchPermit,
261+
);
253262
yield* Ref.update(clones, (map) => {
254263
const existing = map.get(projectId);
255264
if (!existing) return map;
@@ -430,6 +439,7 @@ export const make = Effect.gen(function* () {
430439
discard: (projectId) => locked(discard(projectId)),
431440
get,
432441
stream,
442+
activeCloneCount: FiberSet.size(cloneFibers),
433443
});
434444
});
435445

‎apps/server/src/provider/providerMaintenanceRunner.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ import * as Maintenance from "@t3tools/provider-core/server/maintenance";
33
import * as BackgroundPolicy from "../background/BackgroundPolicy.ts";
44
import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
55
import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts";
6+
import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
7+
import * as ProjectCloneTracker from "../project/ProjectCloneTracker.ts";
8+
import * as PullRequestReviewGuideService from "../pullRequest/PullRequestReviewGuideService.ts";
9+
import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";
610
import * as SqlitePersistence from "../persistence/Sqlite.ts";
711
import * as Option from "effect/Option";
812
import * as ThreadContracts from "@t3tools/contracts";
@@ -3511,6 +3515,26 @@ it.effect.each([
35113515
expected: false,
35123516
updatedInstance: ProviderInstanceId.make("other"),
35133517
},
3518+
...(
3519+
[
3520+
"review-guides",
3521+
"project-clones",
3522+
"workspace-preparation",
3523+
"git-refreshes",
3524+
"missing-guide-tracker",
3525+
] as const
3526+
).flatMap((work) =>
3527+
[true, false].map((restart) => ({
3528+
name: `${work} during ${restart ? "restart" : "provider install"}`,
3529+
restart,
3530+
busy: false,
3531+
installing: false,
3532+
interaction: false,
3533+
expected: !restart,
3534+
updatedInstance: null,
3535+
work,
3536+
})),
3537+
),
35143538
])("rechecks the update window for $name", (scenario) =>
35153539
Effect.gen(function* () {
35163540
let quietReads = 0;
@@ -3519,6 +3543,28 @@ it.effect.each([
35193543
UpdateWindow.layer.pipe(
35203544
Layer.provide(
35213545
Layer.mergeAll(
3546+
"work" in scenario && scenario.work === "missing-guide-tracker"
3547+
? Layer.empty
3548+
: Layer.mock(PullRequestReviewGuideService.PullRequestReviewGuideService)({
3549+
activeGenerationCount: Effect.succeed(
3550+
"work" in scenario && scenario.work === "review-guides" ? 2 : 0,
3551+
),
3552+
}),
3553+
Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({
3554+
activeCloneCount: Effect.succeed(
3555+
"work" in scenario && scenario.work === "project-clones" ? 1 : 0,
3556+
),
3557+
}),
3558+
Layer.mock(ThreadLaunchService.ThreadLaunchService)({
3559+
activePreparationCount: Effect.succeed(
3560+
"work" in scenario && scenario.work === "workspace-preparation" ? 1 : 0,
3561+
),
3562+
}),
3563+
Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({
3564+
activeRefreshCount: Effect.succeed(
3565+
"work" in scenario && scenario.work === "git-refreshes" ? 1 : 0,
3566+
),
3567+
}),
35223568
Layer.mock(ProjectionStore.ProjectionStoreV2)({
35233569
getRecoveryThreadIds: () => Effect.succeed([]),
35243570
getLimitRecoveryCandidates: () => Effect.succeed([]),

0 commit comments

Comments
 (0)