Skip to content

Commit d5afb1a

Browse files
fix(server): slow forge CLI calls no longer delay checkpoints and turn starts (pingdotgg#28431)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent b492e28 commit d5afb1a

4 files changed

Lines changed: 175 additions & 8 deletions

File tree

‎apps/server/src/orchestration-v2/RunFinalizationService.test.ts‎

Lines changed: 109 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,12 @@ import {
55
ThreadId,
66
type OrchestrationV2ThreadShell,
77
} from "@t3tools/contracts";
8+
import * as Deferred from "effect/Deferred";
89
import * as Effect from "effect/Effect";
10+
import * as Fiber from "effect/Fiber";
911
import * as Layer from "effect/Layer";
12+
import * as Option from "effect/Option";
13+
import * as TestClock from "effect/testing/TestClock";
1014

1115
import * as PullRequestService from "../pullRequest/PullRequestService.ts";
1216
import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";
@@ -93,6 +97,7 @@ it.effect.each(
9397
).map((scenario) => [scenario.label, scenario] as const),
9498
)("%s", ([, scenario]) => {
9599
const refreshed: string[] = [];
100+
const lookedUp = Deferred.makeUnsafe<void>();
96101
const threadId = ThreadId.make("thread-pr-refresh");
97102
const runId = RunId.make("completed-run");
98103
const layer = RunFinalization.layerObserver.pipe(
@@ -114,11 +119,11 @@ it.effect.each(
114119
}),
115120
refreshStatus: () =>
116121
Effect.die("turn completion must preserve known PRs and lookup backoff"),
117-
refreshPullRequestStatus: (cwd) =>
118-
Effect.sync(() => {
119-
refreshed.push(cwd);
120-
return null;
121-
}),
122+
// Requested synchronously by refresh; the lookup itself runs in the background.
123+
refreshPullRequestStatus: (cwd) => {
124+
refreshed.push(cwd);
125+
return Deferred.succeed(lookedUp, undefined).pipe(Effect.as(null));
126+
},
122127
}),
123128
Layer.mock(ProjectionStore.ProjectionStoreV2)({
124129
getThreadShell: () =>
@@ -134,6 +139,105 @@ it.effect.each(
134139
return Effect.gen(function* () {
135140
const observer = yield* RunFinalization.RunFinalizationObserver;
136141
yield* observer.refresh({ cwd: "/repo", threadId, runId });
142+
if (scenario.expected.length > 0) yield* Deferred.await(lookedUp);
137143
assert.deepEqual(refreshed, [...scenario.expected]);
138144
}).pipe(Effect.provide(layer));
139145
});
146+
147+
it.effect("returns from the post-turn refresh without waiting for the PR lookup", () => {
148+
const threadId = ThreadId.make("thread-slow-forge");
149+
const runId = RunId.make("completed-run");
150+
const layer = RunFinalization.layerObserver.pipe(
151+
Layer.provide(
152+
Layer.mergeAll(
153+
Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }),
154+
Layer.mock(PullRequestService.PullRequestService)({
155+
refreshAfterTurn: () => Effect.void,
156+
}),
157+
Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({
158+
refreshLocalStatus: () =>
159+
Effect.succeed({
160+
isRepo: true,
161+
hasPrimaryRemote: true,
162+
isDefaultRef: false,
163+
refName: "feature",
164+
hasWorkingTreeChanges: false,
165+
workingTree: { files: [], insertions: 0, deletions: 0 },
166+
}),
167+
refreshPullRequestStatus: () => Effect.never,
168+
}),
169+
Layer.mock(ProjectionStore.ProjectionStoreV2)({
170+
getThreadShell: () =>
171+
Effect.succeed({
172+
id: threadId,
173+
branch: "feature",
174+
activeRunId: null,
175+
} as OrchestrationV2ThreadShell),
176+
}),
177+
),
178+
),
179+
);
180+
return Effect.gen(function* () {
181+
const observer = yield* RunFinalization.RunFinalizationObserver;
182+
const refresh = yield* observer
183+
.refresh({ cwd: "/repo", threadId, runId })
184+
.pipe(Effect.timeoutOption("1 minute"), Effect.forkChild({ startImmediately: true }));
185+
yield* TestClock.adjust("1 minute");
186+
const result = yield* Fiber.join(refresh);
187+
assert.isTrue(
188+
Option.isSome(result),
189+
"turn finalization waited on the forge PR lookup, blocking the thread's next turn",
190+
);
191+
}).pipe(Effect.provide(layer));
192+
});
193+
194+
it.effect("replaces a still-running PR lookup when a newer turn finishes", () => {
195+
const threadId = ThreadId.make("thread-back-to-back");
196+
const runId = RunId.make("completed-run");
197+
const firstInterrupted = Deferred.makeUnsafe<void>();
198+
const secondLookedUp = Deferred.makeUnsafe<void>();
199+
let lookups = 0;
200+
const layer = RunFinalization.layerObserver.pipe(
201+
Layer.provide(
202+
Layer.mergeAll(
203+
Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }),
204+
Layer.mock(PullRequestService.PullRequestService)({
205+
refreshAfterTurn: () => Effect.void,
206+
}),
207+
Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({
208+
refreshLocalStatus: () =>
209+
Effect.succeed({
210+
isRepo: true,
211+
hasPrimaryRemote: true,
212+
isDefaultRef: false,
213+
refName: "feature",
214+
hasWorkingTreeChanges: false,
215+
workingTree: { files: [], insertions: 0, deletions: 0 },
216+
}),
217+
refreshPullRequestStatus: () =>
218+
++lookups === 1
219+
? Effect.never.pipe(
220+
Effect.onInterrupt(() => Deferred.succeed(firstInterrupted, undefined)),
221+
)
222+
: Deferred.succeed(secondLookedUp, undefined).pipe(Effect.as(null)),
223+
}),
224+
Layer.mock(ProjectionStore.ProjectionStoreV2)({
225+
getThreadShell: () =>
226+
Effect.succeed({
227+
id: threadId,
228+
branch: "feature",
229+
activeRunId: null,
230+
} as OrchestrationV2ThreadShell),
231+
}),
232+
),
233+
),
234+
);
235+
return Effect.gen(function* () {
236+
const observer = yield* RunFinalization.RunFinalizationObserver;
237+
yield* observer.refresh({ cwd: "/repo", threadId, runId });
238+
yield* observer.refresh({ cwd: "/repo", threadId, runId });
239+
yield* Deferred.await(firstInterrupted);
240+
yield* Deferred.await(secondLookedUp);
241+
assert.strictEqual(lookups, 2);
242+
}).pipe(Effect.provide(layer));
243+
});

‎apps/server/src/orchestration-v2/RunFinalizationService.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { CheckpointScopeId, ProjectId, RunId, ThreadId } from "@t3tools/contracts";
22
import * as Context from "effect/Context";
33
import * as Effect from "effect/Effect";
4+
import * as FiberMap from "effect/FiberMap";
45
import * as Layer from "effect/Layer";
56
import * as Schema from "effect/Schema";
67

@@ -96,6 +97,7 @@ export const layerObserver = Layer.effect(
9697
const vcsStatus = yield* VcsStatusBroadcaster.VcsStatusBroadcaster;
9798
const projections = yield* ProjectionStore.ProjectionStoreV2;
9899
const pullRequests = yield* PullRequestService.PullRequestService;
100+
const pullRequestLookups = yield* FiberMap.make<string>();
99101
return {
100102
refreshAfterTurn: pullRequests.refreshAfterTurn,
101103
refresh: Effect.fn("RunFinalizationService.refresh")(
@@ -108,6 +110,8 @@ export const layerObserver = Layer.effect(
108110
const thread = yield* projections.getThreadShell(threadId);
109111
if (!thread || thread.branch !== local.refName) return;
110112
if (thread.activeRunId !== null && thread.activeRunId !== runId) return;
113+
// The forge CLI can take minutes, so the next turn must not wait for it. A newer
114+
// turn's lookup replaces a running one instead of queueing behind the cwd's lock.
111115
yield* vcsStatus.refreshPullRequestStatus(cwd).pipe(
112116
Effect.catch((error) =>
113117
Effect.logWarning("failed to refresh pull request status after run completion", {
@@ -116,6 +120,7 @@ export const layerObserver = Layer.effect(
116120
detail: error.message,
117121
}),
118122
),
123+
FiberMap.run(pullRequestLookups, cwd),
119124
);
120125
},
121126
(effect, { cwd }) =>

‎apps/server/src/vcs/VcsProcess.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@ import * as Cause from "effect/Cause";
88
import * as Fiber from "effect/Fiber";
99
import * as FileSystem from "effect/FileSystem";
1010
import * as Layer from "effect/Layer";
11+
import * as Option from "effect/Option";
1112
import * as Path from "effect/Path";
1213
import * as Queue from "effect/Queue";
1314
import * as Ref from "effect/Ref";
1415
import * as Schema from "effect/Schema";
16+
import * as TestClock from "effect/testing/TestClock";
1517
import { ChildProcessSpawner } from "effect/process";
1618

1719
import { VcsProcessExitError, VcsProcessSpawnError } from "@t3tools/contracts";
@@ -316,6 +318,60 @@ describe("VcsProcess.run", () => {
316318
}),
317319
);
318320

321+
it.effect("leaves git a slot while slow forge CLI processes are running", () =>
322+
Effect.gen(function* () {
323+
const gate = yield* Deferred.make<void>();
324+
const service = yield* VcsProcess.make.pipe(
325+
Effect.provideService(
326+
ProcessRunner.ProcessRunner,
327+
ProcessRunner.ProcessRunner.of({
328+
run: (input) =>
329+
Effect.gen(function* () {
330+
if (input.command !== "git") yield* Deferred.await(gate);
331+
return {
332+
stdout: "",
333+
stderr: "",
334+
code: ChildProcessSpawner.ExitCode(0),
335+
timedOut: false,
336+
stdoutTruncated: false,
337+
stderrTruncated: false,
338+
stdoutInvalidUtf8: false,
339+
stderrInvalidUtf8: false,
340+
};
341+
}),
342+
}),
343+
),
344+
);
345+
346+
const forgeRuns = yield* Effect.forEach(Array.from({ length: 12 }), () =>
347+
service
348+
.run({
349+
operation: "synthetic.forge.listChangeRequests",
350+
command: "tea",
351+
args: ["pulls", "list"],
352+
cwd: "/workspace",
353+
})
354+
.pipe(Effect.forkChild({ startImmediately: true })),
355+
);
356+
const checkpoint = yield* service
357+
.run({
358+
...baseInput,
359+
operation: VcsProcess.CHECKPOINT_CAPTURE_OPERATION,
360+
args: ["rev-parse", "HEAD"],
361+
})
362+
.pipe(Effect.timeoutOption("1 second"), Effect.forkChild({ startImmediately: true }));
363+
yield* TestClock.adjust("1 second");
364+
365+
const gitResult = yield* Fiber.join(checkpoint);
366+
yield* Deferred.succeed(gate, undefined);
367+
yield* Fiber.joinAll(forgeRuns);
368+
expect(
369+
Option.isSome(gitResult),
370+
"git waited behind forge CLI runs that held every VCS process slot",
371+
).toBe(true);
372+
}),
373+
);
374+
319375
it.effect("keeps git from opening a folder laid out as a bare repository", () =>
320376
Effect.gen(function* () {
321377
const fileSystem = yield* FileSystem.FileSystem;

‎apps/server/src/vcs/VcsProcess.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ const DEFAULT_TIMEOUT_MS = 30_000;
6565
const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
6666
const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";
6767
const VCS_PROCESS_CONCURRENCY = 8;
68+
// Forge CLIs can take seconds per call; cap them together so git, checkpoints included, keeps slots.
69+
const NON_GIT_PROCESS_CONCURRENCY = VCS_PROCESS_CONCURRENCY / 2;
6870

6971
/** Shown to users when a Git command fails because another process holds the index lock. */
7072
export const GIT_INDEX_LOCKED_DETAIL =
@@ -183,6 +185,7 @@ export const make = Effect.gen(function* () {
183185
const processRunner = yield* ProcessRunner.ProcessRunner;
184186
const hostPlatform = yield* HostProcess.Platform;
185187
const vcsProcesses = yield* Semaphore.make(VCS_PROCESS_CONCURRENCY);
188+
const nonGitProcesses = yield* Semaphore.make(NON_GIT_PROCESS_CONCURRENCY);
186189
const keyedProcesses = new Map<string, Semaphore.Semaphore>();
187190
const keyedPermit = (concurrency: NonNullable<VcsProcessInput["concurrency"]>) => {
188191
let semaphore = keyedProcesses.get(concurrency.key);
@@ -365,9 +368,8 @@ export const make = Effect.gen(function* () {
365368
}),
366369
);
367370
}
368-
return yield* input.concurrency === undefined
369-
? bounded
370-
: keyedPermit(input.concurrency)(bounded);
371+
const shared = input.command === "git" ? bounded : nonGitProcesses.withPermits(1)(bounded);
372+
return yield* input.concurrency === undefined ? shared : keyedPermit(input.concurrency)(shared);
371373
});
372374

373375
return VcsProcess.of({ run });

0 commit comments

Comments
 (0)