Skip to content

Commit 0ab622a

Browse files
t3dotggclaude
andcommitted
fix(server): a rate limited stack read also pauses PR sync
A stack read refused for a rate limit was swallowed inside syncGroup, so it never recorded the host's pause and the sweep kept retrying it each minute. It now leaves the pause to the sweep and keeps the stack due for retry. The pause is also checked against the clock as each read starts, so a pause that ends during a sweep no longer holds that sweep's remaining reads. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent e58072f commit 0ab622a

2 files changed

Lines changed: 86 additions & 28 deletions

File tree

‎apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -994,6 +994,51 @@ describe("PullRequestSyncReactor", () => {
994994
);
995995
});
996996

997+
it.effect("pauses the host when only its stack read is rate limited", () =>
998+
Effect.scoped(
999+
Effect.gen(function* () {
1000+
yield* TestClock.setTime(Date.parse(NOW));
1001+
const retryAt = Date.parse(NOW) + 3 * 60_000;
1002+
const fixture = yield* makeHarness({
1003+
snapshot: makeSnapshot([makeThread("one", { pullRequests: [makeLink(7, null)] })]),
1004+
summary: (input) => Effect.succeed(makeSummary(input, { state: "open" })),
1005+
stack: () =>
1006+
Effect.gen(function* () {
1007+
if ((yield* Clock.currentTimeMillis) >= retryAt) return null;
1008+
return yield* new PullRequestOperationError({
1009+
operation: "stack",
1010+
detail: "paused",
1011+
cause: new PullRequestProviderError({
1012+
provider: "github",
1013+
operation: "getChangeRequestStack",
1014+
reason: "rate-limited",
1015+
detail: "paused",
1016+
retryAt,
1017+
}),
1018+
});
1019+
}),
1020+
});
1021+
const reads = Effect.all([
1022+
Ref.get(fixture.summaryCalls).pipe(Effect.map((calls) => calls.length)),
1023+
Ref.get(fixture.stackCalls).pipe(Effect.map((calls) => calls.length)),
1024+
]);
1025+
1026+
yield* Effect.gen(function* () {
1027+
const reactor = yield* startAndSweep(fixture);
1028+
assert.deepStrictEqual(yield* reads, [1, 1]);
1029+
yield* sweepAgain(fixture, reactor);
1030+
yield* sweepAgain(fixture, reactor);
1031+
assert.deepStrictEqual(yield* reads, [1, 1]);
1032+
1033+
// At retryAt the pull request is read again, stack included.
1034+
yield* sweepAgain(fixture, reactor);
1035+
assert.deepStrictEqual(yield* reads, [2, 2]);
1036+
assert.strictEqual((yield* Ref.get(fixture.syncCommands)).length, 1);
1037+
}).pipe(Effect.provide(fixture.layer));
1038+
}),
1039+
),
1040+
);
1041+
9971042
it.effect("reads open links fresh only when a run that ran a merge command ends", () =>
9981043
Effect.scoped(
9991044
Effect.gen(function* () {

‎apps/server/src/orchestration-v2/PullRequestSyncReactor.ts‎

Lines changed: 41 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
visibleThreadPullRequests,
1717
} from "@t3tools/shared/threadPullRequests";
1818
import * as Cause from "effect/Cause";
19+
import * as Clock from "effect/Clock";
1920
import * as Context from "effect/Context";
2021
import * as Crypto from "effect/Crypto";
2122
import * as DateTime from "effect/DateTime";
@@ -305,10 +306,15 @@ export const make = Effect.gen(function* () {
305306
})),
306307
Effect.catchCauseIf(
307308
(cause) => !Cause.hasInterruptsOnly(cause),
308-
() =>
309-
Effect.logWarning("pull request stack lookup failed", {
310-
key,
311-
}).pipe(Effect.as(null)),
309+
(cause) =>
310+
rateLimitRetryAt(cause) !== undefined
311+
? // The sweep records the pause and holds the host's other reads until it ends.
312+
Effect.sync(() => retryStacks.add(key)).pipe(
313+
Effect.andThen(Effect.failCause(cause)),
314+
)
315+
: Effect.logWarning("pull request stack lookup failed", {
316+
key,
317+
}).pipe(Effect.as(null)),
312318
),
313319
);
314320
if (needsStack) {
@@ -340,32 +346,39 @@ export const make = Effect.gen(function* () {
340346

341347
// Failed host reads by reason: how many, and the first key that failed that way.
342348
const skips = new Map<string, { count: number; readonly key: string }>();
343-
yield* Effect.forEach(
344-
groups,
345-
([key, entries]) =>
346-
Effect.suspend(() => {
347-
if (!((scope === "all" || requested.has(key)) && isDue(key, entries, nowMs))) {
348-
return Effect.void;
349+
const readGroup = (key: string, entries: ReadonlyArray<LinkEntry>, pauseKey: string) =>
350+
syncGroup(key, entries).pipe(
351+
Effect.catchCause((cause) => {
352+
if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause);
353+
const retryAt = rateLimitRetryAt(cause);
354+
if (retryAt !== undefined) {
355+
pausedUntil.set(pauseKey, Math.max(retryAt, pausedUntil.get(pauseKey) ?? 0));
349356
}
350-
const first = entries[0]!;
351-
const pauseKey = `${first.thread.projectId}\0${normalizeThreadPullRequestKey(first.link).host}`;
352-
// Checked as each read starts, so a pause found earlier in this sweep holds the rest.
353-
if ((pausedUntil.get(pauseKey) ?? 0) > nowMs) return Effect.void;
354-
return syncGroup(key, entries).pipe(
355-
Effect.catchCause((cause) => {
356-
if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause);
357-
const retryAt = rateLimitRetryAt(cause);
358-
if (retryAt !== undefined) {
359-
pausedUntil.set(pauseKey, Math.max(retryAt, pausedUntil.get(pauseKey) ?? 0));
360-
}
361-
const reason = skipReason(cause);
362-
const skip = skips.get(reason);
363-
if (skip === undefined) skips.set(reason, { count: 1, key });
364-
else skip.count += 1;
365-
return Effect.void;
366-
}),
367-
);
357+
const reason = skipReason(cause);
358+
const skip = skips.get(reason);
359+
if (skip === undefined) skips.set(reason, { count: 1, key });
360+
else skip.count += 1;
361+
return Effect.void;
368362
}),
363+
);
364+
yield* Effect.forEach(
365+
groups,
366+
([key, entries]) => {
367+
if (!((scope === "all" || requested.has(key)) && isDue(key, entries, nowMs))) {
368+
return Effect.void;
369+
}
370+
const first = entries[0]!;
371+
const pauseKey = `${first.thread.projectId}\0${normalizeThreadPullRequestKey(first.link).host}`;
372+
// Checked against the clock as each read starts, so a pause found earlier in this sweep
373+
// holds the rest, and one that ends during the sweep lets the rest through.
374+
return Clock.currentTimeMillis.pipe(
375+
Effect.flatMap((startedAtMs) =>
376+
(pausedUntil.get(pauseKey) ?? 0) > startedAtMs
377+
? Effect.void
378+
: readGroup(key, entries, pauseKey),
379+
),
380+
);
381+
},
369382
// As wide as one batched summary read, so the sweep's reads on a host arrive together and
370383
// GitHub answers them in one request rather than one `gh pr view` apiece.
371384
{ concurrency: 25, discard: true },

0 commit comments

Comments
 (0)