Skip to content

Commit 7f65c9c

Browse files
abiassiclaudejuliusmarminge
authored
fix(server): PR sync stops reading missing pull requests every minute (pingdotgg#17800)
Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent 4bf4ae7 commit 7f65c9c

2 files changed

Lines changed: 208 additions & 15 deletions

File tree

‎apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts‎

Lines changed: 161 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ import * as ServerActivation from "../serverActivation.ts";
4141
import * as Orchestrator from "./Orchestrator.ts";
4242
import * as ProjectionStore from "./ProjectionStore.ts";
4343
import * as PullRequestSyncReactor from "./PullRequestSyncReactor.ts";
44-
import type { PullRequestTestThread } from "./testkit/pullRequestFixtures.ts";
44+
import { type PullRequestTestThread, v2PullRequestThread } from "./testkit/pullRequestFixtures.ts";
4545

4646
const NOW = "2026-08-28T12:00:00.000Z";
4747
const PROJECT_ID = ProjectId.make("sync-project");
@@ -161,6 +161,14 @@ function makeSummary(
161161
};
162162
}
163163

164+
const missingSummary = Effect.fail(
165+
new PullRequestOperationError({
166+
operation: "summary",
167+
detail: "Pull request not found",
168+
reason: "not-found",
169+
}),
170+
);
171+
164172
interface HarnessOptions {
165173
readonly invalidate?: PullRequestService.PullRequestService["Service"]["invalidate"];
166174
readonly snapshot: TestShellSnapshot;
@@ -1262,7 +1270,11 @@ describe("PullRequestSyncReactor", () => {
12621270
makeThread("settled", {
12631271
settledOverride: "settled",
12641272
settledAt: "2026-08-21T00:00:00.000Z",
1265-
pullRequests: [makeLink(5, { state: "open" }), makeLink(6, { state: "closed" })],
1273+
pullRequests: [
1274+
makeLink(5, { state: "open" }),
1275+
makeLink(6, { state: "closed" }),
1276+
makeLink(7),
1277+
],
12661278
}),
12671279
]),
12681280
});
@@ -1283,7 +1295,7 @@ describe("PullRequestSyncReactor", () => {
12831295
yield* sweepAgain(fixture, reactor);
12841296
assert.deepStrictEqual(
12851297
(yield* Ref.get(fixture.summaryCalls)).map((call) => call.number).toSorted(),
1286-
[5, 6],
1298+
[5, 6, 7],
12871299
);
12881300
}).pipe(Effect.provide(fixture.layer));
12891301
}),
@@ -1519,6 +1531,152 @@ describe("PullRequestSyncReactor", () => {
15191531
),
15201532
);
15211533

1534+
it.effect(
1535+
"retries missing links every fifteen minutes and resumes normal reads after recovery",
1536+
() =>
1537+
Effect.scoped(
1538+
Effect.gen(function* () {
1539+
yield* TestClock.setTime(Date.parse(NOW));
1540+
const found = yield* Ref.make(false);
1541+
const fixture = yield* makeHarness({
1542+
snapshot: makeSnapshot([
1543+
makeThread("missing", { pullRequests: [makeLink(9), makeLink(10, {})] }),
1544+
]),
1545+
summary: (input) =>
1546+
Ref.get(found).pipe(
1547+
Effect.flatMap((available) =>
1548+
available ? Effect.succeed(makeSummary(input)) : missingSummary,
1549+
),
1550+
),
1551+
});
1552+
yield* Effect.gen(function* () {
1553+
const reactor = yield* startAndSweep(fixture);
1554+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2);
1555+
for (let index = 0; index < 14; index += 1) yield* sweepAgain(fixture, reactor);
1556+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2);
1557+
yield* sweepAgain(fixture, reactor);
1558+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 4);
1559+
assert.deepStrictEqual(yield* Ref.get(fixture.syncCommands), []);
1560+
1561+
yield* Ref.set(found, true);
1562+
for (let index = 0; index < 14; index += 1) yield* sweepAgain(fixture, reactor);
1563+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 4);
1564+
yield* sweepAgain(fixture, reactor);
1565+
const commands = yield* Ref.get(fixture.syncCommands);
1566+
assert.deepStrictEqual(
1567+
commands.map((command) => command.number),
1568+
[9],
1569+
);
1570+
yield* Ref.update(fixture.snapshots, (snapshot) => applySync(snapshot, commands));
1571+
// Recovered quiet open pull requests return to the five-minute cadence.
1572+
for (let index = 0; index < 5; index += 1) yield* sweepAgain(fixture, reactor);
1573+
assert.deepStrictEqual(
1574+
(yield* Ref.get(fixture.summaryCalls))
1575+
.map((call) => call.number)
1576+
.toSorted((left, right) => left - right),
1577+
[9, 9, 9, 9, 10, 10, 10, 10],
1578+
);
1579+
}).pipe(Effect.provide(fixture.layer));
1580+
}),
1581+
),
1582+
);
1583+
1584+
it.effect(
1585+
"holds missing links through sibling and link updates but honors a reader refresh",
1586+
() =>
1587+
Effect.scoped(
1588+
Effect.gen(function* () {
1589+
yield* TestClock.setTime(Date.parse(NOW));
1590+
const fixture = yield* makeHarness({
1591+
snapshot: makeSnapshot([makeThread("missing", { pullRequests: [makeLink(9)] })]),
1592+
summary: (input) =>
1593+
input.number === 9 ? missingSummary : Effect.succeed(makeSummary(input)),
1594+
});
1595+
const missingReads = Ref.get(fixture.summaryCalls).pipe(
1596+
Effect.map((calls) => calls.filter((call) => call.number === 9).length),
1597+
);
1598+
yield* Effect.gen(function* () {
1599+
const reactor = yield* startAndSweep(fixture);
1600+
const linked = makeThread("missing", { pullRequests: [makeLink(9), makeLink(8)] });
1601+
yield* Ref.set(fixture.snapshots, makeSnapshot([linked]));
1602+
yield* Queue.offer(fixture.domainEvents, {
1603+
type: "thread.pull-request-synced",
1604+
id: EventId.make("sibling-synced"),
1605+
threadId: linked.id,
1606+
occurredAt: DateTime.makeUnsafe(NOW),
1607+
payload: { ...v2PullRequestThread(linked), lastVisitedAt: null },
1608+
});
1609+
yield* Queue.take(fixture.snapshotReads);
1610+
yield* reactor.drain;
1611+
const afterSiblingSync = yield* missingReads;
1612+
assert.deepStrictEqual(
1613+
(yield* Ref.get(fixture.syncCommands)).map((command) => command.number),
1614+
[8],
1615+
);
1616+
1617+
yield* Queue.offer(fixture.domainEvents, {
1618+
type: "thread.pull-request-link-updated",
1619+
id: EventId.make("missing-link-updated"),
1620+
threadId: linked.id,
1621+
occurredAt: DateTime.makeUnsafe(NOW),
1622+
payload: makeLink(9),
1623+
});
1624+
// Let the event consumer finish even when it correctly queues no sweep.
1625+
yield* TestClock.adjust(0);
1626+
yield* reactor.drain;
1627+
const afterLinkUpdate = yield* missingReads;
1628+
1629+
yield* Queue.offer(fixture.stateChanges, makeLink(9));
1630+
yield* TestClock.adjust(0);
1631+
yield* reactor.drain;
1632+
const afterReaderRefresh = yield* missingReads;
1633+
yield* sweepAgain(fixture, reactor);
1634+
assert.deepStrictEqual(
1635+
{
1636+
afterSiblingSync,
1637+
afterLinkUpdate,
1638+
afterReaderRefresh,
1639+
afterSweep: yield* missingReads,
1640+
},
1641+
{ afterSiblingSync: 1, afterLinkUpdate: 1, afterReaderRefresh: 2, afterSweep: 2 },
1642+
);
1643+
}).pipe(Effect.provide(fixture.layer));
1644+
}),
1645+
),
1646+
);
1647+
1648+
it.effect("a pending stack retry respects the missing summary backoff", () =>
1649+
Effect.scoped(
1650+
Effect.gen(function* () {
1651+
yield* TestClock.setTime(Date.parse(NOW));
1652+
const found = yield* Ref.make(true);
1653+
const fixture = yield* makeHarness({
1654+
snapshot: makeSnapshot([makeThread("missing", { pullRequests: [makeLink(4)] })]),
1655+
summary: (input) =>
1656+
Ref.get(found).pipe(
1657+
Effect.flatMap((available) =>
1658+
available ? Effect.succeed(makeSummary(input)) : missingSummary,
1659+
),
1660+
),
1661+
stack: () =>
1662+
Effect.fail(new PullRequestOperationError({ operation: "stack", detail: "host down" })),
1663+
});
1664+
yield* Effect.gen(function* () {
1665+
const reactor = yield* startAndSweep(fixture);
1666+
assert.strictEqual((yield* Ref.get(fixture.stackCalls)).length, 1);
1667+
yield* Ref.set(found, false);
1668+
yield* sweepAgain(fixture, reactor);
1669+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2);
1670+
for (let index = 0; index < 14; index += 1) yield* sweepAgain(fixture, reactor);
1671+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2);
1672+
yield* sweepAgain(fixture, reactor);
1673+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 3);
1674+
assert.strictEqual((yield* Ref.get(fixture.stackCalls)).length, 1);
1675+
}).pipe(Effect.provide(fixture.layer));
1676+
}),
1677+
),
1678+
);
1679+
15221680
it.effect.each([
15231681
"gh pr merge 7 --squash 2>&1 | tail -3",
15241682
"cafe pr merge 7 --squash",

‎apps/server/src/orchestration-v2/PullRequestSyncReactor.ts‎

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -346,22 +346,26 @@ export const make = Effect.gen(function* () {
346346
// linking dozens of pull requests) is read together and shares the summary batches.
347347
let requestedSweepQueued = false;
348348
const retryStacks = new Set<string>();
349+
// Group keys the host answered not-found for; they are read on the slow cadence until found.
350+
const missing = new Set<string>();
349351
// Pause per project and host because credentials are project-specific; leave paused PRs due until reset.
350352
const pausedUntil = new Map<string, number>();
351353

352354
const isDue = (key: string, entries: ReadonlyArray<LinkEntry>, nowMs: number): boolean => {
353-
if (requested.has(key) || retryStacks.has(key)) return true;
354-
if (entries.some((entry) => entry.link.snapshot === null)) return true;
355+
if (requested.has(key)) return true;
355356
const active = entries.filter((entry) => isUnsettled(entry.thread));
357+
if (active.length === 0) return false;
358+
const last = lastSyncedAt.get(key);
359+
const sinceLastMs = last === undefined ? Number.POSITIVE_INFINITY : nowMs - last;
360+
if (missing.has(key)) return sinceLastMs >= SLOW_SYNC_INTERVAL_MS;
361+
if (retryStacks.has(key) || active.some((entry) => entry.link.snapshot === null)) return true;
356362
// Merged snapshots without mergedAt cannot settle threads; reread slowly until supplied.
357363
const isFinal = ({ link: { snapshot } }: LinkEntry) =>
358364
snapshot?.state === "merged"
359365
? snapshot.mergedAt != null
360366
: snapshot?.state === "closed" &&
361367
nowMs - Date.parse(snapshot.closedAt ?? "") >= CLOSED_FINAL_AFTER_MS;
362368
if (active.every(isFinal)) return false;
363-
const last = lastSyncedAt.get(key);
364-
const sinceLastMs = last === undefined ? Number.POSITIVE_INFINITY : nowMs - last;
365369
const open = active.filter((entry) => entry.link.snapshot?.state === "open");
366370
if (open.length > 0) {
367371
const quiet = open.every((entry) => {
@@ -403,6 +407,7 @@ export const make = Effect.gen(function* () {
403407

404408
for (const key of lastSyncedAt.keys()) if (!groups.has(key)) lastSyncedAt.delete(key);
405409
for (const key of retryStacks) if (!groups.has(key)) retryStacks.delete(key);
410+
for (const key of missing) if (!groups.has(key)) missing.delete(key);
406411
for (const key of requested.keys()) if (!groups.has(key)) requested.delete(key);
407412

408413
const linkedThisSweep = new Set<string>();
@@ -488,7 +493,19 @@ export const make = Effect.gen(function* () {
488493
// A stack retry reads past the cache, or it could get back the layer list it is retrying.
489494
const invalidated = refreshRequested || missingMergedAt || retryStacks.has(key);
490495
if (invalidated) yield* pullRequests.invalidate({ reference: ref });
491-
const summary = yield* pullRequests.summary(ref, { recoverTransientFailure: false });
496+
const summary = yield* pullRequests.summary(ref, { recoverTransientFailure: false }).pipe(
497+
Effect.tapError((error) =>
498+
error.reason === "not-found"
499+
? Effect.sync(() => {
500+
// Not-found is a host answer: it consumes the refresh and starts the backoff.
501+
missing.add(key);
502+
lastSyncedAt.set(key, nowMs);
503+
requested.delete(key);
504+
})
505+
: Effect.void,
506+
),
507+
);
508+
missing.delete(key);
492509
const fields = snapshotFieldsOf(summary);
493510
const needsStack =
494511
refreshRequested ||
@@ -645,6 +662,27 @@ export const make = Effect.gen(function* () {
645662
Effect.catchCause(logSkipped("pull request refresh after run skipped", { threadId })),
646663
);
647664

665+
// A thread's sibling syncs and relinks must not re-read a pull request its project's host
666+
// could not find; the slow cadence and explicit refreshes still do.
667+
const requestUnsyncedLinks = Effect.fnUntraced(
668+
function* (threadId: ThreadId, links: ReadonlyArray<ThreadPullRequestLink>) {
669+
const unsynced = links.filter((link) => link.snapshot === null);
670+
if (unsynced.length === 0) return;
671+
const [thread] =
672+
missing.size === 0 ? [] : yield* projections.getThreadsWithPullRequests(threadId);
673+
for (const link of unsynced) {
674+
if (
675+
thread !== undefined &&
676+
missing.has(`${thread.projectId}\0${threadPullRequestKeyOf(link)}`)
677+
) {
678+
continue;
679+
}
680+
yield* requestSync(link);
681+
}
682+
},
683+
Effect.catchCause(logSkipped("pull request refresh after link update skipped", {})),
684+
);
685+
648686
// An agent that opens a pull request from a shell gets it linked as link_pull_request would.
649687
const linkCreatedChangeRequests = Effect.fn("PullRequestSyncReactor.linkCreatedChangeRequests")(
650688
function* (threadId: ThreadId, item: CommandExecution) {
@@ -690,14 +728,11 @@ export const make = Effect.gen(function* () {
690728
Stream.runForEach(events, (event) => {
691729
switch (event.type) {
692730
case "thread.pull-request-link-updated":
693-
return event.payload.snapshot === null ? requestSync(event.payload) : Effect.void;
731+
return requestUnsyncedLinks(event.threadId, [event.payload]);
694732
case "thread.pull-request-synced":
695-
return Effect.forEach(
696-
visibleThreadPullRequests(event.payload.pullRequests ?? []).filter(
697-
(link) => link.snapshot === null,
698-
),
699-
requestSync,
700-
{ discard: true },
733+
return requestUnsyncedLinks(
734+
event.threadId,
735+
visibleThreadPullRequests(event.payload.pullRequests ?? []),
701736
);
702737
// An agent can merge or close its pull request from a shell with its host's CLI, which
703738
// sends no merge notification. Read the thread's unmerged links fresh once that command

0 commit comments

Comments
 (0)