Skip to content

Commit a35fd81

Browse files
fix(server): settlement sweep reads quiet open pull requests every 5 minutes (pingdotgg#34953)
1 parent a472342 commit a35fd81

3 files changed

Lines changed: 185 additions & 7 deletions

File tree

‎apps/server/src/orchestration-v2/PullRequestSyncReactor.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ const SLOW_SYNC_INTERVAL_MS = 15 * 60 * 1_000;
4242
// Every server polls every open linked pull request, and most of them sit untouched for days.
4343
// One nobody has updated for QUIET_AFTER_MS is read on the quiet cadence instead of each sweep;
4444
// requested refreshes, watches and a host CLI's merge command still read it at once.
45-
const QUIET_AFTER_MS = 30 * 60 * 1_000;
46-
const QUIET_SYNC_INTERVAL_MS = 5 * 60 * 1_000;
45+
export const QUIET_AFTER_MS = 30 * 60 * 1_000;
46+
export const QUIET_SYNC_INTERVAL_MS = 5 * 60 * 1_000;
4747
// A pull request closed this long ago is rarely reopened, so the sweep stops reading it, as it
4848
// does merged ones. A reader that sees it open, or a requested refresh, still reads it at once.
4949
const CLOSED_FINAL_AFTER_MS = 24 * 60 * 60 * 1_000;

‎apps/server/src/orchestration-v2/ThreadSettlementService.test.ts‎

Lines changed: 144 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options:
602602
const settingsReadCount = yield* Ref.make(0);
603603
const settingsChanges = yield* PubSub.unbounded<ContractServerSettings>();
604604
const mergedPullRequests = yield* PubSub.unbounded<PullRequestService.PullRequestMergeEvent>();
605+
const pullRequestRefreshes = yield* PubSub.unbounded<number>();
605606
const commands = yield* Ref.make<ReadonlyArray<AutoSettleCommand>>([]);
606607
const branchCalls = yield* Ref.make<
607608
ReadonlyArray<{ readonly cwd: string; readonly branch: string }>
@@ -737,6 +738,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options:
737738
subscribeMerges: PubSub.subscribe(mergedPullRequests).pipe(
738739
Effect.map((subscription) => Stream.fromSubscription(subscription)),
739740
),
741+
subscribeRefreshes: Stream.fromPubSub(pullRequestRefreshes),
740742
}),
741743
Layer.succeed(ServerSettings.ServerSettingsService, serverSettings),
742744
Layer.succeed(ServerActivation.ServerActivation, Deferred.await(activation)),
@@ -775,6 +777,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options:
775777
observedEvents,
776778
publishEvent: (event: OrchestrationV2DomainEvent) => PubSub.publish(domainEvents, event),
777779
updateSettings,
780+
publishRefresh: PubSub.publish(pullRequestRefreshes, 1),
778781
publishMerge: PubSub.publish(mergedPullRequests, {
779782
projectId: PROJECT_ID,
780783
repository: "owner/repository",
@@ -1623,20 +1626,40 @@ describe("ThreadSettlementServiceV2 worker", () => {
16231626
}),
16241627
);
16251628

1626-
it.effect("settles an unlinked branch thread its agent merged from a shell, past the cache", () =>
1629+
it.effect.each([
1630+
{ kind: "an unlinked", linked: false },
1631+
{ kind: "a quiet linked", linked: true },
1632+
])("settles $kind branch thread its agent merged from a shell, past the cache", ({ linked }) =>
16271633
Effect.gen(function* () {
16281634
yield* TestClock.setTime(Date.parse(NOW));
1629-
const settled = yield* Deferred.make<void>();
16301635
const thread = makeThread("shell-merged", {
16311636
branch: "saved-feature",
16321637
latestUserMessageAt: DateTime.makeUnsafe("2026-08-27T00:00:00.000Z"),
1638+
...(linked
1639+
? {
1640+
branchPullRequest: {
1641+
projectId: PROJECT_ID,
1642+
repository: "owner/repository",
1643+
number: 42,
1644+
url: "https://example.test/owner/repository/pull/42",
1645+
},
1646+
}
1647+
: {}),
16331648
});
16341649
const fixture = yield* makeHarness({
16351650
snapshot: makeSnapshot([thread]),
16361651
// The host already merged it; only a fresh read sees that past the cached open lookup.
16371652
branchPullRequest: (_input, lookupOptions) =>
16381653
Effect.succeed(makeBranchPullRequest(lookupOptions?.refresh ? "merged" : "open")),
1639-
onDispatch: () => Deferred.succeed(settled, undefined),
1654+
pullRequestSummary: (input, readOptions) =>
1655+
Effect.succeed({
1656+
...makePullRequestSummary({
1657+
...input,
1658+
state: readOptions?.bypassCache ? "merged" : "open",
1659+
updatedAt: DateTime.formatIso(DateTime.makeUnsafe(Date.parse(NOW) - 2 * DAY_MS)),
1660+
}),
1661+
...(readOptions?.bypassCache ? { mergedAt: NOW } : {}),
1662+
}),
16401663
});
16411664
const at = DateTime.makeUnsafe(NOW);
16421665
const runId = RunId.make("run:shell-merged");
@@ -1681,7 +1704,8 @@ describe("ThreadSettlementServiceV2 worker", () => {
16811704
input: "gh pr merge 42 --squash",
16821705
},
16831706
});
1684-
yield* Deferred.await(settled);
1707+
yield* Queue.take(fixture.snapshotReads);
1708+
yield* reactor.drain;
16851709
assert.deepStrictEqual(
16861710
(yield* Ref.get(fixture.commands)).map((command) => command.threadId),
16871711
[thread.id],
@@ -1728,6 +1752,122 @@ describe("ThreadSettlementServiceV2 worker", () => {
17281752
}),
17291753
);
17301754

1755+
it.effect("reads a quiet open branch pull request every 5 minutes", () =>
1756+
Effect.gen(function* () {
1757+
yield* TestClock.setTime(Date.parse(NOW));
1758+
const fixture = yield* makeHarness({
1759+
snapshot: makeSnapshot([
1760+
makeThread("active-branch", {
1761+
branch: "feature",
1762+
latestUserMessageAt: DateTime.makeUnsafe("2026-08-27T00:00:00.000Z"),
1763+
branchPullRequest: {
1764+
projectId: PROJECT_ID,
1765+
repository: "owner/repository",
1766+
number: 42,
1767+
url: "https://example.test/owner/repository/pull/42",
1768+
},
1769+
}),
1770+
]),
1771+
pullRequestSummary: (input) =>
1772+
Effect.succeed(
1773+
makePullRequestSummary({
1774+
...input,
1775+
state: "open",
1776+
updatedAt: DateTime.formatIso(DateTime.makeUnsafe(Date.parse(NOW) - 2 * DAY_MS)),
1777+
}),
1778+
),
1779+
});
1780+
1781+
yield* Effect.gen(function* () {
1782+
const reactor = yield* ThreadSettlementService.ThreadSettlementServiceV2;
1783+
yield* startHarness(reactor, fixture.activation, fixture.snapshotReads);
1784+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 1);
1785+
1786+
for (let minute = 0; minute < 4; minute++) {
1787+
yield* TestClock.adjust("1 minute");
1788+
yield* Queue.take(fixture.snapshotReads);
1789+
yield* reactor.drain;
1790+
}
1791+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 1);
1792+
1793+
yield* TestClock.adjust("1 minute");
1794+
yield* Queue.take(fixture.snapshotReads);
1795+
yield* reactor.drain;
1796+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2);
1797+
1798+
// An app merge the host completes later sends only a refresh; reads stay on the minute
1799+
// cadence while the pull request still reads open.
1800+
yield* fixture.publishRefresh;
1801+
for (const expected of [3, 4]) {
1802+
yield* TestClock.adjust("1 minute");
1803+
yield* Queue.take(fixture.snapshotReads);
1804+
yield* reactor.drain;
1805+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, expected);
1806+
}
1807+
}).pipe(Effect.provide(fixture.layer));
1808+
}),
1809+
);
1810+
1811+
it.effect.each([
1812+
{ when: "after", mergeDuringRead: false },
1813+
{ when: "during", mergeDuringRead: true },
1814+
])(
1815+
"a merge $when a quiet pull request's read, while its thread is not a candidate, rereads it next sweep",
1816+
({ mergeDuringRead }) =>
1817+
Effect.gen(function* () {
1818+
yield* TestClock.setTime(Date.parse(NOW));
1819+
const thread = makeThread("busy-branch", {
1820+
branch: "feature",
1821+
latestUserMessageAt: DateTime.makeUnsafe("2026-08-27T00:00:00.000Z"),
1822+
branchPullRequest: {
1823+
projectId: PROJECT_ID,
1824+
repository: "owner/repository",
1825+
number: 42,
1826+
url: "https://example.test/owner/repository/pull/42",
1827+
},
1828+
});
1829+
const readStarted = yield* Deferred.make<void>();
1830+
const releaseRead = yield* Deferred.make<void>();
1831+
if (!mergeDuringRead) yield* Deferred.succeed(releaseRead, undefined);
1832+
const fixture = yield* makeHarness({
1833+
snapshot: makeSnapshot([thread]),
1834+
pullRequestSummary: (input) =>
1835+
Deferred.succeed(readStarted, undefined).pipe(
1836+
Effect.andThen(Deferred.await(releaseRead)),
1837+
Effect.as(
1838+
makePullRequestSummary({
1839+
...input,
1840+
state: "open",
1841+
updatedAt: DateTime.formatIso(DateTime.makeUnsafe(Date.parse(NOW) - 2 * DAY_MS)),
1842+
}),
1843+
),
1844+
),
1845+
});
1846+
1847+
yield* Effect.gen(function* () {
1848+
const reactor = yield* ThreadSettlementService.ThreadSettlementServiceV2;
1849+
yield* reactor.start();
1850+
yield* Deferred.succeed(fixture.activation, undefined);
1851+
yield* Queue.take(fixture.snapshotReads);
1852+
yield* Deferred.await(readStarted);
1853+
if (!mergeDuringRead) yield* reactor.drain;
1854+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 1);
1855+
1856+
yield* Ref.set(fixture.snapshots, makeSnapshot([]));
1857+
yield* fixture.publishMerge;
1858+
yield* Queue.take(fixture.snapshotReads);
1859+
yield* Deferred.succeed(releaseRead, undefined);
1860+
yield* reactor.drain;
1861+
1862+
yield* Ref.set(fixture.snapshots, makeSnapshot([thread]));
1863+
yield* TestClock.adjust("1 minute");
1864+
yield* Queue.take(fixture.snapshotReads);
1865+
yield* reactor.drain;
1866+
assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2);
1867+
}).pipe(Effect.provide(fixture.layer));
1868+
}),
1869+
);
1870+
17311871
it.effect("rereads a branch without a pull request less often the longer its thread idles", () =>
17321872
Effect.gen(function* () {
17331873
yield* TestClock.setTime(Date.parse(NOW));

‎apps/server/src/orchestration-v2/ThreadSettlementService.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import { sourceControlDefinitions } from "../sourceControl/sourceControlDefiniti
3838
import { makeNoPullRequestRechecks } from "./noPullRequestRechecks.ts";
3939
import * as Orchestrator from "./Orchestrator.ts";
4040
import * as ProjectionStore from "./ProjectionStore.ts";
41+
import { QUIET_AFTER_MS, QUIET_SYNC_INTERVAL_MS } from "./PullRequestSyncReactor.ts";
4142

4243
export interface SettlementPullRequest {
4344
readonly state: "open" | "closed" | "merged";
@@ -320,6 +321,22 @@ export const make = Effect.gen(function* () {
320321
// A shell merge or close sends no event, and the branch lookup caches an open pull request for
321322
// a minute, so these threads' next lookup reads the host fresh.
322323
const shellClosedThreads = new Set<ThreadId>();
324+
// Each summary read can be a host CLI process (one `az` per read on Azure DevOps), so a quiet
325+
// open pull request is read on the sync reactor's quiet cadence rather than every sweep.
326+
const quietOpenReadAt = new Map<string, number>();
327+
const quietReadKey = (reference: {
328+
readonly projectId: string;
329+
readonly repository: string;
330+
readonly number: number;
331+
}) => `${reference.projectId}\0${reference.repository.toLowerCase()}\0${reference.number}`;
332+
// A host can finish a merge minutes after the app's merge action and its refresh, and Azure's
333+
// updatedAt never moves while open, so pull requests stay on the fast cadence for a while after.
334+
// Sweeps that started before the pause fail the same check, so their reads record nothing.
335+
let quietPausedAtMs = Number.NEGATIVE_INFINITY;
336+
const pauseQuietReads = Effect.fn("ThreadSettlementServiceV2.pauseQuietReads")(function* () {
337+
quietPausedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
338+
quietOpenReadAt.clear();
339+
});
323340
const terminals = yield* TerminalManager.TerminalManager;
324341
const projectScripts = yield* ProjectSetupScriptRunner.ProjectSetupScriptRunner;
325342
// Repeated thread.settled retains settledAt; run the action only once per settlement.
@@ -330,13 +347,18 @@ export const make = Effect.gen(function* () {
330347
mergedPullRequest: PullRequestService.PullRequestMergeEvent | null,
331348
threadId?: ThreadId,
332349
) {
350+
// A thread that is not a candidate now (still running) must not keep a merged PR's quiet entry.
351+
if (mergedPullRequest !== null) yield* pauseQuietReads();
333352
const settings = yield* settingsService.getSettings;
334353
const automaticSettlementEnabled = autoSettlementConfigured(settings);
335354
const threads = yield* projections.getSettlementCandidates(threadId);
336355
if (threads.length === 0) return;
337356
const projectShells = yield* projectStore.listShells();
338357
const nowMs = DateTime.toEpochMillis(yield* DateTime.now);
339358
const projects = new Map(projectShells.map((project) => [project.id, project]));
359+
for (const [key, readAtMs] of quietOpenReadAt) {
360+
if (nowMs - readAtMs >= QUIET_SYNC_INTERVAL_MS) quietOpenReadAt.delete(key);
361+
}
340362
// Merge events resweep all candidates because unlinked threads settle through branch lookup.
341363
const candidates = threads.filter(
342364
(thread) =>
@@ -509,6 +531,13 @@ export const make = Effect.gen(function* () {
509531
if (!matchesMerge && !projects.has(reference.projectId)) {
510532
return yield* Effect.die(new Error("linked pull request project not found"));
511533
}
534+
const readKey = quietReadKey(reference);
535+
const shellClosed =
536+
group.filter((member) => shellClosedThreads.delete(member.id)).length > 0;
537+
if (shellClosed) quietOpenReadAt.delete(readKey);
538+
else if (!matchesMerge && quietOpenReadAt.has(readKey)) {
539+
return { state: "open", closedAt: null, mergedAt: null } satisfies SettlementPullRequest;
540+
}
512541
const summary = matchesMerge
513542
? ({
514543
state: "merged",
@@ -521,8 +550,16 @@ export const make = Effect.gen(function* () {
521550
repository: reference.repository,
522551
number: reference.number,
523552
},
524-
{ recoverTransientFailure: false },
553+
{ recoverTransientFailure: false, bypassCache: shellClosed },
525554
);
555+
if (
556+
"updatedAt" in summary &&
557+
summary.state === "open" &&
558+
nowMs - quietPausedAtMs >= QUIET_SYNC_INTERVAL_MS &&
559+
nowMs - Date.parse(summary.updatedAt) >= QUIET_AFTER_MS
560+
) {
561+
quietOpenReadAt.set(readKey, nowMs);
562+
}
526563
const terminal = {
527564
state: summary.state,
528565
closedAt: summary.closedAt ?? null,
@@ -786,6 +823,7 @@ export const make = Effect.gen(function* () {
786823
}),
787824
);
788825
yield* forkParked(Stream.runForEach(mergedPullRequests, (event) => runSweep(event)));
826+
yield* forkParked(Stream.runForEach(pullRequests.subscribeRefreshes, pauseQuietReads));
789827
yield* forkParked(
790828
Stream.runForEach(events, processEvent).pipe(
791829
Effect.catchCause((cause) =>

0 commit comments

Comments
 (0)