Skip to content

Commit 0af2d21

Browse files
fix(server): drop stale queued pull-request watch wakes (pingdotgg#21239)
Co-authored-by: Sinan Turna <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent 4901343 commit 0af2d21

4 files changed

Lines changed: 520 additions & 65 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 140 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -940,6 +940,106 @@ function usageLimitHoldsQueue(
940940
return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now);
941941
}
942942

943+
function pullRequestSnapshotEndsWatch(
944+
snapshot: ThreadPullRequestLink["snapshot"],
945+
watchStartedAt: ThreadPullRequestWatch["startedAt"],
946+
): boolean {
947+
return (
948+
snapshot?.state === "merged" ||
949+
// A restarted watch can see a closed snapshot from before the PR reopened.
950+
(snapshot?.state === "closed" &&
951+
DateTime.isGreaterThanOrEqualTo(
952+
DateTime.makeUnsafe(snapshot.syncedAt),
953+
DateTime.makeUnsafe(watchStartedAt),
954+
))
955+
);
956+
}
957+
958+
function isStalePullRequestWatchWake(
959+
projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
960+
run: OrchestrationV2Run,
961+
): boolean {
962+
const source = projection.messages.find((message) => message.id === run.userMessageId)
963+
?.notification?.source;
964+
if (source?.kind !== "monitor" || source.pullRequest === undefined) return false;
965+
const stamp = source.pullRequest;
966+
const link = threadPullRequestsOf(projection.thread).find(
967+
(candidate) =>
968+
candidate.source !== "stack-dismissed" && threadPullRequestKeysEqual(candidate, stamp),
969+
);
970+
if (link === undefined) return true;
971+
if (stamp.watchEnded === true) {
972+
// The final notice outlives its watch, but a replacement watch supersedes it.
973+
const latestWatch = link.watch ?? link.endedWatch;
974+
return latestWatch?.startedAt !== stamp.watchStartedAt;
975+
}
976+
if (link.watch === undefined) {
977+
// A watch the server ended with a final notice still owes the wakes it queued before it.
978+
const endedWithNotice = projection.messages.some((message) => {
979+
const ending = message.notification?.source;
980+
return (
981+
ending?.kind === "monitor" &&
982+
ending.pullRequest?.watchEnded === true &&
983+
ending.pullRequest.watchStartedAt === stamp.watchStartedAt &&
984+
threadPullRequestKeysEqual(ending.pullRequest, stamp)
985+
);
986+
});
987+
return (
988+
!endedWithNotice ||
989+
link.endedWatch?.startedAt !== stamp.watchStartedAt ||
990+
pullRequestSnapshotEndsWatch(link.snapshot, stamp.watchStartedAt)
991+
);
992+
}
993+
return (
994+
link.watch.startedAt !== stamp.watchStartedAt ||
995+
(stamp.headSha !== null && link.watch.headSha !== stamp.headSha) ||
996+
pullRequestSnapshotEndsWatch(link.snapshot, link.watch.startedAt)
997+
);
998+
}
999+
1000+
function queuedRunCancellationEvents(
1001+
projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "attempts">,
1002+
run: OrchestrationV2Run,
1003+
now: DateTime.Utc,
1004+
): ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> {
1005+
const rootNode = projection.nodes.find((node) => node.id === run.rootNodeId);
1006+
const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
1007+
const events: Array<Omit<OrchestrationV2DomainEvent, "id">> = [
1008+
{
1009+
type: "run.updated",
1010+
threadId: run.threadId,
1011+
runId: run.id,
1012+
...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
1013+
providerInstanceId: run.providerInstanceId,
1014+
occurredAt: now,
1015+
payload: { ...run, status: "cancelled", queuePosition: null, completedAt: now },
1016+
},
1017+
];
1018+
if (attempt !== undefined && rootNode !== undefined) {
1019+
events.push({
1020+
type: "run-attempt.updated",
1021+
threadId: run.threadId,
1022+
runId: run.id,
1023+
nodeId: rootNode.id,
1024+
providerInstanceId: run.providerInstanceId,
1025+
occurredAt: now,
1026+
payload: { ...attempt, status: "cancelled", completedAt: now },
1027+
});
1028+
}
1029+
if (rootNode !== undefined) {
1030+
events.push({
1031+
type: "node.updated",
1032+
threadId: run.threadId,
1033+
runId: run.id,
1034+
nodeId: rootNode.id,
1035+
providerInstanceId: run.providerInstanceId,
1036+
occurredAt: now,
1037+
payload: { ...rootNode, status: "cancelled", completedAt: now },
1038+
});
1039+
}
1040+
return events;
1041+
}
1042+
9431043
function latestStableRun(
9441044
projection: Pick<OrchestrationV2ThreadProjection, "runs">,
9451045
): OrchestrationV2Run | null {
@@ -1769,60 +1869,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
17691869
readonly run: OrchestrationV2Run;
17701870
readonly now: DateTime.Utc;
17711871
}) {
1772-
const rootNode =
1773-
input.run.rootNodeId === null
1774-
? undefined
1775-
: input.projection.nodes.find((candidate) => candidate.id === input.run.rootNodeId);
1776-
const attempt =
1777-
input.run.activeAttemptId === null
1778-
? undefined
1779-
: input.projection.attempts.find(
1780-
(candidate) => candidate.id === input.run.activeAttemptId,
1781-
);
17821872
const emitEvent = emit(input.events, input.command);
1783-
yield* emitEvent({
1784-
type: "run.updated",
1785-
threadId: input.run.threadId,
1786-
runId: input.run.id,
1787-
...(input.run.rootNodeId === null ? {} : { nodeId: input.run.rootNodeId }),
1788-
providerInstanceId: input.run.providerInstanceId,
1789-
occurredAt: input.now,
1790-
payload: {
1791-
...input.run,
1792-
status: "cancelled",
1793-
queuePosition: null,
1794-
completedAt: input.now,
1795-
},
1796-
});
1797-
if (attempt !== undefined && rootNode !== undefined) {
1798-
yield* emitEvent({
1799-
type: "run-attempt.updated",
1800-
threadId: input.run.threadId,
1801-
runId: input.run.id,
1802-
nodeId: rootNode.id,
1803-
providerInstanceId: input.run.providerInstanceId,
1804-
occurredAt: input.now,
1805-
payload: {
1806-
...attempt,
1807-
status: "cancelled",
1808-
completedAt: input.now,
1809-
},
1810-
});
1811-
}
1812-
if (rootNode !== undefined) {
1813-
yield* emitEvent({
1814-
type: "node.updated",
1815-
threadId: input.run.threadId,
1816-
runId: input.run.id,
1817-
nodeId: rootNode.id,
1818-
providerInstanceId: input.run.providerInstanceId,
1819-
occurredAt: input.now,
1820-
payload: {
1821-
...rootNode,
1822-
status: "cancelled",
1823-
completedAt: input.now,
1824-
},
1825-
});
1873+
for (const event of queuedRunCancellationEvents(input.projection, input.run, input.now)) {
1874+
yield* emitEvent(event);
18261875
}
18271876
},
18281877
);
@@ -1949,7 +1998,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
19491998
},
19501999
) {
19512000
if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
1952-
const projection = yield* readCommandProjection(threadId);
2001+
let projection = yield* readCommandProjection(threadId);
19532002
if (
19542003
projection.thread.archivedAt !== null ||
19552004
projection.thread.deletedAt !== null ||
@@ -1958,12 +2007,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
19582007
return;
19592008
}
19602009

1961-
const storedQueuedRun =
2010+
let storedQueuedRun =
19622011
options?.queuedRunId === undefined
19632012
? nextQueuedRun(projection)
19642013
: projection.runs.find(
19652014
(run) => run.id === options.queuedRunId && run.status === "queued",
19662015
);
2016+
while (
2017+
storedQueuedRun !== undefined &&
2018+
isStalePullRequestWatchWake(projection, storedQueuedRun)
2019+
) {
2020+
yield* writeSystemEvents(
2021+
queuedRunCancellationEvents(projection, storedQueuedRun, yield* DateTime.now),
2022+
);
2023+
projection = yield* readCommandProjection(threadId);
2024+
storedQueuedRun = nextQueuedRun(projection);
2025+
}
19672026
if (storedQueuedRun === undefined) {
19682027
return;
19692028
}
@@ -3417,14 +3476,39 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
34173476
}
34183477
yield* dispatchThreadMutation(command, events, effects);
34193478
if (command.wake === undefined) return;
3479+
// An idle thread dispatches immediately, bypassing queued-wake validation.
3480+
if (
3481+
command.wake.notification.source.kind === "monitor" &&
3482+
command.watch !== null &&
3483+
pullRequestSnapshotEndsWatch(link.snapshot, command.startedAt)
3484+
) {
3485+
return;
3486+
}
34203487
yield* dispatchMessage(
34213488
{
34223489
type: "message.dispatch",
34233490
commandId: command.commandId,
34243491
threadId: command.threadId,
34253492
messageId: command.wake.messageId,
34263493
text: command.wake.text,
3427-
notification: command.wake.notification,
3494+
notification: {
3495+
...command.wake.notification,
3496+
source:
3497+
command.wake.notification.source.kind === "monitor"
3498+
? {
3499+
kind: "monitor",
3500+
pullRequest: {
3501+
...key,
3502+
watchStartedAt: command.startedAt,
3503+
headSha:
3504+
command.wake.includesRemarks === true
3505+
? null
3506+
: (command.watch?.headSha ?? null),
3507+
...(command.watch === null ? { watchEnded: true } : {}),
3508+
},
3509+
}
3510+
: command.wake.notification.source,
3511+
},
34283512
attachments: [],
34293513
dispatchMode: { type: "queue_after_active" },
34303514
createdBy: "agent",

‎apps/server/src/orchestration-v2/PullRequestWatchReactor.ts‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,11 @@ export const make = Effect.gen(function* () {
284284
const record = Effect.fn("PullRequestWatchReactor.record")(function* (
285285
target: WatchTarget,
286286
next: ThreadPullRequestWatch | null,
287-
wake?: { readonly text: string; readonly notification: OrchestrationV2Notification },
287+
wake?: {
288+
readonly text: string;
289+
readonly notification: OrchestrationV2Notification;
290+
readonly includesRemarks?: boolean;
291+
},
288292
endedWatch?: ThreadPullRequestWatch,
289293
) {
290294
const uuid = yield* random.uuidV4();
@@ -570,13 +574,16 @@ export const make = Effect.gen(function* () {
570574
yield* eachTarget(group, (target) => {
571575
const report = evaluatePullRequestWatch(target.watch, detail, remarks, now);
572576
if (report.changes.length > 0) {
573-
const wake = pullRequestWatchMessage({
574-
number: target.link.number,
575-
url: target.link.url,
576-
baseBranch: detail.baseBranch,
577-
headSha: report.next.headSha,
578-
report,
579-
});
577+
const wake = {
578+
...pullRequestWatchMessage({
579+
number: target.link.number,
580+
url: target.link.url,
581+
baseBranch: detail.baseBranch,
582+
headSha: report.next.headSha,
583+
report,
584+
}),
585+
includesRemarks: report.changes.some((change) => change.kind === "remarks"),
586+
};
580587
// An ending watch keeps what this pass saw, so watching again starts from it.
581588
return (
582589
report.exhausted

0 commit comments

Comments
 (0)