Skip to content

Commit c8198d1

Browse files
committed
fix(server): drop stale queued pull-request watch wakes
1 parent 2be4045 commit c8198d1

2 files changed

Lines changed: 145 additions & 110 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 133 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,102 @@ function nextQueuedRun(
599599
return queuedRunsInDeliveryOrder(projection)[0];
600600
}
601601

602+
function isStalePullRequestWatchWake(
603+
projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
604+
run: OrchestrationV2Run,
605+
): boolean {
606+
const source = projection.messages.find((message) => message.id === run.userMessageId)
607+
?.notification?.source;
608+
if (source?.kind !== "monitor" || source.pullRequest === undefined) {
609+
return false;
610+
}
611+
const stamp = source.pullRequest;
612+
const link = threadPullRequestsOf(projection.thread).find(
613+
(candidate) =>
614+
candidate.source !== "stack-dismissed" && threadPullRequestKeysEqual(candidate, stamp),
615+
);
616+
if (link === undefined) {
617+
return true;
618+
}
619+
if (stamp.watchEnded === true) {
620+
// The closed notice is the watch's final delivery, so it still goes out once
621+
// the watch is gone; only a newer watch generation supersedes it.
622+
return link.watch !== undefined && link.watch.startedAt !== stamp.watchStartedAt;
623+
}
624+
return (
625+
link.watch === undefined ||
626+
link.watch.startedAt !== stamp.watchStartedAt ||
627+
stamp.headSha !== link.watch.headSha ||
628+
link.snapshot?.state === "merged" ||
629+
link.snapshot?.state === "closed"
630+
);
631+
}
632+
633+
function queuedRunCancellationEvents(
634+
projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "attempts">,
635+
run: OrchestrationV2Run,
636+
now: DateTime.Utc,
637+
): ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> {
638+
const rootNode =
639+
run.rootNodeId === null
640+
? undefined
641+
: projection.nodes.find((candidate) => candidate.id === run.rootNodeId);
642+
const attempt =
643+
run.activeAttemptId === null
644+
? undefined
645+
: projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
646+
return [
647+
{
648+
type: "run.updated" as const,
649+
threadId: run.threadId,
650+
runId: run.id,
651+
...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
652+
providerInstanceId: run.providerInstanceId,
653+
occurredAt: now,
654+
payload: {
655+
...run,
656+
status: "cancelled" as const,
657+
queuePosition: null,
658+
completedAt: now,
659+
},
660+
},
661+
...(attempt === undefined || rootNode === undefined
662+
? []
663+
: [
664+
{
665+
type: "run-attempt.updated" as const,
666+
threadId: run.threadId,
667+
runId: run.id,
668+
nodeId: rootNode.id,
669+
providerInstanceId: run.providerInstanceId,
670+
occurredAt: now,
671+
payload: {
672+
...attempt,
673+
status: "cancelled" as const,
674+
completedAt: now,
675+
},
676+
},
677+
]),
678+
...(rootNode === undefined
679+
? []
680+
: [
681+
{
682+
type: "node.updated" as const,
683+
threadId: run.threadId,
684+
runId: run.id,
685+
nodeId: rootNode.id,
686+
providerInstanceId: run.providerInstanceId,
687+
occurredAt: now,
688+
payload: {
689+
...rootNode,
690+
status: "cancelled" as const,
691+
completedAt: now,
692+
},
693+
},
694+
]),
695+
];
696+
}
697+
602698
function latestStableRun(
603699
projection: Pick<OrchestrationV2ThreadProjection, "runs">,
604700
): OrchestrationV2Run | null {
@@ -1099,60 +1195,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10991195
readonly now: DateTime.Utc;
11001196
}) =>
11011197
Effect.gen(function* () {
1102-
const rootNode =
1103-
input.run.rootNodeId === null
1104-
? undefined
1105-
: input.projection.nodes.find((candidate) => candidate.id === input.run.rootNodeId);
1106-
const attempt =
1107-
input.run.activeAttemptId === null
1108-
? undefined
1109-
: input.projection.attempts.find(
1110-
(candidate) => candidate.id === input.run.activeAttemptId,
1111-
);
11121198
const emitEvent = emit(input.events, input.command);
1113-
yield* emitEvent({
1114-
type: "run.updated",
1115-
threadId: input.run.threadId,
1116-
runId: input.run.id,
1117-
...(input.run.rootNodeId === null ? {} : { nodeId: input.run.rootNodeId }),
1118-
providerInstanceId: input.run.providerInstanceId,
1119-
occurredAt: input.now,
1120-
payload: {
1121-
...input.run,
1122-
status: "cancelled",
1123-
queuePosition: null,
1124-
completedAt: input.now,
1125-
},
1126-
});
1127-
if (attempt !== undefined && rootNode !== undefined) {
1128-
yield* emitEvent({
1129-
type: "run-attempt.updated",
1130-
threadId: input.run.threadId,
1131-
runId: input.run.id,
1132-
nodeId: rootNode.id,
1133-
providerInstanceId: input.run.providerInstanceId,
1134-
occurredAt: input.now,
1135-
payload: {
1136-
...attempt,
1137-
status: "cancelled",
1138-
completedAt: input.now,
1139-
},
1140-
});
1141-
}
1142-
if (rootNode !== undefined) {
1143-
yield* emitEvent({
1144-
type: "node.updated",
1145-
threadId: input.run.threadId,
1146-
runId: input.run.id,
1147-
nodeId: rootNode.id,
1148-
providerInstanceId: input.run.providerInstanceId,
1149-
occurredAt: input.now,
1150-
payload: {
1151-
...rootNode,
1152-
status: "cancelled",
1153-
completedAt: input.now,
1154-
},
1155-
});
1199+
for (const event of queuedRunCancellationEvents(input.projection, input.run, input.now)) {
1200+
yield* emitEvent(event);
11561201
}
11571202
});
11581203

@@ -1267,7 +1312,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12671312
) =>
12681313
Effect.gen(function* () {
12691314
if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
1270-
const projection = yield* readCommandProjection(threadId);
1315+
let projection = yield* readCommandProjection(threadId);
12711316
if (
12721317
projection.thread.archivedAt !== null ||
12731318
projection.thread.deletedAt !== null ||
@@ -1289,7 +1334,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12891334
if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
12901335
return;
12911336
}
1292-
const queuedRun = nextQueuedRun(projection);
1337+
let queuedRun = nextQueuedRun(projection);
1338+
while (queuedRun !== undefined && isStalePullRequestWatchWake(projection, queuedRun)) {
1339+
const cancelledAt = yield* DateTime.now;
1340+
yield* writeSystemEvents(queuedRunCancellationEvents(projection, queuedRun, cancelledAt));
1341+
projection = yield* readCommandProjection(threadId);
1342+
queuedRun = nextQueuedRun(projection);
1343+
}
12931344
if (queuedRun === undefined) {
12941345
return;
12951346
}
@@ -2360,16 +2411,39 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
23602411
cause: "The pull request watch ended or its thread settled while it was read.",
23612412
});
23622413
}
2414+
// An open wake is already obsolete when the link's latest snapshot shows
2415+
// the pull request finished: it would dispatch immediately on an idle
2416+
// thread without ever reaching the queued-run staleness check.
2417+
const staleWatchWake =
2418+
command.wake?.notification.source.kind === "monitor" &&
2419+
command.watch !== null &&
2420+
(link?.snapshot?.state === "merged" || link?.snapshot?.state === "closed");
23632421
yield* dispatchThreadMutation(command, events, effects);
2364-
if (command.wake === undefined) return;
2422+
if (command.wake === undefined || staleWatchWake) return;
23652423
yield* dispatchMessage(
23662424
{
23672425
type: "message.dispatch",
23682426
commandId: command.commandId,
23692427
threadId: command.threadId,
23702428
messageId: command.wake.messageId,
23712429
text: command.wake.text,
2372-
notification: command.wake.notification,
2430+
notification: {
2431+
...command.wake.notification,
2432+
source:
2433+
command.wake.notification.source.kind === "monitor"
2434+
? {
2435+
kind: "monitor",
2436+
pullRequest: {
2437+
host: command.host,
2438+
repository: command.repository,
2439+
number: command.number,
2440+
watchStartedAt: command.startedAt,
2441+
headSha: command.watch?.headSha ?? null,
2442+
...(command.watch === null ? { watchEnded: true } : {}),
2443+
},
2444+
}
2445+
: command.wake.notification.source,
2446+
},
23732447
attachments: [],
23742448
dispatchMode: { type: "queue_after_active" },
23752449
createdBy: "agent",
@@ -7794,60 +7868,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
77947868
});
77957869
return;
77967870
}
7797-
const queuedRootNode =
7798-
queuedRun.rootNodeId === null
7799-
? undefined
7800-
: projection.nodes.find((candidate) => candidate.id === queuedRun.rootNodeId);
7801-
const queuedAttempt =
7802-
queuedRun.activeAttemptId === null
7803-
? undefined
7804-
: projection.attempts.find((candidate) => candidate.id === queuedRun.activeAttemptId);
7805-
78067871
const now = yield* DateTime.now;
78077872
const emitEvent = emit(events, command);
7808-
yield* emitEvent({
7809-
type: "run.updated",
7810-
threadId: command.threadId,
7811-
runId: queuedRun.id,
7812-
...(queuedRun.rootNodeId === null ? {} : { nodeId: queuedRun.rootNodeId }),
7813-
providerInstanceId: queuedRun.providerInstanceId,
7814-
occurredAt: now,
7815-
payload: {
7816-
...queuedRun,
7817-
status: "cancelled",
7818-
queuePosition: null,
7819-
completedAt: now,
7820-
},
7821-
});
7822-
if (queuedAttempt !== undefined && queuedRootNode !== undefined) {
7823-
yield* emitEvent({
7824-
type: "run-attempt.updated",
7825-
threadId: command.threadId,
7826-
runId: queuedRun.id,
7827-
nodeId: queuedRootNode.id,
7828-
providerInstanceId: queuedRun.providerInstanceId,
7829-
occurredAt: now,
7830-
payload: {
7831-
...queuedAttempt,
7832-
status: "cancelled",
7833-
completedAt: now,
7834-
},
7835-
});
7836-
}
7837-
if (queuedRootNode !== undefined) {
7838-
yield* emitEvent({
7839-
type: "node.updated",
7840-
threadId: command.threadId,
7841-
runId: queuedRun.id,
7842-
nodeId: queuedRootNode.id,
7843-
providerInstanceId: queuedRun.providerInstanceId,
7844-
occurredAt: now,
7845-
payload: {
7846-
...queuedRootNode,
7847-
status: "cancelled",
7848-
completedAt: now,
7849-
},
7850-
});
7873+
for (const event of queuedRunCancellationEvents(projection, queuedRun, now)) {
7874+
yield* emitEvent(event);
78517875
}
78527876
});
78537877

‎packages/contracts/src/orchestrationV2.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1065,7 +1065,18 @@ export const OrchestrationV2NotificationSource = kindUnionWithFallback(
10651065
Schema.Struct({ kind: Schema.Literal("background_command").transform("command") }),
10661066
SubagentNotificationSource,
10671067
CommandNotificationSource,
1068-
Schema.Struct({ kind: Schema.Literal("monitor") }),
1068+
Schema.Struct({
1069+
kind: Schema.Literal("monitor"),
1070+
// Optional so monitor sources persisted before the stamp existed still decode.
1071+
pullRequest: Schema.optional(
1072+
Schema.Struct({
1073+
...ThreadPullRequestKey.fields,
1074+
watchStartedAt: IsoDateTime,
1075+
headSha: Schema.NullOr(TrimmedNonEmptyString),
1076+
watchEnded: Schema.optional(Schema.Boolean),
1077+
}),
1078+
),
1079+
}),
10691080
Schema.Struct({ kind: Schema.Literal("background_task") }),
10701081
Schema.Struct({ kind: Schema.Literal("system") }),
10711082
],

0 commit comments

Comments
 (0)