Skip to content

Commit e4eb997

Browse files
fix(server): stop replaying old agent alerts on restart (pingdotgg#13340)
1 parent 894d334 commit e4eb997

2 files changed

Lines changed: 207 additions & 7 deletions

File tree

‎apps/server/src/relay/AgentAwarenessRelay.test.ts‎

Lines changed: 178 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import { CommandId, ProviderInstanceId } from "@t3tools/contracts";
2020
import { RelayClientTracer } from "@t3tools/shared/relayTracing";
2121
import { RELAY_ACTIVITY_PUBLISH_TYP, verifyRelayJwt } from "@t3tools/shared/relayJwt";
2222
import { describe, expect, it } from "@effect/vitest";
23+
import * as DateTime from "effect/DateTime";
2324
import * as Deferred from "effect/Deferred";
2425
import * as Effect from "effect/Effect";
2526
import * as Layer from "effect/Layer";
@@ -325,6 +326,9 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {
325326
const projectId = "project-1" as ProjectId;
326327
const activeThreadId = "thread-active" as ThreadId;
327328
const idleThreadId = "thread-idle" as ThreadId;
329+
const oldCompletedId = "thread-old-completed" as ThreadId;
330+
const newCompletedId = "thread-new-completed" as ThreadId;
331+
const freshMessageId = "thread-fresh-message" as ThreadId;
328332

329333
const baseThread = {
330334
projectId,
@@ -351,6 +355,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {
351355
expect(
352356
AgentAwarenessRelay.resolveAgentAwarenessRelayActiveThreadIds({
353357
environmentId,
358+
startedAt: Date.parse(now),
354359
projects: [
355360
{
356361
id: projectId,
@@ -374,6 +379,44 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {
374379
...baseThread,
375380
id: idleThreadId,
376381
},
382+
{
383+
...baseThread,
384+
id: oldCompletedId,
385+
latestTurn: {
386+
turnId: "turn-old" as TurnId,
387+
state: "completed",
388+
requestedAt: "2026-05-24T00:00:00.000Z",
389+
startedAt: "2026-05-24T00:00:00.000Z",
390+
completedAt: "2026-05-24T00:01:00.000Z",
391+
assistantMessageId: null,
392+
},
393+
},
394+
{
395+
...baseThread,
396+
id: newCompletedId,
397+
latestTurn: {
398+
turnId: "turn-new" as TurnId,
399+
state: "completed",
400+
requestedAt: "2026-05-25T00:00:01.000Z",
401+
startedAt: "2026-05-25T00:00:01.000Z",
402+
completedAt: "2026-05-25T00:00:02.000Z",
403+
assistantMessageId: null,
404+
},
405+
},
406+
{
407+
...baseThread,
408+
id: freshMessageId,
409+
latestUserMessageAt: "2026-05-25T00:00:01.000Z",
410+
session: {
411+
threadId: freshMessageId,
412+
status: "ready",
413+
providerName: "Codex",
414+
runtimeMode: "full-access",
415+
activeTurnId: null,
416+
lastError: null,
417+
updatedAt: "2026-05-25T00:00:02.000Z",
418+
},
419+
},
377420
{
378421
...baseThread,
379422
id: "thread-missing-project" as ThreadId,
@@ -389,7 +432,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {
389432
},
390433
],
391434
}),
392-
).toEqual([activeThreadId]);
435+
).toEqual([activeThreadId, newCompletedId]);
393436
});
394437

395438
it.effect("signs the activity publish JWT and rejects tampering", () =>
@@ -797,4 +840,138 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {
797840
}),
798841
),
799842
);
843+
844+
it.effect("does not alert for historical completions after startup", () =>
845+
Effect.scoped(
846+
Effect.gen(function* () {
847+
const secrets = makeMemorySecretStore();
848+
const now = yield* DateTime.now;
849+
const old = DateTime.formatIso(DateTime.add(now, { days: -7 }));
850+
const threadId = "thread-old" as ThreadId;
851+
const projectId = "project-1" as ProjectId;
852+
const environmentId = "env-1" as EnvironmentId;
853+
const project = {
854+
id: projectId,
855+
title: "T3 Code",
856+
workspaceRoot: "/workspace",
857+
repositoryIdentity: null,
858+
defaultModelSelection: null,
859+
scripts: [],
860+
createdAt: old,
861+
updatedAt: old,
862+
} satisfies OrchestrationProjectShell;
863+
const completedTurn = {
864+
turnId: "turn-1" as TurnId,
865+
state: "completed",
866+
requestedAt: old,
867+
startedAt: old,
868+
completedAt: old,
869+
assistantMessageId: null,
870+
} as const;
871+
const completedThread = {
872+
id: threadId,
873+
projectId,
874+
title: "Old task",
875+
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
876+
runtimeMode: "full-access",
877+
interactionMode: "default",
878+
branch: null,
879+
worktreePath: null,
880+
pullRequests: [],
881+
latestTurn: completedTurn,
882+
createdAt: old,
883+
updatedAt: old,
884+
archivedAt: null,
885+
settledOverride: null,
886+
settledAt: null,
887+
session: null,
888+
latestUserMessageAt: old,
889+
hasPendingApprovals: false,
890+
hasPendingUserInput: false,
891+
hasActionableProposedPlan: false,
892+
} satisfies OrchestrationThreadShell;
893+
let currentThread: OrchestrationThreadShell | null = completedThread;
894+
let publishes = 0;
895+
const originalFetch = globalThis.fetch;
896+
globalThis.fetch = (() => {
897+
publishes += 1;
898+
return Promise.resolve(Response.json({ ok: true, deliveries: [] }));
899+
}) as unknown as typeof fetch;
900+
yield* Effect.addFinalizer(() =>
901+
Effect.sync(() => {
902+
globalThis.fetch = originalFetch;
903+
}),
904+
);
905+
yield* secrets.setString(RELAY_URL_SECRET, "https://relay.example.test");
906+
yield* secrets.setString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "relay-credential");
907+
yield* secrets.setString(PUBLISH_AGENT_ACTIVITY_SECRET, "true");
908+
909+
const layer = Layer.mergeAll(
910+
Layer.succeed(ServerSecretStore.ServerSecretStore, secrets.store),
911+
Layer.succeed(ServerEnvironment.ServerEnvironment, {
912+
getEnvironmentId: Effect.succeed(environmentId),
913+
getDescriptor: Effect.die("unused descriptor"),
914+
}),
915+
Layer.succeed(OrchestrationEngineService, {} as OrchestrationEngineShape),
916+
Layer.succeed(ProjectionSnapshotQuery, {
917+
getThreadShellById: () => Effect.sync(() => Option.fromNullishOr(currentThread)),
918+
getProjectShellById: () => Effect.succeed(Option.some(project)),
919+
} as unknown as ProjectionSnapshotQueryShape),
920+
);
921+
922+
yield* Effect.gen(function* () {
923+
const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
924+
yield* relay.publishThread(threadId);
925+
expect(publishes).toBe(0);
926+
927+
currentThread = {
928+
...completedThread,
929+
latestTurn: null,
930+
latestUserMessageAt: DateTime.formatIso(DateTime.add(now, { seconds: 1 })),
931+
session: {
932+
threadId,
933+
status: "ready",
934+
providerName: "Codex",
935+
runtimeMode: "full-access",
936+
activeTurnId: null,
937+
lastError: null,
938+
updatedAt: DateTime.formatIso(DateTime.add(now, { seconds: 2 })),
939+
},
940+
};
941+
expect(
942+
AgentAwarenessRelay.resolveAgentAwarenessRelayPublishSnapshot({
943+
environmentId,
944+
threadId,
945+
thread: Option.some(currentThread),
946+
project: Option.some(project),
947+
}).state?.phase,
948+
).toBe("completed");
949+
yield* relay.publishThread(threadId);
950+
expect(publishes).toBe(0);
951+
952+
currentThread = {
953+
...completedThread,
954+
session: {
955+
threadId,
956+
status: "error",
957+
providerName: "Codex",
958+
runtimeMode: "full-access",
959+
activeTurnId: null,
960+
lastError: "old failure",
961+
updatedAt: old,
962+
},
963+
};
964+
yield* relay.publishThread(threadId);
965+
expect(publishes).toBe(0);
966+
}).pipe(
967+
Effect.provide(
968+
AgentAwarenessRelay.layer.pipe(
969+
Layer.provide(layer),
970+
Layer.provideMerge(NodeServices.layer),
971+
),
972+
),
973+
);
974+
}),
975+
),
976+
);
800977
});

‎apps/server/src/relay/AgentAwarenessRelay.ts‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -267,8 +267,13 @@ export function resolveAgentAwarenessRelayPublishSnapshot(input: {
267267
};
268268
}
269269

270+
function terminalWorkSinceStart(thread: OrchestrationThreadShell, startedAt: number): boolean {
271+
return Date.parse(thread.latestTurn?.completedAt ?? "") > startedAt;
272+
}
273+
270274
export function resolveAgentAwarenessRelayActiveThreadIds(input: {
271275
readonly environmentId: EnvironmentId;
276+
readonly startedAt: number;
272277
readonly projects: ReadonlyArray<Pick<OrchestrationProjectShell, "id" | "title">>;
273278
readonly threads: ReadonlyArray<OrchestrationThreadShell>;
274279
}): ReadonlyArray<ThreadId> {
@@ -279,12 +284,16 @@ export function resolveAgentAwarenessRelayActiveThreadIds(input: {
279284
if (!project) {
280285
return false;
281286
}
287+
const state = projectThreadAwareness({
288+
environmentId: input.environmentId,
289+
project,
290+
thread,
291+
});
282292
return (
283-
projectThreadAwareness({
284-
environmentId: input.environmentId,
285-
project,
286-
thread,
287-
}) !== null
293+
state !== null &&
294+
(state.phase !== "completed" && state.phase !== "failed"
295+
? true
296+
: terminalWorkSinceStart(thread, input.startedAt))
288297
);
289298
})
290299
.map((thread) => thread.id);
@@ -298,6 +307,7 @@ export const make = Effect.gen(function* () {
298307
const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;
299308
const crypto = yield* Crypto.Crypto;
300309
const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets);
310+
const startedAt = (yield* DateTime.now).epochMilliseconds;
301311
const activeSnapshotPublishedRef = yield* Ref.make(false);
302312
const publishedStateByThreadRef = yield* Ref.make(new Map<ThreadId, string>());
303313

@@ -417,6 +427,14 @@ export const make = Effect.gen(function* () {
417427
});
418428
const publishIdentity = agentAwarenessPublishIdentity(snapshot.state);
419429
const publishedStateByThread = yield* Ref.get(publishedStateByThreadRef);
430+
if (
431+
(snapshot.state?.phase === "completed" || snapshot.state?.phase === "failed") &&
432+
!publishedStateByThread.has(threadId)
433+
) {
434+
// Startup has no publish history. Only work from this server process may
435+
// produce an initial terminal alert; historical threads remain quiet.
436+
if (Option.isNone(thread) || !terminalWorkSinceStart(thread.value, startedAt)) return;
437+
}
420438
if (publishedStateByThread.get(threadId) === publishIdentity) {
421439
// The projection is back at (or never left) the last published state, so
422440
// any pending deferred confirmation is moot. Leaving the deadline in
@@ -495,7 +513,11 @@ export const make = Effect.gen(function* () {
495513
});
496514
yield* Ref.update(publishedStateByThreadRef, (publishedStates) => {
497515
const nextPublishedStates = new Map(publishedStates);
498-
nextPublishedStates.set(threadId, publishIdentity);
516+
if (snapshot.state === null) {
517+
nextPublishedStates.delete(threadId);
518+
} else {
519+
nextPublishedStates.set(threadId, publishIdentity);
520+
}
499521
return nextPublishedStates;
500522
});
501523
});
@@ -529,6 +551,7 @@ export const make = Effect.gen(function* () {
529551
const snapshot = yield* snapshotQuery.getShellSnapshot();
530552
const activeThreadIds = resolveAgentAwarenessRelayActiveThreadIds({
531553
environmentId,
554+
startedAt,
532555
projects: snapshot.projects,
533556
threads: snapshot.threads,
534557
});

0 commit comments

Comments
 (0)