Skip to content

Commit 890cfb6

Browse files
author
Michael Feth
committed
feat(desktop): persistent per-agent circuit-breaker health badge
Previously, when an agent's circuit breaker opened, the only visible sign was a transcript message in whatever channel happened to trigger the crash. Looking anywhere else in the app, or at a different channel, gave no indication the agent was suspended. Add a derived, channel-independent circuit-breaker status (getAgentCircuitStatus/useAgentCircuitStatus) built from the same circuit_open/circuit_recovered observer events, tracked per agent slot so a recovery on one slot can't mask another slot still being open. Render it as a persistent badge on ManagedAgentRow, clickable through to the channel that triggered it when known. Extracted compareObserverEvents/isObserverEventAfter into lib/observerEventOrdering.ts and the new circuit-status logic into agentCircuitStatus.ts to keep observerRelayStore.ts under the desktop file-size ratchet. Signed-off-by: Michael Feth <[email protected]>
1 parent ad44dc4 commit 890cfb6

6 files changed

Lines changed: 500 additions & 52 deletions

File tree

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import { normalizePubkey } from "@/shared/lib/pubkey";
2+
import type { ObserverEvent } from "./ui/agentSessionTypes";
3+
import { isObserverEventAfter } from "./lib/observerEventOrdering";
4+
5+
export type AgentCircuitStatus = {
6+
isOpen: boolean;
7+
message: string | null;
8+
channelId: string | null;
9+
timestamp: string | null;
10+
};
11+
12+
const IDLE_CIRCUIT_STATUS: AgentCircuitStatus = {
13+
isOpen: false,
14+
message: null,
15+
channelId: null,
16+
timestamp: null,
17+
};
18+
19+
// Latest known circuit-breaker state per (agent, slot). Slots are independent
20+
// prompt lanes within one managed-agent process (see SlotCircuit in
21+
// buzz-acp/src/lib.rs) — a recovery on slot 1 must not clear a still-open
22+
// circuit on slot 0, so state is tracked per slot and the derived per-agent
23+
// status reports "open" if ANY slot is open.
24+
type CircuitSlotState = {
25+
isOpen: boolean;
26+
message: string;
27+
channelId: string | null;
28+
timestamp: string;
29+
seq: number;
30+
};
31+
const circuitStateBySlot = new Map<string, CircuitSlotState>();
32+
33+
// Reference-stable per-agent snapshots for useSyncExternalStore, invalidated
34+
// only when that agent's circuit state actually changes.
35+
const circuitStatusCache = new Map<string, AgentCircuitStatus>();
36+
37+
function circuitSlotKey(
38+
agentPubkey: string,
39+
agentIndex: number | null | undefined,
40+
): string {
41+
return `${normalizePubkey(agentPubkey)}:${agentIndex ?? 0}`;
42+
}
43+
44+
/**
45+
* Fold a circuit_open/circuit_recovered event into per-slot circuit state.
46+
* Ignores any other event kind and any event that doesn't sort strictly after
47+
* the slot's currently-stored state, so a late/replayed frame can't regress
48+
* an already-newer status. Returns true if state actually changed.
49+
*/
50+
export function applyCircuitEvent(
51+
agentPubkey: string,
52+
event: ObserverEvent,
53+
): boolean {
54+
if (event.kind !== "circuit_open" && event.kind !== "circuit_recovered") {
55+
return false;
56+
}
57+
const key = circuitSlotKey(agentPubkey, event.agentIndex);
58+
const existing = circuitStateBySlot.get(key);
59+
if (existing && !isObserverEventAfter(event, existing)) {
60+
return false;
61+
}
62+
const payload = event.payload as { error?: unknown } | null;
63+
const message =
64+
typeof payload?.error === "string"
65+
? payload.error
66+
: event.kind === "circuit_open"
67+
? "Agent suspended (repeated crashes)"
68+
: "Agent recovered";
69+
circuitStateBySlot.set(key, {
70+
isOpen: event.kind === "circuit_open",
71+
message,
72+
channelId: event.channelId ?? null,
73+
timestamp: event.timestamp,
74+
seq: event.seq,
75+
});
76+
circuitStatusCache.delete(normalizePubkey(agentPubkey));
77+
return true;
78+
}
79+
80+
/**
81+
* Derived circuit-breaker status for one agent: open if ANY of its slots is
82+
* currently open. Reference-stable while unchanged (useSyncExternalStore-safe
83+
* — see the React.memo/useSyncExternalStore gotcha in this repo's CLAUDE.md).
84+
*/
85+
export function getAgentCircuitStatus(
86+
agentPubkey?: string | null,
87+
): AgentCircuitStatus {
88+
if (!agentPubkey) {
89+
return IDLE_CIRCUIT_STATUS;
90+
}
91+
const key = normalizePubkey(agentPubkey);
92+
const cached = circuitStatusCache.get(key);
93+
if (cached) {
94+
return cached;
95+
}
96+
const prefix = `${key}:`;
97+
let openSlot: CircuitSlotState | null = null;
98+
for (const [slotKey, slot] of circuitStateBySlot) {
99+
if (!slotKey.startsWith(prefix) || !slot.isOpen) continue;
100+
if (!openSlot || isObserverEventAfter(slot, openSlot)) {
101+
openSlot = slot;
102+
}
103+
}
104+
const status: AgentCircuitStatus = openSlot
105+
? {
106+
isOpen: true,
107+
message: openSlot.message,
108+
channelId: openSlot.channelId,
109+
timestamp: openSlot.timestamp,
110+
}
111+
: IDLE_CIRCUIT_STATUS;
112+
circuitStatusCache.set(key, status);
113+
return status;
114+
}
115+
116+
/** Clears all per-agent circuit state. Called from resetAgentObserverStore so
117+
* circuit state can't leak across community switches. */
118+
export function resetCircuitState() {
119+
circuitStateBySlot.clear();
120+
circuitStatusCache.clear();
121+
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import type { ObserverEvent } from "../ui/agentSessionTypes";
2+
3+
/**
4+
* Shared two-key ordering for observer events: later timestamp wins; equal
5+
* timestamp falls back to higher seq. Extracted out of observerRelayStore.ts
6+
* so other per-agent derived stores (e.g. agentCircuitStatus.ts) can apply the
7+
* exact same ordering without importing the whole observer store module and
8+
* without drifting from it.
9+
*/
10+
export function compareObserverEvents(
11+
left: ObserverEvent,
12+
right: ObserverEvent,
13+
) {
14+
const leftTime = Date.parse(left.timestamp);
15+
const rightTime = Date.parse(right.timestamp);
16+
if (Number.isFinite(leftTime) && Number.isFinite(rightTime)) {
17+
const timeDiff = leftTime - rightTime;
18+
if (timeDiff !== 0) {
19+
return timeDiff;
20+
}
21+
}
22+
23+
return left.seq - right.seq;
24+
}
25+
26+
/**
27+
* Returns true if `candidate` sorts strictly after `stored` using the same
28+
* two-key ordering as `compareObserverEvents`: later timestamp wins; equal
29+
* timestamp falls back to higher seq. Extracted so latest-live advancement
30+
* (and any other derived per-agent state) cannot drift from transcript
31+
* ordering.
32+
*/
33+
export function isObserverEventAfter(
34+
candidate: { timestamp: string; seq: number },
35+
stored: { timestamp: string; seq: number },
36+
): boolean {
37+
const candidateTime = Date.parse(candidate.timestamp);
38+
const storedTime = Date.parse(stored.timestamp);
39+
if (Number.isFinite(candidateTime) && Number.isFinite(storedTime)) {
40+
if (candidateTime !== storedTime) {
41+
return candidateTime > storedTime;
42+
}
43+
}
44+
return candidate.seq > stored.seq;
45+
}
Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
import assert from "node:assert/strict";
2+
import { describe, it, beforeEach, afterEach } from "node:test";
3+
4+
import {
5+
getAgentCircuitStatus,
6+
resetAgentObserverStore,
7+
_testProcessLiveObserverEvents,
8+
} from "./observerRelayStore.ts";
9+
10+
const AGENT =
11+
"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234";
12+
13+
function makeEvent(overrides) {
14+
return {
15+
seq: 1,
16+
timestamp: "2024-01-01T00:00:00Z",
17+
kind: "circuit_open",
18+
agentIndex: 0,
19+
channelId: "chan-1",
20+
sessionId: null,
21+
turnId: null,
22+
payload: null,
23+
...overrides,
24+
};
25+
}
26+
27+
describe("getAgentCircuitStatus", () => {
28+
beforeEach(() => {
29+
resetAgentObserverStore();
30+
});
31+
32+
afterEach(() => {
33+
resetAgentObserverStore();
34+
});
35+
36+
it("reports isOpen false when no circuit events have been seen for the agent", () => {
37+
const status = getAgentCircuitStatus(AGENT);
38+
assert.equal(status.isOpen, false);
39+
assert.equal(status.message, null);
40+
assert.equal(status.channelId, null);
41+
assert.equal(status.timestamp, null);
42+
});
43+
44+
it("flips isOpen true on a circuit_open event, surfacing its message/channel/timestamp", () => {
45+
_testProcessLiveObserverEvents(AGENT, [
46+
makeEvent({
47+
seq: 1,
48+
timestamp: "2024-01-01T00:00:00Z",
49+
kind: "circuit_open",
50+
agentIndex: 0,
51+
channelId: "chan-uuid-1",
52+
payload: {
53+
error:
54+
"Agent slot 0 panicked repeatedly and its circuit breaker is now open.",
55+
},
56+
}),
57+
]);
58+
59+
const status = getAgentCircuitStatus(AGENT);
60+
assert.equal(status.isOpen, true);
61+
assert.equal(
62+
status.message,
63+
"Agent slot 0 panicked repeatedly and its circuit breaker is now open.",
64+
);
65+
assert.equal(status.channelId, "chan-uuid-1");
66+
assert.equal(status.timestamp, "2024-01-01T00:00:00Z");
67+
});
68+
69+
it("flips isOpen back to false when a newer circuit_recovered event arrives on the same slot", () => {
70+
_testProcessLiveObserverEvents(AGENT, [
71+
makeEvent({
72+
seq: 1,
73+
timestamp: "2024-01-01T00:00:00Z",
74+
kind: "circuit_open",
75+
agentIndex: 0,
76+
channelId: "chan-uuid-1",
77+
payload: { error: "opened" },
78+
}),
79+
]);
80+
assert.equal(getAgentCircuitStatus(AGENT).isOpen, true);
81+
82+
_testProcessLiveObserverEvents(AGENT, [
83+
makeEvent({
84+
seq: 2,
85+
timestamp: "2024-01-01T00:01:00Z",
86+
kind: "circuit_recovered",
87+
agentIndex: 0,
88+
channelId: "chan-uuid-1",
89+
payload: { error: "recovered" },
90+
}),
91+
]);
92+
93+
const status = getAgentCircuitStatus(AGENT);
94+
assert.equal(status.isOpen, false);
95+
assert.equal(status.message, null);
96+
assert.equal(status.channelId, null);
97+
assert.equal(status.timestamp, null);
98+
});
99+
100+
it("reports isOpen true overall (any-slot-open) with two independent slots, one open and one recovered", () => {
101+
// Slot 0 stays open.
102+
_testProcessLiveObserverEvents(AGENT, [
103+
makeEvent({
104+
seq: 1,
105+
timestamp: "2024-01-01T00:00:00Z",
106+
kind: "circuit_open",
107+
agentIndex: 0,
108+
channelId: "chan-slot-0",
109+
payload: { error: "slot 0 opened" },
110+
}),
111+
]);
112+
// Slot 1 opens then recovers.
113+
_testProcessLiveObserverEvents(AGENT, [
114+
makeEvent({
115+
seq: 2,
116+
timestamp: "2024-01-01T00:00:01Z",
117+
kind: "circuit_open",
118+
agentIndex: 1,
119+
channelId: "chan-slot-1",
120+
payload: { error: "slot 1 opened" },
121+
}),
122+
makeEvent({
123+
seq: 3,
124+
timestamp: "2024-01-01T00:00:02Z",
125+
kind: "circuit_recovered",
126+
agentIndex: 1,
127+
channelId: "chan-slot-1",
128+
payload: { error: "slot 1 recovered" },
129+
}),
130+
]);
131+
132+
const status = getAgentCircuitStatus(AGENT);
133+
assert.equal(
134+
status.isOpen,
135+
true,
136+
"any slot still open must report the agent as open",
137+
);
138+
assert.equal(status.channelId, "chan-slot-0");
139+
assert.equal(status.message, "slot 0 opened");
140+
});
141+
142+
it("ignores a stale/out-of-order event arriving after a recovery", () => {
143+
_testProcessLiveObserverEvents(AGENT, [
144+
makeEvent({
145+
seq: 1,
146+
timestamp: "2024-01-01T00:00:00Z",
147+
kind: "circuit_open",
148+
agentIndex: 0,
149+
channelId: "chan-uuid-1",
150+
payload: { error: "opened" },
151+
}),
152+
makeEvent({
153+
seq: 2,
154+
timestamp: "2024-01-01T00:01:00Z",
155+
kind: "circuit_recovered",
156+
agentIndex: 0,
157+
channelId: "chan-uuid-1",
158+
payload: { error: "recovered" },
159+
}),
160+
]);
161+
assert.equal(getAgentCircuitStatus(AGENT).isOpen, false);
162+
163+
// A delayed circuit_open with an OLDER timestamp+seq than the currently
164+
// stored (recovered) state must be ignored — it must not reopen the
165+
// circuit or otherwise change status.
166+
_testProcessLiveObserverEvents(AGENT, [
167+
makeEvent({
168+
seq: 1,
169+
timestamp: "2024-01-01T00:00:00Z",
170+
kind: "circuit_open",
171+
agentIndex: 0,
172+
channelId: "chan-uuid-1",
173+
payload: { error: "stale reopen" },
174+
}),
175+
]);
176+
177+
const status = getAgentCircuitStatus(AGENT);
178+
assert.equal(
179+
status.isOpen,
180+
false,
181+
"a stale out-of-order event must not regress an already-newer status",
182+
);
183+
assert.equal(status.message, null);
184+
assert.equal(status.channelId, null);
185+
assert.equal(status.timestamp, null);
186+
});
187+
188+
it("returns a stable reference across repeated calls with no intervening event", () => {
189+
_testProcessLiveObserverEvents(AGENT, [
190+
makeEvent({
191+
seq: 1,
192+
timestamp: "2024-01-01T00:00:00Z",
193+
kind: "circuit_open",
194+
agentIndex: 0,
195+
channelId: "chan-uuid-1",
196+
payload: { error: "opened" },
197+
}),
198+
]);
199+
200+
const ref1 = getAgentCircuitStatus(AGENT);
201+
const ref2 = getAgentCircuitStatus(AGENT);
202+
assert.strictEqual(
203+
ref1,
204+
ref2,
205+
"must return the cached object reference when nothing changed — a " +
206+
"regression here would cause an infinite useSyncExternalStore render loop",
207+
);
208+
});
209+
210+
it("resetAgentObserverStore clears circuit state back to isOpen:false", () => {
211+
_testProcessLiveObserverEvents(AGENT, [
212+
makeEvent({
213+
seq: 1,
214+
timestamp: "2024-01-01T00:00:00Z",
215+
kind: "circuit_open",
216+
agentIndex: 0,
217+
channelId: "chan-uuid-1",
218+
payload: { error: "opened" },
219+
}),
220+
]);
221+
assert.equal(getAgentCircuitStatus(AGENT).isOpen, true);
222+
223+
resetAgentObserverStore();
224+
225+
const status = getAgentCircuitStatus(AGENT);
226+
assert.equal(status.isOpen, false);
227+
assert.equal(status.message, null);
228+
assert.equal(status.channelId, null);
229+
assert.equal(status.timestamp, null);
230+
});
231+
});

0 commit comments

Comments
 (0)