Skip to content

Commit a701fdf

Browse files
anandgupta42claude
andcommitted
fix: finalize serve traces on shutdown + harden trace consumer (#886)
Addresses the multi-model consensus review on PR #886. The `TraceConsumer` extraction was sound, but the `serve` integration had lifecycle gaps. `cli/cmd/serve.ts`: - Capture the `subscribeTraceConsumer` handle and wire `SIGINT`/`SIGTERM`/ `beforeExit` to `stop()` so traces are finalized on shutdown instead of left un-`completed`. Mirrors the existing pattern in `cli/cmd/run.ts`. `altimate/observability/trace-consumer.ts`: - `stop()` now drains the event loop (bounded by a 1s timeout) before `flush()`, so finalization can't race an in-flight `handleEvent`. - The `for await` stream loop catches mid-stream errors and reconnects with exponential backoff (250ms -> 30s) instead of dying permanently; backoff sleeps are abortable so shutdown isn't delayed. - Finalize and evict per-session state on `session.deleted` (a real end-of-session signal) so long-lived serve processes don't accumulate `sessionUserMsgIds` / cached traces. - `loadConfig` failure now fails safe (`enabled = false`) instead of silently tracing to the default dir, and logs at debug. - `getOrCreateTrace` and `handleEvent` catch blocks log at debug (still never throw) for headless diagnosability. - Auth header uses `Buffer.from(...).toString("base64")` (UTF-8 safe) instead of `btoa`. - Added a test seam: `subscribeTraceConsumer` accepts an optional injected `consumer` and `subscribe` source (production behaviour unchanged via defaults). Tests: - New `subscribeTraceConsumer` coverage: `stop()` drains + finalizes; a mid-stream throw reconnects rather than killing the loop. - New `TraceConsumer` coverage: incremental snapshots land on disk BEFORE any flush (the real serve path), and `session.deleted` finalizes + releases state. - De-flaked the `reset()` test (poll instead of fixed `setTimeout`). - Registered the `session.deleted` literal in `bridge-merge-invariants`. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 500fed8 commit a701fdf

4 files changed

Lines changed: 272 additions & 39 deletions

File tree

‎packages/opencode/src/altimate/observability/trace-consumer.ts‎

Lines changed: 114 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,12 @@ export class TraceConsumer {
9090
}
9191
this.exporters = exporters
9292
this.maxFiles = tc?.maxFiles
93-
} catch {
93+
} catch (error) {
9494
// Config failure should not prevent the host (TUI/serve) from working
95+
this.enabled = false
96+
Log.Default.debug("[tracing] failed to load config, disabling", {
97+
error: error instanceof Error ? error.message : String(error),
98+
})
9599
}
96100
}
97101

@@ -138,7 +142,10 @@ export class TraceConsumer {
138142
Trace.setActive(trace)
139143
this.sessionTraces.set(sessionID, trace)
140144
return trace
141-
} catch {
145+
} catch (error) {
146+
Log.Default.debug("[tracing] getOrCreateTrace failed", {
147+
error: error instanceof Error ? error.message : String(error),
148+
})
142149
return null
143150
}
144151
}
@@ -250,14 +257,32 @@ export class TraceConsumer {
250257
if (trace) trace.setTitle(String(info.title))
251258
}
252259
}
260+
// Finalize and evict on a real session-end signal. `session.deleted` is a
261+
// true end-of-session event (unlike `idle`, which fires every turn), so
262+
// it is the correct place to flush the trace and release per-session
263+
// state. Without this, in a long-lived `serve` process the trace and its
264+
// sessionUserMsgIds set live until MAX_TRACES eviction (or never, for
265+
// <100 sessions), since reset()/flush() don't fire during normal serve.
266+
if (e.type === "session.deleted") {
267+
const info = e.properties?.info as Record<string, any> | undefined
268+
if (info?.id) {
269+
const trace = this.sessionTraces.get(info.id)
270+
if (trace) void trace.endTrace().catch(() => {})
271+
this.sessionTraces.delete(info.id)
272+
this.sessionUserMsgIds.delete(info.id)
273+
}
274+
}
253275
// DO NOT finalize the trace on session.status=idle. `idle` fires after
254276
// every turn (busy → idle), not at session end. Finalizing per-turn would
255277
// treat each turn as session end and the next turn's events would hit a
256278
// cache miss; getOrCreateTrace rehydrates from disk as defense in depth,
257279
// but the correct behaviour is to keep the Trace live across turns and
258280
// finalize only on flush() (shutdown) and MAX_TRACES eviction.
259-
} catch {
260-
// Trace must never interrupt event forwarding
281+
} catch (error) {
282+
// Trace must never interrupt event forwarding — log at debug only.
283+
Log.Default.debug("[tracing] handleEvent error", {
284+
error: error instanceof Error ? error.message : String(error),
285+
})
261286
}
262287
}
263288

@@ -295,58 +320,116 @@ export class TraceConsumer {
295320
*
296321
* Trace failures must never affect the server, so every step is best-effort.
297322
*/
298-
export function subscribeTraceConsumer(input: { directory: string }): { stop: () => Promise<void> } {
299-
const consumer = new TraceConsumer()
323+
/** One subscription's event stream. */
324+
export interface TraceEventSource {
325+
stream: AsyncIterable<unknown>
326+
}
327+
328+
export interface TraceSubscribeOptions {
329+
/** Test seam: inject the consumer (e.g. with a temp-dir FileExporter). */
330+
consumer?: TraceConsumer
331+
/**
332+
* Test seam: provide the event source. Resolves to a stream of bus events,
333+
* or `undefined` to trigger a backoff+retry. Defaults to the in-process SDK
334+
* subscription. Called once per (re)connect with the shutdown signal.
335+
*/
336+
subscribe?: (signal: AbortSignal) => Promise<TraceEventSource | undefined>
337+
}
338+
339+
export function subscribeTraceConsumer(
340+
input: { directory: string },
341+
options?: TraceSubscribeOptions,
342+
): { stop: () => Promise<void> } {
343+
const consumer = options?.consumer ?? new TraceConsumer()
300344
const abort = new AbortController()
301345
const signal = abort.signal
302346

303-
// In-process fetch against the default app — same pattern as the TUI
304-
// worker. The Bus is process-wide, so events published by sessions served
305-
// by the TCP listener arrive on this subscription too.
306-
const fetchFn = (async (fetchInput: RequestInfo | URL, init?: RequestInit) => {
307-
const request = new Request(fetchInput, init)
308-
const password = Flag.OPENCODE_SERVER_PASSWORD
309-
if (password) {
310-
const username = Flag.OPENCODE_SERVER_USERNAME ?? "opencode"
311-
request.headers.set("Authorization", `Basic ${btoa(`${username}:${password}`)}`)
312-
}
313-
return Server.Default().fetch(request)
314-
}) as typeof globalThis.fetch
347+
// Default event source: the in-process SDK subscription — same pattern as the
348+
// TUI worker. The Bus is process-wide, so events published by sessions served
349+
// by the TCP listener arrive on this subscription too. The SDK client is
350+
// built once; `subscribe` is invoked per (re)connect.
351+
const subscribe =
352+
options?.subscribe ??
353+
(() => {
354+
const fetchFn = (async (fetchInput: RequestInfo | URL, init?: RequestInit) => {
355+
const request = new Request(fetchInput, init)
356+
const password = Flag.OPENCODE_SERVER_PASSWORD
357+
if (password) {
358+
const username = Flag.OPENCODE_SERVER_USERNAME ?? "opencode"
359+
request.headers.set("Authorization", `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`)
360+
}
361+
return Server.Default().fetch(request)
362+
}) as typeof globalThis.fetch
315363

316-
const sdk = createOpencodeClient({
317-
baseUrl: "http://altimate-code.internal",
318-
directory: input.directory,
319-
fetch: fetchFn,
320-
signal,
321-
})
364+
const sdk = createOpencodeClient({
365+
baseUrl: "http://altimate-code.internal",
366+
directory: input.directory,
367+
fetch: fetchFn,
368+
signal,
369+
})
370+
371+
return (sig: AbortSignal): Promise<TraceEventSource | undefined> =>
372+
Promise.resolve(sdk.event.subscribe({}, { signal: sig }))
373+
.then((r) => r as unknown as TraceEventSource)
374+
.catch(() => undefined)
375+
})()
322376

323-
;(async () => {
377+
// Abortable sleep so a pending reconnect-backoff doesn't delay shutdown.
378+
const BASE_BACKOFF_MS = 250
379+
const MAX_BACKOFF_MS = 30_000
380+
const backoffSleep = (ms: number) => sleep(ms, undefined, { signal }).catch(() => {})
381+
382+
const loopPromise = (async () => {
324383
await consumer.loadConfig()
384+
// Exponential backoff on repeated failure so a durably-down stream doesn't
385+
// hot-loop; reset to the base delay after a successful subscription.
386+
let backoff = BASE_BACKOFF_MS
325387
while (!signal.aborted) {
326-
const events = await Promise.resolve(sdk.event.subscribe({}, { signal })).catch(() => undefined)
388+
const events = await subscribe(signal).catch(() => undefined)
327389

328390
if (!events) {
329-
await sleep(250)
391+
await backoffSleep(backoff)
392+
backoff = Math.min(backoff * 2, MAX_BACKOFF_MS)
330393
continue
331394
}
395+
backoff = BASE_BACKOFF_MS
332396

333-
for await (const event of events.stream) {
334-
await consumer.handleEvent(event)
397+
// The try/catch MUST sit inside the while loop: a mid-stream throw
398+
// (network disconnect, server hiccup) would otherwise escape to the
399+
// outer .catch and kill the loop permanently for the server's lifetime.
400+
try {
401+
for await (const event of events.stream) {
402+
await consumer.handleEvent(event)
403+
}
404+
} catch (err) {
405+
if (!signal.aborted) {
406+
Log.Default.warn("[tracing] trace event stream disconnected, reconnecting", {
407+
error: err instanceof Error ? err.message : String(err),
408+
})
409+
}
335410
}
336411

337412
if (!signal.aborted) {
338-
await sleep(250)
413+
await backoffSleep(backoff)
414+
backoff = Math.min(backoff * 2, MAX_BACKOFF_MS)
339415
}
340416
}
341-
})().catch((error) => {
417+
})()
418+
419+
loopPromise.catch((error: unknown) => {
342420
Log.Default.error("trace event stream error", {
343421
error: error instanceof Error ? error.message : error,
344422
})
345423
})
346424

347425
return {
348426
stop: async () => {
427+
// Stop accepting new work, then DRAIN the loop before flushing so we
428+
// don't finalize traces while a handleEvent() is still mid-event.
429+
// Bounded by a timeout so an unresponsive stream can't hang shutdown —
430+
// flush() finalizes whatever is in the cache either way.
349431
abort.abort()
432+
await Promise.race([loopPromise.catch(() => {}), sleep(1000)])
350433
await consumer.flush()
351434
},
352435
}

‎packages/opencode/src/cli/cmd/serve.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,26 @@ export const ServeCommand = cmd({
2929
// serve mode. Subscribe the shared trace consumer to the in-process
3030
// event stream so serve sessions produce the same trace files as the
3131
// terminal entrypoints.
32-
subscribeTraceConsumer({ directory: process.cwd() })
32+
const traceSub = subscribeTraceConsumer({ directory: process.cwd() })
33+
34+
// Finalize traces on shutdown. `serve` blocks forever on the promise below
35+
// and otherwise dies abruptly on signal, so without these handlers the
36+
// consumer's stop()/flush()/endTrace() never runs and serve traces are
37+
// left un-finalized (status never "completed", no summary/narrative).
38+
// Mirrors the SIGINT/SIGTERM/beforeExit pattern in cli/cmd/run.ts.
39+
let isShuttingDown = false
40+
const shutdown = async () => {
41+
if (isShuttingDown) return
42+
isShuttingDown = true
43+
await traceSub.stop()
44+
await server.stop()
45+
process.exit(0)
46+
}
47+
process.once("SIGINT", () => void shutdown())
48+
process.once("SIGTERM", () => void shutdown())
49+
process.once("beforeExit", () => void shutdown())
3350
// altimate_change end
3451

3552
await new Promise(() => {})
36-
await server.stop()
3753
},
3854
})

‎packages/opencode/test/altimate/trace-consumer.test.ts‎

Lines changed: 139 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test"
1818
import fs from "fs/promises"
1919
import path from "path"
2020
import os from "os"
21-
import { TraceConsumer } from "../../src/altimate/observability/trace-consumer"
21+
import {
22+
TraceConsumer,
23+
subscribeTraceConsumer,
24+
type TraceEventSource,
25+
} from "../../src/altimate/observability/trace-consumer"
2226
import { FileExporter, type TraceFile } from "../../src/altimate/observability/tracing"
2327

2428
let tmpDir: string
@@ -47,6 +51,51 @@ async function feed(consumer: TraceConsumer, events: unknown[]) {
4751
}
4852
}
4953

54+
/**
55+
* Poll until `predicate` holds, instead of a fixed sleep — snapshot/endTrace
56+
* writes are async + debounced, so a fixed delay is flaky under CI load.
57+
*/
58+
async function waitFor<T>(read: () => Promise<T>, predicate: (v: T) => boolean, timeoutMs = 5000): Promise<T> {
59+
const start = Date.now()
60+
for (;;) {
61+
try {
62+
const v = await read()
63+
if (predicate(v)) return v
64+
} catch {
65+
// not ready yet (e.g. file not written) — keep polling
66+
}
67+
if (Date.now() - start > timeoutMs) throw new Error("waitFor: condition not met within timeout")
68+
await new Promise((r) => setTimeout(r, 10))
69+
}
70+
}
71+
72+
/**
73+
* Build a single-use event source for the `subscribeTraceConsumer` test seam.
74+
* Optionally throws mid-stream (to exercise reconnect) or holds the stream open
75+
* until the shutdown signal fires (to exercise stop()/drain).
76+
*/
77+
function eventSource(
78+
events: unknown[],
79+
opts?: { throwAfter?: number; holdUntilAbort?: AbortSignal },
80+
): TraceEventSource {
81+
async function* gen() {
82+
let i = 0
83+
for (const e of events) {
84+
yield e
85+
i++
86+
if (opts?.throwAfter !== undefined && i >= opts.throwAfter) throw new Error("simulated stream disconnect")
87+
}
88+
const sig = opts?.holdUntilAbort
89+
if (sig) {
90+
await new Promise<void>((resolve) => {
91+
if (sig.aborted) return resolve()
92+
sig.addEventListener("abort", () => resolve(), { once: true })
93+
})
94+
}
95+
}
96+
return { stream: gen() }
97+
}
98+
5099
/** Event sequence mirroring what a real session emits over the bus. */
51100
function sessionEvents(sessionID: string) {
52101
const now = Date.now()
@@ -260,10 +309,95 @@ describe("TraceConsumer", () => {
260309
const consumer = makeConsumer()
261310
await feed(consumer, sessionEvents("ses_consumer_reset"))
262311
consumer.reset()
263-
// reset finalizes fire-and-forget; give the async endTrace a tick.
264-
await new Promise((r) => setTimeout(r, 100))
265-
266-
const trace = await readTraceFile("ses_consumer_reset")
312+
// reset finalizes fire-and-forget; poll for the write instead of a fixed
313+
// sleep so the test isn't flaky under CI load.
314+
const trace = await waitFor(
315+
() => readTraceFile("ses_consumer_reset"),
316+
(t) => t.spans.some((s) => s.kind === "tool"),
317+
)
267318
expect(trace.spans.some((s) => s.kind === "tool")).toBe(true)
268319
})
320+
321+
test("incremental snapshots land on disk BEFORE any flush (the serve path)", async () => {
322+
// serve never calls flush() during normal operation — it relies entirely
323+
// on the incremental snapshots written as events arrive. This is the path
324+
// every other test skips by flushing first.
325+
const consumer = makeConsumer()
326+
await feed(consumer, sessionEvents("ses_consumer_noflush"))
327+
const trace = await waitFor(
328+
() => readTraceFile("ses_consumer_noflush"),
329+
(t) => t.spans.some((s) => s.kind === "tool"),
330+
)
331+
expect(trace.summary.totalToolCalls).toBe(1)
332+
expect(trace.metadata.prompt).toBe("list my files")
333+
})
334+
335+
test("session.deleted finalizes the trace and releases per-session state", async () => {
336+
const consumer = makeConsumer()
337+
await feed(consumer, sessionEvents("ses_consumer_del"))
338+
await consumer.handleEvent({ type: "session.deleted", properties: { info: { id: "ses_consumer_del" } } })
339+
// endTrace is fire-and-forget on session.deleted; poll for finalization.
340+
const trace = await waitFor(
341+
() => readTraceFile("ses_consumer_del"),
342+
(t) => t.summary.status === "completed",
343+
)
344+
expect(trace.summary.status).toBe("completed")
345+
// State is already released, so a later flush must be a harmless no-op.
346+
await consumer.flush()
347+
})
348+
})
349+
350+
describe("subscribeTraceConsumer (serve integration)", () => {
351+
test("stop() drains the loop and flushes — serve traces are finalized", async () => {
352+
const consumer = makeConsumer()
353+
let calls = 0
354+
const sub = subscribeTraceConsumer(
355+
{ directory: tmpDir },
356+
{
357+
consumer,
358+
subscribe: async (signal) => {
359+
calls++
360+
// Deliver a full session on the first connect, then hold the stream
361+
// open until shutdown so stop() exercises the drain path.
362+
return eventSource(calls === 1 ? sessionEvents("ses_sub_stop") : [], { holdUntilAbort: signal })
363+
},
364+
},
365+
)
366+
// Wait until the session's events have been consumed (snapshot on disk).
367+
await waitFor(
368+
() => readTraceFile("ses_sub_stop"),
369+
(t) => t.spans.some((s) => s.kind === "tool"),
370+
)
371+
// Pre-stop the trace is not yet finalized; stop() must flush → "completed".
372+
await sub.stop()
373+
const trace = await readTraceFile("ses_sub_stop")
374+
expect(trace.summary.status).toBe("completed")
375+
expect(trace.summary.totalToolCalls).toBe(1)
376+
})
377+
378+
test("a mid-stream throw does not kill the loop — it reconnects", async () => {
379+
const consumer = makeConsumer()
380+
let calls = 0
381+
const sub = subscribeTraceConsumer(
382+
{ directory: tmpDir },
383+
{
384+
consumer,
385+
subscribe: async (signal) => {
386+
calls++
387+
// 1st connect throws mid-stream; 2nd connect delivers a different
388+
// session in full. If the throw killed the loop, session B's file
389+
// would never appear and the wait below would time out.
390+
if (calls === 1) return eventSource(sessionEvents("ses_sub_A"), { throwAfter: 2 })
391+
return eventSource(sessionEvents("ses_sub_B"), { holdUntilAbort: signal })
392+
},
393+
},
394+
)
395+
const trace = await waitFor(
396+
() => readTraceFile("ses_sub_B"),
397+
(t) => t.summary.totalToolCalls >= 1,
398+
)
399+
expect(trace.summary.totalToolCalls).toBe(1)
400+
expect(calls).toBeGreaterThanOrEqual(2)
401+
await sub.stop()
402+
})
269403
})

0 commit comments

Comments
 (0)