Skip to content

Commit 3dc245b

Browse files
committed
fix(tracing): harden follow-ups per multi-model review (#901, #902, #903)
Addresses findings from the GPT-5.4 + Kimi-K2.5 review panel: #902 (MAJOR) — key the orphan-writer guard on a monotonic `streamGeneration` counter bumped in startEventStream, not on AbortController object identity. The identity check was correct only while startEventStream always allocates a fresh controller; the counter makes ownership explicit and removes that hidden dependency. The check and the cache insert run in one synchronous turn (no await between), so the insert can't race a later startEventStream. #903 (MAJOR) — capSpansForSerialization now explicitly guarantees the structural root (session) span survives the cut even if it isn't in the head slice, instead of relying on "root happens to be index 0". Rehydrate and the viewer tree both require the root, and the elision marker is parented to it. Added table-driven tests: root-not-at-index-0, tiny caps (1–5) with marker-parent validity, the exact head+tail+1===length boundary, and elided-count accuracy. #901 (Kimi blockers) — interrupted spans now render amber (not red) on EVERY viewer surface, not just the preview/detail: waterfall row + icon class, tree-view meta ("interrupted" in orange), and log-view row (⚠ amber). errSpans already excluded them; this makes the visual treatment consistent so a reconstructed trace never looks like a failed one anywhere. Known minor (deferred, noted on PR): a trace containing ONLY interrupted spans no longer contributes to the summary error count (correct) but also shows no "incomplete" hint in the summary; a dedicated interrupted/incomplete banner is a small follow-up.
1 parent a805df8 commit 3dc245b

4 files changed

Lines changed: 111 additions & 19 deletions

File tree

‎packages/opencode/src/altimate/observability/tracing.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -386,10 +386,19 @@ export function capSpansForSerialization(spans: TraceSpan[], cap: number = MAX_S
386386
// Only elide if the result is actually smaller than the input (+1 for the
387387
// marker we'd add) — otherwise there's nothing to gain.
388388
if (headCount + tailCount + 1 >= spans.length) return spans
389-
const head = spans.slice(0, headCount)
389+
let head = spans.slice(0, headCount)
390390
const tail = spans.slice(spans.length - tailCount)
391+
// Guarantee the structural root (session) span survives the cut even if it
392+
// isn't in the head slice — rehydrate and the viewer's tree both require it,
393+
// and the elision marker is parented to it. In practice the root is index 0
394+
// (pushed first), so this is defensive, but it makes the invariant explicit
395+
// instead of silently depending on span ordering.
396+
const rootSpan = spans.find((s) => s.parentSpanId === null) ?? null
397+
if (rootSpan && !head.some((s) => s.spanId === rootSpan.spanId) && !tail.some((s) => s.spanId === rootSpan.spanId)) {
398+
head = [rootSpan, ...head.slice(0, headCount - 1)]
399+
}
391400
const elided = spans.length - head.length - tail.length
392-
const rootId = spans.find((s) => s.parentSpanId === null)?.spanId ?? null
401+
const rootId = rootSpan?.spanId ?? null
393402
const anchor = head[head.length - 1]
394403
const anchorTime = anchor?.endTime ?? anchor?.startTime ?? 0
395404
const marker: TraceSpan = {

‎packages/opencode/src/altimate/observability/viewer.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1247,12 +1247,12 @@ function showDetail(span) {
12471247
var dur = (span.endTime || Date.now()) - (span.startTime||0);
12481248
var left = (st / tTotal * 100).toFixed(2);
12491249
var width = Math.max(0.5, dur / tTotal * 100).toFixed(2);
1250-
var cls = span.status === 'error' ? 'error' : e(span.kind);
1250+
var cls = (span.status === 'error' && !span.interrupted) ? 'error' : e(span.kind);
12511251
var row = document.createElement('div');
12521252
row.className = 'wf-row';
12531253
row.setAttribute('data-idx', String(idx));
12541254
if (span.spanId) row.setAttribute('data-span-id', span.spanId);
1255-
var iconCls = span.status === 'error' ? 'error' : e(span.kind);
1255+
var iconCls = (span.status === 'error' && !span.interrupted) ? 'error' : e(span.kind);
12561256
var pv = getPreview(span);
12571257
row.innerHTML = '<div class="wf-icon ' + iconCls + '">' + (icons[span.kind]||'\\u2022') + '</div>' +
12581258
'<div class="wf-info"><div class="wf-name">' + e(span.name) + '</div>' + (pv ? '<div class="wf-preview">' + pv + '</div>' : '') + '</div>' +
@@ -1286,7 +1286,8 @@ function showDetail(span) {
12861286
meta.push(fd(dur));
12871287
if (span.tokens) meta.push(Number(span.tokens.total||0) + ' tok');
12881288
if (span.cost) meta.push(fc(span.cost));
1289-
if (span.status === 'error') meta.push('<span style="color:var(--red)">error</span>');
1289+
if (span.interrupted) meta.push('<span style="color:var(--orange)">interrupted</span>');
1290+
else if (span.status === 'error') meta.push('<span style="color:var(--red)">error</span>');
12901291
html += '<div class="tree-node"><div class="tree-item" data-idx="' + idx + '">';
12911292
html += '<div class="tree-head">';
12921293
html += '<span class="tree-type ' + e(span.kind) + '">' + e(span.kind) + '</span>';
@@ -1398,7 +1399,7 @@ function showDetail(span) {
13981399
if (span.kind === 'session') return;
13991400
var idx = spans.indexOf(span);
14001401
var ts = span.startTime ? new Date(span.startTime).toISOString().slice(11,23) : '';
1401-
var kindCls = span.status === 'error' ? 'error' : e(span.kind);
1402+
var kindCls = (span.status === 'error' && !span.interrupted) ? 'error' : e(span.kind);
14021403
html += '<div class="log-entry" data-idx="' + idx + '">';
14031404
html += '<span class="log-ts">' + ts + '</span>';
14041405
var logIcon = span.kind === 'generation' ? '\\u2B50' : span.kind === 'tool' ? '\\u2692' : '\\u25A0';
@@ -1408,7 +1409,8 @@ function showDetail(span) {
14081409
if (span.tokens) html += ' <span style="color:var(--dim);font-size:11px">' + Number(span.tokens.total||0) + ' tok</span>';
14091410
if (span.cost) html += ' <span style="color:var(--orange);font-size:11px">' + fc(span.cost) + '</span>';
14101411
if (span.tool && span.tool.durationMs != null) html += ' <span style="color:var(--dim);font-size:11px">' + fd(span.tool.durationMs) + '</span>';
1411-
if (span.status === 'error') html += ' <span style="color:var(--red);font-size:11px">\\u2718 ' + e((span.statusMessage||'').slice(0,100)) + '</span>';
1412+
if (span.interrupted) html += ' <span style="color:var(--orange);font-size:11px">\\u26A0 ' + e((span.statusMessage||'').slice(0,100)) + '</span>';
1413+
else if (span.status === 'error') html += ' <span style="color:var(--red);font-size:11px">\\u2718 ' + e((span.statusMessage||'').slice(0,100)) + '</span>';
14121414
if (span.kind === 'tool' && span.input) {
14131415
var logPv = getPreview(span);
14141416
if (logPv) html += '<div class="log-data" style="color:var(--cyan);opacity:0.7;max-height:none">' + logPv + '</div>';

‎packages/opencode/src/cli/cmd/tui/worker.ts‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,14 @@ const eventStream = {
4747
abort: undefined as AbortController | undefined,
4848
}
4949

50+
// altimate_change start — trace: monotonic stream generation. Bumped on every
51+
// startEventStream() so an in-flight getOrCreateTrace() can detect that its
52+
// owning stream was torn down while it was suspended at an await. Keyed on a
53+
// counter rather than the AbortController's object identity so the guard does
54+
// not silently depend on startEventStream always allocating a fresh controller.
55+
let streamGeneration = 0
56+
// altimate_change end
57+
5058
// altimate_change start — trace: per-session traces
5159
const sessionTraces = new Map<string, Trace>()
5260
const sessionUserMsgIds = new Map<string, Set<string>>() // Per-session user message IDs (cleaned up on session end)
@@ -83,10 +91,12 @@ async function loadTracingConfig() {
8391
async function getOrCreateTrace(sessionID: string): Promise<Trace | null> {
8492
if (!sessionID || !tracingEnabled) return null
8593
if (sessionTraces.has(sessionID)) return sessionTraces.get(sessionID)!
86-
// altimate_change start — capture the stream that owns this call so we can
87-
// detect a concurrent startEventStream() (e.g. setWorkspace) that aborted us
88-
// and cleared the cache while we were suspended at the rehydrate await below.
89-
const streamAtEntry = eventStream.abort
94+
// altimate_change start — capture the stream generation that owns this call so
95+
// we can detect a concurrent startEventStream() (e.g. setWorkspace) that
96+
// aborted us and cleared the cache while we were suspended at the rehydrate
97+
// await below. A counter (not AbortController identity) so we don't depend on
98+
// startEventStream's allocation strategy.
99+
const generationAtEntry = streamGeneration
90100
// altimate_change end
91101
try {
92102
if (sessionTraces.size >= MAX_TRACES) {
@@ -115,15 +125,17 @@ async function getOrCreateTrace(sessionID: string): Promise<Trace | null> {
115125
// awaiting rehydrate, this Trace belongs to a stream that's already been
116126
// aborted and its cache cleared. Inserting it now would resurrect an orphan
117127
// writer into the freshly-cleared map. Discard it and defer to whatever the
118-
// live stream has for this session.
119-
if (eventStream.abort !== streamAtEntry) {
128+
// live stream has. The check and the set below run in the same synchronous
129+
// turn (no await between them), so the insert can't race a later
130+
// startEventStream — this closes the suspend-at-await hole specifically.
131+
if (streamGeneration !== generationAtEntry) {
120132
void trace.endTrace().catch(() => {})
121133
return sessionTraces.get(sessionID) ?? null
122134
}
123-
// altimate_change end
124135
Trace.setActive(trace)
125136
sessionTraces.set(sessionID, trace)
126137
return trace
138+
// altimate_change end
127139
} catch {
128140
return null
129141
}
@@ -132,6 +144,10 @@ async function getOrCreateTrace(sessionID: string): Promise<Trace | null> {
132144

133145
const startEventStream = (input: { directory: string; workspaceID?: string }) => {
134146
if (eventStream.abort) eventStream.abort.abort()
147+
// altimate_change start — new stream generation; invalidates any in-flight
148+
// getOrCreateTrace() suspended at its rehydrate await (see generationAtEntry).
149+
streamGeneration++
150+
// altimate_change end
135151
// Clear stale per-stream trace state before starting a new stream instance
136152
for (const [, trace] of sessionTraces) {
137153
void trace.endTrace().catch(() => {})

‎packages/opencode/test/altimate/tracing-followups.test.ts‎

Lines changed: 70 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ describe("#901 — interrupted spans are marked and not counted as errors", () =
8383
expect(html).toContain("!sp.interrupted")
8484
// the interrupted span's flag survives into the embedded data
8585
expect(html).toContain('"interrupted":true')
86+
// the secondary surfaces (waterfall row class, tree meta, log row) all
87+
// gate the red/error treatment on !span.interrupted so reconstructed spans
88+
// render amber, not red, everywhere — not just in the preview/detail.
89+
expect(html).toContain("span.status === 'error' && !span.interrupted")
90+
expect(html).toContain("color:var(--orange)\">interrupted</span>")
8691
})
8792
})
8893

@@ -137,29 +142,89 @@ describe("#902 — getOrCreateTrace guards against resurrecting a Trace into a c
137142
// import), so it can't be unit-tested in-process. Lock the guard's shape with
138143
// a scope-bounded source contract, the same approach as
139144
// worker-trace-clearing.test.ts.
140-
test("captures the active stream before the rehydrate await and re-checks it after", async () => {
145+
test("captures the stream generation before the rehydrate await and re-checks it after", async () => {
141146
const workerSrc = await fs.readFile(
142147
path.join(__dirname, "../../src/cli/cmd/tui/worker.ts"),
143148
"utf-8",
144149
)
145150

146-
// The owning stream is captured at entry, before any await.
147-
expect(workerSrc).toMatch(/const streamAtEntry = eventStream\.abort/)
151+
// Ownership is keyed on a monotonic counter, not AbortController identity.
152+
expect(workerSrc).toMatch(/let streamGeneration = 0/)
153+
// The owning generation is captured at entry, before any await.
154+
expect(workerSrc).toMatch(/const generationAtEntry = streamGeneration/)
155+
// A new stream bumps the counter, invalidating in-flight calls.
156+
expect(workerSrc).toMatch(/streamGeneration\+\+/)
148157

149158
// After awaiting rehydrate, if a new stream replaced ours, the freshly built
150159
// Trace is discarded (ended) instead of being inserted into the cleared map.
151160
const guard = workerSrc.match(
152-
/if \(eventStream\.abort !== streamAtEntry\)[\s\S]*?trace\.endTrace\(\)[\s\S]*?return sessionTraces\.get\(sessionID\) \?\? null/,
161+
/if \(streamGeneration !== generationAtEntry\)[\s\S]*?trace\.endTrace\(\)[\s\S]*?return sessionTraces\.get\(sessionID\) \?\? null/,
153162
)
154163
expect(guard).not.toBeNull()
155164

156165
// The guard must sit AFTER the rehydrate await and BEFORE the cache insert,
157166
// otherwise it can't prevent the orphan write.
158167
const awaitIdx = workerSrc.indexOf("await trace.rehydrateFromFile(sessionID)")
159-
const guardIdx = workerSrc.indexOf("if (eventStream.abort !== streamAtEntry)")
168+
const guardIdx = workerSrc.indexOf("if (streamGeneration !== generationAtEntry)")
160169
const setIdx = workerSrc.indexOf("sessionTraces.set(sessionID, trace)")
161170
expect(awaitIdx).toBeGreaterThan(-1)
162171
expect(guardIdx).toBeGreaterThan(awaitIdx)
163172
expect(setIdx).toBeGreaterThan(guardIdx)
173+
174+
// The bump happens inside startEventStream so a workspace switch invalidates
175+
// any suspended getOrCreateTrace.
176+
const startIdx = workerSrc.indexOf("const startEventStream =")
177+
const bumpIdx = workerSrc.indexOf("streamGeneration++")
178+
expect(bumpIdx).toBeGreaterThan(startIdx)
179+
})
180+
})
181+
182+
describe("#903 — capSpansForSerialization structural guarantees (review hardening)", () => {
183+
test("always retains the root span even when it is NOT at index 0", () => {
184+
// Root deliberately placed in the middle so the naive head-slice would drop it.
185+
const spans: TraceSpan[] = []
186+
for (let i = 0; i < 40; i++) spans.push(span(i))
187+
const root: TraceSpan = { spanId: "ROOT", parentSpanId: null, name: "root", kind: "session", startTime: 20, status: "ok" }
188+
spans.splice(20, 0, root) // root now at index 20, not in the head slice
189+
const out = capSpansForSerialization(spans, 10)
190+
expect(out.some((s) => s.spanId === "ROOT")).toBe(true)
191+
// the elision marker is parented to the real root
192+
const marker = out.find((s) => s.name.includes("elided"))
193+
expect(marker?.parentSpanId).toBe("ROOT")
194+
expect(out.length).toBeLessThanOrEqual(10)
195+
})
196+
197+
test.each([1, 2, 3, 4, 5])("tiny cap=%i never throws and never references a missing parent", (cap) => {
198+
const spans: TraceSpan[] = [{ spanId: "root", parentSpanId: null, name: "root", kind: "session", startTime: 0, status: "ok" }]
199+
for (let i = 1; i < 30; i++) spans.push(span(i))
200+
const out = capSpansForSerialization(spans, cap)
201+
// every non-root span's parent (if it's a kept span's parent) must resolve,
202+
// OR be the root, OR be absent (orphan attaches to root in the viewer).
203+
const ids = new Set(out.map((s) => s.spanId))
204+
const marker = out.find((s) => s.name.includes("elided"))
205+
if (marker) {
206+
expect(marker.parentSpanId === null || ids.has(marker.parentSpanId as string)).toBe(true)
207+
}
208+
// a root span is always present in the output
209+
expect(out.some((s) => s.parentSpanId === null)).toBe(true)
210+
})
211+
212+
test("exact boundary head+tail+1 === length returns the original (no pointless marker)", () => {
213+
// cap=10 → head 3, tail 6, +1 marker = 10. length 10 → 10 >= 10 → pass-through.
214+
const spans: TraceSpan[] = [{ spanId: "root", parentSpanId: null, name: "root", kind: "session", startTime: 0, status: "ok" }]
215+
for (let i = 1; i < 10; i++) spans.push(span(i))
216+
expect(spans.length).toBe(10)
217+
expect(capSpansForSerialization(spans, 10)).toBe(spans)
218+
})
219+
220+
test("elision marker reports an accurate elided count", () => {
221+
const spans: TraceSpan[] = [{ spanId: "root", parentSpanId: null, name: "root", kind: "session", startTime: 0, status: "ok" }]
222+
for (let i = 1; i < 1000; i++) spans.push(span(i))
223+
const cap = 100
224+
const out = capSpansForSerialization(spans, cap)
225+
const marker = out.find((s) => s.name.includes("elided"))!
226+
const keptReal = out.length - 1 // minus the marker
227+
expect((marker.attributes as any).elided).toBe(spans.length - keptReal)
228+
expect((marker.attributes as any).totalSpans).toBe(1000)
164229
})
165230
})

0 commit comments

Comments
 (0)