Skip to content

Commit f8191ab

Browse files
feat(observability): carry the client's trace through a turn (#1400)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 9e8f8a3 commit f8191ab

13 files changed

Lines changed: 830 additions & 8 deletions

File tree

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/**
2+
* W3C trace context for a prompt turn.
3+
*
4+
* A client (the VS Code extension) sends `traceparent` with a prompt. The trace is bound to the
5+
* user message that prompt creates — the turn — not to the session, so overlapping prompts on a
6+
* busy session each keep their own trace and nothing outlives its turn. For that turn:
7+
* - calls to the Altimate gateways carry `traceparent` (a fresh child span per call) and
8+
* `x-request-id` (`<trace id>-<span id>`); the backend's Azure Monitor instrumentation adopts
9+
* the trace as the request's `operation_Id`;
10+
* - this process's log lines are tagged `trace=<id>` and its App Insights events get
11+
* `ai.operation.id`.
12+
* One id then joins the client's, altimate-code's and the backend's records of a failure.
13+
*
14+
* Lifecycle: routes bind the turn's message id before prompting; the prompt loop marks the turn
15+
* it is running as the session's active one (for telemetry), lets an automatic compaction or a
16+
* subagent inherit it, and releases every binding it processed when its generation ends.
17+
*/
18+
19+
/** `version-traceid-parentid-flags`, lower-case hex only, as W3C Trace Context requires. */
20+
const TRACEPARENT = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/
21+
const ZERO_TRACE_ID = "0".repeat(32)
22+
const ZERO_SPAN_ID = "0".repeat(16)
23+
/** Bound on remembered turns; the oldest binding is dropped first. */
24+
const MAX_BINDINGS = 500
25+
/** Providers served by Altimate's backend — the only ones that receive the trace. */
26+
const ALTIMATE_PROVIDERS = new Set(["altimate-backend", "altimate-free"])
27+
28+
export interface Trace {
29+
traceId: string
30+
flags: string
31+
}
32+
33+
const byMessage = new Map<string, Trace>()
34+
/** The turn each session is running right now, and its trace, for telemetry. */
35+
const activeBySession = new Map<string, { messageID: string; traceId: string }>()
36+
37+
/**
38+
* Parses a `traceparent` header. Returns undefined for anything W3C says to ignore: a malformed
39+
* header, upper-case hex, or an all-zero trace or parent id.
40+
*/
41+
export function parse(header: string | undefined): Trace | undefined {
42+
const match = header ? TRACEPARENT.exec(header.trim()) : null
43+
if (!match || match[1] === ZERO_TRACE_ID || match[2] === ZERO_SPAN_ID) return undefined
44+
return { traceId: match[1], flags: match[3] }
45+
}
46+
47+
/** Binds the trace a client sent with a prompt to the user message that prompt creates. */
48+
export function bind(messageID: string, traceparent: string | undefined) {
49+
const trace = parse(traceparent)
50+
if (trace) remember(messageID, trace)
51+
else byMessage.delete(messageID)
52+
}
53+
54+
/**
55+
* Gives `childMessageID` the trace of `parentMessageID`'s turn — a subagent's prompt, or an
56+
* automatic compaction inside the turn — or none, when that turn is untraced. Always replaces
57+
* the child's binding, so a resumed subagent never keeps an earlier turn's trace.
58+
*/
59+
export function inherit(childMessageID: string, parentMessageID: string) {
60+
const parent = byMessage.get(parentMessageID)
61+
if (parent) remember(childMessageID, parent)
62+
else byMessage.delete(childMessageID)
63+
}
64+
65+
export function forMessage(messageID: string): Trace | undefined {
66+
return byMessage.get(messageID)
67+
}
68+
69+
export function traceId(messageID: string): string | undefined {
70+
return byMessage.get(messageID)?.traceId
71+
}
72+
73+
/** Marks `messageID`'s turn as the one `sessionID` is running now (or none, if untraced). */
74+
export function activate(sessionID: string, messageID: string) {
75+
const trace = byMessage.get(messageID)
76+
if (trace) activeBySession.set(sessionID, { messageID, traceId: trace.traceId })
77+
else activeBySession.delete(sessionID)
78+
}
79+
80+
/** The trace of the turn `sessionID` is running now. */
81+
export function activeTraceId(sessionID: string): string | undefined {
82+
return activeBySession.get(sessionID)?.traceId
83+
}
84+
85+
/**
86+
* Releases a finished prompt-loop generation: the bindings of exactly the turns it processed
87+
* (a prompt that arrives meanwhile keeps its own), and the session's active turn if it is one of
88+
* them (a newer generation's active turn is left alone).
89+
*/
90+
export function release(sessionID: string, messageIDs: Iterable<string>) {
91+
const released = new Set(messageIDs)
92+
for (const messageID of released) byMessage.delete(messageID)
93+
const active = activeBySession.get(sessionID)
94+
if (active && released.has(active.messageID)) activeBySession.delete(sessionID)
95+
}
96+
97+
/**
98+
* Adds the turn's trace headers to an outgoing request for an Altimate provider. The two headers
99+
* are a unit: if the request headers (model config, plugins) or the provider's configured headers
100+
* already set either, neither is added — the SDK lets per-call headers override provider ones, so
101+
* both must be checked. Each call gets a fresh span id, so the backend can tell its requests
102+
* apart while joining them on the trace.
103+
*/
104+
export function withHeaders(
105+
headers: Record<string, string>,
106+
trace: Trace | undefined,
107+
providerID: string,
108+
providerHeaders?: unknown,
109+
): Record<string, string> {
110+
if (!trace || !ALTIMATE_PROVIDERS.has(providerID)) return headers
111+
const configured = providerHeaders && typeof providerHeaders === "object" ? Object.keys(providerHeaders) : []
112+
const existing = [...Object.keys(headers), ...configured].map((key) => key.toLowerCase())
113+
if (existing.includes("traceparent") || existing.includes("x-request-id")) return headers
114+
const spanId = randomSpanId()
115+
return {
116+
...headers,
117+
traceparent: `00-${trace.traceId}-${spanId}-${trace.flags}`,
118+
"x-request-id": `${trace.traceId}-${spanId}`,
119+
}
120+
}
121+
122+
/** Number of turns currently bound (for tests and diagnostics). */
123+
export function size(): number {
124+
return byMessage.size
125+
}
126+
127+
/** One insertion path for every binding: replace, then evict the oldest past the bound. */
128+
function remember(messageID: string, trace: Trace) {
129+
byMessage.delete(messageID)
130+
byMessage.set(messageID, trace)
131+
while (byMessage.size > MAX_BINDINGS) byMessage.delete(byMessage.keys().next().value!)
132+
}
133+
134+
function randomSpanId(): string {
135+
const bytes = crypto.getRandomValues(new Uint8Array(8))
136+
const id = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")
137+
return id === ZERO_SPAN_ID ? randomSpanId() : id
138+
}
139+
140+
export * as TraceContext from "./trace-context"

‎packages/opencode/src/altimate/telemetry/index.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { Config } from "@/config/config"
33
import { Flag } from "@/flag/flag"
44
import { InstallationVersion } from "@opencode-ai/core/installation/version"
55
import { Log } from "@/altimate/util/log"
6+
import { TraceContext } from "@/altimate/observability/trace-context"
67
// altimate_change — shared machine-id helper (race-safe, UUID-validated, size-capped)
78
import { getOrCreateMachineId } from "@/altimate/util/machine-id"
89
import { createHash, randomUUID } from "crypto"
@@ -1938,7 +1939,7 @@ export namespace Telemetry {
19381939
const measurements: Record<string, number> = {}
19391940

19401941
for (const [k, v] of Object.entries(fields)) {
1941-
if (k === "session_id" || k === "project_id" || k === "_retried") continue
1942+
if (k === "session_id" || k === "project_id" || k === "_retried" || k === "_operation_id") continue
19421943
if (typeof v === "number") {
19431944
measurements[k] = v
19441945
} else if (v !== undefined && v !== null) {
@@ -1961,6 +1962,7 @@ export namespace Telemetry {
19611962
// altimate_change end
19621963
"ai.cloud.role": "altimate",
19631964
"ai.application.ver": InstallationVersion,
1965+
...(fields._operation_id && { "ai.operation.id": fields._operation_id }),
19641966
},
19651967
data: {
19661968
baseType: "EventData",
@@ -2228,6 +2230,14 @@ export namespace Telemetry {
22282230
// Before init completes: buffer (flushed once init enables, or cleared if disabled).
22292231
// After init completed and disabled telemetry: drop silently.
22302232
if (initDone && !enabled) return
2233+
// altimate_change start — stamp the client trace of the turn the event's own session is
2234+
// running, now: the event is serialised at flush time, when that turn may be over. Only an
2235+
// explicit per-event `session_id` is used — the process-global session can be another one.
2236+
// Becomes the envelope's `ai.operation.id`, joining it to the extension's and backend's records.
2237+
const eventSession = (event as { session_id?: unknown }).session_id
2238+
const operationId = typeof eventSession === "string" ? TraceContext.activeTraceId(eventSession) : undefined
2239+
if (operationId) (event as any)._operation_id = operationId
2240+
// altimate_change end
22312241
buffer.push(event)
22322242
if (buffer.length > MAX_BUFFER_SIZE) {
22332243
buffer.shift()

‎packages/opencode/src/server/routes/session.ts‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ import { Todo } from "../../session/todo"
1717
import { Agent } from "../../agent/agent"
1818
import { Snapshot } from "@/snapshot"
1919
import { Log } from "../../util/log"
20+
// altimate_change start — client trace propagation
21+
import { TraceContext } from "@/altimate/observability/trace-context"
22+
// altimate_change end
2023
// altimate_change start — upstream_fix: bridge merge wired routes to Effect-TS Permission, but the runtime
2124
// asks come from PermissionNext. Use PermissionNext here so this deprecated reply route hits the same
2225
// pending map. See packages/opencode/src/server/routes/permission.ts for the full explanation.
@@ -922,7 +925,15 @@ export const SessionRoutes = lazy(() =>
922925
return stream(c, async (stream) => {
923926
const sessionID = c.req.valid("param").sessionID
924927
const body = c.req.valid("json")
925-
const msg = await SessionPrompt.prompt({ ...body, sessionID })
928+
// altimate_change start — bind the client's trace to this turn's user message (see TraceContext).
929+
// A noReply prompt runs no turn, so binds nothing; a prompt that fails before its loop is released.
930+
const messageID = body.messageID ?? MessageID.ascending()
931+
TraceContext.bind(messageID, body.noReply ? undefined : c.req.header("traceparent"))
932+
const msg = await SessionPrompt.prompt({ ...body, sessionID, messageID }).catch((error) => {
933+
TraceContext.release(sessionID, [messageID])
934+
throw error
935+
})
936+
// altimate_change end
926937
stream.write(JSON.stringify(msg))
927938
})
928939
},
@@ -954,7 +965,15 @@ export const SessionRoutes = lazy(() =>
954965
return stream(c, async () => {
955966
const sessionID = c.req.valid("param").sessionID
956967
const body = c.req.valid("json")
957-
SessionPrompt.prompt({ ...body, sessionID }).catch((err) => {
968+
// altimate_change start — bind the client's trace to this turn's user message (see TraceContext).
969+
// A noReply prompt runs no turn, so binds nothing.
970+
const messageID = body.messageID ?? MessageID.ascending()
971+
TraceContext.bind(messageID, body.noReply ? undefined : c.req.header("traceparent"))
972+
// altimate_change end
973+
SessionPrompt.prompt({ ...body, sessionID, messageID }).catch((err) => {
974+
// altimate_change start — a prompt that failed before its loop is released here
975+
TraceContext.release(sessionID, [messageID])
976+
// altimate_change end
958977
log.error("prompt_async failed", { sessionID, error: err })
959978
Bus.publish(Session.Event.Error, {
960979
sessionID,
@@ -997,7 +1016,15 @@ export const SessionRoutes = lazy(() =>
9971016
async (c) => {
9981017
const sessionID = c.req.valid("param").sessionID
9991018
const body = c.req.valid("json")
1000-
const msg = await SessionPrompt.command({ ...body, sessionID })
1019+
// altimate_change start — bind the client's trace to this turn's user message (see TraceContext);
1020+
// a command that fails before its loop is released
1021+
const messageID = body.messageID ?? MessageID.ascending()
1022+
TraceContext.bind(messageID, c.req.header("traceparent"))
1023+
const msg = await SessionPrompt.command({ ...body, sessionID, messageID }).catch((error) => {
1024+
TraceContext.release(sessionID, [messageID])
1025+
throw error
1026+
})
1027+
// altimate_change end
10011028
return c.json(msg)
10021029
},
10031030
)

‎packages/opencode/src/session/compaction.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import { Context, Effect, Layer } from "effect"
3232
import { LayerNode } from "@opencode-ai/core/effect/layer-node"
3333
import { serviceUse } from "@opencode-ai/core/effect/service-use"
3434
import path from "node:path"
35+
import { TraceContext } from "@/altimate/observability/trace-context" // altimate_change — client trace per turn
3536
// altimate_change end
3637

3738
export namespace SessionCompaction {
@@ -1638,6 +1639,9 @@ When constructing the summary, try to stick to this template:
16381639
system: original.system,
16391640
variant: original.variant,
16401641
})
1642+
// altimate_change start — the replayed prompt continues the compacted turn (client trace)
1643+
TraceContext.inherit(replayMsg.id, input.parentID)
1644+
// altimate_change end
16411645
for (const part of replay.parts) {
16421646
if (part.type === "compaction") continue
16431647
const replayPart =
@@ -1683,6 +1687,8 @@ When constructing the summary, try to stick to this template:
16831687
system: latestField("system") ?? userMessage.system,
16841688
variant: latestField("variant") ?? userMessage.variant,
16851689
})
1690+
// The continue message carries the compacted turn on (client trace).
1691+
TraceContext.inherit(continueMsg.id, input.parentID)
16861692
// altimate_change end
16871693
// altimate_change start — deterministic corroborated-facts-only
16881694
// state ledger appended to the synthetic continue message (all-modes, compaction-gated).

‎packages/opencode/src/session/llm.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import { Context, Effect, Layer, Stream } from "effect"
3939
import { LayerNode } from "@opencode-ai/core/effect/layer-node"
4040
import type { LLMEvent } from "@opencode-ai/llm"
4141
import { LLMAISDK } from "./llm/ai-sdk"
42+
import { TraceContext } from "@/altimate/observability/trace-context"
4243
// altimate_change end
4344

4445
export namespace LLM {
@@ -84,6 +85,11 @@ export namespace LLM {
8485
.tag("small", (input.small ?? false).toString())
8586
.tag("agent", input.agent.name)
8687
.tag("mode", input.agent.mode)
88+
// altimate_change start — the client trace of this request's turn (its user message), for the
89+
// log lines below and the outgoing headers; resolved once so both use the same trace
90+
const trace = TraceContext.forMessage(input.user.id)
91+
if (trace) l.tag("trace", trace.traceId)
92+
// altimate_change end
8793
l.info("stream", {
8894
modelID: input.model.id,
8995
providerID: input.model.providerID,
@@ -315,8 +321,13 @@ export namespace LLM {
315321
maxOutputTokens,
316322
// altimate_change end
317323
abortSignal: input.abort,
318-
// altimate_change start — send the canonical headers used by the budget estimator, bound to the current Altimate Base session
319-
headers: withManagedSessionHeaders(input.model.providerID, input.sessionID, requestHeaders),
324+
// altimate_change start — send the canonical headers used by the budget estimator, bound to the
325+
// current Altimate Base session, plus the turn's client trace for the Altimate gateways
326+
headers: withManagedSessionHeaders(
327+
input.model.providerID,
328+
input.sessionID,
329+
TraceContext.withHeaders(requestHeaders, trace, input.model.providerID, provider.options?.headers),
330+
),
320331
// altimate_change end
321332
maxRetries: input.retries ?? 0,
322333
messages: [

‎packages/opencode/src/session/processor.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { MessageV2 } from "./message-v2"
22
import { Log } from "@/util/log"
3+
// altimate_change start — client trace propagation
4+
import { TraceContext } from "@/altimate/observability/trace-context"
5+
// altimate_change end
36
import { Session } from "."
47
import { Agent } from "@/agent/agent"
58
import { Snapshot } from "@/snapshot"
@@ -1160,9 +1163,15 @@ export namespace SessionProcessor {
11601163
// altimate_change end
11611164
}
11621165
} catch (e: any) {
1166+
// altimate_change start — join this failure to its turn's client trace (when traced)
1167+
const traceId = TraceContext.traceId(input.assistantMessage.parentID)
1168+
// altimate_change end
11631169
log.error("process", {
11641170
error: e,
11651171
stack: JSON.stringify(e.stack),
1172+
// altimate_change start — client trace (see above)
1173+
...(traceId && { trace: traceId }),
1174+
// altimate_change end
11661175
})
11671176
const error = MessageV2.fromError(e, { providerID: input.model.providerID })
11681177
if (MessageV2.ContextOverflowError.isInstance(error)) {

‎packages/opencode/src/session/prompt.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ import { stampRegistryToolSource, describeMcpTool } from "../altimate/tool-sourc
100100
// altimate_change end
101101
// altimate_change end
102102
import { Telemetry } from "@/telemetry" // altimate_change — session telemetry
103+
import { TraceContext } from "@/altimate/observability/trace-context" // altimate_change — client trace per turn
103104
import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" // altimate_change — onboarding funnel
104105
// altimate_change start — keyless public Zen predicate (flat module)
105106
import { isPublicZen } from "@/provider/public-zen"
@@ -634,6 +635,12 @@ export namespace SessionPrompt {
634635
const nudgeGeneration = NudgeArbiter.begin(sessionID)
635636
using _nudgeGeneration = defer(() => NudgeArbiter.clear(sessionID, nudgeGeneration))
636637
// altimate_change end
638+
// altimate_change start — client trace: the turns this generation runs. Released when it ends
639+
// (disposed before the generation's own cleanup, so no newer generation exists yet).
640+
const tracedTurns = new Set<string>()
641+
let currentTurn: string | undefined
642+
using _traceGeneration = defer(() => TraceContext.release(sessionID, tracedTurns))
643+
// altimate_change end
637644

638645
// Structured output state
639646
// Note: On session resumption, state is reset but outputFormat is preserved
@@ -815,6 +822,21 @@ export namespace SessionPrompt {
815822
}
816823

817824
if (!lastUser) throw new Error("No user message found in stream. This should never happen.")
825+
// altimate_change start — client trace for this step. A user message carrying a compaction
826+
// part was created by this loop: it continues the turn that triggered it. Any other user
827+
// message is a turn of its own, traced or not.
828+
const stepUser = lastUser
829+
const isCompaction = msgs.some(
830+
(msg) => msg.info.id === stepUser.id && msg.parts.some((part) => part.type === "compaction"),
831+
)
832+
if (isCompaction && currentTurn && !TraceContext.forMessage(stepUser.id)) {
833+
TraceContext.inherit(stepUser.id, currentTurn)
834+
} else if (!isCompaction) {
835+
currentTurn = stepUser.id
836+
}
837+
tracedTurns.add(stepUser.id)
838+
TraceContext.activate(sessionID, stepUser.id)
839+
// altimate_change end
818840
// altimate_change start — always track the current agent name so early breaks still report it
819841
if (lastUser.agent) sessionAgentName = lastUser.agent
820842
// altimate_change end
@@ -1042,6 +1064,9 @@ export namespace SessionPrompt {
10421064
model: lastUser.model,
10431065
}
10441066
await Session.updateMessage(summaryUserMsg)
1067+
// altimate_change start — this loop-written message continues the step's turn (client trace)
1068+
TraceContext.inherit(summaryUserMsg.id, lastUser.id)
1069+
// altimate_change end
10451070
await Session.updatePart({
10461071
id: PartID.ascending(),
10471072
messageID: summaryUserMsg.id,
@@ -1862,6 +1887,8 @@ export namespace SessionPrompt {
18621887
agent: lastUser.agent,
18631888
model: lastUser.model,
18641889
} as MessageV2.Info)
1890+
// This loop-written message continues the step's turn (client trace).
1891+
TraceContext.inherit(syntheticMessageID, lastUser.id)
18651892

18661893
// Append the validator body as a text part on the new user message.
18671894
await Session.updatePart({

0 commit comments

Comments
 (0)