Skip to content

Commit ec9fc07

Browse files
committed
fix(ag2): count each model call's tokens once per trace
record_usage model_call repeats the chat span's input/output tokens (ag2 1.1.2 telemetry.py 437-440 vs 501-504). fi-collector promotes gen_ai.usage.* into the token columns on any span (adapter.go inputTokenKeys/outputTokenKeys/totalTokenKeys) and Observe sums total_tokens over every span of a trace (span_reader v2, tasks/session.py), so the trace total doubled. On model_call and aggregation usage spans the processor now moves the promoted keys to ag2.usage.*. Values are kept. subtask and compaction usage spans keep their tokens: no chat span repeats them. Tests first: the contract test now asserts the summed promoted input tokens equal one model call; 2 new processor tests. 52 passed on ag2 1.1.2, 50 passed + 2 skipped on 1.0.3. approved: Nikhil 2026-10-03 blanket Refs: TH-8236
1 parent e11a3d5 commit ec9fc07

4 files changed

Lines changed: 97 additions & 5 deletions

File tree

‎python/frameworks/ag2/README.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,13 @@ Notes:
128128
- From ag2 1.0.3, AG2 also emits a `record_usage {kind}` span per usage event,
129129
which carries the same token counts as the `chat` span it accounts for.
130130
It has no operation name and gets no span kind; the aliases are applied.
131-
- Network trace propagation (`ag2.otel.traceparent`) is upstream. The
132-
processor never removes attributes it did not hide by request.
131+
Future AGI totals tokens by summing every span in a trace, so on
132+
`model_call` (and `aggregation`) usage spans the processor moves
133+
`gen_ai.usage.input_tokens` / `output_tokens` / `total_tokens` to
134+
`ag2.usage.*`. The values stay on the span; each model call counts once.
135+
- Network trace propagation (`ag2.otel.traceparent`) is upstream. Apart from
136+
that usage move, the processor never removes attributes it did not hide by
137+
request.
133138

134139
## Troubleshooting
135140

‎python/frameworks/ag2/tests/test_contract_harness.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,20 @@ def test_one_tool_run_reaches_collector_path_with_normalized_gen_ai_spans(fi_env
101101
"gen_ai.usage.reasoning.output_tokens": 5,
102102
}
103103
usage_spans = [chat]
104-
if "record_usage model_call" in by_name: # ag2 >= 1.0.3
105-
usage_spans.append(by_name["record_usage model_call"])
106104
for source in usage_spans:
107105
assert {k: source.get(k) for k in expected_usage} == expected_usage
106+
if "record_usage model_call" in by_name: # ag2 >= 1.0.3
107+
# The duplicate of the chat span's tokens is kept under ag2.usage.*
108+
# so the trace total (sum of promoted gen_ai.usage.* columns) counts once.
109+
record = by_name["record_usage model_call"]
110+
assert "gen_ai.usage.input_tokens" not in record
111+
assert "gen_ai.usage.output_tokens" not in record
112+
assert record["ag2.usage.input_tokens"] == 11
113+
assert record["ag2.usage.output_tokens"] == 7
114+
promoted_input = sum(
115+
a.get("gen_ai.usage.input_tokens", 0) for a in by_name.values()
116+
)
117+
assert promoted_input == 11, "trace-level input tokens must count the model call once"
108118

109119
# session: AG2 emits none; content: capture is off
110120
for name, attributes in by_name.items():

‎python/frameworks/ag2/tests/test_processor.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,49 @@ def test_aliases_apply_to_record_usage_spans(pipeline):
155155
assert "gen_ai.span.kind" not in got
156156

157157

158+
def test_model_call_usage_span_does_not_double_count_trace_tokens(pipeline):
159+
"""record_usage model_call repeats the chat span's tokens (telemetry.py:437-440 vs 501-504).
160+
161+
fi-collector promotes gen_ai.usage.* on any span and Observe sums
162+
total_tokens over every span in a trace, so the duplicate moves to ag2.usage.*.
163+
"""
164+
provider, exporter = pipeline
165+
_emit(
166+
provider,
167+
{
168+
"ag2.span.type": "usage",
169+
"ag2.usage.kind": "model_call",
170+
"gen_ai.usage.input_tokens": 11,
171+
"gen_ai.usage.output_tokens": 7,
172+
"ag2.usage.total_tokens": 18,
173+
},
174+
name="record_usage model_call",
175+
)
176+
(span,) = exporter.get_finished_spans()
177+
got = attrs(span)
178+
assert "gen_ai.usage.input_tokens" not in got
179+
assert "gen_ai.usage.output_tokens" not in got
180+
assert got["ag2.usage.input_tokens"] == 11
181+
assert got["ag2.usage.output_tokens"] == 7
182+
assert got["ag2.usage.total_tokens"] == 18
183+
184+
185+
def test_subtask_usage_span_keeps_promoted_tokens(pipeline):
186+
"""Only kinds that duplicate a chat span are demoted; others keep their tokens."""
187+
provider, exporter = pipeline
188+
_emit(
189+
provider,
190+
{
191+
"ag2.span.type": "usage",
192+
"ag2.usage.kind": "subtask",
193+
"gen_ai.usage.input_tokens": 4,
194+
},
195+
name="record_usage subtask",
196+
)
197+
(span,) = exporter.get_finished_spans()
198+
assert attrs(span)["gen_ai.usage.input_tokens"] == 4
199+
200+
158201
def test_processor_only_adds_keys_by_default(pipeline):
159202
"""Nothing AG2 set is stripped, including propagation-related keys."""
160203
provider, exporter = pipeline

‎python/frameworks/ag2/traceai_ag2/_processor.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ def normalize_attributes(
7979
) -> Dict[str, Any]:
8080
"""Return a new attribute dict with the kind, aliases and content gate applied.
8181
82-
Never removes a key AG2 set except where ``config`` asks to hide content.
82+
Never removes a key AG2 set, except where ``config`` asks to hide content
83+
and where a duplicate usage span's promoted token keys move under
84+
``ag2.usage.*`` (see ``_demote_duplicate_usage``).
8385
"""
8486
mapped: Dict[str, Any] = dict(attributes)
8587

@@ -92,11 +94,43 @@ def normalize_attributes(
9294
if source in mapped and target not in mapped:
9395
mapped[target] = mapped[source]
9496

97+
_demote_duplicate_usage(mapped)
98+
9599
if config is not None:
96100
mapped = _apply_trace_config(mapped, config)
97101
return mapped
98102

99103

104+
# fi-collector promotes these keys into the token columns on any span
105+
# (fi-collector/pkg/adapter/adapter.go inputTokenKeys/outputTokenKeys/
106+
# totalTokenKeys), and Observe sums total_tokens over every span of a trace.
107+
_PROMOTED_TOKEN_KEYS = (
108+
"gen_ai.usage.input_tokens",
109+
"gen_ai.usage.output_tokens",
110+
"gen_ai.usage.total_tokens",
111+
)
112+
# ``record_usage model_call`` repeats the chat span's tokens (telemetry.py
113+
# 437-440 vs 501-504 at 1.1.2). ``aggregation`` is a sum by definition.
114+
# ``subtask`` and ``compaction`` are left alone: no chat span repeats them.
115+
_DUPLICATE_USAGE_KINDS = frozenset({"model_call", "aggregation"})
116+
117+
118+
def _demote_duplicate_usage(mapped: Dict[str, Any]) -> None:
119+
"""Move a duplicate usage span's promoted token keys under ``ag2.usage.*``.
120+
121+
The values are kept, so nothing AG2 reported is lost, but the trace total
122+
counts each model call once.
123+
"""
124+
if mapped.get("ag2.span.type") != "usage":
125+
return
126+
if mapped.get("ag2.usage.kind") not in _DUPLICATE_USAGE_KINDS:
127+
return
128+
for key in _PROMOTED_TOKEN_KEYS:
129+
if key in mapped:
130+
value = mapped.pop(key)
131+
mapped.setdefault("ag2.usage." + key.rsplit(".", 1)[1], value)
132+
133+
100134
def _apply_trace_config(attributes: Dict[str, Any], config: TraceConfig) -> Dict[str, Any]:
101135
hidden = set()
102136
if config.hide_inputs:

0 commit comments

Comments
 (0)