|
| 1 | +"""Trace token totals against AG2's own ``UsageReport``, with real AG2. |
| 2 | +
|
| 3 | +fi-collector promotes ``gen_ai.usage.input_tokens`` / ``output_tokens`` / |
| 4 | +``total_tokens`` into its token columns on any span, and Observe sums them over |
| 5 | +every span of a trace. So the sum of those keys over a trace must equal the |
| 6 | +total AG2 itself reports for the run. The model is AG2's scripted |
| 7 | +``ag2.testing.TestConfig``: no network, no API key. |
| 8 | +""" |
| 9 | + |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +import asyncio |
| 13 | +from typing import Any, Iterable, Tuple |
| 14 | + |
| 15 | +import pytest |
| 16 | + |
| 17 | +from traceai_ag2 import setup |
| 18 | + |
| 19 | +from ._support import attrs, emits_record_usage, memory_provider |
| 20 | + |
| 21 | +PROMOTED_INPUT = "gen_ai.usage.input_tokens" |
| 22 | +PROMOTED_OUTPUT = "gen_ai.usage.output_tokens" |
| 23 | +PROMOTED_TOTAL = "gen_ai.usage.total_tokens" |
| 24 | + |
| 25 | +needs_record_usage = pytest.mark.skipif( |
| 26 | + not emits_record_usage(), |
| 27 | + reason="record_usage spans (the only carrier of aggregation/compaction/sub-task rollup spend) start at ag2 1.0.3", |
| 28 | +) |
| 29 | + |
| 30 | + |
| 31 | +def _ag2_version() -> Tuple[int, ...]: |
| 32 | + from importlib.metadata import version |
| 33 | + |
| 34 | + parts = [] |
| 35 | + for piece in version("ag2").split(".")[:3]: |
| 36 | + digits = "".join(ch for ch in piece if ch.isdigit()) |
| 37 | + parts.append(int(digits or 0)) |
| 38 | + return tuple(parts) |
| 39 | + |
| 40 | + |
| 41 | +def _response(text: str, prompt_tokens: int, completion_tokens: int) -> Any: |
| 42 | + from ag2.events import ModelMessage, ModelResponse |
| 43 | + from ag2.usage import Usage |
| 44 | + |
| 45 | + return ModelResponse( |
| 46 | + ModelMessage(text), |
| 47 | + usage=Usage( |
| 48 | + prompt_tokens=prompt_tokens, |
| 49 | + completion_tokens=completion_tokens, |
| 50 | + total_tokens=prompt_tokens + completion_tokens, |
| 51 | + ), |
| 52 | + model="fake-model", |
| 53 | + ) |
| 54 | + |
| 55 | + |
| 56 | +def _trace_sums(spans: Iterable[Any]) -> Tuple[int, int, int]: |
| 57 | + spans = list(spans) |
| 58 | + assert len({s.context.trace_id for s in spans}) == 1, "expected one trace" |
| 59 | + total_in = sum(attrs(s).get(PROMOTED_INPUT, 0) for s in spans) |
| 60 | + total_out = sum(attrs(s).get(PROMOTED_OUTPUT, 0) for s in spans) |
| 61 | + total = sum(attrs(s).get(PROMOTED_TOTAL, 0) for s in spans) |
| 62 | + return total_in, total_out, total |
| 63 | + |
| 64 | + |
| 65 | +def _run(agent: Any, prompt: str) -> Any: |
| 66 | + async def _go() -> Any: |
| 67 | + reply = await agent.ask(prompt) |
| 68 | + return await reply.usage() |
| 69 | + |
| 70 | + return asyncio.run(_go()) |
| 71 | + |
| 72 | + |
| 73 | +def _assert_trace_matches_report(spans: Any, report: Any) -> None: |
| 74 | + total_in, total_out, total = _trace_sums(spans) |
| 75 | + assert (total_in, total_out) == (report.total.prompt_tokens, report.total.completion_tokens) |
| 76 | + # AG2 never sets gen_ai.usage.total_tokens; if a later release does, it must |
| 77 | + # not be counted twice either. |
| 78 | + assert total in (0, report.total.prompt_tokens + report.total.completion_tokens) |
| 79 | + |
| 80 | + |
| 81 | +def _knowledge(*, aggregate: bool = False, compact: bool = False) -> Any: |
| 82 | + from ag2.knowledge import MemoryKnowledgeStore |
| 83 | + from ag2.knowledge.config import KnowledgeConfig |
| 84 | + from ag2.testing import TestConfig |
| 85 | + |
| 86 | + kwargs: dict = {"store": MemoryKnowledgeStore(), "expose_tool": False, "write_event_log": False} |
| 87 | + if aggregate: |
| 88 | + from ag2.aggregate import AggregateTrigger, ConversationSummaryAggregate |
| 89 | + |
| 90 | + kwargs["aggregate"] = ConversationSummaryAggregate(TestConfig(_response("summary", 100, 50))) |
| 91 | + kwargs["aggregate_trigger"] = AggregateTrigger(on_end=True) |
| 92 | + if compact: |
| 93 | + from ag2.compact import CompactTrigger, SummarizeCompact |
| 94 | + |
| 95 | + kwargs["compact"] = SummarizeCompact(target=2, config=TestConfig(_response("compacted", 300, 30))) |
| 96 | + kwargs["compact_trigger"] = CompactTrigger(max_events=2) |
| 97 | + return KnowledgeConfig(**kwargs) |
| 98 | + |
| 99 | + |
| 100 | +def _delegating_planner(worker: Any, **planner_kwargs: Any) -> Any: |
| 101 | + from ag2 import Agent |
| 102 | + from ag2.events import ToolCallEvent |
| 103 | + from ag2.testing import TestConfig |
| 104 | + from ag2.tools.subagents import subagent_tool |
| 105 | + |
| 106 | + return Agent( |
| 107 | + "planner", |
| 108 | + config=TestConfig( |
| 109 | + ToolCallEvent("task_worker", arguments='{"objective": "x"}', id="call_task_1"), |
| 110 | + _response("ok", 11, 7), |
| 111 | + ), |
| 112 | + tools=[subagent_tool(worker, description="delegate")], |
| 113 | + **planner_kwargs, |
| 114 | + ) |
| 115 | + |
| 116 | + |
| 117 | +def _worker() -> Any: |
| 118 | + from ag2 import Agent |
| 119 | + from ag2.testing import TestConfig |
| 120 | + |
| 121 | + return Agent("worker", config=TestConfig(_response("done", 40, 4))) |
| 122 | + |
| 123 | + |
| 124 | +# --- R1: memory aggregation calls the model outside on_llm_call --------------- |
| 125 | + |
| 126 | + |
| 127 | +@needs_record_usage |
| 128 | +def test_aggregation_usage_counts_once_in_trace_total(): |
| 129 | + """aggregate.py calls the client on a throwaway stream: no chat span repeats it.""" |
| 130 | + from ag2 import Agent |
| 131 | + from ag2.testing import TestConfig |
| 132 | + |
| 133 | + provider, exporter = memory_provider() |
| 134 | + agent = Agent("mem_bot", config=TestConfig(_response("hello", 11, 7)), knowledge=_knowledge(aggregate=True)) |
| 135 | + setup(agent, tracer_provider=provider) |
| 136 | + report = _run(agent, "hi") |
| 137 | + |
| 138 | + assert (report.total.prompt_tokens, report.total.completion_tokens) == (111, 57) |
| 139 | + spans = exporter.get_finished_spans() |
| 140 | + (aggregation,) = [s for s in spans if s.name == "record_usage aggregation"] |
| 141 | + assert attrs(aggregation)[PROMOTED_INPUT] == 100 |
| 142 | + assert attrs(aggregation)[PROMOTED_OUTPUT] == 50 |
| 143 | + _assert_trace_matches_report(spans, report) |
| 144 | + |
| 145 | + |
| 146 | +# --- R2: a sub-task rollup repeats an instrumented worker's chat spans -------- |
| 147 | + |
| 148 | + |
| 149 | +@needs_record_usage |
| 150 | +def test_subtask_rollup_keeps_tokens_when_worker_is_not_instrumented(): |
| 151 | + """The rollup is the only copy of the worker's spend: it must stay promoted.""" |
| 152 | + provider, exporter = memory_provider() |
| 153 | + worker = _worker() |
| 154 | + planner = _delegating_planner(worker) |
| 155 | + setup(planner, tracer_provider=provider) |
| 156 | + report = _run(planner, "plan") |
| 157 | + |
| 158 | + spans = exporter.get_finished_spans() |
| 159 | + (rollup,) = [s for s in spans if s.name == "record_usage subtask"] |
| 160 | + assert attrs(rollup)["ag2.usage.label"] == "worker" |
| 161 | + assert attrs(rollup)[PROMOTED_INPUT] == 40 |
| 162 | + assert attrs(rollup)[PROMOTED_OUTPUT] == 4 |
| 163 | + _assert_trace_matches_report(spans, report) |
| 164 | + |
| 165 | + |
| 166 | +def test_subtask_rollup_counts_once_when_worker_is_instrumented(): |
| 167 | + """setup(planner, worker): the worker's chat spans already carry its tokens.""" |
| 168 | + provider, exporter = memory_provider() |
| 169 | + worker = _worker() |
| 170 | + planner = _delegating_planner(worker) |
| 171 | + setup(planner, worker, tracer_provider=provider) |
| 172 | + report = _run(planner, "plan") |
| 173 | + |
| 174 | + spans = exporter.get_finished_spans() |
| 175 | + assert [s.name for s in spans if s.name.startswith("invoke_agent")] == ["invoke_agent worker", "invoke_agent planner"] |
| 176 | + for rollup in [s for s in spans if s.name == "record_usage subtask"]: |
| 177 | + a = attrs(rollup) |
| 178 | + assert PROMOTED_INPUT not in a and PROMOTED_OUTPUT not in a |
| 179 | + assert a["ag2.usage.input_tokens"] == 40 |
| 180 | + assert a["ag2.usage.output_tokens"] == 4 |
| 181 | + _assert_trace_matches_report(spans, report) |
| 182 | + |
| 183 | + |
| 184 | +@needs_record_usage |
| 185 | +def test_subtask_rollup_for_an_agent_seen_in_another_trace_keeps_tokens(): |
| 186 | + """Agent names are tracked per trace: an instrumented worker's earlier, |
| 187 | + separate run must not demote a later rollup from an uninstrumented copy.""" |
| 188 | + from ag2 import Agent |
| 189 | + from ag2.testing import TestConfig |
| 190 | + |
| 191 | + provider, exporter = memory_provider() |
| 192 | + instrumented_namesake = Agent("worker", config=TestConfig(_response("solo", 1, 1))) |
| 193 | + setup(instrumented_namesake, tracer_provider=provider) |
| 194 | + _run(instrumented_namesake, "warm up") |
| 195 | + exporter.clear() |
| 196 | + |
| 197 | + planner = _delegating_planner(_worker()) |
| 198 | + setup(planner, tracer_provider=provider) |
| 199 | + report = _run(planner, "plan") |
| 200 | + _assert_trace_matches_report(exporter.get_finished_spans(), report) |
| 201 | + |
| 202 | + |
| 203 | +# --- every usage route in one run --------------------------------------------- |
| 204 | + |
| 205 | + |
| 206 | +@needs_record_usage |
| 207 | +@pytest.mark.parametrize("instrument_worker", [False, True], ids=["worker-uninstrumented", "worker-instrumented"]) |
| 208 | +def test_trace_tokens_equal_usage_report_across_every_usage_route(instrument_worker): |
| 209 | + """Main loop (chat + record_usage model_call), compaction, aggregation and a |
| 210 | + sub-task rollup in one run: summed promoted input and output tokens equal |
| 211 | + AG2's UsageReport. |
| 212 | +
|
| 213 | + Compaction is left out on ag2 < 1.0.4: there ``UsageReport`` reads the |
| 214 | + compacted history, so it drops usage events compaction removed (measured on |
| 215 | + 1.0.3: report 111 input tokens, trace and actual spend 451). The trace is |
| 216 | + right in that case and the report is not, so it is no ground truth. |
| 217 | + """ |
| 218 | + compact = _ag2_version() >= (1, 0, 4) |
| 219 | + provider, exporter = memory_provider() |
| 220 | + worker = _worker() |
| 221 | + planner = _delegating_planner(worker, knowledge=_knowledge(aggregate=True, compact=compact)) |
| 222 | + agents = (planner, worker) if instrument_worker else (planner,) |
| 223 | + setup(*agents, tracer_provider=provider) |
| 224 | + report = _run(planner, "plan") |
| 225 | + |
| 226 | + spans = exporter.get_finished_spans() |
| 227 | + names = {s.name for s in spans} |
| 228 | + expected_routes = ["record_usage model_call", "record_usage aggregation", "record_usage subtask"] |
| 229 | + if compact: |
| 230 | + expected_routes.append("record_usage compaction") |
| 231 | + for expected in expected_routes: |
| 232 | + assert expected in names, f"{expected} not emitted; the scenario no longer covers it" |
| 233 | + spent_in = 11 + 40 + 100 + (300 if compact else 0) |
| 234 | + spent_out = 7 + 4 + 50 + (30 if compact else 0) |
| 235 | + assert (report.total.prompt_tokens, report.total.completion_tokens) == (spent_in, spent_out) |
| 236 | + _assert_trace_matches_report(spans, report) |
0 commit comments