Skip to content

Commit 7c85d52

Browse files
committed
feat(MCP): Break down tool result size by content type
MCP clients differ in whether they render text blocks, structured content, or both into the agent's context (see modelcontextprotocol/modelcontextprotocol#1624), so no single number represents a call's token cost. Label flagsmith_mcp_tool_result_bytes with content={unstructured,structured,total}, observing all three per call so counts stay aligned. Sizes are measured with compact JSON encoding to match the wire. beep boop
1 parent 032f138 commit 7c85d52

2 files changed

Lines changed: 48 additions & 26 deletions

File tree

‎mcp/src/flagsmith_mcp/metrics.py‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
import json
21
import time
32
from collections.abc import Sequence
43

54
import mcp.types as mt
5+
import pydantic_core
66
from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext
77
from fastmcp.tools.base import Tool, ToolResult
88
from prometheus_client import Gauge, Histogram
@@ -15,11 +15,12 @@
1515
)
1616
flagsmith_mcp_tool_result_bytes = Histogram(
1717
"flagsmith_mcp_tool_result_bytes",
18-
"Size of a successful MCP tool call result's text content — what a "
19-
"client renders into the calling agent's context, and so a proxy for "
20-
"the token cost of the call. Excludes the structuredContent duplicate "
21-
"and JSON-RPC framing also present on the wire.",
22-
labelnames=["tool"],
18+
"Size of a successful MCP tool call result: its unstructured text "
19+
"blocks, its structuredContent, or their total. A proxy for the token "
20+
"cost a call incurs on the calling agent's context — MCP clients "
21+
"differ in which content they render to the agent, so the cost sits "
22+
"between the larger component and the total.",
23+
labelnames=["tool", "content"],
2324
buckets=(256, 1024, 4096, 16384, 65536, 262144, 1048576, float("inf")),
2425
)
2526
flagsmith_mcp_tool_catalogue_bytes = Gauge(
@@ -49,15 +50,26 @@ async def on_call_tool(
4950
flagsmith_mcp_tool_call_duration_seconds.labels(
5051
tool=tool, status="success"
5152
).observe(time.perf_counter() - start)
52-
# The text blocks already hold the serialised payload; measuring them
53-
# costs no extra marshalling.
54-
flagsmith_mcp_tool_result_bytes.labels(tool=tool).observe(
55-
sum(
56-
len(block.text.encode())
57-
for block in result.content
58-
if isinstance(block, mt.TextContent)
59-
)
53+
# Text blocks already hold the serialised payload; structuredContent
54+
# costs one compact dump, matching its wire encoding.
55+
unstructured_bytes = sum(
56+
len(block.text.encode())
57+
for block in result.content
58+
if isinstance(block, mt.TextContent)
59+
)
60+
structured_bytes = (
61+
len(pydantic_core.to_json(result.structured_content, fallback=str))
62+
if result.structured_content is not None
63+
else 0
6064
)
65+
for content, size in (
66+
("unstructured", unstructured_bytes),
67+
("structured", structured_bytes),
68+
("total", unstructured_bytes + structured_bytes),
69+
):
70+
flagsmith_mcp_tool_result_bytes.labels(tool=tool, content=content).observe(
71+
size
72+
)
6173
return result
6274

6375
async def on_list_tools(
@@ -68,12 +80,12 @@ async def on_list_tools(
6880
tools = await call_next(context)
6981
flagsmith_mcp_tool_catalogue_bytes.set(
7082
len(
71-
json.dumps(
83+
pydantic_core.to_json(
7284
[
7385
tool.to_mcp_tool().model_dump(exclude_none=True, by_alias=True)
7486
for tool in tools
7587
]
76-
).encode()
88+
)
7789
)
7890
)
7991
return tools

‎mcp/tests/integration/test_metrics.py‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,25 @@ async def test_metrics__successful_tool_call__records_duration_and_result_size(
3131
labels={"tool": "list_environments", "status": "success"},
3232
value=1,
3333
)
34-
assert_metric(
35-
name="flagsmith_mcp_tool_result_bytes_count",
36-
labels={"tool": "list_environments"},
37-
value=1,
38-
)
39-
result_bytes_sum = REGISTRY.get_sample_value(
40-
"flagsmith_mcp_tool_result_bytes_sum", {"tool": "list_environments"}
34+
content_sums: dict[str, float] = {}
35+
for content in ("unstructured", "structured", "total"):
36+
assert_metric(
37+
name="flagsmith_mcp_tool_result_bytes_count",
38+
labels={"tool": "list_environments", "content": content},
39+
value=1,
40+
)
41+
content_sum = REGISTRY.get_sample_value(
42+
"flagsmith_mcp_tool_result_bytes_sum",
43+
{"tool": "list_environments", "content": content},
44+
)
45+
assert content_sum is not None
46+
content_sums[content] = content_sum
47+
assert content_sums["unstructured"] > 0
48+
assert content_sums["structured"] > 0
49+
assert (
50+
content_sums["total"]
51+
== content_sums["unstructured"] + content_sums["structured"]
4152
)
42-
assert result_bytes_sum is not None
43-
assert result_bytes_sum > 0
4453

4554

4655
async def test_metrics__failing_tool_call__records_error_duration_only(
@@ -63,7 +72,8 @@ async def test_metrics__failing_tool_call__records_error_duration_only(
6372
)
6473
assert (
6574
REGISTRY.get_sample_value(
66-
"flagsmith_mcp_tool_result_bytes_count", {"tool": "list_environments"}
75+
"flagsmith_mcp_tool_result_bytes_count",
76+
{"tool": "list_environments", "content": "total"},
6777
)
6878
is None
6979
)

0 commit comments

Comments
 (0)