Repository navigation
Expand file tree
/
Copy pathrunner.py
More file actions
218 lines (203 loc) · 9.16 KB
/
Copy pathrunner.py
File metadata and controls
218 lines (203 loc) · 9.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
from __future__ import annotations
from typing import Any
import anyio
from mcp import MCPError
from .client import DiscoveryLimitError, InitializationTimeoutError, discover_tools, open_session
from .models import BenchReport, CaseReceipt, Outcome, RunSpec
from .redaction import BoundedTextCapture, Redactor, bound_value
from .schemas import validate_instance, validate_schema
from .snapshot import diff_snapshots, make_snapshot
def _clean(redactor: Redactor, value: Any, limit: int) -> Any:
return bound_value(redactor.value(value), limit)
def _exception_kind(exc: BaseException) -> Outcome:
text = str(exc).lower()
if isinstance(exc, MCPError):
return Outcome.CALL_TIMEOUT if "timed out" in text else Outcome.PROTOCOL_ERROR
if isinstance(exc, TimeoutError):
return Outcome.CALL_TIMEOUT
if isinstance(exc, (anyio.BrokenResourceError, anyio.ClosedResourceError, anyio.EndOfStream)):
return Outcome.TRANSPORT_ERROR
if isinstance(exc, RuntimeError) and str(exc).startswith(
"Invalid structured content returned by tool"
):
return Outcome.SDK_OUTPUT_SCHEMA_REJECTED
return Outcome.CLIENT_RUNTIME_ERROR
def _contains_exception(exc: BaseException, kind: type[BaseException]) -> bool:
if isinstance(exc, kind):
return True
if isinstance(exc, BaseExceptionGroup):
return any(_contains_exception(child, kind) for child in exc.exceptions)
return False
def _contains_runtime_message(exc: BaseException, prefix: str) -> bool:
if isinstance(exc, RuntimeError) and str(exc).startswith(prefix):
return True
if isinstance(exc, BaseExceptionGroup):
return any(_contains_runtime_message(child, prefix) for child in exc.exceptions)
return False
def _exception_details(exc: BaseException) -> str:
if isinstance(exc, BaseExceptionGroup):
details = [_exception_details(child) for child in exc.exceptions]
return "; ".join(dict.fromkeys(detail for detail in details if detail))
return str(exc) or type(exc).__name__
async def run_bench(spec: RunSpec, baseline: dict[str, Any] | None = None) -> BenchReport:
redactor = Redactor(spec.redaction_values)
errlog = BoundedTextCapture(spec.limits.max_stderr_bytes, redactor)
report = BenchReport(server="configured stdio server", connections_opened=1)
try:
async with open_session(
spec.server, errlog, spec.limits.initialization_timeout_seconds
) as session:
try:
with anyio.fail_after(spec.limits.discovery_timeout_seconds):
tools = await discover_tools(
session, spec.limits.max_tool_pages, spec.limits.max_tools
)
except TimeoutError as exc:
raise TimeoutError("MCP tool discovery timed out") from exc
current_snapshot = redactor.value(make_snapshot(tools))
report.tools = current_snapshot["tools"]
if baseline is not None:
report.drift = diff_snapshots(baseline, current_snapshot)
for case in spec.cases:
arguments = _clean(redactor, case.arguments, spec.limits.max_output_bytes)
if case.tool not in spec.allow_tools:
report.cases.append(
CaseReceipt(
case.id,
case.tool,
Outcome.SAFE_SKIP_NOT_ALLOWLISTED,
"tool not in explicit allowlist",
arguments,
)
)
continue
tool = tools.get(case.tool)
if tool is None:
report.cases.append(
CaseReceipt(
case.id,
case.tool,
Outcome.SAFE_SKIP_TOOL_MISSING,
"tool was not discovered",
arguments,
)
)
continue
schema_problem = validate_schema(tool.input_schema)
if schema_problem:
report.cases.append(
CaseReceipt(
case.id,
case.tool,
Outcome.INPUT_SCHEMA_INVALID,
redactor.text(schema_problem),
arguments,
)
)
continue
input_errors = validate_instance(tool.input_schema, case.arguments)
if input_errors:
report.cases.append(
CaseReceipt(
case.id,
case.tool,
Outcome.INPUT_VALIDATION_FAILED,
redactor.text("; ".join(input_errors)),
arguments,
)
)
continue
if tool.output_schema is not None:
schema_problem = validate_schema(tool.output_schema)
if schema_problem:
report.cases.append(
CaseReceipt(
case.id,
case.tool,
Outcome.OUTPUT_SCHEMA_INVALID,
redactor.text(schema_problem),
arguments,
)
)
continue
try:
result = await session.call_tool(
case.tool,
arguments=case.arguments,
read_timeout_seconds=case.timeout_seconds
or spec.limits.default_timeout_seconds,
)
content = [
block.model_dump(mode="json", by_alias=True, exclude_none=True)
for block in result.content
]
clean_content = _clean(redactor, content, spec.limits.max_output_bytes)
clean_structured = _clean(
redactor, result.structured_content, spec.limits.max_output_bytes
)
if result.is_error:
outcome, detail = Outcome.TOOL_RESULT_ERROR, "server returned isError=true"
elif tool.output_schema is not None:
errors = validate_instance(tool.output_schema, result.structured_content)
outcome = Outcome.OUTPUT_SCHEMA_MISMATCH if errors else Outcome.PASS
detail = (
redactor.text("; ".join(errors)) if errors else "contract satisfied"
)
else:
outcome, detail = Outcome.PASS, "contract satisfied"
report.cases.append(
CaseReceipt(
case.id,
case.tool,
outcome,
detail,
arguments,
clean_content,
clean_structured,
)
)
except Exception as exc:
report.cases.append(
CaseReceipt(
case.id,
case.tool,
_exception_kind(exc),
redactor.text(str(exc)),
arguments,
)
)
except Exception as exc:
text = redactor.text(_exception_details(exc))
if _contains_exception(exc, InitializationTimeoutError):
system_kind = "INITIALIZATION_TIMEOUT"
elif _contains_exception(exc, DiscoveryLimitError):
system_kind = "DISCOVERY_LIMIT"
elif _contains_exception(exc, TimeoutError):
system_kind = "DISCOVERY_TIMEOUT"
else:
transport_fault = (
_contains_exception(exc, anyio.BrokenResourceError)
or _contains_exception(exc, anyio.ClosedResourceError)
or _contains_exception(exc, anyio.EndOfStream)
or any(
marker in text.lower()
for marker in ("connection closed", "end of stream", "broken resource")
)
)
protocol_fault = _contains_exception(exc, MCPError) or (
_contains_runtime_message(exc, "duplicate tool name")
or _contains_runtime_message(exc, "repeated tools/list cursor")
)
system_kind = (
"TRANSPORT_ERROR"
if transport_fault
else "PROTOCOL_ERROR"
if protocol_fault
else "TRANSPORT_ERROR"
)
report.system_error = {
"kind": system_kind,
"detail": text,
}
report.stderr = errlog.getvalue()
return report