Skip to content

Commit 02740b3

Browse files
cliffhallclaude
andauthored
test(fetch): drive serve() through the protocol and reach 100% coverage
Add an in-process protocol harness (serve() over in-memory streams with a ClientSession, HTTP served by httpx.MockTransport) and a characterization suite covering tools/list, tools/call validation, robots.txt handling, markdown extraction in both readabilipy modes, pagination, raw mode, every error path and the fetch prompt. Pin current behavior that open issues ask to change: call_tool and get_prompt ignore the name, redirects have no private-IP guard (#4838), hard-coded timeouts (#4448), proxy config errors as bare text (#767, #1401) and serverInfo.version being the SDK's (#360). Cover main(), __main__ and the __init__ guard in-process, plus one stdio smoke of the console script. Add pytest-cov, branch coverage config and a README development section with the coverage command. Remove two dead branches in call_tool (empty URL after AnyUrl validation, empty slice when start_index < length and max_length > 0). Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent fc478d1 commit 02740b3

10 files changed

Lines changed: 1230 additions & 274 deletions

File tree

‎src/fetch/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
coverage.json
2+
.coverage
3+
.coverage.*

‎src/fetch/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,25 @@ cd path/to/servers/src/fetch
231231
npx @modelcontextprotocol/inspector uv run mcp-server-fetch
232232
```
233233

234+
## Development
235+
236+
From `src/fetch`:
237+
238+
```
239+
uv sync --frozen --all-extras --dev
240+
uv run --frozen pytest
241+
```
242+
243+
The tests drive `serve()` through an MCP `ClientSession` in-process, with HTTP mocked, so they need no network. The HTML extraction tests need Node.js on `PATH` (readabilipy uses Readability.js when Node is present and a pure-Python fallback when it is not).
244+
245+
Coverage, measured per file with branch coverage:
246+
247+
```
248+
uv run --frozen pytest --cov --cov-report=term-missing --cov-report=json
249+
```
250+
251+
This writes `coverage.json` (git-ignored). Every file under `src/mcp_server_fetch` is held to at least 90% of lines and 90% of branches.
252+
234253
## Contributing
235254

236255
We encourage contributions to help improve mcp-server-fetch. Bug reports, ideas for new tools, enhancements to existing functionality, and documentation improvements are all valuable.

‎src/fetch/pyproject.toml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,21 @@ requires = ["hatchling"]
3737
build-backend = "hatchling.build"
3838

3939
[tool.uv]
40-
dev-dependencies = ["pyright>=1.1.389", "ruff>=0.7.3", "pytest>=8.0.0", "pytest-asyncio>=0.21.0"]
40+
dev-dependencies = [
41+
"pyright>=1.1.389",
42+
"ruff>=0.7.3",
43+
"pytest>=8.0.0",
44+
"pytest-asyncio>=0.21.0",
45+
"pytest-cov>=7.1.0",
46+
]
4147

4248
[tool.pytest.ini_options]
4349
testpaths = ["tests"]
4450
asyncio_mode = "auto"
51+
52+
[tool.coverage.run]
53+
branch = true
54+
source = ["src/mcp_server_fetch"]
55+
56+
[tool.coverage.report]
57+
show_missing = true

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -240,8 +240,6 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
240240
raise McpError(ErrorData(code=INVALID_PARAMS, message=str(e)))
241241

242242
url = str(args.url)
243-
if not url:
244-
raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required"))
245243

246244
if not ignore_robots_txt:
247245
await check_may_autonomously_fetch_url(
@@ -255,21 +253,19 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
255253
if args.start_index >= original_length:
256254
content = "<error>No more content available.</error>"
257255
else:
256+
# Never empty: start_index < original_length and max_length > 0.
258257
truncated_content = content[
259258
args.start_index : args.start_index + args.max_length
260259
]
261-
if not truncated_content:
262-
content = "<error>No more content available.</error>"
263-
else:
264-
content = truncated_content
265-
actual_content_length = len(truncated_content)
266-
remaining_content = original_length - (
267-
args.start_index + actual_content_length
268-
)
269-
# Only add the prompt to continue fetching if there is still remaining content
270-
if actual_content_length == args.max_length and remaining_content > 0:
271-
next_start = args.start_index + actual_content_length
272-
content += f"\n\n<error>Content truncated. Call the fetch tool with a start_index of {next_start} to get more content.</error>"
260+
content = truncated_content
261+
actual_content_length = len(truncated_content)
262+
remaining_content = original_length - (
263+
args.start_index + actual_content_length
264+
)
265+
# Only add the prompt to continue fetching if there is still remaining content
266+
if actual_content_length == args.max_length and remaining_content > 0:
267+
next_start = args.start_index + actual_content_length
268+
content += f"\n\n<error>Content truncated. Call the fetch tool with a start_index of {next_start} to get more content.</error>"
273269
return [TextContent(type="text", text=f"{prefix}Contents of {url}:\n{content}")]
274270

275271
@server.get_prompt()

‎src/fetch/tests/conftest.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from .harness import node_readability, private_tempdir, python_readability, web
2+
3+
__all__ = ["node_readability", "private_tempdir", "python_readability", "web"]

‎src/fetch/tests/harness.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
"""Shared harness for the fetch server tests (conftest.py re-exports the fixtures).
2+
3+
Two pieces:
4+
5+
- ``connect(...)`` runs the real ``serve()`` in-process and yields an
6+
initialized ``ClientSession`` linked to it over in-memory streams, so tests
7+
go through the SDK's validation and error mapping exactly as a client would.
8+
- ``web`` replaces ``httpx.AsyncClient`` with one backed by an
9+
``httpx.MockTransport``, so no test touches the network. It records every
10+
request and the keyword arguments each client was constructed with.
11+
"""
12+
13+
from __future__ import annotations
14+
15+
import tempfile
16+
from contextlib import asynccontextmanager
17+
from dataclasses import dataclass, field
18+
from pathlib import Path
19+
from typing import Any, AsyncIterator, Callable, Union
20+
21+
import anyio
22+
import httpx
23+
import pytest
24+
import readabilipy.simple_json
25+
from mcp import ClientSession
26+
from mcp.shared.memory import create_client_server_memory_streams
27+
from mcp.types import InitializeResult
28+
29+
import mcp_server_fetch.server as server_module
30+
31+
Responder = Union[httpx.Response, Callable[[httpx.Request], httpx.Response]]
32+
33+
34+
@dataclass
35+
class FakeWeb:
36+
"""A router of URL -> response used as the transport of every AsyncClient."""
37+
38+
routes: dict[str, Responder] = field(default_factory=dict)
39+
requests: list[httpx.Request] = field(default_factory=list)
40+
client_kwargs: list[dict[str, Any]] = field(default_factory=list)
41+
42+
def add(self, url: str, response: Responder) -> None:
43+
self.routes[url] = response
44+
45+
def handle(self, request: httpx.Request) -> httpx.Response:
46+
self.requests.append(request)
47+
responder = self.routes.get(str(request.url))
48+
if responder is None:
49+
return httpx.Response(404, content=b"no route in FakeWeb")
50+
if isinstance(responder, httpx.Response):
51+
return responder
52+
return responder(request)
53+
54+
def urls(self) -> list[str]:
55+
return [str(r.url) for r in self.requests]
56+
57+
58+
@pytest.fixture
59+
def web(monkeypatch: pytest.MonkeyPatch) -> FakeWeb:
60+
fake = FakeWeb()
61+
real_async_client = httpx.AsyncClient
62+
63+
def factory(**kwargs: Any) -> httpx.AsyncClient:
64+
fake.client_kwargs.append(dict(kwargs))
65+
# A proxy would mount a real proxy transport in front of ours, so it is
66+
# recorded above and dropped here. Passing a transport also stops httpx
67+
# from picking up HTTP(S)_PROXY from the environment.
68+
kwargs.pop("proxy", None)
69+
return real_async_client(transport=httpx.MockTransport(fake.handle), **kwargs)
70+
71+
# server.py imports AsyncClient from httpx inside each function, so
72+
# patching the attribute on the httpx module is what it picks up.
73+
monkeypatch.setattr(httpx, "AsyncClient", factory)
74+
return fake
75+
76+
77+
@asynccontextmanager
78+
async def connect(
79+
**serve_kwargs: Any,
80+
) -> AsyncIterator[tuple[ClientSession, InitializeResult]]:
81+
"""Run ``serve(**serve_kwargs)`` in-process and yield a connected client."""
82+
async with create_client_server_memory_streams() as (
83+
client_streams,
84+
server_streams,
85+
):
86+
87+
@asynccontextmanager
88+
async def fake_stdio_server() -> AsyncIterator[Any]:
89+
yield server_streams
90+
91+
original = server_module.stdio_server
92+
server_module.stdio_server = fake_stdio_server # type: ignore[assignment]
93+
try:
94+
async with anyio.create_task_group() as tg:
95+
tg.start_soon(lambda: server_module.serve(**serve_kwargs))
96+
async with ClientSession(*client_streams) as session:
97+
init = await session.initialize()
98+
yield session, init
99+
tg.cancel_scope.cancel()
100+
finally:
101+
server_module.stdio_server = original
102+
103+
104+
@pytest.fixture
105+
def node_readability() -> None:
106+
"""Require the Readability.js path that readabilipy takes when Node is on PATH.
107+
108+
readabilipy silently falls back to a pure-Python extractor without Node,
109+
and the two produce different Markdown, so each HTML test names the mode
110+
it pins. Node is a prerequisite of this repository's gate, so a missing
111+
Node fails the test rather than skipping it.
112+
"""
113+
if not readabilipy.simple_json.have_node():
114+
pytest.fail("Node.js >= 10 must be on PATH for the Readability.js path")
115+
116+
117+
@pytest.fixture
118+
def python_readability(monkeypatch: pytest.MonkeyPatch) -> None:
119+
"""Force readabilipy's pure-Python fallback, as on a machine without Node."""
120+
monkeypatch.setattr(readabilipy.simple_json, "have_node", lambda: False)
121+
122+
123+
@pytest.fixture(autouse=True)
124+
def private_tempdir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
125+
"""Give each test its own ``tempfile.gettempdir()``.
126+
127+
readabilipy's Readability.js path writes its input and output to fixed
128+
names (``full.html``, ``article.json``) in the shared temp directory, so
129+
two test runs at once (parallel gate runs in separate worktrees, say) read
130+
each other's pages.
131+
"""
132+
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
"""Tests for the command-line entry points: ``main()``, ``__main__`` and the console script."""
2+
3+
from __future__ import annotations
4+
5+
import os
6+
import runpy
7+
import sys
8+
from pathlib import Path
9+
from unittest.mock import AsyncMock, MagicMock
10+
11+
import pytest
12+
from mcp import ClientSession, StdioServerParameters
13+
from mcp.client.stdio import stdio_client
14+
15+
import mcp_server_fetch
16+
import mcp_server_fetch.server
17+
18+
19+
def run_main(monkeypatch: pytest.MonkeyPatch, *argv: str) -> AsyncMock:
20+
serve = AsyncMock()
21+
monkeypatch.setattr(mcp_server_fetch, "serve", serve)
22+
monkeypatch.setattr(sys, "argv", ["mcp-server-fetch", *argv])
23+
mcp_server_fetch.main()
24+
return serve
25+
26+
27+
def test_main_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
28+
serve = run_main(monkeypatch)
29+
serve.assert_awaited_once_with(None, False, None)
30+
31+
32+
def test_main_passes_every_flag(monkeypatch: pytest.MonkeyPatch) -> None:
33+
serve = run_main(
34+
monkeypatch,
35+
"--user-agent",
36+
"MyBot/1.0",
37+
"--ignore-robots-txt",
38+
"--proxy-url",
39+
"http://proxy.example.com:8080",
40+
)
41+
serve.assert_awaited_once_with("MyBot/1.0", True, "http://proxy.example.com:8080")
42+
43+
44+
def test_main_help_exits_zero(
45+
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
46+
) -> None:
47+
with pytest.raises(SystemExit) as excinfo:
48+
run_main(monkeypatch, "--help")
49+
assert excinfo.value.code == 0
50+
out = capsys.readouterr().out
51+
assert "give a model the ability to make web requests" in out
52+
for flag in ("--user-agent", "--ignore-robots-txt", "--proxy-url"):
53+
assert flag in out
54+
55+
56+
def test_main_rejects_unknown_flag(
57+
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
58+
) -> None:
59+
with pytest.raises(SystemExit) as excinfo:
60+
run_main(monkeypatch, "--bogus")
61+
assert excinfo.value.code == 2
62+
assert "unrecognized arguments: --bogus" in capsys.readouterr().err
63+
64+
65+
def test_dunder_main_calls_main(monkeypatch: pytest.MonkeyPatch) -> None:
66+
# `python -m mcp_server_fetch`, in-process.
67+
main = MagicMock()
68+
monkeypatch.setattr(mcp_server_fetch, "main", main)
69+
monkeypatch.delitem(sys.modules, "mcp_server_fetch.__main__", raising=False)
70+
runpy.run_module("mcp_server_fetch", run_name="__main__")
71+
main.assert_called_once_with()
72+
73+
74+
def test_package_init_run_as_main_calls_main(monkeypatch: pytest.MonkeyPatch) -> None:
75+
# Covers the `if __name__ == "__main__"` guard at the foot of __init__.py.
76+
serve = AsyncMock()
77+
monkeypatch.setattr(mcp_server_fetch.server, "serve", serve)
78+
monkeypatch.setattr(sys, "argv", ["mcp-server-fetch"])
79+
runpy.run_module("mcp_server_fetch.__init__", run_name="__main__")
80+
serve.assert_awaited_once_with(None, False, None)
81+
82+
83+
def console_script() -> Path:
84+
bin_dir = Path(sys.executable).parent
85+
name = "mcp-server-fetch.exe" if os.name == "nt" else "mcp-server-fetch"
86+
return bin_dir / name
87+
88+
89+
async def test_console_script_serves_over_stdio() -> None:
90+
# The one subprocess smoke: the installed console script boots and answers
91+
# tools/list over real stdio. Everything else runs in-process.
92+
script = console_script()
93+
assert script.exists(), f"console script not installed at {script}"
94+
params = StdioServerParameters(command=str(script), args=[])
95+
with open(os.devnull, "w") as devnull:
96+
async with stdio_client(params, errlog=devnull) as (read, write):
97+
async with ClientSession(read, write) as session:
98+
await session.initialize()
99+
tools = await session.list_tools()
100+
assert [t.name for t in tools.tools] == ["fetch"]

0 commit comments

Comments
 (0)