Skip to content

Commit c544ab4

Browse files
cliffhallclaude
andcommitted
feat(fetch): add --no-readability-js to keep extraction off Node.js
readabilipy runs Readability.js in a Node.js subprocess, with no timeout, whenever Node is on PATH, so a hung or misconfigured Node stalls the tool call. --no-readability-js asks readabilipy for its pure-Python extractor, which never starts Node or npm. The README documents both paths. Closes #4199 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 8520fb2 commit c544ab4

5 files changed

Lines changed: 174 additions & 7 deletions

File tree

‎src/fetch/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ The server speaks both eras of the Model Context Protocol over stdio, with nothi
4646
## Installation
4747

4848
Optionally: Install node.js, this will cause the fetch server to use a different HTML simplifier that is more robust.
49+
To keep the server from ever starting Node.js, see [Customization - HTML extraction](#customization---html-extraction).
4950

5051
### Using uv (recommended)
5152

@@ -210,6 +211,17 @@ environment variables (honoring `NO_PROXY`). Both `http(s)://` and SOCKS5 proxie
210211
installed with the server). A `socks://` proxy URL, as many desktop proxy settings export it, is treated as
211212
`socks5://`. A proxy setting the server cannot use makes each fetch fail with a tool error that says so.
212213

214+
### Customization - HTML extraction
215+
216+
HTML is converted to Markdown by [readabilipy](https://github.com/alan-turing-institute/ReadabiliPy). When Node.js is
217+
on the `PATH`, readabilipy runs Mozilla's Readability.js in a Node.js subprocess; otherwise it uses its own
218+
pure-Python extractor, which keeps headings and text but drops link targets. The Node.js subprocess runs with no
219+
timeout, so a Node.js that hangs, or fails in a way readabilipy does not detect, stalls the tool call until the client
220+
gives up.
221+
222+
To always use the pure-Python extractor, add the argument `--no-readability-js` to the `args` list in the
223+
configuration. The server then never starts Node.js (or npm), whether or not it is installed.
224+
213225
## Windows Configuration
214226

215227
If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding:

‎src/fetch/src/mcp_server_fetch/__init__.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ def main():
2121
action="store_true",
2222
help="Allow fetching private, loopback and link-local addresses (refused by default)",
2323
)
24+
parser.add_argument(
25+
"--no-readability-js",
26+
action="store_true",
27+
help="Extract HTML with the pure-Python extractor only, never starting Node.js "
28+
"or npm (by default Readability.js is used when Node.js is on PATH)",
29+
)
2430

2531
args = parser.parse_args()
2632
asyncio.run(
@@ -29,6 +35,7 @@ def main():
2935
args.ignore_robots_txt,
3036
args.proxy_url,
3137
args.allow_private_ips,
38+
not args.no_readability_js,
3239
)
3340
)
3441

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -167,19 +167,22 @@ async def _refuse_private_destination(request: httpx.Request) -> None:
167167
)
168168

169169

170-
def extract_content_from_html(html: str) -> str:
170+
def extract_content_from_html(html: str, use_readability_js: bool = True) -> str:
171171
"""Extract and convert HTML content to Markdown format.
172172
173173
Args:
174174
html: Raw HTML content to process
175+
use_readability_js: Ask readabilipy for Readability.js, which it runs
176+
under Node when Node is on PATH. False always uses its pure-Python
177+
extractor, so no Node (or npm) process is started (#4199).
175178
176179
Returns:
177180
Simplified markdown version of the content
178181
"""
179182
failed = "<error>Page failed to be simplified from HTML</error>"
180183
try:
181184
ret = readabilipy.simple_json.simple_json_from_html_string(
182-
html, use_readability=True
185+
html, use_readability=use_readability_js
183186
)
184187
except IndexError:
185188
# readabilipy's pure-Python extractor wraps its simplified tree with
@@ -326,6 +329,7 @@ async def fetch_url(
326329
proxy_url: str | None = None,
327330
allow_private_ips: bool = False,
328331
check_robots_txt: bool = False,
332+
use_readability_js: bool = True,
329333
) -> Tuple[str, str]:
330334
"""
331335
Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information.
@@ -378,7 +382,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None:
378382
)
379383

380384
if is_page_html and not force_raw:
381-
return extract_content_from_html(page_raw), ""
385+
return extract_content_from_html(page_raw, use_readability_js), ""
382386

383387
return (
384388
page_raw,
@@ -433,6 +437,7 @@ def create_server(
433437
ignore_robots_txt: bool = False,
434438
proxy_url: str | None = None,
435439
allow_private_ips: bool = False,
440+
use_readability_js: bool = True,
436441
) -> MCPServer:
437442
"""Build the fetch server: the `fetch` tool, the `fetch` prompt, and the guard.
438443
@@ -511,6 +516,7 @@ async def fetch(
511516
proxy_url=proxy_url,
512517
allow_private_ips=allow_private_ips,
513518
check_robots_txt=not ignore_robots_txt,
519+
use_readability_js=use_readability_js,
514520
)
515521
original_length = len(content)
516522
if start_index >= original_length:
@@ -542,6 +548,7 @@ async def fetch_prompt(
542548
user_agent_manual,
543549
proxy_url=proxy_url,
544550
allow_private_ips=allow_private_ips,
551+
use_readability_js=use_readability_js,
545552
)
546553
except ToolError as e:
547554
return str(e)
@@ -555,6 +562,7 @@ async def serve(
555562
ignore_robots_txt: bool = False,
556563
proxy_url: str | None = None,
557564
allow_private_ips: bool = False,
565+
use_readability_js: bool = True,
558566
) -> None:
559567
"""Run the fetch MCP server over stdio.
560568
@@ -564,9 +572,15 @@ async def serve(
564572
proxy_url: Optional proxy URL to use for requests
565573
allow_private_ips: Allow fetching private, loopback and link-local
566574
addresses, which are refused by default
575+
use_readability_js: Extract HTML with Readability.js under Node when
576+
Node is on PATH; False always uses the pure-Python extractor
567577
"""
568578
server = create_server(
569-
custom_user_agent, ignore_robots_txt, proxy_url, allow_private_ips
579+
custom_user_agent,
580+
ignore_robots_txt,
581+
proxy_url,
582+
allow_private_ips,
583+
use_readability_js,
570584
)
571585
# Both eras (#4853): the SDK's dual-era loop lets the client's first
572586
# request decide. `initialize` gets the 2025-11-25 handshake;

‎src/fetch/tests/test_entrypoints.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def run_main(monkeypatch: pytest.MonkeyPatch, *argv: str) -> AsyncMock:
3131

3232
def test_main_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
3333
serve = run_main(monkeypatch)
34-
serve.assert_awaited_once_with(None, False, None, False)
34+
serve.assert_awaited_once_with(None, False, None, False, True)
3535

3636

3737
def test_main_passes_every_flag(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -45,7 +45,7 @@ def test_main_passes_every_flag(monkeypatch: pytest.MonkeyPatch) -> None:
4545
"--allow-private-ips",
4646
)
4747
serve.assert_awaited_once_with(
48-
"MyBot/1.0", True, "http://proxy.example.com:8080", True
48+
"MyBot/1.0", True, "http://proxy.example.com:8080", True, True
4949
)
5050

5151

@@ -90,7 +90,7 @@ def test_package_init_run_as_main_calls_main(monkeypatch: pytest.MonkeyPatch) ->
9090
monkeypatch.setattr(mcp_server_fetch.server, "serve", serve)
9191
monkeypatch.setattr(sys, "argv", ["mcp-server-fetch"])
9292
runpy.run_module("mcp_server_fetch.__init__", run_name="__main__")
93-
serve.assert_awaited_once_with(None, False, None, False)
93+
serve.assert_awaited_once_with(None, False, None, False, True)
9494

9595

9696
def console_script() -> Path:
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
"""Tests for ``--no-readability-js`` (#4199): an operator opt-out from Node.js.
2+
3+
readabilipy runs Mozilla's Readability.js under Node whenever Node is on PATH
4+
(and readabilipy 0.3 may first run ``npm install``), in a subprocess with no
5+
timeout. A Node or npm that hangs, or a deployment that must not start them,
6+
had no remedy short of editing the installed package. With the flag the
7+
server asks readabilipy for its pure-Python extractor, which never probes for
8+
Node at all.
9+
10+
The opt-out tests replace readabilipy's ``subprocess`` module with one that
11+
fails on any use, so they prove no process is started rather than inferring
12+
it from the Markdown that comes back. The default-path test needs Node on PATH,
13+
like the repository's other Readability.js tests.
14+
"""
15+
16+
from __future__ import annotations
17+
18+
import sys
19+
from typing import Any
20+
from unittest.mock import AsyncMock
21+
22+
import httpx
23+
import pytest
24+
import readabilipy.simple_json
25+
from mcp.types import CallToolResult
26+
27+
import mcp_server_fetch
28+
29+
from .harness import FakeWeb, connect
30+
31+
PAGE = "https://example.com/article"
32+
33+
ARTICLE = """<html><head><title>Article</title></head><body><article>
34+
<h1>Heading</h1>
35+
<p>This is a <a href="https://example.com/x">link</a> in a paragraph with
36+
enough words to count as the main content of the page.</p>
37+
</article></body></html>"""
38+
39+
40+
class NoProcesses:
41+
"""Stands in for readabilipy's ``subprocess`` module: any use fails the test."""
42+
43+
def __getattr__(self, name: str) -> Any:
44+
raise AssertionError(f"readabilipy used subprocess.{name}")
45+
46+
47+
@pytest.fixture
48+
def no_processes(monkeypatch: pytest.MonkeyPatch) -> None:
49+
# Both readabilipy's Node probe (`have_node`) and its Readability.js call
50+
# go through this module attribute, as does 0.3's `npm install`.
51+
monkeypatch.setattr(readabilipy.simple_json, "subprocess", NoProcesses())
52+
53+
54+
def wire(result: CallToolResult) -> dict[str, Any]:
55+
return result.model_dump(
56+
by_alias=True, mode="json", exclude_none=True, exclude_unset=True
57+
)
58+
59+
60+
def html(body: str) -> httpx.Response:
61+
return httpx.Response(
62+
200, content=body.encode(), headers={"content-type": "text/html"}
63+
)
64+
65+
66+
def test_no_processes_fixture_catches_the_node_probe(no_processes: None) -> None:
67+
# The guard the opt-out tests rely on does fire on the default path.
68+
with pytest.raises(AssertionError, match="readabilipy used subprocess.run"):
69+
readabilipy.simple_json.simple_json_from_html_string(
70+
ARTICLE, use_readability=True
71+
)
72+
73+
74+
def test_flag_is_passed_to_serve(monkeypatch: pytest.MonkeyPatch) -> None:
75+
serve = AsyncMock()
76+
monkeypatch.setattr(mcp_server_fetch, "serve", serve)
77+
monkeypatch.setattr(sys, "argv", ["mcp-server-fetch", "--no-readability-js"])
78+
mcp_server_fetch.main()
79+
serve.assert_awaited_once_with(None, False, None, False, False)
80+
81+
82+
def test_flag_is_in_help(
83+
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
84+
) -> None:
85+
monkeypatch.setattr(mcp_server_fetch, "serve", AsyncMock())
86+
monkeypatch.setattr(sys, "argv", ["mcp-server-fetch", "--help"])
87+
with pytest.raises(SystemExit) as excinfo:
88+
mcp_server_fetch.main()
89+
assert excinfo.value.code == 0
90+
assert "--no-readability-js" in capsys.readouterr().out
91+
92+
93+
@pytest.mark.usefixtures("no_processes")
94+
async def test_tool_call_without_readability_js_starts_no_process(
95+
web: FakeWeb,
96+
) -> None:
97+
web.add(PAGE, html(ARTICLE))
98+
async with connect(ignore_robots_txt=True, use_readability_js=False) as (
99+
session,
100+
_,
101+
):
102+
result = await session.call_tool("fetch", {"url": PAGE})
103+
data = wire(result)
104+
assert data["isError"] is False
105+
[content] = data["content"]
106+
# The pure-Python extractor's Markdown: the heading and the text survive,
107+
# the link target does not (Readability.js keeps it).
108+
assert content["text"].startswith(f"Contents of {PAGE}:\n")
109+
assert "# Heading" in content["text"]
110+
assert "This is a link in a paragraph" in content["text"]
111+
assert "https://example.com/x" not in content["text"]
112+
113+
114+
@pytest.mark.usefixtures("no_processes")
115+
async def test_prompt_without_readability_js_starts_no_process(
116+
web: FakeWeb,
117+
) -> None:
118+
web.add(PAGE, html(ARTICLE))
119+
async with connect(use_readability_js=False) as (session, _):
120+
result = await session.get_prompt("fetch", {"url": PAGE})
121+
[message] = result.messages
122+
assert message.content.type == "text"
123+
assert "This is a link in a paragraph" in message.content.text
124+
assert "https://example.com/x" not in message.content.text
125+
126+
127+
@pytest.mark.usefixtures("node_readability")
128+
async def test_default_still_uses_readability_js(web: FakeWeb) -> None:
129+
# Without the flag nothing changes: Node is used, and it keeps link targets.
130+
web.add(PAGE, html(ARTICLE))
131+
async with connect(ignore_robots_txt=True) as (session, _):
132+
result = await session.call_tool("fetch", {"url": PAGE})
133+
[content] = wire(result)["content"]
134+
assert "[link](https://example.com/x)" in content["text"]

0 commit comments

Comments
 (0)