Skip to content

Commit 2a9dd44

Browse files
cliffhallclaude
andcommitted
feat(fetch): open the simplified result with the page title
Readability.js drops an article heading that repeats the document title, so a page like pandas' 2.1.0 release note came back without the line naming it. A new title module reads the title from the raw HTML (og:title, the h1 when <title> holds it, <title>, the h1) and puts it on top of the markdown unless the markdown already opens with it. Raw results and failed simplifications are untouched. Issue: modelcontextprotocol#2472 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 65cbd2c commit 2a9dd44

4 files changed

Lines changed: 253 additions & 1 deletion

File tree

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ Requires MCP Python SDK 2.x (`mcp>=2.3,<3`).
1515
1616
The fetch tool will truncate the response, but by using the `start_index` argument, you can specify where to start the content extraction. This lets models read a webpage in chunks, until they find the information they need.
1717

18+
The markdown of an HTML page opens with the page's title as a `#` heading (its `og:title`, else its `<h1>` when the `<title>` holds it, else its `<title>` or `<h1>`), because the extraction drops a heading that repeats the title. It is not added when the markdown already opens with it, to a raw result, or to a page that failed to simplify.
19+
1820
### Available Tools
1921

2022
- `fetch` - Fetches a URL from the internet and extracts its contents as markdown.

‎src/mcp_server_fetch/server.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
from protego import Protego
4242
from pydantic import AnyUrl, Field
4343

44+
from .title import with_page_title
45+
4446
# The version this server reports in serverInfo, read from the installed
4547
# distribution's metadata (pyproject.toml) so it cannot drift from the
4648
# published version (#360). Without it the SDK reports its own `mcp` version.
@@ -601,7 +603,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None:
601603
content = await asyncio.to_thread(
602604
extract_content_from_html, page_raw, use_readability_js
603605
)
604-
return content, ""
606+
return with_page_title(page_raw, content), ""
605607

606608
# A response with no Content-Type would otherwise leave an empty slot here.
607609
declared_type = content_type or "(not declared)"

‎src/mcp_server_fetch/title.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# The page title for the fetch server's simplified (Markdown) result (#2472).
2+
#
3+
# Readability.js removes an article heading that repeats the document title,
4+
# and neither extractor returns the `<title>` itself in the content the server
5+
# converts, so a page such as a Sphinx release note came back without the line
6+
# that says what it is. This module reads the title from the raw HTML and puts
7+
# it at the top of the Markdown as a level-one heading, unless the Markdown
8+
# already opens with it.
9+
#
10+
# It works from the raw page rather than from readabilipy's result, so it is
11+
# independent of how the extractor is invoked and is applied at the one call
12+
# site in `fetch_url` that simplifies HTML. The raw result (`raw: true`) and
13+
# non-HTML bodies never pass through it.
14+
15+
import re
16+
17+
from bs4 import BeautifulSoup, Tag
18+
19+
# The marker `extract_content_from_html` returns when a page simplifies to
20+
# nothing. A title on top of it would make the failure read as content.
21+
_FAILURE_PREFIX = "<error>"
22+
23+
# Markdown heading markers, emphasis, escapes, Sphinx's "¶" permalink and any
24+
# other punctuation are ignored when comparing a heading with the title.
25+
_NOT_WORD = re.compile(r"[\W_]+")
26+
27+
28+
def _normalized(text: str) -> str:
29+
return _NOT_WORD.sub("", text).casefold()
30+
31+
32+
def _collapsed(text: str) -> str:
33+
return " ".join(text.split())
34+
35+
36+
def _text(candidate: str) -> str:
37+
# A trailing "¶" is a permalink that is not marked up as one.
38+
return _collapsed(candidate).rstrip("¶").strip()
39+
40+
41+
def page_title(html: str) -> str | None:
42+
"""The page's title, or None when the page names none.
43+
44+
In order: `og:title`; the first `<h1>` when `<title>` holds it, since
45+
`<title>` then only adds the site's name ("What's new — pandas docs");
46+
`<title>`; the first `<h1>`.
47+
"""
48+
soup = BeautifulSoup(html, "html.parser")
49+
og = soup.find("meta", attrs={"property": "og:title"})
50+
og_title = _text(str(og.get("content", ""))) if isinstance(og, Tag) else ""
51+
title = _text(soup.title.get_text(" ")) if soup.title else ""
52+
h1 = soup.find("h1")
53+
heading = ""
54+
if isinstance(h1, Tag):
55+
# Sphinx appends a permalink anchor ("¶", or "#" in some themes) to
56+
# each heading; it is not part of the heading's text.
57+
for link in h1.find_all("a", class_="headerlink"):
58+
link.decompose()
59+
heading = _text(h1.get_text(" "))
60+
heading_in_title = bool(_normalized(heading)) and _normalized(
61+
heading
62+
) in _normalized(title)
63+
if og_title:
64+
return og_title
65+
if heading_in_title:
66+
return heading
67+
return title or heading or None
68+
69+
70+
def _opens_with_title(content: str, title: str) -> bool:
71+
"""Whether the Markdown's first non-empty line already carries the title.
72+
73+
A first line holding the whole title counts (the pure-Python extractor
74+
opens with the title in a paragraph), and so does a first heading that the
75+
title holds: `<title>` often adds the site name ("What's new — pandas
76+
docs") to the page's own heading ("What's new").
77+
"""
78+
first = next((line for line in content.splitlines() if line.strip()), "")
79+
line, wanted = _normalized(first), _normalized(title)
80+
if not line or not wanted:
81+
return False
82+
if wanted in line:
83+
return True
84+
return first.lstrip().startswith("#") and line in wanted
85+
86+
87+
def with_page_title(html: str, content: str) -> str:
88+
"""``content`` with the page's title as a heading on top, if it lacks one."""
89+
if content.startswith(_FAILURE_PREFIX):
90+
return content
91+
title = page_title(html)
92+
if title is None or _opens_with_title(content, title):
93+
return content
94+
return f"# {title}\n\n{content.lstrip()}"

‎tests/test_page_title.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
"""Tests for the page title on the fetch tool's simplified result (#2472).
2+
3+
Readability.js drops an article heading that repeats the document title, so a
4+
page such as pandas' "What's new in 2.1.0" release note came back without the
5+
line naming it. ``with_page_title`` now puts the title on top as a heading
6+
unless the Markdown already opens with it. The protocol-level tests drive the
7+
real server through an MCP client with HTTP served by ``harness.FakeWeb``, on
8+
both extractor paths, and pin that the raw result is untouched; the unit tests
9+
below pin each rule for choosing the title and spotting a duplicate.
10+
"""
11+
12+
from __future__ import annotations
13+
14+
from typing import Any
15+
16+
import httpx
17+
import pytest
18+
from mcp import ClientSession
19+
from mcp.types import CallToolResult
20+
21+
from mcp_server_fetch.title import page_title, with_page_title
22+
23+
from .harness import FakeWeb, connect
24+
25+
PAGE = "https://example.com/page"
26+
ROBOTS = "https://example.com/robots.txt"
27+
PREFIX = f"Contents of {PAGE}:\n"
28+
29+
# The shape of a Sphinx (pydata theme) page: `<title>` adds the site name to
30+
# the `<h1>`, which carries a "#" permalink.
31+
BODY = "<p>These are the changes in pandas 2.1.0, with a long enough body. " * 8
32+
SPHINX_PAGE = f"""<!DOCTYPE html>
33+
<html><head><title>What’s new in 2.1.0 (Aug 30, 2023) — pandas documentation</title></head>
34+
<body><article>
35+
<h1>What’s new in 2.1.0 (Aug 30, 2023)<a class="headerlink" href="#x">#</a></h1>
36+
{BODY}</p>
37+
</article></body></html>"""
38+
39+
40+
def text_of(result: CallToolResult) -> str:
41+
assert not result.is_error, result
42+
data = result.model_dump(by_alias=True, mode="json", exclude_none=True)
43+
assert len(data["content"]) == 1
44+
return data["content"][0]["text"]
45+
46+
47+
async def fetch(web: FakeWeb, html: str, **arguments: Any) -> str:
48+
web.add(ROBOTS, httpx.Response(404))
49+
web.add(
50+
PAGE,
51+
httpx.Response(200, text=html, headers={"content-type": "text/html"}),
52+
)
53+
async with connect() as (session, _):
54+
assert isinstance(session, ClientSession)
55+
result = await session.call_tool("fetch", {"url": PAGE, **arguments})
56+
return text_of(result)
57+
58+
59+
@pytest.mark.usefixtures("node_readability", "private_tempdir")
60+
async def test_readability_js_result_gains_the_dropped_title(web: FakeWeb) -> None:
61+
text = await fetch(web, SPHINX_PAGE)
62+
assert text.startswith(f"{PREFIX}# What’s new in 2.1.0 (Aug 30, 2023)\n\n")
63+
assert text.count("What’s new in 2.1.0") == 1
64+
assert "These are the changes in pandas 2.1.0" in text
65+
66+
67+
@pytest.mark.usefixtures("python_readability")
68+
async def test_python_result_is_not_given_a_second_title(web: FakeWeb) -> None:
69+
# The pure-Python extractor keeps the `<h1>` (and opens with the
70+
# `<title>`), so the title is already there and nothing is added.
71+
text = await fetch(web, SPHINX_PAGE)
72+
assert not text.startswith(f"{PREFIX}# ")
73+
assert "What’s new in 2.1.0" in text
74+
assert "These are the changes in pandas 2.1.0" in text
75+
76+
77+
async def test_raw_result_is_untouched(web: FakeWeb) -> None:
78+
text = await fetch(web, SPHINX_PAGE, raw=True)
79+
assert text.endswith(SPHINX_PAGE)
80+
assert "\n# What’s new" not in text
81+
82+
83+
@pytest.mark.usefixtures("node_readability", "private_tempdir")
84+
async def test_failed_simplification_is_not_titled(web: FakeWeb) -> None:
85+
text = await fetch(
86+
web, "<html><head><title>Shell</title></head><body></body></html>"
87+
)
88+
assert text == f"{PREFIX}<error>Page failed to be simplified from HTML</error>"
89+
90+
91+
@pytest.mark.parametrize(
92+
("html", "expected"),
93+
[
94+
# og:title wins over everything else.
95+
(
96+
'<head><meta property="og:title" content=" Social title "><title>T</title></head><h1>T</h1>',
97+
"Social title",
98+
),
99+
# The `<h1>` wins when `<title>` holds it.
100+
("<title>Heading | Site</title><h1>Heading</h1>", "Heading"),
101+
# Otherwise `<title>`.
102+
("<title>Document title</title><h1>Something else</h1>", "Document title"),
103+
# A trailing "¶" permalink that is plain text is dropped.
104+
("<title>Doc ¶</title>", "Doc"),
105+
# No `<title>`: the `<h1>`.
106+
("<body><h1>Only heading</h1></body>", "Only heading"),
107+
# An empty og:title and title fall through to the `<h1>`.
108+
(
109+
'<meta property="og:title" content=""><title> </title><h1>H</h1>',
110+
"H",
111+
),
112+
# Nothing names the page.
113+
("<body><p>No title</p></body>", None),
114+
("<title></title><h1><a class='headerlink'>#</a></h1>", None),
115+
],
116+
)
117+
def test_page_title(html: str, expected: str | None) -> None:
118+
assert page_title(html) == expected
119+
120+
121+
@pytest.mark.parametrize(
122+
("content", "expected"),
123+
[
124+
# No heading: the title goes on top.
125+
("Body text.\n", "# Title Here\n\nBody text.\n"),
126+
# Leading blank lines are folded into the separator.
127+
("\n\nBody text.\n", "# Title Here\n\nBody text.\n"),
128+
# Already opens with the title as a heading, at any level.
129+
("# Title Here\n\nBody.", "# Title Here\n\nBody."),
130+
("\n## title here ¶\n\nBody.", "\n## title here ¶\n\nBody."),
131+
# A first line that holds the whole title, heading or not.
132+
("<p>Title Here — Site</p>Skip\n", "<p>Title Here — Site</p>Skip\n"),
133+
# A first heading the title holds (the title adds a site name).
134+
("# Title\n\nBody.", "# Title\n\nBody."),
135+
# A first paragraph the title holds is not a heading: title added.
136+
("Title\n\nBody.", "# Title Here\n\nTitle\n\nBody."),
137+
# A first line of punctuation only cannot match.
138+
("---\n\nBody.", "# Title Here\n\n---\n\nBody."),
139+
# Empty content still gains the title.
140+
("", "# Title Here\n\n"),
141+
],
142+
)
143+
def test_with_page_title(content: str, expected: str) -> None:
144+
assert with_page_title("<title>Title Here</title>", content) == expected
145+
146+
147+
def test_with_page_title_leaves_untitled_page_and_failure_alone() -> None:
148+
assert with_page_title("<p>x</p>", "Body.") == "Body."
149+
failure = "<error>Page failed to be simplified from HTML</error>"
150+
assert with_page_title("<title>T</title>", failure) == failure
151+
152+
153+
def test_title_with_no_word_characters_is_not_matched() -> None:
154+
assert with_page_title("<title>—</title>", "Body.") == "# —\n\nBody."

0 commit comments

Comments
 (0)