Skip to content

Commit efbeb27

Browse files
cliffhallclaude
andcommitted
fix(fetch): recover server-rendered content hidden until hydration
Progressively rendered pages stream a visible loading shell, then the real content in a container hidden until the client hydrates it. Readability.js drops hidden nodes, so fetch returned only the shell. When a hidden element that is not page furniture holds more text than the whole extraction, unhide it and run Readability.js again, keeping the second result only if it has more text. Pages whose hidden elements are smaller are extracted once, as before. Closes #3878 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 66e3dec commit efbeb27

3 files changed

Lines changed: 316 additions & 14 deletions

File tree

‎src/fetch/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,12 @@ pure-Python extractor, which keeps headings and text but drops link targets. The
219219
timeout, so a Node.js that hangs, or fails in a way readabilipy does not detect, stalls the tool call until the client
220220
gives up.
221221

222+
Readability.js discards hidden elements. Pages rendered progressively on the server often send a visible loading
223+
shell, then the real content in a container hidden (`display: none`, `visibility: hidden`, `hidden`) until scripts
224+
run in the browser. When a hidden element other than navigation or similar page furniture holds more text than
225+
Readability.js extracted, the server unhides it and runs Readability.js again, keeping that result only if it has more
226+
text. Other pages are extracted once, exactly as before.
227+
222228
To always use the pure-Python extractor, add the argument `--no-readability-js` to the `args` list in the
223229
configuration. The server then never starts Node.js (or npm), whether or not it is installed.
224230

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 100 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import httpx
3232
import markdownify
3333
import readabilipy.simple_json
34+
from bs4 import BeautifulSoup, Tag
3435
from mcp.server import MCPServer, ServerRequestContext
3536
from mcp.server.context import CallNext, HandlerResult, ServerMiddleware
3637
from mcp.server.mcpserver.exceptions import ToolError
@@ -167,6 +168,98 @@ async def _refuse_private_destination(request: httpx.Request) -> None:
167168
)
168169

169170

171+
def _readability_content(html: str, use_readability_js: bool) -> str:
172+
"""readabilipy's simplified HTML for ``html``, or "" when it has none."""
173+
try:
174+
ret = readabilipy.simple_json.simple_json_from_html_string(
175+
html, use_readability=use_readability_js
176+
)
177+
except IndexError:
178+
# readabilipy's pure-Python extractor wraps its simplified tree with
179+
# `Tag.append`, which beautifulsoup4 4.13 implements as
180+
# `self.insert(...)[0]`. A page that simplifies to nothing (an SPA
181+
# shell, an empty body) inserts nothing, so that index raises (#5158).
182+
# Under beautifulsoup4 <= 4.12 the same page comes back as an empty
183+
# tree, reported by the caller as this same failure.
184+
return ""
185+
return ret["content"] or ""
186+
187+
188+
def _hiding_declaration(declaration: str) -> bool:
189+
"""Whether one inline-style declaration is one Readability.js treats as hidden."""
190+
prop, _, value = declaration.partition(":")
191+
value = value.replace("!important", "").strip().lower()
192+
return (prop.strip().lower(), value) in {
193+
("display", "none"),
194+
("visibility", "hidden"),
195+
}
196+
197+
198+
def _is_hidden(tag: Tag) -> bool:
199+
"""Whether Readability.js drops ``tag`` as invisible (its `_isProbablyVisible`)."""
200+
if tag.has_attr("hidden") or tag.get("aria-hidden") == "true":
201+
return True
202+
return any(_hiding_declaration(d) for d in str(tag.get("style", "")).split(";"))
203+
204+
205+
# Hidden page furniture that is expected to stay hidden (a collapsed menu, a
206+
# closed dialog): never a hydration container, so never unhidden.
207+
_PAGE_CHROME_TAGS = frozenset({"nav", "header", "footer", "aside", "menu", "dialog"})
208+
_PAGE_CHROME_ROLES = frozenset(
209+
{"navigation", "banner", "contentinfo", "menu", "menubar", "dialog", "alertdialog"}
210+
)
211+
212+
213+
def _is_page_chrome(tag: Tag) -> bool:
214+
return tag.name in _PAGE_CHROME_TAGS or tag.get("role") in _PAGE_CHROME_ROLES
215+
216+
217+
def _text_length(html: Tag) -> int:
218+
"""The length of the text in ``html``, whitespace collapsed."""
219+
return len(" ".join(html.get_text(" ").split()))
220+
221+
222+
def _recover_hidden_content(html: str, content: str) -> str:
223+
"""Re-extract a page whose content Readability.js dropped as hidden (#3878).
224+
225+
Progressive server-side rendering streams a visible loading shell first,
226+
then the real page in a container hidden (``visibility: hidden``,
227+
``display: none``, ``hidden``) until the client hydrates it. Readability.js
228+
discards hidden nodes, so only the shell's text survived. When a hidden
229+
element holds more text than the whole extraction, those elements are
230+
unhidden and Readability.js runs again; the second result is kept only if
231+
it has more text. A page whose hidden elements are smaller than its
232+
extracted content, or are page furniture (navigation, menus, dialogs), is
233+
returned exactly as before, from the one Readability.js run.
234+
"""
235+
extracted = _text_length(BeautifulSoup(content, "html.parser"))
236+
soup = BeautifulSoup(html, "html.parser")
237+
larger = [
238+
tag
239+
for tag in soup.find_all(_is_hidden)
240+
# Outermost only: unhiding a container exposes what it holds, and
241+
# anything hidden inside it stays hidden.
242+
if not any(_is_hidden(parent) for parent in tag.parents)
243+
and not _is_page_chrome(tag)
244+
and _text_length(tag) > extracted
245+
]
246+
if not larger:
247+
return content
248+
for tag in larger:
249+
del tag["hidden"]
250+
del tag["aria-hidden"]
251+
style = [
252+
d
253+
for d in str(tag.get("style", "")).split(";")
254+
if not _hiding_declaration(d)
255+
]
256+
tag["style"] = ";".join(style)
257+
recovered = _readability_content(str(soup), use_readability_js=True)
258+
if _text_length(BeautifulSoup(recovered, "html.parser")) > extracted:
259+
return recovered
260+
return content
261+
262+
170263
def extract_content_from_html(html: str, use_readability_js: bool = True) -> str:
171264
"""Extract and convert HTML content to Markdown format.
172265
@@ -180,22 +273,15 @@ def extract_content_from_html(html: str, use_readability_js: bool = True) -> str
180273
Simplified markdown version of the content
181274
"""
182275
failed = "<error>Page failed to be simplified from HTML</error>"
183-
try:
184-
ret = readabilipy.simple_json.simple_json_from_html_string(
185-
html, use_readability=use_readability_js
186-
)
187-
except IndexError:
188-
# readabilipy's pure-Python extractor wraps its simplified tree with
189-
# `Tag.append`, which beautifulsoup4 4.13 implements as
190-
# `self.insert(...)[0]`. A page that simplifies to nothing (an SPA
191-
# shell, an empty body) inserts nothing, so that index raises (#5158).
192-
# Under beautifulsoup4 <= 4.12 the same page comes back as an empty
193-
# tree, reported below as this same failure.
194-
return failed
195-
if not ret["content"]:
276+
content_html = _readability_content(html, use_readability_js)
277+
if use_readability_js:
278+
# Only Readability.js drops hidden elements; the pure-Python extractor
279+
# keeps them, so its result never needs recovering.
280+
content_html = _recover_hidden_content(html, content_html)
281+
if not content_html:
196282
return failed
197283
content = markdownify.markdownify(
198-
ret["content"],
284+
content_html,
199285
heading_style=markdownify.ATX,
200286
)
201287
# Without Node, readabilipy's pure-Python extractor never returns empty

‎src/fetch/tests/test_hidden_ssr.py‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
"""Tests for #3878: content that server-side rendering hides until hydration.
2+
3+
A progressively rendered page streams a visible loading shell, then the real
4+
page in a container hidden until the client hydrates it. Readability.js drops
5+
hidden nodes, so fetch returned only the shell. The server now re-runs
6+
Readability.js with such a container unhidden, but only when a hidden element
7+
(that is not navigation or other page furniture) holds more text than the whole
8+
first extraction, and keeps the second result only when it has more text.
9+
10+
What counts as hidden depends on the Readability.js that readabilipy runs: the
11+
copy bundled with readabilipy 0.2 (the lockfile's) drops ``display: none`` and
12+
``hidden``; the ``@mozilla/readability`` that readabilipy 0.3 installs also
13+
drops ``visibility: hidden`` and ``aria-hidden="true"``. The protocol tests
14+
that prove the second run happens use the two hidings both versions drop. They
15+
count readabilipy calls, so "a normal page is unchanged" is asserted as "it
16+
was extracted once, as before", not inferred from the Markdown.
17+
"""
18+
19+
from __future__ import annotations
20+
21+
from typing import Any
22+
23+
import httpx
24+
import pytest
25+
import readabilipy.simple_json
26+
from bs4 import BeautifulSoup
27+
from mcp.types import CallToolResult
28+
29+
from mcp_server_fetch.server import _is_hidden
30+
31+
from .harness import FakeWeb, connect
32+
33+
PAGE = "https://example.com/docs"
34+
35+
ARTICLE = """<article><h1>Developer Concepts</h1>
36+
<p>Everything you need to integrate the framework: guides, concepts and
37+
references for developers and architects, rendered on the server.</p>
38+
<h2>Install</h2>
39+
<p>Install the package and initialize your first project. The command line
40+
handles project setup and configuration automatically.</p></article>"""
41+
42+
43+
def ssr_page(hiding: str) -> str:
44+
# The shape from the issue: a visible shell, then the content, hidden.
45+
return f"""<html><head><title>Docs</title></head><body>
46+
<div class="shell"><p>Unified framework for full-stack applications</p></div>
47+
<div {hiding}>{ARTICLE}</div></body></html>"""
48+
49+
50+
@pytest.fixture
51+
def calls(monkeypatch: pytest.MonkeyPatch) -> list[str]:
52+
"""Record every HTML string readabilipy is asked to simplify."""
53+
seen: list[str] = []
54+
original = readabilipy.simple_json.simple_json_from_html_string
55+
56+
def spy(html: str, **kwargs: Any) -> Any:
57+
seen.append(html)
58+
return original(html, **kwargs)
59+
60+
monkeypatch.setattr(readabilipy.simple_json, "simple_json_from_html_string", spy)
61+
return seen
62+
63+
64+
def html(body: str) -> httpx.Response:
65+
return httpx.Response(
66+
200, content=body.encode(), headers={"content-type": "text/html"}
67+
)
68+
69+
70+
def text_of(result: CallToolResult) -> str:
71+
data = result.model_dump(
72+
by_alias=True, mode="json", exclude_none=True, exclude_unset=True
73+
)
74+
assert data["isError"] is False
75+
[content] = data["content"]
76+
assert content["type"] == "text"
77+
return content["text"]
78+
79+
80+
async def fetch(web: FakeWeb, page: str, **serve_kwargs: Any) -> str:
81+
web.add(PAGE, html(page))
82+
async with connect(ignore_robots_txt=True, **serve_kwargs) as (session, _):
83+
return text_of(await session.call_tool("fetch", {"url": PAGE}))
84+
85+
86+
@pytest.mark.usefixtures("node_readability")
87+
@pytest.mark.parametrize(
88+
"hiding",
89+
[
90+
'style="position:absolute;top:-9999px;display:none"',
91+
"hidden",
92+
'style="display: NONE !important"',
93+
],
94+
)
95+
async def test_hidden_ssr_content_is_recovered(
96+
web: FakeWeb, calls: list[str], hiding: str
97+
) -> None:
98+
text = await fetch(web, ssr_page(hiding))
99+
assert "## Install" in text
100+
assert "rendered on the server" in text
101+
assert "configuration automatically" in text
102+
# Extracted once as is, then once with the container unhidden.
103+
assert len(calls) == 2
104+
105+
106+
@pytest.mark.usefixtures("node_readability")
107+
@pytest.mark.parametrize("hiding", ['style="visibility: hidden"', 'aria-hidden="true"'])
108+
async def test_newer_readability_hidings_come_back_too(
109+
web: FakeWeb, hiding: str
110+
) -> None:
111+
# Dropped by readabilipy 0.3's Readability.js, kept by 0.2's: either way
112+
# the content reaches the model.
113+
text = await fetch(web, ssr_page(hiding))
114+
assert "rendered on the server" in text
115+
116+
117+
@pytest.mark.usefixtures("node_readability")
118+
async def test_page_with_small_hidden_element_is_extracted_once(
119+
web: FakeWeb, calls: list[str]
120+
) -> None:
121+
page = f"""<html><body>{ARTICLE}
122+
<div style="display:none">Copied to clipboard</div></body></html>"""
123+
text = await fetch(web, page)
124+
assert "rendered on the server" in text
125+
assert "Copied to clipboard" not in text
126+
assert len(calls) == 1
127+
128+
129+
@pytest.mark.usefixtures("node_readability")
130+
@pytest.mark.parametrize(
131+
("opening", "closing"),
132+
[("<nav hidden>", "</nav>"), ('<div role="navigation" hidden>', "</div>")],
133+
)
134+
async def test_hidden_navigation_is_left_hidden(
135+
web: FakeWeb, calls: list[str], opening: str, closing: str
136+
) -> None:
137+
# A collapsed menu larger than a short page is not a hydration container.
138+
links = "".join(
139+
f'<li><a href="/p{i}">Navigation link number {i}</a></li>' for i in range(30)
140+
)
141+
page = f"""<html><body><p>A short page with one visible paragraph.</p>
142+
{opening}<ul>{links}</ul>{closing}</body></html>"""
143+
text = await fetch(web, page)
144+
assert "one visible paragraph" in text
145+
assert "Navigation link" not in text
146+
assert len(calls) == 1
147+
148+
149+
@pytest.mark.usefixtures("node_readability")
150+
async def test_second_run_is_kept_only_when_it_gains_text(
151+
web: FakeWeb, calls: list[str]
152+
) -> None:
153+
# Readability.js discards a <select> even when it is visible, so unhiding
154+
# it gains nothing and the first extraction is returned.
155+
options = "".join(f"<option>Option number {i}</option>" for i in range(30))
156+
page = f"""<html><body><p>A short page with one visible paragraph.</p>
157+
<div hidden><select>{options}</select></div></body></html>"""
158+
text = await fetch(web, page)
159+
assert "one visible paragraph" in text
160+
assert "Option number" not in text
161+
assert len(calls) == 2
162+
163+
164+
@pytest.mark.usefixtures("node_readability")
165+
async def test_shell_with_nothing_hidden_still_fails_to_simplify(
166+
web: FakeWeb, calls: list[str]
167+
) -> None:
168+
text = await fetch(web, "<html><body></body></html>")
169+
assert text == (
170+
f"Contents of {PAGE}:\n<error>Page failed to be simplified from HTML</error>"
171+
)
172+
assert len(calls) == 1
173+
174+
175+
@pytest.mark.usefixtures("python_readability")
176+
async def test_pure_python_fallback_keeps_hidden_content_in_one_run(
177+
web: FakeWeb, calls: list[str]
178+
) -> None:
179+
# Without Node, readabilipy's own extractor ignores visibility, so the
180+
# content is there the first time and there is nothing to recover.
181+
text = await fetch(web, ssr_page("hidden"))
182+
assert "rendered on the server" in text
183+
assert len(calls) == 1
184+
185+
186+
async def test_no_readability_js_is_never_rerun(web: FakeWeb, calls: list[str]) -> None:
187+
text = await fetch(web, ssr_page("hidden"), use_readability_js=False)
188+
assert "rendered on the server" in text
189+
assert len(calls) == 1
190+
191+
192+
@pytest.mark.parametrize(
193+
("element", "hidden"),
194+
[
195+
('<div style="display:none">', True),
196+
('<div style="color: red; Display : None ;">', True),
197+
('<div style="visibility:hidden !important">', True),
198+
("<div hidden>", True),
199+
('<div aria-hidden="true">', True),
200+
("<div>", False),
201+
('<div style="display:block">', False),
202+
('<div style="visibility: visible">', False),
203+
('<div style="display:none-ish">', False),
204+
('<div aria-hidden="false">', False),
205+
],
206+
)
207+
def test_is_hidden_follows_readability(element: str, hidden: bool) -> None:
208+
tag = BeautifulSoup(f"{element}text</div>", "html.parser").div
209+
assert tag is not None
210+
assert _is_hidden(tag) is hidden

0 commit comments

Comments
 (0)