|
| 1 | +"""Tests for #3878: content that server-side rendering hides until hydration. |
| 2 | +
|
| 3 | +A progressively rendered page streams a visible loading shell, then the real |
| 4 | +page in a container hidden until the client hydrates it. Readability.js drops |
| 5 | +hidden nodes, so fetch returned only the shell. The server now re-runs |
| 6 | +Readability.js with such a container unhidden, but only when a hidden element |
| 7 | +(that is not navigation or other page furniture) holds more text than the whole |
| 8 | +first extraction, and keeps the second result only when it has more text. |
| 9 | +
|
| 10 | +What counts as hidden depends on the Readability.js that readabilipy runs: the |
| 11 | +copy bundled with readabilipy 0.2 (the lockfile's) drops ``display: none`` and |
| 12 | +``hidden``; the ``@mozilla/readability`` that readabilipy 0.3 installs also |
| 13 | +drops ``visibility: hidden`` and ``aria-hidden="true"``. The protocol tests |
| 14 | +that prove the second run happens use the two hidings both versions drop. They |
| 15 | +count readabilipy calls, so "a normal page is unchanged" is asserted as "it |
| 16 | +was extracted once, as before", not inferred from the Markdown. |
| 17 | +""" |
| 18 | + |
| 19 | +from __future__ import annotations |
| 20 | + |
| 21 | +from typing import Any |
| 22 | + |
| 23 | +import httpx |
| 24 | +import pytest |
| 25 | +import readabilipy.simple_json |
| 26 | +from bs4 import BeautifulSoup |
| 27 | +from mcp.types import CallToolResult |
| 28 | + |
| 29 | +from mcp_server_fetch.server import _is_hidden |
| 30 | + |
| 31 | +from .harness import FakeWeb, connect |
| 32 | + |
| 33 | +PAGE = "https://example.com/docs" |
| 34 | + |
| 35 | +ARTICLE = """<article><h1>Developer Concepts</h1> |
| 36 | +<p>Everything you need to integrate the framework: guides, concepts and |
| 37 | +references for developers and architects, rendered on the server.</p> |
| 38 | +<h2>Install</h2> |
| 39 | +<p>Install the package and initialize your first project. The command line |
| 40 | +handles project setup and configuration automatically.</p></article>""" |
| 41 | + |
| 42 | + |
| 43 | +def ssr_page(hiding: str) -> str: |
| 44 | + # The shape from the issue: a visible shell, then the content, hidden. |
| 45 | + return f"""<html><head><title>Docs</title></head><body> |
| 46 | +<div class="shell"><p>Unified framework for full-stack applications</p></div> |
| 47 | +<div {hiding}>{ARTICLE}</div></body></html>""" |
| 48 | + |
| 49 | + |
| 50 | +@pytest.fixture |
| 51 | +def calls(monkeypatch: pytest.MonkeyPatch) -> list[str]: |
| 52 | + """Record every HTML string readabilipy is asked to simplify.""" |
| 53 | + seen: list[str] = [] |
| 54 | + original = readabilipy.simple_json.simple_json_from_html_string |
| 55 | + |
| 56 | + def spy(html: str, **kwargs: Any) -> Any: |
| 57 | + seen.append(html) |
| 58 | + return original(html, **kwargs) |
| 59 | + |
| 60 | + monkeypatch.setattr(readabilipy.simple_json, "simple_json_from_html_string", spy) |
| 61 | + return seen |
| 62 | + |
| 63 | + |
| 64 | +def html(body: str) -> httpx.Response: |
| 65 | + return httpx.Response( |
| 66 | + 200, content=body.encode(), headers={"content-type": "text/html"} |
| 67 | + ) |
| 68 | + |
| 69 | + |
| 70 | +def text_of(result: CallToolResult) -> str: |
| 71 | + data = result.model_dump( |
| 72 | + by_alias=True, mode="json", exclude_none=True, exclude_unset=True |
| 73 | + ) |
| 74 | + assert data["isError"] is False |
| 75 | + [content] = data["content"] |
| 76 | + assert content["type"] == "text" |
| 77 | + return content["text"] |
| 78 | + |
| 79 | + |
| 80 | +async def fetch(web: FakeWeb, page: str, **serve_kwargs: Any) -> str: |
| 81 | + web.add(PAGE, html(page)) |
| 82 | + async with connect(ignore_robots_txt=True, **serve_kwargs) as (session, _): |
| 83 | + return text_of(await session.call_tool("fetch", {"url": PAGE})) |
| 84 | + |
| 85 | + |
| 86 | +@pytest.mark.usefixtures("node_readability") |
| 87 | +@pytest.mark.parametrize( |
| 88 | + "hiding", |
| 89 | + [ |
| 90 | + 'style="position:absolute;top:-9999px;display:none"', |
| 91 | + "hidden", |
| 92 | + 'style="display: NONE !important"', |
| 93 | + ], |
| 94 | +) |
| 95 | +async def test_hidden_ssr_content_is_recovered( |
| 96 | + web: FakeWeb, calls: list[str], hiding: str |
| 97 | +) -> None: |
| 98 | + text = await fetch(web, ssr_page(hiding)) |
| 99 | + assert "## Install" in text |
| 100 | + assert "rendered on the server" in text |
| 101 | + assert "configuration automatically" in text |
| 102 | + # Extracted once as is, then once with the container unhidden. |
| 103 | + assert len(calls) == 2 |
| 104 | + |
| 105 | + |
| 106 | +@pytest.mark.usefixtures("node_readability") |
| 107 | +@pytest.mark.parametrize("hiding", ['style="visibility: hidden"', 'aria-hidden="true"']) |
| 108 | +async def test_newer_readability_hidings_come_back_too( |
| 109 | + web: FakeWeb, hiding: str |
| 110 | +) -> None: |
| 111 | + # Dropped by readabilipy 0.3's Readability.js, kept by 0.2's: either way |
| 112 | + # the content reaches the model. |
| 113 | + text = await fetch(web, ssr_page(hiding)) |
| 114 | + assert "rendered on the server" in text |
| 115 | + |
| 116 | + |
| 117 | +@pytest.mark.usefixtures("node_readability") |
| 118 | +async def test_page_with_small_hidden_element_is_extracted_once( |
| 119 | + web: FakeWeb, calls: list[str] |
| 120 | +) -> None: |
| 121 | + page = f"""<html><body>{ARTICLE} |
| 122 | +<div style="display:none">Copied to clipboard</div></body></html>""" |
| 123 | + text = await fetch(web, page) |
| 124 | + assert "rendered on the server" in text |
| 125 | + assert "Copied to clipboard" not in text |
| 126 | + assert len(calls) == 1 |
| 127 | + |
| 128 | + |
| 129 | +@pytest.mark.usefixtures("node_readability") |
| 130 | +@pytest.mark.parametrize( |
| 131 | + ("opening", "closing"), |
| 132 | + [("<nav hidden>", "</nav>"), ('<div role="navigation" hidden>', "</div>")], |
| 133 | +) |
| 134 | +async def test_hidden_navigation_is_left_hidden( |
| 135 | + web: FakeWeb, calls: list[str], opening: str, closing: str |
| 136 | +) -> None: |
| 137 | + # A collapsed menu larger than a short page is not a hydration container. |
| 138 | + links = "".join( |
| 139 | + f'<li><a href="/p{i}">Navigation link number {i}</a></li>' for i in range(30) |
| 140 | + ) |
| 141 | + page = f"""<html><body><p>A short page with one visible paragraph.</p> |
| 142 | +{opening}<ul>{links}</ul>{closing}</body></html>""" |
| 143 | + text = await fetch(web, page) |
| 144 | + assert "one visible paragraph" in text |
| 145 | + assert "Navigation link" not in text |
| 146 | + assert len(calls) == 1 |
| 147 | + |
| 148 | + |
| 149 | +@pytest.mark.usefixtures("node_readability") |
| 150 | +async def test_second_run_is_kept_only_when_it_gains_text( |
| 151 | + web: FakeWeb, calls: list[str] |
| 152 | +) -> None: |
| 153 | + # Readability.js discards a <select> even when it is visible, so unhiding |
| 154 | + # it gains nothing and the first extraction is returned. |
| 155 | + options = "".join(f"<option>Option number {i}</option>" for i in range(30)) |
| 156 | + page = f"""<html><body><p>A short page with one visible paragraph.</p> |
| 157 | +<div hidden><select>{options}</select></div></body></html>""" |
| 158 | + text = await fetch(web, page) |
| 159 | + assert "one visible paragraph" in text |
| 160 | + assert "Option number" not in text |
| 161 | + assert len(calls) == 2 |
| 162 | + |
| 163 | + |
| 164 | +@pytest.mark.usefixtures("node_readability") |
| 165 | +async def test_shell_with_nothing_hidden_still_fails_to_simplify( |
| 166 | + web: FakeWeb, calls: list[str] |
| 167 | +) -> None: |
| 168 | + text = await fetch(web, "<html><body></body></html>") |
| 169 | + assert text == ( |
| 170 | + f"Contents of {PAGE}:\n<error>Page failed to be simplified from HTML</error>" |
| 171 | + ) |
| 172 | + assert len(calls) == 1 |
| 173 | + |
| 174 | + |
| 175 | +@pytest.mark.usefixtures("python_readability") |
| 176 | +async def test_pure_python_fallback_keeps_hidden_content_in_one_run( |
| 177 | + web: FakeWeb, calls: list[str] |
| 178 | +) -> None: |
| 179 | + # Without Node, readabilipy's own extractor ignores visibility, so the |
| 180 | + # content is there the first time and there is nothing to recover. |
| 181 | + text = await fetch(web, ssr_page("hidden")) |
| 182 | + assert "rendered on the server" in text |
| 183 | + assert len(calls) == 1 |
| 184 | + |
| 185 | + |
| 186 | +async def test_no_readability_js_is_never_rerun(web: FakeWeb, calls: list[str]) -> None: |
| 187 | + text = await fetch(web, ssr_page("hidden"), use_readability_js=False) |
| 188 | + assert "rendered on the server" in text |
| 189 | + assert len(calls) == 1 |
| 190 | + |
| 191 | + |
| 192 | +@pytest.mark.parametrize( |
| 193 | + ("element", "hidden"), |
| 194 | + [ |
| 195 | + ('<div style="display:none">', True), |
| 196 | + ('<div style="color: red; Display : None ;">', True), |
| 197 | + ('<div style="visibility:hidden !important">', True), |
| 198 | + ("<div hidden>", True), |
| 199 | + ('<div aria-hidden="true">', True), |
| 200 | + ("<div>", False), |
| 201 | + ('<div style="display:block">', False), |
| 202 | + ('<div style="visibility: visible">', False), |
| 203 | + ('<div style="display:none-ish">', False), |
| 204 | + ('<div aria-hidden="false">', False), |
| 205 | + ], |
| 206 | +) |
| 207 | +def test_is_hidden_follows_readability(element: str, hidden: bool) -> None: |
| 208 | + tag = BeautifulSoup(f"{element}text</div>", "html.parser").div |
| 209 | + assert tag is not None |
| 210 | + assert _is_hidden(tag) is hidden |
0 commit comments