|
| 1 | +"""Regression tests for #5158: pages that simplify to an empty tree under beautifulsoup4 4.13. |
| 2 | +
|
| 3 | +readabilipy's pure-Python extractor (the path taken without Node) finishes by |
| 4 | +wrapping its simplified tree in a ``<div>`` with ``Tag.append``. beautifulsoup4 |
| 5 | +4.13 implements ``append`` as ``self.insert(...)[0]``, and appending an empty |
| 6 | +tree inserts nothing, so a page with no extractable content (an SPA shell, an |
| 7 | +empty body) raised ``IndexError`` out of the tool call instead of returning the |
| 8 | +server's "Page failed to be simplified" error. |
| 9 | +
|
| 10 | +The lockfile pins beautifulsoup4 4.12, where ``append`` returns nothing and the |
| 11 | +crash cannot happen, and pyproject.toml is not where this is fixed: users who |
| 12 | +install the server unpinned get 4.13. So ``bs4_413_append`` reproduces 4.13's |
| 13 | +behavior on the installed version, and the first test proves the emulation |
| 14 | +raises the crash the issue reports before the others rely on it. |
| 15 | +""" |
| 16 | + |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +from typing import Any |
| 20 | + |
| 21 | +import httpx |
| 22 | +import pytest |
| 23 | +import readabilipy.simple_json |
| 24 | +from bs4 import BeautifulSoup |
| 25 | +from bs4.element import Tag |
| 26 | +from mcp.types import CallToolResult |
| 27 | + |
| 28 | +from .harness import FakeWeb, connect |
| 29 | + |
| 30 | +PAGE = "https://example.com/app" |
| 31 | +FAILED = "<error>Page failed to be simplified from HTML</error>" |
| 32 | + |
| 33 | +# The shape of the page in the issue's reproduction: client-rendered, nothing |
| 34 | +# to extract. (A <title> would survive simplification as text, so it has none.) |
| 35 | +SPA_SHELL = """<!doctype html><html><head><meta charset="utf-8"> |
| 36 | +<script type="module" src="/assets/index.js"></script></head> |
| 37 | +<body><div id="root"></div></body></html>""" |
| 38 | + |
| 39 | +ARTICLE = """<html><head><title>Article</title></head><body><article> |
| 40 | +<h1>Heading</h1><p>A paragraph with enough words to be the page's content.</p> |
| 41 | +</article></body></html>""" |
| 42 | + |
| 43 | + |
| 44 | +@pytest.fixture |
| 45 | +def bs4_413_append(monkeypatch: pytest.MonkeyPatch) -> None: |
| 46 | + """Make ``Tag.append`` index the elements it inserted, as beautifulsoup4 4.13 does.""" |
| 47 | + original = Tag.append |
| 48 | + |
| 49 | + def append(self: Tag, tag: Any) -> Any: |
| 50 | + # Appending a BeautifulSoup object inserts its children, so an empty |
| 51 | + # one inserts nothing, and 4.13's `insert(...)[0]` has nothing to index. |
| 52 | + inserted = list(tag.contents) if isinstance(tag, BeautifulSoup) else [tag] |
| 53 | + original(self, tag) |
| 54 | + return inserted[0] |
| 55 | + |
| 56 | + monkeypatch.setattr(Tag, "append", append) |
| 57 | + |
| 58 | + |
| 59 | +def wire(result: CallToolResult) -> dict[str, Any]: |
| 60 | + # The result as it crosses the wire (2025-11-25 fields only). |
| 61 | + return result.model_dump( |
| 62 | + by_alias=True, mode="json", exclude_none=True, exclude_unset=True |
| 63 | + ) |
| 64 | + |
| 65 | + |
| 66 | +def html(body: str) -> httpx.Response: |
| 67 | + return httpx.Response( |
| 68 | + 200, content=body.encode(), headers={"content-type": "text/html"} |
| 69 | + ) |
| 70 | + |
| 71 | + |
| 72 | +@pytest.mark.usefixtures("python_readability", "bs4_413_append") |
| 73 | +@pytest.mark.parametrize("page", [SPA_SHELL, "<html><body></body></html>", ""]) |
| 74 | +def test_emulation_reproduces_the_reported_crash(page: str) -> None: |
| 75 | + with pytest.raises(IndexError, match="list index out of range"): |
| 76 | + readabilipy.simple_json.simple_json_from_html_string(page, use_readability=True) |
| 77 | + |
| 78 | + |
| 79 | +@pytest.mark.usefixtures("python_readability", "bs4_413_append") |
| 80 | +async def test_empty_tree_tool_call_reports_simplification_failure( |
| 81 | + web: FakeWeb, |
| 82 | +) -> None: |
| 83 | + web.add(PAGE, html(SPA_SHELL)) |
| 84 | + async with connect(ignore_robots_txt=True) as (session, _): |
| 85 | + result = await session.call_tool("fetch", {"url": PAGE}) |
| 86 | + assert wire(result) == { |
| 87 | + "content": [{"type": "text", "text": f"Contents of {PAGE}:\n{FAILED}"}], |
| 88 | + "isError": False, |
| 89 | + } |
| 90 | + |
| 91 | + |
| 92 | +@pytest.mark.usefixtures("python_readability", "bs4_413_append") |
| 93 | +async def test_empty_tree_prompt_reports_simplification_failure( |
| 94 | + web: FakeWeb, |
| 95 | +) -> None: |
| 96 | + web.add(PAGE, html(SPA_SHELL)) |
| 97 | + async with connect() as (session, _): |
| 98 | + result = await session.get_prompt("fetch", {"url": PAGE}) |
| 99 | + assert len(result.messages) == 1 |
| 100 | + content = result.messages[0].content |
| 101 | + assert content.type == "text" |
| 102 | + assert content.text == FAILED |
| 103 | + |
| 104 | + |
| 105 | +@pytest.mark.usefixtures("python_readability", "bs4_413_append") |
| 106 | +async def test_page_with_content_still_extracts_under_bs4_413(web: FakeWeb) -> None: |
| 107 | + # Only the empty tree trips the index; a page with content is unaffected. |
| 108 | + web.add(PAGE, html(ARTICLE)) |
| 109 | + async with connect(ignore_robots_txt=True) as (session, _): |
| 110 | + result = await session.call_tool("fetch", {"url": PAGE}) |
| 111 | + data = wire(result) |
| 112 | + assert data["isError"] is False |
| 113 | + [content] = data["content"] |
| 114 | + assert content["type"] == "text" |
| 115 | + assert "# Heading" in content["text"] |
| 116 | + assert "A paragraph with enough words" in content["text"] |
0 commit comments