Skip to content

Commit 8b912e1

Browse files
cliffhallclaude
andcommitted
fix(fetch): match the page title by visible whole words, off the loop
Copilot round 1 on #5398: the duplicate check compared characters with word boundaries removed and read link destinations and tags, so 'Rust' matched 'Trust' and a title only in a URL suppressed the heading. Compare the first line's visible words instead. Title extraction parses the whole page, so run it in a worker thread like the extraction itself. Issue: #2472 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 9b1570a commit 8b912e1

3 files changed

Lines changed: 45 additions & 6 deletions

File tree

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -603,7 +603,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None:
603603
content = await asyncio.to_thread(
604604
extract_content_from_html, page_raw, use_readability_js
605605
)
606-
return with_page_title(page_raw, content), ""
606+
return await asyncio.to_thread(with_page_title, page_raw, content), ""
607607

608608
# A response with no Content-Type would otherwise leave an empty slot here.
609609
declared_type = content_type or "(not declared)"

‎src/fetch/src/mcp_server_fetch/title.py‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,27 @@
2020
# nothing. A title on top of it would make the failure read as content.
2121
_FAILURE_PREFIX = "<error>"
2222

23-
# Markdown heading markers, emphasis, escapes, Sphinx's "¶" permalink and any
24-
# other punctuation are ignored when comparing a heading with the title.
25-
_NOT_WORD = re.compile(r"[\W_]+")
23+
# Titles are compared word by word: case, Markdown heading markers, emphasis,
24+
# escapes, Sphinx's "¶" permalink and other punctuation are ignored, but word
25+
# boundaries are kept, so "Rust" does not match "Trust".
26+
_WORD = re.compile(r"[^\W_]+")
27+
# What a reader does not see on a Markdown line: a link's destination and any
28+
# HTML tag (an autolink's URL included). A title found only there is not shown.
29+
_LINK_DESTINATION = re.compile(r"\]\([^)]*\)")
30+
_HTML_TAG = re.compile(r"<[^>]*>")
2631

2732

2833
def _normalized(text: str) -> str:
29-
return _NOT_WORD.sub("", text).casefold()
34+
"""``text`` as its casefolded words, one space apart, padded with spaces.
35+
36+
The padding makes ``in`` a whole-word, whole-phrase containment test.
37+
"""
38+
words = _WORD.findall(text.casefold())
39+
return f" {' '.join(words)} " if words else ""
40+
41+
42+
def _visible(line: str) -> str:
43+
return _HTML_TAG.sub(" ", _LINK_DESTINATION.sub("]", line))
3044

3145

3246
def _collapsed(text: str) -> str:
@@ -76,7 +90,7 @@ def _opens_with_title(content: str, title: str) -> bool:
7690
docs") to the page's own heading ("What's new").
7791
"""
7892
first = next((line for line in content.splitlines() if line.strip()), "")
79-
line, wanted = _normalized(first), _normalized(title)
93+
line, wanted = _normalized(_visible(first)), _normalized(title)
8094
if not line or not wanted:
8195
return False
8296
if wanted in line:

‎src/fetch/tests/test_page_title.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,17 @@ def test_page_title(html: str, expected: str | None) -> None:
136136
("Title\n\nBody.", "# Title Here\n\nTitle\n\nBody."),
137137
# A first line of punctuation only cannot match.
138138
("---\n\nBody.", "# Title Here\n\n---\n\nBody."),
139+
# The title only in a link destination or tag is not shown: added.
140+
(
141+
"[Source](https://example.com/title-here)\n",
142+
"# Title Here\n\n[Source](https://example.com/title-here)\n",
143+
),
144+
(
145+
"<a href='title-here'>x</a>\n",
146+
"# Title Here\n\n<a href='title-here'>x</a>\n",
147+
),
148+
# The title as a link's visible text is shown: not added.
149+
("[Title Here](https://example.com)\n", "[Title Here](https://example.com)\n"),
139150
# Empty content still gains the title.
140151
("", "# Title Here\n\n"),
141152
],
@@ -150,5 +161,19 @@ def test_with_page_title_leaves_untitled_page_and_failure_alone() -> None:
150161
assert with_page_title("<title>T</title>", failure) == failure
151162

152163

164+
@pytest.mark.parametrize(
165+
("title", "content"),
166+
[
167+
# Matching is by whole words, in both directions.
168+
("Rust", "Trust the compiler."),
169+
("Go", "Algorithms make this possible."),
170+
("Rust Guide", "# Rust Gui\n"),
171+
],
172+
)
173+
def test_partial_words_do_not_match(title: str, content: str) -> None:
174+
html = f"<title>{title}</title>"
175+
assert with_page_title(html, content) == f"# {title}\n\n{content}"
176+
177+
153178
def test_title_with_no_word_characters_is_not_matched() -> None:
154179
assert with_page_title("<title>—</title>", "Body.") == "# —\n\nBody."

0 commit comments

Comments
 (0)