[linkcheck] Use stricter HTML parsing during anchor-checking.

jayaddison · jayaddison · commit 328233289c0b · 2024-03-24T17:45:22.000Z
diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py
@@ -582,7 +582,7 @@ def _get_request_headers(
     return {}
 
 
-def contains_anchor(response: Response, anchor: str) -> bool:
+def contains_anchor(response: Response, anchor: str, *, lenient: bool = False) -> bool:
     """Determine if an anchor is contained within an HTTP response."""
     parser = AnchorCheckParser(unquote(anchor))
     # Read file in chunks. If we find a matching anchor, we break
@@ -595,6 +595,10 @@ def contains_anchor(response: Response, anchor: str) -> bool:
         if parser.found:
             break
     parser.close()
+
+    if parser.errors and not lenient:
+        raise ValueError(parser.errors[0])
+
     return parser.found
 
 
@@ -606,8 +610,17 @@ def __init__(self, search_anchor: str) -> None:
 
         self.search_anchor = search_anchor
         self.found = False
+        self.errors: list[str] = []
+        self.decl: str | None = None
+
+    def handle_decl(self, decl: str) -> None:
+        self.decl = decl
 
     def handle_starttag(self, tag: Any, attrs: Any) -> None:
+        if self.errors:
+            return
+        if self.decl is None:
+            self.errors.append(f'encountered start tag "{tag}" before a doctype declaration')
         for key, value in attrs:
             if key in ('id', 'name') and value == self.search_anchor:
                 self.found = True
diff --git a/tests/test_builders/test_build_linkcheck.py b/tests/test_builders/test_build_linkcheck.py
@@ -320,6 +320,7 @@ def do_GET(self):
         self._send_chunked(content)
 
 
+@pytest.mark.xfail()  # AnchorsIgnoreForUrlHandler returns incomplete HTML documents
 @pytest.mark.sphinx(
     'linkcheck', testroot='linkcheck-anchors-ignore-for-url', freshenv=True,
     confoverrides={'linkcheck_anchors_ignore_for_url': [
@@ -380,6 +381,29 @@ def do_GET(self):
     )
 
 
+@pytest.mark.sphinx('linkcheck', testroot='linkcheck-localserver-anchor', freshenv=True)
+def test_incomplete_html_anchor(app):
+    class IncompleteHTMLDocumentHandler(BaseHTTPRequestHandler):
+        protocol_version = 'HTTP/1.1'
+
+        def do_GET(self):
+            content = b'this is <div id="anchor">not</div> a valid HTML document'
+            self.send_response(200, 'OK')
+            self.send_header('Content-Length', str(len(content)))
+            self.end_headers()
+            self.wfile.write(content)
+
+    with http_server(IncompleteHTMLDocumentHandler):
+        app.build()
+
+    content = (app.outdir / 'output.json').read_text(encoding='utf8')
+    assert len(content.splitlines()) == 1
+
+    row = json.loads(content)
+    assert row['status'] == 'broken'
+    assert row['info'] == 'encountered start tag "div" before a doctype declaration'
+
+
 def custom_handler(valid_credentials=(), success_criteria=lambda _: True):
     """
     Returns an HTTP request handler that authenticates the client and then determines
