|
| 1 | +""" |
| 2 | +Build hooks for the URLs the site serves and the files crawlers read. |
| 3 | +1. Link to each page by the URL it is served at (`page`, `dir/`), not its `.html` file, |
| 4 | + which the host redirects |
| 5 | +2. Write sitemap.xml with the URL of every page |
| 6 | +3. Write llms.txt (https://llmstxt.org), an index of the docs for language models |
| 7 | +4. Write llms-full.txt, the plain text of every page |
| 8 | +""" |
| 9 | + |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +from typing import TYPE_CHECKING |
| 13 | +from urllib.parse import quote |
| 14 | +from xml.sax.saxutils import escape |
| 15 | + |
| 16 | +from sphinx.builders.text import TextBuilder |
| 17 | +from sphinx.writers.text import TextTranslator |
| 18 | + |
| 19 | +if TYPE_CHECKING: |
| 20 | + from collections.abc import Iterator |
| 21 | + from typing import Any |
| 22 | + |
| 23 | + from sphinx.application import Sphinx |
| 24 | + |
| 25 | +# Pages that Sphinx generates and that hold no documentation. |
| 26 | +_GENERATED = {"genindex", "py-modindex", "search"} |
| 27 | + |
| 28 | + |
| 29 | +def _target_uri(docname: str, typ: str | None = None) -> str: |
| 30 | + if docname == "index": |
| 31 | + return "" |
| 32 | + if docname.endswith("/index"): |
| 33 | + return quote(docname.removesuffix("index")) |
| 34 | + return quote(docname) |
| 35 | + |
| 36 | + |
| 37 | +def _clean_links(app: Sphinx) -> None: |
| 38 | + if app.builder.name == "html": |
| 39 | + app.builder.get_target_uri = _target_uri |
| 40 | + |
| 41 | + |
| 42 | +def _url(app: Sphinx, docname: str) -> str: |
| 43 | + return app.config.html_baseurl.rstrip("/") + "/" + _target_uri(docname) |
| 44 | + |
| 45 | + |
| 46 | +def _toctree_order(app: Sphinx, docname: str = "index") -> Iterator[str]: |
| 47 | + """Yield `docname` and the documents its toctrees include, in reading order.""" |
| 48 | + yield docname |
| 49 | + for child in app.env.toctree_includes.get(docname, ()): |
| 50 | + yield from _toctree_order(app, child) |
| 51 | + |
| 52 | + |
| 53 | +def _pages(app: Sphinx) -> list[str]: |
| 54 | + ordered = list(dict.fromkeys(_toctree_order(app))) |
| 55 | + rest = sorted(app.env.found_docs - set(ordered)) |
| 56 | + return [doc for doc in ordered + rest if doc not in _GENERATED] |
| 57 | + |
| 58 | + |
| 59 | +def _title(app: Sphinx, docname: str) -> str: |
| 60 | + return app.env.titles[docname].astext() |
| 61 | + |
| 62 | + |
| 63 | +def _sitemap(app: Sphinx, pages: list[str]) -> str: |
| 64 | + urls = "".join(f" <url><loc>{escape(_url(app, doc))}</loc></url>\n" for doc in pages) |
| 65 | + return ( |
| 66 | + '<?xml version="1.0" encoding="UTF-8"?>\n' |
| 67 | + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n' |
| 68 | + f"{urls}" |
| 69 | + "</urlset>\n" |
| 70 | + ) |
| 71 | + |
| 72 | + |
| 73 | +def _summary(app: Sphinx) -> str: |
| 74 | + from docutils import nodes |
| 75 | + |
| 76 | + doctree = app.env.get_doctree("index") |
| 77 | + first = next(iter(doctree.findall(nodes.paragraph)), None) |
| 78 | + return " ".join(first.astext().split()) if first else "" |
| 79 | + |
| 80 | + |
| 81 | +def _llms(app: Sphinx) -> str: |
| 82 | + base = app.config.html_baseurl.rstrip("/") |
| 83 | + lines = [f"# {app.config.project}", "", f"> {_summary(app)}", ""] |
| 84 | + lines += [ |
| 85 | + f"The documentation for {app.config.project}, by section. " |
| 86 | + f"[llms-full.txt]({base}/llms-full.txt) holds the text of every page in one file.", |
| 87 | + "", |
| 88 | + ] |
| 89 | + for section in app.env.toctree_includes.get("index", ()): |
| 90 | + lines += [f"## {_title(app, section)}", ""] |
| 91 | + lines += [f"- [{_title(app, doc)}]({_url(app, doc)})" for doc in dict.fromkeys(_toctree_order(app, section))] |
| 92 | + lines.append("") |
| 93 | + lines += [ |
| 94 | + "## Optional", |
| 95 | + "", |
| 96 | + f"- [Full text of the documentation]({base}/llms-full.txt)", |
| 97 | + f"- [Java API reference (Javadoc)]({base}/_static/vortex-jni/)", |
| 98 | + f"- [Spark connector reference (Javadoc)]({base}/_static/vortex-spark/)", |
| 99 | + "- [Source code](https://github.com/vortex-data/vortex)", |
| 100 | + "", |
| 101 | + ] |
| 102 | + return "\n".join(lines) |
| 103 | + |
| 104 | + |
| 105 | +def _llms_full(app: Sphinx, pages: list[str]) -> str: |
| 106 | + text_builder = TextBuilder(app, app.env) |
| 107 | + text_builder.init() |
| 108 | + parts = [] |
| 109 | + for doc in pages: |
| 110 | + doctree = app.env.get_and_resolve_doctree(doc, app.builder) |
| 111 | + translator = TextTranslator(doctree, text_builder) |
| 112 | + doctree.walkabout(translator) |
| 113 | + parts.append(f"Source: {_url(app, doc)}\n\n{translator.body.strip()}\n") |
| 114 | + return "\n\n".join(parts) |
| 115 | + |
| 116 | + |
| 117 | +def _write(app: Sphinx, exception: Exception | None) -> None: |
| 118 | + if exception is not None or app.builder.name != "html": |
| 119 | + return |
| 120 | + out = app.builder.outdir |
| 121 | + pages = _pages(app) |
| 122 | + (out / "sitemap.xml").write_text(_sitemap(app, pages), encoding="utf-8") |
| 123 | + (out / "llms.txt").write_text(_llms(app), encoding="utf-8") |
| 124 | + (out / "llms-full.txt").write_text(_llms_full(app, pages), encoding="utf-8") |
| 125 | + |
| 126 | + |
| 127 | +def setup(app: Sphinx) -> dict[str, Any]: |
| 128 | + app.connect("builder-inited", _clean_links) |
| 129 | + app.connect("build-finished", _write) |
| 130 | + return {"parallel_read_safe": True, "parallel_write_safe": True} |
0 commit comments