Skip to content

Commit e59b236

Browse files
authored
docs: add robots.txt, a sitemap, llms.txt, and a favicon (#11166)
## Summary Crawlers and language models read docs.vortex.dev. This change gives them the files they look for, and it removes a redirect from every link. ## Changes - Links and canonical URLs use the URL that the host serves (`page`, `dir/`). Before, they used `page.html`, which the host redirects. - `robots.txt` allows all crawlers and names the sitemap. - The build writes `sitemap.xml`, `llms.txt` (https://llmstxt.org), and `llms-full.txt`, which holds the text of all pages. - `favicon.ico` is rendered from the existing logo. The pages do not change otherwise. `make check` passes. A local check found no internal link that redirects or fails, apart from the Javadoc, which only CI builds. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: Will Manning <[email protected]>
1 parent b1e8448 commit e59b236

5 files changed

Lines changed: 143 additions & 2 deletions

File tree

‎docs/_ext/vortex_site_files.py‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
"""
2+
Build hooks for the URLs the site serves and the files crawlers read.
3+
1. Link to each page by the URL it is served at (`page`, `dir/`), not its `.html` file,
4+
which the host redirects
5+
2. Write sitemap.xml with the URL of every page
6+
3. Write llms.txt (https://llmstxt.org), an index of the docs for language models
7+
4. Write llms-full.txt, the plain text of every page
8+
"""
9+
10+
from __future__ import annotations
11+
12+
from typing import TYPE_CHECKING
13+
from urllib.parse import quote
14+
from xml.sax.saxutils import escape
15+
16+
from sphinx.builders.text import TextBuilder
17+
from sphinx.writers.text import TextTranslator
18+
19+
if TYPE_CHECKING:
20+
from collections.abc import Iterator
21+
from typing import Any
22+
23+
from sphinx.application import Sphinx
24+
25+
# Pages that Sphinx generates and that hold no documentation.
26+
_GENERATED = {"genindex", "py-modindex", "search"}
27+
28+
29+
def _target_uri(docname: str, typ: str | None = None) -> str:
30+
if docname == "index":
31+
return ""
32+
if docname.endswith("/index"):
33+
return quote(docname.removesuffix("index"))
34+
return quote(docname)
35+
36+
37+
def _clean_links(app: Sphinx) -> None:
38+
if app.builder.name == "html":
39+
app.builder.get_target_uri = _target_uri
40+
41+
42+
def _url(app: Sphinx, docname: str) -> str:
43+
return app.config.html_baseurl.rstrip("/") + "/" + _target_uri(docname)
44+
45+
46+
def _toctree_order(app: Sphinx, docname: str = "index") -> Iterator[str]:
47+
"""Yield `docname` and the documents its toctrees include, in reading order."""
48+
yield docname
49+
for child in app.env.toctree_includes.get(docname, ()):
50+
yield from _toctree_order(app, child)
51+
52+
53+
def _pages(app: Sphinx) -> list[str]:
54+
ordered = list(dict.fromkeys(_toctree_order(app)))
55+
rest = sorted(app.env.found_docs - set(ordered))
56+
return [doc for doc in ordered + rest if doc not in _GENERATED]
57+
58+
59+
def _title(app: Sphinx, docname: str) -> str:
60+
return app.env.titles[docname].astext()
61+
62+
63+
def _sitemap(app: Sphinx, pages: list[str]) -> str:
64+
urls = "".join(f" <url><loc>{escape(_url(app, doc))}</loc></url>\n" for doc in pages)
65+
return (
66+
'<?xml version="1.0" encoding="UTF-8"?>\n'
67+
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
68+
f"{urls}"
69+
"</urlset>\n"
70+
)
71+
72+
73+
def _summary(app: Sphinx) -> str:
74+
from docutils import nodes
75+
76+
doctree = app.env.get_doctree("index")
77+
first = next(iter(doctree.findall(nodes.paragraph)), None)
78+
return " ".join(first.astext().split()) if first else ""
79+
80+
81+
def _llms(app: Sphinx) -> str:
82+
base = app.config.html_baseurl.rstrip("/")
83+
lines = [f"# {app.config.project}", "", f"> {_summary(app)}", ""]
84+
lines += [
85+
f"The documentation for {app.config.project}, by section. "
86+
f"[llms-full.txt]({base}/llms-full.txt) holds the text of every page in one file.",
87+
"",
88+
]
89+
for section in app.env.toctree_includes.get("index", ()):
90+
lines += [f"## {_title(app, section)}", ""]
91+
lines += [f"- [{_title(app, doc)}]({_url(app, doc)})" for doc in dict.fromkeys(_toctree_order(app, section))]
92+
lines.append("")
93+
lines += [
94+
"## Optional",
95+
"",
96+
f"- [Full text of the documentation]({base}/llms-full.txt)",
97+
f"- [Java API reference (Javadoc)]({base}/_static/vortex-jni/)",
98+
f"- [Spark connector reference (Javadoc)]({base}/_static/vortex-spark/)",
99+
"- [Source code](https://github.com/vortex-data/vortex)",
100+
"",
101+
]
102+
return "\n".join(lines)
103+
104+
105+
def _llms_full(app: Sphinx, pages: list[str]) -> str:
106+
text_builder = TextBuilder(app, app.env)
107+
text_builder.init()
108+
parts = []
109+
for doc in pages:
110+
doctree = app.env.get_and_resolve_doctree(doc, app.builder)
111+
translator = TextTranslator(doctree, text_builder)
112+
doctree.walkabout(translator)
113+
parts.append(f"Source: {_url(app, doc)}\n\n{translator.body.strip()}\n")
114+
return "\n\n".join(parts)
115+
116+
117+
def _write(app: Sphinx, exception: Exception | None) -> None:
118+
if exception is not None or app.builder.name != "html":
119+
return
120+
out = app.builder.outdir
121+
pages = _pages(app)
122+
(out / "sitemap.xml").write_text(_sitemap(app, pages), encoding="utf-8")
123+
(out / "llms.txt").write_text(_llms(app), encoding="utf-8")
124+
(out / "llms-full.txt").write_text(_llms_full(app, pages), encoding="utf-8")
125+
126+
127+
def setup(app: Sphinx) -> dict[str, Any]:
128+
app.connect("builder-inited", _clean_links)
129+
app.connect("build-finished", _write)
130+
return {"parallel_read_safe": True, "parallel_write_safe": True}

‎docs/_extra/favicon.ico‎

6.06 KB
Binary file not shown.

‎docs/_extra/robots.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Every page is open to crawlers, including AI crawlers.
2+
# https://docs.vortex.dev/llms.txt indexes the docs for language models.
3+
User-agent: *
4+
Allow: /
5+
6+
Sitemap: https://docs.vortex.dev/sitemap.xml

‎docs/api/java/index.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ The API is split into two main components:
1313
<div class="api-links">
1414
<h3>API Documentation</h3>
1515
<ul>
16-
<li><a href="../../_static/vortex-jni/index.html">Vortex JNI API</a> - Core Java bindings for Vortex</li>
17-
<li><a href="../../_static/vortex-spark/index.html">Vortex Spark API</a> - Apache Spark integration</li>
16+
<li><a href="../../_static/vortex-jni/">Vortex JNI API</a> - Core Java bindings for Vortex</li>
17+
<li><a href="../../_static/vortex-spark/">Vortex Spark API</a> - Apache Spark integration</li>
1818
</ul>
1919
</div>
2020

‎docs/conf.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"sphinx.ext.intersphinx",
3434
"sphinx.ext.napoleon",
3535
"vortex_theme",
36+
"vortex_site_files",
3637
]
3738

3839
templates_path = ["_templates"]
@@ -87,6 +88,10 @@
8788
html_baseurl = "https://docs.vortex.dev/"
8889
html_static_path = ["_static", "_build/_gen_static"]
8990
html_favicon = "_static/vortex_logo.svg" # relative to _static/
91+
# Copied to the site root: robots.txt, and favicon.ico for clients that ignore the icon link.
92+
html_extra_path = ["_extra"]
93+
# Pages are served without `.html`. vortex_site_files sets the page links; this sets the search results'.
94+
html_link_suffix = ""
9095

9196
os.makedirs(Path(__file__).parent / "_build" / "_gen_static", exist_ok=True)
9297

0 commit comments

Comments
 (0)