Skip to content

Commit 40e64f6

Browse files
fix(docs): restore legacy documentation URLs
Map missing applications, MySQL, kexec, LINSTOR and roadmap URLs to their current content. Keep distro-hosted on the v0 bundles archive. The LINSTOR mapping uses the storage overview because its historical page was not found. Serve static HTML copies with the target's canonical and robots policy, including the current-major paths. Preserve queries and existing fragments without redirects or JavaScript. Validate all mappings before output is replaced and include compatibility pages in ownership-based cleanup. Cover HTTP responses, release rollover, preview policy, archived content, invalid mappings, symlinks and collisions with real pages. Assisted-By: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 4be61a8 commit 40e64f6

5 files changed

Lines changed: 252 additions & 5 deletions

File tree

‎CONTRIBUTING.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,15 @@ If you need to understand or change the versioning machinery itself:
262262
not. Local `make serve` keeps the version-selection landing and its navigation
263263
links at `/docs/`. The landing template carries `data-docs-version-landing` so
264264
repeat mirror runs can identify it independently of Open Graph metadata.
265+
- `data/docs-legacy-urls.json` maps old English URLs to current docs or a numbered
266+
archive. The same generator serves full HTML copies at those paths, with the
267+
source page's canonical and robots policy. Current-doc mappings also produce
268+
their major-version paths. There is no redirect, so queries and existing
269+
fragments stay on the requested URL. Copies carry `data-docs-legacy-url` and
270+
are tracked in the manifest. Missing targets, symlinks and collisions with
271+
real pages fail before output is replaced.
272+
`distro-hosted` uses the v0 archive; the old LINSTOR path uses the storage
273+
overview because its exact historical page was not identified.
265274
- `layouts/partials/docs-seo.html` supplies canonical and robots rules to the
266275
document head and sitemap. Current numbered docs and their copies canonicalize
267276
to `/docs/`. Older versions remain noindex and stay out of sitemaps.

‎data/docs-legacy-urls.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"/docs/reference/applications/": "/docs/applications/",
3+
"/docs/reference/applications/mysql/": "/docs/applications/mariadb/",
4+
"/docs/install/talos/kexec/": "/docs/install/talos/boot-to-talos/",
5+
"/roadmap/": "/docs/roadmap/",
6+
"/docs/bundles/distro-hosted/": "/docs/v0/operations/configuration/bundles/",
7+
"/docs/operations/storage/linstor/": "/docs/storage/"
8+
}

‎hack/mirror_latest.py‎

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ def __init__(self, html):
1515
super().__init__()
1616
self.url = None
1717
self.version_landing = False
18+
self.redirect = False
19+
self.legacy_url = None
1820
self.feed(html)
1921

2022
def handle_starttag(self, tag, attrs):
@@ -24,6 +26,10 @@ def handle_starttag(self, tag, attrs):
2426
self.url = attrs.get("content")
2527
if "data-docs-version-landing" in attrs:
2628
self.version_landing = True
29+
if tag == "meta" and attrs.get("http-equiv", "").lower() == "refresh":
30+
self.redirect = True
31+
if "data-docs-legacy-url" in attrs:
32+
self.legacy_url = attrs["data-docs-legacy-url"]
2733

2834

2935
def rewrite_og_url(text, source_url, target_url):
@@ -70,6 +76,59 @@ def root_target(public, name, *, allow_major=False):
7076
return target
7177

7278

79+
def legacy_page(public, url):
80+
if (not isinstance(url, str) or not re.fullmatch(r"/(?:[A-Za-z0-9_.-]+/)+", url)
81+
or any(part in (".", "..") for part in url.split("/"))):
82+
raise ValueError(f"invalid legacy URL: {url}")
83+
path = public / url.lstrip("/") / "index.html"
84+
if (path.resolve() != path or path.is_dir()
85+
or any(p.is_file() for p in path.parents if p == public or public in p.parents)):
86+
raise ValueError(f"invalid legacy page path: {path}")
87+
return path
88+
89+
90+
def legacy_copies(public, latest, copies, previous):
91+
mappings = json.loads(Path("data/docs-legacy-urls.json").read_text())
92+
if not isinstance(mappings, dict):
93+
raise ValueError("data/docs-legacy-urls.json must be a URL-to-URL object")
94+
result = {}
95+
major = latest.split(".")[0]
96+
for old, new in mappings.items():
97+
legacy_page(public, old)
98+
source = legacy_page(public, new)
99+
parts = old.strip("/").split("/")
100+
if old != "/roadmap/" and (len(parts) < 2 or parts[0] != "docs" or reserved(parts[1])):
101+
raise ValueError(f"legacy destination must be an unversioned docs path: {old}")
102+
if old == new or not new.startswith("/docs/"):
103+
raise ValueError(f"invalid legacy target: {old} -> {new}")
104+
if source in copies:
105+
content = copies[source]
106+
elif re.match(r"^/docs/v[0-9]+(?:\.[0-9]+)?/", new) and source.is_file():
107+
content = source.read_bytes()
108+
else:
109+
raise ValueError(f"legacy target is not built: {old} -> {new}")
110+
if PageURL(content.decode()).redirect:
111+
raise ValueError(f"legacy target must be a content page: {new}")
112+
urls = [old]
113+
if source in copies and old.startswith("/docs/") and major != latest:
114+
urls.append(f"/docs/{major}/" + old[len("/docs/"):])
115+
for url in urls:
116+
target = legacy_page(public, url)
117+
if target in copies and not PageURL(copies[target].decode()).redirect:
118+
raise ValueError(f"legacy URL collides with current content: {url}")
119+
if target.is_file():
120+
existing = target.read_bytes()
121+
page = PageURL(existing.decode())
122+
owned = previous.get(str(target.relative_to(public))) == sha256(existing).hexdigest()
123+
if not (page.redirect or page.legacy_url == url or owned):
124+
raise ValueError(f"legacy URL collides with existing content: {url}")
125+
text, count = re.subn(r"<html\b", f'<html data-docs-legacy-url="{url}"', content.decode(), count=1)
126+
if not count:
127+
raise ValueError(f"legacy target has no HTML document: {new}")
128+
result[target] = text.encode()
129+
return result
130+
131+
73132
def mirror(public):
74133
config = Path("hugo.yaml").read_text()
75134
match = re.search(r'^ latest_version_id:\s*"(v[0-9]+(?:\.[0-9]+)?)"\s*$', config, re.M)
@@ -86,7 +145,10 @@ def mirror(public):
86145
if not isinstance(previous, dict):
87146
raise ValueError(f"invalid root mirror manifest: {manifest}")
88147
for name, digest in previous.items():
89-
root_target(public, name, allow_major=True)
148+
if name == "roadmap/index.html":
149+
legacy_page(public, "/roadmap/")
150+
else:
151+
root_target(public, name, allow_major=True)
90152
if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest):
91153
raise ValueError(f"invalid root mirror digest: {name}")
92154

@@ -124,14 +186,20 @@ def mirror(public):
124186
root_target(public, target.relative_to(public), allow_major=bool(alias))
125187
plans.append((file, source_url, target, alias))
126188

127-
for versions, content in landings:
128-
versions.parent.mkdir(parents=True, exist_ok=True)
129-
versions.write_bytes(content)
130-
current = {}
189+
copies = {}
131190
for file, source_url, target, alias in plans:
132191
content = file.read_bytes()
133192
if file.suffix in (".html", ".xml"):
134193
content = rewrite(content.decode(), source_url, alias).encode()
194+
copies[target] = content
195+
legacy = legacy_copies(public, latest, copies, previous)
196+
copies.update(legacy)
197+
198+
for versions, content in landings:
199+
versions.parent.mkdir(parents=True, exist_ok=True)
200+
versions.write_bytes(content)
201+
current = {}
202+
for target, content in copies.items():
135203
target.parent.mkdir(parents=True, exist_ok=True)
136204
target.write_bytes(content)
137205
current[str(target.relative_to(public))] = sha256(content).hexdigest()

‎hack/test_docs_seo.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,13 @@ def __init__(self, text):
2424
self.languages = {}
2525
self.og_urls = []
2626
self.formats = {}
27+
self.ids = set()
2728
self.feed(text)
2829

2930
def handle_starttag(self, tag, attrs):
3031
attrs = dict(attrs)
32+
if attrs.get("id"):
33+
self.ids.add(attrs["id"])
3134
if tag == "link" and attrs.get("rel") == "canonical":
3235
self.canonical.append(attrs.get("href"))
3336
if tag == "meta" and attrs.get("name", "").lower() == "robots":

‎hack/test_docs_seo_fixtures.py‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ def setUp(self):
5555
self.write("content/en/docs/v1.6/removed.md", '---\ntitle: Removed\n---\nOld page')
5656
self.write("content/en/private.md", '---\ntitle: Private\nparams:\n noindex: true\n---\nPrivate')
5757
self.write("content/en/print.md", '---\ntitle: Print\noutputs: [HTML, PRINT]\n---\nPrint')
58+
self.write("data/docs-legacy-urls.json", "{}")
5859
self.configure()
5960

6061
def write(self, name, text):
@@ -371,6 +372,164 @@ def log_message(self, *args):
371372
thread.join()
372373
server.server_close()
373374

375+
def legacy_site(self):
376+
self.write("data/docs-legacy-urls.json", (REPO / "data/docs-legacy-urls.json").read_text())
377+
for version in ("v1.6", "v1.7"):
378+
for path, name in (("applications/_index.md", "Applications"),
379+
("applications/mariadb.md", "MariaDB"),
380+
("install/talos/boot-to-talos.md", "Boot to Talos"),
381+
("roadmap.md", "Roadmap"), ("storage/_index.md", "Storage")):
382+
self.write(f"content/en/docs/{version}/{path}", f'---\ntitle: {name}\n---\n{name} {version}\n\n## Backups\n')
383+
self.write("content/en/docs/v0/operations/configuration/bundles.md",
384+
'---\ntitle: Archived bundles\n---\nArchived bundles\n\n## distro-hosted\n')
385+
return {
386+
"/docs/reference/applications/": "/docs/applications/",
387+
"/docs/reference/applications/mysql/": "/docs/applications/mariadb/",
388+
"/docs/install/talos/kexec/": "/docs/install/talos/boot-to-talos/",
389+
"/roadmap/": "/docs/roadmap/",
390+
"/docs/bundles/distro-hosted/": None,
391+
"/docs/operations/storage/linstor/": "/docs/storage/",
392+
"/docs/v1/reference/applications/": "/docs/applications/",
393+
"/docs/v1/reference/applications/mysql/": "/docs/applications/mariadb/",
394+
"/docs/v1/install/talos/kexec/": "/docs/install/talos/boot-to-talos/",
395+
"/docs/v1/operations/storage/linstor/": "/docs/storage/",
396+
}
397+
398+
def test_legacy_urls_serve_current_and_archived_content(self):
399+
expected = self.legacy_site()
400+
self.build()
401+
for old, target in expected.items():
402+
with self.subTest(url=old):
403+
page = self.head(old.lstrip("/") + "index.html")
404+
self.assertFalse(page.refresh)
405+
if target:
406+
self.assertEqual(page.canonical, ["https://example.org" + target])
407+
self.assertEqual(page.og_urls, page.canonical)
408+
self.assertEqual(page.robots, ["index, follow"])
409+
self.assertIn("backups", page.ids)
410+
else:
411+
self.assertEqual(page.canonical, [])
412+
self.assertEqual(page.robots, ["noindex, follow"])
413+
self.assertIn("distro-hosted", page.ids)
414+
self.configure("v1.7")
415+
self.build()
416+
for old, target in expected.items():
417+
html = (self.public / old.lstrip("/") / "index.html").read_text()
418+
self.assertIn("v1.7" if target else "Archived bundles", html)
419+
archive = (self.public / "docs/v1.6/applications/mariadb/index.html").read_bytes()
420+
self.mirror()
421+
self.assertEqual((self.public / "docs/v1.6/applications/mariadb/index.html").read_bytes(), archive)
422+
423+
def test_legacy_urls_preserve_queries_and_fragments_without_javascript(self):
424+
expected = self.legacy_site()
425+
self.configure(base="https://example.org/project/")
426+
self.build()
427+
class QuietHandler(SimpleHTTPRequestHandler):
428+
def log_message(self, *args):
429+
pass
430+
server = ThreadingHTTPServer(("127.0.0.1", 0), functools.partial(QuietHandler, directory=self.public))
431+
thread = threading.Thread(target=server.serve_forever, daemon=True)
432+
thread.start()
433+
try:
434+
for old, target in expected.items():
435+
fragment = "backups" if target else "distro-hosted"
436+
url = f"http://127.0.0.1:{server.server_port}{old}?source=test#{fragment}"
437+
with urlopen(url) as response:
438+
page = Head(response.read().decode())
439+
self.assertEqual(response.status, 200)
440+
self.assertEqual(response.geturl(), url)
441+
self.assertFalse(page.refresh)
442+
self.assertIn(fragment, page.ids)
443+
if target:
444+
self.assertEqual(page.canonical, ["https://example.org/project" + target])
445+
finally:
446+
server.shutdown()
447+
thread.join()
448+
server.server_close()
449+
450+
def test_invalid_legacy_map_preserves_previous_output(self):
451+
expected = self.legacy_site()
452+
self.build()
453+
before = {old: (self.public / old.lstrip("/") / "index.html").read_bytes() for old in expected}
454+
source = self.public / "docs/v1.6/applications/index.html"
455+
source.write_text(source.read_text().replace("Applications v1.6", "Changed applications"))
456+
self.write("public/docs/blocked", "existing file")
457+
for old, target in (("/docs/unknown/", "/docs/missing/"),
458+
("/docs/v0/overwrite/", "/docs/storage/"),
459+
("/docs/../outside/", "/docs/storage/"),
460+
("/docs/unknown/", "https://other.example/"),
461+
("/docs/blocked/guide/", "/docs/storage/"),
462+
("/docs/applications/", "/docs/storage/")):
463+
with self.subTest(old=old, target=target):
464+
mapping = {"/docs/reference/applications/": "/docs/applications/", old: target}
465+
self.write("data/docs-legacy-urls.json", json.dumps(mapping))
466+
self.mirror(success=False)
467+
for path, content in before.items():
468+
self.assertEqual((self.public / path.lstrip("/") / "index.html").read_bytes(), content)
469+
for mapping in ([], None, {"/docs/unknown/": None},
470+
{"/docs/unknown/": "/docs/storage/?x=one"},
471+
{"/docs/%2e%2e/outside/": "/docs/storage/"}):
472+
self.write("data/docs-legacy-urls.json", json.dumps(mapping))
473+
self.mirror(success=False)
474+
for path, content in before.items():
475+
self.assertEqual((self.public / path.lstrip("/") / "index.html").read_bytes(), content)
476+
477+
def test_legacy_map_refuses_symlinks(self):
478+
self.legacy_site()
479+
self.build()
480+
archive = self.public / "docs/v0/guide/index.html"
481+
original = archive.read_bytes()
482+
old = self.public / "docs/reference/applications/mysql/index.html"
483+
old.unlink()
484+
old.symlink_to(archive)
485+
self.mirror(success=False)
486+
self.assertTrue(old.is_symlink())
487+
self.assertEqual(archive.read_bytes(), original)
488+
489+
def test_legacy_map_refuses_archive_source_symlink_and_alias_target(self):
490+
self.legacy_site()
491+
self.write("content/en/docs/v1.6/guide.md", '---\ntitle: Guide\naliases: [/docs/v1.6/alias-guide/]\n---\nGuide')
492+
self.build()
493+
self.write("data/docs-legacy-urls.json", json.dumps({"/docs/old-guide/": "/docs/alias-guide/"}))
494+
self.mirror(success=False)
495+
self.write("data/docs-legacy-urls.json", (REPO / "data/docs-legacy-urls.json").read_text())
496+
source = self.public / "docs/v0/operations/configuration/bundles/index.html"
497+
source.unlink()
498+
source.symlink_to(self.public / "docs/v0/guide/index.html")
499+
self.mirror(success=False)
500+
self.assertTrue(source.is_symlink())
501+
502+
def test_legacy_map_preserves_real_roadmap_page(self):
503+
self.legacy_site()
504+
self.write("content/en/roadmap.md", '---\ntitle: Standalone roadmap\n---\nStandalone roadmap')
505+
result = subprocess.run([HUGO, "--minify"], cwd=self.root, capture_output=True, text=True)
506+
self.assertEqual(result.returncode, 0, result.stderr)
507+
before = (self.public / "roadmap/index.html").read_bytes()
508+
self.mirror(success=False)
509+
self.assertEqual((self.public / "roadmap/index.html").read_bytes(), before)
510+
511+
def test_legacy_preview_has_no_indexable_copies(self):
512+
expected = self.legacy_site()
513+
self.build("preview")
514+
for old in expected:
515+
page = self.head(old.lstrip("/") + "index.html")
516+
self.assertEqual(page.robots, ["noindex, nofollow"])
517+
self.assertEqual(page.canonical, [])
518+
self.assertEqual(self.locations(), [])
519+
520+
def test_legacy_major_aliases_are_cleaned_on_major_rollover(self):
521+
self.legacy_site()
522+
self.build()
523+
source = self.root / "content/en/docs/v1.7"
524+
for file in source.rglob("*.md"):
525+
self.write("content/en/docs/v2.0/" + str(file.relative_to(source)),
526+
file.read_text().replace("v1.7", "v2.0"))
527+
self.configure("v2.0")
528+
self.build()
529+
self.assertFalse((self.public / "docs/v1/reference/applications/mysql/index.html").exists())
530+
self.assertIn("v2.0", (self.public / "docs/v2/reference/applications/mysql/index.html").read_text())
531+
self.assertTrue((self.public / "docs/v1.6/applications/mariadb/index.html").is_file())
532+
374533

375534
if __name__ == "__main__":
376535
unittest.main()

0 commit comments

Comments
 (0)