Skip to content

Commit 9205406

Browse files
committed
ci: share native Lychee hooks and retry reporting through pre-commit [skip ci]
Use the configured native hooks for authored and rendered checks, collecting strict native reports across pre-commit chunks and applying the existing retry policies. Signed-off-by: Ralf W. Grosse-Kunstleve <[email protected]>
1 parent fad2ab1 commit 9205406

10 files changed

Lines changed: 701 additions & 66 deletions

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ __pycache__/
1111
.cython-stdlib
1212
.cython-bindings
1313
.lycheecache
14+
.lychee-rendered/
1415
.pytest_cache/
1516
.benchmarks/
1617
*.cpp

‎.pre-commit-config.yaml‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,18 +111,39 @@ repos:
111111
name: lychee
112112
entry: python ./ci/tools/run_lychee.py
113113
language: python
114-
additional_dependencies:
115-
- lychee-bin==0.24.2 # Must match check-doc-links/action.yml
114+
additional_dependencies: &lychee_dependencies
115+
- lychee-bin==0.24.2
116116
types: [text]
117117
require_serial: true
118118
args:
119119
- --cache
120+
- --max-cache-age=1d
120121
- --max-concurrency=4
121122
- --max-retries=3
122123
- --no-progress
123124
files: '\.(md|rst)$'
124125
exclude: ^qa/
125126

127+
# Generated HTML is checked explicitly after the documentation build.
128+
- id: lychee-rendered
129+
name: lychee (rendered documentation)
130+
entry: python ./ci/tools/run_lychee.py --rendered
131+
language: python
132+
additional_dependencies: *lychee_dependencies
133+
stages: [manual]
134+
pass_filenames: false
135+
always_run: true
136+
args:
137+
- --cache
138+
- --max-cache-age=1d
139+
- --max-concurrency=16
140+
- --host-concurrency=2
141+
- --host-request-interval=250ms
142+
- --max-retries=3
143+
- --retry-wait-time=5
144+
- --timeout=30
145+
- --no-progress
146+
126147
# Standard hooks
127148
- repo: https://github.com/pre-commit/pre-commit-hooks
128149
rev: "3e8a8703264a2f4a69428a0aa4dcb512790b2c8c" # frozen: v6.0.0

‎ci/tools/lychee_cache.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Describe the URL-result caches shared by pre-commit and documentation CI."""
5+
6+
from __future__ import annotations
7+
8+
import argparse
9+
import hashlib
10+
import os
11+
from pathlib import Path
12+
13+
ROOT = Path(__file__).resolve().parents[2]
14+
POLICY_FILES = (
15+
".pre-commit-config.yaml",
16+
"lychee.toml",
17+
"ci/tools/run_lychee.py",
18+
"ci/tools/run_lychee_precommit.py",
19+
"ci/tools/retry_lychee.py",
20+
"ci/tools/prepare_lychee_inputs.py",
21+
"ci/tools/lychee_cache.py",
22+
)
23+
24+
25+
def cache_prefix(kind: str, root: Path = ROOT) -> str:
26+
# The hook configuration includes the native wheel pin. Separate kinds
27+
# prevent authored successes from bypassing rendered fragment checks.
28+
digest = hashlib.sha256()
29+
for relative in POLICY_FILES:
30+
digest.update(relative.encode("utf-8") + b"\0")
31+
digest.update((root / relative).read_bytes() + b"\0")
32+
return f"lychee-precommit-v1-{kind}-{digest.hexdigest()[:24]}-baseline-"
33+
34+
35+
def main() -> None:
36+
parser = argparse.ArgumentParser(description=__doc__)
37+
parser.add_argument("kind", choices=("authored", "rendered"))
38+
parser.add_argument("--github-output", type=Path, required=True)
39+
args = parser.parse_args()
40+
prefix = cache_prefix(args.kind)
41+
cache_path = (
42+
Path(".lycheecache")
43+
if args.kind == "authored"
44+
else Path(os.environ.get("LYCHEE_CACHE_DIR", str(ROOT / ".lychee-rendered"))) / ".lycheecache"
45+
)
46+
outputs = {
47+
"prefix": prefix,
48+
"key": f"{prefix}{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}",
49+
"path": str(cache_path),
50+
}
51+
with args.github_output.open("a", encoding="utf-8") as stream:
52+
for name, value in outputs.items():
53+
if "\n" in value or "\r" in value:
54+
raise ValueError(f"Cache output {name} contains a newline")
55+
stream.write(f"{name}={value}\n")
56+
57+
58+
if __name__ == "__main__":
59+
main()

‎ci/tools/retry_lychee.py‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import shutil
1515
import subprocess
1616
import time
17+
from collections.abc import Callable
1718
from dataclasses import dataclass
1819
from pathlib import Path
1920
from urllib.parse import urlsplit
@@ -62,7 +63,7 @@ def transient(self) -> bool:
6263
return bool(self.failures) and all(failure.transient for failure in self.failures)
6364

6465

65-
def read_report(path: Path, exit_code: int) -> Report:
66+
def read_report(path: Path, exit_code: int, *, allow_empty: bool = False) -> Report:
6667
"""Validate the pinned lychee JSON schema and its agreement with the exit code."""
6768
data = json.loads(path.read_text(encoding="utf-8"))
6869
if not isinstance(data, dict):
@@ -73,7 +74,7 @@ def read_report(path: Path, exit_code: int) -> Report:
7374
if type(value) is not int or value < 0:
7475
raise ValueError(f"Lychee report {name} must be a nonnegative integer: {path}")
7576
counts[name] = value
76-
if counts["total"] == 0:
77+
if counts["total"] == 0 and not allow_empty:
7778
raise ValueError(f"Lychee checked no links: {path}")
7879
if counts["errors"] + counts["timeouts"] > counts["total"]:
7980
raise ValueError(f"Lychee failure counts exceed the total: {path}")
@@ -173,7 +174,14 @@ def finish_link_failures(report: Path, data: Report, exit_codes: list[int], mess
173174
return 0 if warning else LINK_CHECK_FAILURE
174175

175176

176-
def retry_lychee(initial_exit_code: int, report: Path, max_attempts: int = 3, *, policy: str = "check") -> int:
177+
def retry_lychee(
178+
initial_exit_code: int,
179+
report: Path,
180+
max_attempts: int = 3,
181+
*,
182+
policy: str = "check",
183+
run_attempt: Callable[[Path], int] | None = None,
184+
) -> int:
177185
"""Retry only if every remaining failure is a recognized transient HTTP error."""
178186
if initial_exit_code not in (0, 1, 2, 3):
179187
raise ValueError(f"Unexpected initial lychee exit code: {initial_exit_code}")
@@ -210,7 +218,7 @@ def retry_lychee(initial_exit_code: int, report: Path, max_attempts: int = 3, *,
210218
f"Lychee still has transient link failures after {max_attempts} attempts.",
211219
policy,
212220
)
213-
if lychee_args is None:
221+
if run_attempt is None and lychee_args is None:
214222
raw_args = os.environ.get("LYCHEE_ARGS")
215223
if not raw_args or not (lychee_args := shlex.split(raw_args)):
216224
raise ValueError("LYCHEE_ARGS must contain the same arguments used by the first lychee pass")
@@ -222,20 +230,23 @@ def retry_lychee(initial_exit_code: int, report: Path, max_attempts: int = 3, *,
222230
)
223231
time.sleep(cooldown)
224232
current_report = attempt_report(report, attempt + 1)
225-
result = subprocess.run( # noqa: S603
226-
["lychee", *lychee_args, "--mode", "task", "--format", "json", "--output", str(current_report)], # noqa: S607
227-
check=False,
228-
)
229-
exit_codes.append(result.returncode)
230-
if result.returncode not in (0, LINK_CHECK_FAILURE):
233+
if run_attempt is None:
234+
code = subprocess.run( # noqa: S603
235+
["lychee", *lychee_args, "--mode", "task", "--format", "json", "--output", str(current_report)], # noqa: S607
236+
check=False,
237+
).returncode
238+
else:
239+
code = run_attempt(current_report)
240+
exit_codes.append(code)
241+
if code not in (0, LINK_CHECK_FAILURE):
231242
finish(
232243
report,
233244
None,
234245
exit_codes,
235-
f"Lychee stopped on attempt {attempt + 1} with exit code {result.returncode}; no retry.",
246+
f"Lychee stopped on attempt {attempt + 1} with exit code {code}; no retry.",
236247
)
237-
return result.returncode if result.returncode > 0 else 1
238-
data = read_report(current_report, result.returncode)
248+
return code if code > 0 else 1
249+
data = read_report(current_report, code)
239250
shutil.copyfile(current_report, report)
240251

241252

‎ci/tools/run_lychee.py‎

Lines changed: 55 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,41 @@
11
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
# SPDX-License-Identifier: Apache-2.0
33

4-
"""Run Lychee with this checkout's main-branch source links mapped to local paths."""
4+
"""Run the installed native Lychee for authored or rendered documentation."""
55

66
from __future__ import annotations
77

8+
import json
9+
import os
810
import subprocess
911
import sys
1012
import sysconfig
13+
import uuid
1114
from pathlib import Path
1215

16+
from prepare_lychee_inputs import rendered_inputs, write_inputs
17+
1318
ROOT = Path(__file__).resolve().parents[2]
1419

1520

16-
def repo_link_remaps(root: Path = ROOT) -> list[str]:
17-
"""Apply the same URL-prefix boundaries as the dedicated CI link check."""
18-
local_prefix = root.resolve().as_uri() + "/"
21+
def repo_link_remaps(root: Path = ROOT, *, rendered: bool = False) -> list[str]:
22+
"""Check proposed main sources locally, retaining GitHub views for HTML anchors."""
23+
if rendered:
24+
commit = subprocess.run( # noqa: S603
25+
["git", "-C", str(root), "rev-parse", "HEAD"], # noqa: S607
26+
check=True,
27+
capture_output=True,
28+
text=True,
29+
).stdout.strip()
30+
server = os.environ.get("GITHUB_SERVER_URL", "https://github.com").rstrip("/")
31+
repository = os.environ.get("GITHUB_REPOSITORY", "NVIDIA/cuda-python")
32+
prefix = f"{server}/{repository}"
33+
else:
34+
prefix = root.resolve().as_uri()
1935
arguments = []
2036
for view in ("blob", "tree"):
21-
arguments.extend(["--remap", f"^https://github[.]com/NVIDIA/cuda-python/{view}/main/ {local_prefix}"])
37+
replacement = f"{prefix}/{view}/{commit}/" if rendered else f"{prefix}/"
38+
arguments.extend(["--remap", f"^https://github[.]com/NVIDIA/cuda-python/{view}/main/ {replacement}"])
2239
return arguments
2340

2441

@@ -29,8 +46,39 @@ def main(arguments: list[str] | None = None) -> int:
2946
raise SystemExit(
3047
f"lychee was not found at {executable}; run this script through the configured pre-commit hook"
3148
)
32-
arguments = sys.argv[1:] if arguments is None else arguments
33-
return subprocess.run([str(executable), *repo_link_remaps(), *arguments], check=False).returncode # noqa: S603
49+
arguments = list(sys.argv[1:] if arguments is None else arguments)
50+
rendered = bool(arguments and arguments[0] == "--rendered")
51+
options = {}
52+
if rendered:
53+
arguments.pop(0)
54+
cache = Path(os.environ.get("LYCHEE_CACHE_DIR", ROOT / ".lychee-rendered")).resolve()
55+
cache.mkdir(parents=True, exist_ok=True)
56+
inputs = cache / "inputs.txt"
57+
write_inputs(rendered_inputs(ROOT / "artifacts/docs"), inputs)
58+
arguments = [
59+
"--files-from",
60+
str(inputs),
61+
"--include-fragments=full",
62+
"--config",
63+
str(ROOT / "lychee.toml"),
64+
*arguments,
65+
]
66+
options["cwd"] = cache
67+
68+
report = None
69+
if report_dir := os.environ.get("LYCHEE_REPORT_DIR"):
70+
directory = Path(report_dir).resolve()
71+
directory.mkdir(parents=True, exist_ok=True)
72+
# pre-commit may split a long file list into several native invocations.
73+
report = directory / f"{uuid.uuid4().hex}.json"
74+
arguments = ["--format", "json", "--output", str(report), *arguments]
75+
76+
code = subprocess.run( # noqa: S603
77+
[str(executable), *repo_link_remaps(ROOT, rendered=rendered), *arguments], check=False, **options
78+
).returncode
79+
if report is not None:
80+
report.with_suffix(".status.json").write_text(json.dumps({"exit_code": code}), encoding="utf-8")
81+
return code
3482

3583

3684
if __name__ == "__main__":

‎ci/tools/run_lychee_precommit.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Warm link caches or check rendered HTML through the configured pre-commit hooks."""
5+
6+
from __future__ import annotations
7+
8+
import argparse
9+
import json
10+
import os
11+
import subprocess
12+
import sys
13+
import uuid
14+
from pathlib import Path
15+
16+
from retry_lychee import LINK_CHECK_FAILURE, read_report, retry_lychee
17+
18+
ROOT = Path(__file__).resolve().parents[2]
19+
20+
21+
def collect_reports(directory: Path, output: Path, precommit_code: int) -> int:
22+
"""Distinguish native link failures from pre-commit setup or hook failures."""
23+
statuses = sorted(directory.glob("*.status.json"))
24+
reports = set(directory.glob("*.json")) - set(statuses)
25+
expected = {path.with_name(path.name.removesuffix(".status.json") + ".json") for path in statuses}
26+
if not statuses:
27+
raise ValueError("The configured Lychee hook produced no native status; pre-commit setup or execution failed")
28+
merged = {"total": 0, "errors": 0, "timeouts": 0, "error_map": {}, "timeout_map": {}}
29+
native_codes = []
30+
for status in statuses:
31+
metadata = json.loads(status.read_text(encoding="utf-8"))
32+
if not isinstance(metadata, dict) or type(metadata.get("exit_code")) is not int:
33+
raise ValueError(f"Invalid native Lychee status: {status}")
34+
code = metadata["exit_code"]
35+
if code not in (0, LINK_CHECK_FAILURE):
36+
raise ValueError(f"Native Lychee execution failed with exit code {code}: {status}")
37+
native_codes.append(code)
38+
report = status.with_name(status.name.removesuffix(".status.json") + ".json")
39+
# A pre-commit chunk can contain no links; the complete scan must not be empty.
40+
read_report(report, code, allow_empty=True)
41+
data = json.loads(report.read_text(encoding="utf-8"))
42+
for name in ("total", "errors", "timeouts"):
43+
merged[name] += data[name]
44+
for name in ("error_map", "timeout_map"):
45+
for source, responses in data[name].items():
46+
merged[name].setdefault(source, []).extend(responses)
47+
if reports != expected:
48+
raise ValueError("Native Lychee reports and completed invocation statuses do not match")
49+
code = LINK_CHECK_FAILURE if LINK_CHECK_FAILURE in native_codes else 0
50+
if precommit_code != int(code != 0):
51+
raise ValueError(f"pre-commit exit {precommit_code} disagrees with native Lychee exit {code}")
52+
output.write_text(json.dumps(merged), encoding="utf-8")
53+
read_report(output, code)
54+
return code
55+
56+
57+
def run_hook(kind: str, report: Path) -> int:
58+
"""Run the real hook, retaining every native chunk report for diagnostics."""
59+
directory = report.parent / f"{report.stem}-parts" / uuid.uuid4().hex
60+
directory.mkdir(parents=True)
61+
hook = "lychee" if kind == "authored" else "lychee-rendered"
62+
environment = os.environ.copy()
63+
environment["LYCHEE_REPORT_DIR"] = str(directory)
64+
environment["PYTHONUTF8"] = "1"
65+
environment["SKIP"] = ",".join(
66+
name.strip() for name in environment.get("SKIP", "").split(",") if name.strip() and name.strip() != hook
67+
)
68+
command = [sys.executable, "-m", "pre_commit", "run", hook, "--all-files", "--show-diff-on-failure"]
69+
if kind == "rendered":
70+
command.extend(["--hook-stage", "manual"])
71+
result = subprocess.run(command, cwd=ROOT, env=environment, check=False) # noqa: S603
72+
return collect_reports(directory, report, result.returncode)
73+
74+
75+
def run_checks(kind: str, report: Path, max_attempts: int = 3, *, policy: str = "check") -> int:
76+
if max_attempts < 1:
77+
raise ValueError("max-attempts must be positive")
78+
report = report.resolve()
79+
report.parent.mkdir(parents=True, exist_ok=True)
80+
return retry_lychee(
81+
run_hook(kind, report),
82+
report,
83+
max_attempts,
84+
policy=policy,
85+
run_attempt=lambda path: run_hook(kind, path),
86+
)
87+
88+
89+
def main(arguments: list[str] | None = None) -> int:
90+
parser = argparse.ArgumentParser(description=__doc__)
91+
parser.add_argument("kind", choices=("authored", "rendered"))
92+
parser.add_argument("--policy", choices=("cache-warm", "check"), default="check")
93+
parser.add_argument("--max-attempts", type=int, default=3)
94+
parser.add_argument("--report", type=Path, required=True)
95+
args = parser.parse_args(arguments)
96+
try:
97+
return run_checks(args.kind, args.report, args.max_attempts, policy=args.policy)
98+
except (OSError, ValueError) as error:
99+
print(f"::error::{error}", flush=True)
100+
return 1
101+
102+
103+
if __name__ == "__main__":
104+
raise SystemExit(main())

0 commit comments

Comments
 (0)