|
1 | 1 | # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
2 | 2 | # SPDX-License-Identifier: Apache-2.0 |
3 | 3 |
|
4 | | -"""Retry complete lychee passes while retaining successful checks in its cache.""" |
| 4 | +"""Retry transient lychee failures while retaining successful checks in its cache.""" |
5 | 5 |
|
6 | 6 | from __future__ import annotations |
7 | 7 |
|
8 | 8 | import argparse |
| 9 | +import html |
| 10 | +import json |
9 | 11 | import os |
10 | 12 | import re |
11 | 13 | import shlex |
12 | 14 | import shutil |
13 | 15 | import subprocess |
14 | 16 | import time |
| 17 | +from dataclasses import dataclass |
15 | 18 | from pathlib import Path |
| 19 | +from urllib.parse import urlsplit |
16 | 20 |
|
17 | 21 | LINK_CHECK_FAILURE = 2 |
18 | 22 |
|
19 | 23 |
|
20 | | -def read_report(path: Path) -> str: |
21 | | - """Reject missing, malformed, or empty-check reports instead of accepting them.""" |
22 | | - text = path.read_text(encoding="utf-8") |
23 | | - total = re.search(r"^\|[^|\n]*\bTotal\s*\|\s*(\d+)\s*\|", text, re.MULTILINE) |
24 | | - if total is None: |
25 | | - raise ValueError(f"Lychee report has no Total count: {path}") |
26 | | - if int(total.group(1)) == 0: |
| 24 | +@dataclass(frozen=True) |
| 25 | +class Failure: |
| 26 | + source: str |
| 27 | + url: str |
| 28 | + text: str |
| 29 | + details: str | None |
| 30 | + code: int | None |
| 31 | + timed_out: bool |
| 32 | + |
| 33 | + @property |
| 34 | + def transient(self) -> bool: |
| 35 | + try: |
| 36 | + parsed = urlsplit(self.url) |
| 37 | + is_http = parsed.scheme in ("http", "https") and bool(parsed.hostname) |
| 38 | + except ValueError: |
| 39 | + return False |
| 40 | + return is_http and ( |
| 41 | + self.timed_out or self.code in (408, 429) or (self.code is not None and 500 <= self.code <= 599) |
| 42 | + ) |
| 43 | + |
| 44 | + |
| 45 | +@dataclass(frozen=True) |
| 46 | +class Report: |
| 47 | + total: int |
| 48 | + errors: int |
| 49 | + timeouts: int |
| 50 | + failures: tuple[Failure, ...] |
| 51 | + |
| 52 | + @property |
| 53 | + def transient(self) -> bool: |
| 54 | + return bool(self.failures) and all(failure.transient for failure in self.failures) |
| 55 | + |
| 56 | + |
| 57 | +def read_report(path: Path, exit_code: int) -> Report: |
| 58 | + """Validate the pinned lychee JSON schema and its agreement with the exit code.""" |
| 59 | + data = json.loads(path.read_text(encoding="utf-8")) |
| 60 | + if not isinstance(data, dict): |
| 61 | + raise ValueError(f"Lychee report must be a JSON object: {path}") |
| 62 | + counts = {} |
| 63 | + for name in ("total", "errors", "timeouts"): |
| 64 | + value = data.get(name) |
| 65 | + if type(value) is not int or value < 0: |
| 66 | + raise ValueError(f"Lychee report {name} must be a nonnegative integer: {path}") |
| 67 | + counts[name] = value |
| 68 | + if counts["total"] == 0: |
27 | 69 | raise ValueError(f"Lychee checked no links: {path}") |
28 | | - return text |
| 70 | + if counts["errors"] + counts["timeouts"] > counts["total"]: |
| 71 | + raise ValueError(f"Lychee failure counts exceed the total: {path}") |
| 72 | + |
| 73 | + failures = [] |
| 74 | + for name, timed_out, count in (("error_map", False, counts["errors"]), ("timeout_map", True, counts["timeouts"])): |
| 75 | + mapping = data.get(name) |
| 76 | + if not isinstance(mapping, dict): |
| 77 | + raise ValueError(f"Lychee report {name} must be an object: {path}") |
| 78 | + entries = [] |
| 79 | + for source, responses in mapping.items(): |
| 80 | + if not isinstance(source, str) or not source or not isinstance(responses, list): |
| 81 | + raise ValueError(f"Lychee report {name} must map source names to response lists: {path}") |
| 82 | + for response in responses: |
| 83 | + if not isinstance(response, dict) or not isinstance(response.get("url"), str) or not response["url"]: |
| 84 | + raise ValueError(f"Lychee report has a malformed failure URL: {path}") |
| 85 | + status = response.get("status") |
| 86 | + if not isinstance(status, dict) or not isinstance(status.get("text"), str) or not status["text"]: |
| 87 | + raise ValueError(f"Lychee report has a malformed failure status: {path}") |
| 88 | + code = status.get("code") |
| 89 | + details = status.get("details") |
| 90 | + if "code" in status: |
| 91 | + if type(code) is not int or not 100 <= code <= 999: |
| 92 | + raise ValueError(f"Lychee report has a malformed HTTP status code: {path}") |
| 93 | + elif not isinstance(details, str): |
| 94 | + raise ValueError(f"Lychee report non-HTTP status must contain string details: {path}") |
| 95 | + if "details" in status and not isinstance(details, str): |
| 96 | + raise ValueError(f"Lychee report status details must be a string: {path}") |
| 97 | + entries.append(Failure(source, response["url"], status["text"], details, code, timed_out)) |
| 98 | + # Counters count occurrences; response maps deduplicate equivalent checks. |
| 99 | + if bool(count) != bool(entries): |
| 100 | + raise ValueError(f"Lychee report {name} disagrees with its failure count: {path}") |
| 101 | + failures.extend(entries) |
| 102 | + |
| 103 | + if (exit_code == 0) != (not failures): |
| 104 | + raise ValueError(f"Lychee report failures disagree with exit code {exit_code}: {path}") |
| 105 | + return Report(counts["total"], counts["errors"], counts["timeouts"], tuple(failures)) |
| 106 | + |
| 107 | + |
| 108 | +def escape_markdown(text: str) -> str: |
| 109 | + text = html.escape(text).replace("\r", " ").replace("\n", " ") |
| 110 | + return re.sub(r"([\\`*_~\[\]|])", r"\\\1", text) |
| 111 | + |
| 112 | + |
| 113 | +def render_report(report: Report) -> str: |
| 114 | + result = ( |
| 115 | + "| Status | Count |\n| --- | --- |\n" |
| 116 | + f"| Total | {report.total} |\n| Errors | {report.errors} |\n| Timeouts | {report.timeouts} |\n" |
| 117 | + ) |
| 118 | + if report.failures: |
| 119 | + result += "\n| Source | URL | Status | Details |\n| --- | --- | --- | --- |\n" |
| 120 | + for failure in report.failures: |
| 121 | + cells = (failure.source, failure.url, failure.text, failure.details or "") |
| 122 | + result += "| " + " | ".join(escape_markdown(cell) for cell in cells) + " |\n" |
| 123 | + return result |
29 | 124 |
|
30 | 125 |
|
31 | 126 | def attempt_report(report: Path, attempt: int) -> Path: |
32 | 127 | return report.with_name(f"{report.stem}-attempt-{attempt}{report.suffix}") |
33 | 128 |
|
34 | 129 |
|
35 | | -def finish(report_text: str | None, exit_codes: list[int], message: str) -> None: |
| 130 | +def finish(report: Path, data: Report | None, exit_codes: list[int], message: str) -> None: |
36 | 131 | """Publish the final result without presenting recovered failures as current.""" |
37 | 132 | summary = f"## Documentation links\n\n{message}\n\n| Attempt | Exit code |\n| --- | --- |\n" |
38 | 133 | summary += "".join(f"| {attempt} | {code} |\n" for attempt, code in enumerate(exit_codes, start=1)) |
39 | | - if report_text is not None: |
40 | | - summary += f"\n### Final attempt\n\n{report_text}\n" |
| 134 | + if data is not None: |
| 135 | + summary += f"\n### Final attempt\n\n{render_report(data)}\n" |
41 | 136 | else: |
42 | 137 | summary += ( |
43 | 138 | "\nThe final attempt did not produce a link-check report. Earlier reports are retained as artifacts.\n" |
44 | 139 | ) |
| 140 | + report.with_suffix(".md").write_text(summary, encoding="utf-8") |
45 | 141 | print(message, flush=True) |
46 | 142 | if summary_path := os.environ.get("GITHUB_STEP_SUMMARY"): |
47 | 143 | with Path(summary_path).open("a", encoding="utf-8") as stream: |
48 | 144 | stream.write(summary) |
49 | 145 |
|
50 | 146 |
|
51 | 147 | def retry_lychee(initial_exit_code: int, report: Path, max_attempts: int = 10) -> int: |
52 | | - """Count the action's first pass toward the limit, and retry only link failures.""" |
| 148 | + """Retry only if every remaining failure is a recognized transient HTTP error.""" |
53 | 149 | if initial_exit_code not in (0, 1, 2, 3): |
54 | 150 | raise ValueError(f"Unexpected initial lychee exit code: {initial_exit_code}") |
55 | 151 | if max_attempts < 1: |
56 | 152 | raise ValueError("max_attempts must be positive") |
57 | 153 | exit_codes = [initial_exit_code] |
58 | 154 | if initial_exit_code not in (0, LINK_CHECK_FAILURE): |
59 | | - finish(None, exit_codes, f"Lychee stopped on attempt 1 with exit code {initial_exit_code}; no retry.") |
| 155 | + finish(report, None, exit_codes, f"Lychee stopped on attempt 1 with exit code {initial_exit_code}; no retry.") |
60 | 156 | return initial_exit_code |
61 | 157 |
|
62 | | - report_text = read_report(report) |
| 158 | + data = read_report(report, initial_exit_code) |
63 | 159 | shutil.copyfile(report, attempt_report(report, 1)) |
64 | | - if initial_exit_code == 0: |
65 | | - finish(report_text, exit_codes, "Lychee passed on attempt 1.") |
66 | | - return 0 |
67 | | - if max_attempts == 1: |
68 | | - finish(report_text, exit_codes, "Lychee still has broken links after 1 attempt.") |
69 | | - return LINK_CHECK_FAILURE |
70 | | - |
71 | | - raw_args = os.environ.get("LYCHEE_ARGS") |
72 | | - if not raw_args or not (lychee_args := shlex.split(raw_args)): |
73 | | - raise ValueError("LYCHEE_ARGS must contain the same arguments used by the first lychee pass") |
74 | | - |
75 | | - for attempt in range(2, max_attempts + 1): |
76 | | - cooldown = min(60 * (attempt - 1), 120) |
| 160 | + lychee_args = None |
| 161 | + while True: |
| 162 | + attempt = len(exit_codes) |
| 163 | + if exit_codes[-1] == 0: |
| 164 | + finish(report, data, exit_codes, f"Lychee passed on attempt {attempt}/{max_attempts}.") |
| 165 | + return 0 |
| 166 | + if not data.transient: |
| 167 | + finish( |
| 168 | + report, |
| 169 | + data, |
| 170 | + exit_codes, |
| 171 | + f"Lychee has permanent or unclassified link failures on attempt {attempt}; no retry.", |
| 172 | + ) |
| 173 | + return LINK_CHECK_FAILURE |
| 174 | + if attempt >= max_attempts: |
| 175 | + finish(report, data, exit_codes, f"Lychee still has transient link failures after {max_attempts} attempts.") |
| 176 | + return LINK_CHECK_FAILURE |
| 177 | + if lychee_args is None: |
| 178 | + raw_args = os.environ.get("LYCHEE_ARGS") |
| 179 | + if not raw_args or not (lychee_args := shlex.split(raw_args)): |
| 180 | + raise ValueError("LYCHEE_ARGS must contain the same arguments used by the first lychee pass") |
| 181 | + |
| 182 | + cooldown = min(60 * attempt, 120) |
77 | 183 | print( |
78 | | - f"Lychee attempt {attempt - 1} failed; waiting {cooldown}s before attempt {attempt}/{max_attempts}.", |
| 184 | + f"Lychee attempt {attempt} had transient failures; waiting {cooldown}s before attempt {attempt + 1}/{max_attempts}.", |
79 | 185 | flush=True, |
80 | 186 | ) |
81 | 187 | time.sleep(cooldown) |
82 | | - current_report = attempt_report(report, attempt) |
| 188 | + current_report = attempt_report(report, attempt + 1) |
83 | 189 | result = subprocess.run( # noqa: S603 |
84 | | - ["lychee", *lychee_args, "--mode", "task", "--format", "markdown", "--output", str(current_report)], # noqa: S607 |
| 190 | + ["lychee", *lychee_args, "--mode", "task", "--format", "json", "--output", str(current_report)], # noqa: S607 |
85 | 191 | check=False, |
86 | 192 | ) |
87 | 193 | exit_codes.append(result.returncode) |
88 | 194 | if result.returncode not in (0, LINK_CHECK_FAILURE): |
89 | 195 | finish( |
90 | | - None, exit_codes, f"Lychee stopped on attempt {attempt} with exit code {result.returncode}; no retry." |
| 196 | + report, |
| 197 | + None, |
| 198 | + exit_codes, |
| 199 | + f"Lychee stopped on attempt {attempt + 1} with exit code {result.returncode}; no retry.", |
91 | 200 | ) |
92 | 201 | return result.returncode if result.returncode > 0 else 1 |
93 | | - report_text = read_report(current_report) |
| 202 | + data = read_report(current_report, result.returncode) |
94 | 203 | shutil.copyfile(current_report, report) |
95 | | - if result.returncode == 0: |
96 | | - finish( |
97 | | - report_text, |
98 | | - exit_codes, |
99 | | - f"Lychee passed on attempt {attempt}/{max_attempts}; successful checks were retained between passes.", |
100 | | - ) |
101 | | - return 0 |
102 | | - |
103 | | - finish(report_text, exit_codes, f"Lychee still has broken links after {max_attempts} attempts.") |
104 | | - return LINK_CHECK_FAILURE |
105 | 204 |
|
106 | 205 |
|
107 | 206 | def main() -> int: |
|
0 commit comments