Created
October 28, 2025 12:41
-
-
Save maxrjones/a97c65facb2197376f49c6fe5ce2702e to your computer and use it in GitHub Desktop.
Script for checking object inventory consistency
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # /// script | |
| # dependencies = [ | |
| # "requests", | |
| # "pandas", | |
| # ] | |
| # /// | |
| """ | |
| Compare objects.inv from stable docs with local docs. | |
| This script: | |
| 1. Downloads and parses objects.inv from stable docs | |
| 2. Parses objects.inv from newly built local docs | |
| 3. Compares the two inventories to identify differences | |
| 4. Generates a comprehensive report | |
| Generated by Claude. | |
| One can also manually inspect differences using: | |
| `uv run --with sphinx python -m sphinx.ext.intersphinx "https://zarr.readthedocs.io/en/stable/objects.inv"` | |
| `uv run --with sphinx python -m sphinx.ext.intersphinx "site/objects.inv"` | |
| """ | |
| import zlib | |
| from io import BytesIO | |
| from pathlib import Path | |
| import pandas as pd | |
| import requests | |
| # Configuration | |
| STABLE_OBJECTS_INV_URL = "https://zarr.readthedocs.io/en/stable/objects.inv" | |
| LOCAL_OBJECTS_INV = Path("site/objects.inv") | |
| def parse_objects_inv(file_obj: bytes) -> pd.DataFrame: | |
| """ | |
| Parse a Sphinx objects.inv file and extract object information. | |
| Args: | |
| file_obj: File-like object containing objects.inv data | |
| Returns: | |
| DataFrame with columns: name, type, url | |
| """ | |
| # Read header (4 lines) | |
| file_obj.readline() # project line | |
| file_obj.readline() # version line | |
| file_obj.readline() # compression line | |
| file_obj.readline() # empty line | |
| # Decompress the rest | |
| compressed = file_obj.read() | |
| decompressed = zlib.decompress(compressed) | |
| # Parse entries | |
| data = [] | |
| lines = decompressed.decode("utf-8").split("\n") | |
| for line in lines: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| # Format: name domain:role priority uri dispname | |
| # Example: Array py:class 1 api/array.html#zarr.Array - | |
| parts = line.split(None, 4) | |
| if len(parts) >= 4: | |
| name = parts[0] | |
| obj_type = parts[1] # e.g., "py:class", "std:doc" | |
| uri = parts[3] | |
| # Handle special placeholder | |
| if uri == "-": | |
| continue | |
| # Store full URI (with anchor) and page-only URI | |
| uri_full = uri | |
| uri_page = uri.split("#")[0] if "#" in uri else uri | |
| data.append( | |
| {"name": name, "type": obj_type, "url_full": uri_full, "url_page": uri_page} | |
| ) | |
| return pd.DataFrame(data) | |
| def download_stable_inventory() -> pd.DataFrame: | |
| """Download and parse the stable docs objects.inv.""" | |
| try: | |
| # Use requests with User-Agent header | |
| response = requests.get( | |
| STABLE_OBJECTS_INV_URL, | |
| headers={"User-Agent": "Mozilla/5.0 (compatible; zarr-docs-validator/1.0)"}, | |
| timeout=30, | |
| ) | |
| response.raise_for_status() | |
| # Convert bytes to file-like object for parsing | |
| file_obj: bytes = BytesIO(response.content) | |
| return parse_objects_inv(file_obj) | |
| except Exception as e: | |
| print(f"Error downloading stable inventory: {e}") | |
| return pd.DataFrame() | |
| def load_local_inventory() -> pd.DataFrame: | |
| """Parse the locally built objects.inv.""" | |
| if not LOCAL_OBJECTS_INV.exists(): | |
| return pd.DataFrame() | |
| with open(LOCAL_OBJECTS_INV, "rb") as f: | |
| return parse_objects_inv(f) | |
| def compare_inventories( | |
| stable_inv: pd.DataFrame, | |
| local_inv: pd.DataFrame, | |
| ) -> pd.DataFrame: | |
| """ | |
| Compare stable and local inventories to find differences. | |
| Returns: | |
| DataFrame with comparison results | |
| """ | |
| # Prepare stable inventory for comparison | |
| stable_urls = stable_inv[["name", "type", "url_page"]].copy() | |
| stable_urls = stable_urls.rename(columns={"url_page": "stable_url"}) | |
| # Prepare local inventory for comparison | |
| local_urls = local_inv[["name", "type", "url_page"]].copy() | |
| local_urls = local_urls.rename(columns={"url_page": "local_url"}) | |
| # Full outer merge to find all entries | |
| return stable_urls.merge(local_urls, on=["name", "type"], how="outer", indicator=True) | |
| def main() -> None: | |
| stable_inv = download_stable_inventory() | |
| local_inv = load_local_inventory() | |
| compare_inventories(stable_inv, local_inv) | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment