Skip to content

Instantly share code, notes, and snippets.

@pawamoy
Created September 4, 2026 10:21
Show Gist options
  • Select an option

  • Save pawamoy/ecf3a8c9f22d538752989c907949a00d to your computer and use it in GitHub Desktop.

Select an option

Save pawamoy/ecf3a8c9f22d538752989c907949a00d to your computer and use it in GitHub Desktop.
objects.inv stats top 500 packages
#!/usr/bin/env python3
"""Survey Python-domain object types used by the top 500 PyPI projects.
``INVENTORIES`` is a reproducible snapshot of the trailing-30-day download
ranking from ``top-pypi-packages``. PyPI metadata does not provide an
inventory URL, so unavailable or unverified inventories are recorded with
``None`` and skipped.
Five sorted lists are written to stdout: each ``py:objtype`` with the number of
inventories containing it, each project and its unique Python object types,
each documentation builder with its unique invalid object types, and each valid
and invalid object type with its total number of inventory entries. Diagnostics
are written to stderr so that stdout can be redirected. Downloaded inventories
are cached in ``.invcache`` beside this script. Each ``INVENTORIES`` entry also
records the documentation builder when it could be identified. Object types
not listed in ``VALID_OBJECT_TYPES`` are marked as invalid in the output.
"""
from __future__ import annotations
import argparse
import sys
import time
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal
from urllib.request import Request, urlopen
try:
import sphobjinv
except ImportError:
sphobjinv = None # type: ignore[assignment]
DEFAULT_CACHE_DIR: Final = Path(__file__).resolve().with_name(".invcache")
RANKING_SOURCE: Final = (
"https://hugovk.dev/top-pypi-packages/top-pypi-packages.min.json"
)
RANKING_SNAPSHOT: Final = "2026-09-01 06:34:08"
DocumentationBuilder = Literal[
"astro", "mkdocs", "sphinx", "unknown", "zensical"
]
VALID_OBJECT_TYPES: Final[frozenset[str]] = frozenset(
{
"function",
"data",
"class",
"exception",
"method",
"classmethod",
"staticmethod",
"attribute",
"property",
"type",
"module",
"obj",
}
)
INVALID_STYLE: Final = "\033[1;31m"
RESET_STYLE: Final = "\033[0m"
# Projects are ordered by download rank. A ``None`` URL means no valid
# Sphinx-format inventory could be found at the time of the snapshot.
INVENTORIES: Final[
tuple[tuple[str, DocumentationBuilder | None, str | None], ...]
] = (
# PyPI download ranks 1-50.
(
"boto3",
"sphinx",
"https://boto3.amazonaws.com/v1/documentation/api/latest/objects.inv",
),
("packaging", "sphinx", "https://packaging.pypa.io/en/stable/objects.inv"),
(
"typing-extensions",
"sphinx",
"https://typing-extensions.readthedocs.io/en/stable/objects.inv",
),
("certifi", None, None),
("idna", None, None),
("urllib3", "sphinx", "https://urllib3.readthedocs.io/en/stable/objects.inv"),
("requests", "sphinx", "https://requests.readthedocs.io/en/latest/objects.inv"),
(
"charset-normalizer",
"sphinx",
"https://charset-normalizer.readthedocs.io/en/stable/objects.inv",
),
("setuptools", "sphinx", "https://setuptools.pypa.io/en/stable/objects.inv"),
("cryptography", "sphinx", "https://cryptography.io/en/latest/objects.inv"),
("cffi", "sphinx", "https://cffi.readthedocs.io/en/latest/objects.inv"),
("pluggy", "sphinx", "https://pluggy.readthedocs.io/en/stable/objects.inv"),
("pygments", "sphinx", "https://pygments.org/objects.inv"),
("pyyaml", None, None),
(
"botocore",
"sphinx",
"https://botocore.amazonaws.com/v1/documentation/api/latest/objects.inv",
),
(
"python-dateutil",
"sphinx",
"https://dateutil.readthedocs.io/en/stable/objects.inv",
),
("six", "sphinx", "https://six.readthedocs.io/objects.inv"),
("pydantic", "astro", "https://pydantic.dev/docs/validation/latest/objects.inv"),
("numpy", "sphinx", "https://numpy.org/doc/stable/objects.inv"),
("click", "sphinx", "https://click.palletsprojects.com/en/stable/objects.inv"),
("pycparser", None, None),
("anyio", "sphinx", "https://anyio.readthedocs.io/en/stable/objects.inv"),
("pytest", "sphinx", "https://docs.pytest.org/en/stable/objects.inv"),
("pydantic-core", None, None),
("iniconfig", None, None),
("aiobotocore", None, None),
("annotated-types", None, None),
("h11", "sphinx", "https://h11.readthedocs.io/en/latest/objects.inv"),
("attrs", "sphinx", "https://www.attrs.org/en/stable/objects.inv"),
(
"typing-inspection",
"mkdocs",
"https://pydantic.github.io/typing-inspection/dev/objects.inv",
),
("protobuf", None, None),
(
"fsspec",
"sphinx",
"https://filesystem-spec.readthedocs.io/en/latest/objects.inv",
),
("httpx", None, None),
(
"markupsafe",
"sphinx",
"https://markupsafe.palletsprojects.com/en/stable/objects.inv",
),
("httpcore", "mkdocs", "https://www.encode.io/httpcore/objects.inv"),
("s3transfer", None, None),
("python-dotenv", None, None),
(
"platformdirs",
"sphinx",
"https://platformdirs.readthedocs.io/en/stable/objects.inv",
),
("pandas", "sphinx", "https://pandas.pydata.org/docs/objects.inv"),
("jinja2", "sphinx", "https://jinja.palletsprojects.com/en/stable/objects.inv"),
(
"pathspec",
"sphinx",
"https://python-path-specification.readthedocs.io/en/latest/objects.inv",
),
("grpcio-status", "sphinx", "https://grpc.github.io/grpc/python/objects.inv"),
("filelock", "sphinx", "https://py-filelock.readthedocs.io/en/stable/objects.inv"),
("pip", "sphinx", "https://pip.pypa.io/en/stable/objects.inv"),
("pyjwt", None, None),
("starlette", "zensical", "https://starlette.dev/objects.inv"),
("uvicorn", None, None),
("litellm", None, None),
("aiohttp", "sphinx", "https://docs.aiohttp.org/en/stable/objects.inv"),
("tqdm", None, None),
# PyPI download ranks 51-100.
("jmespath", None, None),
("rpds-py", "sphinx", "https://rpds.readthedocs.io/en/stable/objects.inv"),
("yarl", "sphinx", "https://yarl.aio-libs.org/en/latest/objects.inv"),
(
"jsonschema",
"sphinx",
"https://python-jsonschema.readthedocs.io/en/stable/objects.inv",
),
("rich", "sphinx", "https://rich.readthedocs.io/en/stable/objects.inv"),
(
"markdown-it-py",
"sphinx",
"https://markdown-it-py.readthedocs.io/en/stable/objects.inv",
),
("multidict", "sphinx", "https://multidict.aio-libs.org/en/latest/objects.inv"),
("propcache", "sphinx", "https://propcache.readthedocs.io/en/latest/objects.inv"),
("s3fs", None, None),
("fastapi", "zensical", "https://fastapi.tiangolo.com/objects.inv"),
(
"referencing",
"sphinx",
"https://referencing.readthedocs.io/en/stable/objects.inv",
),
("tomlkit", None, None),
("frozenlist", None, None),
(
"jsonschema-specifications",
"sphinx",
"https://jsonschema-specifications.readthedocs.io/en/stable/objects.inv",
),
("pyasn1", "sphinx", "https://pyasn1.readthedocs.io/en/stable/objects.inv"),
("wheel", "sphinx", "https://wheel.readthedocs.io/en/stable/objects.inv"),
("websockets", "sphinx", "https://websockets.readthedocs.io/en/stable/objects.inv"),
(
"aiohappyeyeballs",
"sphinx",
"https://aiohappyeyeballs.readthedocs.io/en/stable/objects.inv",
),
("mdurl", None, None),
("aiosignal", None, None),
(
"google-auth",
"sphinx",
"https://googleapis.dev/python/google-auth/latest/objects.inv",
),
("pillow", "sphinx", "https://pillow.readthedocs.io/en/stable/objects.inv"),
("googleapis-common-protos", None, None),
("pytz", "sphinx", "https://pythonhosted.org/pytz/objects.inv"),
("sniffio", "sphinx", "https://sniffio.readthedocs.io/en/stable/objects.inv"),
("importlib-metadata", None, None),
("trove-classifiers", None, None),
("zipp", None, None),
("opentelemetry-semantic-conventions", None, None),
("annotated-doc", None, None),
("virtualenv", "sphinx", "https://virtualenv.pypa.io/en/latest/objects.inv"),
("hatchling", "mkdocs", "https://hatch.pypa.io/dev/objects.inv"),
("pydantic-settings", "astro", "https://docs.pydantic.dev/dev-v2/objects.inv"),
("tzdata", "sphinx", "https://tzdata.python.org/objects.inv"),
("wrapt", "sphinx", "https://wrapt.readthedocs.io/en/latest/objects.inv"),
("ghapi", None, None),
("greenlet", "sphinx", "https://greenlet.readthedocs.io/en/stable/objects.inv"),
("opentelemetry-api", None, None),
("huggingface-hub", None, None),
("opentelemetry-sdk", None, None),
("pyasn1-modules", None, None),
("tenacity", None, None),
(
"google-api-core",
"sphinx",
"https://googleapis.dev/python/google-api-core/latest/objects.inv",
),
("regex", None, None),
("pyarrow", "sphinx", "https://arrow.apache.org/docs/objects.inv"),
(
"python-multipart",
"mkdocs",
"https://kludex.github.io/python-multipart/objects.inv",
),
("textual", "mkdocs", "https://textual.textualize.io/objects.inv"),
("openai", None, None),
("grpcio", "sphinx", "https://grpc.github.io/grpc/python/objects.inv"),
("soupsieve", None, None),
# PyPI download ranks 101-150.
(
"beautifulsoup4",
"sphinx",
"https://www.crummy.com/software/BeautifulSoup/bs4/doc/objects.inv",
),
("jiter", None, None),
(
"pyparsing",
"sphinx",
"https://pyparsing-docs.readthedocs.io/en/latest/objects.inv",
),
("sqlalchemy", "sphinx", "https://docs.sqlalchemy.org/en/20/objects.inv"),
("distro", None, None),
("scipy", None, None),
("colorama", None, None),
("tomli", None, None),
("lxml", "sphinx", "https://lxml.de/apidoc/objects.inv"),
("watchfiles", "mkdocs", "https://watchfiles.helpmanual.io/objects.inv"),
("opentelemetry-proto", None, None),
("psutil", None, None),
("shellingham", None, None),
("opentelemetry-instrumentation", None, None),
("editables", "sphinx", "https://editables.readthedocs.io/en/stable/objects.inv"),
("typer", "zensical", "https://typer.tiangolo.com/objects.inv"),
("requests-oauthlib", None, None),
("opentelemetry-exporter-otlp-proto-http", None, None),
(
"proto-plus",
"sphinx",
"https://googleapis.dev/python/proto-plus/latest/objects.inv",
),
("openpyxl", "sphinx", "https://openpyxl.readthedocs.io/en/stable/objects.inv"),
(
"et-xmlfile",
"sphinx",
"https://openpyxl.pages.heptapod.net/et_xmlfile/objects.inv",
),
("mcp", "zensical", "https://py.sdk.modelcontextprotocol.io/objects.inv"),
("ruff", None, None),
("opentelemetry-exporter-otlp-proto-common", None, None),
("coverage", "sphinx", "https://coverage.readthedocs.io/en/latest/objects.inv"),
("exceptiongroup", None, None),
("sglang", None, None),
("oauthlib", None, None),
("mypy-extensions", None, None),
("dnspython", "sphinx", "https://dnspython.readthedocs.io/en/stable/objects.inv"),
(
"more-itertools",
"sphinx",
"https://more-itertools.readthedocs.io/en/stable/objects.inv",
),
("wcwidth", None, None),
("cachetools", "sphinx", "https://cachetools.readthedocs.io/en/stable/objects.inv"),
("distlib", "sphinx", "https://distlib.readthedocs.io/en/latest/objects.inv"),
("sse-starlette", None, None),
("hf-xet", None, None),
(
"multiprocess",
"sphinx",
"https://multiprocess.readthedocs.io/en/stable/objects.inv",
),
(
"pytest-asyncio",
"sphinx",
"https://pytest-asyncio.readthedocs.io/en/stable/objects.inv",
),
(
"python-discovery",
"sphinx",
"https://python-discovery.readthedocs.io/en/stable/objects.inv",
),
("docstring-parser", None, None),
(
"sortedcontainers",
"sphinx",
"https://grantjenks.com/docs/sortedcontainers/objects.inv",
),
(
"requests-toolbelt",
"sphinx",
"https://toolbelt.readthedocs.io/en/stable/objects.inv",
),
("pyopenssl", "sphinx", "https://www.pyopenssl.org/en/stable/objects.inv"),
("psycopg2-binary", "sphinx", "https://www.psycopg.org/docs/objects.inv"),
("rsa", None, None),
("uvloop", None, None),
(
"msgpack",
"sphinx",
"https://msgpack-python.readthedocs.io/en/stable/objects.inv",
),
("opentelemetry-exporter-otlp-proto-grpc", None, None),
("tiktoken", None, None),
("httptools", None, None),
# PyPI download ranks 151-200.
("google-genai", None, None),
("networkx", "sphinx", "https://networkx.org/documentation/stable/objects.inv"),
(
"werkzeug",
"sphinx",
"https://werkzeug.palletsprojects.com/en/stable/objects.inv",
),
("tokenizers", None, None),
("redis", "sphinx", "https://redis.readthedocs.io/en/stable/objects.inv"),
("gitpython", None, None),
("fonttools", None, None),
("langchain", None, None),
(
"prompt-toolkit",
"sphinx",
"https://python-prompt-toolkit.readthedocs.io/en/stable/objects.inv",
),
("defusedxml", None, None),
("grpcio-tools", None, None),
("scikit-learn", "sphinx", "https://scikit-learn.org/stable/objects.inv"),
("smmap", None, None),
("pytest-cov", "sphinx", "https://pytest-cov.readthedocs.io/en/stable/objects.inv"),
("google-cloud-storage", None, None),
("matplotlib", "sphinx", "https://matplotlib.org/stable/objects.inv"),
("orjson", None, None),
("email-validator", None, None),
("pynacl", "sphinx", "https://pynacl.readthedocs.io/en/latest/objects.inv"),
("httpx-sse", None, None),
("isodate", None, None),
("chardet", "sphinx", "https://chardet.readthedocs.io/en/stable/objects.inv"),
(
"azure-core",
"sphinx",
"https://azuresdkdocs.z19.web.core.windows.net/python/azure-core/latest/objects.inv",
),
("gitdb", None, None),
("tzlocal", None, None),
("alembic", "sphinx", "https://alembic.sqlalchemy.org/en/latest/objects.inv"),
(
"itsdangerous",
"sphinx",
"https://itsdangerous.palletsprojects.com/en/stable/objects.inv",
),
("dill", "sphinx", "https://dill.readthedocs.io/en/stable/objects.inv"),
(
"websocket-client",
"sphinx",
"https://websocket-client.readthedocs.io/en/latest/objects.inv",
),
("nodeenv", None, None),
("joblib", "sphinx", "https://joblib.readthedocs.io/en/stable/objects.inv"),
("tabulate", None, None),
("mako", "sphinx", "https://docs.makotemplates.org/objects.inv"),
("opentelemetry-instrumentation-requests", None, None),
("threadpoolctl", None, None),
("ruamel-yaml", None, None),
("h2", None, None),
(
"azure-identity",
"sphinx",
"https://azuresdkdocs.z19.web.core.windows.net/python/azure-identity/latest/objects.inv",
),
("keyring", None, None),
(
"google-cloud-core",
"sphinx",
"https://googleapis.dev/python/google-cloud-core/latest/objects.inv",
),
("mypy", "sphinx", "https://mypy.readthedocs.io/en/stable/objects.inv"),
("kiwisolver", "sphinx", "https://kiwisolver.readthedocs.io/en/latest/objects.inv"),
("bcrypt", None, None),
("hpack", None, None),
("pytest-json-ctrf", None, None),
("flask", "sphinx", "https://flask.palletsprojects.com/en/stable/objects.inv"),
("opentelemetry-util-http", None, None),
("contourpy", "sphinx", "https://contourpy.readthedocs.io/en/stable/objects.inv"),
("jaraco-classes", None, None),
("msal", "sphinx", "https://msal-python.readthedocs.io/en/stable/objects.inv"),
# PyPI download ranks 201-250.
("docutils", None, None),
("truststore", "sphinx", "https://truststore.readthedocs.io/en/stable/objects.inv"),
("hyperframe", None, None),
("jeepney", "sphinx", "https://jeepney.readthedocs.io/en/latest/objects.inv"),
("jaraco-functools", None, None),
("decorator", None, None),
(
"zstandard",
"sphinx",
"https://python-zstandard.readthedocs.io/en/latest/objects.inv",
),
("anthropic", None, None),
("pre-commit", None, None),
(
"secretstorage",
"sphinx",
"https://secretstorage.readthedocs.io/en/latest/objects.inv",
),
("identify", None, None),
("cfgv", None, None),
("toml", None, None),
("docker", "sphinx", "https://docker-py.readthedocs.io/en/stable/objects.inv"),
("xxhash", None, None),
("jaraco-context", None, None),
("blinker", "sphinx", "https://blinker.readthedocs.io/objects.inv"),
("uv", None, None),
("cycler", "sphinx", "https://matplotlib.org/cycler/objects.inv"),
(
"google-resumable-media",
"sphinx",
"https://googleapis.dev/python/google-resumable-media/latest/objects.inv",
),
("httpx2", None, None),
("ptyprocess", None, None),
("google-crc32c", None, None),
("sympy", "sphinx", "https://docs.sympy.org/latest/objects.inv"),
("transformers", None, None),
("httpcore2", None, None),
("pexpect", "sphinx", "https://pexpect.readthedocs.io/en/stable/objects.inv"),
("google-cloud-bigquery", None, None),
("google-api-python-client", None, None),
(
"uritemplate",
"sphinx",
"https://uritemplate.readthedocs.org/en/stable/objects.inv",
),
("black", "sphinx", "https://black.readthedocs.io/en/stable/objects.inv"),
("prometheus-client", None, None),
("jsonpointer", None, None),
(
"google-auth-httplib2",
"sphinx",
"https://googleapis.dev/python/google-auth-httplib2/latest/objects.inv",
),
("sentry-sdk", None, None),
("langchain-core", None, None),
("opentelemetry-instrumentation-threading", None, None),
("deprecated", "sphinx", "https://deprecated.readthedocs.io/en/latest/objects.inv"),
("backoff", None, None),
("snowflake-connector-python", None, None),
("async-timeout", None, None),
("google-auth-oauthlib", None, None),
("rapidfuzz", "sphinx", "https://rapidfuzz.github.io/RapidFuzz/objects.inv"),
("mpmath", "sphinx", "https://mpmath.org/doc/current/objects.inv"),
("asn1crypto", None, None),
("httplib2", None, None),
("opentelemetry-exporter-otlp", None, None),
("ipython", "sphinx", "https://ipython.readthedocs.io/en/stable/objects.inv"),
("awscli", "sphinx", "https://docs.aws.amazon.com/cli/latest/objects.inv"),
(
"pytest-xdist",
"sphinx",
"https://pytest-xdist.readthedocs.io/en/latest/objects.inv",
),
# PyPI download ranks 251-300.
("narwhals", "zensical", "https://narwhals-dev.github.io/narwhals/objects.inv"),
("librt", None, None),
("aiofiles", None, None),
("types-requests", None, None),
("msal-extensions", None, None),
("poetry-core", None, None),
("datasets", None, None),
("pypdf", "sphinx", "https://pypdf.readthedocs.io/en/latest/objects.inv"),
("execnet", "sphinx", "https://execnet.readthedocs.io/en/latest/objects.inv"),
("traitlets", "sphinx", "https://traitlets.readthedocs.io/en/stable/objects.inv"),
("cloudpickle", None, None),
("asgiref", "sphinx", "https://asgi.readthedocs.io/en/stable/objects.inv"),
("ydb", None, None),
("paramiko", "sphinx", "https://docs.paramiko.org/en/stable/objects.inv"),
(
"jsonpatch",
"sphinx",
"https://python-json-patch.readthedocs.io/en/stable/objects.inv",
),
("sqlparse", "sphinx", "https://sqlparse.readthedocs.io/en/stable/objects.inv"),
("google-cloud-secret-manager", None, None),
(
"pyproject-hooks",
"sphinx",
"https://pyproject-hooks.readthedocs.io/en/stable/objects.inv",
),
("webencodings", "sphinx", "https://pythonhosted.org/webencodings/objects.inv"),
(
"fastjsonschema",
"sphinx",
"https://horejsek.github.io/python-fastjsonschema/objects.inv",
),
("authlib", None, None),
("psycopg", "sphinx", "https://www.psycopg.org/psycopg3/docs/objects.inv"),
("build", "sphinx", "https://build.pypa.io/en/latest/objects.inv"),
("grpc-google-iam-v1", None, None),
("gunicorn", None, None),
("executing", None, None),
(
"databricks-sdk",
"sphinx",
"https://databricks-sdk-py.readthedocs.io/en/stable/objects.inv",
),
("isort", "mkdocs", "https://isort.readthedocs.io/en/latest/objects.inv"),
("joserfc", None, None),
("azure-storage-blob", None, None),
("jedi", "sphinx", "https://jedi.readthedocs.io/en/latest/objects.inv"),
("parso", "sphinx", "https://parso.readthedocs.io/en/latest/objects.inv"),
("babel", None, None),
("asttokens", "sphinx", "https://asttokens.readthedocs.io/en/latest/objects.inv"),
("psycopg-binary", "sphinx", "https://psycopg.org/psycopg3/docs/objects.inv"),
("google-cloud-aiplatform", None, None),
("matplotlib-inline", None, None),
("cython", "sphinx", "https://cython.readthedocs.io/en/latest/objects.inv"),
("tornado", "sphinx", "https://www.tornadoweb.org/en/stable/objects.inv"),
("markdown", "mkdocs", "https://Python-Markdown.github.io/objects.inv"),
("setuptools-scm", None, None),
("xmltodict", None, None),
("pydantic-ai-slim", "astro", "https://pydantic.dev/docs/ai/objects.inv"),
("stack-data", None, None),
("pure-eval", None, None),
("asyncpg", None, None),
("gcsfs", "sphinx", "https://gcsfs.readthedocs.io/en/stable/objects.inv"),
("google-cloud-batch", None, None),
(
"marshmallow",
"sphinx",
"https://marshmallow.readthedocs.io/en/stable/objects.inv",
),
("cachecontrol", None, None),
# PyPI download ranks 301-350.
("langsmith", None, None),
("python-json-logger", None, None),
("vcs-versioning", None, None),
("fastuuid", None, None),
(
"python-docx",
"sphinx",
"https://python-docx.readthedocs.org/en/latest/objects.inv",
),
("structlog", None, None),
("watchdog", "sphinx", "https://pythonhosted.org/watchdog/objects.inv"),
("playwright", None, None),
("google-cloud-kms", None, None),
("safetensors", None, None),
("tinycss2", None, None),
("termcolor", None, None),
("google-analytics-admin", None, None),
("google-cloud-compute", None, None),
("pytokens", None, None),
(
"aioitertools",
"sphinx",
"https://aioitertools.omnilib.dev/en/latest/objects.inv",
),
("kubernetes", None, None),
("brotli", None, None),
("pymupdf", "sphinx", "https://pymupdf.readthedocs.io/en/latest/objects.inv"),
("pyzmq", "sphinx", "https://pyzmq.readthedocs.org/en/stable/objects.inv"),
("pyee", "mkdocs", "https://pyee.readthedocs.io/en/stable/objects.inv"),
("xlsxwriter", None, None),
("pymongo", "sphinx", "https://pymongo.readthedocs.io/en/stable/objects.inv"),
("lz4", None, None),
(
"mdit-py-plugins",
"mkdocs",
"https://mdit-py-plugins.readthedocs.io/en/stable/objects.inv",
),
("google-cloud-pubsub", None, None),
("pydantic-graph", "astro", "https://pydantic.dev/docs/ai/objects.inv"),
("cbor2", "sphinx", "https://cbor2.readthedocs.org/en/latest/objects.inv"),
(
"pytest-mock",
"sphinx",
"https://pytest-mock.readthedocs.io/en/latest/objects.inv",
),
("pytest-timeout", None, None),
("py4j", "sphinx", "https://www.py4j.org/objects.inv"),
("pymysql", "sphinx", "https://pymysql.readthedocs.io/en/latest/objects.inv"),
("linkify-it-py", None, None),
("flatbuffers", None, None),
("google-cloud-dlp", None, None),
("pyperclip", None, None),
("nest-asyncio", None, None),
("google-cloud-speech", None, None),
("dbt-adapters", None, None),
("wsproto", None, None),
("debugpy", None, None),
("typedload", None, None),
("google-cloud-texttospeech", None, None),
("ruamel-yaml-clib", None, None),
("croniter", None, None),
("slack-sdk", None, None),
("uc-micro-py", None, None),
(
"weaviate-client",
"sphinx",
"https://weaviate-python-client.readthedocs.io/en/stable/objects.inv",
),
("griffelib", None, None),
("tomli-w", None, None),
# PyPI download ranks 351-400.
("types-toml", None, None),
(
"importlib-resources",
"sphinx",
"https://importlib-resources.readthedocs.io/en/latest/objects.inv",
),
("pycryptodome", "sphinx", "https://www.pycryptodome.org/objects.inv"),
("google-cloud-logging", None, None),
("opentelemetry-instrumentation-fastapi", None, None),
("reportlab", None, None),
(
"awswrangler",
"sphinx",
"https://aws-sdk-pandas.readthedocs.io/en/stable/objects.inv",
),
("torch", "sphinx", "https://docs.pytorch.org/docs/stable/objects.inv"),
("fastmcp", None, None),
("mccabe", None, None),
(
"tree-sitter",
"sphinx",
"https://tree-sitter.github.io/py-tree-sitter/objects.inv",
),
(
"jupyter-core",
"sphinx",
"https://jupyter-core.readthedocs.io/en/stable/objects.inv",
),
(
"deprecation",
"sphinx",
"https://deprecation.readthedocs.io/en/stable/objects.inv",
),
("beartype", "sphinx", "https://beartype.readthedocs.io/en/stable/objects.inv"),
("opentelemetry-instrumentation-asgi", None, None),
("cyclopts", None, None),
("text-unidecode", None, None),
("semver", "sphinx", "https://python-semver.rtfd.io/en/stable/objects.inv"),
("loguru", "sphinx", "https://loguru.readthedocs.io/en/stable/objects.inv"),
("ast-serialize", None, None),
("grpclib", None, None),
("nbformat", "sphinx", "https://nbformat.readthedocs.io/en/latest/objects.inv"),
("uuid-utils", None, None),
("jsonschema-path", None, None),
("sqlglot", None, None),
("typing-inspect", None, None),
("python-slugify", None, None),
("onnxruntime", None, None),
("dulwich", None, None),
("xlrd", None, None),
("types-certifi", None, None),
("aiosqlite", None, None),
("jsonref", "sphinx", "https://jsonref.readthedocs.io/en/latest/objects.inv"),
("dbt-core", None, None),
("invoke", "sphinx", "https://www.pyinvoke.org/objects.inv"),
("opentelemetry-exporter-prometheus", None, None),
(
"jupyter-client",
"sphinx",
"https://jupyter-client.readthedocs.io/en/stable/objects.inv",
),
("google-cloud-monitoring", None, None),
("pathable", None, None),
("fastmcp-slim", None, None),
("modal", None, None),
("synchronicity", None, None),
("graphql-core", None, None),
("pycodestyle", "sphinx", "https://pycodestyle.pycqa.org/en/latest/objects.inv"),
("types-pyyaml", None, None),
("google-cloud-tasks", None, None),
("argcomplete", "sphinx", "https://kislyuk.github.io/argcomplete/objects.inv"),
("faker", "sphinx", "https://faker.readthedocs.io/en/master/objects.inv"),
("fastapi-cli", "zensical", "https://fastapi.tiangolo.com/objects.inv"),
("databricks-sql-connector", None, None),
# PyPI download ranks 401-450.
("pydantic-extra-types", "astro", "https://docs.pydantic.dev/latest/objects.inv"),
("installer", None, None),
("rfc3339-validator", None, None),
("pkginfo", "sphinx", "https://pythonhosted.org/pkginfo/objects.inv"),
("lark", None, None),
("rich-rst", "sphinx", "https://rich-rst.readthedocs.io/en/latest/objects.inv"),
("shapely", "sphinx", "https://shapely.readthedocs.io/en/stable/objects.inv"),
("py-key-value-aio", None, None),
("backports-zstd", None, None),
("ipython-pygments-lexers", None, None),
("numba", "sphinx", "https://numba.readthedocs.io/en/stable/objects.inv"),
("arrow", "sphinx", "https://arrow.readthedocs.io/en/stable/objects.inv"),
("markdownify", None, None),
("deepdiff", None, None),
("dbt-common", None, None),
("smart-open", None, None),
("requests-file", None, None),
("rich-toolkit", None, None),
("polars", "sphinx", "https://docs.pola.rs/api/python/stable/objects.inv"),
("mistune", None, None),
("argon2-cffi-bindings", None, None),
("bleach", "sphinx", "https://pythonhosted.org/bleach/objects.inv"),
("durationpy", None, None),
("msrest", None, None),
("mmh3", "sphinx", "https://mmh3.readthedocs.io/en/stable/objects.inv"),
("snowflake-snowpark-python", None, None),
("pypdfium2", "sphinx", "https://pypdfium2.readthedocs.io/en/stable/objects.inv"),
("llama-parse", None, None),
(
"argon2-cffi",
"sphinx",
"https://argon2-cffi.readthedocs.io/en/stable/objects.inv",
),
("llama-cloud-services", None, None),
("google-cloud-bigtable", None, None),
("langchain-openai", None, None),
("poetry", None, None),
("pendulum", None, None),
("typeguard", "sphinx", "https://typeguard.readthedocs.io/en/latest/objects.inv"),
("pygithub", "sphinx", "https://pygithub.readthedocs.io/en/stable/objects.inv"),
("types-protobuf", None, None),
("jsonpath-ng", None, None),
("uncalled-for", None, None),
("scramp", None, None),
("llvmlite", "sphinx", "https://llvmlite.readthedocs.io/en/latest/objects.inv"),
("sphinx", "sphinx", "https://www.sphinx-doc.org/en/master/objects.inv"),
(
"grpcio-health-checking",
"sphinx",
"https://grpc.github.io/grpc/python/objects.inv",
),
("nltk", "sphinx", "https://www.nltk.org/objects.inv"),
("fastcore", None, None),
("comm", None, None),
("plotly", "sphinx", "https://plotly.com/python-api-reference/objects.inv"),
("crashtest", None, None),
("ipykernel", "sphinx", "https://ipykernel.readthedocs.io/en/stable/objects.inv"),
("langgraph", None, None),
# PyPI download ranks 451-500.
("aiofile", None, None),
("duckdb", None, None),
("cleo", None, None),
("pdfminer-six", None, None),
("pyarrow-hotfix", None, None),
("responses", None, None),
("caio", None, None),
("humanize", "mkdocs", "https://humanize.readthedocs.io/en/stable/objects.inv"),
("psycopg2", "sphinx", "https://www.psycopg.org/docs/objects.inv"),
("cfn-lint", None, None),
("google-cloud-vision", None, None),
("triton", None, None),
("cattrs", "sphinx", "https://catt.rs/en/stable/objects.inv"),
("dataclasses-json", None, None),
(
"portalocker",
"sphinx",
"https://portalocker.readthedocs.io/en/latest/objects.inv",
),
("google-cloud-resource-manager", None, None),
("fastapi-cloud-cli", "zensical", "https://fastapi.tiangolo.com/objects.inv"),
("datadog", "sphinx", "https://datadogpy.readthedocs.io/en/latest/objects.inv"),
("toolz", "sphinx", "https://toolz.readthedocs.io/en/latest/objects.inv"),
("google-cloud-run", None, None),
("mcp-types", "zensical", "https://py.sdk.modelcontextprotocol.io/objects.inv"),
("poetry-plugin-export", None, None),
("nbconvert", "sphinx", "https://nbconvert.readthedocs.io/en/latest/objects.inv"),
(
"python-pptx",
"sphinx",
"https://python-pptx.readthedocs.io/en/latest/objects.inv",
),
("py", "sphinx", "https://pylib.readthedocs.io/en/latest/objects.inv"),
("langchain-protocol", None, None),
("pycryptodomex", "sphinx", "https://www.pycryptodome.org/objects.inv"),
(
"polars-runtime-32",
"sphinx",
"https://docs.pola.rs/api/python/stable/objects.inv",
),
("pyflakes", None, None),
("backports-tarfile", None, None),
("snowflake-sqlalchemy", None, None),
("future", "sphinx", "https://python-future.org/objects.inv"),
("pysocks", None, None),
("nbclient", "sphinx", "https://nbclient.readthedocs.io/en/stable/objects.inv"),
("google-cloud-videointelligence", None, None),
("google-cloud-language", None, None),
("google-cloud-workflows", None, None),
("colorlog", None, None),
("unidiff", None, None),
("sqlalchemy-bigquery", None, None),
("confluent-kafka", None, None),
("absl-py", None, None),
("google-cloud-dataform", None, None),
(
"zope-interface",
"sphinx",
"https://zopeinterface.readthedocs.io/en/latest/objects.inv",
),
("google-cloud-redis", None, None),
("openapi-pydantic", None, None),
(
"pbs-installer",
"mkdocs",
"https://pbs-installer.readthedocs.io/en/stable/objects.inv",
),
("libcst", "sphinx", "https://libcst.readthedocs.io/en/latest/objects.inv"),
("google-cloud-os-login", None, None),
("google-cloud-bigquery-datatransfer", None, None),
)
assert len(INVENTORIES) == 500
assert len({name for name, _, _ in INVENTORIES}) == len(INVENTORIES)
assert all(
(builder is None) == (url is None)
for _, builder, url in INVENTORIES
)
assert sum(url is not None for _, _, url in INVENTORIES) == 223
# Builder values were checked from HTML fingerprints and upstream build config.
assert Counter(builder for _, builder, _ in INVENTORIES if builder) == Counter(
sphinx=198,
zensical=8,
mkdocs=12,
astro=5,
)
@dataclass(frozen=True)
class InventoryResult:
name: str
url: str
project: str
version: str
object_count: int
object_types: frozenset[str]
object_type_counts: Counter[str]
from_cache: bool
def _parse_inventory(
name: str,
url: str,
*,
payload: bytes,
from_cache: bool,
) -> InventoryResult:
assert sphobjinv is not None
inventory = sphobjinv.Inventory(zlib=payload)
# sphobjinv calls the second inventory-field component ``role``;
# in Sphinx's inventory format it is the long object type, e.g.
# ``function`` in ``py:function`` (not the xref role ``py:func``).
object_type_counts = Counter(
f"{obj.domain}:{obj.role}"
for obj in inventory.objects
if obj.domain == "py" and obj.role
)
return InventoryResult(
name=name,
url=url,
project=str(inventory.project or ""),
version=str(inventory.version or ""),
object_count=inventory.count,
object_types=frozenset(object_type_counts),
object_type_counts=object_type_counts,
from_cache=from_cache,
)
def _load_inventory(
name: str,
url: str,
*,
cache_dir: Path,
refresh: bool,
timeout: float,
retries: int,
) -> InventoryResult:
cache_path = cache_dir / f"{name}.objects.inv"
if not refresh and cache_path.is_file():
try:
return _parse_inventory(
name,
url,
payload=cache_path.read_bytes(),
from_cache=True,
)
except Exception:
# A truncated or otherwise invalid cache entry should heal itself.
# Keep it in place until a valid replacement has been downloaded.
pass
request = Request(
url,
headers={
"Accept": "application/octet-stream,*/*;q=0.8",
"User-Agent": "inventory-object-type-survey/1.0",
},
)
for attempt in range(retries + 1):
try:
with urlopen(request, timeout=timeout) as response:
payload = response.read()
result = _parse_inventory(
name,
url,
payload=payload,
from_cache=False,
)
# Replace atomically so an interrupted write cannot corrupt an
# already valid cache entry.
temporary_path = cache_path.with_suffix(cache_path.suffix + ".tmp")
temporary_path.write_bytes(payload)
temporary_path.replace(cache_path)
return result
except Exception:
if attempt == retries:
raise
time.sleep(0.5 * (2**attempt))
raise AssertionError("retry loop exited unexpectedly")
def _positive_int(value: str) -> int:
parsed = int(value)
if parsed < 1:
raise argparse.ArgumentTypeError("must be at least 1")
return parsed
def _mark_invalid(text: str, *, color: bool) -> str:
marked = f"{text} [INVALID]"
if color:
return f"{INVALID_STYLE}{marked}{RESET_STYLE}"
return marked
def _nonnegative_int(value: str) -> int:
parsed = int(value)
if parsed < 0:
raise argparse.ArgumentTypeError("must be nonnegative")
return parsed
def _positive_float(value: str) -> float:
parsed = float(value)
if parsed <= 0:
raise argparse.ArgumentTypeError("must be greater than 0")
return parsed
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--cache-dir",
type=Path,
default=DEFAULT_CACHE_DIR,
help=f"inventory cache directory (default: {DEFAULT_CACHE_DIR})",
)
parser.add_argument(
"--refresh",
action="store_true",
help="download and replace inventories even when cached",
)
parser.add_argument(
"--workers",
type=_positive_int,
default=12,
help="maximum concurrent downloads (default: 12)",
)
parser.add_argument(
"--timeout",
type=_positive_float,
default=20.0,
help="per-request timeout in seconds (default: 20)",
)
parser.add_argument(
"--retries",
type=_nonnegative_int,
default=1,
help="retries after the initial request (default: 1)",
)
parser.add_argument(
"--strict",
action="store_true",
help="exit unsuccessfully if any inventory could not be loaded",
)
parser.add_argument(
"--verbose",
action="store_true",
help="report each successful inventory on stderr",
)
return parser.parse_args()
def main() -> int:
args = _parse_args()
if sphobjinv is None:
print(
"sphobjinv is required; install it with your package manager, e.g.: "
f"uv pip install --python {sys.executable} sphobjinv",
file=sys.stderr,
)
return 2
try:
args.cache_dir.mkdir(parents=True, exist_ok=True)
except OSError as exc:
print(f"cannot create cache directory {args.cache_dir}: {exc}", file=sys.stderr)
return 2
object_type_inventory_counts: Counter[str] = Counter()
object_type_occurrence_counts: Counter[str] = Counter()
object_type_projects: defaultdict[str, set[str]] = defaultdict(set)
invalid_object_type_counts_by_builder: defaultdict[str, Counter[str]] = defaultdict(
Counter
)
builders_by_project = {
name: builder
for name, builder, url in INVENTORIES
if builder is not None and url is not None
}
builder_project_counts = Counter(builders_by_project.values())
successes: list[InventoryResult] = []
failures: list[tuple[str, str, Exception]] = []
with ThreadPoolExecutor(max_workers=args.workers) as executor:
futures = {
executor.submit(
_load_inventory,
name,
url,
cache_dir=args.cache_dir,
refresh=args.refresh,
timeout=args.timeout,
retries=args.retries,
): (name, url)
for name, _builder, url in INVENTORIES
if url is not None
}
for future in as_completed(futures):
name, url = futures[future]
try:
result = future.result()
except Exception as exc:
failures.append((name, url, exc))
continue
successes.append(result)
# object_types is a set, so each type contributes at most once per
# inventory regardless of how many objects have that type.
object_type_inventory_counts.update(result.object_types)
object_type_occurrence_counts.update(result.object_type_counts)
for object_type in result.object_types:
object_type_projects[object_type].add(result.name)
object_type_name = object_type.removeprefix("py:")
if object_type_name not in VALID_OBJECT_TYPES:
invalid_object_type_counts_by_builder[
builders_by_project[result.name]
][object_type_name] += 1
if args.verbose:
project_version = " ".join(
part for part in (result.project, result.version) if part
)
print(
f"[{'cached' if result.from_cache else 'downloaded'}] "
f"{result.name}: {result.object_count} objects"
+ (f" ({project_version})" if project_version else ""),
file=sys.stderr,
)
color = sys.stdout.isatty()
print("Object types by inventory count:")
for object_type in sorted(object_type_inventory_counts):
inventory_count = object_type_inventory_counts[object_type]
project_suffix = ""
if inventory_count < 5:
project_names = sorted(object_type_projects[object_type], key=str.casefold)
project_suffix = f" ({', '.join(project_names)})"
object_type_name = object_type.removeprefix("py:")
displayed_object_type = object_type
if object_type_name not in VALID_OBJECT_TYPES:
displayed_object_type = _mark_invalid(object_type, color=color)
print(f"{displayed_object_type}: {inventory_count}{project_suffix}")
print("\nProjects and object types:")
for result in sorted(successes, key=lambda item: item.name.casefold()):
builder = builders_by_project[result.name]
object_types = sorted(
object_type.removeprefix("py:") for object_type in result.object_types
)
invalid_object_types = set(object_types) - VALID_OBJECT_TYPES
displayed_name = result.name
if invalid_object_types:
displayed_name = _mark_invalid(displayed_name, color=color)
displayed_name += f" (builder: {builder})"
displayed_object_types = [
_mark_invalid(object_type, color=color)
if object_type in invalid_object_types
else object_type
for object_type in object_types
]
print(
f"{displayed_name}: "
f"{', '.join(displayed_object_types) if object_types else '(none)'}"
)
print("\nInvalid object types by builder:")
for builder in sorted(set(builders_by_project.values())):
invalid_object_type_counts = invalid_object_type_counts_by_builder[builder]
displayed_object_types = [
f"{_mark_invalid(object_type, color=color)} "
f"({invalid_object_type_counts[object_type]})"
for object_type in sorted(invalid_object_type_counts)
]
print(
f"{builder} ({builder_project_counts[builder]}): "
f"{', '.join(displayed_object_types) if displayed_object_types else '(none)'}"
)
print("\nValid object types by total occurrence count:")
for object_type in sorted(VALID_OBJECT_TYPES):
qualified_object_type = f"py:{object_type}"
print(
f"{qualified_object_type}: "
f"{object_type_occurrence_counts[qualified_object_type]}"
)
print("\nInvalid object types by total occurrence count:")
invalid_object_types = sorted(
object_type
for object_type in object_type_occurrence_counts
if object_type.removeprefix("py:") not in VALID_OBJECT_TYPES
)
for object_type in invalid_object_types:
displayed_object_type = _mark_invalid(object_type, color=color)
print(f"{displayed_object_type}: {object_type_occurrence_counts[object_type]}")
for name, url, exc in sorted(failures, key=lambda failure: failure[0]):
print(f"[failed] {name}: {url}: {type(exc).__name__}: {exc}", file=sys.stderr)
if not successes:
return 1
if args.strict and failures:
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment