Skip to content

Commit 59936e6

Browse files
committed
GH-55596: [Python][Annotations] Add runtime docstrings to the type stubs with docstring-adder
Squash the stub-only pyarrow.__lib package into lib.pyi, with a section per .pxi file, since pyarrow.lib is a single extension module built from the .pxi files it includes. The per-file __all__ lists go away, and so does the aliasing of pyarrow.__lib modules to pyarrow.lib in run_stubtest.py. Replace the libcst codemod in update_stub_docstrings.py with docstring-adder (https://github.com/astral-sh/docstring-adder). The script still assembles an importable pyarrow from the source and install trees, then runs docstring-adder on the installed stubs and checks that lib.pyi got docstrings. Build the Cython modules with embedsignature.format=clinic, so that the embedded signatures go to __text_signature__ instead of the first lines of __doc__ and don't end up in the docstrings of the stubs. Install the stubs into wheels again and add the docstrings in the wheel builds (PYARROW_REQUIRE_STUB_DOCSTRINGS=ON). py.typed stays excluded from wheels until the stubs are complete (GH-49831). Check the stubs in each built wheel against the modules in the same wheel with stubtest (run_stubtest.py --wheel) in the wheel build scripts. The wheel builds don't install the pandas and scipy stubs, so mypy treats those modules as Any when their stubs are missing. libcst is no longer a build requirement. docstring-adder isn't on PyPI, so requirements-wheel-build.txt pins it to a commit.
1 parent cfca462 commit 59936e6

27 files changed

Lines changed: 2195 additions & 2848 deletions

‎ci/conda_env_python.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ cython>=3.1
2424
cloudpickle
2525
fsspec
2626
hypothesis
27-
libcst>=1.8.6
2827
numpy>=2.0
2928
pytest
3029
pytest-faulthandler

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,9 @@ else
4848
exit 1
4949
fi
5050

51-
wheel_build_binary_flag="--only-binary=:all:"
52-
if python -c "import sys; sys.exit(0 if sys.version_info[:2] == (3, 15) else 1)"; then
53-
# TODO: 3.15 workaround due to pyyaml (via libcst). Remove once pyyaml ships 3.15 wheels.
54-
wheel_build_binary_flag=""
55-
fi
56-
5751
pip install \
5852
--force-reinstall \
59-
${wheel_build_binary_flag} \
53+
--only-binary=:all: \
6054
--upgrade \
6155
-r "${source_dir}/python/requirements-wheel-build.txt"
6256
pip install "delocate>=0.10.3"
@@ -154,8 +148,7 @@ popd
154148

155149
echo "=== (${PYTHON_VERSION}) Building wheel ==="
156150
export PYARROW_BUNDLE_ARROW_CPP=ON
157-
# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
158-
# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
151+
export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
159152
export PYARROW_WITH_ACERO="${ARROW_ACERO}"
160153
export PYARROW_WITH_AZURE="${ARROW_AZURE}"
161154
export PYARROW_WITH_DATASET="${ARROW_DATASET}"
@@ -193,3 +186,7 @@ fi
193186
# Move the verified wheels
194187
mkdir -p "${source_dir}/python/repaired_wheels"
195188
mv "${source_dir}"/python/dist/*.whl "${source_dir}"/python/repaired_wheels/
189+
190+
echo "=== (${PYTHON_VERSION}) Check the type stubs against the wheel ==="
191+
python "${source_dir}/python/scripts/run_stubtest.py" \
192+
--wheel "${source_dir}"/python/repaired_wheels/pyarrow-*.whl

‎ci/scripts/python_wheel_windows_build.bat‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,7 @@ popd
116116

117117
echo "=== (%PYTHON%) Building wheel ==="
118118
set PYARROW_BUNDLE_ARROW_CPP=ON
119-
rem TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
120-
rem set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
119+
set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
121120
set PYARROW_WITH_ACERO=%ARROW_ACERO%
122121
set PYARROW_WITH_AZURE=%ARROW_AZURE%
123122
set PYARROW_WITH_DATASET=%ARROW_DATASET%
@@ -155,4 +154,9 @@ echo "Wheel name: %WHEEL_NAME%"
155154
--ignore-existing --with-mangle ^
156155
-w repaired_wheels %WHEEL_NAME% || exit /B 1
157156

157+
@REM Check the type stubs against the wheel
158+
for %%w in (repaired_wheels\pyarrow-*.whl) do (
159+
%PYTHON_CMD% scripts\run_stubtest.py --wheel %%w || exit /B 1
160+
)
161+
158162
popd

‎ci/scripts/python_wheel_xlinux_build.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,7 @@ check_arrow_visibility
168168

169169
echo "=== (${PYTHON_VERSION}) Building wheel ==="
170170
export PYARROW_BUNDLE_ARROW_CPP=ON
171-
# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
172-
# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
171+
export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
173172
export PYARROW_WITH_ACERO="${ARROW_ACERO}"
174173
export PYARROW_WITH_AZURE="${ARROW_AZURE}"
175174
export PYARROW_WITH_DATASET="${ARROW_DATASET}"
@@ -224,4 +223,7 @@ rm -rf dist/temp-fix-wheel
224223

225224
echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ==="
226225
auditwheel repair dist/pyarrow-*.whl -w repaired_wheels
226+
227+
echo "=== (${PYTHON_VERSION}) Check the type stubs against the wheel ==="
228+
python scripts/run_stubtest.py --wheel repaired_wheels/pyarrow-*.whl
227229
popd

‎cpp/cmake_modules/UseCython.cmake‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,11 @@ function(compile_pyx
145145
${no_docstrings_arg}
146146
${cython_debug_arg}
147147
${CYTHON_FLAGS}
148-
# Necessary for autodoc of function arguments
148+
# Necessary for autodoc of function arguments. The clinic format
149+
# stores the signature in __text_signature__ rather than in
150+
# __doc__, which keeps it out of the docstrings of the type stubs.
149151
--directive embedsignature=True
152+
--directive embedsignature.format=clinic
150153
# Necessary for Cython code coverage
151154
--working
152155
${CMAKE_CURRENT_SOURCE_DIR}

‎docs/source/developers/python/development.rst‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,11 @@ Type Checking
107107
PyArrow provides type stubs (``*.pyi`` files) for static type checking. These
108108
stubs are located in the ``pyarrow-stubs/`` directory, which is laid out as a
109109
:pep:`561` stub-only package for ``pyarrow``, so type checkers and IDEs pick
110-
it up from the ``arrow/python`` directory. The stubs aren't included in the
111-
distributed wheel packages until they are complete (see
110+
it up from the ``arrow/python`` directory. The stubs are installed alongside
111+
the package, and the wheel builds add the docstrings of the built modules to
112+
them with `docstring-adder <https://github.com/astral-sh/docstring-adder>`_
113+
(``PYARROW_REQUIRE_STUB_DOCSTRINGS=ON``). The ``py.typed`` marker is omitted
114+
from the wheels until the stubs are complete (see
112115
`GH-49831 <https://github.com/apache/arrow/issues/49831>`_).
113116

114117
Run all the commands in this section from the ``arrow/python`` directory,
@@ -194,6 +197,13 @@ Stubs that aren't written yet are skipped. The known mismatches are listed in
194197
``pyarrow-stubs-tests/stubtest_allowlist.txt``; remove the entries of the
195198
mismatches you fix, as stubtest fails on unused entries.
196199

200+
The wheel builds also check the stubs shipped in each wheel against the
201+
modules in the same wheel, without installing it:
202+
203+
.. code-block::
204+
205+
$ python scripts/run_stubtest.py --wheel dist/pyarrow-*.whl
206+
197207
Type Checking in an IDE
198208
-----------------------
199209

‎python/CMakeLists.txt‎

Lines changed: 19 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1075,40 +1075,26 @@ endif()
10751075
#
10761076
# Type stubs with docstring injection
10771077
#
1078-
# TODO(GH-32609): Reintroduce type stubs into PyArrow wheels.
1079-
# GH-49831: Temporarily do not install pyarrow-stubs into wheels.
10801078
# Stubs live in pyarrow-stubs/ during development but are installed
10811079
# alongside the package so type checkers can find them (PEP 561).
1082-
# The stubs are currently incomplete, and some type checkers consume .pyi files
1083-
# even without the py.typed marker. Re-enable this when the stubs are complete.
1080+
# TODO(GH-32609): py.typed is still omitted from wheels (see pyproject.toml)
1081+
# until the stubs are complete (GH-49831).
1082+
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs/"
1083+
DESTINATION "."
1084+
FILES_MATCHING
1085+
PATTERN "*.pyi")
1086+
10841087
if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
1085-
message(FATAL_ERROR "PYARROW_REQUIRE_STUB_DOCSTRINGS cannot be used while "
1086-
"pyarrow-stubs are omitted from wheels (GH-49831).")
1088+
install(CODE "
1089+
execute_process(
1090+
COMMAND \"${Python3_EXECUTABLE}\"
1091+
\"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
1092+
\"\${CMAKE_INSTALL_PREFIX}\"
1093+
\"${CMAKE_CURRENT_SOURCE_DIR}\"
1094+
RESULT_VARIABLE _pyarrow_stub_docstrings_result
1095+
)
1096+
if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
1097+
message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
1098+
endif()
1099+
")
10871100
endif()
1088-
# set(PYARROW_STUBS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs")
1089-
# if(EXISTS "${PYARROW_STUBS_SOURCE_DIR}")
1090-
# install(DIRECTORY "${PYARROW_STUBS_SOURCE_DIR}/"
1091-
# DESTINATION "."
1092-
# FILES_MATCHING
1093-
# PATTERN "*.pyi")
1094-
#
1095-
# if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
1096-
# install(CODE "
1097-
# execute_process(
1098-
# COMMAND \"${Python3_EXECUTABLE}\"
1099-
# \"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
1100-
# \"${CMAKE_INSTALL_PREFIX}\"
1101-
# \"${CMAKE_CURRENT_SOURCE_DIR}\"
1102-
# RESULT_VARIABLE _pyarrow_stub_docstrings_result
1103-
# )
1104-
# if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
1105-
# message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
1106-
# endif()
1107-
# ")
1108-
# endif()
1109-
# else()
1110-
# if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
1111-
# message(FATAL_ERROR "PyArrow stub source directory not found at ${PYARROW_STUBS_SOURCE_DIR}; "
1112-
# "cannot build wheel without .pyi files.")
1113-
# endif()
1114-
# endif()

‎python/pyarrow-stubs-tests/stubtest_allowlist.txt‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,8 @@
2121
# current mismatches with
2222
# python scripts/run_stubtest.py --generate-allowlist
2323

24-
# Stub-only modules. The pyarrow.__lib modules are checked against
25-
# pyarrow.lib, which has no __all__.
24+
# Stub-only module.
2625
pyarrow\._stubs_typing
27-
pyarrow\.__lib\.[a-z]+\.__all__
2826

2927
# The __init__ of Cython extension types takes (*args, **kwargs) at runtime.
30-
pyarrow\.__lib\.[a-z]+\.[A-Za-z_]+\.__init__
28+
pyarrow\.lib\.[A-Za-z_]+\.__init__

‎python/pyarrow-stubs/__lib/__init__.pyi‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

0 commit comments

Comments
 (0)