Skip to content

Commit b44e1d7

Browse files
committed
GH-55596: [Python][Annotations] Add runtime docstrings to the type stubs with docstring-adder
Squash the stub-only pyarrow.__lib package into lib.pyi, with a section per .pxi file, since pyarrow.lib is a single extension module built from the .pxi files it includes. The per-file __all__ lists go away, and so does the aliasing of pyarrow.__lib modules to pyarrow.lib in run_stubtest.py. Replace the libcst codemod in update_stub_docstrings.py with docstring-adder (https://github.com/astral-sh/docstring-adder). The script still assembles an importable pyarrow from the source and install trees, then runs docstring-adder on the installed stubs and checks that lib.pyi got docstrings. Build the Cython modules with embedsignature.format=clinic, so that the embedded signatures go to __text_signature__ instead of the first lines of __doc__ and don't end up in the docstrings of the stubs. Install the stubs into wheels again and add the docstrings in the wheel builds (PYARROW_REQUIRE_STUB_DOCSTRINGS=ON). py.typed stays excluded from wheels until the stubs are complete (GH-49831). Check the stubs in each built wheel against the modules in the same wheel with stubtest (run_stubtest.py --wheel) in the wheel build scripts. The wheel builds don't install the pandas and scipy stubs, so mypy treats those modules as Any when their stubs are missing. The wheel content validation checks again that the wheels contain the stubs of pyarrow-stubs/ with docstrings, and still that they don't contain py.typed. libcst is no longer a build requirement. docstring-adder isn't on PyPI, so requirements-wheel-build.txt pins it to a commit.
1 parent cfca462 commit b44e1d7

28 files changed

Lines changed: 2200 additions & 2855 deletions

‎ci/conda_env_python.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ cython>=3.1
2424
cloudpickle
2525
fsspec
2626
hypothesis
27-
libcst>=1.8.6
2827
numpy>=2.0
2928
pytest
3029
pytest-faulthandler

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,9 @@ else
4848
exit 1
4949
fi
5050

51-
wheel_build_binary_flag="--only-binary=:all:"
52-
if python -c "import sys; sys.exit(0 if sys.version_info[:2] == (3, 15) else 1)"; then
53-
# TODO: 3.15 workaround due to pyyaml (via libcst). Remove once pyyaml ships 3.15 wheels.
54-
wheel_build_binary_flag=""
55-
fi
56-
5751
pip install \
5852
--force-reinstall \
59-
${wheel_build_binary_flag} \
53+
--only-binary=:all: \
6054
--upgrade \
6155
-r "${source_dir}/python/requirements-wheel-build.txt"
6256
pip install "delocate>=0.10.3"
@@ -154,8 +148,7 @@ popd
154148

155149
echo "=== (${PYTHON_VERSION}) Building wheel ==="
156150
export PYARROW_BUNDLE_ARROW_CPP=ON
157-
# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
158-
# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
151+
export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
159152
export PYARROW_WITH_ACERO="${ARROW_ACERO}"
160153
export PYARROW_WITH_AZURE="${ARROW_AZURE}"
161154
export PYARROW_WITH_DATASET="${ARROW_DATASET}"
@@ -193,3 +186,7 @@ fi
193186
# Move the verified wheels
194187
mkdir -p "${source_dir}/python/repaired_wheels"
195188
mv "${source_dir}"/python/dist/*.whl "${source_dir}"/python/repaired_wheels/
189+
190+
echo "=== (${PYTHON_VERSION}) Check the type stubs against the wheel ==="
191+
python "${source_dir}/python/scripts/run_stubtest.py" \
192+
--wheel "${source_dir}"/python/repaired_wheels/pyarrow-*.whl

‎ci/scripts/python_wheel_validate_contents.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ def _count_docstrings(source):
3737
return count
3838

3939

40-
# TODO(GH-48970): Check stubs ARE present once annotations are complete
4140
def validate_wheel(path):
4241
p = Path(path)
4342
wheels = list(p.glob('*.whl'))
@@ -55,6 +54,8 @@ def validate_wheel(path):
5554
info.filename.split("/")[-1] == filename for info in wheel_zip.filelist
5655
), f"{filename} is missing from the wheel."
5756

57+
# TODO(GH-48970): Check that py.typed IS present once the stubs are
58+
# complete.
5859
assert not any(
5960
info.filename == "pyarrow/py.typed" for info in wheel_zip.filelist
6061
), "pyarrow/py.typed is present in the wheel."
@@ -74,22 +75,19 @@ def validate_wheel(path):
7475
if info.filename.startswith("pyarrow/") and info.filename.endswith(".pyi")
7576
}
7677

77-
assert not (wheel_stub_files == expected_stub_files), (
78-
"Wheel .pyi files do not differ from python/pyarrow-stubs.\n"
78+
assert wheel_stub_files == expected_stub_files, (
79+
"Wheel .pyi files differ from python/pyarrow-stubs.\n"
7980
f"Missing in wheel: {sorted(expected_stub_files - wheel_stub_files)}\n"
8081
f"Unexpected in wheel: {sorted(wheel_stub_files - expected_stub_files)}"
8182
)
82-
assert not wheel_stub_files, (
83-
f"Wheel contains unexpected .pyi files: {sorted(wheel_stub_files)}"
84-
)
8583

8684
wheel_docstring_count = sum(
8785
_count_docstrings(wheel_zip.read(wsf).decode("utf-8"))
8886
for wsf in wheel_stub_files
8987
)
9088

9189
print(f"Found {wheel_docstring_count} docstring(s) in wheel stubs.")
92-
assert wheel_docstring_count == 0, "Docstrings found in wheel stub files."
90+
assert wheel_docstring_count, "No docstrings found in wheel stub files."
9391

9492
print(f"The wheel: {wheels[0]} seems valid.")
9593

‎ci/scripts/python_wheel_windows_build.bat‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,7 @@ popd
116116

117117
echo "=== (%PYTHON%) Building wheel ==="
118118
set PYARROW_BUNDLE_ARROW_CPP=ON
119-
rem TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
120-
rem set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
119+
set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
121120
set PYARROW_WITH_ACERO=%ARROW_ACERO%
122121
set PYARROW_WITH_AZURE=%ARROW_AZURE%
123122
set PYARROW_WITH_DATASET=%ARROW_DATASET%
@@ -155,4 +154,9 @@ echo "Wheel name: %WHEEL_NAME%"
155154
--ignore-existing --with-mangle ^
156155
-w repaired_wheels %WHEEL_NAME% || exit /B 1
157156

157+
@REM Check the type stubs against the wheel
158+
for %%w in (repaired_wheels\pyarrow-*.whl) do (
159+
%PYTHON_CMD% scripts\run_stubtest.py --wheel %%w || exit /B 1
160+
)
161+
158162
popd

‎ci/scripts/python_wheel_xlinux_build.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,7 @@ check_arrow_visibility
168168

169169
echo "=== (${PYTHON_VERSION}) Building wheel ==="
170170
export PYARROW_BUNDLE_ARROW_CPP=ON
171-
# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
172-
# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
171+
export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
173172
export PYARROW_WITH_ACERO="${ARROW_ACERO}"
174173
export PYARROW_WITH_AZURE="${ARROW_AZURE}"
175174
export PYARROW_WITH_DATASET="${ARROW_DATASET}"
@@ -224,4 +223,7 @@ rm -rf dist/temp-fix-wheel
224223

225224
echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ==="
226225
auditwheel repair dist/pyarrow-*.whl -w repaired_wheels
226+
227+
echo "=== (${PYTHON_VERSION}) Check the type stubs against the wheel ==="
228+
python scripts/run_stubtest.py --wheel repaired_wheels/pyarrow-*.whl
227229
popd

‎cpp/cmake_modules/UseCython.cmake‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,11 @@ function(compile_pyx
145145
${no_docstrings_arg}
146146
${cython_debug_arg}
147147
${CYTHON_FLAGS}
148-
# Necessary for autodoc of function arguments
148+
# Necessary for autodoc of function arguments. The clinic format
149+
# stores the signature in __text_signature__ rather than in
150+
# __doc__, which keeps it out of the docstrings of the type stubs.
149151
--directive embedsignature=True
152+
--directive embedsignature.format=clinic
150153
# Necessary for Cython code coverage
151154
--working
152155
${CMAKE_CURRENT_SOURCE_DIR}

‎docs/source/developers/python/development.rst‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,11 @@ Type Checking
107107
PyArrow provides type stubs (``*.pyi`` files) for static type checking. These
108108
stubs are located in the ``pyarrow-stubs/`` directory, which is laid out as a
109109
:pep:`561` stub-only package for ``pyarrow``, so type checkers and IDEs pick
110-
it up from the ``arrow/python`` directory. The stubs aren't included in the
111-
distributed wheel packages until they are complete (see
110+
it up from the ``arrow/python`` directory. The stubs are installed alongside
111+
the package, and the wheel builds add the docstrings of the built modules to
112+
them with `docstring-adder <https://github.com/astral-sh/docstring-adder>`_
113+
(``PYARROW_REQUIRE_STUB_DOCSTRINGS=ON``). The ``py.typed`` marker is omitted
114+
from the wheels until the stubs are complete (see
112115
`GH-49831 <https://github.com/apache/arrow/issues/49831>`_).
113116

114117
Run all the commands in this section from the ``arrow/python`` directory,
@@ -194,6 +197,13 @@ Stubs that aren't written yet are skipped. The known mismatches are listed in
194197
``pyarrow-stubs-tests/stubtest_allowlist.txt``; remove the entries of the
195198
mismatches you fix, as stubtest fails on unused entries.
196199

200+
The wheel builds also check the stubs shipped in each wheel against the
201+
modules in the same wheel, without installing it:
202+
203+
.. code-block::
204+
205+
$ python scripts/run_stubtest.py --wheel dist/pyarrow-*.whl
206+
197207
Type Checking in an IDE
198208
-----------------------
199209

‎python/CMakeLists.txt‎

Lines changed: 19 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1075,40 +1075,26 @@ endif()
10751075
#
10761076
# Type stubs with docstring injection
10771077
#
1078-
# TODO(GH-32609): Reintroduce type stubs into PyArrow wheels.
1079-
# GH-49831: Temporarily do not install pyarrow-stubs into wheels.
10801078
# Stubs live in pyarrow-stubs/ during development but are installed
10811079
# alongside the package so type checkers can find them (PEP 561).
1082-
# The stubs are currently incomplete, and some type checkers consume .pyi files
1083-
# even without the py.typed marker. Re-enable this when the stubs are complete.
1080+
# TODO(GH-32609): py.typed is still omitted from wheels (see pyproject.toml)
1081+
# until the stubs are complete (GH-49831).
1082+
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs/"
1083+
DESTINATION "."
1084+
FILES_MATCHING
1085+
PATTERN "*.pyi")
1086+
10841087
if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
1085-
message(FATAL_ERROR "PYARROW_REQUIRE_STUB_DOCSTRINGS cannot be used while "
1086-
"pyarrow-stubs are omitted from wheels (GH-49831).")
1088+
install(CODE "
1089+
execute_process(
1090+
COMMAND \"${Python3_EXECUTABLE}\"
1091+
\"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
1092+
\"\${CMAKE_INSTALL_PREFIX}\"
1093+
\"${CMAKE_CURRENT_SOURCE_DIR}\"
1094+
RESULT_VARIABLE _pyarrow_stub_docstrings_result
1095+
)
1096+
if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
1097+
message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
1098+
endif()
1099+
")
10871100
endif()
1088-
# set(PYARROW_STUBS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs")
1089-
# if(EXISTS "${PYARROW_STUBS_SOURCE_DIR}")
1090-
# install(DIRECTORY "${PYARROW_STUBS_SOURCE_DIR}/"
1091-
# DESTINATION "."
1092-
# FILES_MATCHING
1093-
# PATTERN "*.pyi")
1094-
#
1095-
# if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
1096-
# install(CODE "
1097-
# execute_process(
1098-
# COMMAND \"${Python3_EXECUTABLE}\"
1099-
# \"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
1100-
# \"${CMAKE_INSTALL_PREFIX}\"
1101-
# \"${CMAKE_CURRENT_SOURCE_DIR}\"
1102-
# RESULT_VARIABLE _pyarrow_stub_docstrings_result
1103-
# )
1104-
# if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
1105-
# message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
1106-
# endif()
1107-
# ")
1108-
# endif()
1109-
# else()
1110-
# if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
1111-
# message(FATAL_ERROR "PyArrow stub source directory not found at ${PYARROW_STUBS_SOURCE_DIR}; "
1112-
# "cannot build wheel without .pyi files.")
1113-
# endif()
1114-
# endif()

‎python/pyarrow-stubs-tests/stubtest_allowlist.txt‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,8 @@
2121
# current mismatches with
2222
# python scripts/run_stubtest.py --generate-allowlist
2323

24-
# Stub-only modules. The pyarrow.__lib modules are checked against
25-
# pyarrow.lib, which has no __all__.
24+
# Stub-only module.
2625
pyarrow\._stubs_typing
27-
pyarrow\.__lib\.[a-z]+\.__all__
2826

2927
# The __init__ of Cython extension types takes (*args, **kwargs) at runtime.
30-
pyarrow\.__lib\.[a-z]+\.[A-Za-z_]+\.__init__
28+
pyarrow\.lib\.[A-Za-z_]+\.__init__

‎python/pyarrow-stubs/__lib/__init__.pyi‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

0 commit comments

Comments
 (0)