Skip to content

Commit a59a2a2

Browse files
Merge remote-tracking branch 'upstream/main' into apachegh-43683-pandas-string-dtype
2 parents ea4cbf4 + 2b5f56c commit a59a2a2

42 files changed

Lines changed: 1085 additions & 339 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ ULIMIT_CORE=-1
4747

4848
# Default versions for platforms
4949
ALMALINUX=8
50-
ALPINE_LINUX=3.16
50+
ALPINE_LINUX=3.18
5151
DEBIAN=12
5252
FEDORA=39
5353
UBUNTU=22.04
@@ -92,10 +92,11 @@ TZ=UTC
9292
VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release
9393

9494
# This must be updated when we update
95-
# ci/docker/python-wheel-windows-vs2019.dockerfile.
95+
# ci/docker/python-*-windows-*.dockerfile.
9696
# This is a workaround for our CI problem that "archery docker build" doesn't
9797
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
98-
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-08-06
98+
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-01-08
99+
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-01-08
99100

100101
# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
101102
# See https://github.com/conan-io/conan-docker-tools#readme and

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ repos:
3939
files: >-
4040
(
4141
?^ci/docker/conda-python-emscripten\.dockerfile$|
42-
?^ci/docker/python-wheel-windows-test-vs2019\.dockerfile$|
42+
?^ci/docker/python-.*-wheel-windows-test-vs2019.*\.dockerfile$|
4343
)
4444
types: []
4545
- repo: https://github.com/pycqa/flake8

ci/docker/alpine-linux-3.16-cpp.dockerfile renamed to ci/docker/alpine-linux-3.18-cpp.dockerfile

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@
1616
# under the License.
1717

1818
ARG arch=amd64
19-
FROM ${arch}/alpine:3.16
19+
FROM ${arch}/alpine:3.18
2020

2121
RUN apk add \
22+
apache-orc-dev \
2223
bash \
2324
benchmark-dev \
2425
boost-dev \
@@ -39,8 +40,8 @@ RUN apk add \
3940
grpc-dev \
4041
gtest-dev \
4142
libxml2-dev \
42-
llvm13-dev \
43-
llvm13-static \
43+
llvm16-dev \
44+
llvm16-static \
4445
lz4-dev \
4546
make \
4647
musl-locales \
@@ -61,6 +62,7 @@ RUN apk add \
6162
thrift-dev \
6263
tzdata \
6364
utf8proc-dev \
65+
xsimd-dev \
6466
zlib-dev \
6567
zstd-dev && \
6668
rm -rf /var/cache/apk/* && \
@@ -98,6 +100,5 @@ ENV ARROW_ACERO=ON \
98100
ARROW_WITH_ZSTD=ON \
99101
AWSSDK_SOURCE=BUNDLED \
100102
google_cloud_cpp_storage_SOURCE=BUNDLED \
101-
ORC_SOURCE=BUNDLED \
102-
PATH=/usr/lib/ccache/:$PATH \
103-
xsimd_SOURCE=BUNDLED
103+
MUSL_LOCPATH=/usr/share/i18n/locales/musl \
104+
PATH=/usr/lib/ccache/bin:$PATH
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env
19+
# when you update this file.
20+
21+
ARG base
22+
# https://github.com/hadolint/hadolint/wiki/DL3006
23+
# (Hadolint does not expand variables and thinks '${base}' is an untagged image)
24+
# hadolint ignore=DL3006
25+
FROM ${base}
26+
27+
ARG python=3.13
28+
29+
SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
30+
RUN $filename = 'python-3.13.1-amd64.exe'; \
31+
$url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
32+
Invoke-WebRequest -Uri $url -OutFile $filename; \
33+
Start-Process -FilePath $filename -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
34+
35+
ENV PYTHON_CMD="py -${python}t"
36+
37+
SHELL ["cmd", "/S", "/C"]
38+
RUN %PYTHON_CMD% -m pip install -U pip setuptools
39+
40+
COPY python/requirements-wheel-test.txt C:/arrow/python/
41+
# Cython and Pandas wheels for 3.13 free-threaded are not released yet
42+
RUN %PYTHON_CMD% -m pip install \
43+
--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
44+
--pre \
45+
--prefer-binary \
46+
-r C:/arrow/python/requirements-wheel-test.txt
47+
# cffi-based tests would crash when importing cffi.
48+
# hadolint ignore=DL3059
49+
RUN %PYTHON_CMD% -m pip uninstall -y cffi
50+
51+
ENV PYTHON="${python}t"
52+
ENV PYTHON_GIL=0
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
19+
# when you update this file.
20+
21+
ARG base
22+
# https://github.com/hadolint/hadolint/wiki/DL3006
23+
# (Hadolint does not expand variables and thinks '${base}' is an untagged image)
24+
# hadolint ignore=DL3006
25+
FROM ${base}
26+
27+
ARG python=3.13
28+
29+
SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
30+
RUN $filename = 'python-3.13.1-amd64.exe'; \
31+
$url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
32+
Invoke-WebRequest -Uri $url -OutFile $filename; \
33+
Start-Process -FilePath $filename -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
34+
35+
ENV PYTHON_CMD="py -${python}t"
36+
37+
SHELL ["cmd", "/S", "/C"]
38+
RUN %PYTHON_CMD% -m pip install -U pip setuptools
39+
40+
COPY python/requirements-wheel-build.txt C:/arrow/python/
41+
# Cython wheels for 3.13 free-threaded are not released yet
42+
RUN %PYTHON_CMD% -m pip install \
43+
--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
44+
--pre \
45+
--prefer-binary \
46+
cython
47+
RUN %PYTHON_CMD% -m pip install -r C:/arrow/python/requirements-wheel-build.txt
48+
49+
ENV PYTHON="${python}t"
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env
19+
# when you update this file.
20+
21+
# based on mcr.microsoft.com/windows/servercore:ltsc2019
22+
# contains choco and vs2019 preinstalled
23+
FROM abrarov/msvc-2019:2.11.0
24+
25+
# hadolint shell=cmd.exe
26+
27+
# Add unix tools to path
28+
RUN setx path "%path%;C:\Program Files\Git\usr\bin"
29+
30+
# 1. Remove previous installations of Python from the base image
31+
# NOTE: a more recent base image (tried with 2.12.1) comes with Python 3.9.7
32+
# and the MSI installers are failing to remove pip and tcl/tk "products" making
33+
# the subsequent choco python installation step failing for installing Python
34+
# version 3.9.* due to existing python version
35+
# 2. Install Minio for S3 testing.
36+
RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
37+
rm -rf Python* && \
38+
curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z \
39+
--output "C:\Windows\Minio.exe"
40+
41+
# Install archiver to extract xz archives (for timezone database).
42+
# Install the GCS testbench using a well-known Python version.
43+
# NOTE: cannot use pipx's `--fetch-missing-python` because of
44+
# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves.
45+
RUN choco install --no-progress -r -y archiver && \
46+
choco install -r -y --pre --no-progress python --version=3.11.9
47+
ENV PIPX_BIN_DIR=C:\\Windows\\
48+
ENV PIPX_PYTHON="C:\Python311\python.exe"
49+
COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/
50+
RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
51+
storage-testbench -h

‎ci/docker/python-wheel-windows-test-vs2019.dockerfile‎

Lines changed: 19 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -15,49 +15,31 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
18+
# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env
1919
# when you update this file.
2020

21-
# based on mcr.microsoft.com/windows/servercore:ltsc2019
22-
# contains choco and vs2019 preinstalled
23-
FROM abrarov/msvc-2019:2.11.0
21+
ARG base
22+
# https://github.com/hadolint/hadolint/wiki/DL3006
23+
# (Hadolint does not expand variables and thinks '${base}' is an untagged image)
24+
# hadolint ignore=DL3006
25+
FROM ${base}
2426

2527
# hadolint shell=cmd.exe
2628

27-
# Add unix tools to path
28-
RUN setx path "%path%;C:\Program Files\Git\usr\bin"
29-
30-
# 1. Remove previous installations of python from the base image
31-
# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
32-
# and the msi installers are failing to remove pip and tcl/tk "products" making
33-
# the subsequent choco python installation step failing for installing python
34-
# version 3.9.* due to existing python version
35-
# 2. Install Minio for S3 testing.
36-
RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
37-
rm -rf Python* && \
38-
curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z \
39-
--output "C:\Windows\Minio.exe"
40-
41-
# Install the GCS testbench using a well-known Python version.
42-
# NOTE: cannot use pipx's `--fetch-missing-python` because of
43-
# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves.
44-
RUN choco install -r -y --pre --no-progress python --version=3.11.9
45-
ENV PIPX_BIN_DIR=C:\\Windows\\
46-
ENV PIPX_PYTHON="C:\Python311\python.exe"
47-
COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/
48-
RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
49-
storage-testbench -h
50-
5129
# Define the full version number otherwise choco falls back to patch number 0 (3.9 => 3.9.0)
5230
ARG python=3.9
53-
RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13") & \
54-
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11") & \
55-
(if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9") & \
56-
(if "%python%"=="3.12" setx PYTHON_VERSION "3.12.5") & \
57-
(if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1")
58-
59-
# Install archiver to extract xz archives
60-
RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION% && \
61-
choco install --no-progress -r -y archiver
31+
RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PYTHON_CMD "C:\Python39\python") & \
32+
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD "py -3.10") & \
33+
(if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py -3.11") & \
34+
(if "%python%"=="3.12" setx PYTHON_VERSION "3.12.8" && setx PYTHON_CMD "py -3.12") & \
35+
(if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1" && setx PYTHON_CMD "py -3.13")
36+
37+
# hadolint ignore=DL3059
38+
RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION%
39+
# hadolint ignore=DL3059
40+
RUN %PYTHON_CMD% -m pip install -U pip setuptools
41+
42+
COPY python/requirements-wheel-test.txt C:/arrow/python/
43+
RUN %PYTHON_CMD% -m pip install -r C:/arrow/python/requirements-wheel-test.txt
6244

6345
ENV PYTHON=$python
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
19+
# when you update this file.
20+
21+
# based on mcr.microsoft.com/windows/servercore:ltsc2019
22+
# contains choco and vs2019 preinstalled
23+
FROM abrarov/msvc-2019:2.11.0
24+
25+
# Install CMake and Ninja
26+
ARG cmake=3.31.2
27+
RUN choco install --no-progress -r -y cmake --version=%cmake% --installargs 'ADD_CMAKE_TO_PATH=System' && \
28+
choco install --no-progress -r -y gzip wget ninja
29+
30+
# Add unix tools to path
31+
RUN setx path "%path%;C:\Program Files\Git\usr\bin"
32+
33+
# Install vcpkg
34+
#
35+
# Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has
36+
# started to ship precompiled binaries for the vcpkg-tool.
37+
ARG vcpkg
38+
COPY ci/vcpkg/*.patch \
39+
ci/vcpkg/*windows*.cmake \
40+
arrow/ci/vcpkg/
41+
COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
42+
ENV VCPKG_ROOT=C:\\vcpkg
43+
RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
44+
setx PATH "%PATH%;%VCPKG_ROOT%"
45+
46+
# Configure vcpkg and install dependencies
47+
# NOTE: use windows batch environment notation for build arguments in RUN
48+
# statements but bash notation in ENV statements
49+
# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system
50+
# cmake's and ninja's versions are recent enough
51+
ARG build_type=release
52+
ENV CMAKE_BUILD_TYPE=${build_type} \
53+
VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
54+
VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
55+
VCPKG_FEATURE_FLAGS="manifests"
56+
COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
57+
# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
58+
# ssl related fixes as well as we can patch the vcpkg portfile to support
59+
# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
60+
# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
61+
# previous versions => use bundled S3 build
62+
RUN vcpkg install \
63+
--clean-after-build \
64+
--x-install-root=%VCPKG_ROOT%\installed \
65+
--x-manifest-root=arrow/ci/vcpkg \
66+
--x-feature=flight \
67+
--x-feature=gcs \
68+
--x-feature=json \
69+
--x-feature=orc \
70+
--x-feature=parquet \
71+
--x-feature=s3
72+
73+
# Remove previous installations of Python from the base image
74+
# NOTE: a more recent base image (tried with 2.12.1) comes with Python 3.9.7
75+
# and the MSI installers are failing to remove pip and tcl/tk "products" making
76+
# the subsequent choco python installation step failing for installing Python
77+
# version 3.9.* due to existing Python version
78+
RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
79+
rm -rf Python*

0 commit comments

Comments
 (0)