Skip to content

Commit 045f680

Browse files
Merge remote-tracking branch 'upstream/master' into ARROW-16728-ParquetDataset-switch-default
2 parents 5aadb41 + 093a4fe commit 045f680

1,281 files changed

Lines changed: 69889 additions & 16035 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,18 +53,18 @@ FEDORA=35
5353
UBUNTU=20.04
5454

5555
# Default versions for various dependencies
56-
CLANG_TOOLS=12
56+
CLANG_TOOLS=14
5757
CUDA=11.0.3
5858
DASK=latest
5959
DOTNET=6.0
6060
GCC_VERSION=""
61-
GO=1.16
61+
GO=1.17
6262
STATICCHECK=v0.2.2
6363
HDFS=3.2.1
6464
JDK=8
6565
KARTOTHEK=latest
6666
# LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
67-
LLVM=13
67+
LLVM=14
6868
MAVEN=3.5.4
6969
NODE=16
7070
NUMBA=latest
@@ -86,8 +86,8 @@ ARROW_R_DEV=TRUE
8686
R_PRUNE_DEPS=FALSE
8787
TZ=UTC
8888

89-
# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
90-
DEVTOOLSET_VERSION=-1
89+
# Any non-empty string will install devtoolset-${DEVTOOLSET_VERSION}
90+
DEVTOOLSET_VERSION=
9191

9292
# Used through docker-compose.yml and serves as the default version for the
9393
# ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the

‎.github/workflows/archery.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ concurrency:
3535
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
3636
cancel-in-progress: true
3737

38+
permissions:
39+
contents: read
40+
3841
jobs:
3942

4043
test:

‎.github/workflows/comment_bot.yml‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ on:
2424
- created
2525
- edited
2626

27+
permissions:
28+
contents: read
29+
pull-requests: write
30+
2731
jobs:
2832
crossbow:
2933
name: Listen!
@@ -66,8 +70,9 @@ jobs:
6670
DEFAULT_BRANCH=${{ github.event.repository.default_branch }}
6771
git remote add upstream https://github.com/apache/arrow
6872
git fetch upstream
73+
6974
changed() {
70-
git diff --name-only HEAD..upstream/$DEFAULT_BRANCH | grep -e "$1" >/dev/null 2>&1
75+
git diff --name-only upstream/$DEFAULT_BRANCH... | grep -e "$1" >/dev/null 2>&1
7176
}
7277
if changed '^r/.*\.R$'; then
7378
echo "R_DOCS=true" >> $GITHUB_ENV
@@ -82,7 +87,7 @@ jobs:
8287
if changed '^r/src'; then
8388
echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV
8489
fi
85-
- name: Ensure clang-format has the appropriate versoin
90+
- name: Ensure clang-format has the appropriate version
8691
if: env.CMAKE_FORMAT == 'true' ||
8792
env.CLANG_FORMAT_CPP == 'true' ||
8893
env.CLANG_FORMAT_R == 'true' ||
@@ -124,18 +129,16 @@ jobs:
124129
run: |
125130
source("ci/etc/rprofile")
126131
install.packages(c("remotes", "roxygen2"))
127-
# We currently need dev roxygen2 (> 7.1.1) until they release
128-
remotes::install_github("r-lib/roxygen2")
129132
remotes::install_deps("r")
130133
roxygen2::roxygenize("r")
131134
- name: Style R code
132135
if: env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything')
133136
shell: Rscript {0}
134137
run: |
135-
changed_files <- system("git diff --name-only HEAD..upstream/${{ github.event.repository.default_branch }} 2>&1", intern = TRUE)
138+
changed_files <- system("git diff --name-only upstream/${{ github.event.repository.default_branch }}... 2>&1", intern = TRUE)
136139
# only grab the .R files under r/
137140
changed_files <- grep('^r/.*\\.R$', changed_files, value = TRUE)
138-
# remove latin1 which is unstylable due to encoding and codegen.R which is unique
141+
# remove codegen.R and other possible exclusions
139142
changed_files <- changed_files[!changed_files %in% file.path("r", source("r/.styler_excludes.R")$value)]
140143
source("ci/etc/rprofile")
141144
install.packages(c("remotes", "styler"))

‎.github/workflows/cpp.yml‎

Lines changed: 27 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -45,18 +45,19 @@ concurrency:
4545
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
4646
cancel-in-progress: true
4747

48+
permissions:
49+
contents: read
50+
4851
env:
4952
ARROW_ENABLE_TIMING_TESTS: OFF
5053
DOCKER_VOLUME_PREFIX: ".docker/"
51-
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
52-
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
5354

5455
jobs:
5556
docker:
5657
name: ${{ matrix.title }}
5758
runs-on: ubuntu-latest
5859
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
59-
timeout-minutes: 60
60+
timeout-minutes: 75
6061
strategy:
6162
fail-fast: false
6263
matrix:
@@ -75,7 +76,7 @@ jobs:
7576
fetch-depth: 0
7677
submodules: recursive
7778
- name: Cache Docker Volumes
78-
uses: actions/cache@v2
79+
uses: actions/cache@v3
7980
with:
8081
path: .docker
8182
key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
@@ -87,12 +88,18 @@ jobs:
8788
- name: Setup Archery
8889
run: pip install -e dev/archery[docker]
8990
- name: Execute Docker Build
91+
env:
92+
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
93+
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
9094
run: |
9195
sudo sysctl -w kernel.core_pattern="core.%e.%p"
9296
ulimit -c unlimited
9397
archery docker run ${{ matrix.image }}
9498
- name: Docker Push
9599
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
100+
env:
101+
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
102+
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
96103
continue-on-error: true
97104
run: archery docker push ${{ matrix.image }}
98105

@@ -101,8 +108,6 @@ jobs:
101108
runs-on: ubuntu-latest
102109
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
103110
timeout-minutes: 45
104-
strategy:
105-
fail-fast: false
106111
steps:
107112
- name: Checkout Arrow
108113
uses: actions/checkout@v3
@@ -122,9 +127,7 @@ jobs:
122127
name: AMD64 macOS 11 C++
123128
runs-on: macos-latest
124129
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
125-
timeout-minutes: 60
126-
strategy:
127-
fail-fast: false
130+
timeout-minutes: 75
128131
env:
129132
ARROW_BUILD_TESTS: ON
130133
ARROW_DATASET: ON
@@ -146,8 +149,7 @@ jobs:
146149
ARROW_WITH_SNAPPY: ON
147150
ARROW_WITH_ZLIB: ON
148151
ARROW_WITH_ZSTD: ON
149-
# System Abseil installed by Homebrew uses C++ 17
150-
CMAKE_CXX_STANDARD: 17
152+
GTest_SOURCE: BUNDLED
151153
steps:
152154
- name: Checkout Arrow
153155
uses: actions/checkout@v3
@@ -174,13 +176,14 @@ jobs:
174176
run: |
175177
echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)"
176178
- name: Cache ccache
177-
uses: actions/cache@v2
179+
uses: actions/cache@v3
178180
with:
179181
path: ${{ steps.ccache-info.outputs.cache-dir }}
180182
key: cpp-ccache-macos-${{ hashFiles('cpp/**') }}
181183
restore-keys: cpp-ccache-macos-
182184
- name: Build
183-
run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
185+
run: |
186+
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
184187
- name: Test
185188
shell: bash
186189
run: |
@@ -257,7 +260,7 @@ jobs:
257260
- name: Install ccache
258261
shell: bash
259262
run: |
260-
ci/scripts/install_ccache.sh 4.6.2 /usr
263+
ci/scripts/install_ccache.sh 4.6.3 /usr
261264
- name: Setup ccache
262265
shell: bash
263266
run: |
@@ -268,11 +271,14 @@ jobs:
268271
run: |
269272
echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)"
270273
- name: Cache ccache
271-
uses: actions/cache@v2
274+
uses: actions/cache@v3
272275
with:
273276
path: ${{ steps.ccache-info.outputs.cache-dir }}
274-
key: cpp-ccache-windows-${{ hashFiles('cpp/**') }}
275-
restore-keys: cpp-ccache-windows-
277+
key: cpp-ccache-windows-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }}
278+
restore-keys: cpp-ccache-windows-${{ env.CACHE_VERSION }}-
279+
env:
280+
# We can invalidate the current cache by updating this.
281+
CACHE_VERSION: "2022-09-13"
276282
- name: Build
277283
shell: cmd
278284
run: |
@@ -351,7 +357,7 @@ jobs:
351357
shell: msys2 {0}
352358
run: ci/scripts/msys2_setup.sh cpp
353359
- name: Cache ccache
354-
uses: actions/cache@v2
360+
uses: actions/cache@v3
355361
with:
356362
path: ccache
357363
key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }}
@@ -372,23 +378,13 @@ jobs:
372378
--output-document /usr/local/bin/minio.exe \
373379
https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z
374380
chmod +x /usr/local/bin/minio.exe
375-
- name: Cache Python wheels
376-
uses: actions/cache@v2
377-
with:
378-
path: "${{ env.PIP_CACHE_DIR }}"
379-
key: cpp-wheels-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('ci/scripts/install_gcs_testbench.sh') }}
380-
restore-keys: cpp-wheels-mingw${{ matrix.mingw-n-bits }}-
381381
- name: Install Google Cloud Storage Testbench
382-
shell: msys2 {0}
382+
shell: bash
383383
run: |
384384
ci/scripts/install_gcs_testbench.sh default
385+
echo "PYTHON_BIN_DIR=$(cygpath --windows $(dirname $(which python3.exe)))" >> $GITHUB_ENV
385386
- name: Test
386387
shell: msys2 {0}
387388
run: |
388-
python_version=$(python3 -c "import sys; print('.'.join(map(str, sys.version_info[0:2])))")
389-
export PYTHONHOME="$(cygpath --windows ${MINGW_PREFIX})\lib\python${python_version}"
390-
PYTHONPATH="${PYTHONHOME}"
391-
PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\lib-dynload"
392-
PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\site-packages"
393-
export PYTHONPATH
389+
PATH="$(cygpath --unix ${PYTHON_BIN_DIR}):${PATH}"
394390
ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"

‎.github/workflows/csharp.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ concurrency:
3333
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
3434
cancel-in-progress: true
3535

36+
permissions:
37+
contents: read
38+
3639
jobs:
3740

3841
ubuntu:
@@ -46,7 +49,7 @@ jobs:
4649
dotnet: ['6.0.x']
4750
steps:
4851
- name: Install C#
49-
uses: actions/setup-dotnet@v1
52+
uses: actions/setup-dotnet@v2
5053
with:
5154
dotnet-version: ${{ matrix.dotnet }}
5255
- name: Checkout Arrow
@@ -74,7 +77,7 @@ jobs:
7477
dotnet: ['6.0.x']
7578
steps:
7679
- name: Install C#
77-
uses: actions/setup-dotnet@v1
80+
uses: actions/setup-dotnet@v2
7881
with:
7982
dotnet-version: ${{ matrix.dotnet }}
8083
- name: Checkout Arrow
@@ -101,7 +104,7 @@ jobs:
101104
dotnet: ['6.0.x']
102105
steps:
103106
- name: Install C#
104-
uses: actions/setup-dotnet@v1
107+
uses: actions/setup-dotnet@v2
105108
with:
106109
dotnet-version: ${{ matrix.dotnet }}
107110
- name: Checkout Arrow

‎.github/workflows/dev.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,8 @@ concurrency:
2626
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
2727
cancel-in-progress: true
2828

29-
env:
30-
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
31-
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
29+
permissions:
30+
contents: read
3231

3332
jobs:
3433

@@ -48,12 +47,18 @@ jobs:
4847
- name: Setup Archery
4948
run: pip install -e dev/archery[docker]
5049
- name: Execute Docker Build
50+
env:
51+
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
52+
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
5153
run: |
5254
sudo sysctl -w kernel.core_pattern="core.%e.%p"
5355
ulimit -c unlimited
5456
archery docker run -e GITHUB_ACTIONS=true ubuntu-lint
5557
- name: Docker Push
5658
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
59+
env:
60+
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
61+
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
5762
continue-on-error: true
5863
run: archery docker push ubuntu-lint
5964

‎.github/workflows/dev_pr.yml‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,14 @@ on:
2929
- edited
3030
- synchronize
3131

32-
# NOTE: not using the "cancel-in-progress" feature here as the group key
33-
# does not have enough information for linking it to a particular PR
32+
concurrency:
33+
group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.number }}
34+
cancel-in-progress: true
3435

36+
permissions:
37+
contents: read
38+
pull-requests: write
39+
3540
jobs:
3641
process:
3742
name: Process
@@ -41,9 +46,8 @@ jobs:
4146

4247
- name: Comment JIRA link
4348
if: |
44-
github.event_name == 'pull_request_target' &&
45-
(github.event.action == 'opened' ||
46-
github.event.action == 'edited')
49+
(github.event.action == 'opened' ||
50+
github.event.action == 'edited')
4751
uses: actions/github-script@v3
4852
with:
4953
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -53,9 +57,8 @@ jobs:
5357
5458
- name: Check title
5559
if: |
56-
github.event_name == 'pull_request_target' &&
57-
(github.event.action == 'opened' ||
58-
github.event.action == 'edited')
60+
(github.event.action == 'opened' ||
61+
github.event.action == 'edited')
5962
uses: actions/github-script@v3
6063
with:
6164
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -65,9 +68,8 @@ jobs:
6568
6669
- name: Check Jira Issue
6770
if: |
68-
github.event_name == 'pull_request_target' &&
69-
(github.event.action == 'opened' ||
70-
github.event.action == 'edited')
71+
(github.event.action == 'opened' ||
72+
github.event.action == 'edited')
7173
uses: actions/github-script@v3
7274
with:
7375
debug: true
@@ -78,10 +80,9 @@ jobs:
7880
7981
- name: Assign GitHub labels
8082
if: |
81-
github.event_name == 'pull_request_target' &&
82-
(github.event.action == 'opened' ||
83-
github.event.action == 'synchronize')
84-
uses: actions/[email protected]
83+
(github.event.action == 'opened' ||
84+
github.event.action == 'synchronize')
85+
uses: actions/labeler@v4
8586
with:
8687
repo-token: ${{ secrets.GITHUB_TOKEN }}
8788
configuration-path: .github/workflows/dev_pr/labeler.yml

0 commit comments

Comments
 (0)