Skip to content

Commit 91efbc7

Browse files
committed
Merge remote-tracking branch 'apache/main' into apacheGH-44950
2 parents 8001188 + ffcc117 commit 91efbc7

27 files changed

Lines changed: 1442 additions & 253 deletions

‎.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ CMAKE_VERSION=3.25.0
5858
CUDA=11.2.2
5959
DASK=latest
6060
DOTNET=8.0
61-
GCC_VERSION=""
61+
GCC=
6262
HDFS=3.2.1
6363
JDK=11
6464
KARTOTHEK=latest

‎.github/workflows/csharp.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
dotnet: ['8.0.x']
5555
steps:
5656
- name: Install C#
57-
uses: actions/setup-dotnet@v4.1.0
57+
uses: actions/setup-dotnet@v4.2.0
5858
with:
5959
dotnet-version: ${{ matrix.dotnet }}
6060
- name: Setup Python
@@ -86,7 +86,7 @@ jobs:
8686
dotnet: ['8.0.x']
8787
steps:
8888
- name: Install C#
89-
uses: actions/setup-dotnet@v4.1.0
89+
uses: actions/setup-dotnet@v4.2.0
9090
with:
9191
dotnet-version: ${{ matrix.dotnet }}
9292
- name: Checkout Arrow
@@ -113,7 +113,7 @@ jobs:
113113
dotnet: ['8.0.x']
114114
steps:
115115
- name: Install C#
116-
uses: actions/setup-dotnet@v4.1.0
116+
uses: actions/setup-dotnet@v4.2.0
117117
with:
118118
dotnet-version: ${{ matrix.dotnet }}
119119
- name: Setup Python

‎.github/workflows/dev.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ jobs:
122122
with:
123123
ruby-version: ruby
124124
- name: Install .NET
125-
uses: actions/setup-dotnet@3e891b0cb619bf60e2c25674b222b8940e2c1c25 # v4.1.0
125+
uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2.0
126126
with:
127127
dotnet-version: '8.0.x'
128128
- name: Install Dependencies
Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
ARG arch=amd64
19+
FROM ${arch}/debian:experimental
20+
ARG arch
21+
22+
ENV DEBIAN_FRONTEND noninteractive
23+
24+
ARG gcc
25+
ARG llvm
26+
RUN if [ -n "${gcc}" ]; then \
27+
gcc_package_suffix="-${gcc}"; \
28+
else \
29+
gcc_package_suffix=""; \
30+
fi && \
31+
if [ -n "${llvm}" ]; then \
32+
llvm_package_suffix="-${llvm}"; \
33+
else \
34+
llvm_package_suffix=""; \
35+
fi && \
36+
apt-get update -y -q && \
37+
apt-get install -y -q --no-install-recommends \
38+
autoconf \
39+
ccache \
40+
cmake \
41+
curl \
42+
g++ \
43+
gcc \
44+
gdb \
45+
git \
46+
libbenchmark-dev \
47+
libboost-filesystem-dev \
48+
libboost-system-dev \
49+
libbrotli-dev \
50+
libbz2-dev \
51+
libc-ares-dev \
52+
libcurl4-openssl-dev \
53+
libgflags-dev \
54+
libgmock-dev \
55+
libgoogle-glog-dev \
56+
libgrpc++-dev \
57+
libidn2-dev \
58+
libkrb5-dev \
59+
libldap-dev \
60+
liblz4-dev \
61+
libnghttp2-dev \
62+
libprotobuf-dev \
63+
libprotoc-dev \
64+
libpsl-dev \
65+
libre2-dev \
66+
librtmp-dev \
67+
libsnappy-dev \
68+
libsqlite3-dev \
69+
libssh-dev \
70+
libssh2-1-dev \
71+
libssl-dev \
72+
libthrift-dev \
73+
libutf8proc-dev \
74+
libxml2-dev \
75+
libxsimd-dev \
76+
libzstd-dev \
77+
make \
78+
ninja-build \
79+
nlohmann-json3-dev \
80+
npm \
81+
opentelemetry-cpp-dev \
82+
pkg-config \
83+
protobuf-compiler-grpc \
84+
python3-dev \
85+
python3-pip \
86+
python3-venv \
87+
rapidjson-dev \
88+
rsync \
89+
tzdata \
90+
zlib1g-dev && \
91+
apt-get install -y -q --no-install-recommends -t experimental \
92+
clang${llvm_package_suffix} \
93+
g++${gcc_package_suffix} \
94+
gcc${gcc_package_suffix} \
95+
llvm${llvm_package_suffix}-dev && \
96+
apt-get clean && \
97+
rm -rf /var/lib/apt/lists/*
98+
99+
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
100+
RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
101+
102+
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
103+
RUN /arrow/ci/scripts/install_gcs_testbench.sh default
104+
105+
COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
106+
RUN /arrow/ci/scripts/install_azurite.sh
107+
108+
COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
109+
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
110+
111+
# Prioritize system packages and local installation.
112+
ENV ARROW_ACERO=ON \
113+
ARROW_AZURE=ON \
114+
ARROW_BUILD_TESTS=ON \
115+
ARROW_DATASET=ON \
116+
ARROW_DEPENDENCY_SOURCE=SYSTEM \
117+
ARROW_DATASET=ON \
118+
ARROW_FLIGHT=ON \
119+
ARROW_FLIGHT_SQL=ON \
120+
ARROW_GANDIVA=ON \
121+
ARROW_GCS=ON \
122+
ARROW_HOME=/usr/local \
123+
ARROW_JEMALLOC=ON \
124+
ARROW_ORC=ON \
125+
ARROW_PARQUET=ON \
126+
ARROW_S3=ON \
127+
ARROW_SUBSTRAIT=ON \
128+
ARROW_USE_CCACHE=ON \
129+
ARROW_WITH_BROTLI=ON \
130+
ARROW_WITH_BZ2=ON \
131+
ARROW_WITH_LZ4=ON \
132+
ARROW_WITH_OPENTELEMETRY=ON \
133+
ARROW_WITH_SNAPPY=ON \
134+
ARROW_WITH_ZLIB=ON \
135+
ARROW_WITH_ZSTD=ON \
136+
AWSSDK_SOURCE=BUNDLED \
137+
Azure_SOURCE=BUNDLED \
138+
CC=gcc${gcc:+-${gcc}} \
139+
CXX=g++${gcc:+-${gcc}} \
140+
google_cloud_cpp_storage_SOURCE=BUNDLED \
141+
ORC_SOURCE=BUNDLED \
142+
PATH=/usr/lib/ccache/:$PATH \
143+
PYTHON=python3

‎ci/docker/linux-apt-r.dockerfile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,10 @@ RUN apt-get update -y && \
6565
apt-get clean && \
6666
rm -rf /var/lib/apt/lists/*
6767

68-
ARG gcc_version=""
69-
RUN if [ "${gcc_version}" != "" ]; then \
70-
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
71-
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
68+
ARG gcc=""
69+
RUN if [ "${gcc}" != "" ]; then \
70+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \
71+
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \
7272
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 && \
7373
update-alternatives --set cc /usr/bin/gcc && \
7474
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 && \

‎ci/docker/ubuntu-22.04-cpp.dockerfile‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -136,32 +136,27 @@ RUN cd ~ && git clone https://github.com/emscripten-core/emsdk.git && \
136136
echo "Installed emsdk to:" ~/emsdk
137137

138138

139-
ARG gcc_version=""
140-
RUN if [ "${gcc_version}" = "" ]; then \
139+
ARG gcc=""
140+
RUN if [ "${gcc}" = "" ]; then \
141141
apt-get update -y -q && \
142142
apt-get install -y -q --no-install-recommends \
143143
g++ \
144144
gcc; \
145145
else \
146-
if [ "${gcc_version}" -gt "12" ]; then \
147-
apt-get update -y -q && \
148-
apt-get install -y -q --no-install-recommends software-properties-common && \
149-
add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
150-
fi; \
151146
apt-get update -y -q && \
152147
apt-get install -y -q --no-install-recommends \
153-
g++-${gcc_version} \
154-
gcc-${gcc_version} && \
155-
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
156-
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
148+
g++-${gcc} \
149+
gcc-${gcc} && \
150+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \
151+
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \
157152
update-alternatives --install \
158153
/usr/bin/$(uname --machine)-linux-gnu-gcc \
159154
$(uname --machine)-linux-gnu-gcc \
160-
/usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc_version} 100 && \
155+
/usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc} 100 && \
161156
update-alternatives --install \
162157
/usr/bin/$(uname --machine)-linux-gnu-g++ \
163158
$(uname --machine)-linux-gnu-g++ \
164-
/usr/bin/$(uname --machine)-linux-gnu-g++-${gcc_version} 100 && \
159+
/usr/bin/$(uname --machine)-linux-gnu-g++-${gcc} 100 && \
165160
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
166161
update-alternatives --set cc /usr/bin/gcc && \
167162
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \

‎ci/docker/ubuntu-24.04-cpp.dockerfile‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -128,32 +128,27 @@ RUN apt-get update -y -q && \
128128
apt-get clean && \
129129
rm -rf /var/lib/apt/lists*
130130

131-
ARG gcc_version=""
132-
RUN if [ "${gcc_version}" = "" ]; then \
131+
ARG gcc=""
132+
RUN if [ "${gcc}" = "" ]; then \
133133
apt-get update -y -q && \
134134
apt-get install -y -q --no-install-recommends \
135135
g++ \
136136
gcc; \
137137
else \
138-
if [ "${gcc_version}" -gt "14" ]; then \
139-
apt-get update -y -q && \
140-
apt-get install -y -q --no-install-recommends software-properties-common && \
141-
add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
142-
fi; \
143138
apt-get update -y -q && \
144139
apt-get install -y -q --no-install-recommends \
145-
g++-${gcc_version} \
146-
gcc-${gcc_version} && \
147-
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
148-
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
140+
g++-${gcc} \
141+
gcc-${gcc} && \
142+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \
143+
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \
149144
update-alternatives --install \
150145
/usr/bin/$(uname --machine)-linux-gnu-gcc \
151146
$(uname --machine)-linux-gnu-gcc \
152-
/usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc_version} 100 && \
147+
/usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc} 100 && \
153148
update-alternatives --install \
154149
/usr/bin/$(uname --machine)-linux-gnu-g++ \
155150
$(uname --machine)-linux-gnu-g++ \
156-
/usr/bin/$(uname --machine)-linux-gnu-g++-${gcc_version} 100 && \
151+
/usr/bin/$(uname --machine)-linux-gnu-g++-${gcc} 100 && \
157152
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
158153
update-alternatives --set cc /usr/bin/gcc && \
159154
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \

‎cpp/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ endif()
9292
string(TOLOWER ${CMAKE_BUILD_TYPE} LOWERCASE_BUILD_TYPE)
9393
string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
9494

95-
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
95+
list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
9696

9797
# this must be included before the project() command, because of the way
9898
# vcpkg (ab)uses CMAKE_TOOLCHAIN_FILE to inject its logic into CMake

‎cpp/src/arrow/acero/swiss_join.cc‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2462,7 +2462,6 @@ class SwissJoin : public HashJoinImpl {
24622462
output_batch_callback_ = std::move(output_batch_callback);
24632463
finished_callback_ = std::move(finished_callback);
24642464

2465-
hash_table_ready_.store(false);
24662465
cancelled_.store(false);
24672466
{
24682467
std::lock_guard<std::mutex> lock(state_mutex_);
@@ -2474,7 +2473,6 @@ class SwissJoin : public HashJoinImpl {
24742473
local_states_.resize(num_threads_);
24752474
for (int i = 0; i < num_threads_; ++i) {
24762475
RETURN_NOT_OK(local_states_[i].stack.Init(pool_, kTempStackUsage));
2477-
local_states_[i].hash_table_ready = false;
24782476
local_states_[i].num_output_batches = 0;
24792477
local_states_[i].materialize.Init(pool_, proj_map_left, proj_map_right);
24802478
}
@@ -2520,11 +2518,6 @@ class SwissJoin : public HashJoinImpl {
25202518
return status();
25212519
}
25222520

2523-
if (!local_states_[thread_index].hash_table_ready) {
2524-
local_states_[thread_index].hash_table_ready = hash_table_ready_.load();
2525-
}
2526-
ARROW_DCHECK(local_states_[thread_index].hash_table_ready);
2527-
25282521
ExecBatch keypayload_batch;
25292522
ARROW_ASSIGN_OR_RAISE(keypayload_batch, KeyPayloadFromInput(/*side=*/0, &batch));
25302523
arrow::util::TempVectorStack* temp_stack = &local_states_[thread_index].stack;
@@ -2689,7 +2682,6 @@ class SwissJoin : public HashJoinImpl {
26892682
hash_table_.payloads(),
26902683
hash_table_.key_to_payload() == nullptr);
26912684
}
2692-
hash_table_ready_.store(true);
26932685

26942686
residual_filter_.OnBuildFinished();
26952687

@@ -2910,7 +2902,6 @@ class SwissJoin : public HashJoinImpl {
29102902
JoinResultMaterialize materialize;
29112903
std::vector<KeyColumnArray> temp_column_arrays;
29122904
int64_t num_output_batches;
2913-
bool hash_table_ready;
29142905
};
29152906
std::vector<ThreadLocalState> local_states_;
29162907

@@ -2927,7 +2918,6 @@ class SwissJoin : public HashJoinImpl {
29272918
// The other flags that follow them, protected by mutex, will be queried or
29282919
// updated only a fixed number of times during entire join processing.
29292920
//
2930-
std::atomic<bool> hash_table_ready_;
29312921
std::atomic<bool> cancelled_;
29322922

29332923
// Mutex protecting state flags.

0 commit comments

Comments
 (0)