Skip to content

Commit 9882df5

Browse files
authored
ARROW-17451: [CI][Java] Use manylinux2014 image for JNI (#13920)
Because our official .jar packages are built in manylinux2014 image. Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
1 parent 0600bb2 commit 9882df5

9 files changed

Lines changed: 87 additions & 63 deletions

File tree

‎.github/workflows/java_jni.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ env:
4747
jobs:
4848

4949
docker:
50-
name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset)
50+
name: AMD64 manylinux2014 Java JNI
5151
runs-on: ubuntu-latest
5252
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
5353
timeout-minutes: 90
@@ -61,23 +61,23 @@ jobs:
6161
uses: actions/cache@v2
6262
with:
6363
path: .docker
64-
key: maven-${{ hashFiles('java/**') }}
65-
restore-keys: maven-
64+
key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }}
65+
restore-keys: java-jni-manylinux-2014-
6666
- name: Setup Python
6767
uses: actions/setup-python@v4
6868
with:
6969
python-version: 3.8
7070
- name: Setup Archery
7171
run: pip install -e dev/archery[docker]
7272
- name: Execute Docker Build
73-
run: archery docker run debian-java-jni
73+
run: archery docker run java-jni-manylinux-2014
7474
- name: Docker Push
7575
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
7676
continue-on-error: true
77-
run: archery docker push debian-java-jni
77+
run: archery docker push java-jni-manylinux-2014
7878

7979
docker_integration_python:
80-
name: AMD64 Debian 9 Java C Data Interface Integration
80+
name: AMD64 Conda Java C Data Interface Integration
8181
runs-on: ubuntu-latest
8282
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
8383
timeout-minutes: 90

‎ci/docker/java-jni-manylinux-201x.dockerfile‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,13 @@ RUN vcpkg install \
3333

3434
# Install Java
3535
ARG java=1.8.0
36-
RUN yum install -y java-$java-openjdk-devel && yum clean all
36+
RUN yum install -y java-$java-openjdk-devel rh-maven35 && yum clean all
3737
ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/
38+
39+
# For ci/scripts/java_*.sh
40+
ENV ARROW_GANDIVA_JAVA=ON \
41+
ARROW_HOME=/tmp/local \
42+
ARROW_JAVA_CDATA=ON \
43+
ARROW_JNI=ON \
44+
ARROW_PLASMA=ON \
45+
ARROW_USE_CCACHE=ON

‎ci/scripts/java_build.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@
1818

1919
set -ex
2020

21+
if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then
22+
exit
23+
fi
24+
2125
arrow_dir=${1}
2226
source_dir=${1}/java
2327
build_dir=${2}
24-
cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug}
2528
java_jni_dist_dir=${3}
2629

2730
: ${BUILD_DOCS_JAVA:=OFF}
@@ -85,7 +88,7 @@ if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
8588
fi
8689

8790
if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
88-
${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install
91+
${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni install
8992
fi
9093

9194
if [ "${ARROW_PLASMA}" = "ON" ]; then

‎ci/scripts/java_jni_macos_build.sh‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ echo "=== Clear output directories and leftovers ==="
2929
rm -rf ${build_dir}
3030

3131
echo "=== Building Arrow C++ libraries ==="
32+
install_dir=${build_dir}/cpp-install
3233
: ${ARROW_BUILD_TESTS:=OFF}
3334
: ${ARROW_DATASET:=ON}
3435
: ${ARROW_FILESYSTEM:=ON}
@@ -40,9 +41,15 @@ echo "=== Building Arrow C++ libraries ==="
4041
: ${ARROW_PLASMA:=ON}
4142
: ${ARROW_PYTHON:=OFF}
4243
: ${ARROW_S3:=ON}
44+
: ${ARROW_USE_CCACHE:=OFF}
4345
: ${CMAKE_BUILD_TYPE:=Release}
4446
: ${CMAKE_UNITY_BUILD:=ON}
4547

48+
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
49+
echo "=== ccache statistics before build ==="
50+
ccache -s
51+
fi
52+
4653
export ARROW_TEST_DATA="${arrow_dir}/testing/data"
4754
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
4855
export AWS_EC2_METADATA_DISABLED=TRUE
@@ -75,12 +82,13 @@ cmake \
7582
-DARROW_S3=${ARROW_S3} \
7683
-DARROW_SNAPPY_USE_SHARED=OFF \
7784
-DARROW_THRIFT_USE_SHARED=OFF \
85+
-DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
7886
-DARROW_UTF8PROC_USE_SHARED=OFF \
7987
-DARROW_ZSTD_USE_SHARED=OFF \
8088
-DAWSSDK_SOURCE=BUNDLED \
8189
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
8290
-DCMAKE_INSTALL_LIBDIR=lib \
83-
-DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
91+
-DCMAKE_INSTALL_PREFIX=${install_dir} \
8492
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
8593
-DPARQUET_BUILD_EXAMPLES=OFF \
8694
-DPARQUET_BUILD_EXECUTABLES=OFF \
@@ -102,12 +110,17 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \
102110
${build_dir} \
103111
${dist_dir}
104112

113+
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
114+
echo "=== ccache statistics after build ==="
115+
ccache -s
116+
fi
105117

106118
echo "=== Copying libraries to the distribution folder ==="
107119
mkdir -p "${dist_dir}"
108-
cp -L ${build_dir}/cpp/lib/libgandiva_jni.dylib ${dist_dir}
109-
cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.dylib ${dist_dir}
110-
cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.dylib ${dist_dir}
120+
cp -L ${install_dir}/lib/libarrow_dataset_jni.dylib ${dist_dir}
121+
cp -L ${install_dir}/lib/libarrow_orc_jni.dylib ${dist_dir}
122+
cp -L ${install_dir}/lib/libgandiva_jni.dylib ${dist_dir}
123+
cp -L ${build_dir}/cpp/*/libplasma_java.dylib ${dist_dir}
111124

112125
echo "=== Checking shared dependencies for libraries ==="
113126

@@ -122,9 +135,11 @@ archery linking check-dependencies \
122135
--allow libcurl \
123136
--allow libgandiva_jni \
124137
--allow libncurses \
138+
--allow libplasma_java \
125139
--allow libz \
126140
libarrow_cdata_jni.dylib \
127141
libarrow_dataset_jni.dylib \
128142
libarrow_orc_jni.dylib \
129-
libgandiva_jni.dylib
143+
libgandiva_jni.dylib \
144+
libplasma_java.dylib
130145
popd

‎ci/scripts/java_jni_manylinux_build.sh‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ echo "=== Building Arrow C++ libraries ==="
3232
devtoolset_version=$(rpm -qa "devtoolset-*-gcc" --queryformat %{VERSION} | \
3333
grep -o "^[0-9]*")
3434
devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
35+
: ${ARROW_BUILD_TESTS:=OFF}
3536
: ${ARROW_DATASET:=ON}
3637
: ${ARROW_GANDIVA:=ON}
3738
: ${ARROW_GANDIVA_JAVA:=ON}
@@ -44,14 +45,19 @@ devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/includ
4445
: ${ARROW_PLASMA_JAVA_CLIENT:=ON}
4546
: ${ARROW_PYTHON:=OFF}
4647
: ${ARROW_S3:=ON}
47-
: ${ARROW_BUILD_TESTS:=OFF}
48+
: ${ARROW_USE_CCACHE:=OFF}
4849
: ${CMAKE_BUILD_TYPE:=Release}
4950
: ${CMAKE_UNITY_BUILD:=ON}
5051
: ${VCPKG_ROOT:=/opt/vcpkg}
5152
: ${VCPKG_FEATURE_FLAGS:=-manifests}
5253
: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}
5354
: ${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-isystem;-lpthread}
5455

56+
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
57+
echo "=== ccache statistics before build ==="
58+
ccache -s
59+
fi
60+
5561
export ARROW_TEST_DATA="${arrow_dir}/testing/data"
5662
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
5763
export AWS_EC2_METADATA_DISABLED=TRUE
@@ -87,11 +93,12 @@ cmake \
8793
-DARROW_S3=${ARROW_S3} \
8894
-DARROW_SNAPPY_USE_SHARED=OFF \
8995
-DARROW_THRIFT_USE_SHARED=OFF \
96+
-DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
9097
-DARROW_UTF8PROC_USE_SHARED=OFF \
9198
-DARROW_ZSTD_USE_SHARED=OFF \
9299
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
93100
-DCMAKE_INSTALL_LIBDIR=lib \
94-
-DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
101+
-DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \
95102
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
96103
-DORC_SOURCE=BUNDLED \
97104
-DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
@@ -126,11 +133,17 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \
126133
${build_dir} \
127134
${dist_dir}
128135

136+
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
137+
echo "=== ccache statistics after build ==="
138+
ccache -s
139+
fi
140+
129141

130142
echo "=== Copying libraries to the distribution folder ==="
131-
cp -L ${build_dir}/cpp/lib/libgandiva_jni.so ${dist_dir}
132-
cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.so ${dist_dir}
133-
cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.so ${dist_dir}
143+
cp -L ${ARROW_HOME}/lib/libarrow_dataset_jni.so ${dist_dir}
144+
cp -L ${ARROW_HOME}/lib/libarrow_orc_jni.so ${dist_dir}
145+
cp -L ${ARROW_HOME}/lib/libgandiva_jni.so ${dist_dir}
146+
cp -L ${build_dir}/cpp/*/libplasma_java.so ${dist_dir}
134147

135148
echo "=== Checking shared dependencies for libraries ==="
136149

@@ -149,5 +162,6 @@ archery linking check-dependencies \
149162
libarrow_cdata_jni.so \
150163
libarrow_dataset_jni.so \
151164
libarrow_orc_jni.so \
152-
libgandiva_jni.so
165+
libgandiva_jni.so \
166+
libplasma_java.so
153167
popd

‎ci/scripts/java_test.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,12 @@
1818

1919
set -ex
2020

21+
if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then
22+
exit
23+
fi
24+
2125
arrow_dir=${1}
2226
source_dir=${1}/java
23-
cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
2427
java_jni_dist_dir=${3}
2528

2629
# For JNI and Plasma tests
@@ -36,7 +39,7 @@ pushd ${source_dir}
3639
${mvn} test
3740

3841
if [ "${ARROW_JNI}" = "ON" ]; then
39-
${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir}
42+
${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${java_jni_dist_dir}
4043
fi
4144

4245
if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
@@ -46,7 +49,7 @@ fi
4649
if [ "${ARROW_PLASMA}" = "ON" ]; then
4750
pushd ${source_dir}/plasma
4851
java -cp target/test-classes:target/classes \
49-
-Djava.library.path=${cpp_build_dir} \
52+
-Djava.library.path=${java_jni_dist_dir} \
5053
org.apache.arrow.plasma.PlasmaClientTest
5154
popd
5255
fi

‎cpp/src/plasma/CMakeLists.txt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -185,14 +185,14 @@ if(ARROW_PLASMA_JAVA_CLIENT)
185185

186186
if(APPLE)
187187
target_link_libraries(plasma_java
188-
plasma_shared
189-
${PLASMA_LINK_LIBS}
188+
plasma_static
189+
${PLASMA_STATIC_LINK_LIBS}
190190
"-undefined dynamic_lookup"
191191
${PTHREAD_LIBRARY})
192-
else(APPLE)
193-
target_link_libraries(plasma_java plasma_shared ${PLASMA_LINK_LIBS}
192+
else()
193+
target_link_libraries(plasma_java plasma_static ${PLASMA_STATIC_LINK_LIBS}
194194
${PTHREAD_LIBRARY})
195-
endif(APPLE)
195+
endif()
196196
endif()
197197
#
198198
# Unit tests

‎dev/tasks/java-jars/github.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@ jobs:
2828
{{ macros.github_checkout_arrow()|indent }}
2929
{{ macros.github_install_archery()|indent }}
3030
- name: Build C++ Libs
31-
run: archery docker run java-jni-manylinux-2014
31+
run: |
32+
archery docker run \
33+
-e ARROW_JAVA_BUILD=OFF \
34+
-e ARROW_JAVA_TEST=OFF \
35+
java-jni-manylinux-2014
3236
- name: Compress into single artifact to keep directory structure
3337
run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/
3438
- name: Upload Artifacts
@@ -61,6 +65,9 @@ jobs:
6165
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
6266
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
6367
brew uninstall aws-sdk-cpp
68+
- name: Setup ccache
69+
run: |
70+
arrow/ci/scripts/ccache_setup.sh
6471
- name: Build C++ Libs
6572
run: |
6673
set -e
@@ -100,11 +107,13 @@ jobs:
100107
test -f arrow/java-dist/libarrow_cdata_jni.dylib
101108
test -f arrow/java-dist/libarrow_dataset_jni.dylib
102109
test -f arrow/java-dist/libgandiva_jni.dylib
110+
test -f arrow/java-dist/libplasma_java.dylib
103111
test -f arrow/java-dist/libarrow_orc_jni.dylib
104112
test -f arrow/java-dist/libarrow_cdata_jni.so
105113
test -f arrow/java-dist/libarrow_dataset_jni.so
106114
test -f arrow/java-dist/libarrow_orc_jni.so
107115
test -f arrow/java-dist/libgandiva_jni.so
116+
test -f arrow/java-dist/libplasma_java.so
108117
- name: Build Bundled Jar
109118
run: |
110119
set -e

‎docker-compose.yml‎

Lines changed: 6 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,6 @@ x-hierarchy:
122122
- debian-go-cgo
123123
- debian-go-cgo-python
124124
- debian-java
125-
- debian-java-jni
126125
- debian-js
127126
- fedora-cpp:
128127
- fedora-python
@@ -999,10 +998,14 @@ services:
999998
<<: *ccache
1000999
volumes:
10011000
- .:/arrow:delegated
1001+
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
10021002
- ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
10031003
command:
1004-
["pip install -e /arrow/dev/archery &&
1005-
/arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"]
1004+
["pip install -e /arrow/dev/archery && \
1005+
/arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \
1006+
source /opt/rh/rh-maven35/enable && \
1007+
/arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \
1008+
/arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"]
10061009

10071010
############################## Integration #################################
10081011

@@ -1541,37 +1544,6 @@ services:
15411544
/arrow/ci/scripts/java_build.sh /arrow /build &&
15421545
/arrow/ci/scripts/java_test.sh /arrow /build"
15431546

1544-
debian-java-jni:
1545-
# Includes plasma test, jni for gandiva and orc, and C data interface.
1546-
# Usage:
1547-
# docker-compose build debian-java
1548-
# docker-compose build debian-java-jni
1549-
# docker-compose run debian-java-jni
1550-
image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
1551-
build:
1552-
context: .
1553-
dockerfile: ci/docker/linux-apt-jni.dockerfile
1554-
cache_from:
1555-
- ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
1556-
args:
1557-
llvm: ${LLVM}
1558-
shm_size: *shm-size
1559-
environment:
1560-
<<: *ccache
1561-
ARROW_BUILD_TESTS: "OFF"
1562-
ARROW_S3: "OFF"
1563-
ARROW_SUBSTRAIT: "OFF"
1564-
volumes:
1565-
- .:/arrow:delegated
1566-
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
1567-
- ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
1568-
command:
1569-
/bin/bash -c "
1570-
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
1571-
/arrow/ci/scripts/java_jni_build.sh /arrow /build /tmp/java_dist &&
1572-
/arrow/ci/scripts/java_build.sh /arrow /build /tmp/java_dist &&
1573-
/arrow/ci/scripts/java_test.sh /arrow /build /tmp/java_dist"
1574-
15751547
oracle-java:
15761548
# Usage:
15771549
# docker-compose build oracle-java

0 commit comments

Comments
 (0)