Skip to content

Commit b319661

Browse files
committed
ARROW-17451: [CI][Java] Use manylinux2014 image for JNI
Because our official .jar packages are built in manylinux2014 image.
1 parent 6e8f0e4 commit b319661

9 files changed

Lines changed: 56 additions & 60 deletions

File tree

‎.github/workflows/java_jni.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ env:
4747
jobs:
4848

4949
docker:
50-
name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset)
50+
name: AMD64 manylinux2014 Java JNI
5151
runs-on: ubuntu-latest
5252
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
5353
timeout-minutes: 90
@@ -70,14 +70,14 @@ jobs:
7070
- name: Setup Archery
7171
run: pip install -e dev/archery[docker]
7272
- name: Execute Docker Build
73-
run: archery docker run debian-java-jni
73+
run: archery docker run java-jni-manylinux-2014
7474
- name: Docker Push
7575
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
7676
continue-on-error: true
77-
run: archery docker push debian-java-jni
77+
run: archery docker push java-jni-manylinux-2014
7878

7979
docker_integration_python:
80-
name: AMD64 Debian 9 Java C Data Interface Integration
80+
name: AMD64 Conda Java C Data Interface Integration
8181
runs-on: ubuntu-latest
8282
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
8383
timeout-minutes: 90

‎ci/docker/java-jni-manylinux-201x.dockerfile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,12 @@ RUN vcpkg install \
3333

3434
# Install Java
3535
ARG java=1.8.0
36-
RUN yum install -y java-$java-openjdk-devel && yum clean all
36+
RUN yum install -y java-$java-openjdk-devel rh-maven35 && yum clean all
3737
ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/
38+
39+
# For ci/scripts/java_*.sh
40+
ENV ARROW_GANDIVA_JAVA=ON \
41+
ARROW_HOME=/tmp/local \
42+
ARROW_JAVA_CDATA=ON \
43+
ARROW_JNI=ON \
44+
ARROW_PLASMA=ON

‎ci/scripts/java_build.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@
1818

1919
set -ex
2020

21+
if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then
22+
exit
23+
fi
24+
2125
arrow_dir=${1}
2226
source_dir=${1}/java
2327
build_dir=${2}
24-
cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug}
2528
java_jni_dist_dir=${3}
2629

2730
: ${BUILD_DOCS_JAVA:=OFF}
@@ -80,7 +83,7 @@ if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
8083
fi
8184

8285
if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
83-
${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install
86+
${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni install
8487
fi
8588

8689
if [ "${ARROW_PLASMA}" = "ON" ]; then

‎ci/scripts/java_jni_macos_build.sh‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ echo "=== Clear output directories and leftovers ==="
2929
rm -rf ${build_dir}
3030

3131
echo "=== Building Arrow C++ libraries ==="
32+
install_dir=${build_dir}/cpp-install
3233
: ${ARROW_BUILD_TESTS:=OFF}
3334
: ${ARROW_DATASET:=ON}
3435
: ${ARROW_FILESYSTEM:=ON}
@@ -80,7 +81,7 @@ cmake \
8081
-DAWSSDK_SOURCE=BUNDLED \
8182
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
8283
-DCMAKE_INSTALL_LIBDIR=lib \
83-
-DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
84+
-DCMAKE_INSTALL_PREFIX=${install_dir} \
8485
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
8586
-DPARQUET_BUILD_EXAMPLES=OFF \
8687
-DPARQUET_BUILD_EXECUTABLES=OFF \
@@ -105,9 +106,10 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \
105106

106107
echo "=== Copying libraries to the distribution folder ==="
107108
mkdir -p "${dist_dir}"
108-
cp -L ${build_dir}/cpp/lib/libgandiva_jni.dylib ${dist_dir}
109-
cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.dylib ${dist_dir}
110-
cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.dylib ${dist_dir}
109+
cp -L ${install_dir}/lib/libarrow_dataset_jni.dylib ${dist_dir}
110+
cp -L ${install_dir}/lib/libarrow_orc_jni.dylib ${dist_dir}
111+
cp -L ${install_dir}/lib/libgandiva_jni.dylib ${dist_dir}
112+
cp -L ${build_dir}/cpp/*/libplasma_java.dylib ${dist_dir}
111113

112114
echo "=== Checking shared dependencies for libraries ==="
113115

@@ -126,5 +128,6 @@ archery linking check-dependencies \
126128
libarrow_cdata_jni.dylib \
127129
libarrow_dataset_jni.dylib \
128130
libarrow_orc_jni.dylib \
129-
libgandiva_jni.dylib
131+
libgandiva_jni.dylib \
132+
libplasma_java.dylib
130133
popd

‎ci/scripts/java_jni_manylinux_build.sh‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ cmake \
9191
-DARROW_ZSTD_USE_SHARED=OFF \
9292
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
9393
-DCMAKE_INSTALL_LIBDIR=lib \
94-
-DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
94+
-DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \
9595
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
9696
-DORC_SOURCE=BUNDLED \
9797
-DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
@@ -128,9 +128,10 @@ ${arrow_dir}/ci/scripts/java_jni_build.sh \
128128

129129

130130
echo "=== Copying libraries to the distribution folder ==="
131-
cp -L ${build_dir}/cpp/lib/libgandiva_jni.so ${dist_dir}
132-
cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.so ${dist_dir}
133-
cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.so ${dist_dir}
131+
cp -L ${ARROW_HOME}/lib/libarrow_dataset_jni.so ${dist_dir}
132+
cp -L ${ARROW_HOME}/lib/libarrow_orc_jni.so ${dist_dir}
133+
cp -L ${ARROW_HOME}/lib/libgandiva_jni.so ${dist_dir}
134+
cp -L ${build_dir}/cpp/*/libplasma_java.so ${dist_dir}
134135

135136
echo "=== Checking shared dependencies for libraries ==="
136137

@@ -149,5 +150,6 @@ archery linking check-dependencies \
149150
libarrow_cdata_jni.so \
150151
libarrow_dataset_jni.so \
151152
libarrow_orc_jni.so \
152-
libgandiva_jni.so
153+
libgandiva_jni.so \
154+
libplasma_java.so
153155
popd

‎ci/scripts/java_test.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,12 @@
1818

1919
set -ex
2020

21+
if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then
22+
exit
23+
fi
24+
2125
arrow_dir=${1}
2226
source_dir=${1}/java
23-
cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
2427
java_jni_dist_dir=${3}
2528

2629
# For JNI and Plasma tests
@@ -36,7 +39,7 @@ pushd ${source_dir}
3639
${mvn} test
3740

3841
if [ "${ARROW_JNI}" = "ON" ]; then
39-
${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir}
42+
${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${java_jni_dist_dir}
4043
fi
4144

4245
if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
@@ -46,7 +49,7 @@ fi
4649
if [ "${ARROW_PLASMA}" = "ON" ]; then
4750
pushd ${source_dir}/plasma
4851
java -cp target/test-classes:target/classes \
49-
-Djava.library.path=${cpp_build_dir} \
52+
-Djava.library.path=${java_jni_dist_dir} \
5053
org.apache.arrow.plasma.PlasmaClientTest
5154
popd
5255
fi

‎cpp/src/plasma/CMakeLists.txt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -185,14 +185,14 @@ if(ARROW_PLASMA_JAVA_CLIENT)
185185

186186
if(APPLE)
187187
target_link_libraries(plasma_java
188-
plasma_shared
189-
${PLASMA_LINK_LIBS}
188+
plasma_static
189+
${PLASMA_STATIC_LINK_LIBS}
190190
"-undefined dynamic_lookup"
191191
${PTHREAD_LIBRARY})
192-
else(APPLE)
193-
target_link_libraries(plasma_java plasma_shared ${PLASMA_LINK_LIBS}
192+
else()
193+
target_link_libraries(plasma_java plasma_static ${PLASMA_STATIC_LINK_LIBS}
194194
${PTHREAD_LIBRARY})
195-
endif(APPLE)
195+
endif()
196196
endif()
197197
#
198198
# Unit tests

‎dev/tasks/java-jars/github.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@ jobs:
2828
{{ macros.github_checkout_arrow()|indent }}
2929
{{ macros.github_install_archery()|indent }}
3030
- name: Build C++ Libs
31-
run: archery docker run java-jni-manylinux-2014
31+
run: |
32+
archery docker run \
33+
-e ARROW_JAVA_BUILD=OFF \
34+
-e ARROW_JAVA_TEST=OFF \
35+
java-jni-manylinux-2014
3236
- name: Compress into single artifact to keep directory structure
3337
run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/
3438
- name: Upload Artifacts
@@ -100,11 +104,13 @@ jobs:
100104
test -f arrow/java-dist/libarrow_cdata_jni.dylib
101105
test -f arrow/java-dist/libarrow_dataset_jni.dylib
102106
test -f arrow/java-dist/libgandiva_jni.dylib
107+
test -f arrow/java-dist/libplasma_jni.dylib
103108
test -f arrow/java-dist/libarrow_orc_jni.dylib
104109
test -f arrow/java-dist/libarrow_cdata_jni.so
105110
test -f arrow/java-dist/libarrow_dataset_jni.so
106111
test -f arrow/java-dist/libarrow_orc_jni.so
107112
test -f arrow/java-dist/libgandiva_jni.so
113+
test -f arrow/java-dist/libplasma_jni.so
108114
- name: Build Bundled Jar
109115
run: |
110116
set -e

‎docker-compose.yml‎

Lines changed: 6 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,6 @@ x-hierarchy:
122122
- debian-go-cgo
123123
- debian-go-cgo-python
124124
- debian-java
125-
- debian-java-jni
126125
- debian-js
127126
- fedora-cpp:
128127
- fedora-python
@@ -999,10 +998,14 @@ services:
999998
<<: *ccache
1000999
volumes:
10011000
- .:/arrow:delegated
1001+
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
10021002
- ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
10031003
command:
1004-
["pip install -e /arrow/dev/archery &&
1005-
/arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"]
1004+
["pip install -e /arrow/dev/archery && \
1005+
/arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \
1006+
source /opt/rh/rh-maven35/enable && \
1007+
/arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \
1008+
/arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"]
10061009

10071010
############################## Integration #################################
10081011

@@ -1541,37 +1544,6 @@ services:
15411544
/arrow/ci/scripts/java_build.sh /arrow /build &&
15421545
/arrow/ci/scripts/java_test.sh /arrow /build"
15431546

1544-
debian-java-jni:
1545-
# Includes plasma test, jni for gandiva and orc, and C data interface.
1546-
# Usage:
1547-
# docker-compose build debian-java
1548-
# docker-compose build debian-java-jni
1549-
# docker-compose run debian-java-jni
1550-
image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
1551-
build:
1552-
context: .
1553-
dockerfile: ci/docker/linux-apt-jni.dockerfile
1554-
cache_from:
1555-
- ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
1556-
args:
1557-
llvm: ${LLVM}
1558-
shm_size: *shm-size
1559-
environment:
1560-
<<: *ccache
1561-
ARROW_BUILD_TESTS: "OFF"
1562-
ARROW_S3: "OFF"
1563-
ARROW_SUBSTRAIT: "OFF"
1564-
volumes:
1565-
- .:/arrow:delegated
1566-
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
1567-
- ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
1568-
command:
1569-
/bin/bash -c "
1570-
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
1571-
/arrow/ci/scripts/java_jni_build.sh /arrow /build /tmp/java_dist &&
1572-
/arrow/ci/scripts/java_build.sh /arrow /build /tmp/java_dist &&
1573-
/arrow/ci/scripts/java_test.sh /arrow /build /tmp/java_dist"
1574-
15751547
oracle-java:
15761548
# Usage:
15771549
# docker-compose build oracle-java

0 commit comments

Comments
 (0)