Skip to content

Commit 98b4a21

Browse files
[improvement](regression) Align regression framework arrow/netty with fe-core and target JDK17 (apache#64677)
## Proposed changes Align the regression-test framework's dependency versions with fe-core (`fe/pom.xml`): - `arrow.version`: `17.0.0` -> `19.0.0` - `netty-all`: `4.1.104.Final` -> `4.2.15.Final` - `maven.compiler.source/target`: `1.8` -> `17` (Arrow 18+ requires JDK 17, and the regression build already runs on JDK 17) This is a **dependency-alignment / JDK17-target change only**. It keeps the framework's Arrow Flight SQL JDBC driver and netty in lockstep with fe-core instead of drifting on older versions. ### Scope / risk - Only `regression-test/framework/pom.xml` is touched. - Raising `maven.compiler.target` to 17 affects the compiled output of the whole regression framework, so this needs full regression CI to validate, not just a local build. - It does **not** fix the intermittent `arrow_flight_sql_p0` close-time buffer leak — that has a separate root cause and will be handled in a follow-up. --------- Co-authored-by: Dongyang Li <[email protected]>
1 parent 7718451 commit 98b4a21

6 files changed

Lines changed: 45 additions & 18 deletions

File tree

‎regression-test/framework/pom.xml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,17 +69,18 @@ under the License.
6969
<doris.thrift.executable>${doris.home}/thirdparty/installed/bin/thrift</doris.thrift.executable>
7070
<doris.thrift.source>${doris.home}/gensrc/thrift</doris.thrift.source>
7171
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
72-
<maven.compiler.source>1.8</maven.compiler.source>
73-
<maven.compiler.target>1.8</maven.compiler.target>
72+
<maven.compiler.source>17</maven.compiler.source>
73+
<maven.compiler.target>17</maven.compiler.target>
7474
<revision>1.0-SNAPSHOT</revision>
7575
<project.scm.id>github</project.scm.id>
7676
<groovy.version>4.0.19</groovy.version>
7777
<antlr.version>4.9.3</antlr.version>
7878
<hadoop.version>2.8.0</hadoop.version>
7979
<aws-java-sdk-s3.version>1.11.95</aws-java-sdk-s3.version>
8080
<jackson.version>2.13.5</jackson.version>
81-
<!-- Arrow 18 only supports jdk17 -->
82-
<arrow.version>17.0.0</arrow.version>
81+
<!-- Aligned with fe (fe/pom.xml). Arrow 18+ requires jdk17; the regression
82+
framework now targets jdk17, so this matches fe-core. -->
83+
<arrow.version>19.0.0</arrow.version>
8384
</properties>
8485
<profiles>
8586
<profile>
@@ -488,7 +489,7 @@ under the License.
488489
<dependency>
489490
<groupId>io.netty</groupId>
490491
<artifactId>netty-all</artifactId>
491-
<version>4.1.104.Final</version>
492+
<version>4.2.15.Final</version>
492493
</dependency>
493494
<dependency>
494495
<groupId>org.awaitility</groupId>

‎regression-test/pipeline/cloud_p0/run.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,11 @@ run() {
7474
sed -i "s/oss-cn-hongkong.aliyuncs.com/oss-cn-hongkong-internal.aliyuncs.com/" "${teamcity_build_checkoutDir}"/docker/thirdparties/custom_settings.env
7575
if bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh --stop; then echo; fi
7676
if bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh -c kafka; then echo; else echo "ERROR: start kafka docker failed"; fi
77-
JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
77+
# Run the regression suite under JDK 17: the Arrow Flight SQL JDBC driver (arrow 19)
78+
# is Java 11+ bytecode and cannot be loaded by a JDK 8 runtime.
79+
JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*' | sed -n '1p')"
7880
export JAVA_HOME
81+
export PATH="${JAVA_HOME}/bin:${PATH}"
7982
if "${teamcity_build_checkoutDir}"/run-regression-test.sh \
8083
--teamcity \
8184
--run \

‎regression-test/pipeline/common/doris-utils.sh‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -985,9 +985,9 @@ prepare_java_udf() {
985985
# custom_lib相关的case需要在fe启动前把编译好的jar放到 $DORIS_HOME/fe/custom_lib/
986986
install_java
987987
install_maven
988-
OLD_JAVA_HOME=${JAVA_HOME}
989-
JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
990-
export JAVA_HOME
988+
# Build with the ambient JDK (>=17 in CI): the regression framework is compiled with it,
989+
# while run-regression-test.sh internally switches to JDK 8 only for the Java UDF module
990+
# (which must stay Java 8 for the BE embedded JVM).
991991
bash "${DORIS_HOME}"/../run-regression-test.sh --clean
992992
if bash "${DORIS_HOME}"/../run-regression-test.sh --compile ||
993993
bash "${DORIS_HOME}"/../run-regression-test.sh --compile ||
@@ -996,8 +996,6 @@ prepare_java_udf() {
996996
else
997997
echo "ERROR: failed to compile java udf"
998998
fi
999-
JAVA_HOME=${OLD_JAVA_HOME}
1000-
export JAVA_HOME
1001999

10021000
if ls "${DORIS_HOME}"/fe/custom_lib/*.jar &&
10031001
ls "${DORIS_HOME}"/be/custom_lib/*.jar; then

‎regression-test/pipeline/vault_p0/run.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,11 @@ services:
114114
network_mode: "host"
115115
'
116116
if echo "${docker_compose_hdfs_yaml}" >docker-compose.yaml && docker-compose up -d; then echo; else echo "ERROR: start hdfs docker failed"; fi
117-
JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
117+
# Run the regression suite under JDK 17: the Arrow Flight SQL JDBC driver (arrow 19)
118+
# is Java 11+ bytecode and cannot be loaded by a JDK 8 runtime.
119+
JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*' | sed -n '1p')"
118120
export JAVA_HOME
121+
export PATH="${JAVA_HOME}/bin:${PATH}"
119122
if "${teamcity_build_checkoutDir}"/run-regression-test.sh \
120123
--teamcity \
121124
--run \

‎regression-test/suites/vault_p0/create/test_nereids_show_create_storage_vault.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,11 @@ suite("test_show_create_storage_vault_command") {
8383
def hdfsVaultInfo = try_sql """SHOW CREATE STORAGE VAULT ${hdfsVaultName}"""
8484
def s3VaultInfo = try_sql """SHOW CREATE STORAGE VAULT ${s3VaultName}"""
8585

86-
if (hdfsVaultInfo.size == 1) {
86+
if (hdfsVaultInfo.size() == 1) {
8787
hdfsVaultExist = true
8888
}
8989

90-
if (s3VaultInfo.size == 1) {
90+
if (s3VaultInfo.size() == 1) {
9191
s3VaultExist = true
9292
}
9393

‎run-regression-test.sh‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -225,13 +225,35 @@ if ! test -f ${RUN_JAR:+${RUN_JAR}}; then
225225
echo "===== BUILD JAVA_UDF_SRC TO GENERATE JAR ====="
226226
mkdir -p "${DORIS_HOME}"/regression-test/suites/javaudf_p0/jars
227227
cd "${DORIS_HOME}"/regression-test/java-udf-src || { echo "Failed to change directory to java-udf-src"; exit 1; }
228-
228+
229+
# The Java UDF jar must be built with JDK 8 (source/target 8, loaded by the BE embedded
230+
# JVM), while the framework above is built with the ambient JDK (>=17 in CI). Switch to a
231+
# JDK 8 just for this module, falling back to the ambient JDK if none is installed
232+
# (e.g. local dev boxes).
233+
udf_prev_java_home="${JAVA_HOME:-}"
234+
udf_jdk8_home="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' 2>/dev/null | sed -n '1p')"
235+
if [[ -n "${udf_jdk8_home}" ]]; then
236+
echo "Building Java UDF with JDK 8 at ${udf_jdk8_home}"
237+
export JAVA_HOME="${udf_jdk8_home}"
238+
else
239+
echo "WARNING: no JDK 8 found under /usr/lib/jvm; building Java UDF with ${JAVA_HOME:-the current JDK}"
240+
fi
241+
229242
# Build UDF with retry
230-
execute_maven_with_retry "${MVN_CMD} clean package -B -DskipTests=true -Dmaven.javadoc.skip=true" || {
243+
udf_build_rc=0
244+
execute_maven_with_retry "${MVN_CMD} clean package -B -DskipTests=true -Dmaven.javadoc.skip=true" || udf_build_rc=1
245+
246+
# Restore the framework JDK regardless of the UDF build result.
247+
if [[ -n "${udf_prev_java_home}" ]]; then
248+
export JAVA_HOME="${udf_prev_java_home}"
249+
else
250+
unset JAVA_HOME
251+
fi
252+
if [[ "${udf_build_rc}" -ne 0 ]]; then
231253
echo "Failed to build UDF package"
232254
exit 1
233-
}
234-
255+
fi
256+
235257
cp target/java-udf-case-jar-with-dependencies.jar "${DORIS_HOME}"/regression-test/suites/javaudf_p0/jars/
236258
# be and fe dir is compiled output
237259
mkdir -p "${DORIS_HOME}"/output/fe/custom_lib/

0 commit comments

Comments
 (0)