Skip to content

Commit e5f6299

Browse files
authored
[fix](build) Keep Arrow 17 and 24 in shared thirdparty (#66546)
### What problem does this PR solve? Related PR: #66221 Problem Summary: master and branch-4.1 consume the same prebuilt thirdparty prefix. Upgrading the unversioned Arrow/Paimon stack to Arrow 24 makes unchanged branch-4.1 compile against incompatible headers and libraries. Keep Arrow 17/Paimon 17 in the legacy unversioned prefix, install Arrow 24/Paimon 24 in a versioned prefix selected by master, build and validate both source closures independently, and recover only stale stacks. ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [x] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [x] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [x] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [x] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent b6a52ff commit e5f6299

13 files changed

Lines changed: 1261 additions & 130 deletions

‎be/CMakeLists.txt‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,6 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for
200200
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")
201201

202202
option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
203-
set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
204203
option(ENABLE_TDE "Enable TDE feature module" OFF)
205204
set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src")
206205
option(ENABLE_TLS "Enable TLS feature module" OFF)
@@ -222,8 +221,31 @@ endif()
222221
if (DEFINED ENV{ENABLE_PAIMON_CPP})
223222
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
224223
endif()
225-
if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
226-
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
224+
225+
# Refresh automatic selections on every configure so a reused CMake cache
226+
# follows the current DORIS_THIRDPARTY. Environment overrides are explicit and
227+
# likewise replace any earlier automatic or explicit cache value.
228+
if (DEFINED ENV{ARROW_HOME})
229+
set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "Selected Arrow install prefix" FORCE)
230+
else()
231+
set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH
232+
"Selected Arrow install prefix" FORCE)
233+
endif()
234+
if (DEFINED ENV{PAIMON_HOME})
235+
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "Selected Paimon install prefix" FORCE)
236+
else()
237+
set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "Selected Paimon install prefix" FORCE)
238+
endif()
239+
240+
if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h")
241+
message(FATAL_ERROR "Selected Arrow installation is incomplete: ${ARROW_HOME}")
242+
endif()
243+
message(STATUS "ARROW_HOME is ${ARROW_HOME}")
244+
if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon")
245+
message(FATAL_ERROR "Selected Paimon installation is incomplete: ${PAIMON_HOME}")
246+
endif()
247+
if (ENABLE_PAIMON_CPP)
248+
message(STATUS "PAIMON_HOME is ${PAIMON_HOME}")
227249
endif()
228250

229251
set(CMAKE_SKIP_RPATH TRUE)
@@ -596,6 +618,12 @@ include_directories(
596618
${SRC_DIR}/
597619
)
598620

621+
include_directories(
622+
BEFORE SYSTEM
623+
${ARROW_HOME}/include
624+
${PAIMON_HOME}/include
625+
)
626+
599627
include_directories(
600628
SYSTEM
601629
${COMMON_SRC_DIR}
@@ -741,10 +769,10 @@ endif()
741769

742770
set(PAIMON_FACTORY_REGISTRY_LIBS)
743771
if (ENABLE_PAIMON_CPP)
744-
# Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
745-
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
746-
# thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
747-
# No dual-stack selection needed — single Arrow for everything.
772+
# The Arrow selected for this BE build includes COMPUTE/DATASET/ACERO/FILESYSTEM,
773+
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY
774+
# via thirdparty.cmake. Paimon uses this same selected Arrow (no paimon_deps);
775+
# the legacy Arrow 17 stack remains installed only for pre-upgrade branch-4.1 consumers.
748776

749777
# paimon_parquet_file_format depends on Arrow Dataset symbols.
750778
# Force-link it with --whole-archive so its factory registration runs.

‎be/cmake/thirdparty.cmake‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ set(COMMON_THIRDPARTY)
2525
# if arg exist noadd, not append to COMMON_THIRDPARTY variable
2626
# if arg exist libname, use libname to find library
2727
# if arg exist wholelibpath, use wholelibpath to find library
28+
# if arg exist root, resolve the library below that install prefix
2829
function(add_thirdparty)
2930
cmake_parse_arguments(DORIS_THIRDPARTY
3031
"NOTADD;LIB64"
31-
"LIBNAME;WHOLELIBPATH"
32+
"LIBNAME;WHOLELIBPATH;ROOT"
3233
""
3334
${ARGN})
3435

@@ -39,14 +40,20 @@ function(add_thirdparty)
3940
set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME} PARENT_SCOPE)
4041
endif()
4142

43+
if (DORIS_THIRDPARTY_ROOT)
44+
set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT})
45+
else()
46+
set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR})
47+
endif()
48+
4249
if (DORIS_THIRDPARTY_LIB64)
43-
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
50+
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
4451
elseif (DORIS_THIRDPARTY_LIBNAME)
45-
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME})
52+
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME})
4653
elseif (DORIS_THIRDPARTY_WHOLELIBPATH)
4754
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH})
4855
else()
49-
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
56+
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
5057
endif()
5158
set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH})
5259
endfunction()
@@ -100,18 +107,18 @@ add_thirdparty(cares LIB64)
100107
add_thirdparty(address_sorting LIB64)
101108
add_thirdparty(z LIB64)
102109

103-
add_thirdparty(brotlicommon LIB64)
104-
add_thirdparty(brotlidec LIB64)
105-
add_thirdparty(brotlienc LIB64)
110+
add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME})
111+
add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME})
112+
add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME})
106113
add_thirdparty(zstd LIB64)
107-
add_thirdparty(arrow LIB64)
108-
add_thirdparty(arrow_compute LIB64)
109-
add_thirdparty(arrow_flight LIB64)
110-
add_thirdparty(arrow_flight_sql LIB64)
111-
add_thirdparty(arrow_dataset LIB64)
112-
add_thirdparty(arrow_acero LIB64)
114+
add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME})
115+
add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME})
116+
add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME})
117+
add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME})
118+
add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME})
119+
add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME})
113120
add_thirdparty(adbc_driver_manager LIB64)
114-
add_thirdparty(parquet LIB64)
121+
add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME})
115122
# liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it
116123
# so the final linker resolves C math symbols from the system library first.
117124
add_thirdparty(lance_c LIB64 NOTADD)
@@ -191,16 +198,16 @@ add_thirdparty(icudata LIB64)
191198
add_thirdparty(pugixml LIB64)
192199

193200
if (ENABLE_PAIMON_CPP)
194-
add_thirdparty(paimon LIB64)
195-
add_thirdparty(paimon_parquet_file_format LIB64)
196-
add_thirdparty(paimon_orc_file_format LIB64)
197-
add_thirdparty(paimon_blob_file_format LIB64)
198-
add_thirdparty(paimon_local_file_system LIB64)
199-
add_thirdparty(paimon_file_index LIB64)
200-
add_thirdparty(paimon_global_index LIB64)
201-
202-
add_thirdparty(roaring_bitmap_paimon LIB64)
203-
add_thirdparty(xxhash_paimon LIB64)
204-
add_thirdparty(fmt_paimon LIB64)
205-
add_thirdparty(tbb_paimon LIB64)
201+
add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME})
202+
add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME})
203+
add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME})
204+
add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME})
205+
add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME})
206+
add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME})
207+
add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME})
208+
209+
add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME})
210+
add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME})
211+
add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME})
212+
add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME})
206213
endif()

‎build.sh‎

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,18 @@ if [[ "${HELP}" -eq 1 ]]; then
472472
usage
473473
fi
474474

475+
# Normalize compile-bench before dependency selection. The mode is a BE build,
476+
# even when --compile-bench is the only command-line target.
477+
if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
478+
BUILD_BE=1
479+
BUILD_FE=0
480+
BUILD_CLOUD=0
481+
BUILD_HIVE_UDF=0
482+
BUILD_BE_JAVA_EXTENSIONS=0
483+
BUILD_BE_CDC_CLIENT=0
484+
OUTPUT_BE_BINARY=0
485+
fi
486+
475487
if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_CLOUD} -eq 0 ]]; then
476488
clean_gensrc
477489
clean_be
@@ -491,12 +503,27 @@ fi
491503
# shellcheck source=thirdparty/arrow-paimon-vars.sh
492504
. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh"
493505
NEED_ARROW_PAIMON_THIRDPARTY=false
494-
if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 ||
495-
"${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
506+
if [[ "${BUILD_BE}" -eq 1 || "${BUILD_META_TOOL}" == "ON" ||
507+
"${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
496508
"${BUILD_INDEX_TOOL}" == "ON" ]]; then
497509
NEED_ARROW_PAIMON_THIRDPARTY=true
498510
fi
499511

512+
if [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
513+
DEFAULT_ARROW_PAIMON_HOME="${DORIS_THIRDPARTY}/installed/${ARROW_INSTALL_SUBDIR}"
514+
SELECTED_ARROW_HOME="${ARROW_HOME:-${DEFAULT_ARROW_PAIMON_HOME}}"
515+
SELECTED_PAIMON_HOME="${PAIMON_HOME:-${SELECTED_ARROW_HOME}}"
516+
if [[ "${SELECTED_ARROW_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ||
517+
"${SELECTED_PAIMON_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ]]; then
518+
echo "build.sh only supports the Arrow/Paimon stack selected from DORIS_THIRDPARTY." >&2
519+
echo "Expected ARROW_HOME=${DEFAULT_ARROW_PAIMON_HOME} and PAIMON_HOME=${DEFAULT_ARROW_PAIMON_HOME}." >&2
520+
echo "Unset ARROW_HOME and PAIMON_HOME, or point DORIS_THIRDPARTY at the matching thirdparty tree." >&2
521+
exit 1
522+
fi
523+
export ARROW_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
524+
export PAIMON_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
525+
fi
526+
500527
rebuild_thirdparty_libraries() {
501528
local remove_installed="$1"
502529
shift
@@ -526,7 +553,7 @@ rebuild_thirdparty_libraries() {
526553
build_args+=(--clean)
527554
fi
528555
bash "${build_script}" "${build_args[@]}" "$@"
529-
if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
556+
if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
530557
echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2
531558
exit 1
532559
fi
@@ -535,10 +562,12 @@ rebuild_thirdparty_libraries() {
535562
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
536563
echo "Thirdparty libraries need to be build ..."
537564
rebuild_thirdparty_libraries true
538-
elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] &&
539-
! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
540-
echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
541-
rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
565+
elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
566+
select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed"
567+
if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then
568+
echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
569+
rebuild_thirdparty_libraries false "${ARROW_PAIMON_REBUILD_PACKAGES[@]}"
570+
fi
542571
fi
543572

544573
update_submodule() {
@@ -728,16 +757,6 @@ for ((i = 0; i < ${#CLOUD_EXTRA_FEATURE_KEYS[@]}; i++)); do
728757
done
729758

730759
if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
731-
# BE compile benchmark mode: measure a cold, cache-free BE C++ build.
732-
# Everything that is not the BE C++ build would only add noise, so force
733-
# a BE-only build regardless of the other options.
734-
BUILD_BE=1
735-
BUILD_FE=0
736-
BUILD_CLOUD=0
737-
BUILD_HIVE_UDF=0
738-
BUILD_BE_JAVA_EXTENSIONS=0
739-
BUILD_BE_CDC_CLIENT=0
740-
OUTPUT_BE_BINARY=0
741760
# shellcheck source=build-support/compile-bench/bench-lib.sh
742761
. "${DORIS_HOME}/build-support/compile-bench/bench-lib.sh"
743762
compile_bench_init "${DORIS_HOME}"

‎regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,12 @@ suite("test_adbc_type_mapping", "p0,external") {
227227
// ---- floating point ----
228228

229229
qt_desc_float """DESC ${catalogName}.${dbName}.t_float"""
230-
qt_select_float """SELECT id, c_float, c_double FROM ${catalogName}.${dbName}.t_float ORDER BY id"""
230+
// Render the double with the historical 16-digit baseline so DBL_MAX is not reparsed by the
231+
// test client. The bit-for-bit comparison below still validates the DOUBLE value inside Doris.
232+
qt_select_float """
233+
SELECT id, c_float, format('{:.16g}', c_double) AS c_double
234+
FROM ${catalogName}.${dbName}.t_float ORDER BY id
235+
"""
231236

232237
// Compared INSIDE Doris rather than through sameAsSource, and not because ADBC needs the
233238
// help: the fixture's DBL_MAX cannot make the trip to the test client at all. Doris renders a

0 commit comments

Comments
 (0)