Skip to content

Commit 0ed1bb2

Browse files
committed
[fix](build) Keep Arrow 17 and 24 in shared thirdparty
### What problem does this PR solve? Issue Number: None Related PR: #66546 Problem Summary: Current branch-4.1 includes the Arrow 24 upgrade and can consume the same thirdparty installation as master. Installing Arrow 24 and Paimon against the legacy root prefix overwrites the Arrow 17 ABI expected by pre-upgrade branch-4.1 revisions and can leave mixed or stale artifacts after interrupted rebuilds. Keep Arrow 17 and Paimon 17 in the unversioned compatibility prefix, install the Arrow 24 and Paimon 24 ABI-coupled stack in a versioned prefix selected by the current branch, fingerprint and validate both stacks independently, and rebuild only the invalid stack. Preserve branch-4.1-specific build and test layout while backporting the shared-stack lifecycle coverage. ### Release note None ### Check List (For Author) - Test: Manual test - PASS: ./thirdparty/test/arrow-paimon-lifecycle-test.sh - PASS: bash -n on all changed shell scripts - PASS: ./build-support/clang-format.sh - PASS: ./build-support/check-format.sh - PASS: non-patch git diff --check and Arrow 17 patch byte-identity checks - Not run: full BE build; the filesystem is 99% used and this worktree does not have a valid dual-stack thirdparty installation, so rebuilding both stacks risks exhausting disk space - Behavior changed: Yes. Current branch-4.1 selects versioned Arrow 24/Paimon 24 while preserving the root Arrow 17/Paimon 17 compatibility stack. - Does this need documentation: No
1 parent 2c3ae38 commit 0ed1bb2

12 files changed

Lines changed: 1229 additions & 120 deletions

‎be/CMakeLists.txt‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for
148148
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")
149149

150150
option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
151-
set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
152151
option(ENABLE_TDE "Enable TDE feature module" OFF)
153152
set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src")
154153
option(ENABLE_TLS "Enable TLS feature module" OFF)
@@ -170,8 +169,31 @@ endif()
170169
if (DEFINED ENV{ENABLE_PAIMON_CPP})
171170
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
172171
endif()
173-
if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
174-
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
172+
173+
# Refresh automatic selections on every configure so a reused CMake cache
174+
# follows the current DORIS_THIRDPARTY. Environment overrides are explicit and
175+
# likewise replace any earlier automatic or explicit cache value.
176+
if (DEFINED ENV{ARROW_HOME})
177+
set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "Selected Arrow install prefix" FORCE)
178+
else()
179+
set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH
180+
"Selected Arrow install prefix" FORCE)
181+
endif()
182+
if (DEFINED ENV{PAIMON_HOME})
183+
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "Selected Paimon install prefix" FORCE)
184+
else()
185+
set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "Selected Paimon install prefix" FORCE)
186+
endif()
187+
188+
if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h")
189+
message(FATAL_ERROR "Selected Arrow installation is incomplete: ${ARROW_HOME}")
190+
endif()
191+
message(STATUS "ARROW_HOME is ${ARROW_HOME}")
192+
if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon")
193+
message(FATAL_ERROR "Selected Paimon installation is incomplete: ${PAIMON_HOME}")
194+
endif()
195+
if (ENABLE_PAIMON_CPP)
196+
message(STATUS "PAIMON_HOME is ${PAIMON_HOME}")
175197
endif()
176198

177199
set(CMAKE_SKIP_RPATH TRUE)
@@ -517,6 +539,12 @@ include_directories(
517539
${SRC_DIR}/
518540
)
519541

542+
include_directories(
543+
BEFORE SYSTEM
544+
${ARROW_HOME}/include
545+
${PAIMON_HOME}/include
546+
)
547+
520548
include_directories(
521549
SYSTEM
522550
${COMMON_SRC_DIR}
@@ -662,10 +690,10 @@ endif()
662690

663691
set(PAIMON_FACTORY_REGISTRY_LIBS)
664692
if (ENABLE_PAIMON_CPP)
665-
# Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
666-
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
667-
# thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
668-
# No dual-stack selection needed — single Arrow for everything.
693+
# The Arrow selected for this BE build includes COMPUTE/DATASET/ACERO/FILESYSTEM,
694+
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY
695+
# via thirdparty.cmake. Paimon uses this same selected Arrow (no paimon_deps);
696+
# the legacy Arrow 17 stack remains installed only for pre-upgrade branch-4.1 consumers.
669697

670698
# paimon_parquet_file_format depends on Arrow Dataset symbols.
671699
# Force-link it with --whole-archive so its factory registration runs.

‎be/cmake/thirdparty.cmake‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ set(COMMON_THIRDPARTY)
2525
# if arg exist noadd, not append to COMMON_THIRDPARTY variable
2626
# if arg exist libname, use libname to find library
2727
# if arg exist wholelibpath, use wholelibpath to find library
28+
# if arg exist root, resolve the library below that install prefix
2829
function(add_thirdparty)
2930
cmake_parse_arguments(DORIS_THIRDPARTY
3031
"NOTADD;LIB64"
31-
"LIBNAME;WHOLELIBPATH"
32+
"LIBNAME;WHOLELIBPATH;ROOT"
3233
""
3334
${ARGN})
3435

@@ -39,14 +40,20 @@ function(add_thirdparty)
3940
set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME} PARENT_SCOPE)
4041
endif()
4142

43+
if (DORIS_THIRDPARTY_ROOT)
44+
set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT})
45+
else()
46+
set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR})
47+
endif()
48+
4249
if (DORIS_THIRDPARTY_LIB64)
43-
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
50+
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
4451
elseif (DORIS_THIRDPARTY_LIBNAME)
45-
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME})
52+
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME})
4653
elseif (DORIS_THIRDPARTY_WHOLELIBPATH)
4754
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH})
4855
else()
49-
set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
56+
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
5057
endif()
5158
set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH})
5259
endfunction()
@@ -100,17 +107,17 @@ add_thirdparty(cares LIB64)
100107
add_thirdparty(address_sorting LIB64)
101108
add_thirdparty(z LIB64)
102109

103-
add_thirdparty(brotlicommon LIB64)
104-
add_thirdparty(brotlidec LIB64)
105-
add_thirdparty(brotlienc LIB64)
110+
add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME})
111+
add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME})
112+
add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME})
106113
add_thirdparty(zstd LIB64)
107-
add_thirdparty(arrow LIB64)
108-
add_thirdparty(arrow_compute LIB64)
109-
add_thirdparty(arrow_flight LIB64)
110-
add_thirdparty(arrow_flight_sql LIB64)
111-
add_thirdparty(arrow_dataset LIB64)
112-
add_thirdparty(arrow_acero LIB64)
113-
add_thirdparty(parquet LIB64)
114+
add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME})
115+
add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME})
116+
add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME})
117+
add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME})
118+
add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME})
119+
add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME})
120+
add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME})
114121
# liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it
115122
# so the final linker resolves C math symbols from the system library first.
116123
add_thirdparty(lance_c LIB64 NOTADD)
@@ -189,16 +196,16 @@ add_thirdparty(icudata LIB64)
189196
add_thirdparty(pugixml LIB64)
190197

191198
if (ENABLE_PAIMON_CPP)
192-
add_thirdparty(paimon LIB64)
193-
add_thirdparty(paimon_parquet_file_format LIB64)
194-
add_thirdparty(paimon_orc_file_format LIB64)
195-
add_thirdparty(paimon_blob_file_format LIB64)
196-
add_thirdparty(paimon_local_file_system LIB64)
197-
add_thirdparty(paimon_file_index LIB64)
198-
add_thirdparty(paimon_global_index LIB64)
199-
200-
add_thirdparty(roaring_bitmap_paimon LIB64)
201-
add_thirdparty(xxhash_paimon LIB64)
202-
add_thirdparty(fmt_paimon LIB64)
203-
add_thirdparty(tbb_paimon LIB64)
199+
add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME})
200+
add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME})
201+
add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME})
202+
add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME})
203+
add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME})
204+
add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME})
205+
add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME})
206+
207+
add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME})
208+
add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME})
209+
add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME})
210+
add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME})
204211
endif()

‎build.sh‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -451,12 +451,27 @@ fi
451451
# shellcheck source=thirdparty/arrow-paimon-vars.sh
452452
. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh"
453453
NEED_ARROW_PAIMON_THIRDPARTY=false
454-
if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 ||
455-
"${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
454+
if [[ "${BUILD_BE}" -eq 1 || "${BUILD_META_TOOL}" == "ON" ||
455+
"${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
456456
"${BUILD_INDEX_TOOL}" == "ON" ]]; then
457457
NEED_ARROW_PAIMON_THIRDPARTY=true
458458
fi
459459

460+
if [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
461+
DEFAULT_ARROW_PAIMON_HOME="${DORIS_THIRDPARTY}/installed/${ARROW_INSTALL_SUBDIR}"
462+
SELECTED_ARROW_HOME="${ARROW_HOME:-${DEFAULT_ARROW_PAIMON_HOME}}"
463+
SELECTED_PAIMON_HOME="${PAIMON_HOME:-${SELECTED_ARROW_HOME}}"
464+
if [[ "${SELECTED_ARROW_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ||
465+
"${SELECTED_PAIMON_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ]]; then
466+
echo "build.sh only supports the Arrow/Paimon stack selected from DORIS_THIRDPARTY." >&2
467+
echo "Expected ARROW_HOME=${DEFAULT_ARROW_PAIMON_HOME} and PAIMON_HOME=${DEFAULT_ARROW_PAIMON_HOME}." >&2
468+
echo "Unset ARROW_HOME and PAIMON_HOME, or point DORIS_THIRDPARTY at the matching thirdparty tree." >&2
469+
exit 1
470+
fi
471+
export ARROW_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
472+
export PAIMON_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
473+
fi
474+
460475
rebuild_thirdparty_libraries() {
461476
local remove_installed="$1"
462477
shift
@@ -486,7 +501,7 @@ rebuild_thirdparty_libraries() {
486501
build_args+=(--clean)
487502
fi
488503
bash "${build_script}" "${build_args[@]}" "$@"
489-
if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
504+
if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
490505
echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2
491506
exit 1
492507
fi
@@ -495,10 +510,12 @@ rebuild_thirdparty_libraries() {
495510
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
496511
echo "Thirdparty libraries need to be build ..."
497512
rebuild_thirdparty_libraries true
498-
elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] &&
499-
! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
500-
echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
501-
rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
513+
elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
514+
select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed"
515+
if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then
516+
echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
517+
rebuild_thirdparty_libraries false "${ARROW_PAIMON_REBUILD_PACKAGES[@]}"
518+
fi
502519
fi
503520

504521
update_submodule() {

0 commit comments

Comments
 (0)