Skip to content

Commit f88735f

Browse files
authored
[chore](build) Add paimon-rust third-party build integration (apache#68562)
### What problem does this PR solve? Related PR: apache#66227 Prepare the third-party dependency for the Paimon Rust reader on master independently of the reader implementation. This extracts all dependency-related changes from apache#66227 after simplifying its Rust archive installation. - Download Apache paimon-rust v0.4.0-rc1 with a pinned checksum and build `paimon-c` with HDFS support. - Generate the C header with pinned cbindgen 0.29.4 and install the header and archive using the existing third-party copy/strip conventions. - Use Rust 1.94.0 by default for both Lance and Paimon. When changing the toolchain, rebuild both libraries with the same `RUSTUP_TOOLCHAIN` to avoid conflicting Rust standard libraries. - Register the dependency in CMake while preserving libm and glibc compatibility link ordering. The changes cover dependency registration, building, and installation checks, plus build-script regression tests. Existing installations missing either Rust archive or the generated Paimon header follow the full dependency rebuild path so Lance and Paimon are rebuilt together. Header generation inherits the selected Cargo executable and offline mode. Reader code and its functional tests remain in apache#66227. The Rust archive identity helper and its dedicated test file are not included. ### Validation - Bash syntax and `git diff --check` passed. - `python3 thirdparty/test/paimon-build-test.py`: all 10 tests passed on both this branch and apache#66227. Eight tests fail before the fix. Coverage includes old Linux/macOS installs, missing archives/header, complete-install reuse, clean forwarding, custom Cargo, and offline metadata settings. External compiler/header-generator commands are test doubles; the tests execute the actual build-script logic. - Six temporary mock-toolchain scenarios passed on this branch, covering installation, strip behavior, offline flags, default toolchain selection, explicitly newer toolchains, and rejection of older toolchains. The same scenarios passed on apache#66227. - CMake fragment checks passed with `GLIBC_COMPATIBILITY` enabled and disabled; compatibility-library ordering was also checked. - The official source archive MD5 matches the manifest. - Full Rust/Doris compilation and final binary ABI validation were not run locally; compilation is requested through CI. ### Release note None ### Check List (For Author) - Test: Build-script smoke checks and dependency/link-order validation described above. - Behavior changed: Yes. Third-party builds include paimon-rust and both Rust libraries default to Rust 1.94.0. - Does this need documentation: No. This prepares an internal build dependency.
1 parent 2eaaf45 commit f88735f

6 files changed

Lines changed: 378 additions & 9 deletions

File tree

‎be/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -860,7 +860,8 @@ if (GLIBC_COMPATIBILITY)
860860
glibc-compatibility-explicit
861861
glibc-compatibility
862862
-lm
863-
lance_c)
863+
lance_c
864+
paimon_c)
864865
endif()
865866

866867
if (NOT OS_MACOSX)

‎be/cmake/thirdparty.cmake‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,13 @@ add_thirdparty(lance_c LIB64 NOTADD)
120120
if (NOT GLIBC_COMPATIBILITY)
121121
list(APPEND COMMON_THIRDPARTY m lance_c)
122122
endif()
123+
# libpaimon_c.a (built from paimon-rust) brings in Rust compiler_builtins that
124+
# would otherwise steal libm symbols. Place libm before it so the final linker
125+
# resolves C math symbols from the system library first.
126+
add_thirdparty(paimon_c LIB64 NOTADD)
127+
if (NOT GLIBC_COMPATIBILITY)
128+
list(APPEND COMMON_THIRDPARTY m paimon_c)
129+
endif()
123130
add_thirdparty(brpc LIB64)
124131
add_thirdparty(rocksdb)
125132
add_thirdparty(cyrus-sasl LIBNAME "lib/libsasl2.a")

‎build.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,12 @@ else
486486
LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a'
487487
fi
488488

489-
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
489+
# Old installs can contain the sentinel but predate Paimon. Rebuild the whole
490+
# dependency set so Lance and Paimon also use the same selected Rust toolchain.
491+
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
492+
! -f "${DORIS_THIRDPARTY}/installed/lib64/liblance_c.a" ||
493+
! -f "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
494+
! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ]]; then
490495
echo "Thirdparty libraries need to be build ..."
491496
# need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists
492497
rm -rf "${DORIS_THIRDPARTY}/installed"

‎thirdparty/build-thirdparty.sh‎

Lines changed: 154 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2372,6 +2372,24 @@ build_pugixml() {
23722372
}
23732373

23742374
# lance-c
2375+
# Publish a complete archive with one rename. Copy/strip must not damage an installed library
2376+
# or expose a partial first installation if either command fails or the build is interrupted.
2377+
install_rust_archive() {
2378+
(
2379+
set -e
2380+
local archive="$1"
2381+
local destination="${TP_INSTALL_DIR}/lib64/${archive##*/}"
2382+
local staged
2383+
staged="$(mktemp "${destination}.tmp.XXXXXX")"
2384+
trap 'rm -f "${staged}"' EXIT
2385+
cp -p "${archive}" "${staged}"
2386+
if [[ "${STRIP_TP_LIB}" = "ON" && "${KERNEL}" != 'Darwin' ]]; then
2387+
strip --strip-debug --strip-unneeded "${staged}"
2388+
fi
2389+
mv -f "${staged}" "${destination}"
2390+
)
2391+
}
2392+
23752393
build_lance_c() {
23762394
check_if_source_exist "${LANCE_C_SOURCE}"
23772395
cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}"
@@ -2381,22 +2399,28 @@ build_lance_c() {
23812399

23822400
local cargo_bin="${LANCE_C_CARGO:-${CARGO:-cargo}}"
23832401
if ! command -v "${cargo_bin}" >/dev/null 2>&1; then
2384-
echo "cargo is required to build lance-c. Install Rust 1.91.0 or set LANCE_C_CARGO."
2402+
echo "cargo is required to build lance-c. Install Rust 1.94.0 or set LANCE_C_CARGO."
23852403
exit 1
23862404
fi
23872405
if [[ ! -x "${TP_INSTALL_DIR}/bin/protoc" ]]; then
23882406
echo "protoc is required to build lance-c. Build protobuf first."
23892407
exit 1
23902408
fi
23912409

2392-
local required_rust_version="1.91.0"
2410+
local required_rust_version="1.94.0"
23932411
local cargo_env=(
23942412
"CARGO_BUILD_JOBS=${PARALLEL}"
23952413
"CARGO_TARGET_DIR=${PWD}/${BUILD_DIR}"
23962414
"PROTOC=${TP_INSTALL_DIR}/bin/protoc"
23972415
)
23982416
if command -v rustup >/dev/null 2>&1 && [[ -z "${RUSTUP_TOOLCHAIN}" ]]; then
2399-
if ! rustup toolchain list | grep -Eq '^1\.91\.0([[:space:]-]|$)'; then
2417+
# The presence check must look for the toolchain the minimum actually
2418+
# requires, not a literal: with only an older toolchain installed the
2419+
# stale check would skip the install below and then force
2420+
# RUSTUP_TOOLCHAIN to a version rustup cannot dispatch, failing the
2421+
# build before any archive is produced.
2422+
local required_rust_regex="${required_rust_version//./\\.}"
2423+
if ! rustup toolchain list | grep -Eq "^${required_rust_regex}([[:space:]-]|$)"; then
24002424
rustup toolchain install "${required_rust_version}" --profile minimal
24012425
fi
24022426
cargo_env+=("RUSTUP_TOOLCHAIN=${required_rust_version}")
@@ -2407,7 +2431,7 @@ build_lance_c() {
24072431
echo "failed to get cargo version for lance-c. Install Rust ${required_rust_version} or set LANCE_C_CARGO/RUSTUP_TOOLCHAIN."
24082432
exit 1
24092433
fi
2410-
# Rust 1.91.0 is the minimum supported version. Allow newer toolchains when
2434+
# Rust 1.94.0 is the minimum supported version. Allow newer toolchains when
24112435
# callers explicitly select one or rustup is unavailable on the system.
24122436
if ! awk -v required="${required_rust_version}" -v actual="${cargo_version}" 'BEGIN {
24132437
split(required, r, ".");
@@ -2440,11 +2464,132 @@ build_lance_c() {
24402464
mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
24412465
rm -rf "${TP_INSTALL_DIR}/include/lance"
24422466
cp -av include/lance "${TP_INSTALL_DIR}/include/"
2443-
cp -v "${BUILD_DIR}/release/liblance_c.a" "${TP_INSTALL_DIR}/lib64/"
2467+
install_rust_archive "${BUILD_DIR}/release/liblance_c.a"
2468+
}
2469+
2470+
build_paimon_rust() {
2471+
check_if_source_exist "${PAIMON_RUST_SOURCE}"
2472+
cd "${TP_SOURCE_DIR}/${PAIMON_RUST_SOURCE}"
2473+
2474+
rm -rf "${BUILD_DIR}"
2475+
mkdir -p "${BUILD_DIR}"
2476+
2477+
local cargo_bin="${PAIMON_RUST_CARGO:-${CARGO:-cargo}}"
2478+
if ! command -v "${cargo_bin}" >/dev/null 2>&1; then
2479+
echo "cargo is required to build paimon-rust. Install Rust 1.94.0 or set PAIMON_RUST_CARGO."
2480+
exit 1
2481+
fi
2482+
2483+
local required_rust_version="1.94.0"
2484+
local cargo_env=(
2485+
"CARGO_BUILD_JOBS=${PARALLEL}"
2486+
"CARGO_TARGET_DIR=${PWD}/${BUILD_DIR}"
2487+
)
2488+
if command -v rustup >/dev/null 2>&1 && [[ -z "${RUSTUP_TOOLCHAIN}" ]]; then
2489+
# The presence check must look for the toolchain the minimum actually
2490+
# requires, not a literal: with only an older toolchain installed the
2491+
# stale check would skip the install below and then force
2492+
# RUSTUP_TOOLCHAIN to a version rustup cannot dispatch, failing the
2493+
# build before any archive is produced.
2494+
local required_rust_regex="${required_rust_version//./\\.}"
2495+
if ! rustup toolchain list | grep -Eq "^${required_rust_regex}([[:space:]-]|$)"; then
2496+
rustup toolchain install "${required_rust_version}" --profile minimal
2497+
fi
2498+
cargo_env+=("RUSTUP_TOOLCHAIN=${required_rust_version}")
2499+
fi
24442500

2445-
if [[ "${STRIP_TP_LIB}" = "ON" && "${KERNEL}" != 'Darwin' ]]; then
2446-
strip --strip-debug --strip-unneeded "${TP_INSTALL_DIR}/lib64/liblance_c.a"
2501+
local cargo_version
2502+
if ! cargo_version="$(env "${cargo_env[@]}" "${cargo_bin}" --version | awk '{print $2}')"; then
2503+
echo "failed to get cargo version for paimon-rust. Install Rust ${required_rust_version} or set PAIMON_RUST_CARGO/RUSTUP_TOOLCHAIN."
2504+
exit 1
24472505
fi
2506+
# Rust 1.94.0 is the minimum supported version. Allow newer toolchains when
2507+
# callers explicitly select one or rustup is unavailable on the system.
2508+
# NOTE: paimon_c and lance_c are both Rust staticlibs linked into the same
2509+
# BE binary; they must be built with the SAME rustc toolchain so the linker
2510+
# resolves both crates' std references against a single std copy. Mixing
2511+
# toolchains makes the precompiled std hashes differ and the linker pulls
2512+
# both std copies in, colliding on the unmangled `rust_eh_personality`
2513+
# (duplicate symbol). Rebuild both lance_c and paimon_rust whenever the
2514+
# toolchain changes, using the same RUSTUP_TOOLCHAIN for both builds.
2515+
if ! awk -v required="${required_rust_version}" -v actual="${cargo_version}" 'BEGIN {
2516+
split(required, r, ".");
2517+
split(actual, a, ".");
2518+
for (i = 1; i <= 3; i++) {
2519+
if ((a[i] + 0) > (r[i] + 0)) {
2520+
exit 0;
2521+
}
2522+
if ((a[i] + 0) < (r[i] + 0)) {
2523+
exit 1;
2524+
}
2525+
}
2526+
exit 0;
2527+
}'; then
2528+
echo "paimon-rust requires Rust/Cargo ${required_rust_version} or newer, but found ${cargo_version}."
2529+
echo "Install Rust ${required_rust_version} or set PAIMON_RUST_CARGO/RUSTUP_TOOLCHAIN."
2530+
exit 1
2531+
fi
2532+
2533+
if [[ "${KERNEL}" != 'Darwin' ]]; then
2534+
cargo_env+=("CFLAGS=${CFLAGS:-} -std=gnu17")
2535+
fi
2536+
2537+
local cargo_args=(build --release --locked -p paimon-c --features paimon/storage-hdfs)
2538+
# cbindgen invokes cargo metadata itself; command-line flags on the build
2539+
# and install calls do not propagate to that child process.
2540+
cargo_env+=("CARGO=${cargo_bin}")
2541+
if [[ "$(echo "${PAIMON_RUST_CARGO_OFFLINE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then
2542+
cargo_args+=(--offline)
2543+
cargo_env+=("CARGO_NET_OFFLINE=true")
2544+
fi
2545+
env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}"
2546+
2547+
# Generate the C header from the Rust extern "C" surface via cbindgen.
2548+
# cbindgen is a pinned, Doris-controlled input: an unpinned "current"
2549+
# release would regenerate paimon.h differently between builds. The
2550+
# pinned version installs under a Doris-controlled --root and its
2551+
# resolved absolute path is invoked directly (a custom CARGO_HOME does
2552+
# not necessarily put cargo-installed binaries on PATH). Offline builds
2553+
# pass --offline to the install command, exactly like the fetch/build
2554+
# handling above — cargo fails on a missing local crate cache instead
2555+
# of reaching for the network.
2556+
local cbindgen_version="0.29.4"
2557+
local cbindgen_bin="${PAIMON_RUST_CBINDGEN:-}"
2558+
if [[ -z "${cbindgen_bin}" ]]; then
2559+
local cbindgen_root="${TP_SOURCE_DIR}/.doris-cbindgen-${cbindgen_version}"
2560+
cbindgen_bin="${cbindgen_root}/bin/cbindgen"
2561+
if [[ ! -x "${cbindgen_bin}" ]]; then
2562+
local cbindgen_install_args=(install cbindgen
2563+
--version "${cbindgen_version}" --locked --root "${cbindgen_root}")
2564+
if [[ "$(echo "${PAIMON_RUST_CARGO_OFFLINE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then
2565+
cbindgen_install_args+=(--offline)
2566+
fi
2567+
echo "cbindgen not found; installing pinned ${cbindgen_version} via cargo install ..."
2568+
env "${cargo_env[@]}" "${cargo_bin}" "${cbindgen_install_args[@]}"
2569+
fi
2570+
elif [[ ! -x "${cbindgen_bin}" ]]; then
2571+
echo "PAIMON_RUST_CBINDGEN=${cbindgen_bin} is not an executable file."
2572+
exit 1
2573+
fi
2574+
# Write a temporary cbindgen.toml so the generated header carries our
2575+
# include-guard / cpp-compat settings without touching the upstream tree.
2576+
local cbindgen_toml="${BUILD_DIR}/cbindgen.toml"
2577+
mkdir -p "${BUILD_DIR}"
2578+
cat >"${cbindgen_toml}" <<'EOF'
2579+
language = "C"
2580+
include_guard = "PAIMON_C_H"
2581+
pragma_once = true
2582+
cpp_compat = true
2583+
EOF
2584+
env "${cargo_env[@]}" "${cbindgen_bin}" bindings/c \
2585+
--config "${cbindgen_toml}" \
2586+
--output "${BUILD_DIR}/release/paimon.h"
2587+
2588+
mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
2589+
rm -rf "${TP_INSTALL_DIR}/include/paimon_rust"
2590+
mkdir -p "${TP_INSTALL_DIR}/include/paimon_rust"
2591+
cp -v "${BUILD_DIR}/release/paimon.h" "${TP_INSTALL_DIR}/include/paimon_rust/"
2592+
install_rust_archive "${BUILD_DIR}/release/libpaimon_c.a"
24482593
}
24492594

24502595
if [[ "${#packages[@]}" -eq 0 ]]; then
@@ -2526,6 +2671,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
25262671
icu
25272672
mecab_ipadic
25282673
pugixml
2674+
paimon_rust
25292675
)
25302676
if [[ "$(uname -s)" == 'Darwin' ]]; then
25312677
read -r -a packages <<<"binutils gettext ${packages[*]}"
@@ -2636,6 +2782,7 @@ cleanup_package_source() {
26362782
juicefs) src_var="JUICEFS_SOURCE" ;;
26372783
pugixml) src_var="PUGIXML_SOURCE" ;;
26382784
lance_c) src_var="LANCE_C_SOURCE" ;;
2785+
paimon_rust) src_var="PAIMON_RUST_SOURCE" ;;
26392786
aws_sdk) src_var="AWS_SDK_SOURCE" ;;
26402787
lzma) src_var="LZMA_SOURCE" ;;
26412788
xml2) src_var="XML2_SOURCE" ;;

0 commit comments

Comments
 (0)