Skip to content

Commit 717a851

Browse files
committed
[fix](build) Validate Lance rebuild inputs before removing dependencies
### What problem does this PR solve? Problem Summary: An incomplete or outdated external third-party tree could erase installed dependencies before its builder failed. Check required inputs and compare the external Lance fingerprint with the checkout before removal. Make the installation harness use independent external definitions, verify preservation on rejected rebuilds, and run it in the third-party CI script job. ### Release note Preserve installed third-party dependencies when Lance rebuild inputs are incomplete or do not match the checkout. ### Check List (For Author) - Test: Installation harness passed on master and branch-4.1; the new missing-helper regression failed before the fix. Shell syntax, workflow YAML, and diff checks passed. - Behavior changed: Yes. Reject invalid Lance rebuild inputs before deleting installed dependencies. - Does this need documentation: No.
1 parent c285848 commit 717a851

3 files changed

Lines changed: 53 additions & 11 deletions

File tree

‎.github/workflows/build-thirdparty.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ jobs:
4949
- 'thirdparty/**'
5050
- 'env.sh'
5151
focused_test_changes:
52+
- 'build.sh'
5253
- 'regression-test/pipeline/external/conf/fe.conf'
5354
- 'thirdparty/test/adbc-jni-config-test.sh'
5455
- '.github/workflows/build-thirdparty.yml'
@@ -82,6 +83,9 @@ jobs:
8283
run: |
8384
thirdparty/test/adbc-jni-config-test.sh
8485
86+
- name: Test Lance installation
87+
run: bash thirdparty/test/lance-install-test.sh
88+
8589
build_linux:
8690
name: Build Third Party Libraries (Linux)
8791
needs: changes

‎build.sh‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -498,9 +498,19 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
498498
! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
499499
-e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]] ||
500500
! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then
501-
# Compilation images may contain only installed artifacts; never erase them without a rebuild source.
502-
if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then
503-
echo "Third-party dependencies require a rebuild, but build-thirdparty.sh is missing." >&2
501+
# External trees can be partially updated or pinned to another revision. Preserve
502+
# the existing prefix unless their build inputs can produce the requested Lance version.
503+
for input in build-thirdparty.sh download-thirdparty.sh vars.sh lance-install.sh patches/lance-c-foyer.patch; do
504+
if [[ ! -f "${DORIS_THIRDPARTY}/${input}" || ! -r "${DORIS_THIRDPARTY}/${input}" ]]; then
505+
echo "Third-party dependencies require a rebuild, but ${input} is missing or unreadable." >&2
506+
echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
507+
exit 1
508+
fi
509+
done
510+
if ! expected_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_HOME}/thirdparty")" ||
511+
! rebuild_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_THIRDPARTY}")" ||
512+
[[ "${rebuild_lance_fingerprint}" != "${expected_lance_fingerprint}" ]]; then
513+
echo "Lance rebuild sources do not match this checkout; installed dependencies have been preserved." >&2
504514
echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
505515
exit 1
506516
fi

‎thirdparty/test/lance-install-test.sh‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,20 @@ mkdir -p "${work}/repo/thirdparty/patches" "${work}/external"
2424
cp "${ROOT}/thirdparty/vars.sh" "${work}/repo/thirdparty/"
2525
cp "${ROOT}/thirdparty/patches/lance-c-foyer.patch" "${work}/repo/thirdparty/patches/"
2626
# Extract the real build gate; all destructive operations stay inside this temporary install.
27+
echo 'set -eo pipefail' > "${work}/gate.sh"
2728
sed -n '/^# build thirdparty libraries if necessary/,/^update_submodule()/p' "${ROOT}/build.sh" \
28-
| sed '$d' > "${work}/gate.sh"
29+
| sed '$d' >> "${work}/gate.sh"
2930
export DORIS_HOME="${work}/repo" DORIS_THIRDPARTY="${work}/external"
3031
export TARGET_SYSTEM=Linux CLEAN=0 PARALLEL=1
3132
export TEST_HELPER="${ROOT}/thirdparty/lance-install.sh"
32-
if [[ -f "${TEST_HELPER}" ]]; then
33-
cp "${TEST_HELPER}" "${work}/repo/thirdparty/"
34-
fi
33+
cp "${TEST_HELPER}" "${work}/repo/thirdparty/"
34+
cp -r "${DORIS_HOME}/thirdparty/." "${DORIS_THIRDPARTY}/"
35+
printf '#!/usr/bin/env bash\nexit 0\n' > "${DORIS_THIRDPARTY}/download-thirdparty.sh"
3536
cat > "${DORIS_THIRDPARTY}/build-thirdparty.sh" <<'BUILDER'
3637
set -euo pipefail
37-
source "${TEST_HELPER}"
38+
TP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
39+
source "${TP_DIR}/lance-install.sh"
40+
fingerprint="$(lance_c_install_fingerprint "${TP_DIR}")"
3841
echo rebuilt >> "${DORIS_THIRDPARTY}/builds"
3942
installed="${DORIS_THIRDPARTY}/installed"
4043
mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \
@@ -44,7 +47,7 @@ for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs.
4447
include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do
4548
echo artifact > "${installed}/${file}"
4649
done
47-
lance_c_install_fingerprint "${DORIS_HOME}/thirdparty" > "${installed}/lib64/.lance-c-fingerprint"
50+
printf '%s\n' "${fingerprint}" > "${installed}/lib64/.lance-c-fingerprint"
4851
BUILDER
4952
# A complete legacy image has all old sentinels but no Lance revision marker.
5053
installed="${DORIS_THIRDPARTY}/installed"
@@ -66,19 +69,43 @@ check_builds 1
6669
bash "${work}/gate.sh"
6770
check_builds 1
6871
echo 'PASS: legacy image rebuilt; matching install reused'
72+
# Failed preflight must preserve every dependency, not just the Lance archive.
73+
expect_preserved_install() {
74+
cp -a "${installed}" "${work}/saved-install"
75+
cp "${DORIS_THIRDPARTY}/builds" "${work}/saved-builds"
76+
if bash "${work}/gate.sh" > "${work}/preflight.log" 2>&1; then
77+
echo 'FAIL: accepted incomplete or mismatched rebuild sources'; exit 1
78+
fi
79+
if ! diff -r "${work}/saved-install" "${installed}"; then
80+
echo 'FAIL: rejected rebuild sources after changing installed dependencies'; exit 1
81+
fi
82+
cmp "${work}/saved-builds" "${DORIS_THIRDPARTY}/builds"
83+
rm -rf "${work}/saved-install"
84+
}
85+
echo stale > "${installed}/lib64/.lance-c-fingerprint"
86+
for file in lance-install.sh vars.sh patches/lance-c-foyer.patch download-thirdparty.sh build-thirdparty.sh; do
87+
mv "${DORIS_THIRDPARTY}/${file}" "${work}/missing-input"
88+
expect_preserved_install
89+
mv "${work}/missing-input" "${DORIS_THIRDPARTY}/${file}"
90+
done
91+
echo 'PASS: incomplete external sources rejected before changing installed dependencies'
6992
echo stale > "${installed}/lib64/.lance-c-fingerprint"
7093
bash "${work}/gate.sh"
7194
check_builds 2
7295
# Pin and patch updates both invalidate installed artifacts, even with identical ABI.
7396
sed 's/^LANCE_C_SOURCE=.*/LANCE_C_SOURCE="lance-c-test-revision"/' "${DORIS_HOME}/thirdparty/vars.sh" \
7497
> "${work}/new-vars.sh"
7598
mv "${work}/new-vars.sh" "${DORIS_HOME}/thirdparty/vars.sh"
99+
expect_preserved_install
100+
cp "${DORIS_HOME}/thirdparty/vars.sh" "${DORIS_THIRDPARTY}/vars.sh"
76101
bash "${work}/gate.sh"
77102
check_builds 3
78103
echo '# test patch update' >> "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch"
104+
expect_preserved_install
105+
cp "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch" "${DORIS_THIRDPARTY}/patches/"
79106
bash "${work}/gate.sh"
80107
check_builds 4
81-
echo 'PASS: stale marker, changed pin and changed patch rebuild'
108+
echo 'PASS: mismatched sources rejected; synchronized pin and patch updates rebuild'
82109
rm "${installed}/include/lance/lance.h"
83110
bash "${work}/gate.sh"
84111
check_builds 5
@@ -94,7 +121,8 @@ if bash "${work}/gate.sh" > "${work}/old-builder.log" 2>&1; then
94121
echo 'FAIL: accepted output from a stale external builder'; exit 1
95122
fi
96123
grep -q 'Lance dependency revision does not match' "${work}/old-builder.log"
97-
bash "${work}/good-builder.sh"
124+
cp "${work}/good-builder.sh" "${DORIS_THIRDPARTY}/build-thirdparty.sh"
125+
bash "${DORIS_THIRDPARTY}/build-thirdparty.sh"
98126
echo 'PASS: stale external builder cannot silently satisfy the revision gate'
99127
# Images without rebuild sources must fail before deleting installed dependencies.
100128
rm "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${installed}/lib64/.lance-c-fingerprint"

0 commit comments

Comments
 (0)