Skip to content

Commit f805ff3

Browse files
committed
[fix](build) Refresh Paimon dependencies and preserve Cargo settings
### What problem does this PR solve? Related PR: #68562, #66227 Existing third-party installs can satisfy the legacy installation sentinel while missing the newly required Paimon archive. Check both Rust archives and the generated header before reusing an install; incomplete installs use the existing full rebuild path, rebuilding Lance and Paimon together. Pass the selected Cargo executable and offline mode to cbindgen's metadata subprocess so header generation honors the same settings as archive builds. ### Release note None ### Check List (For Author) - Test: Ten build-script tests pass; eight reproduce the bugs before the fix. Bash syntax and git diff --check pass. Full Rust/Doris compilation was not run. - Behavior changed: Yes. Incomplete dependency installs rebuild, and header generation honors custom Cargo and offline settings. - Does this need documentation: No.
1 parent ab9fac3 commit f805ff3

3 files changed

Lines changed: 164 additions & 1 deletion

File tree

‎build.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,12 @@ else
486486
LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a'
487487
fi
488488

489-
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
489+
# Old installs can contain the sentinel but predate Paimon. Rebuild the whole
490+
# dependency set so Lance and Paimon also use the same selected Rust toolchain.
491+
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
492+
! -f "${DORIS_THIRDPARTY}/installed/lib64/liblance_c.a" ||
493+
! -f "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
494+
! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ]]; then
490495
echo "Thirdparty libraries need to be build ..."
491496
# need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists
492497
rm -rf "${DORIS_THIRDPARTY}/installed"

‎thirdparty/build-thirdparty.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2521,8 +2521,12 @@ build_paimon_rust() {
25212521
fi
25222522

25232523
local cargo_args=(build --release --locked -p paimon-c --features paimon/storage-hdfs)
2524+
# cbindgen invokes cargo metadata itself; command-line flags on the build
2525+
# and install calls do not propagate to that child process.
2526+
cargo_env+=("CARGO=${cargo_bin}")
25242527
if [[ "$(echo "${PAIMON_RUST_CARGO_OFFLINE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then
25252528
cargo_args+=(--offline)
2529+
cargo_env+=("CARGO_NET_OFFLINE=true")
25262530
fi
25272531
env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}"
25282532

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#!/usr/bin/env python3
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
"""Run with python3 thirdparty/test/paimon-build-test.py; no Rust downloads required."""
20+
21+
import os
22+
from pathlib import Path
23+
import re
24+
import subprocess
25+
import tempfile
26+
import unittest
27+
28+
29+
ROOT = Path(__file__).resolve().parents[2]
30+
31+
32+
class PaimonBuildTest(unittest.TestCase):
33+
def setUp(self):
34+
self.temp = tempfile.TemporaryDirectory()
35+
self.addCleanup(self.temp.cleanup)
36+
self.work = Path(self.temp.name)
37+
self.env = dict(os.environ)
38+
for key in ("CARGO", "CARGO_NET_OFFLINE", "RUSTUP_TOOLCHAIN"):
39+
self.env.pop(key, None)
40+
41+
def write(self, name, text=""):
42+
path = self.work / name
43+
path.parent.mkdir(parents=True, exist_ok=True)
44+
path.write_text(text)
45+
return path
46+
47+
def executable(self, name, text):
48+
path = self.write(name, "#!/bin/bash\nset -e\n" + text)
49+
path.chmod(0o755)
50+
return path
51+
52+
def run_bash(self, script):
53+
return subprocess.run(["bash", "-ec", script], env=self.env,
54+
cwd=self.work, text=True, capture_output=True)
55+
56+
def check_install(self, system, missing, clean=0):
57+
script = (ROOT / "build.sh").read_text()
58+
gate = script[script.index("# build thirdparty libraries if necessary."):
59+
script.index("update_submodule() {")]
60+
sentinel = re.search(r"else\n LAST_THIRDPARTY_LIB='([^']+)'", gate)[1]
61+
if system == "Darwin":
62+
sentinel = "libbrotlienc.a"
63+
files = ["lib/" + sentinel, "lib64/liblance_c.a", "lib64/libpaimon_c.a",
64+
"include/paimon_rust/paimon.h"]
65+
for name in files:
66+
if name != missing:
67+
self.write("installed/" + name)
68+
# Execute the real installation gate, replacing only the expensive build.
69+
self.executable("build-thirdparty.sh", 'printf "%s\\n" "$*" > "$BUILD_LOG"\n')
70+
self.env.update(DORIS_THIRDPARTY=str(self.work), TARGET_SYSTEM=system,
71+
CLEAN=str(clean), PARALLEL="2", BUILD_LOG=str(self.work / "build.log"))
72+
result = self.run_bash(gate)
73+
self.assertEqual(result.returncode, 0, result.stderr)
74+
self.assertEqual((self.work / "build.log").exists(), missing is not None)
75+
if missing is not None:
76+
args = (self.work / "build.log").read_text().strip()
77+
self.assertEqual(args, "-j 2" + (" --clean" if clean else ""))
78+
# A full rebuild also replaces Lance built with the previous Rust version.
79+
self.assertFalse((self.work / "installed/lib64/liblance_c.a").exists())
80+
81+
def test_old_linux_install_rebuilds(self):
82+
self.check_install("Linux", "lib64/libpaimon_c.a")
83+
84+
def test_old_macos_install_rebuilds(self):
85+
self.check_install("Darwin", "lib64/libpaimon_c.a")
86+
87+
def test_missing_header_rebuilds(self):
88+
self.check_install("Linux", "include/paimon_rust/paimon.h")
89+
90+
def test_missing_lance_rebuilds(self):
91+
self.check_install("Linux", "lib64/liblance_c.a")
92+
93+
def test_clean_option_is_preserved(self):
94+
self.check_install("Linux", "lib64/libpaimon_c.a", clean=1)
95+
96+
def test_complete_install_is_reused(self):
97+
self.check_install("Linux", None)
98+
99+
def check_header(self, offline, inherited_offline="", cargo_override=True):
100+
script = (ROOT / "thirdparty/build-thirdparty.sh").read_text()
101+
function = re.search(r"^build_paimon_rust\(\) \{\n.*?^\}", script, re.M | re.S)[0]
102+
cargo = self.executable("custom/cargo", r'''case "$1" in
103+
--version) echo 'cargo 1.94.0' ;;
104+
build)
105+
mkdir -p "$CARGO_TARGET_DIR/release"
106+
touch "$CARGO_TARGET_DIR/release/libpaimon_c.a"
107+
;;
108+
metadata) printf '%s' "${CARGO_NET_OFFLINE:-unset}" > "$METADATA_LOG" ;;
109+
*) exit 90 ;;
110+
esac
111+
''')
112+
# PATH's Cargo deliberately fails: metadata must use the selected executable.
113+
self.executable("bin/cargo", "exit 91\n")
114+
cbindgen = self.executable("bin/cbindgen", r'''"${CARGO:-cargo}" metadata
115+
while [[ "$1" != --output ]]; do shift; done
116+
printf '/* Generated test header */\n' > "$2"
117+
''')
118+
self.write("src/paimon/Cargo.toml")
119+
self.env.update(PATH=str(self.work / "bin") + os.pathsep + os.environ["PATH"],
120+
TP_SOURCE_DIR=str(self.work / "src"), PAIMON_RUST_SOURCE="paimon",
121+
TP_INSTALL_DIR=str(self.work / "installed"), BUILD_DIR="build",
122+
PARALLEL="1", KERNEL="Linux", STRIP_TP_LIB="OFF",
123+
RUSTUP_TOOLCHAIN="1.94.0", PAIMON_RUST_CARGO_OFFLINE=offline,
124+
PAIMON_RUST_CBINDGEN=str(cbindgen),
125+
METADATA_LOG=str(self.work / "metadata.log"))
126+
if cargo_override:
127+
self.env["PAIMON_RUST_CARGO"] = str(cargo)
128+
else:
129+
self.env.pop("PAIMON_RUST_CARGO", None)
130+
self.env["CARGO"] = str(cargo)
131+
if inherited_offline:
132+
self.env["CARGO_NET_OFFLINE"] = inherited_offline
133+
result = self.run_bash("check_if_source_exist() { :; }\n" + function + "\nbuild_paimon_rust")
134+
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
135+
expected = "true" if offline.upper() == "ON" else inherited_offline or "unset"
136+
self.assertEqual((self.work / "metadata.log").read_text(), expected)
137+
self.assertTrue((self.work / "installed/include/paimon_rust/paimon.h").exists())
138+
self.assertTrue((self.work / "installed/lib64/libpaimon_c.a").exists())
139+
140+
def test_header_uses_selected_cargo(self):
141+
self.check_header("OFF")
142+
143+
def test_header_metadata_is_offline(self):
144+
self.check_header("on", cargo_override=False)
145+
146+
def test_offline_override_wins(self):
147+
self.check_header("ON", inherited_offline="false", cargo_override=False)
148+
149+
def test_inherited_offline_is_preserved(self):
150+
self.check_header("OFF", inherited_offline="true", cargo_override=False)
151+
152+
153+
if __name__ == "__main__":
154+
unittest.main(verbosity=2)

0 commit comments

Comments
 (0)