Skip to content

Commit f534dcf

Browse files
committed
[fix](build) Recover interrupted Paimon header and archive installation
1 parent d2cfcc1 commit f534dcf

3 files changed

Lines changed: 96 additions & 12 deletions

File tree

‎build.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -488,10 +488,14 @@ fi
488488

489489
# Old installs can contain the sentinel but predate Paimon. Rebuild the whole
490490
# dependency set so Lance and Paimon also use the same selected Rust toolchain.
491+
# An interrupted header/archive publication must be repaired before either file is reused.
491492
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
492493
! -f "${DORIS_THIRDPARTY}/installed/lib64/liblance_c.a" ||
493494
! -f "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
494-
! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ]]; then
495+
! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
496+
! -s "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
497+
! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
498+
-e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]]; then
495499
echo "Thirdparty libraries need to be build ..."
496500
# need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists
497501
rm -rf "${DORIS_THIRDPARTY}/installed"

‎thirdparty/build-thirdparty.sh‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2390,6 +2390,29 @@ install_rust_archive() {
23902390
)
23912391
}
23922392

2393+
install_paimon_rust() {
2394+
(
2395+
set -e
2396+
local archive="$1"
2397+
local header="$2"
2398+
local destination="${TP_INSTALL_DIR}/include/paimon_rust/paimon.h"
2399+
local incomplete="${TP_INSTALL_DIR}/lib64/.paimon-installing"
2400+
mkdir -p "${TP_INSTALL_DIR}/include/paimon_rust" "${TP_INSTALL_DIR}/lib64"
2401+
local staged
2402+
staged="$(mktemp "${destination}.tmp.XXXXXX")"
2403+
trap 'rm -f "${staged}"' EXIT
2404+
cp -p "${header}" "${staged}"
2405+
test -s "${staged}"
2406+
test -s "${archive}"
2407+
# The two renames cannot be atomic together. Keep this recovery marker until
2408+
# both succeed so build.sh never reuses a mixed pair after interruption.
2409+
touch "${incomplete}"
2410+
install_rust_archive "${archive}"
2411+
mv -f "${staged}" "${destination}"
2412+
rm -f "${incomplete}"
2413+
)
2414+
}
2415+
23932416
build_lance_c() {
23942417
check_if_source_exist "${LANCE_C_SOURCE}"
23952418
cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}"
@@ -2585,11 +2608,7 @@ EOF
25852608
--config "${cbindgen_toml}" \
25862609
--output "${BUILD_DIR}/release/paimon.h"
25872610

2588-
mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
2589-
rm -rf "${TP_INSTALL_DIR}/include/paimon_rust"
2590-
mkdir -p "${TP_INSTALL_DIR}/include/paimon_rust"
2591-
cp -v "${BUILD_DIR}/release/paimon.h" "${TP_INSTALL_DIR}/include/paimon_rust/"
2592-
install_rust_archive "${BUILD_DIR}/release/libpaimon_c.a"
2611+
install_paimon_rust "${BUILD_DIR}/release/libpaimon_c.a" "${BUILD_DIR}/release/paimon.h"
25932612
}
25942613

25952614
if [[ "${#packages[@]}" -eq 0 ]]; then

‎thirdparty/test/paimon-build-test.py‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ def run_bash(self, script):
5353
return subprocess.run(["bash", "-ec", script], env=self.env,
5454
cwd=self.work, text=True, capture_output=True)
5555

56-
def check_install(self, system, missing, clean=0):
56+
def check_install(self, system, missing, clean=0, empty=None, expect_rebuild=False):
5757
script = (ROOT / "build.sh").read_text()
5858
gate = script[script.index("# build thirdparty libraries if necessary."):
5959
script.index("update_submodule() {")]
@@ -64,15 +64,15 @@ def check_install(self, system, missing, clean=0):
6464
"include/paimon_rust/paimon.h"]
6565
for name in files:
6666
if name != missing:
67-
self.write("installed/" + name)
67+
self.write("installed/" + name, "" if name == empty else "complete")
6868
# Execute the real installation gate, replacing only the expensive build.
6969
self.executable("build-thirdparty.sh", 'printf "%s\\n" "$*" > "$BUILD_LOG"\n')
7070
self.env.update(DORIS_THIRDPARTY=str(self.work), TARGET_SYSTEM=system,
7171
CLEAN=str(clean), PARALLEL="2", BUILD_LOG=str(self.work / "build.log"))
7272
result = self.run_bash(gate)
7373
self.assertEqual(result.returncode, 0, result.stderr)
74-
self.assertEqual((self.work / "build.log").exists(), missing is not None)
75-
if missing is not None:
74+
self.assertEqual((self.work / "build.log").exists(), missing is not None or expect_rebuild)
75+
if missing is not None or expect_rebuild:
7676
args = (self.work / "build.log").read_text().strip()
7777
self.assertEqual(args, "-j 2" + (" --clean" if clean else ""))
7878
# A full rebuild also replaces Lance built with the previous Rust version.
@@ -96,9 +96,70 @@ def test_clean_option_is_preserved(self):
9696
def test_complete_install_is_reused(self):
9797
self.check_install("Linux", None)
9898

99+
def test_incomplete_pair_rebuilds(self):
100+
self.write("installed/lib64/.paimon-installing")
101+
self.check_install("Linux", None, expect_rebuild=True)
102+
103+
def test_empty_header_rebuilds(self):
104+
self.check_install("Linux", None, empty="include/paimon_rust/paimon.h", expect_rebuild=True)
105+
106+
def test_empty_archive_rebuilds(self):
107+
self.check_install("Linux", None, empty="lib64/libpaimon_c.a", expect_rebuild=True)
108+
99109
def archive_installer(self):
100110
script = (ROOT / "thirdparty/build-thirdparty.sh").read_text()
101-
return re.search(r"^install_rust_archive\(\) \{\n.*?^\}", script, re.M | re.S)[0]
111+
return "\n".join(re.search(r"^" + name + r"\(\) \{\n.*?^\}", script, re.M | re.S)[0]
112+
for name in ("install_rust_archive", "install_paimon_rust"))
113+
114+
def test_header_archive_pair_publication(self):
115+
archive = self.write("build/libpaimon_c.a", "new archive")
116+
header = self.write("build/paimon.h", "new header")
117+
target_archive = self.work / "installed/lib64/libpaimon_c.a"
118+
target_header = self.work / "installed/include/paimon_rust/paimon.h"
119+
marker = target_archive.parent / ".paimon-installing"
120+
self.env.update(TP_INSTALL_DIR=str(self.work / "installed"), STRIP_TP_LIB="OFF",
121+
KERNEL="Linux", ARCHIVE=str(archive), HEADER=str(header))
122+
failures = {
123+
"header-copy": 'cp() { if [[ "$1" = -p && "$2" = "$HEADER" ]]; then '
124+
'printf partial > "${@: -1}"; return 1; fi; command cp "$@"; }',
125+
"archive-publish": 'mv() { if [[ "${@: -1}" = */libpaimon_c.a ]]; then return 1; fi; '
126+
'command mv "$@"; }',
127+
"header-publish": 'mv() { if [[ "${@: -1}" = */paimon.h ]]; then return 1; fi; '
128+
'command mv "$@"; }',
129+
"success": "",
130+
}
131+
for replacement in (False, True):
132+
for stage, injection in failures.items():
133+
with self.subTest(replacement=replacement, stage=stage):
134+
for path in (target_archive, target_header, marker):
135+
if path.exists():
136+
path.unlink()
137+
path.parent.mkdir(parents=True, exist_ok=True)
138+
if replacement:
139+
target_archive.write_text("old archive")
140+
target_header.write_text("old header")
141+
result = self.run_bash(self.archive_installer() + "\n" + injection
142+
+ '\ninstall_paimon_rust "$ARCHIVE" "$HEADER"')
143+
if stage == "success":
144+
self.assertEqual(result.returncode, 0, result.stderr)
145+
self.assertEqual(target_header.read_text(), "new header")
146+
self.assertEqual(target_archive.read_text(), "new archive")
147+
self.assertFalse(marker.exists())
148+
else:
149+
self.assertNotEqual(result.returncode, 0)
150+
if replacement:
151+
self.assertEqual(target_header.read_text(), "old header")
152+
else:
153+
self.assertFalse(target_header.exists())
154+
self.assertEqual(marker.exists(), stage != "header-copy")
155+
# A retry must repair the pair and clear its incomplete marker.
156+
retry = self.run_bash(self.archive_installer()
157+
+ '\ninstall_paimon_rust "$ARCHIVE" "$HEADER"')
158+
self.assertEqual(retry.returncode, 0, retry.stderr)
159+
self.assertEqual(target_header.read_text(), "new header")
160+
self.assertEqual(target_archive.read_text(), "new archive")
161+
self.assertFalse(marker.exists())
162+
self.assertEqual(list(target_header.parent.glob("*.tmp.*")), [])
102163

103164
def check_archive_publication(self, name):
104165
source = self.write("build/" + name, "new archive")
@@ -150,7 +211,7 @@ def check_header(self, offline, inherited_offline="", cargo_override=True):
150211
--version) echo 'cargo 1.94.0' ;;
151212
build)
152213
mkdir -p "$CARGO_TARGET_DIR/release"
153-
touch "$CARGO_TARGET_DIR/release/libpaimon_c.a"
214+
printf 'archive' > "$CARGO_TARGET_DIR/release/libpaimon_c.a"
154215
;;
155216
metadata) printf '%s' "${CARGO_NET_OFFLINE:-unset}" > "$METADATA_LOG" ;;
156217
*) exit 90 ;;

0 commit comments

Comments
 (0)