|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 3 | +# or more contributor license agreements. See the NOTICE file |
| 4 | +# distributed with this work for additional information |
| 5 | +# regarding copyright ownership. The ASF licenses this file |
| 6 | +# to you under the Apache License, Version 2.0 (the |
| 7 | +# "License"); you may not use this file except in compliance |
| 8 | +# with the License. You may obtain a copy of the License at |
| 9 | +# |
| 10 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | +# |
| 12 | +# Unless required by applicable law or agreed to in writing, |
| 13 | +# software distributed under the License is distributed on an |
| 14 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | +# KIND, either express or implied. See the License for the |
| 16 | +# specific language governing permissions and limitations |
| 17 | +# under the License. |
| 18 | + |
| 19 | +"""Run with python3 thirdparty/test/paimon-build-test.py; no Rust downloads required.""" |
| 20 | + |
| 21 | +import os |
| 22 | +from pathlib import Path |
| 23 | +import re |
| 24 | +import subprocess |
| 25 | +import tempfile |
| 26 | +import unittest |
| 27 | + |
| 28 | + |
| 29 | +ROOT = Path(__file__).resolve().parents[2] |
| 30 | + |
| 31 | + |
| 32 | +class PaimonBuildTest(unittest.TestCase): |
| 33 | + def setUp(self): |
| 34 | + self.temp = tempfile.TemporaryDirectory() |
| 35 | + self.addCleanup(self.temp.cleanup) |
| 36 | + self.work = Path(self.temp.name) |
| 37 | + self.env = dict(os.environ) |
| 38 | + for key in ("CARGO", "CARGO_NET_OFFLINE", "RUSTUP_TOOLCHAIN"): |
| 39 | + self.env.pop(key, None) |
| 40 | + |
| 41 | + def write(self, name, text=""): |
| 42 | + path = self.work / name |
| 43 | + path.parent.mkdir(parents=True, exist_ok=True) |
| 44 | + path.write_text(text) |
| 45 | + return path |
| 46 | + |
| 47 | + def executable(self, name, text): |
| 48 | + path = self.write(name, "#!/bin/bash\nset -e\n" + text) |
| 49 | + path.chmod(0o755) |
| 50 | + return path |
| 51 | + |
| 52 | + def run_bash(self, script): |
| 53 | + return subprocess.run(["bash", "-ec", script], env=self.env, |
| 54 | + cwd=self.work, text=True, capture_output=True) |
| 55 | + |
| 56 | + def check_install(self, system, missing, clean=0): |
| 57 | + script = (ROOT / "build.sh").read_text() |
| 58 | + gate = script[script.index("# build thirdparty libraries if necessary."): |
| 59 | + script.index("update_submodule() {")] |
| 60 | + sentinel = re.search(r"else\n LAST_THIRDPARTY_LIB='([^']+)'", gate)[1] |
| 61 | + if system == "Darwin": |
| 62 | + sentinel = "libbrotlienc.a" |
| 63 | + files = ["lib/" + sentinel, "lib64/liblance_c.a", "lib64/libpaimon_c.a", |
| 64 | + "include/paimon_rust/paimon.h"] |
| 65 | + for name in files: |
| 66 | + if name != missing: |
| 67 | + self.write("installed/" + name) |
| 68 | + # Execute the real installation gate, replacing only the expensive build. |
| 69 | + self.executable("build-thirdparty.sh", 'printf "%s\\n" "$*" > "$BUILD_LOG"\n') |
| 70 | + self.env.update(DORIS_THIRDPARTY=str(self.work), TARGET_SYSTEM=system, |
| 71 | + CLEAN=str(clean), PARALLEL="2", BUILD_LOG=str(self.work / "build.log")) |
| 72 | + result = self.run_bash(gate) |
| 73 | + self.assertEqual(result.returncode, 0, result.stderr) |
| 74 | + self.assertEqual((self.work / "build.log").exists(), missing is not None) |
| 75 | + if missing is not None: |
| 76 | + args = (self.work / "build.log").read_text().strip() |
| 77 | + self.assertEqual(args, "-j 2" + (" --clean" if clean else "")) |
| 78 | + # A full rebuild also replaces Lance built with the previous Rust version. |
| 79 | + self.assertFalse((self.work / "installed/lib64/liblance_c.a").exists()) |
| 80 | + |
| 81 | + def test_old_linux_install_rebuilds(self): |
| 82 | + self.check_install("Linux", "lib64/libpaimon_c.a") |
| 83 | + |
| 84 | + def test_old_macos_install_rebuilds(self): |
| 85 | + self.check_install("Darwin", "lib64/libpaimon_c.a") |
| 86 | + |
| 87 | + def test_missing_header_rebuilds(self): |
| 88 | + self.check_install("Linux", "include/paimon_rust/paimon.h") |
| 89 | + |
| 90 | + def test_missing_lance_rebuilds(self): |
| 91 | + self.check_install("Linux", "lib64/liblance_c.a") |
| 92 | + |
| 93 | + def test_clean_option_is_preserved(self): |
| 94 | + self.check_install("Linux", "lib64/libpaimon_c.a", clean=1) |
| 95 | + |
| 96 | + def test_complete_install_is_reused(self): |
| 97 | + self.check_install("Linux", None) |
| 98 | + |
| 99 | + def check_header(self, offline, inherited_offline="", cargo_override=True): |
| 100 | + script = (ROOT / "thirdparty/build-thirdparty.sh").read_text() |
| 101 | + function = re.search(r"^build_paimon_rust\(\) \{\n.*?^\}", script, re.M | re.S)[0] |
| 102 | + cargo = self.executable("custom/cargo", r'''case "$1" in |
| 103 | + --version) echo 'cargo 1.94.0' ;; |
| 104 | + build) |
| 105 | + mkdir -p "$CARGO_TARGET_DIR/release" |
| 106 | + touch "$CARGO_TARGET_DIR/release/libpaimon_c.a" |
| 107 | + ;; |
| 108 | + metadata) printf '%s' "${CARGO_NET_OFFLINE:-unset}" > "$METADATA_LOG" ;; |
| 109 | + *) exit 90 ;; |
| 110 | +esac |
| 111 | +''') |
| 112 | + # PATH's Cargo deliberately fails: metadata must use the selected executable. |
| 113 | + self.executable("bin/cargo", "exit 91\n") |
| 114 | + cbindgen = self.executable("bin/cbindgen", r'''"${CARGO:-cargo}" metadata |
| 115 | +while [[ "$1" != --output ]]; do shift; done |
| 116 | +printf '/* Generated test header */\n' > "$2" |
| 117 | +''') |
| 118 | + self.write("src/paimon/Cargo.toml") |
| 119 | + self.env.update(PATH=str(self.work / "bin") + os.pathsep + os.environ["PATH"], |
| 120 | + TP_SOURCE_DIR=str(self.work / "src"), PAIMON_RUST_SOURCE="paimon", |
| 121 | + TP_INSTALL_DIR=str(self.work / "installed"), BUILD_DIR="build", |
| 122 | + PARALLEL="1", KERNEL="Linux", STRIP_TP_LIB="OFF", |
| 123 | + RUSTUP_TOOLCHAIN="1.94.0", PAIMON_RUST_CARGO_OFFLINE=offline, |
| 124 | + PAIMON_RUST_CBINDGEN=str(cbindgen), |
| 125 | + METADATA_LOG=str(self.work / "metadata.log")) |
| 126 | + if cargo_override: |
| 127 | + self.env["PAIMON_RUST_CARGO"] = str(cargo) |
| 128 | + else: |
| 129 | + self.env.pop("PAIMON_RUST_CARGO", None) |
| 130 | + self.env["CARGO"] = str(cargo) |
| 131 | + if inherited_offline: |
| 132 | + self.env["CARGO_NET_OFFLINE"] = inherited_offline |
| 133 | + result = self.run_bash("check_if_source_exist() { :; }\n" + function + "\nbuild_paimon_rust") |
| 134 | + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) |
| 135 | + expected = "true" if offline.upper() == "ON" else inherited_offline or "unset" |
| 136 | + self.assertEqual((self.work / "metadata.log").read_text(), expected) |
| 137 | + self.assertTrue((self.work / "installed/include/paimon_rust/paimon.h").exists()) |
| 138 | + self.assertTrue((self.work / "installed/lib64/libpaimon_c.a").exists()) |
| 139 | + |
| 140 | + def test_header_uses_selected_cargo(self): |
| 141 | + self.check_header("OFF") |
| 142 | + |
| 143 | + def test_header_metadata_is_offline(self): |
| 144 | + self.check_header("on", cargo_override=False) |
| 145 | + |
| 146 | + def test_offline_override_wins(self): |
| 147 | + self.check_header("ON", inherited_offline="false", cargo_override=False) |
| 148 | + |
| 149 | + def test_inherited_offline_is_preserved(self): |
| 150 | + self.check_header("OFF", inherited_offline="true", cargo_override=False) |
| 151 | + |
| 152 | + |
| 153 | +if __name__ == "__main__": |
| 154 | + unittest.main(verbosity=2) |
0 commit comments