Skip to content

Commit f0e0080

Browse files
authored
branch-4.1:[chore](thirdparty) Upgrade Apache Arrow to 24.0.0(apache#66221) (apache#66480)
### What problem does this PR solve? bp apache#66221 Problem Summary: Doris used Apache Arrow 17.0.0, which blocked consuming newer Arrow fixes and APIs. Upgrade the C++ third-party dependency to 24.0.0, align xsimd with Arrow 24, port the Paimon row-group, INT96, and Parquet LZO compatibility patches, add Arrow Compute to the static dependency graph, and migrate removed Arrow and Parquet APIs while preserving Decimal128 semantics. ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [X] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [x] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [x] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [x] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent e48fc7e commit f0e0080

27 files changed

Lines changed: 1442 additions & 307 deletions

‎.github/workflows/be-ut-mac.yml‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,17 +47,34 @@ jobs:
4747
- 'be/**'
4848
- 'gensrc/proto/**'
4949
- 'gensrc/thrift/**'
50+
arrow_paimon_changes:
51+
- '.github/workflows/be-ut-mac.yml'
52+
- 'build.sh'
53+
- 'env.sh'
54+
- 'thirdparty/arrow-paimon-vars.sh'
55+
- 'thirdparty/vars.sh'
56+
- 'thirdparty/download-thirdparty.sh'
57+
- 'thirdparty/build-thirdparty.sh'
58+
- 'thirdparty/paimon-cpp-cache.cmake'
59+
- 'thirdparty/patches/apache-arrow-*.patch'
60+
- 'thirdparty/patches/paimon-cpp-*.patch'
5061
5162
- name: Ccache ${{ github.ref }}
52-
if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }}
63+
if: |
64+
github.event_name == 'schedule' ||
65+
steps.filter.outputs.be_changes == 'true' ||
66+
steps.filter.outputs.arrow_paimon_changes == 'true'
5367
uses: ./.github/actions/ccache-action
5468
with:
5569
key: BE-UT-macOS
5670
max-size: "5G"
5771
restore-keys: BE-UT-macOS-
5872

5973
- name: Run UT ${{ github.ref }}
60-
if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }}
74+
if: |
75+
github.event_name == 'schedule' ||
76+
steps.filter.outputs.be_changes == 'true' ||
77+
steps.filter.outputs.arrow_paimon_changes == 'true'
6178
run: |
6279
cellars=(
6380
'm4'
@@ -84,6 +101,15 @@ jobs:
84101
)
85102
brew install "${cellars[@]}" || true
86103
104+
# Paimon's bundled dependencies still require pre-CMake-4 policy
105+
# compatibility. Match the supported version used by the full
106+
# thirdparty jobs instead of relying on Homebrew's latest CMake.
107+
brew unlink cmake || true
108+
wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz
109+
tar -xzf cmake-3.25.3-macos-universal.tar.gz
110+
sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/
111+
export CUSTOM_CMAKE=/usr/local/bin/cmake
112+
"${CUSTOM_CMAKE}" --version
87113
pushd thirdparty
88114
branch="${{ github.base_ref }}"
89115
if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' ]]; then
@@ -94,6 +120,26 @@ jobs:
94120
-o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz
95121
fi
96122
tar -xvf doris-thirdparty-prebuilt-darwin-x86_64.tar.xz
123+
124+
# Rebuild the Arrow/Paimon stack when its inputs change or the shared
125+
# prebuilt predates the selected Arrow version/component closure.
126+
# The artifact check also covers scheduled and later BE-only builds,
127+
# where Paths Filter is skipped or reports no Arrow/Paimon changes.
128+
# shellcheck source=thirdparty/arrow-paimon-vars.sh
129+
. ./arrow-paimon-vars.sh
130+
arrow_paimon_prebuilt_is_valid=false
131+
if arrow_paimon_prebuilt_valid installed; then
132+
arrow_paimon_prebuilt_is_valid=true
133+
fi
134+
if [[ "${{ steps.filter.outputs.arrow_paimon_changes }}" == "true" ||
135+
"${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then
136+
curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \
137+
-o doris-thirdparty-source.tgz
138+
tar -zxvf doris-thirdparty-source.tgz
139+
./download-thirdparty.sh arrow paimon_cpp xsimd brotli
140+
export MACOSX_DEPLOYMENT_TARGET=12.0
141+
./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp
142+
fi
97143
popd
98144
99145
export JAVA_HOME="${JAVA_HOME_17_X64%\/}"

‎.github/workflows/build-thirdparty.yml‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ jobs:
3939
github.event.issue.user.login == 'github-actions[bot]')
4040
outputs:
4141
thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }}
42+
arrow_paimon_lifecycle_changes: ${{ steps.filter.outputs.arrow_paimon_lifecycle_changes }}
4243
steps:
4344
- name: Checkout ${{ github.ref }}
4445
uses: actions/checkout@v4
@@ -54,6 +55,30 @@ jobs:
5455
thirdparty_changes:
5556
- 'thirdparty/**'
5657
- 'env.sh'
58+
arrow_paimon_lifecycle_changes:
59+
- 'build.sh'
60+
- 'env.sh'
61+
- 'thirdparty/arrow-paimon-vars.sh'
62+
- 'thirdparty/vars.sh'
63+
- 'thirdparty/build-thirdparty.sh'
64+
- 'thirdparty/download-thirdparty.sh'
65+
- 'thirdparty/paimon-cpp-cache.cmake'
66+
- 'thirdparty/patches/apache-arrow-*.patch'
67+
- 'thirdparty/patches/paimon-cpp-*.patch'
68+
- 'thirdparty/test/arrow-paimon-lifecycle-test.sh'
69+
- '.github/workflows/build-thirdparty.yml'
70+
71+
arrow_paimon_lifecycle_test:
72+
name: Arrow/Paimon Lifecycle Test
73+
needs: changes
74+
if: ${{ needs.changes.outputs.arrow_paimon_lifecycle_changes == 'true' }}
75+
runs-on: ubuntu-22.04
76+
steps:
77+
- name: Checkout ${{ github.ref }}
78+
uses: actions/checkout@v4
79+
80+
- name: Test focused thirdparty lifecycle
81+
run: thirdparty/test/arrow-paimon-lifecycle-test.sh
5782

5883
build_linux:
5984
name: Build Third Party Libraries (Linux)
@@ -195,9 +220,9 @@ jobs:
195220
196221
# Install specific version of cmake
197222
brew unlink cmake || true
198-
wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
199-
tar -xzf cmake-3.22.1-macos-universal.tar.gz
200-
sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/
223+
wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz
224+
tar -xzf cmake-3.25.3-macos-universal.tar.gz
225+
sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/
201226
cmake --version
202227
203228
- name: Build
@@ -260,9 +285,9 @@ jobs:
260285
261286
# Install specific version of cmake
262287
brew unlink cmake || true
263-
wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
264-
tar -xzf cmake-3.22.1-macos-universal.tar.gz
265-
sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/
288+
wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz
289+
tar -xzf cmake-3.25.3-macos-universal.tar.gz
290+
sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/
266291
cmake --version
267292
268293
- name: Build
@@ -278,4 +303,3 @@ jobs:
278303
cd thirdparty
279304
#./build-thirdparty.sh -j "$(nproc)"
280305
./build-thirdparty.sh -j 2
281-

‎be/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -653,7 +653,7 @@ endif()
653653
set(PAIMON_FACTORY_REGISTRY_LIBS)
654654
if (ENABLE_PAIMON_CPP)
655655
# Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
656-
# so arrow, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
656+
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
657657
# thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
658658
# No dual-stack selection needed — single Arrow for everything.
659659

‎be/cmake/thirdparty.cmake‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ add_thirdparty(brotlidec LIB64)
105105
add_thirdparty(brotlienc LIB64)
106106
add_thirdparty(zstd LIB64)
107107
add_thirdparty(arrow LIB64)
108+
add_thirdparty(arrow_compute LIB64)
108109
add_thirdparty(arrow_flight LIB64)
109110
add_thirdparty(arrow_flight_sql LIB64)
110111
add_thirdparty(arrow_dataset LIB64)

‎be/src/format/table/iceberg/arrow_schema_util.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
9898

9999
case iceberg::TypeID::DECIMAL: {
100100
auto* dt = dynamic_cast<DecimalType*>(field.field_type());
101-
arrow_type = arrow::decimal(dt->get_precision(), dt->get_scale());
101+
arrow_type = arrow::decimal128(dt->get_precision(), dt->get_scale());
102102
break;
103103
}
104104

‎be/test/core/data_type_serde/data_type_serde_decimal_test.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ TEST_F(DataTypeDecimalSerDeTest, serdes) {
292292
// Run with UBSan enabled to catch misalignment errors.
293293
TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) {
294294
// 1.Prepare the data.
295-
arrow::Decimal128Builder builder(arrow::decimal(38, 30));
295+
arrow::Decimal128Builder builder(arrow::decimal128(38, 30));
296296
std::vector<std::string> decimal_strings = {"12345.67", "89.10", "1112.13", "1415.16",
297297
"1718.19"};
298298

@@ -318,7 +318,7 @@ TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) {
318318
// 4. Create Arrow array with unaligned memory
319319
auto unaligned_buffer = arrow::Buffer::Wrap(unaligned_data, num_elements * element_size);
320320

321-
auto arr = std::make_shared<arrow::DecimalArray>(arrow::decimal(38, 30), num_elements,
321+
auto arr = std::make_shared<arrow::DecimalArray>(arrow::decimal128(38, 30), num_elements,
322322
unaligned_buffer);
323323

324324
const auto* raw_values_ptr = arr->raw_values();

‎be/test/format/parquet/parquet_expr_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ class ParquetExprTest : public testing::Test {
339339
outfile = std::move(result_file).ValueUnsafe();
340340

341341
::parquet::WriterProperties::Builder builder;
342-
builder.version(::parquet::ParquetVersion::PARQUET_2_0);
342+
builder.version(::parquet::ParquetVersion::PARQUET_2_6);
343343
builder.data_page_version(::parquet::ParquetDataPageVersion::V2);
344344
builder.enable_write_page_index();
345345
builder.compression(::parquet::Compression::SNAPPY);

‎be/test/format_v2/parquet/parquet_reader_test.cpp‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,16 +1004,25 @@ void write_sparse_filter_nested_parquet_file(const std::string& file_path) {
10041004

10051005
::parquet::WriterProperties::Builder builder;
10061006
builder.version(::parquet::ParquetVersion::PARQUET_2_6);
1007-
// V2 and page-index writers preserve record boundaries, so use V1 here to produce the
1008-
// continuation pages that the reader must still handle correctly.
1007+
// Arrow 24 enables page indexes by default. V2 and page-index writers preserve record
1008+
// boundaries, so use V1 without a page index to produce the continuation pages that the
1009+
// reader must still handle correctly.
10091010
builder.data_page_version(::parquet::ParquetDataPageVersion::V1);
1011+
builder.disable_write_page_index();
10101012
builder.compression(::parquet::Compression::UNCOMPRESSED);
10111013
builder.disable_dictionary();
10121014
builder.write_batch_size(8);
1013-
builder.data_pagesize(64);
1015+
// Arrow 24 checks repeated-column page limits only at record or WriteBatch boundaries.
1016+
// A one-byte target deterministically flushes at every eligible boundary, including the
1017+
// deliberately split wide record below.
1018+
builder.data_pagesize(1);
10141019
auto writer = ::parquet::ParquetFileWriter::Open(out, schema, builder.build());
10151020
auto* row_group = writer->AppendRowGroup();
10161021

1022+
// Split each wide record across WriteBatch calls. The first half flushes at the one-byte page
1023+
// limit, so the second call deterministically creates a page whose first repetition level is 1.
1024+
constexpr int64_t SPANNING_BATCH_VALUES = SPANNING_NESTED_VALUES / 2;
1025+
10171026
auto* id_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
10181027
const int32_t ids[] = {1, 2, 3, 4, 5, 6};
10191028
EXPECT_EQ(id_writer->WriteBatch(6, nullptr, nullptr, ids), 6);
@@ -1042,16 +1051,32 @@ void write_sparse_filter_nested_parquet_file(const std::string& file_path) {
10421051
map_keys.push_back(6000);
10431052

10441053
auto* map_key_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
1045-
EXPECT_EQ(map_key_writer->WriteBatch(static_cast<int64_t>(map_repetition_levels.size()),
1046-
map_key_definition_levels.data(),
1054+
constexpr int64_t MAP_PREFIX_LEVELS = 4;
1055+
constexpr int64_t MAP_KEY_PREFIX_VALUES = 2;
1056+
constexpr int64_t MAP_SPLIT_LEVELS = MAP_PREFIX_LEVELS + SPANNING_BATCH_VALUES;
1057+
constexpr int64_t MAP_KEY_SPLIT_VALUES = MAP_KEY_PREFIX_VALUES + SPANNING_BATCH_VALUES;
1058+
EXPECT_EQ(map_key_writer->WriteBatch(MAP_SPLIT_LEVELS, map_key_definition_levels.data(),
10471059
map_repetition_levels.data(), map_keys.data()),
1048-
static_cast<int64_t>(map_keys.size()));
1060+
MAP_KEY_SPLIT_VALUES);
1061+
EXPECT_EQ(map_key_writer->WriteBatch(
1062+
static_cast<int64_t>(map_repetition_levels.size()) - MAP_SPLIT_LEVELS,
1063+
map_key_definition_levels.data() + MAP_SPLIT_LEVELS,
1064+
map_repetition_levels.data() + MAP_SPLIT_LEVELS,
1065+
map_keys.data() + MAP_KEY_SPLIT_VALUES),
1066+
static_cast<int64_t>(map_keys.size()) - MAP_KEY_SPLIT_VALUES);
10491067
map_key_writer->Close();
10501068
auto* map_value_writer = static_cast<::parquet::ByteArrayWriter*>(row_group->NextColumn());
1051-
EXPECT_EQ(map_value_writer->WriteBatch(static_cast<int64_t>(map_repetition_levels.size()),
1052-
map_value_definition_levels.data(),
1069+
constexpr int64_t MAP_VALUE_PREFIX_VALUES = 1;
1070+
constexpr int64_t MAP_VALUE_SPLIT_VALUES = MAP_VALUE_PREFIX_VALUES + SPANNING_BATCH_VALUES;
1071+
EXPECT_EQ(map_value_writer->WriteBatch(MAP_SPLIT_LEVELS, map_value_definition_levels.data(),
10531072
map_repetition_levels.data(), map_values.data()),
1054-
static_cast<int64_t>(map_values.size()));
1073+
MAP_VALUE_SPLIT_VALUES);
1074+
EXPECT_EQ(map_value_writer->WriteBatch(
1075+
static_cast<int64_t>(map_repetition_levels.size()) - MAP_SPLIT_LEVELS,
1076+
map_value_definition_levels.data() + MAP_SPLIT_LEVELS,
1077+
map_repetition_levels.data() + MAP_SPLIT_LEVELS,
1078+
map_values.data() + MAP_VALUE_SPLIT_VALUES),
1079+
static_cast<int64_t>(map_values.size()) - MAP_VALUE_SPLIT_VALUES);
10551080
map_value_writer->Close();
10561081

10571082
std::vector<int16_t> element_repetition_levels {0, 0, 0, 1, 0};
@@ -1069,10 +1094,19 @@ void write_sparse_filter_nested_parquet_file(const std::string& file_path) {
10691094
element_definition_levels.push_back(3);
10701095

10711096
auto* element_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
1072-
EXPECT_EQ(element_writer->WriteBatch(static_cast<int64_t>(element_repetition_levels.size()),
1073-
element_definition_levels.data(),
1097+
constexpr int64_t ELEMENT_PREFIX_LEVELS = 5;
1098+
constexpr int64_t ELEMENT_PREFIX_VALUES = 2;
1099+
constexpr int64_t ELEMENT_SPLIT_LEVELS = ELEMENT_PREFIX_LEVELS + SPANNING_BATCH_VALUES;
1100+
constexpr int64_t ELEMENT_SPLIT_VALUES = ELEMENT_PREFIX_VALUES + SPANNING_BATCH_VALUES;
1101+
EXPECT_EQ(element_writer->WriteBatch(ELEMENT_SPLIT_LEVELS, element_definition_levels.data(),
10741102
element_repetition_levels.data(), element_values.data()),
1075-
static_cast<int64_t>(element_values.size()));
1103+
ELEMENT_SPLIT_VALUES);
1104+
EXPECT_EQ(element_writer->WriteBatch(
1105+
static_cast<int64_t>(element_repetition_levels.size()) - ELEMENT_SPLIT_LEVELS,
1106+
element_definition_levels.data() + ELEMENT_SPLIT_LEVELS,
1107+
element_repetition_levels.data() + ELEMENT_SPLIT_LEVELS,
1108+
element_values.data() + ELEMENT_SPLIT_VALUES),
1109+
static_cast<int64_t>(element_values.size()) - ELEMENT_SPLIT_VALUES);
10761110
element_writer->Close();
10771111
auto* marker_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
10781112
const int16_t marker_definition_levels[] = {1, 0, 1, 1, 1, 1};

‎be/test/format_v2/table/remote_doris_reader_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ TFileRangeDesc remote_doris_range(const BlockingFlightServer& server) {
213213
auto range = remote_doris_range();
214214
auto& params = range.table_format_params.remote_doris_params;
215215
params.__set_location_uri("grpc://localhost:" + std::to_string(server.port()));
216-
arrow::flight::Ticket ticket {.ticket = "ticket"};
216+
arrow::flight::Ticket ticket {"ticket"};
217217
params.__set_ticket(ticket.SerializeToString().ValueOrDie());
218218
return range;
219219
}

‎build.sh‎

Lines changed: 60 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -432,22 +432,74 @@ fi
432432
if [[ "${HELP}" -eq 1 ]]; then
433433
usage
434434
fi
435+
436+
if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_CLOUD} -eq 0 ]]; then
437+
clean_gensrc
438+
clean_be
439+
clean_fe
440+
exit 0
441+
fi
442+
435443
# build thirdparty libraries if necessary. check last thirdparty lib installation
436444
if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then
437445
LAST_THIRDPARTY_LIB='libbrotlienc.a'
438446
else
439447
LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a'
440448
fi
441-
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
442-
echo "Thirdparty libraries need to be build ..."
443-
# need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists
444-
rm -rf "${DORIS_THIRDPARTY}/installed"
445449

446-
if [[ "${CLEAN}" -eq 0 ]]; then
447-
"${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}"
448-
else
449-
"${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean
450+
# The final-library sentinel only proves that some third-party build completed. It cannot
451+
# distinguish an older prebuilt whose Arrow/Paimon closure predates the selected sources.
452+
# shellcheck source=thirdparty/arrow-paimon-vars.sh
453+
. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh"
454+
NEED_ARROW_PAIMON_THIRDPARTY=false
455+
if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 ||
456+
"${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
457+
"${BUILD_INDEX_TOOL}" == "ON" ]]; then
458+
NEED_ARROW_PAIMON_THIRDPARTY=true
459+
fi
460+
461+
rebuild_thirdparty_libraries() {
462+
local remove_installed="$1"
463+
shift
464+
local build_script="${DORIS_THIRDPARTY}/build-thirdparty.sh"
465+
local build_args=(-j "${PARALLEL}")
466+
local selected_thirdparty_root
467+
local checkout_thirdparty_root
468+
469+
if [[ ! -f "${build_script}" ]]; then
470+
echo "Cannot rebuild thirdparty libraries: ${build_script} is missing." >&2
471+
echo "DORIS_THIRDPARTY=${DORIS_THIRDPARTY} is an install-only or incomplete prefix. Use a matching compilation image/prebuilt, or unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree." >&2
472+
exit 1
473+
fi
474+
selected_thirdparty_root="$(cd "${DORIS_THIRDPARTY}" && pwd -P)"
475+
checkout_thirdparty_root="$(cd "${DORIS_HOME}/thirdparty" && pwd -P)"
476+
if [[ "${selected_thirdparty_root}" != "${checkout_thirdparty_root}" ]]; then
477+
echo "Cannot rebuild thirdparty libraries with an external source tree: ${selected_thirdparty_root}." >&2
478+
echo "Unset DORIS_THIRDPARTY to rebuild with this checkout's thirdparty tree, then use the resulting version-matched installation." >&2
479+
exit 1
450480
fi
481+
build_script="${checkout_thirdparty_root}/build-thirdparty.sh"
482+
if [[ "${remove_installed}" == "true" ]]; then
483+
# Some libraries, such as lz4, fail when an earlier installation remains.
484+
rm -rf "${DORIS_THIRDPARTY}/installed"
485+
fi
486+
if [[ "${CLEAN}" -eq 1 ]]; then
487+
build_args+=(--clean)
488+
fi
489+
bash "${build_script}" "${build_args[@]}" "$@"
490+
if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
491+
echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2
492+
exit 1
493+
fi
494+
}
495+
496+
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
497+
echo "Thirdparty libraries need to be build ..."
498+
rebuild_thirdparty_libraries true
499+
elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] &&
500+
! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
501+
echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
502+
rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
451503
fi
452504

453505
update_submodule() {
@@ -486,13 +538,6 @@ update_submodule() {
486538
fi
487539
}
488540

489-
if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_CLOUD} -eq 0 ]]; then
490-
clean_gensrc
491-
clean_be
492-
clean_fe
493-
exit 0
494-
fi
495-
496541
if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
497542
if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
498543
GLIBC_COMPATIBILITY='ON'

0 commit comments

Comments
 (0)