Skip to content

Commit 5baf045

Browse files
authored
[chore](thirdparty) Upgrade Apache Arrow to 24.0.0 (apache#66221)
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Doris used Apache Arrow 17.0.0, which blocked consuming newer Arrow fixes and APIs. Upgrade the C++ third-party dependency to 24.0.0, align xsimd with Arrow 24, port the Paimon row-group, INT96, and Parquet LZO compatibility patches, add Arrow Compute to the static dependency graph, and migrate removed Arrow and Parquet APIs while preserving Decimal128 semantics. ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [X] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [x] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [x] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [x] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 0bdb7b6 commit 5baf045

24 files changed

Lines changed: 1140 additions & 307 deletions

‎.github/workflows/be-ut-mac.yml‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,34 @@ jobs:
4848
- 'be/**'
4949
- 'gensrc/proto/**'
5050
- 'gensrc/thrift/**'
51+
arrow_paimon_changes:
52+
- '.github/workflows/be-ut-mac.yml'
53+
- 'build.sh'
54+
- 'env.sh'
55+
- 'thirdparty/arrow-paimon-vars.sh'
56+
- 'thirdparty/vars.sh'
57+
- 'thirdparty/download-thirdparty.sh'
58+
- 'thirdparty/build-thirdparty.sh'
59+
- 'thirdparty/paimon-cpp-cache.cmake'
60+
- 'thirdparty/patches/apache-arrow-*.patch'
61+
- 'thirdparty/patches/paimon-cpp-*.patch'
5162
5263
- name: Ccache ${{ github.ref }}
53-
if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }}
64+
if: |
65+
github.event_name == 'schedule' ||
66+
steps.filter.outputs.be_changes == 'true' ||
67+
steps.filter.outputs.arrow_paimon_changes == 'true'
5468
uses: ./.github/actions/ccache-action
5569
with:
5670
key: BE-UT-macOS
5771
max-size: "5G"
5872
restore-keys: BE-UT-macOS-
5973

6074
- name: Build BE ${{ github.ref }}
61-
if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }}
75+
if: |
76+
github.event_name == 'schedule' ||
77+
steps.filter.outputs.be_changes == 'true' ||
78+
steps.filter.outputs.arrow_paimon_changes == 'true'
6279
run: |
6380
cellars=(
6481
'm4'
@@ -86,6 +103,16 @@ jobs:
86103
)
87104
brew install "${cellars[@]}" || true
88105
106+
# Paimon's bundled dependencies still require pre-CMake-4 policy
107+
# compatibility. Match the supported version used by the full
108+
# thirdparty jobs instead of relying on Homebrew's latest CMake.
109+
brew unlink cmake || true
110+
wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz
111+
tar -xzf cmake-3.25.3-macos-universal.tar.gz
112+
sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/
113+
export CUSTOM_CMAKE=/usr/local/bin/cmake
114+
"${CUSTOM_CMAKE}" --version
115+
89116
# macos-15 runners are Apple Silicon (arm64), so download the arm64
90117
# prebuilt thirdparty. Using the x86_64 archive makes the linker ignore
91118
# every thirdparty static library ("found architecture 'x86_64',
@@ -101,6 +128,26 @@ jobs:
101128
-o doris-thirdparty-prebuilt-darwin-arm64.tar.xz
102129
fi
103130
tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz
131+
132+
# Rebuild the Arrow/Paimon stack when its inputs change or the shared
133+
# prebuilt predates the selected Arrow version/component closure.
134+
# The artifact check also covers scheduled and later BE-only builds,
135+
# where Paths Filter is skipped or reports no Arrow/Paimon changes.
136+
# shellcheck source=thirdparty/arrow-paimon-vars.sh
137+
. ./arrow-paimon-vars.sh
138+
arrow_paimon_prebuilt_is_valid=false
139+
if arrow_paimon_prebuilt_valid installed; then
140+
arrow_paimon_prebuilt_is_valid=true
141+
fi
142+
if [[ "${{ steps.filter.outputs.arrow_paimon_changes }}" == "true" ||
143+
"${arrow_paimon_prebuilt_is_valid}" != "true" ]]; then
144+
curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \
145+
-o doris-thirdparty-source.tgz
146+
tar -zxvf doris-thirdparty-source.tgz
147+
./download-thirdparty.sh arrow paimon_cpp xsimd brotli
148+
export MACOSX_DEPLOYMENT_TARGET=12.0
149+
./build-thirdparty.sh -j "$(nproc)" arrow paimon_cpp
150+
fi
104151
popd
105152
106153
# macos-15 runners are Apple Silicon (arm64), so the JDK env var is

‎.github/workflows/build-thirdparty.yml‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ jobs:
3939
github.event.issue.user.login == 'github-actions[bot]')
4040
outputs:
4141
thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }}
42+
arrow_paimon_lifecycle_changes: ${{ steps.filter.outputs.arrow_paimon_lifecycle_changes }}
4243
steps:
4344
- name: Checkout ${{ github.ref }}
4445
uses: actions/checkout@v4
@@ -54,6 +55,30 @@ jobs:
5455
thirdparty_changes:
5556
- 'thirdparty/**'
5657
- 'env.sh'
58+
arrow_paimon_lifecycle_changes:
59+
- 'build.sh'
60+
- 'env.sh'
61+
- 'thirdparty/arrow-paimon-vars.sh'
62+
- 'thirdparty/vars.sh'
63+
- 'thirdparty/build-thirdparty.sh'
64+
- 'thirdparty/download-thirdparty.sh'
65+
- 'thirdparty/paimon-cpp-cache.cmake'
66+
- 'thirdparty/patches/apache-arrow-*.patch'
67+
- 'thirdparty/patches/paimon-cpp-*.patch'
68+
- 'thirdparty/test/arrow-paimon-lifecycle-test.sh'
69+
- '.github/workflows/build-thirdparty.yml'
70+
71+
arrow_paimon_lifecycle_test:
72+
name: Arrow/Paimon Lifecycle Test
73+
needs: changes
74+
if: ${{ needs.changes.outputs.arrow_paimon_lifecycle_changes == 'true' }}
75+
runs-on: ubuntu-22.04
76+
steps:
77+
- name: Checkout ${{ github.ref }}
78+
uses: actions/checkout@v4
79+
80+
- name: Test focused thirdparty lifecycle
81+
run: thirdparty/test/arrow-paimon-lifecycle-test.sh
5782

5883
build_linux:
5984
name: Build Third Party Libraries (Linux)
@@ -195,9 +220,9 @@ jobs:
195220
196221
# Install specific version of cmake
197222
brew unlink cmake || true
198-
wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
199-
tar -xzf cmake-3.22.1-macos-universal.tar.gz
200-
sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/
223+
wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz
224+
tar -xzf cmake-3.25.3-macos-universal.tar.gz
225+
sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/
201226
cmake --version
202227
203228
- name: Build
@@ -260,9 +285,9 @@ jobs:
260285
261286
# Install specific version of cmake
262287
brew unlink cmake || true
263-
wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
264-
tar -xzf cmake-3.22.1-macos-universal.tar.gz
265-
sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/
288+
wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-macos-universal.tar.gz
289+
tar -xzf cmake-3.25.3-macos-universal.tar.gz
290+
sudo cp -r cmake-3.25.3-macos-universal/CMake.app/Contents/* /usr/local/
266291
cmake --version
267292
268293
- name: Build
@@ -278,4 +303,3 @@ jobs:
278303
cd thirdparty
279304
#./build-thirdparty.sh -j "$(nproc)"
280305
./build-thirdparty.sh -j 2
281-

‎be/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -666,7 +666,7 @@ endif()
666666
set(PAIMON_FACTORY_REGISTRY_LIBS)
667667
if (ENABLE_PAIMON_CPP)
668668
# Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
669-
# so arrow, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
669+
# so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via
670670
# thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
671671
# No dual-stack selection needed — single Arrow for everything.
672672

‎be/cmake/thirdparty.cmake‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ add_thirdparty(brotlidec LIB64)
105105
add_thirdparty(brotlienc LIB64)
106106
add_thirdparty(zstd LIB64)
107107
add_thirdparty(arrow LIB64)
108+
add_thirdparty(arrow_compute LIB64)
108109
add_thirdparty(arrow_flight LIB64)
109110
add_thirdparty(arrow_flight_sql LIB64)
110111
add_thirdparty(arrow_dataset LIB64)

‎be/src/format/table/iceberg/arrow_schema_util.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ Status ArrowSchemaUtil::convert_to(const iceberg::NestedField& field,
9797

9898
case iceberg::TypeID::DECIMAL: {
9999
auto* dt = dynamic_cast<DecimalType*>(field.field_type());
100-
arrow_type = arrow::decimal(dt->get_precision(), dt->get_scale());
100+
arrow_type = arrow::decimal128(dt->get_precision(), dt->get_scale());
101101
break;
102102
}
103103

‎be/test/core/data_type_serde/data_type_serde_decimal_test.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ TEST_F(DataTypeDecimalSerDeTest, serdes) {
309309
// Run with UBSan enabled to catch misalignment errors.
310310
TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) {
311311
// 1.Prepare the data.
312-
arrow::Decimal128Builder builder(arrow::decimal(38, 30));
312+
arrow::Decimal128Builder builder(arrow::decimal128(38, 30));
313313
std::vector<std::string> decimal_strings = {"12345.67", "89.10", "1112.13", "1415.16",
314314
"1718.19"};
315315

@@ -335,7 +335,7 @@ TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAligned) {
335335
// 4. Create Arrow array with unaligned memory
336336
auto unaligned_buffer = arrow::Buffer::Wrap(unaligned_data, num_elements * element_size);
337337

338-
auto arr = std::make_shared<arrow::DecimalArray>(arrow::decimal(38, 30), num_elements,
338+
auto arr = std::make_shared<arrow::DecimalArray>(arrow::decimal128(38, 30), num_elements,
339339
unaligned_buffer);
340340

341341
const auto* raw_values_ptr = arr->raw_values();
@@ -432,7 +432,7 @@ uint8_t* find_misaligned_address(std::vector<uint8_t>& storage) {
432432
}
433433

434434
TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAlignedDecimalV2TypeAlignment) {
435-
arrow::Decimal128Builder builder(arrow::decimal(27, 9));
435+
arrow::Decimal128Builder builder(arrow::decimal128(27, 9));
436436
std::vector<std::string> decimal_strings = {"1.000000000", "-123456789012345678.123456789"};
437437

438438
for (const auto& str : decimal_strings) {
@@ -453,7 +453,7 @@ TEST_F(DataTypeDecimalSerDeTest, ArrowMemNotAlignedDecimalV2TypeAlignment) {
453453
memcpy(unaligned_data, original_data, num_elements * element_size);
454454

455455
auto unaligned_buffer = arrow::Buffer::Wrap(unaligned_data, num_elements * element_size);
456-
auto arr = std::make_shared<arrow::DecimalArray>(arrow::decimal(27, 9), num_elements,
456+
auto arr = std::make_shared<arrow::DecimalArray>(arrow::decimal128(27, 9), num_elements,
457457
unaligned_buffer);
458458

459459
const auto* raw_values_ptr = arr->raw_values();

‎be/test/format_v2/parquet/parquet_reader_test.cpp‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -875,16 +875,25 @@ void write_sparse_filter_nested_parquet_file(const std::string& file_path) {
875875

876876
::parquet::WriterProperties::Builder builder;
877877
builder.version(::parquet::ParquetVersion::PARQUET_2_6);
878-
// V2 and page-index writers preserve record boundaries, so use V1 here to produce the
879-
// continuation pages that the reader must still handle correctly.
878+
// Arrow 24 enables page indexes by default. V2 and page-index writers preserve record
879+
// boundaries, so use V1 without a page index to produce the continuation pages that the
880+
// reader must still handle correctly.
880881
builder.data_page_version(::parquet::ParquetDataPageVersion::V1);
882+
builder.disable_write_page_index();
881883
builder.compression(::parquet::Compression::UNCOMPRESSED);
882884
builder.disable_dictionary();
883885
builder.write_batch_size(8);
884-
builder.data_pagesize(64);
886+
// Arrow 24 checks repeated-column page limits only at record or WriteBatch boundaries.
887+
// A one-byte target deterministically flushes at every eligible boundary, including the
888+
// deliberately split wide record below.
889+
builder.data_pagesize(1);
885890
auto writer = ::parquet::ParquetFileWriter::Open(out, schema, builder.build());
886891
auto* row_group = writer->AppendRowGroup();
887892

893+
// Split each wide record across WriteBatch calls. The first half flushes at the one-byte page
894+
// limit, so the second call deterministically creates a page whose first repetition level is 1.
895+
constexpr int64_t SPANNING_BATCH_VALUES = SPANNING_NESTED_VALUES / 2;
896+
888897
auto* id_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
889898
const int32_t ids[] = {1, 2, 3, 4, 5, 6};
890899
EXPECT_EQ(id_writer->WriteBatch(6, nullptr, nullptr, ids), 6);
@@ -913,16 +922,32 @@ void write_sparse_filter_nested_parquet_file(const std::string& file_path) {
913922
map_keys.push_back(6000);
914923

915924
auto* map_key_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
916-
EXPECT_EQ(map_key_writer->WriteBatch(static_cast<int64_t>(map_repetition_levels.size()),
917-
map_key_definition_levels.data(),
925+
constexpr int64_t MAP_PREFIX_LEVELS = 4;
926+
constexpr int64_t MAP_KEY_PREFIX_VALUES = 2;
927+
constexpr int64_t MAP_SPLIT_LEVELS = MAP_PREFIX_LEVELS + SPANNING_BATCH_VALUES;
928+
constexpr int64_t MAP_KEY_SPLIT_VALUES = MAP_KEY_PREFIX_VALUES + SPANNING_BATCH_VALUES;
929+
EXPECT_EQ(map_key_writer->WriteBatch(MAP_SPLIT_LEVELS, map_key_definition_levels.data(),
918930
map_repetition_levels.data(), map_keys.data()),
919-
static_cast<int64_t>(map_keys.size()));
931+
MAP_KEY_SPLIT_VALUES);
932+
EXPECT_EQ(map_key_writer->WriteBatch(
933+
static_cast<int64_t>(map_repetition_levels.size()) - MAP_SPLIT_LEVELS,
934+
map_key_definition_levels.data() + MAP_SPLIT_LEVELS,
935+
map_repetition_levels.data() + MAP_SPLIT_LEVELS,
936+
map_keys.data() + MAP_KEY_SPLIT_VALUES),
937+
static_cast<int64_t>(map_keys.size()) - MAP_KEY_SPLIT_VALUES);
920938
map_key_writer->Close();
921939
auto* map_value_writer = static_cast<::parquet::ByteArrayWriter*>(row_group->NextColumn());
922-
EXPECT_EQ(map_value_writer->WriteBatch(static_cast<int64_t>(map_repetition_levels.size()),
923-
map_value_definition_levels.data(),
940+
constexpr int64_t MAP_VALUE_PREFIX_VALUES = 1;
941+
constexpr int64_t MAP_VALUE_SPLIT_VALUES = MAP_VALUE_PREFIX_VALUES + SPANNING_BATCH_VALUES;
942+
EXPECT_EQ(map_value_writer->WriteBatch(MAP_SPLIT_LEVELS, map_value_definition_levels.data(),
924943
map_repetition_levels.data(), map_values.data()),
925-
static_cast<int64_t>(map_values.size()));
944+
MAP_VALUE_SPLIT_VALUES);
945+
EXPECT_EQ(map_value_writer->WriteBatch(
946+
static_cast<int64_t>(map_repetition_levels.size()) - MAP_SPLIT_LEVELS,
947+
map_value_definition_levels.data() + MAP_SPLIT_LEVELS,
948+
map_repetition_levels.data() + MAP_SPLIT_LEVELS,
949+
map_values.data() + MAP_VALUE_SPLIT_VALUES),
950+
static_cast<int64_t>(map_values.size()) - MAP_VALUE_SPLIT_VALUES);
926951
map_value_writer->Close();
927952

928953
std::vector<int16_t> element_repetition_levels {0, 0, 0, 1, 0};
@@ -940,10 +965,19 @@ void write_sparse_filter_nested_parquet_file(const std::string& file_path) {
940965
element_definition_levels.push_back(3);
941966

942967
auto* element_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
943-
EXPECT_EQ(element_writer->WriteBatch(static_cast<int64_t>(element_repetition_levels.size()),
944-
element_definition_levels.data(),
968+
constexpr int64_t ELEMENT_PREFIX_LEVELS = 5;
969+
constexpr int64_t ELEMENT_PREFIX_VALUES = 2;
970+
constexpr int64_t ELEMENT_SPLIT_LEVELS = ELEMENT_PREFIX_LEVELS + SPANNING_BATCH_VALUES;
971+
constexpr int64_t ELEMENT_SPLIT_VALUES = ELEMENT_PREFIX_VALUES + SPANNING_BATCH_VALUES;
972+
EXPECT_EQ(element_writer->WriteBatch(ELEMENT_SPLIT_LEVELS, element_definition_levels.data(),
945973
element_repetition_levels.data(), element_values.data()),
946-
static_cast<int64_t>(element_values.size()));
974+
ELEMENT_SPLIT_VALUES);
975+
EXPECT_EQ(element_writer->WriteBatch(
976+
static_cast<int64_t>(element_repetition_levels.size()) - ELEMENT_SPLIT_LEVELS,
977+
element_definition_levels.data() + ELEMENT_SPLIT_LEVELS,
978+
element_repetition_levels.data() + ELEMENT_SPLIT_LEVELS,
979+
element_values.data() + ELEMENT_SPLIT_VALUES),
980+
static_cast<int64_t>(element_values.size()) - ELEMENT_SPLIT_VALUES);
947981
element_writer->Close();
948982
auto* marker_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn());
949983
const int16_t marker_definition_levels[] = {1, 0, 1, 1, 1, 1};

‎be/test/format_v2/table/remote_doris_reader_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ TFileRangeDesc remote_doris_range(const BlockingFlightServer& server) {
213213
auto range = remote_doris_range();
214214
auto& params = range.table_format_params.remote_doris_params;
215215
params.__set_location_uri("grpc://localhost:" + std::to_string(server.port()));
216-
arrow::flight::Ticket ticket {.ticket = "ticket"};
216+
arrow::flight::Ticket ticket {"ticket"};
217217
params.__set_ticket(ticket.SerializeToString().ValueOrDie());
218218
return range;
219219
}

0 commit comments

Comments
 (0)