Skip to content

Commit 29f1335

Browse files
committed
[chore](fe) Merge master and extend TSO tracker coverage
### What problem does this PR solve? Issue Number: None Related PR: apache#67181, apache#67594 Problem Summary: Recent master changes to Arrow Flight session handling and statement executor tests conflicted with committed TSO error propagation. Merge current master, preserve the new session and deferred-query behavior while continuing to expose only incremental-window errors, and extend TSOTransactionTracker tests for repeated registration, fence replacement, waits, exact-attempt abandon semantics, metrics, and preconditions. ### Release note None ### Check List (For Author) - Test: Unit Test - TSOTransactionTrackerTest: 9 passed - DorisFlightSqlProducerTest: 12 passed - StmtExecutorTest: 26 passed - Full `./build.sh -j32`: passed - Behavior changed: No. This resolves the master merge and adds coverage. - Does this need documentation: No
2 parents c491a29 + c4dee4b commit 29f1335

197 files changed

Lines changed: 21970 additions & 1009 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/checkstyle.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,31 @@ on:
2222
pull_request:
2323
workflow_dispatch:
2424

25+
permissions:
26+
contents: read
27+
pull-requests: read
28+
2529
jobs:
2630
java-checkstyle:
2731
name: "CheckStyle"
2832
runs-on: ubuntu-latest
33+
concurrency:
34+
group: fe-code-style-${{ github.event.pull_request.number || github.run_id }}-${{ github.sha }}
35+
cancel-in-progress: true
2936
if: github.event_name == 'pull_request'
3037
steps:
38+
- name: Verify current PR head
39+
env:
40+
GH_TOKEN: ${{ github.token }}
41+
PR_NUMBER: ${{ github.event.pull_request.number }}
42+
EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
43+
run: |
44+
current_head_sha=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha')
45+
if [[ "${current_head_sha}" != "${EXPECTED_HEAD_SHA}" ]]; then
46+
echo "::error::PR #${PR_NUMBER} moved from ${EXPECTED_HEAD_SHA} to ${current_head_sha}; refusing stale validation"
47+
exit 1
48+
fi
49+
3150
- name: Checkout
3251
uses: actions/checkout@v3
3352
with:
@@ -56,4 +75,3 @@ jobs:
5675
if: steps.filter.outputs.fe_changes == 'true'
5776
run:
5877
cd fe && mvn clean checkstyle:check
59-

‎.github/workflows/clang-format.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,23 @@ jobs:
3131
clang-format:
3232
name: "Clang Formatter"
3333
runs-on: ubuntu-latest
34+
concurrency:
35+
group: code-formatter-${{ github.event.pull_request.number || github.run_id }}-${{ github.sha }}
36+
cancel-in-progress: true
3437
if: github.event_name == 'pull_request'
3538
steps:
39+
- name: Verify current PR head
40+
env:
41+
GH_TOKEN: ${{ github.token }}
42+
PR_NUMBER: ${{ github.event.pull_request.number }}
43+
EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
44+
run: |
45+
current_head_sha=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha')
46+
if [[ "${current_head_sha}" != "${EXPECTED_HEAD_SHA}" ]]; then
47+
echo "::error::PR #${PR_NUMBER} moved from ${EXPECTED_HEAD_SHA} to ${current_head_sha}; refusing stale validation"
48+
exit 1
49+
fi
50+
3651
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
3752
uses: actions/checkout@v7
3853
with:

‎.github/workflows/gitleaks-pr-check.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,31 @@ on:
2323

2424
permissions:
2525
contents: read
26+
pull-requests: read
2627

2728
jobs:
2829
gitleaks:
2930
name: Check for secrets
3031
runs-on: ubuntu-latest
32+
concurrency:
33+
group: gitleaks-${{ github.event.pull_request.number || github.run_id }}-${{ github.sha }}
34+
cancel-in-progress: true
3135
env:
3236
GITLEAKS_VERSION: 8.30.0
3337
GITLEAKS_SHA256: 79a3ab579b53f71efd634f3aaf7e04a0fa0cf206b7ed434638d1547a2470a66e
3438
steps:
39+
- name: Verify current PR head
40+
env:
41+
GH_TOKEN: ${{ github.token }}
42+
PR_NUMBER: ${{ github.event.pull_request.number }}
43+
EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
44+
run: |
45+
current_head_sha=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha')
46+
if [[ "${current_head_sha}" != "${EXPECTED_HEAD_SHA}" ]]; then
47+
echo "::error::PR #${PR_NUMBER} moved from ${EXPECTED_HEAD_SHA} to ${current_head_sha}; refusing stale validation"
48+
exit 1
49+
fi
50+
3551
- name: Checkout
3652
uses: actions/checkout@v4
3753
with:

‎.github/workflows/license-eyes.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,26 @@ jobs:
3232
license-check:
3333
name: "License Check"
3434
runs-on: ubuntu-latest
35+
concurrency:
36+
group: license-check-${{ github.event.pull_request.number || github.run_id }}-${{ github.sha }}
37+
cancel-in-progress: true
3538
if: |
3639
(github.event_name == 'pull_request') ||
3740
(github.event_name == 'push' && github.ref == 'refs/heads/master')
3841
steps:
42+
- name: Verify current PR head
43+
if: github.event_name == 'pull_request'
44+
env:
45+
GH_TOKEN: ${{ github.token }}
46+
PR_NUMBER: ${{ github.event.pull_request.number }}
47+
EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
48+
run: |
49+
current_head_sha=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha')
50+
if [[ "${current_head_sha}" != "${EXPECTED_HEAD_SHA}" ]]; then
51+
echo "::error::PR #${PR_NUMBER} moved from ${EXPECTED_HEAD_SHA} to ${current_head_sha}; refusing stale validation"
52+
exit 1
53+
fi
54+
3955
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
4056
uses: actions/checkout@v7
4157
with:

‎.licenserc.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ header:
6565
# the whole file and compares it to this one line for line, so a header would read back
6666
# as extra rows; it prints the regenerated path on mismatch instead.
6767
- "fe/fe-core/src/test/resources/access-control-behavior-baseline.txt"
68+
# Hive-generated CSV oracle. The FE test compares every line with regenerated
69+
# data, and both BE readers consume every line as a record; a header would
70+
# become unexpected test data. Exclude only this fixture, not all TSV files.
71+
- "fe/fe-connector/fe-connector-hive/src/test/resources/hive-csv-oracle.tsv"
6872
# Golden protocol traffic. MysqlPacketGoldenTest and FlightResultGoldenTest regenerate
6973
# each file in full and compare it to this one byte for byte, so a header would read back
7074
# as unexpected leading content; both tests print the regeneration command on mismatch.

‎be/src/agent/be_exec_version_manager.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,10 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers
133133
// b. support Paimon default fixed-bucket routing in the external sink exchange.
134134
// 14: start from master
135135
// a. support TIMESTAMP_NS in Thrift descriptors and PBlock exchange.
136+
// 15: start from master
137+
// a. distinguish Hive OpenCSVSerde row semantics from generic CSV decoding during upgrades.
136138

137-
const int BeExecVersionManager::max_be_exec_version = SUPPORT_TIMESTAMP_NS_VERSION;
139+
const int BeExecVersionManager::max_be_exec_version = SUPPORT_HIVE_OPEN_CSV_VERSION;
138140
const int BeExecVersionManager::min_be_exec_version = 0;
139141
std::map<std::string, std::set<int>> BeExecVersionManager::_function_change_map {};
140142
std::set<std::string> BeExecVersionManager::_function_restrict_map;

‎be/src/agent/be_exec_version_manager.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11;
3434
constexpr inline int SUPPORT_ICEBERG_VARIANT_VERSION = 12;
3535
constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 13;
3636
constexpr inline int SUPPORT_TIMESTAMP_NS_VERSION = 14;
37+
constexpr inline int SUPPORT_HIVE_OPEN_CSV_VERSION = 15;
3738

3839
class BeExecVersionManager {
3940
public:

‎be/src/core/value/ip_address_cidr.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,17 @@ class IPAddressVariant {
8585
return 0;
8686
}
8787

88+
bool is_v4() const { return std::holds_alternative<IPv4AddrType>(_addr); }
89+
8890
const UInt8* as_v6() const {
8991
if (const auto* val = std::get_if<IPv6AddrType>(&_addr)) {
9092
return val->data();
9193
}
9294
return nullptr;
9395
}
9496

97+
bool is_v6() const { return std::holds_alternative<IPv6AddrType>(_addr); }
98+
9599
private:
96100
using IPv4AddrType = UInt32;
97101
using IPv6AddrType = std::array<UInt8, IPV6_BINARY_LENGTH>;
@@ -175,7 +179,7 @@ inline IPAddressCIDR parse_ip_with_cidr(std::string_view cidr_str) {
175179

176180
const auto* prefix_str_end = prefix_str.data() + prefix_str.size();
177181
auto [parse_end, parse_error] = std::from_chars(prefix_str.data(), prefix_str_end, prefix);
178-
uint8_t max_prefix = (addr.as_v6() ? IPV6_BINARY_LENGTH : IPV4_BINARY_LENGTH) * 8;
182+
uint8_t max_prefix = (addr.is_v6() ? IPV6_BINARY_LENGTH : IPV4_BINARY_LENGTH) * 8;
179183

180184
if (parse_error != std::errc() || parse_end != prefix_str_end || prefix > max_prefix) {
181185
throw Exception(ErrorCode::INVALID_ARGUMENT, "The CIDR has a malformed prefix bits: {}",

‎be/src/exec/operator/olap_scan_operator.cpp‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <fmt/format.h>
2121

22+
#include <algorithm>
2223
#include <memory>
2324
#include <numeric>
2425
#include <optional>
@@ -140,6 +141,13 @@ Status OlapScanLocalState::_init_profile() {
140141
// 2. init timer and counters
141142
_reader_init_timer = ADD_TIMER(_scanner_profile, "ReaderInitTime");
142143
_scanner_init_timer = ADD_TIMER(_scanner_profile, "ScannerInitTime");
144+
_rowset_tso_prune_timer = ADD_TIMER(custom_profile(), "RowsetTsoPruneTime");
145+
_rowsets_pruned_by_tso_counter =
146+
ADD_COUNTER(custom_profile(), "RowsetsPrunedByTso", TUnit::UNIT);
147+
_segments_pruned_by_tso_counter =
148+
ADD_COUNTER(custom_profile(), "SegmentsPrunedByTso", TUnit::UNIT);
149+
_tablets_pruned_by_tso_counter =
150+
ADD_COUNTER(custom_profile(), "TabletsPrunedByTso", TUnit::UNIT);
143151
_process_conjunct_timer = ADD_TIMER(custom_profile(), "ProcessConjunctTime");
144152
_read_compressed_counter = ADD_COUNTER(_segment_profile, "CompressedBytesRead", TUnit::BYTES);
145153
_read_uncompressed_counter =
@@ -541,7 +549,6 @@ Status OlapScanLocalState::_should_push_down_function_filter(VectorizedFnCall* f
541549
const auto& children = fn_call->children();
542550
doris::FunctionContext* func_cxt = expr_ctx->fn_context(fn_call->fn_context_index());
543551
DCHECK(func_cxt != nullptr);
544-
DCHECK(children.size() == 2);
545552
for (size_t i = 0; i < children.size(); i++) {
546553
if (VExpr::expr_without_cast(children[i])->node_type() != TExprNodeType::SLOT_REF) {
547554
// not a slot ref(column)
@@ -635,6 +642,32 @@ bool OlapScanLocalState::_is_binlog_merge_scan() const {
635642
return scan_type == TBinlogScanType::MIN_DELTA || scan_type == TBinlogScanType::DETAIL;
636643
}
637644

645+
void OlapScanLocalState::_prune_rowsets_by_tso(const TPaloScanRange& scan_range,
646+
TabletReadSource& read_source) {
647+
SCOPED_TIMER(_rowset_tso_prune_timer);
648+
int64_t pruned_segments = 0;
649+
const auto pruned_rowsets = std::erase_if(read_source.rs_splits, [&](const auto& split) {
650+
const auto& rowset = split.rs_reader->rowset();
651+
const auto tso = rowset->commit_tso();
652+
// Old rowsets can lack commit TSO metadata, including compaction inputs with an
653+
// unknown endpoint. Keep them for the existing segment/row-level predicates.
654+
if (tso.start_tso() < 0 || tso.end_tso() < 0) {
655+
return false;
656+
}
657+
DCHECK_LE(tso.start_tso(), tso.end_tso());
658+
// Rowset metadata is inclusive [min, max]; the query is half-open [start, end).
659+
const bool outside_window =
660+
(scan_range.__isset.start_tso && tso.end_tso() < scan_range.start_tso) ||
661+
(scan_range.__isset.end_tso && tso.start_tso() >= scan_range.end_tso);
662+
if (outside_window) {
663+
pruned_segments += rowset->num_segments();
664+
}
665+
return outside_window;
666+
});
667+
COUNTER_UPDATE(_rowsets_pruned_by_tso_counter, pruned_rowsets);
668+
COUNTER_UPDATE(_segments_pruned_by_tso_counter, pruned_segments);
669+
}
670+
638671
Status OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
639672
if (_scan_ranges.empty()) {
640673
_eos = true;
@@ -795,6 +828,22 @@ Status OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
795828
int scanners_per_tablet = std::max(1, 64 / (int)_scan_ranges.size());
796829
for (size_t scan_range_idx = 0; scan_range_idx < _scan_ranges.size(); scan_range_idx++) {
797830
const auto& palo_scan_range = *_scan_ranges[scan_range_idx];
831+
if (read_row_binlog &&
832+
(palo_scan_range.__isset.start_tso || palo_scan_range.__isset.end_tso)) {
833+
auto& read_source = _read_sources[scan_range_idx];
834+
// The version-consistent read source and delete predicates have already been
835+
// captured. Prune before cloning readers or opening any segment footers.
836+
_prune_rowsets_by_tso(palo_scan_range, read_source);
837+
if (std::all_of(read_source.rs_splits.begin(), read_source.rs_splits.end(),
838+
[](const auto& split) {
839+
return split.rs_reader->rowset()->num_rows() == 0;
840+
})) {
841+
// Empty bootstrap rowsets may have no TSO. Skip the tablet even if those
842+
// remain, and do not let OlapScanner recapture an empty read source.
843+
COUNTER_UPDATE(_tablets_pruned_by_tso_counter, 1);
844+
continue;
845+
}
846+
}
798847
int64_t version = 0;
799848
std::from_chars(palo_scan_range.version.data(),
800849
palo_scan_range.version.data() + palo_scan_range.version.size(), version);

‎be/src/exec/operator/olap_scan_operator.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
135135

136136
Status _init_scanners(std::list<ScannerSPtr>* scanners) override;
137137

138+
void _prune_rowsets_by_tso(const TPaloScanRange& scan_range, TabletReadSource& read_source);
139+
138140
Status _build_key_ranges_and_filters();
139141

140142
bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int32_t bucket_seq,
@@ -165,6 +167,10 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
165167
RuntimeProfile::Counter* _key_range_counter = nullptr;
166168
RuntimeProfile::Counter* _reader_init_timer = nullptr;
167169
RuntimeProfile::Counter* _scanner_init_timer = nullptr;
170+
RuntimeProfile::Counter* _rowset_tso_prune_timer = nullptr;
171+
RuntimeProfile::Counter* _rowsets_pruned_by_tso_counter = nullptr;
172+
RuntimeProfile::Counter* _segments_pruned_by_tso_counter = nullptr;
173+
RuntimeProfile::Counter* _tablets_pruned_by_tso_counter = nullptr;
168174
RuntimeProfile::Counter* _process_conjunct_timer = nullptr;
169175

170176
RuntimeProfile::Counter* _io_timer = nullptr;

0 commit comments

Comments
 (0)