Skip to content

Commit a225ec4

Browse files
committed
[refactor](be) Check cancellation before each window peer fill
### What problem does this PR solve? Issue Number: None Related PR: apache#68500 Problem Summary: Window spill peer replay fills a constant result for each peer/output intersection, but retained row-based cancellation checks by splitting fills at 4096-row boundaries. Check cancellation before each fill instead and let the peer and output boundaries determine the fill size. Replay output is already bounded by the runtime batch size. Add unit coverage for cancellation before filling, a large peer filled without artificial row splits, and cancellation between uneven peers. SQL results are unchanged. ### Release note None ### Check List (For Author) - Test: Unit test compilation and ASAN BE build - Unit Test: Added two focused peer-fill tests. The modified source and test translation units compiled through run-be-ut.sh, but test execution was blocked by existing function_string_test.cpp compilation errors. - Manual test: ASAN ./build.sh --be -j16, clang-format v16, build hygiene, and git diff --check passed. clang-tidy was attempted but blocked by the toolchain's missing stddef.h resource header. - Regression test: Not run for this cancellation-check refactoring. - Behavior changed: Yes (check cancellation before each peer/output fill instead of every 4096 output rows; SQL results unchanged). - Does this need documentation: No.
1 parent af6a803 commit a225ec4

2 files changed

Lines changed: 59 additions & 6 deletions

File tree

‎be/src/exec/operator/analytic_source_operator.cpp‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -432,19 +432,15 @@ Status AnalyticLocalState::_append_peer_group_results(
432432
RuntimeState* state, size_t rows, const std::vector<IColumn*>& computed_result_columns) {
433433
const auto function_count = _current_batch->strategies.size();
434434
for (size_t offset = 0; offset < rows;) {
435-
if ((offset & 4095) == 0) {
436-
RETURN_IF_CANCELLED(state);
437-
}
435+
RETURN_IF_CANCELLED(state);
438436
const int64_t batch_row = _batch_output_position + offset;
439437
while (batch_row >= _peer_group_end) {
440438
RETURN_IF_ERROR(_next_peer_group_end(state));
441439
}
442440
DCHECK_GE(_peer_group_start, _partition_start);
443441
DCHECK_LE(_peer_group_end, _partition_end);
444-
// Keep the existing cancellation cadence even when one peer spans many output rows.
445442
const size_t count =
446-
std::min({rows - offset, static_cast<size_t>(_peer_group_end - batch_row),
447-
size_t {4096} - (offset & 4095)});
443+
std::min(rows - offset, static_cast<size_t>(_peer_group_end - batch_row));
448444
for (size_t i = 0; i < function_count; ++i) {
449445
if (_current_batch->strategies[i] != WindowSpillStrategy::PEER_GROUP) {
450446
continue;

‎be/test/exec/operator/analytic_sink_operator_test.cpp‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "core/column/column_const.h"
3434
#include "core/column/column_dummy.h"
3535
#include "core/column/column_nullable.h"
36+
#include "core/column/column_vector.h"
3637
#include "core/cow.h"
3738
#include "core/data_type/data_type.h"
3839
#include "core/data_type/data_type_nullable.h"
@@ -1709,6 +1710,62 @@ TEST_F(AnalyticSinkOperatorTest, SpillPathFillsPeerRangesAcrossOutputBlocksAndSi
17091710
expect_large_cume_dist_peer_output(source.get(), state.get());
17101711
}
17111712

1713+
TEST_F(AnalyticSinkOperatorTest, SpillPeerRangeChecksCancellationBeforeFillingLargePeer) {
1714+
Initialize(8192);
1715+
prepare_cume_dist_without_partition();
1716+
auto batch = std::make_shared<AnalyticSpillBatch>();
1717+
batch->rows = state->batch_size();
1718+
batch->strategies = {WindowSpillStrategy::PEER_GROUP};
1719+
batch->peer_functions = {WindowSpillPeerFunction::CUME_DIST};
1720+
source_local_state->_current_batch = batch;
1721+
source_local_state->_partition_end = state->batch_size();
1722+
source_local_state->_peer_group_end = state->batch_size();
1723+
auto result = ColumnFloat64::create();
1724+
const std::vector<IColumn*> result_columns {result.get()};
1725+
1726+
CancelAfterChecksRuntimeState cancelled_state;
1727+
cancelled_state.reset_cancel_after(0);
1728+
expect_cancelled(source_local_state->_append_peer_group_results(
1729+
&cancelled_state, state->batch_size(), result_columns));
1730+
EXPECT_TRUE(result->empty());
1731+
1732+
// One peer/output intersection is filled in one iteration, without artificial row splits.
1733+
cancelled_state.reset_cancel_after(1);
1734+
const auto status = source_local_state->_append_peer_group_results(
1735+
&cancelled_state, state->batch_size(), result_columns);
1736+
ASSERT_TRUE(status.ok()) << status.to_string();
1737+
ASSERT_EQ(result->size(), state->batch_size());
1738+
for (const auto value : result->get_data()) {
1739+
EXPECT_DOUBLE_EQ(value, 1.0);
1740+
}
1741+
}
1742+
1743+
TEST_F(AnalyticSinkOperatorTest, SpillPeerRangeChecksCancellationBetweenUnevenPeers) {
1744+
Initialize(16);
1745+
prepare_cume_dist_without_partition();
1746+
auto batch = std::make_shared<AnalyticSpillBatch>();
1747+
batch->rows = state->batch_size();
1748+
batch->strategies = {WindowSpillStrategy::PEER_GROUP};
1749+
batch->peer_functions = {WindowSpillPeerFunction::CUME_DIST};
1750+
batch->peer_group_ends = {3, 10, 16};
1751+
source_local_state->_current_batch = batch;
1752+
source_local_state->_partition_end = batch->rows;
1753+
ASSERT_TRUE(source_local_state->_next_peer_group_end(state.get()).ok());
1754+
auto result = ColumnFloat64::create();
1755+
const std::vector<IColumn*> result_columns {result.get()};
1756+
1757+
CancelAfterChecksRuntimeState cancelled_state;
1758+
cancelled_state.reset_cancel_after(1);
1759+
expect_cancelled(source_local_state->_append_peer_group_results(
1760+
&cancelled_state, state->batch_size(), result_columns));
1761+
ASSERT_EQ(result->size(), 3);
1762+
for (const auto value : result->get_data()) {
1763+
EXPECT_DOUBLE_EQ(value, 3.0 / 16);
1764+
}
1765+
EXPECT_EQ(source_local_state->_peer_group_end, 3);
1766+
EXPECT_EQ(source_local_state->_in_memory_peer_group_index, 1);
1767+
}
1768+
17121769
TEST_F(AnalyticSinkOperatorTest, SpillPathFlushesPeerGroupsAtSpillBufferSize) {
17131770
ASSERT_TRUE(spill_file_manager.init().ok());
17141771

0 commit comments

Comments
 (0)