Skip to content

Commit 38a8f54

Browse files
BiteTheDDDDtCopilot
andcommitted
[fix](be) Fix TopN runtime filter activation
### What problem does this PR solve? Issue Number: N/A Related PR: apache#59088 Problem Summary: apache#59088 changed TopN runtime predicate target initialization to rely on a storage column id. For targets that cannot create a storage column predicate, such as non-pushdown TopN predicates or unsupported storage columns, init_target returned before marking the target as detected. That left RuntimePredicate disabled, so the scan side ignored the TopN source even though FE had sent the source id. This change keeps the target detected when no storage predicate is created, removes obsolete compatibility skips for missing runtime predicate descs, and adds FE/BE coverage for the source marking and no-column target paths. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.qe.runtime.ThriftPlansBuilderTest,org.apache.doris.qe.CoordinatorTest - ./run-be-ut.sh --run --filter=RuntimePredicateTest.* - cd fe && mvn checkstyle:check -pl fe-core -q - build-support/check-format.sh - Behavior changed: No - Does this need documentation: No Co-authored-by: Copilot <[email protected]>
1 parent d99156a commit 38a8f54

5 files changed

Lines changed: 117 additions & 10 deletions

File tree

‎be/src/exec/operator/scan_operator.cpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1250,11 +1250,6 @@ Status ScanOperatorX<LocalStateType>::prepare(RuntimeState* state) {
12501250
_slot_id_to_slot_desc[slot->id()] = slot;
12511251
}
12521252
for (auto id : _topn_filter_source_node_ids) {
1253-
if (!state->get_query_ctx()->has_runtime_predicate(id)) {
1254-
// compatible with older versions fe
1255-
continue;
1256-
}
1257-
12581253
int cid = -1;
12591254
if (state->get_query_ctx()->get_runtime_predicate(id).target_is_slot(node_id())) {
12601255
auto s = _slot_id_to_slot_desc[state->get_query_ctx()

‎be/src/exec/operator/scan_operator.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -258,11 +258,6 @@ class ScanLocalState : public ScanLocalStateBase {
258258
std::vector<int> get_topn_filter_source_node_ids(RuntimeState* state, bool push_down) {
259259
std::vector<int> result;
260260
for (int id : _parent->cast<typename Derived::Parent>()._topn_filter_source_node_ids) {
261-
if (!state->get_query_ctx()->has_runtime_predicate(id)) {
262-
// compatible with older versions fe
263-
continue;
264-
}
265-
266261
const auto& pred = state->get_query_ctx()->get_runtime_predicate(id);
267262
if (!pred.enable()) {
268263
continue;

‎be/src/runtime/runtime_predicate.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ Status RuntimePredicate::init_target(
5959
int32_t target_node_id, phmap::flat_hash_map<int, SlotDescriptor*> slot_id_to_slot_desc,
6060
const int column_id) {
6161
if (column_id < 0) {
62+
_detected_target = true;
6263
return Status::OK();
6364
}
6465
std::unique_lock<std::shared_mutex> wlock(_rwlock);
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "runtime/runtime_predicate.h"
19+
20+
#include <gtest/gtest.h>
21+
22+
#include "core/data_type/data_type_factory.hpp"
23+
#include "core/field.h"
24+
#include "exec/pipeline/thrift_builder.h"
25+
#include "runtime/descriptors.h"
26+
27+
namespace doris {
28+
namespace {
29+
30+
constexpr TPlanNodeId SOURCE_NODE_ID = 10;
31+
constexpr TPlanNodeId TARGET_NODE_ID = 20;
32+
constexpr SlotId SLOT_ID = 0;
33+
34+
TTopnFilterDesc create_topn_filter_desc() {
35+
auto target_expr = TRuntimeFilterDescBuilder::get_default_expr();
36+
37+
TTopnFilterDesc desc;
38+
desc.__set_source_node_id(SOURCE_NODE_ID);
39+
desc.__set_is_asc(true);
40+
desc.__set_null_first(false);
41+
desc.__set_target_node_id_to_target_expr({{TARGET_NODE_ID, target_expr}});
42+
return desc;
43+
}
44+
45+
SlotDescriptor create_int_slot_descriptor() {
46+
SlotDescriptor slot_desc;
47+
slot_desc._id = SLOT_ID;
48+
slot_desc._col_name = "k1";
49+
slot_desc._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, false);
50+
return slot_desc;
51+
}
52+
53+
} // namespace
54+
55+
TEST(RuntimePredicateTest, init_target_creates_column_predicate_for_valid_column_id) {
56+
RuntimePredicate predicate(create_topn_filter_desc());
57+
predicate.set_detected_source();
58+
59+
auto slot_desc = create_int_slot_descriptor();
60+
phmap::flat_hash_map<int, SlotDescriptor*> slot_id_to_slot_desc;
61+
slot_id_to_slot_desc[SLOT_ID] = &slot_desc;
62+
63+
ASSERT_TRUE(predicate.init_target(TARGET_NODE_ID, slot_id_to_slot_desc, 0).ok());
64+
65+
EXPECT_TRUE(predicate.enable());
66+
EXPECT_EQ("k1", predicate.get_col_name(TARGET_NODE_ID));
67+
EXPECT_NE(nullptr, predicate.get_predicate(TARGET_NODE_ID));
68+
}
69+
70+
TEST(RuntimePredicateTest, init_target_without_column_predicate_still_enables_runtime_filter) {
71+
RuntimePredicate predicate(create_topn_filter_desc());
72+
predicate.set_detected_source();
73+
74+
phmap::flat_hash_map<int, SlotDescriptor*> slot_id_to_slot_desc;
75+
ASSERT_TRUE(predicate.init_target(TARGET_NODE_ID, slot_id_to_slot_desc, -1).ok());
76+
77+
EXPECT_TRUE(predicate.enable());
78+
EXPECT_EQ(nullptr, predicate.get_predicate(TARGET_NODE_ID));
79+
80+
auto top_value = Field::create_field<TYPE_INT>(10);
81+
ASSERT_TRUE(predicate.update(top_value).ok());
82+
EXPECT_TRUE(predicate.has_value());
83+
EXPECT_EQ(top_value, predicate.get_value());
84+
}
85+
86+
} // namespace doris

‎fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,20 @@
1717

1818
package org.apache.doris.qe;
1919

20+
import org.apache.doris.analysis.DescriptorTable;
2021
import org.apache.doris.catalog.EnvFactory;
2122
import org.apache.doris.common.FeConstants;
2223
import org.apache.doris.nereids.NereidsPlanner;
2324
import org.apache.doris.nereids.trees.plans.physical.TopnFilter;
2425
import org.apache.doris.nereids.util.PlanChecker;
26+
import org.apache.doris.planner.PlanFragment;
2527
import org.apache.doris.planner.PlanFragmentId;
28+
import org.apache.doris.planner.ScanNode;
29+
import org.apache.doris.planner.SortNode;
2630
import org.apache.doris.thrift.TNetworkAddress;
2731
import org.apache.doris.thrift.TPipelineFragmentParams;
2832
import org.apache.doris.thrift.TPipelineInstanceParams;
33+
import org.apache.doris.thrift.TPlanFragment;
2934
import org.apache.doris.thrift.TTopnFilterDesc;
3035
import org.apache.doris.thrift.TUniqueId;
3136
import org.apache.doris.utframe.TestWithFeService;
@@ -34,9 +39,11 @@
3439
import org.junit.jupiter.api.Assumptions;
3540
import org.junit.jupiter.api.BeforeAll;
3641
import org.junit.jupiter.api.Test;
42+
import org.mockito.Mockito;
3743

3844
import java.io.IOException;
3945
import java.lang.reflect.Field;
46+
import java.util.Collections;
4047
import java.util.List;
4148
import java.util.Map;
4249
import java.util.UUID;
@@ -121,6 +128,29 @@ public void testTopnFilterDescsSharedAmongInstances() throws Exception {
121128
}
122129
}
123130

131+
@Test
132+
public void testFragmentExecParamsMarksNonOlapTopnFilterSource() {
133+
ScanNode scanNode = Mockito.mock(ScanNode.class);
134+
SortNode sortNode = Mockito.mock(SortNode.class);
135+
Mockito.when(scanNode.getTopnFilterSortNodes()).thenReturn(Collections.singletonList(sortNode));
136+
137+
PlanFragment fragment = Mockito.mock(PlanFragment.class);
138+
Mockito.when(fragment.getFragmentId()).thenReturn(new PlanFragmentId(0));
139+
Mockito.when(fragment.toThrift()).thenReturn(new TPlanFragment());
140+
Mockito.when(fragment.isTransferQueryStatisticsWithEveryBatch()).thenReturn(false);
141+
142+
Coordinator.FragmentExecParams fragParams = new Coordinator(0L, new TUniqueId(1L, 1L),
143+
new DescriptorTable(), Collections.singletonList(fragment), Collections.singletonList(scanNode),
144+
"UTC", false, false).new FragmentExecParams(fragment);
145+
TNetworkAddress host = new TNetworkAddress("127.0.0.1", 9060);
146+
fragParams.instanceExecParams.add(
147+
new Coordinator.FInstanceExecParam(new TUniqueId(2L, 2L), host, fragParams));
148+
149+
fragParams.toThrift(0);
150+
151+
Mockito.verify(sortNode).setHasRuntimePredicate();
152+
}
153+
124154
private NereidsPlanner plan(String sql) throws IOException {
125155
connectContext.getSessionVariable().setDisableNereidsRules(
126156
"PRUNE_EMPTY_PARTITION,OLAP_SCAN_TABLET_PRUNE");

0 commit comments

Comments
 (0)