Skip to content

Commit 85478ae

Browse files
committed
[fix](be) Preserve predicate order in split pruning
Issue Number: None Related PR: #65449 Problem Summary: Split pruning could skip an unsafe conjunct and still pre-execute later predicates. A later predicate could then prune a split before the earlier non-deterministic or error-preserving predicate reached its normal row-level evaluation point, changing conjunct ordering semantics. Preserve only the safe prefix for V2 partition pruning, V2 constant pruning, and the legacy FileScanner runtime-filter partition pruning path. Add unit coverage for all three paths. Split pruning now preserves row-level conjunct order when predicates are unsafe to pre-execute. - Test: Unit Test - ASAN BE unit tests: TableReaderTest.PrepareSplit*, TableReaderTest.ConstantPruningStopsAtUnsafePredicate, FileScannerTest.PartitionPruningStopsAtUnsafePredicate - C++ format check - Behavior changed: Yes. Split pruning stops at the first predicate that is unsafe to pre-execute. - Does this need documentation: No
1 parent 6e0020d commit 85478ae

6 files changed

Lines changed: 154 additions & 14 deletions

File tree

‎be/src/exec/scan/file_scanner.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,11 @@ void FileScanner::_init_runtime_filter_partition_prune_ctxs() {
352352
auto impl = conjunct->root()->get_impl();
353353
// If impl is not null, which means this a conjuncts from runtime filter.
354354
auto expr = impl ? impl : conjunct->root();
355+
// Preserve a safe prefix of the row-level conjunct order. Considering later predicates
356+
// after an unsafe one could prune the split before the unsafe predicate is evaluated.
357+
if (!expr->is_safe_to_execute_on_selected_rows()) {
358+
break;
359+
}
355360
if (_check_partition_prune_expr(expr)) {
356361
_runtime_filter_partition_prune_ctxs.emplace_back(conjunct);
357362
}

‎be/src/exec/scan/file_scanner.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,19 @@ class FileScanner : public Scanner {
6868
static const std::string FileReadBytesProfile;
6969
static const std::string FileReadTimeProfile;
7070

71+
#ifdef BE_TEST
72+
void TEST_init_runtime_filter_partition_prune_ctxs(
73+
const VExprContextSPtrs& conjuncts,
74+
const std::unordered_map<SlotId, int>& partition_slot_index_map) {
75+
_conjuncts = conjuncts;
76+
_partition_slot_index_map = partition_slot_index_map;
77+
_init_runtime_filter_partition_prune_ctxs();
78+
}
79+
const VExprContextSPtrs& TEST_runtime_filter_partition_prune_ctxs() const {
80+
return _runtime_filter_partition_prune_ctxs;
81+
}
82+
#endif
83+
7184
FileScanner(RuntimeState* state, FileScanLocalState* parent, int64_t limit,
7285
std::shared_ptr<SplitSourceConnector> split_source, RuntimeProfile* profile,
7386
ShardedKVCache* kv_cache,

‎be/src/format_v2/table_reader.cpp‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -803,14 +803,11 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs&
803803
for (const auto& conjunct : conjuncts) {
804804
DORIS_CHECK(conjunct != nullptr);
805805
DORIS_CHECK(conjunct->root() != nullptr);
806-
const auto root = conjunct->root();
807-
const auto impl = root->get_impl();
808-
const auto predicate = impl != nullptr ? impl : root;
809-
// Split pruning evaluates a predicate once before any file rows are read. Reordering
810-
// non-deterministic or error-preserving expressions can change their row-level semantics,
811-
// even when every referenced slot is a partition column.
812-
if (!predicate->is_safe_to_execute_on_selected_rows()) {
813-
continue;
806+
// Keep only the safe prefix of the original conjunct order. If an unsafe conjunct is
807+
// skipped, a later predicate could prune the split before the unsafe one reaches its
808+
// normal row-level evaluation point.
809+
if (!_is_safe_to_pre_execute(conjunct)) {
810+
break;
814811
}
815812
std::set<GlobalIndex> global_indices;
816813
collect_global_indices(conjunct->root(), &global_indices);
@@ -845,6 +842,18 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs&
845842
can_filter_all);
846843
}
847844

845+
bool TableReader::_is_safe_to_pre_execute(const VExprContextSPtr& conjunct) {
846+
DORIS_CHECK(conjunct != nullptr);
847+
DORIS_CHECK(conjunct->root() != nullptr);
848+
const auto root = conjunct->root();
849+
const auto impl = root->get_impl();
850+
const auto predicate = impl != nullptr ? impl : root;
851+
// Split pruning evaluates a predicate once before any file rows are read. Reordering
852+
// non-deterministic or error-preserving expressions can change their row-level semantics,
853+
// even when every referenced slot is a partition column or maps to a constant entry.
854+
return predicate->is_safe_to_execute_on_selected_rows();
855+
}
856+
848857
Status TableReader::_build_partition_prune_block(Block* block) const {
849858
DORIS_CHECK(block != nullptr);
850859
DORIS_CHECK(!_projected_columns.empty());

‎be/src/format_v2/table_reader.h‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,7 @@ class TableReader {
413413
Status _build_table_filters_from_conjuncts();
414414
Status _evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts,
415415
bool* can_filter_all);
416+
static bool _is_safe_to_pre_execute(const VExprContextSPtr& conjunct);
416417
Status _build_partition_prune_block(Block* block) const;
417418
Status _open_local_filter_exprs(const FileScanRequest& file_request);
418419
Status _init_reader_condition_cache(const FileScanRequest& file_request);
@@ -423,12 +424,20 @@ class TableReader {
423424
DORIS_CHECK(can_filter_all != nullptr);
424425
*can_filter_all = false;
425426
for (const auto& table_filter : _table_filters) {
426-
if (table_filter.conjunct == nullptr ||
427-
// RuntimeFilterExpr does not implement execute_column_impl(); it is evaluated by
428-
// the row-level filter path through execute_filter(). Constant split pruning uses
429-
// VExprContext::execute() on a one-row synthetic block, so runtime filters must not
430-
// be pre-executed here even when their referenced slot maps to a constant value.
431-
table_filter.conjunct->root()->is_rf_wrapper() ||
427+
if (table_filter.conjunct == nullptr) {
428+
continue;
429+
}
430+
// Constant pruning must preserve a safe prefix of the row-level conjunct order. Once
431+
// an unsafe predicate is reached, evaluating any later constant predicate could hide
432+
// its error or other row-level behavior by pruning the split first.
433+
if (!_is_safe_to_pre_execute(table_filter.conjunct)) {
434+
break;
435+
}
436+
// RuntimeFilterExpr does not implement execute_column_impl(); it is evaluated by the
437+
// row-level filter path through execute_filter(). Constant split pruning uses
438+
// VExprContext::execute() on a one-row synthetic block, so runtime filters must not be
439+
// pre-executed here even when their referenced slot maps to a constant value.
440+
if (table_filter.conjunct->root()->is_rf_wrapper() ||
432441
!_table_filter_has_only_constant_entries(table_filter)) {
433442
continue;
434443
}

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,13 @@
2929

3030
#include "common/consts.h"
3131
#include "core/assert_cast.h"
32+
#include "core/block/block.h"
33+
#include "core/column/column_vector.h"
3234
#include "core/data_type/data_type_number.h"
3335
#include "core/data_type/data_type_string.h"
3436
#include "exec/operator/file_scan_operator.h"
37+
#include "exec/scan/file_scanner.h"
38+
#include "exec/scan/split_source_connector.h"
3539
#include "exprs/runtime_filter_expr.h"
3640
#include "exprs/vdirect_in_predicate.h"
3741
#include "exprs/vslot_ref.h"
@@ -63,6 +67,33 @@ VExprSPtr slot_ref(int slot_id, int column_id, DataTypePtr type, const std::stri
6367
return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name);
6468
}
6569

70+
TExprNode bool_in_pred_node();
71+
72+
class UnsafePartitionPredicate final : public VExpr {
73+
public:
74+
UnsafePartitionPredicate() : VExpr(std::make_shared<DataTypeUInt8>(), false) {}
75+
76+
Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count,
77+
ColumnPtr& result_column) const override {
78+
auto result = ColumnUInt8::create();
79+
result->get_data().resize_fill(count, 1);
80+
result_column = std::move(result);
81+
return Status::OK();
82+
}
83+
84+
const std::string& expr_name() const override { return _expr_name; }
85+
bool is_safe_to_execute_on_selected_rows() const override { return false; }
86+
87+
private:
88+
const std::string _expr_name = "UnsafePartitionPredicate";
89+
};
90+
91+
VExprContextSPtr runtime_filter_context(VExprSPtr impl, int filter_id) {
92+
const auto node = bool_in_pred_node();
93+
return VExprContext::create_shared(
94+
RuntimeFilterExpr::create_shared(node, std::move(impl), 0.4, false, filter_id));
95+
}
96+
6697
TExprNode bool_in_pred_node() {
6798
TTypeDesc bool_type;
6899
TTypeNode bool_node;
@@ -512,4 +543,24 @@ TEST(FileScannerV2Test, RewriteSlotRefsToGlobalIndexMatrix) {
512543
}
513544
}
514545

546+
TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) {
547+
const auto bool_type = std::make_shared<DataTypeUInt8>();
548+
auto unsafe_predicate = std::make_shared<UnsafePartitionPredicate>();
549+
unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part"));
550+
VExprContextSPtrs conjuncts {
551+
runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 1),
552+
runtime_filter_context(std::move(unsafe_predicate), 2),
553+
runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 3),
554+
};
555+
556+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
557+
RuntimeProfile profile("file_scanner");
558+
FileScanner scanner(&state, &profile, nullptr, nullptr, nullptr);
559+
scanner.TEST_init_runtime_filter_partition_prune_ctxs(conjuncts, {{1, 0}});
560+
561+
const auto& partition_conjuncts = scanner.TEST_runtime_filter_partition_prune_ctxs();
562+
ASSERT_EQ(partition_conjuncts.size(), 1);
563+
EXPECT_EQ(partition_conjuncts[0], conjuncts[0]);
564+
}
565+
515566
} // namespace doris

‎be/test/format_v2/table_reader_test.cpp‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,12 @@ class NonDeterministicPartitionPredicate final : public VExpr {
283283
const std::string& expr_name() const override { return _expr_name; }
284284
bool is_deterministic() const override { return false; }
285285

286+
Status clone_node(VExprSPtr* cloned_expr) const override {
287+
DORIS_CHECK(cloned_expr != nullptr);
288+
*cloned_expr = std::make_shared<NonDeterministicPartitionPredicate>(_executed);
289+
return Status::OK();
290+
}
291+
286292
private:
287293
bool* const _executed;
288294
const std::string _expr_name = "NonDeterministicPartitionPredicate";
@@ -1267,6 +1273,8 @@ TEST(TableReaderTest, PrepareSplitDoesNotEvaluateNonDeterministicPartitionPredic
12671273
split.partition_values.emplace("part", Field::create_field<TYPE_INT>(7));
12681274
split.partition_prune_conjuncts.push_back(
12691275
VExprContext::create_shared(runtime_filter_wrapper_expr(std::move(predicate))));
1276+
split.partition_prune_conjuncts.push_back(VExprContext::create_shared(
1277+
runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 10))));
12701278

12711279
ASSERT_TRUE(reader.prepare_split(split).ok());
12721280
EXPECT_FALSE(predicate_executed);
@@ -1275,6 +1283,51 @@ TEST(TableReaderTest, PrepareSplitDoesNotEvaluateNonDeterministicPartitionPredic
12751283
EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 0);
12761284
}
12771285

1286+
TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) {
1287+
std::vector<ColumnDefinition> projected_columns;
1288+
auto partition_column = make_table_column(0, "part", std::make_shared<DataTypeInt32>());
1289+
partition_column.is_partition_key = true;
1290+
projected_columns.push_back(std::move(partition_column));
1291+
set_name_identifiers(&projected_columns);
1292+
1293+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
1294+
bool predicate_executed = false;
1295+
auto unsafe_predicate =
1296+
std::make_shared<NonDeterministicPartitionPredicate>(&predicate_executed);
1297+
unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "part"));
1298+
auto fake_state = std::make_shared<FakeFileReaderState>();
1299+
FakeTableReader reader({}, fake_state);
1300+
ASSERT_TRUE(reader.init({
1301+
.projected_columns = projected_columns,
1302+
.conjuncts =
1303+
{
1304+
prepared_conjunct(&state, unsafe_predicate),
1305+
prepared_conjunct(&state,
1306+
table_int32_greater_than_expr(
1307+
0, 0, 10)),
1308+
},
1309+
.format = FileFormat::PARQUET,
1310+
.scan_params = nullptr,
1311+
.io_ctx = nullptr,
1312+
.runtime_state = &state,
1313+
.scanner_profile = nullptr,
1314+
})
1315+
.ok());
1316+
1317+
SplitReadOptions split;
1318+
split.current_range.__set_path("fake-table-reader-input");
1319+
split.partition_values.emplace("part", Field::create_field<TYPE_INT>(7));
1320+
ASSERT_TRUE(reader.prepare_split(split).ok());
1321+
1322+
Block block = build_table_block(projected_columns);
1323+
bool eos = false;
1324+
ASSERT_TRUE(reader.get_block(&block, &eos).ok());
1325+
EXPECT_FALSE(predicate_executed);
1326+
EXPECT_FALSE(eos);
1327+
EXPECT_EQ(fake_state->open_count, 1);
1328+
ASSERT_TRUE(reader.close().ok());
1329+
}
1330+
12781331
TEST(TableReaderTest, CanUseInjectedFileReaderForStandaloneUnitTest) {
12791332
std::vector<ColumnDefinition> file_schema;
12801333
file_schema.push_back(make_file_column(0, "id", std::make_shared<DataTypeInt32>()));

0 commit comments

Comments
 (0)