Skip to content

Commit 742c07f

Browse files
committed
[fix](be) Preserve FileScannerV2 pruning semantics
1 parent fb3af51 commit 742c07f

6 files changed

Lines changed: 103 additions & 28 deletions

File tree

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -632,9 +632,12 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
632632
VExprContextSPtrs conjuncts;
633633
RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts));
634634
VExprContextSPtrs partition_prune_conjuncts;
635-
// FileScannerV2 owns its complete pruning pipeline, so safe partition predicates must not
636-
// inherit the legacy scanner's session gate.
637-
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
635+
if (!partition_values.empty()) {
636+
// A split without partition constants cannot be pruned here, so avoid cloning every
637+
// conjunct solely for a consumer that must return immediately. FileScannerV2 otherwise
638+
// keeps safe partition pruning enabled independently of the legacy session gate.
639+
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
640+
}
638641
RETURN_IF_ERROR(_table_reader->prepare_split({
639642
.partition_values = std::move(partition_values),
640643
.conjuncts = std::move(conjuncts),

‎be/src/format_v2/parquet/parquet_statistics.cpp‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -452,10 +452,28 @@ std::optional<format::LocalColumnId> file_column_id_by_block_position(
452452
return std::nullopt;
453453
}
454454

455-
bool has_expr_zonemap_filter(const format::FileScanRequest& request, const RuntimeState*) {
456-
// FileScannerV2 metadata pruning is a fixed part of its scan pipeline and must not inherit
457-
// the legacy scanner's expression ZoneMap session gate.
458-
for (const auto& conjunct : request.conjuncts) {
455+
VExprContextSPtrs metadata_pruning_safe_conjunct_prefix(const VExprContextSPtrs& conjuncts) {
456+
VExprContextSPtrs safe_prefix;
457+
safe_prefix.reserve(conjuncts.size());
458+
for (const auto& conjunct : conjuncts) {
459+
if (conjunct == nullptr || conjunct->root() == nullptr) {
460+
break;
461+
}
462+
const auto root = conjunct->root();
463+
const auto impl = root->get_impl();
464+
const auto predicate = impl != nullptr ? impl : root;
465+
// Metadata pruning must not skip an earlier error-preserving predicate and discard rows
466+
// with a later predicate before the earlier one reaches row-level evaluation.
467+
if (!predicate->is_safe_to_execute_on_selected_rows()) {
468+
break;
469+
}
470+
safe_prefix.push_back(conjunct);
471+
}
472+
return safe_prefix;
473+
}
474+
475+
bool has_expr_zonemap_filter(const VExprContextSPtrs& conjuncts) {
476+
for (const auto& conjunct : conjuncts) {
459477
if (conjunct != nullptr && conjunct->root() != nullptr &&
460478
conjunct->root()->can_evaluate_zonemap_filter()) {
461479
return true;
@@ -531,9 +549,11 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats
531549

532550
} // namespace
533551

534-
bool can_use_parquet_page_index(const format::FileScanRequest& request,
535-
const RuntimeState* runtime_state) {
536-
return config::enable_parquet_page_index && has_expr_zonemap_filter(request, runtime_state);
552+
bool can_use_parquet_page_index(const format::FileScanRequest& request, const RuntimeState*) {
553+
// FileScannerV2 metadata pruning is a fixed part of its scan pipeline and must not inherit
554+
// the legacy scanner's expression ZoneMap session gate.
555+
return config::enable_parquet_page_index &&
556+
has_expr_zonemap_filter(metadata_pruning_safe_conjunct_prefix(request.conjuncts));
537557
}
538558

539559
std::shared_ptr<segment_v2::ZoneMap> ParquetStatisticsUtils::MakeZoneMap(
@@ -632,8 +652,9 @@ bool check_native_statistics(const tparquet::FileMetaData& metadata,
632652
const tparquet::RowGroup& row_group,
633653
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
634654
const format::FileScanRequest& request,
635-
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) {
636-
const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts);
655+
const VExprContextSPtrs& conjuncts, ParquetPruningStats* pruning_stats,
656+
const cctz::time_zone* timezone) {
657+
const auto slot_indexes = collect_expr_zonemap_slot_indexes(conjuncts);
637658
if (slot_indexes.empty()) {
638659
return false;
639660
}
@@ -668,7 +689,7 @@ bool check_native_statistics(const tparquet::FileMetaData& metadata,
668689
}
669690
add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map));
670691
}
671-
const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx);
692+
const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx);
672693
accumulate_zonemap_stats(ctx, pruning_stats);
673694
return result == ZoneMapFilterResult::kNoMatch;
674695
}
@@ -881,9 +902,8 @@ Status select_row_groups_by_metadata(
881902
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
882903
const format::FileScanRequest& request, const std::vector<int>* candidate_row_groups,
883904
std::vector<int>* selected_row_groups, bool enable_bloom_filter,
884-
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone,
885-
const RuntimeState* runtime_state, ParquetFileContext* file_context,
886-
const ParquetColumnReaderProfile& column_reader_profile,
905+
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState*,
906+
ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile,
887907
ParquetMetadataProbeMode probe_mode) {
888908
int64_t timer_sink = 0;
889909
SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink
@@ -897,6 +917,7 @@ Status select_row_groups_by_metadata(
897917
if (pruning_stats != nullptr) {
898918
pruning_stats->total_row_groups = cast_set<int64_t>(candidate_size);
899919
}
920+
const auto zonemap_conjuncts = metadata_pruning_safe_conjunct_prefix(request.conjuncts);
900921
selected_row_groups->reserve(candidate_size);
901922
for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) {
902923
const int row_group_idx = candidate_row_groups == nullptr
@@ -920,9 +941,9 @@ Status select_row_groups_by_metadata(
920941
}
921942
ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE;
922943
if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY &&
923-
has_expr_zonemap_filter(request, runtime_state) &&
924-
check_native_statistics(metadata, row_group, file_schema, request, pruning_stats,
925-
timezone)) {
944+
has_expr_zonemap_filter(zonemap_conjuncts) &&
945+
check_native_statistics(metadata, row_group, file_schema, request, zonemap_conjuncts,
946+
pruning_stats, timezone)) {
926947
prune_reason = ParquetRowGroupPruneReason::STATISTICS;
927948
}
928949
if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY &&
@@ -1241,8 +1262,7 @@ Status select_row_group_ranges_by_native_page_index(
12411262
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
12421263
const format::FileScanRequest& request, int64_t row_group_rows,
12431264
std::vector<RowRange>* selected_ranges, std::map<int, ParquetPageSkipPlan>* page_skip_plans,
1244-
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone,
1245-
const RuntimeState* runtime_state) {
1265+
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState*) {
12461266
int64_t filter_time_sink = 0;
12471267
SCOPED_RAW_TIMER(pruning_stats == nullptr ? &filter_time_sink
12481268
: &pruning_stats->page_index_filter_time);
@@ -1252,16 +1272,17 @@ Status select_row_group_ranges_by_native_page_index(
12521272
if (page_skip_plans != nullptr) {
12531273
page_skip_plans->clear();
12541274
}
1275+
const auto zonemap_conjuncts = metadata_pruning_safe_conjunct_prefix(request.conjuncts);
12551276
if (row_group_rows <= 0 || !config::enable_parquet_page_index ||
1256-
!has_expr_zonemap_filter(request, runtime_state) || page_indexes.empty()) {
1277+
!has_expr_zonemap_filter(zonemap_conjuncts) || page_indexes.empty()) {
12571278
return Status::OK();
12581279
}
12591280
if (pruning_stats != nullptr) {
12601281
++pruning_stats->page_index_read_calls;
12611282
}
12621283

12631284
std::map<int, VExprContextSPtrs> conjuncts_by_slot;
1264-
for (const auto& conjunct : request.conjuncts) {
1285+
for (const auto& conjunct : zonemap_conjuncts) {
12651286
const auto slot_index = expr_zonemap::single_slot_zonemap_index(conjunct);
12661287
if (slot_index >= 0) {
12671288
conjuncts_by_slot[slot_index].push_back(conjunct);

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,6 +529,12 @@ TEST(FileScannerV2Test, PartitionPruningRemainsEnabledWhenSessionSwitchIsFalse)
529529
const auto range = range_with_format("hive", TFileFormatType::FORMAT_PARQUET);
530530
ASSERT_TRUE(scanner._prepare_table_reader_split(range, {}).ok());
531531
EXPECT_EQ(captured->conjunct_count, 1);
532+
EXPECT_EQ(captured->partition_prune_conjunct_count, 0);
533+
534+
ASSERT_TRUE(scanner._prepare_table_reader_split(
535+
range, {{"partition_col", Field::create_field<TYPE_INT>(1)}})
536+
.ok());
537+
EXPECT_EQ(captured->conjunct_count, 1);
532538
EXPECT_EQ(captured->partition_prune_conjunct_count, 1);
533539
}
534540

‎be/test/format_v2/parquet/parquet_statistics_test.cpp‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,21 @@ class MetadataInt32GreaterThanExpr final : public VExpr {
165165
const std::string _expr_name = "MetadataInt32GreaterThanExpr";
166166
};
167167

168+
class UnsafeMetadataBarrierExpr final : public VExpr {
169+
public:
170+
UnsafeMetadataBarrierExpr() : VExpr(std::make_shared<DataTypeUInt8>(), false) {}
171+
172+
const std::string& expr_name() const override { return _expr_name; }
173+
Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t,
174+
ColumnPtr&) const override {
175+
return Status::InternalError("unsafe metadata barrier must remain on the row path");
176+
}
177+
bool is_safe_to_execute_on_selected_rows() const override { return false; }
178+
179+
private:
180+
const std::string _expr_name = "UnsafeMetadataBarrierExpr";
181+
};
182+
168183
class MetadataBoundsProbeExpr final : public VExpr {
169184
public:
170185
explicit MetadataBoundsProbeExpr(bool require_false_boolean = false)
@@ -670,7 +685,7 @@ TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder)
670685
EXPECT_TRUE(selected_ranges.empty());
671686
}
672687

673-
TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresDisabledSessionSwitch) {
688+
TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresSwitchAndPreservesUnsafePrefix) {
674689
auto encode_int32 = [](int32_t value) {
675690
std::string bytes(sizeof(value), '\0');
676691
memcpy(bytes.data(), &value, sizeof(value));
@@ -741,6 +756,23 @@ TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresDisabledSessionSwitch) {
741756
nullptr, nullptr, &state)
742757
.ok());
743758
EXPECT_TRUE(selected_ranges.empty());
759+
760+
request.conjuncts.insert(
761+
request.conjuncts.begin(),
762+
VExprContext::create_shared(std::make_shared<UnsafeMetadataBarrierExpr>()));
763+
selected_row_groups.clear();
764+
ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr,
765+
&selected_row_groups, false, nullptr,
766+
nullptr, &state)
767+
.ok());
768+
EXPECT_EQ(selected_row_groups, std::vector<int>({0}));
769+
ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
770+
metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans,
771+
nullptr, nullptr, &state)
772+
.ok());
773+
ASSERT_EQ(selected_ranges.size(), 1);
774+
EXPECT_EQ(selected_ranges[0].start, 0);
775+
EXPECT_EQ(selected_ranges[0].length, 1);
744776
}
745777

746778
TEST(NativeParquetStatisticsTest, ContradictoryAllNullPageCountsDisablePruning) {

‎fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2198,7 +2198,15 @@ public boolean isEnableHboNonStrictMatchingMode() {
21982198
@VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_PRUNE, needForward = true, fuzzy = true)
21992199
public boolean enableRuntimeFilterPrune = true;
22002200

2201-
@VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_PARTITION_PRUNE, needForward = true, fuzzy = true)
2201+
@VarAttrDef.VarAttr(
2202+
name = ENABLE_RUNTIME_FILTER_PARTITION_PRUNE,
2203+
description = {"控制支持该变量的 scanner 是否启用运行时过滤器分区裁剪。"
2204+
+ "File Scanner V2 始终启用安全的分区裁剪。默认为 true。",
2205+
"Controls runtime-filter partition pruning in scanners that honor this variable. "
2206+
+ "File Scanner V2 always enables safe partition pruning. "
2207+
+ "The default value is true."},
2208+
needForward = true,
2209+
fuzzy = true)
22022210
public boolean enableRuntimeFilterPartitionPrune = true;
22032211

22042212
/**
@@ -2719,8 +2727,10 @@ public static boolean isEagerAggregationOnJoin() {
27192727
@VarAttrDef.VarAttr(
27202728
name = ENABLE_EXPR_ZONEMAP_FILTER,
27212729
fuzzy = true,
2722-
description = {"控制 scanner 是否启用表达式 ZoneMap 过滤。默认为 true。",
2723-
"Controls whether to enable expression ZoneMap filtering in scanners. "
2730+
description = {"控制支持该变量的 scanner 是否启用表达式 ZoneMap 过滤。"
2731+
+ "File Scanner V2 始终启用安全的表达式 ZoneMap 过滤。默认为 true。",
2732+
"Controls expression ZoneMap filtering in scanners that honor this variable. "
2733+
+ "File Scanner V2 always enables safe expression ZoneMap filtering. "
27242734
+ "The default value is true."},
27252735
needForward = true)
27262736
public boolean enableExprZonemapFilter = true;

‎gensrc/thrift/PaloInternalService.thrift‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,8 @@ struct TQueryOptions {
374374

375375
148: optional i32 min_scanners_concurrency = 1;
376376
149: optional i32 min_scan_scheduler_concurrency = 0; //deprecated
377+
// Controls runtime-filter partition pruning for readers that honor this option.
378+
// FileScannerV2 always enables safe partition pruning.
377379
150: optional bool enable_runtime_filter_partition_prune = true;
378380

379381
// The minimum memory that an operator required to run.
@@ -501,7 +503,8 @@ struct TQueryOptions {
501503
// enable plan local exchange node in fe
502504
223: optional bool enable_local_shuffle_planner;
503505

504-
// To control whether BE scan readers may apply expression-based ZoneMap pruning.
506+
// Controls expression-based ZoneMap pruning for readers that honor this option.
507+
// FileScannerV2 always enables safe expression ZoneMap pruning.
505508
224: optional bool enable_expr_zonemap_filter = true
506509

507510
225: optional i64 runtime_filter_tree_publish_max_send_bytes = 268435456

0 commit comments

Comments
 (0)