Skip to content

Commit b374eb2

Browse files
committed
[improvement](be) Defer unsafe ZoneMap prefix handling
1 parent 742c07f commit b374eb2

2 files changed

Lines changed: 23 additions & 74 deletions

File tree

‎be/src/format_v2/parquet/parquet_statistics.cpp‎

Lines changed: 22 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -452,28 +452,12 @@ std::optional<format::LocalColumnId> file_column_id_by_block_position(
452452
return std::nullopt;
453453
}
454454

455-
VExprContextSPtrs metadata_pruning_safe_conjunct_prefix(const VExprContextSPtrs& conjuncts) {
456-
VExprContextSPtrs safe_prefix;
457-
safe_prefix.reserve(conjuncts.size());
458-
for (const auto& conjunct : conjuncts) {
459-
if (conjunct == nullptr || conjunct->root() == nullptr) {
460-
break;
461-
}
462-
const auto root = conjunct->root();
463-
const auto impl = root->get_impl();
464-
const auto predicate = impl != nullptr ? impl : root;
465-
// Metadata pruning must not skip an earlier error-preserving predicate and discard rows
466-
// with a later predicate before the earlier one reaches row-level evaluation.
467-
if (!predicate->is_safe_to_execute_on_selected_rows()) {
468-
break;
469-
}
470-
safe_prefix.push_back(conjunct);
471-
}
472-
return safe_prefix;
473-
}
474-
475-
bool has_expr_zonemap_filter(const VExprContextSPtrs& conjuncts) {
476-
for (const auto& conjunct : conjuncts) {
455+
bool has_expr_zonemap_filter(const format::FileScanRequest& request, const RuntimeState*) {
456+
// FileScannerV2 metadata pruning is a fixed part of its scan pipeline and must not inherit
457+
// the legacy scanner's expression ZoneMap session gate.
458+
// TODO: Fence metadata pruning at the first unsafe/error-preserving conjunct so a later
459+
// ZoneMap predicate cannot bypass its row-level evaluation.
460+
for (const auto& conjunct : request.conjuncts) {
477461
if (conjunct != nullptr && conjunct->root() != nullptr &&
478462
conjunct->root()->can_evaluate_zonemap_filter()) {
479463
return true;
@@ -549,11 +533,9 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats
549533

550534
} // namespace
551535

552-
bool can_use_parquet_page_index(const format::FileScanRequest& request, const RuntimeState*) {
553-
// FileScannerV2 metadata pruning is a fixed part of its scan pipeline and must not inherit
554-
// the legacy scanner's expression ZoneMap session gate.
555-
return config::enable_parquet_page_index &&
556-
has_expr_zonemap_filter(metadata_pruning_safe_conjunct_prefix(request.conjuncts));
536+
bool can_use_parquet_page_index(const format::FileScanRequest& request,
537+
const RuntimeState* runtime_state) {
538+
return config::enable_parquet_page_index && has_expr_zonemap_filter(request, runtime_state);
557539
}
558540

559541
std::shared_ptr<segment_v2::ZoneMap> ParquetStatisticsUtils::MakeZoneMap(
@@ -652,9 +634,8 @@ bool check_native_statistics(const tparquet::FileMetaData& metadata,
652634
const tparquet::RowGroup& row_group,
653635
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
654636
const format::FileScanRequest& request,
655-
const VExprContextSPtrs& conjuncts, ParquetPruningStats* pruning_stats,
656-
const cctz::time_zone* timezone) {
657-
const auto slot_indexes = collect_expr_zonemap_slot_indexes(conjuncts);
637+
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) {
638+
const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts);
658639
if (slot_indexes.empty()) {
659640
return false;
660641
}
@@ -689,7 +670,7 @@ bool check_native_statistics(const tparquet::FileMetaData& metadata,
689670
}
690671
add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map));
691672
}
692-
const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx);
673+
const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx);
693674
accumulate_zonemap_stats(ctx, pruning_stats);
694675
return result == ZoneMapFilterResult::kNoMatch;
695676
}
@@ -902,8 +883,9 @@ Status select_row_groups_by_metadata(
902883
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
903884
const format::FileScanRequest& request, const std::vector<int>* candidate_row_groups,
904885
std::vector<int>* selected_row_groups, bool enable_bloom_filter,
905-
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState*,
906-
ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile,
886+
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone,
887+
const RuntimeState* runtime_state, ParquetFileContext* file_context,
888+
const ParquetColumnReaderProfile& column_reader_profile,
907889
ParquetMetadataProbeMode probe_mode) {
908890
int64_t timer_sink = 0;
909891
SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink
@@ -917,7 +899,6 @@ Status select_row_groups_by_metadata(
917899
if (pruning_stats != nullptr) {
918900
pruning_stats->total_row_groups = cast_set<int64_t>(candidate_size);
919901
}
920-
const auto zonemap_conjuncts = metadata_pruning_safe_conjunct_prefix(request.conjuncts);
921902
selected_row_groups->reserve(candidate_size);
922903
for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) {
923904
const int row_group_idx = candidate_row_groups == nullptr
@@ -941,9 +922,9 @@ Status select_row_groups_by_metadata(
941922
}
942923
ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE;
943924
if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY &&
944-
has_expr_zonemap_filter(zonemap_conjuncts) &&
945-
check_native_statistics(metadata, row_group, file_schema, request, zonemap_conjuncts,
946-
pruning_stats, timezone)) {
925+
has_expr_zonemap_filter(request, runtime_state) &&
926+
check_native_statistics(metadata, row_group, file_schema, request, pruning_stats,
927+
timezone)) {
947928
prune_reason = ParquetRowGroupPruneReason::STATISTICS;
948929
}
949930
if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY &&
@@ -1262,7 +1243,8 @@ Status select_row_group_ranges_by_native_page_index(
12621243
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
12631244
const format::FileScanRequest& request, int64_t row_group_rows,
12641245
std::vector<RowRange>* selected_ranges, std::map<int, ParquetPageSkipPlan>* page_skip_plans,
1265-
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState*) {
1246+
ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone,
1247+
const RuntimeState* runtime_state) {
12661248
int64_t filter_time_sink = 0;
12671249
SCOPED_RAW_TIMER(pruning_stats == nullptr ? &filter_time_sink
12681250
: &pruning_stats->page_index_filter_time);
@@ -1272,17 +1254,16 @@ Status select_row_group_ranges_by_native_page_index(
12721254
if (page_skip_plans != nullptr) {
12731255
page_skip_plans->clear();
12741256
}
1275-
const auto zonemap_conjuncts = metadata_pruning_safe_conjunct_prefix(request.conjuncts);
12761257
if (row_group_rows <= 0 || !config::enable_parquet_page_index ||
1277-
!has_expr_zonemap_filter(zonemap_conjuncts) || page_indexes.empty()) {
1258+
!has_expr_zonemap_filter(request, runtime_state) || page_indexes.empty()) {
12781259
return Status::OK();
12791260
}
12801261
if (pruning_stats != nullptr) {
12811262
++pruning_stats->page_index_read_calls;
12821263
}
12831264

12841265
std::map<int, VExprContextSPtrs> conjuncts_by_slot;
1285-
for (const auto& conjunct : zonemap_conjuncts) {
1266+
for (const auto& conjunct : request.conjuncts) {
12861267
const auto slot_index = expr_zonemap::single_slot_zonemap_index(conjunct);
12871268
if (slot_index >= 0) {
12881269
conjuncts_by_slot[slot_index].push_back(conjunct);

‎be/test/format_v2/parquet/parquet_statistics_test.cpp‎

Lines changed: 1 addition & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -165,21 +165,6 @@ class MetadataInt32GreaterThanExpr final : public VExpr {
165165
const std::string _expr_name = "MetadataInt32GreaterThanExpr";
166166
};
167167

168-
class UnsafeMetadataBarrierExpr final : public VExpr {
169-
public:
170-
UnsafeMetadataBarrierExpr() : VExpr(std::make_shared<DataTypeUInt8>(), false) {}
171-
172-
const std::string& expr_name() const override { return _expr_name; }
173-
Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t,
174-
ColumnPtr&) const override {
175-
return Status::InternalError("unsafe metadata barrier must remain on the row path");
176-
}
177-
bool is_safe_to_execute_on_selected_rows() const override { return false; }
178-
179-
private:
180-
const std::string _expr_name = "UnsafeMetadataBarrierExpr";
181-
};
182-
183168
class MetadataBoundsProbeExpr final : public VExpr {
184169
public:
185170
explicit MetadataBoundsProbeExpr(bool require_false_boolean = false)
@@ -685,7 +670,7 @@ TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder)
685670
EXPECT_TRUE(selected_ranges.empty());
686671
}
687672

688-
TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresSwitchAndPreservesUnsafePrefix) {
673+
TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresDisabledSessionSwitch) {
689674
auto encode_int32 = [](int32_t value) {
690675
std::string bytes(sizeof(value), '\0');
691676
memcpy(bytes.data(), &value, sizeof(value));
@@ -756,23 +741,6 @@ TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresSwitchAndPreservesUnsafeP
756741
nullptr, nullptr, &state)
757742
.ok());
758743
EXPECT_TRUE(selected_ranges.empty());
759-
760-
request.conjuncts.insert(
761-
request.conjuncts.begin(),
762-
VExprContext::create_shared(std::make_shared<UnsafeMetadataBarrierExpr>()));
763-
selected_row_groups.clear();
764-
ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr,
765-
&selected_row_groups, false, nullptr,
766-
nullptr, &state)
767-
.ok());
768-
EXPECT_EQ(selected_row_groups, std::vector<int>({0}));
769-
ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
770-
metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans,
771-
nullptr, nullptr, &state)
772-
.ok());
773-
ASSERT_EQ(selected_ranges.size(), 1);
774-
EXPECT_EQ(selected_ranges[0].start, 0);
775-
EXPECT_EQ(selected_ranges[0].length, 1);
776744
}
777745

778746
TEST(NativeParquetStatisticsTest, ContradictoryAllNullPageCountsDisablePruning) {

0 commit comments

Comments
 (0)