Skip to content

Commit fb3af51

Browse files
committed
[improvement](be) Keep FileScannerV2 predicate pruning enabled
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: FileScannerV2 inherited legacy session gates that could disable partition predicate pruning and Parquet expression ZoneMap pruning. ### Release note FileScannerV2 always applies partition predicate pruning and Parquet expression ZoneMap pruning. ### Check List (For Author) - Test: Unit Test\n- Behavior changed: Yes. FileScannerV2 ignores the two legacy pruning switches.\n- Does this need documentation: No
1 parent 82646c3 commit fb3af51

4 files changed

Lines changed: 134 additions & 8 deletions

File tree

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -632,9 +632,9 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
632632
VExprContextSPtrs conjuncts;
633633
RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts));
634634
VExprContextSPtrs partition_prune_conjuncts;
635-
if (_state->query_options().enable_runtime_filter_partition_prune) {
636-
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
637-
}
635+
// FileScannerV2 owns its complete pruning pipeline, so safe partition predicates must not
636+
// inherit the legacy scanner's session gate.
637+
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
638638
RETURN_IF_ERROR(_table_reader->prepare_split({
639639
.partition_values = std::move(partition_values),
640640
.conjuncts = std::move(conjuncts),

‎be/src/format_v2/parquet/parquet_statistics.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -452,11 +452,9 @@ std::optional<format::LocalColumnId> file_column_id_by_block_position(
452452
return std::nullopt;
453453
}
454454

455-
bool has_expr_zonemap_filter(const format::FileScanRequest& request,
456-
const RuntimeState* runtime_state) {
457-
if (!expr_zonemap::is_expr_zonemap_filter_enabled(runtime_state)) {
458-
return false;
459-
}
455+
bool has_expr_zonemap_filter(const format::FileScanRequest& request, const RuntimeState*) {
456+
// FileScannerV2 metadata pruning is a fixed part of its scan pipeline and must not inherit
457+
// the legacy scanner's expression ZoneMap session gate.
460458
for (const auto& conjunct : request.conjuncts) {
461459
if (conjunct != nullptr && conjunct->root() != nullptr &&
462460
conjunct->root()->can_evaluate_zonemap_filter()) {

‎be/test/exec/scan/file_scanner_v2_test.cpp‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,18 @@ class RetryableCloseTableReader final : public format::TableReader {
144144
std::shared_ptr<RetryableCloseState> _state;
145145
};
146146

147+
class CapturingSplitTableReader final : public format::TableReader {
148+
public:
149+
Status prepare_split(const format::SplitReadOptions& options) override {
150+
conjunct_count = options.conjuncts.has_value() ? options.conjuncts->size() : 0;
151+
partition_prune_conjunct_count = options.partition_prune_conjuncts.size();
152+
return Status::OK();
153+
}
154+
155+
size_t conjunct_count = 0;
156+
size_t partition_prune_conjunct_count = 0;
157+
};
158+
147159
VExprSPtr slot_ref(int slot_id, int column_id, DataTypePtr type, const std::string& name) {
148160
return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name);
149161
}
@@ -478,6 +490,48 @@ TEST(FileScannerV2Test, FailedTableReaderCloseCanBeRetriedThroughScanner) {
478490
EXPECT_EQ(close_state->close_calls, 2);
479491
}
480492

493+
TEST(FileScannerV2Test, PartitionPruningRemainsEnabledWhenSessionSwitchIsFalse) {
494+
TQueryOptions query_options;
495+
query_options.__set_enable_runtime_filter_partition_prune(false);
496+
RuntimeState state {query_options, TQueryGlobals()};
497+
ObjectPool pool;
498+
TDescriptorTable thrift_descriptors;
499+
TTupleDescriptor tuple_descriptor;
500+
tuple_descriptor.id = 0;
501+
tuple_descriptor.byteSize = 0;
502+
tuple_descriptor.numNullBytes = 0;
503+
thrift_descriptors.tupleDescriptors.push_back(tuple_descriptor);
504+
DescriptorTbl* descriptors = nullptr;
505+
ASSERT_TRUE(DescriptorTbl::create(&pool, thrift_descriptors, &descriptors).ok());
506+
TPlanNode plan_node;
507+
plan_node.node_id = 0;
508+
plan_node.node_type = TPlanNodeType::FILE_SCAN_NODE;
509+
plan_node.num_children = 0;
510+
plan_node.limit = -1;
511+
plan_node.row_tuples.push_back(0);
512+
plan_node.file_scan_node.tuple_id = 0;
513+
plan_node.__isset.file_scan_node = true;
514+
FileScanOperatorX parent(&pool, plan_node, 0, *descriptors, 1);
515+
FileScanLocalState local_state(&state, &parent);
516+
RuntimeProfile profile("file_scanner_v2_partition_prune");
517+
auto table_reader = std::make_unique<CapturingSplitTableReader>();
518+
auto* captured = table_reader.get();
519+
FileScannerV2 scanner(&state, &profile, std::move(table_reader));
520+
scanner._local_state = &local_state;
521+
522+
TFileScanRangeParams params;
523+
params.__set_format_type(TFileFormatType::FORMAT_PARQUET);
524+
scanner._params = &params;
525+
scanner._slot_id_to_global_index.emplace(7, format::GlobalIndex(0));
526+
scanner._conjuncts = {VExprContext::create_shared(
527+
slot_ref(7, 7, std::make_shared<DataTypeInt32>(), "partition_col"))};
528+
529+
const auto range = range_with_format("hive", TFileFormatType::FORMAT_PARQUET);
530+
ASSERT_TRUE(scanner._prepare_table_reader_split(range, {}).ok());
531+
EXPECT_EQ(captured->conjunct_count, 1);
532+
EXPECT_EQ(captured->partition_prune_conjunct_count, 1);
533+
}
534+
481535
// Scenario: Once FileScannerV2 is selected, an unsupported range must fail instead of falling back
482536
// to FileScanner.
483537
TEST(FileScannerV2Test, ValidateScanRangeRejectsUnsupportedRange) {

‎be/test/format_v2/parquet/parquet_statistics_test.cpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
#include "format_v2/parquet/parquet_file_context.h"
4848
#include "format_v2/parquet/reader/native/block_split_bloom_filter.h"
4949
#include "io/fs/file_reader.h"
50+
#include "runtime/runtime_state.h"
5051
#include "util/thrift_util.h"
5152
namespace doris {
5253
namespace {
@@ -669,6 +670,79 @@ TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder)
669670
EXPECT_TRUE(selected_ranges.empty());
670671
}
671672

673+
TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresDisabledSessionSwitch) {
674+
auto encode_int32 = [](int32_t value) {
675+
std::string bytes(sizeof(value), '\0');
676+
memcpy(bytes.data(), &value, sizeof(value));
677+
return bytes;
678+
};
679+
680+
auto column_schema = std::make_unique<format::parquet::ParquetColumnSchema>();
681+
column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
682+
column_schema->local_id = 0;
683+
column_schema->leaf_column_id = 0;
684+
column_schema->type = std::make_shared<DataTypeInt32>();
685+
column_schema->type_descriptor.doris_type = column_schema->type;
686+
column_schema->type_descriptor.physical_type = tparquet::Type::INT32;
687+
std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
688+
schema.push_back(std::move(column_schema));
689+
690+
tparquet::Statistics statistics;
691+
statistics.__set_min_value(encode_int32(1));
692+
statistics.__set_max_value(encode_int32(2));
693+
statistics.__set_null_count(0);
694+
tparquet::ColumnMetaData column_metadata;
695+
column_metadata.__set_type(tparquet::Type::INT32);
696+
column_metadata.__set_num_values(1);
697+
column_metadata.__set_statistics(statistics);
698+
tparquet::ColumnChunk chunk;
699+
chunk.__set_meta_data(column_metadata);
700+
tparquet::RowGroup row_group;
701+
row_group.__set_columns({chunk});
702+
row_group.__set_num_rows(1);
703+
tparquet::ColumnOrder order;
704+
order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
705+
tparquet::FileMetaData metadata;
706+
metadata.__set_column_orders({order});
707+
metadata.__set_row_groups({row_group});
708+
709+
format::FileScanRequest request;
710+
request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0));
711+
request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
712+
request.conjuncts = {
713+
VExprContext::create_shared(std::make_shared<MetadataInt32GreaterThanExpr>(100))};
714+
715+
TQueryOptions query_options;
716+
query_options.__set_enable_expr_zonemap_filter(false);
717+
RuntimeState state {query_options, TQueryGlobals()};
718+
std::vector<int> selected_row_groups;
719+
ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr,
720+
&selected_row_groups, false, nullptr,
721+
nullptr, &state)
722+
.ok());
723+
EXPECT_TRUE(selected_row_groups.empty());
724+
725+
format::parquet::NativeParquetPageIndex page_index;
726+
page_index.column_index.__set_min_values({encode_int32(1)});
727+
page_index.column_index.__set_max_values({encode_int32(2)});
728+
page_index.column_index.__set_null_pages({false});
729+
page_index.column_index.__set_null_counts({0});
730+
tparquet::PageLocation location;
731+
location.__set_offset(0);
732+
location.__set_compressed_page_size(10);
733+
location.__set_first_row_index(0);
734+
page_index.offset_index.__set_page_locations({location});
735+
std::unordered_map<int, format::parquet::NativeParquetPageIndex> page_indexes;
736+
page_indexes.emplace(0, std::move(page_index));
737+
std::vector<format::parquet::RowRange> selected_ranges;
738+
std::map<int, format::parquet::ParquetPageSkipPlan> skip_plans;
739+
ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
740+
metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans,
741+
nullptr, nullptr, &state)
742+
.ok());
743+
EXPECT_TRUE(selected_ranges.empty());
744+
}
745+
672746
TEST(NativeParquetStatisticsTest, ContradictoryAllNullPageCountsDisablePruning) {
673747
auto column_schema = std::make_unique<format::parquet::ParquetColumnSchema>();
674748
column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;

0 commit comments

Comments
 (0)