Skip to content

Commit 5f7f4d8

Browse files
committed
[fix](be) Keep residual parquet dictionary predicates
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Row-level dictionary predicate filtering treated can_evaluate_dictionary_filter() as proof that the whole conjunct could be replaced by a dictionary bitmap. For compound AND predicates, that method may only mean one child can be evaluated by dictionary. The old behavior could skip residual predicate evaluation after dictionary prefiltering. This change builds the dictionary bitmap through evaluate_dictionary_filter() entry by entry, making it a safe prefilter, and always runs the original single-column conjuncts on surviving rows. ### Release note None ### Check List (For Author) - Test: Unit Test / Manual test - Added BE UT coverage for a dictionary prefilter with a stricter residual row predicate. - Ran build-support/clang-format.sh on changed C++ files. - Ran git diff --check on changed files. - On [email protected]:/mnt/disk3/gabriel/Workspace/dev3/doris, built parquet_scan.cpp.o and parquet_reader_test.cpp.o successfully. - Behavior changed: No - Does this need documentation: No
1 parent d7f0b99 commit 5f7f4d8

2 files changed

Lines changed: 141 additions & 82 deletions

File tree

‎be/src/format_v2/parquet/parquet_scan.cpp‎

Lines changed: 34 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@
2929
#include "common/status.h"
3030
#include "core/assert_cast.h"
3131
#include "core/block/block.h"
32-
#include "core/column/column_nullable.h"
33-
#include "core/column/column_string.h"
3432
#include "core/column/column_vector.h"
3533
#include "exprs/vexpr_context.h"
3634
#include "format_v2/parquet/parquet_column_schema.h"
@@ -789,65 +787,6 @@ bool can_evaluate_all_with_dictionary(const VExprContextSPtrs& conjuncts) {
789787
});
790788
}
791789

792-
size_t file_block_column_count(const format::FileScanRequest& request) {
793-
size_t column_count = 0;
794-
for (const auto& [_, block_position] : request.local_positions) {
795-
column_count = std::max(column_count, static_cast<size_t>(block_position.value()) + 1);
796-
}
797-
return column_count;
798-
}
799-
800-
Status build_dictionary_value_column(const ParquetColumnSchema& column_schema,
801-
const ParquetDictionaryWords& dict_words,
802-
ColumnWithTypeAndName* column) {
803-
DORIS_CHECK(column != nullptr);
804-
auto string_column = ColumnString::create();
805-
for (const auto& value : dict_words.values) {
806-
string_column->insert_data(value.data(), value.size());
807-
}
808-
809-
MutableColumnPtr result_column = std::move(string_column);
810-
if (column_schema.type->is_nullable()) {
811-
result_column = ColumnNullable::create(std::move(result_column),
812-
ColumnUInt8::create(dict_words.values.size(), 0));
813-
}
814-
*column =
815-
ColumnWithTypeAndName(std::move(result_column), column_schema.type, column_schema.name);
816-
return Status::OK();
817-
}
818-
819-
Status build_dictionary_filter_block(
820-
const format::FileScanRequest& request,
821-
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
822-
format::LocalColumnId dictionary_column_id, const ParquetDictionaryWords& dict_words,
823-
Block* dict_block) {
824-
DORIS_CHECK(dict_block != nullptr);
825-
dict_block->clear();
826-
// VExpr conjuncts address columns by their file-block position, not by Parquet local column id.
827-
// Keep the temporary dictionary block in the same layout as normal scan blocks so existing
828-
// expression evaluation can be reused without a dictionary-specific expression path.
829-
std::vector<ColumnWithTypeAndName> columns(file_block_column_count(request));
830-
for (const auto& [file_column_id, block_position] : request.local_positions) {
831-
DORIS_CHECK(file_column_id.is_valid() &&
832-
file_column_id.value() < static_cast<int>(file_schema.size()));
833-
const auto& column_schema = file_schema[file_column_id.value()];
834-
DORIS_CHECK(column_schema != nullptr);
835-
ColumnWithTypeAndName column;
836-
if (file_column_id == dictionary_column_id) {
837-
RETURN_IF_ERROR(build_dictionary_value_column(*column_schema, dict_words, &column));
838-
} else {
839-
column = ColumnWithTypeAndName(column_schema->type->create_column(),
840-
column_schema->type, column_schema->name);
841-
}
842-
columns[static_cast<size_t>(block_position.value())] = std::move(column);
843-
}
844-
for (auto& column : columns) {
845-
DORIS_CHECK(column.column.get() != nullptr);
846-
dict_block->insert(std::move(column));
847-
}
848-
return Status::OK();
849-
}
850-
851790
uint16_t count_selected_rows(const IColumn::Filter& filter) {
852791
uint16_t selected_rows = 0;
853792
for (const auto value : filter) {
@@ -865,6 +804,31 @@ Status filter_read_predicate_columns(Block* file_block, const std::vector<uint32
865804
return Status::OK();
866805
}
867806

807+
IColumn::Filter build_dictionary_entry_filter(size_t block_position,
808+
const ParquetColumnSchema& column_schema,
809+
const VExprContextSPtrs& conjuncts,
810+
const ParquetDictionaryWords& dict_words) {
811+
auto fields = dictionary_fields_from_words(dict_words);
812+
IColumn::Filter dictionary_filter(fields.size(), 1);
813+
DictionaryEvalContext ctx;
814+
auto& slot = ctx.slots
815+
.emplace(static_cast<int>(block_position),
816+
DictionaryEvalContext::SlotDictionary {
817+
.data_type = column_schema.type, .values = {}})
818+
.first->second;
819+
slot.values.reserve(1);
820+
821+
for (size_t dict_idx = 0; dict_idx < fields.size(); ++dict_idx) {
822+
slot.values.clear();
823+
slot.values.push_back(fields[dict_idx]);
824+
dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter(conjuncts, ctx) ==
825+
ZoneMapFilterResult::kNoMatch
826+
? 0
827+
: 1;
828+
}
829+
return dictionary_filter;
830+
}
831+
868832
} // namespace
869833

870834
Status ParquetScanScheduler::prepare_current_dictionary_filters(
@@ -921,20 +885,12 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters(
921885
continue;
922886
}
923887

924-
Block dict_block;
925-
RETURN_IF_ERROR(build_dictionary_filter_block(request, file_schema, col.column_id(),
926-
dict_words, &dict_block));
927-
IColumn::Filter dictionary_filter;
928-
bool can_filter_all = false;
929-
// Evaluate the original conjuncts against dictionary entries once. The resulting bitmap is
930-
// indexed by Parquet dictionary id, so the data-page reader only performs an integer lookup
931-
// for each selected row instead of materializing every predicate-column value first.
932-
RETURN_IF_ERROR(execute_compact_filter_conjuncts(conjunct_it->second,
933-
dict_words.values.size(), &dict_block,
934-
&dictionary_filter, &can_filter_all));
935-
if (can_filter_all) {
936-
dictionary_filter.resize_fill(dict_words.values.size(), 0);
937-
}
888+
// Build a safe dictionary prefilter from the dictionary-filter interface instead of
889+
// executing the row expression on a temporary dictionary block. For compound AND,
890+
// VCompoundPred intentionally evaluates only dictionary-capable children, so residual
891+
// predicates still run later on surviving rows.
892+
auto dictionary_filter = build_dictionary_entry_filter(block_position, *column_schema,
893+
conjunct_it->second, dict_words);
938894

939895
// The bitmap is keyed by Parquet dictionary id. Later data-page reads evaluate the
940896
// predicate with an integer lookup and only materialize STRING values for surviving rows.
@@ -1142,17 +1098,13 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows,
11421098
}
11431099
return Status::OK();
11441100
}
1145-
if (used_dictionary_filter) {
1146-
// The dictionary bitmap has already applied the single-column conjunct for this
1147-
// predicate column while reading dictionary ids. Running the same conjunct again
1148-
// would be redundant and would require materializing rows that were filtered out.
1149-
continue;
1150-
}
1151-
11521101
const auto conjunct_it = schedule.single_column_conjuncts.find(block_position);
11531102
if (conjunct_it == schedule.single_column_conjuncts.end()) {
11541103
continue;
11551104
}
1105+
// The dictionary bitmap is a prefilter. It may be equivalent for simple dictionary
1106+
// predicates, but compound AND can contain residual children that are not safe to
1107+
// evaluate on dictionary entries. Always run the original conjuncts on survivors.
11561108
RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second));
11571109
if (*selected_rows != 0) {
11581110
continue;

‎be/test/format_v2/parquet/parquet_reader_test.cpp‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,61 @@ class StringInExpr final : public VExpr {
232232
const std::string _expr_name = "StringInExpr";
233233
};
234234

235+
class StringDictionaryPrefilterExpr final : public VExpr {
236+
public:
237+
StringDictionaryPrefilterExpr(int column_id, std::vector<std::string> dictionary_values,
238+
std::string row_value)
239+
: VExpr(std::make_shared<DataTypeUInt8>(), false),
240+
_column_id(column_id),
241+
_dictionary_values(std::move(dictionary_values)),
242+
_row_value(std::move(row_value)) {}
243+
244+
Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector,
245+
size_t count, ColumnPtr& result_column) const override {
246+
const auto& input = nullable_nested_column<ColumnString>(*block, _column_id);
247+
auto result = ColumnUInt8::create();
248+
auto& result_data = result->get_data();
249+
result_data.resize(count);
250+
for (size_t row = 0; row < count; ++row) {
251+
const size_t input_row = selector == nullptr ? row : (*selector)[row];
252+
result_data[row] = input.get_data_at(input_row).to_string() == _row_value;
253+
}
254+
result_column = std::move(result);
255+
return Status::OK();
256+
}
257+
258+
const std::string& expr_name() const override { return _expr_name; }
259+
260+
bool can_evaluate_dictionary_filter() const override { return true; }
261+
262+
ZoneMapFilterResult evaluate_dictionary_filter(
263+
const DictionaryEvalContext& ctx) const override {
264+
const auto* dictionary = ctx.slot(_column_id);
265+
if (dictionary == nullptr) {
266+
return ZoneMapFilterResult::kUnsupported;
267+
}
268+
for (const auto& value : _dictionary_values) {
269+
const auto field = Field::create_field<TYPE_STRING>(value);
270+
for (const auto& dictionary_value : dictionary->values) {
271+
if (dictionary_value == field) {
272+
return ZoneMapFilterResult::kMayMatch;
273+
}
274+
}
275+
}
276+
return ZoneMapFilterResult::kNoMatch;
277+
}
278+
279+
void collect_slot_column_ids(std::set<int>& column_ids) const override {
280+
column_ids.insert(_column_id);
281+
}
282+
283+
private:
284+
const int _column_id;
285+
const std::vector<std::string> _dictionary_values;
286+
const std::string _row_value;
287+
const std::string _expr_name = "StringDictionaryPrefilterExpr";
288+
};
289+
235290
VExprContextSPtr create_int32_greater_than_conjunct(int column_id, int32_t value) {
236291
auto ctx =
237292
VExprContext::create_shared(std::make_shared<Int32GreaterThanExpr>(column_id, value));
@@ -257,6 +312,15 @@ VExprContextSPtr create_string_in_conjunct(int column_id, std::vector<std::strin
257312
return ctx;
258313
}
259314

315+
VExprContextSPtr create_string_dictionary_prefilter_conjunct(
316+
int column_id, std::vector<std::string> dictionary_values, std::string row_value) {
317+
auto ctx = VExprContext::create_shared(std::make_shared<StringDictionaryPrefilterExpr>(
318+
column_id, std::move(dictionary_values), std::move(row_value)));
319+
ctx->_prepared = true;
320+
ctx->_opened = true;
321+
return ctx;
322+
}
323+
260324
std::shared_ptr<arrow::Array> finish_array(arrow::ArrayBuilder* builder) {
261325
std::shared_ptr<arrow::Array> array;
262326
EXPECT_TRUE(builder->Finish(&array).ok());
@@ -1754,6 +1818,49 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateSkipsRemainingPredicateColumnsWh
17541818
EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 6);
17551819
}
17561820

1821+
TEST_F(NewParquetReaderTest, DictionaryPredicateRunsResidualConjunctOnSurvivors) {
1822+
write_single_row_group_dictionary_filter_parquet_file(_file_path);
1823+
1824+
RuntimeProfile profile("new_parquet_reader_dictionary_prefilter_residual_profile");
1825+
auto reader = create_reader(0, -1, &profile);
1826+
RuntimeState state {TQueryOptions(), TQueryGlobals()};
1827+
ASSERT_TRUE(reader->init(&state).ok());
1828+
1829+
std::vector<format::ColumnDefinition> schema;
1830+
ASSERT_TRUE(reader->get_schema(&schema).ok());
1831+
auto request = std::make_shared<format::FileScanRequest>();
1832+
request->predicate_columns = {field_projection(1)};
1833+
request->non_predicate_columns = {field_projection(0)};
1834+
request->conjuncts.push_back(
1835+
create_string_dictionary_prefilter_conjunct(1, {"az", "za"}, "za"));
1836+
use_schema_order_positions(request.get(), schema);
1837+
ASSERT_TRUE(reader->open(request).ok());
1838+
1839+
std::vector<int32_t> ids;
1840+
std::vector<std::string> values;
1841+
bool eof = false;
1842+
while (!eof) {
1843+
Block block = build_file_block(schema);
1844+
size_t rows = 0;
1845+
ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
1846+
if (rows == 0) {
1847+
continue;
1848+
}
1849+
const auto& id_column = nullable_nested_column<ColumnInt32>(block, 0);
1850+
const auto& value_column = nullable_nested_column<ColumnString>(block, 1);
1851+
for (size_t row = 0; row < rows; ++row) {
1852+
ids.push_back(id_column.get_element(row));
1853+
values.push_back(value_column.get_data_at(row).to_string());
1854+
}
1855+
}
1856+
1857+
EXPECT_EQ(ids, std::vector<int32_t>({5}));
1858+
EXPECT_EQ(values, std::vector<std::string>({"za"}));
1859+
EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4);
1860+
EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 5);
1861+
EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1);
1862+
}
1863+
17571864
TEST_F(NewParquetReaderTest, ScanRangeFiltersRowGroupsBeforeDictionaryPruning) {
17581865
write_dictionary_filter_parquet_file(_file_path);
17591866
auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false);

0 commit comments

Comments
 (0)