2929#include " common/status.h"
3030#include " core/assert_cast.h"
3131#include " core/block/block.h"
32- #include " core/column/column_nullable.h"
33- #include " core/column/column_string.h"
3432#include " core/column/column_vector.h"
3533#include " exprs/vexpr_context.h"
3634#include " format_v2/parquet/parquet_column_schema.h"
@@ -789,65 +787,6 @@ bool can_evaluate_all_with_dictionary(const VExprContextSPtrs& conjuncts) {
789787 });
790788}
791789
792- size_t file_block_column_count (const format::FileScanRequest& request) {
793- size_t column_count = 0 ;
794- for (const auto & [_, block_position] : request.local_positions ) {
795- column_count = std::max (column_count, static_cast <size_t >(block_position.value ()) + 1 );
796- }
797- return column_count;
798- }
799-
800- Status build_dictionary_value_column (const ParquetColumnSchema& column_schema,
801- const ParquetDictionaryWords& dict_words,
802- ColumnWithTypeAndName* column) {
803- DORIS_CHECK (column != nullptr );
804- auto string_column = ColumnString::create ();
805- for (const auto & value : dict_words.values ) {
806- string_column->insert_data (value.data (), value.size ());
807- }
808-
809- MutableColumnPtr result_column = std::move (string_column);
810- if (column_schema.type ->is_nullable ()) {
811- result_column = ColumnNullable::create (std::move (result_column),
812- ColumnUInt8::create (dict_words.values .size (), 0 ));
813- }
814- *column =
815- ColumnWithTypeAndName (std::move (result_column), column_schema.type , column_schema.name );
816- return Status::OK ();
817- }
818-
819- Status build_dictionary_filter_block (
820- const format::FileScanRequest& request,
821- const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
822- format::LocalColumnId dictionary_column_id, const ParquetDictionaryWords& dict_words,
823- Block* dict_block) {
824- DORIS_CHECK (dict_block != nullptr );
825- dict_block->clear ();
826- // VExpr conjuncts address columns by their file-block position, not by Parquet local column id.
827- // Keep the temporary dictionary block in the same layout as normal scan blocks so existing
828- // expression evaluation can be reused without a dictionary-specific expression path.
829- std::vector<ColumnWithTypeAndName> columns (file_block_column_count (request));
830- for (const auto & [file_column_id, block_position] : request.local_positions ) {
831- DORIS_CHECK (file_column_id.is_valid () &&
832- file_column_id.value () < static_cast <int >(file_schema.size ()));
833- const auto & column_schema = file_schema[file_column_id.value ()];
834- DORIS_CHECK (column_schema != nullptr );
835- ColumnWithTypeAndName column;
836- if (file_column_id == dictionary_column_id) {
837- RETURN_IF_ERROR (build_dictionary_value_column (*column_schema, dict_words, &column));
838- } else {
839- column = ColumnWithTypeAndName (column_schema->type ->create_column (),
840- column_schema->type , column_schema->name );
841- }
842- columns[static_cast <size_t >(block_position.value ())] = std::move (column);
843- }
844- for (auto & column : columns) {
845- DORIS_CHECK (column.column .get () != nullptr );
846- dict_block->insert (std::move (column));
847- }
848- return Status::OK ();
849- }
850-
851790uint16_t count_selected_rows (const IColumn::Filter& filter) {
852791 uint16_t selected_rows = 0 ;
853792 for (const auto value : filter) {
@@ -865,6 +804,31 @@ Status filter_read_predicate_columns(Block* file_block, const std::vector<uint32
865804 return Status::OK ();
866805}
867806
807+ IColumn::Filter build_dictionary_entry_filter (size_t block_position,
808+ const ParquetColumnSchema& column_schema,
809+ const VExprContextSPtrs& conjuncts,
810+ const ParquetDictionaryWords& dict_words) {
811+ auto fields = dictionary_fields_from_words (dict_words);
812+ IColumn::Filter dictionary_filter (fields.size (), 1 );
813+ DictionaryEvalContext ctx;
814+ auto & slot = ctx.slots
815+ .emplace (static_cast <int >(block_position),
816+ DictionaryEvalContext::SlotDictionary {
817+ .data_type = column_schema.type , .values = {}})
818+ .first ->second ;
819+ slot.values .reserve (1 );
820+
821+ for (size_t dict_idx = 0 ; dict_idx < fields.size (); ++dict_idx) {
822+ slot.values .clear ();
823+ slot.values .push_back (fields[dict_idx]);
824+ dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter (conjuncts, ctx) ==
825+ ZoneMapFilterResult::kNoMatch
826+ ? 0
827+ : 1 ;
828+ }
829+ return dictionary_filter;
830+ }
831+
868832} // namespace
869833
870834Status ParquetScanScheduler::prepare_current_dictionary_filters (
@@ -921,20 +885,12 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters(
921885 continue ;
922886 }
923887
924- Block dict_block;
925- RETURN_IF_ERROR (build_dictionary_filter_block (request, file_schema, col.column_id (),
926- dict_words, &dict_block));
927- IColumn::Filter dictionary_filter;
928- bool can_filter_all = false ;
929- // Evaluate the original conjuncts against dictionary entries once. The resulting bitmap is
930- // indexed by Parquet dictionary id, so the data-page reader only performs an integer lookup
931- // for each selected row instead of materializing every predicate-column value first.
932- RETURN_IF_ERROR (execute_compact_filter_conjuncts (conjunct_it->second ,
933- dict_words.values .size (), &dict_block,
934- &dictionary_filter, &can_filter_all));
935- if (can_filter_all) {
936- dictionary_filter.resize_fill (dict_words.values .size (), 0 );
937- }
888+ // Build a safe dictionary prefilter from the dictionary-filter interface instead of
889+ // executing the row expression on a temporary dictionary block. For compound AND,
890+ // VCompoundPred intentionally evaluates only dictionary-capable children, so residual
891+ // predicates still run later on surviving rows.
892+ auto dictionary_filter = build_dictionary_entry_filter (block_position, *column_schema,
893+ conjunct_it->second , dict_words);
938894
939895 // The bitmap is keyed by Parquet dictionary id. Later data-page reads evaluate the
940896 // predicate with an integer lookup and only materialize STRING values for surviving rows.
@@ -1142,17 +1098,13 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows,
11421098 }
11431099 return Status::OK ();
11441100 }
1145- if (used_dictionary_filter) {
1146- // The dictionary bitmap has already applied the single-column conjunct for this
1147- // predicate column while reading dictionary ids. Running the same conjunct again
1148- // would be redundant and would require materializing rows that were filtered out.
1149- continue ;
1150- }
1151-
11521101 const auto conjunct_it = schedule.single_column_conjuncts .find (block_position);
11531102 if (conjunct_it == schedule.single_column_conjuncts .end ()) {
11541103 continue ;
11551104 }
1105+ // The dictionary bitmap is a prefilter. It may be equivalent for simple dictionary
1106+ // predicates, but compound AND can contain residual children that are not safe to
1107+ // evaluate on dictionary entries. Always run the original conjuncts on survivors.
11561108 RETURN_IF_ERROR (execute_scheduled_conjuncts_with_profile (conjunct_it->second ));
11571109 if (*selected_rows != 0 ) {
11581110 continue ;
0 commit comments