Skip to content

Commit ece564a

Browse files
committed
[fix](zonemap) Require one type domain for v1 Parquet zone map bounds
parse_min_max_value decodes row-group and page bounds in the file's own logical type, while _type_matches compares only primitive types. A DECIMAL bound decoded at the file's scale was then compared against a table slot at a different scale, so the interval rules could prune a scan unit holding matching rows. Leave the zone map out unless the file and table types agree, which is the check the format_v2 reader already applies.
1 parent 69e63d6 commit ece564a

2 files changed

Lines changed: 56 additions & 0 deletions

File tree

‎be/src/format/parquet/vparquet_reader.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1392,6 +1392,11 @@ Status ParquetReader::_process_expr_zonemap_page_filter(
13921392
if (!cached_page_index->get_stat_func(&stat, cid) || stat == nullptr || !stat->available) {
13931393
continue;
13941394
}
1395+
// Same domain requirement as the row-group path above.
1396+
if (stat->col_schema == nullptr ||
1397+
!expr_zonemap::data_types_compatible(stat->col_schema->data_type, slot->type())) {
1398+
continue;
1399+
}
13951400
RowRanges expr_ranges;
13961401
ZoneMapEvalStats page_stats;
13971402
for (int64_t page_id = 0; page_id < stat->num_of_pages; ++page_id) {
@@ -1673,6 +1678,14 @@ Status ParquetReader::_process_expr_zonemap_filter(const tparquet::RowGroup& row
16731678
const auto& file_col_name =
16741679
_table_info_node_ptr->children_file_column_name(slot->col_name());
16751680
const FieldSchema* col_schema = _file_metadata->schema().get_column(file_col_name);
1681+
// parse_min_max_value decodes the bounds in the file's own logical type, while _type_matches
1682+
// only compares primitive types. A DECIMAL bound decoded at the file's scale would then be
1683+
// compared against the table's scale as if the payloads shared a domain, so leave the zone
1684+
// map out unless the two types agree exactly.
1685+
if (!expr_zonemap::data_types_compatible(col_schema->data_type, slot->type())) {
1686+
ctx.slots.emplace(cid, std::move(slot_zone_map));
1687+
continue;
1688+
}
16761689
int parquet_col_id = col_schema->physical_column_index;
16771690
if (parquet_col_id < 0) {
16781691
// Complex parent fields do not map to a physical Parquet column.

‎be/test/exprs/expr_zonemap_filter_test.cpp‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2202,6 +2202,49 @@ TEST(ExprZonemapFilterTest, SlotSlotBailsOutWhenAFloatingNanCountIsUnknown) {
22022202
expect_bails_out(FunctionComparison<GreaterOrEqualsOp, NameGreaterOrEquals> {}, "a >= b");
22032203
}
22042204

2205+
TEST(ExprZonemapFilterTest, SlotSlotComparesDecimalBoundsInOneDomain) {
2206+
// Decimal bounds are unscaled payloads, so the rule is only meaningful when both sides share a
2207+
// scale. The v1 Parquet reader decodes row-group bounds in the file's own logical type while
2208+
// _type_matches compares only primitive types, so it now leaves the zone map out when the file
2209+
// and table types differ; this pins the behaviour the rule needs when they agree.
2210+
auto decimal_type = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL64, false, 18, 2);
2211+
auto left = make_slot(0, decimal_type);
2212+
auto right = make_slot(1, decimal_type);
2213+
FunctionComparison<LessOp, NameLess> less;
2214+
2215+
auto make_decimal_zonemap = [](int64_t unscaled_min, int64_t unscaled_max) {
2216+
segment_v2::ZoneMap zone_map;
2217+
zone_map.min_value = Field::create_field<TYPE_DECIMAL64>(Decimal64(unscaled_min));
2218+
zone_map.max_value = Field::create_field<TYPE_DECIMAL64>(Decimal64(unscaled_max));
2219+
zone_map.has_not_null = true;
2220+
return zone_map;
2221+
};
2222+
2223+
// 10.00 .. 20.00 against 0.50 .. 1.00: a < b cannot hold.
2224+
auto separated =
2225+
make_two_slot_context(make_decimal_zonemap(1000, 2000), make_decimal_zonemap(50, 100),
2226+
decimal_type, decimal_type);
2227+
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
2228+
less.evaluate_zonemap_filter(separated, {left, right}));
2229+
2230+
// Touching at 10.00, which is what separates `lmin >= rmax` from `lmin > rmax`: a < b still
2231+
// cannot hold, so LT must prune while LE must not.
2232+
auto touching =
2233+
make_two_slot_context(make_decimal_zonemap(1000, 2000), make_decimal_zonemap(500, 1000),
2234+
decimal_type, decimal_type);
2235+
EXPECT_EQ(ZoneMapFilterResult::kNoMatch, less.evaluate_zonemap_filter(touching, {left, right}));
2236+
FunctionComparison<LessOrEqualsOp, NameLessOrEquals> less_equals;
2237+
EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
2238+
less_equals.evaluate_zonemap_filter(touching, {left, right}));
2239+
2240+
// Overlapping in the same scale: nothing can be concluded.
2241+
auto overlapping =
2242+
make_two_slot_context(make_decimal_zonemap(1000, 2000),
2243+
make_decimal_zonemap(1500, 2500), decimal_type, decimal_type);
2244+
EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
2245+
less.evaluate_zonemap_filter(overlapping, {left, right}));
2246+
}
2247+
22052248
TEST(ExprZonemapFilterTest, ParquetSlotZoneMapMarksFloatingNanCountUnknown) {
22062249
// The rule used to be spelled out in format_v2's add_slot_zonemap only, and the v1 Parquet
22072250
// reader assigned data_type directly and so left the flag false. Two double columns whose

0 commit comments

Comments
 (0)