Skip to content

Commit b39c77f

Browse files
csun5285claude
andcommitted
[fix](zonemap) Treat reversed zone map bounds as invalid
NaN and infinity are recorded in the has_nan/has_positive_inf/ has_negative_inf flags and never move min/max, so a page holding nothing else leaves both at the values add_values() starts from: min = DBL_MAX and max = -DBL_MAX. Those are written out as if they were data, and the flags stand in for at most one side, so reading them back reports a DBL_MAX that is nowhere in the page -- SELECT MIN(v) answers with it while SELECT COUNT(*) WHERE v = that value answers 0. On data written before 4.0 there is no flag at all and both bounds are used, which prunes the page away: 25 rows of NaN return 0 for WHERE v > 5. The bounds round-trip exactly, so neither the flags nor the parse failure #67341 keys on can tell such a zone map from a sound one. Comparing against the values the writer starts from does not work either: 4.0 and 4.1 wrote them with digits10 + 1 digits, so a FLOAT page recorded 3.402823e+38, which no longer equals FLT_MAX but is still reversed. Mark the zone map pass_all when its bounds come back reversed, which only FLOAT and DOUBLE can do -- any value of any other type moves both bounds. Reversed is exactly the state of a page that held no finite value, so the test walks all 127 non-empty subsets of {NaN, +inf, -inf, max, lowest, 1.5, 20.5} through the writer and checks that each zone map either reports itself unusable or covers every value the page holds, and that the seven subsets without a finite value are the only unusable ones. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01TPTGDEAgu5sQVYYvfvyTJz
1 parent 5fbe360 commit b39c77f

2 files changed

Lines changed: 209 additions & 8 deletions

File tree

‎be/src/storage/index/zone_map/zone_map_index.cpp‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,21 @@ namespace doris {
4545
struct uint24_t;
4646

4747
namespace segment_v2 {
48+
namespace {
49+
50+
// Only FLOAT and DOUBLE can come out reversed: any value of any other type moves both bounds.
51+
bool is_reversed(const Field& min_value, const Field& max_value, FieldType field_type) {
52+
if (FieldType::OLAP_FIELD_TYPE_FLOAT == field_type) {
53+
return min_value.get<TYPE_FLOAT>() > max_value.get<TYPE_FLOAT>();
54+
}
55+
if (FieldType::OLAP_FIELD_TYPE_DOUBLE == field_type) {
56+
return min_value.get<TYPE_DOUBLE>() > max_value.get<TYPE_DOUBLE>();
57+
}
58+
return false;
59+
}
60+
61+
} // namespace
62+
4863
Status ZoneMap::from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_type,
4964
ZoneMap& zone_map_info) {
5065
zone_map_info.has_null = zone_map.has_null();
@@ -65,6 +80,19 @@ Status ZoneMap::from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_ty
6580
auto field_type = data_type->get_storage_field_type();
6681
// min value and max value are valid if has_not_null is true
6782
if (zone_map.has_not_null()) {
83+
if (!zone_map_info.pass_all) {
84+
parse_bound(zone_map.min(), zone_map_info.min_value);
85+
parse_bound(zone_map.max(), zone_map_info.max_value);
86+
}
87+
88+
// NaN and infinity only set the flags below, never min/max, so a page holding nothing
89+
// else leaves both at the values add_values() starts from: min = DBL_MAX and
90+
// max = -DBL_MAX, neither of which is a value in the page.
91+
if (!zone_map_info.pass_all &&
92+
is_reversed(zone_map_info.min_value, zone_map_info.max_value, field_type)) {
93+
zone_map_info.pass_all = true;
94+
}
95+
6896
if (zone_map.has_negative_inf()) {
6997
if (FieldType::OLAP_FIELD_TYPE_FLOAT == field_type) {
7098
static auto constexpr float_neg_inf = -std::numeric_limits<float>::infinity();
@@ -75,10 +103,6 @@ Status ZoneMap::from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_ty
75103
} else {
76104
return Status::InternalError("invalid zone map with negative Infinity");
77105
}
78-
} else {
79-
if (!zone_map_info.pass_all) {
80-
parse_bound(zone_map.min(), zone_map_info.min_value);
81-
}
82106
}
83107

84108
if (zone_map.has_nan()) {
@@ -101,10 +125,6 @@ Status ZoneMap::from_proto(const ZoneMapPB& zone_map, const DataTypePtr& data_ty
101125
} else {
102126
return Status::InternalError("invalid zone map with positive Infinity");
103127
}
104-
} else {
105-
if (!zone_map_info.pass_all) {
106-
parse_bound(zone_map.max(), zone_map_info.max_value);
107-
}
108128
}
109129
}
110130
return Status::OK();

‎be/test/storage/segment/zone_map_index_test.cpp‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,17 @@
1717

1818
#include "storage/index/zone_map/zone_map_index.h"
1919

20+
#include <fmt/format.h>
2021
#include <gtest/gtest-message.h>
2122
#include <gtest/gtest-test-part.h>
2223

2324
#include <cstdint>
2425
#include <limits>
2526
#include <memory>
2627
#include <string>
28+
#include <vector>
2729

30+
#include "common/config.h"
2831
#include "core/data_type/data_type_factory.hpp"
2932
#include "core/data_type/define_primitive_type.h"
3033
#include "core/decimal12.h"
@@ -981,6 +984,184 @@ TEST_F(ColumnZoneMapTest, LegacyUnparsableDoubleBoundDegradesToPassAll) {
981984
}
982985
}
983986

987+
// Every value written to a FLOAT or DOUBLE zone map is one of seven shapes, and only an ordinary
988+
// finite value moves the recorded bounds -- NaN and infinity go to the flags instead, and
989+
// DBL_MAX/-DBL_MAX happen to be the very values the bounds start from. Walk every non-empty subset
990+
// of the seven and check the one property a zone map has to hold: either it says it is unusable,
991+
// or its bounds cover every value the page holds, so nothing it contains is ever pruned away.
992+
template <PrimitiveType Type>
993+
void test_every_value_combination(const std::string& test_dir) {
994+
using CppType = typename PrimitiveTypeTraits<Type>::CppType;
995+
constexpr bool is_double = Type == TYPE_DOUBLE;
996+
const std::vector<std::pair<const char*, CppType>> candidates = {
997+
{"NaN", std::numeric_limits<CppType>::quiet_NaN()},
998+
{"+inf", std::numeric_limits<CppType>::infinity()},
999+
{"-inf", -std::numeric_limits<CppType>::infinity()},
1000+
{"max", std::numeric_limits<CppType>::max()},
1001+
{"lowest", std::numeric_limits<CppType>::lowest()},
1002+
{"1.5", static_cast<CppType>(1.5)},
1003+
{"20.5", static_cast<CppType>(20.5)}};
1004+
1005+
// Doris orders NaN above every number, so rank it beyond infinity to compare bounds the way
1006+
// the scan does.
1007+
auto rank = [](CppType v) {
1008+
return std::isnan(v) ? std::numeric_limits<double>::infinity() : static_cast<double>(v);
1009+
};
1010+
auto covers = [&](CppType low, CppType high, CppType v) {
1011+
if (std::isnan(v)) {
1012+
return std::isnan(high);
1013+
}
1014+
return (std::isnan(low) ? false : rank(low) <= rank(v)) &&
1015+
(std::isnan(high) ? true : rank(v) <= rank(high));
1016+
};
1017+
1018+
auto fs = io::global_local_filesystem();
1019+
auto column = create_float_column < is_double ? FieldType::OLAP_FIELD_TYPE_DOUBLE
1020+
: FieldType::OLAP_FIELD_TYPE_FLOAT > (0, true);
1021+
const TabletColumn* field = &(*column);
1022+
auto data_type_ptr = DataTypeFactory::instance().create_data_type(Type, false);
1023+
1024+
size_t pass_all_count = 0;
1025+
for (uint32_t mask = 1; mask < (1u << candidates.size()); ++mask) {
1026+
std::vector<CppType> values;
1027+
std::string label;
1028+
for (size_t i = 0; i < candidates.size(); ++i) {
1029+
if (mask & (1u << i)) {
1030+
values.push_back(candidates[i].second);
1031+
label += (label.empty() ? "" : ",");
1032+
label += candidates[i].first;
1033+
}
1034+
}
1035+
1036+
std::string filename =
1037+
fmt::format("{}/every_value_{}_{}", test_dir, is_double ? "double" : "float", mask);
1038+
std::unique_ptr<ZoneMapIndexWriter> builder(nullptr);
1039+
static_cast<void>(ZoneMapIndexWriter::create(data_type_ptr, field, builder));
1040+
// Stay above zone_map_row_num_threshold so the writer does not invalidate the page for
1041+
// being small, which would hide what is being tested here.
1042+
const size_t rows = config::zone_map_row_num_threshold + 5;
1043+
for (size_t i = 0; i < rows; ++i) {
1044+
CppType value = values[i % values.size()];
1045+
builder->add_values((const uint8_t*)&value, 1);
1046+
}
1047+
ASSERT_TRUE(builder->flush().ok()) << label;
1048+
1049+
ColumnIndexMetaPB index_meta;
1050+
{
1051+
io::FileWriterPtr file_writer;
1052+
ASSERT_TRUE(fs->create_file(filename, &file_writer).ok()) << label;
1053+
ASSERT_TRUE(builder->finish(file_writer.get(), &index_meta).ok()) << label;
1054+
ASSERT_TRUE(file_writer->close().ok()) << label;
1055+
}
1056+
1057+
ZoneMap zone_map;
1058+
ASSERT_TRUE(ZoneMap::from_proto(index_meta.zone_map_index().segment_zone_map(),
1059+
data_type_ptr, zone_map)
1060+
.ok())
1061+
<< label;
1062+
ASSERT_TRUE(zone_map.has_not_null) << label;
1063+
1064+
if (zone_map.pass_all) {
1065+
++pass_all_count;
1066+
continue;
1067+
}
1068+
auto low = zone_map.min_value.get<Type>();
1069+
auto high = zone_map.max_value.get<Type>();
1070+
for (auto value : values) {
1071+
EXPECT_TRUE(covers(low, high, value))
1072+
<< "values {" << label << "} left bounds that do not cover " << value;
1073+
}
1074+
}
1075+
1076+
// A page reports no bounds exactly when it held no finite value, which is every non-empty
1077+
// subset of NaN, +inf and -inf: seven of the 127.
1078+
EXPECT_EQ(7, pass_all_count) << (is_double ? "double" : "float");
1079+
}
1080+
1081+
TEST_F(ColumnZoneMapTest, EveryValueCombinationDouble) {
1082+
test_every_value_combination<TYPE_DOUBLE>(kTestDir);
1083+
}
1084+
1085+
TEST_F(ColumnZoneMapTest, EveryValueCombinationFloat) {
1086+
test_every_value_combination<TYPE_FLOAT>(kTestDir);
1087+
}
1088+
1089+
TEST_F(ColumnZoneMapTest, ReversedBoundsDegradeToPassAll) {
1090+
auto make_zone_map = [](const std::string& min, const std::string& max) {
1091+
ZoneMapPB pb;
1092+
pb.set_min(min);
1093+
pb.set_max(max);
1094+
pb.set_has_null(false);
1095+
pb.set_has_not_null(true);
1096+
pb.set_pass_all(false);
1097+
return pb;
1098+
};
1099+
// What a page of only NaN leaves behind before 4.0: bounds that never moved off the values
1100+
// the writer starts from, and that round-trip exactly, so only the reversal gives them away.
1101+
const std::string double_lowest = "-1.7976931348623157e+308";
1102+
const std::string double_highest = "1.7976931348623157e+308";
1103+
const std::string float_lowest = "-3.4028235e+38";
1104+
const std::string float_highest = "3.4028235e+38";
1105+
1106+
for (bool nullable : {false, true}) {
1107+
for (auto type : {TYPE_DOUBLE, TYPE_FLOAT}) {
1108+
const bool is_double = type == TYPE_DOUBLE;
1109+
auto data_type = DataTypeFactory::instance().create_data_type(type, nullable);
1110+
const auto& lowest = is_double ? double_lowest : float_lowest;
1111+
const auto& highest = is_double ? double_highest : float_highest;
1112+
1113+
ZoneMap reversed;
1114+
ASSERT_TRUE(
1115+
ZoneMap::from_proto(make_zone_map(highest, lowest), data_type, reversed).ok());
1116+
EXPECT_TRUE(reversed.pass_all) << "nullable=" << nullable << ", double=" << is_double;
1117+
1118+
// The same bounds the right way round stay usable.
1119+
ZoneMap sound;
1120+
ASSERT_TRUE(ZoneMap::from_proto(make_zone_map(lowest, highest), data_type, sound).ok());
1121+
EXPECT_FALSE(sound.pass_all) << "nullable=" << nullable << ", double=" << is_double;
1122+
}
1123+
}
1124+
1125+
// 4.0 and later add a flag for what the page held instead, but the bounds are still the
1126+
// reversed pair whatever the flags say.
1127+
auto double_type = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false);
1128+
for (bool nan : {false, true}) {
1129+
for (bool pos_inf : {false, true}) {
1130+
for (bool neg_inf : {false, true}) {
1131+
auto pb = make_zone_map(double_highest, double_lowest);
1132+
pb.set_has_nan(nan);
1133+
pb.set_has_positive_inf(pos_inf);
1134+
pb.set_has_negative_inf(neg_inf);
1135+
ZoneMap flagged;
1136+
ASSERT_TRUE(ZoneMap::from_proto(pb, double_type, flagged).ok());
1137+
EXPECT_TRUE(flagged.pass_all)
1138+
<< "nan=" << nan << ", +inf=" << pos_inf << ", -inf=" << neg_inf;
1139+
}
1140+
}
1141+
}
1142+
1143+
// 4.0 and 4.1 wrote bounds with digits10 + 1 digits, so a FLOAT page of only NaN recorded
1144+
// 3.402823e+38 rather than FLT_MAX. It parses back finite and no longer equals the value the
1145+
// writer starts from -- the reversal survives the lossy round trip where the value does not.
1146+
auto truncated = make_zone_map("3.402823e+38", "-3.402823e+38");
1147+
truncated.set_has_nan(true);
1148+
ZoneMap from_4_0;
1149+
ASSERT_TRUE(ZoneMap::from_proto(truncated,
1150+
DataTypeFactory::instance().create_data_type(TYPE_FLOAT, false),
1151+
from_4_0)
1152+
.ok());
1153+
EXPECT_TRUE(from_4_0.pass_all);
1154+
1155+
// A flag on top of bounds that do describe finite values leaves the zone map usable.
1156+
auto pb = make_zone_map("1.5", "20.5");
1157+
pb.set_has_nan(true);
1158+
ZoneMap partly_nan;
1159+
ASSERT_TRUE(ZoneMap::from_proto(pb, double_type, partly_nan).ok());
1160+
EXPECT_FALSE(partly_nan.pass_all);
1161+
EXPECT_TRUE(std::isnan(partly_nan.max_value.get<TYPE_DOUBLE>()));
1162+
EXPECT_EQ(partly_nan.min_value.get<TYPE_DOUBLE>(), 1.5);
1163+
}
1164+
9841165
TabletColumnPtr create_timestamptz_column(int32_t id, bool is_nullable) {
9851166
auto column = std::make_shared<TabletColumn>();
9861167
column->_unique_id = id;

0 commit comments

Comments
 (0)