|
17 | 17 |
|
18 | 18 | #include "storage/index/zone_map/zone_map_index.h" |
19 | 19 |
|
| 20 | +#include <fmt/format.h> |
20 | 21 | #include <gtest/gtest-message.h> |
21 | 22 | #include <gtest/gtest-test-part.h> |
22 | 23 |
|
23 | 24 | #include <cstdint> |
24 | 25 | #include <limits> |
25 | 26 | #include <memory> |
26 | 27 | #include <string> |
| 28 | +#include <vector> |
27 | 29 |
|
| 30 | +#include "common/config.h" |
28 | 31 | #include "core/data_type/data_type_factory.hpp" |
29 | 32 | #include "core/data_type/define_primitive_type.h" |
30 | 33 | #include "core/decimal12.h" |
@@ -981,6 +984,184 @@ TEST_F(ColumnZoneMapTest, LegacyUnparsableDoubleBoundDegradesToPassAll) { |
981 | 984 | } |
982 | 985 | } |
983 | 986 |
|
| 987 | +// Every value written to a FLOAT or DOUBLE zone map is one of seven shapes, and only an ordinary |
| 988 | +// finite value moves the recorded bounds -- NaN and infinity go to the flags instead, and |
| 989 | +// DBL_MAX/-DBL_MAX happen to be the very values the bounds start from. Walk every non-empty subset |
| 990 | +// of the seven and check the one property a zone map has to hold: either it says it is unusable, |
| 991 | +// or its bounds cover every value the page holds, so nothing it contains is ever pruned away. |
| 992 | +template <PrimitiveType Type> |
| 993 | +void test_every_value_combination(const std::string& test_dir) { |
| 994 | + using CppType = typename PrimitiveTypeTraits<Type>::CppType; |
| 995 | + constexpr bool is_double = Type == TYPE_DOUBLE; |
| 996 | + const std::vector<std::pair<const char*, CppType>> candidates = { |
| 997 | + {"NaN", std::numeric_limits<CppType>::quiet_NaN()}, |
| 998 | + {"+inf", std::numeric_limits<CppType>::infinity()}, |
| 999 | + {"-inf", -std::numeric_limits<CppType>::infinity()}, |
| 1000 | + {"max", std::numeric_limits<CppType>::max()}, |
| 1001 | + {"lowest", std::numeric_limits<CppType>::lowest()}, |
| 1002 | + {"1.5", static_cast<CppType>(1.5)}, |
| 1003 | + {"20.5", static_cast<CppType>(20.5)}}; |
| 1004 | + |
| 1005 | + // Doris orders NaN above every number, so rank it beyond infinity to compare bounds the way |
| 1006 | + // the scan does. |
| 1007 | + auto rank = [](CppType v) { |
| 1008 | + return std::isnan(v) ? std::numeric_limits<double>::infinity() : static_cast<double>(v); |
| 1009 | + }; |
| 1010 | + auto covers = [&](CppType low, CppType high, CppType v) { |
| 1011 | + if (std::isnan(v)) { |
| 1012 | + return std::isnan(high); |
| 1013 | + } |
| 1014 | + return (std::isnan(low) ? false : rank(low) <= rank(v)) && |
| 1015 | + (std::isnan(high) ? true : rank(v) <= rank(high)); |
| 1016 | + }; |
| 1017 | + |
| 1018 | + auto fs = io::global_local_filesystem(); |
| 1019 | + auto column = create_float_column < is_double ? FieldType::OLAP_FIELD_TYPE_DOUBLE |
| 1020 | + : FieldType::OLAP_FIELD_TYPE_FLOAT > (0, true); |
| 1021 | + const TabletColumn* field = &(*column); |
| 1022 | + auto data_type_ptr = DataTypeFactory::instance().create_data_type(Type, false); |
| 1023 | + |
| 1024 | + size_t pass_all_count = 0; |
| 1025 | + for (uint32_t mask = 1; mask < (1u << candidates.size()); ++mask) { |
| 1026 | + std::vector<CppType> values; |
| 1027 | + std::string label; |
| 1028 | + for (size_t i = 0; i < candidates.size(); ++i) { |
| 1029 | + if (mask & (1u << i)) { |
| 1030 | + values.push_back(candidates[i].second); |
| 1031 | + label += (label.empty() ? "" : ","); |
| 1032 | + label += candidates[i].first; |
| 1033 | + } |
| 1034 | + } |
| 1035 | + |
| 1036 | + std::string filename = |
| 1037 | + fmt::format("{}/every_value_{}_{}", test_dir, is_double ? "double" : "float", mask); |
| 1038 | + std::unique_ptr<ZoneMapIndexWriter> builder(nullptr); |
| 1039 | + static_cast<void>(ZoneMapIndexWriter::create(data_type_ptr, field, builder)); |
| 1040 | + // Stay above zone_map_row_num_threshold so the writer does not invalidate the page for |
| 1041 | + // being small, which would hide what is being tested here. |
| 1042 | + const size_t rows = config::zone_map_row_num_threshold + 5; |
| 1043 | + for (size_t i = 0; i < rows; ++i) { |
| 1044 | + CppType value = values[i % values.size()]; |
| 1045 | + builder->add_values((const uint8_t*)&value, 1); |
| 1046 | + } |
| 1047 | + ASSERT_TRUE(builder->flush().ok()) << label; |
| 1048 | + |
| 1049 | + ColumnIndexMetaPB index_meta; |
| 1050 | + { |
| 1051 | + io::FileWriterPtr file_writer; |
| 1052 | + ASSERT_TRUE(fs->create_file(filename, &file_writer).ok()) << label; |
| 1053 | + ASSERT_TRUE(builder->finish(file_writer.get(), &index_meta).ok()) << label; |
| 1054 | + ASSERT_TRUE(file_writer->close().ok()) << label; |
| 1055 | + } |
| 1056 | + |
| 1057 | + ZoneMap zone_map; |
| 1058 | + ASSERT_TRUE(ZoneMap::from_proto(index_meta.zone_map_index().segment_zone_map(), |
| 1059 | + data_type_ptr, zone_map) |
| 1060 | + .ok()) |
| 1061 | + << label; |
| 1062 | + ASSERT_TRUE(zone_map.has_not_null) << label; |
| 1063 | + |
| 1064 | + if (zone_map.pass_all) { |
| 1065 | + ++pass_all_count; |
| 1066 | + continue; |
| 1067 | + } |
| 1068 | + auto low = zone_map.min_value.get<Type>(); |
| 1069 | + auto high = zone_map.max_value.get<Type>(); |
| 1070 | + for (auto value : values) { |
| 1071 | + EXPECT_TRUE(covers(low, high, value)) |
| 1072 | + << "values {" << label << "} left bounds that do not cover " << value; |
| 1073 | + } |
| 1074 | + } |
| 1075 | + |
| 1076 | + // A page reports no bounds exactly when it held no finite value, which is every non-empty |
| 1077 | + // subset of NaN, +inf and -inf: seven of the 127. |
| 1078 | + EXPECT_EQ(7, pass_all_count) << (is_double ? "double" : "float"); |
| 1079 | +} |
| 1080 | + |
| 1081 | +TEST_F(ColumnZoneMapTest, EveryValueCombinationDouble) { |
| 1082 | + test_every_value_combination<TYPE_DOUBLE>(kTestDir); |
| 1083 | +} |
| 1084 | + |
| 1085 | +TEST_F(ColumnZoneMapTest, EveryValueCombinationFloat) { |
| 1086 | + test_every_value_combination<TYPE_FLOAT>(kTestDir); |
| 1087 | +} |
| 1088 | + |
| 1089 | +TEST_F(ColumnZoneMapTest, ReversedBoundsDegradeToPassAll) { |
| 1090 | + auto make_zone_map = [](const std::string& min, const std::string& max) { |
| 1091 | + ZoneMapPB pb; |
| 1092 | + pb.set_min(min); |
| 1093 | + pb.set_max(max); |
| 1094 | + pb.set_has_null(false); |
| 1095 | + pb.set_has_not_null(true); |
| 1096 | + pb.set_pass_all(false); |
| 1097 | + return pb; |
| 1098 | + }; |
| 1099 | + // What a page of only NaN leaves behind before 4.0: bounds that never moved off the values |
| 1100 | + // the writer starts from, and that round-trip exactly, so only the reversal gives them away. |
| 1101 | + const std::string double_lowest = "-1.7976931348623157e+308"; |
| 1102 | + const std::string double_highest = "1.7976931348623157e+308"; |
| 1103 | + const std::string float_lowest = "-3.4028235e+38"; |
| 1104 | + const std::string float_highest = "3.4028235e+38"; |
| 1105 | + |
| 1106 | + for (bool nullable : {false, true}) { |
| 1107 | + for (auto type : {TYPE_DOUBLE, TYPE_FLOAT}) { |
| 1108 | + const bool is_double = type == TYPE_DOUBLE; |
| 1109 | + auto data_type = DataTypeFactory::instance().create_data_type(type, nullable); |
| 1110 | + const auto& lowest = is_double ? double_lowest : float_lowest; |
| 1111 | + const auto& highest = is_double ? double_highest : float_highest; |
| 1112 | + |
| 1113 | + ZoneMap reversed; |
| 1114 | + ASSERT_TRUE( |
| 1115 | + ZoneMap::from_proto(make_zone_map(highest, lowest), data_type, reversed).ok()); |
| 1116 | + EXPECT_TRUE(reversed.pass_all) << "nullable=" << nullable << ", double=" << is_double; |
| 1117 | + |
| 1118 | + // The same bounds the right way round stay usable. |
| 1119 | + ZoneMap sound; |
| 1120 | + ASSERT_TRUE(ZoneMap::from_proto(make_zone_map(lowest, highest), data_type, sound).ok()); |
| 1121 | + EXPECT_FALSE(sound.pass_all) << "nullable=" << nullable << ", double=" << is_double; |
| 1122 | + } |
| 1123 | + } |
| 1124 | + |
| 1125 | + // 4.0 and later add a flag for what the page held instead, but the bounds are still the |
| 1126 | + // reversed pair whatever the flags say. |
| 1127 | + auto double_type = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false); |
| 1128 | + for (bool nan : {false, true}) { |
| 1129 | + for (bool pos_inf : {false, true}) { |
| 1130 | + for (bool neg_inf : {false, true}) { |
| 1131 | + auto pb = make_zone_map(double_highest, double_lowest); |
| 1132 | + pb.set_has_nan(nan); |
| 1133 | + pb.set_has_positive_inf(pos_inf); |
| 1134 | + pb.set_has_negative_inf(neg_inf); |
| 1135 | + ZoneMap flagged; |
| 1136 | + ASSERT_TRUE(ZoneMap::from_proto(pb, double_type, flagged).ok()); |
| 1137 | + EXPECT_TRUE(flagged.pass_all) |
| 1138 | + << "nan=" << nan << ", +inf=" << pos_inf << ", -inf=" << neg_inf; |
| 1139 | + } |
| 1140 | + } |
| 1141 | + } |
| 1142 | + |
| 1143 | + // 4.0 and 4.1 wrote bounds with digits10 + 1 digits, so a FLOAT page of only NaN recorded |
| 1144 | + // 3.402823e+38 rather than FLT_MAX. It parses back finite and no longer equals the value the |
| 1145 | + // writer starts from -- the reversal survives the lossy round trip where the value does not. |
| 1146 | + auto truncated = make_zone_map("3.402823e+38", "-3.402823e+38"); |
| 1147 | + truncated.set_has_nan(true); |
| 1148 | + ZoneMap from_4_0; |
| 1149 | + ASSERT_TRUE(ZoneMap::from_proto(truncated, |
| 1150 | + DataTypeFactory::instance().create_data_type(TYPE_FLOAT, false), |
| 1151 | + from_4_0) |
| 1152 | + .ok()); |
| 1153 | + EXPECT_TRUE(from_4_0.pass_all); |
| 1154 | + |
| 1155 | + // A flag on top of bounds that do describe finite values leaves the zone map usable. |
| 1156 | + auto pb = make_zone_map("1.5", "20.5"); |
| 1157 | + pb.set_has_nan(true); |
| 1158 | + ZoneMap partly_nan; |
| 1159 | + ASSERT_TRUE(ZoneMap::from_proto(pb, double_type, partly_nan).ok()); |
| 1160 | + EXPECT_FALSE(partly_nan.pass_all); |
| 1161 | + EXPECT_TRUE(std::isnan(partly_nan.max_value.get<TYPE_DOUBLE>())); |
| 1162 | + EXPECT_EQ(partly_nan.min_value.get<TYPE_DOUBLE>(), 1.5); |
| 1163 | +} |
| 1164 | + |
984 | 1165 | TabletColumnPtr create_timestamptz_column(int32_t id, bool is_nullable) { |
985 | 1166 | auto column = std::make_shared<TabletColumn>(); |
986 | 1167 | column->_unique_id = id; |
|
0 commit comments