Skip to content

Commit f9554cb

Browse files
committed
[fix](binlog) Support nullable HLL conversion for row binlog
### What problem does this PR solve? Issue Number: close apache#65865 Related PR: apache#66553 Problem Summary: ROW Binlog wraps BEFORE/AFTER value columns in Nullable. PR apache#53124 removed the previous nullable HLL conversion and made the HLL OLAP converter reject any column carrying a null map. Consequently, a user-declared HLL NOT NULL column still fails when ROW Binlog converts its internal Nullable(HLL) BEFORE/AFTER representation. Restore nullable conversion only for this internal representation: serialize non-null nested HLL values, emit empty slices for null rows, retain the null map, and preserve the no-nullmap fast path. This does not allow users to define HLL NULL columns. ### Release note Fix ROW Binlog writes for NOT NULL HLL columns. ### Check List (For Author) - Test - [x] Regression test - [x] Unit Test - [ ] Manual test - [ ] No need to test or manual test. - Behavior changed: - [ ] No. - [x] Yes. ROW Binlog can persist NOT NULL HLL values through its internal nullable columns. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label
1 parent 62707fd commit f9554cb

5 files changed

Lines changed: 399 additions & 44 deletions

File tree

‎be/src/storage/iterator/olap_data_convertor.cpp‎

Lines changed: 50 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -474,20 +474,36 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
474474
assert(_typed_column.column);
475475
const ColumnHLL* column_hll = nullptr;
476476
if (_nullmap) {
477-
return Status::NotSupported("QuantileState column does not support nullable");
477+
// PR #53124 removed the previous nullable HLL conversion. ROW Binlog still wraps
478+
// BEFORE/AFTER value columns in Nullable internally, so support that representation here;
479+
// this does not change the user-visible restriction that HLL columns cannot be nullable.
480+
const auto* nullable_column =
481+
assert_cast<const ColumnNullable*>(_typed_column.column.get());
482+
column_hll = assert_cast<const ColumnHLL*>(nullable_column->get_nested_column_ptr().get());
483+
} else {
484+
column_hll = assert_cast<const ColumnHLL*>(_typed_column.column.get());
478485
}
479486

480-
column_hll = assert_cast<const ColumnHLL*>(_typed_column.column.get());
481-
482487
assert(column_hll);
483488
const HyperLogLog* hll_value = column_hll->get_data().data() + _row_pos;
484489
const HyperLogLog* hll_value_cur = hll_value;
485490
const HyperLogLog* hll_value_end = hll_value_cur + _num_rows;
486491

487492
size_t total_size = 0;
488-
while (hll_value_cur != hll_value_end) {
489-
total_size += hll_value_cur->max_serialized_size();
490-
++hll_value_cur;
493+
if (_nullmap) {
494+
const UInt8* nullmap_cur = _nullmap + _row_pos;
495+
while (hll_value_cur != hll_value_end) {
496+
if (!*nullmap_cur) {
497+
total_size += hll_value_cur->max_serialized_size();
498+
}
499+
++nullmap_cur;
500+
++hll_value_cur;
501+
}
502+
} else {
503+
while (hll_value_cur != hll_value_end) {
504+
total_size += hll_value_cur->max_serialized_size();
505+
++hll_value_cur;
506+
}
491507
}
492508
_raw_data.resize(total_size);
493509

@@ -496,17 +512,37 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
496512
Slice* slice = _slice.data();
497513

498514
hll_value_cur = hll_value;
499-
while (hll_value_cur != hll_value_end) {
500-
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
515+
if (_nullmap) {
516+
const UInt8* nullmap_cur = _nullmap + _row_pos;
517+
while (hll_value_cur != hll_value_end) {
518+
if (!*nullmap_cur) {
519+
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
501520

502-
slice->data = raw_data;
503-
slice->size = slice_size;
504-
raw_data += slice_size;
521+
slice->data = raw_data;
522+
slice->size = slice_size;
523+
raw_data += slice_size;
524+
} else {
525+
slice->data = nullptr;
526+
slice->size = 0;
527+
}
528+
++slice;
529+
++nullmap_cur;
530+
++hll_value_cur;
531+
}
532+
assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice == _slice.get_end_ptr());
533+
} else {
534+
while (hll_value_cur != hll_value_end) {
535+
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
505536

506-
++slice;
507-
++hll_value_cur;
537+
slice->data = raw_data;
538+
slice->size = slice_size;
539+
raw_data += slice_size;
540+
541+
++slice;
542+
++hll_value_cur;
543+
}
544+
assert(slice == _slice.get_end_ptr());
508545
}
509-
assert(slice == _slice.get_end_ptr());
510546
return Status::OK();
511547
}
512548

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "storage/iterator/olap_data_convertor.h"
19+
20+
#include <gtest/gtest.h>
21+
22+
#include <cstddef>
23+
#include <cstdint>
24+
#include <memory>
25+
#include <utility>
26+
#include <vector>
27+
28+
#include "core/block/block.h"
29+
#include "core/column/column_complex.h"
30+
#include "core/column/column_nullable.h"
31+
#include "core/column/column_vector.h"
32+
#include "core/data_type/data_type_hll.h"
33+
#include "core/data_type/data_type_nullable.h"
34+
#include "core/value/hll.h"
35+
#include "storage/tablet/tablet_schema.h"
36+
#include "util/slice.h"
37+
38+
namespace doris {
39+
namespace {
40+
41+
Block create_nullable_hll_block(const std::vector<UInt8>& nullmap) {
42+
auto hll_column = ColumnHLL::create();
43+
auto nullmap_column = ColumnUInt8::create();
44+
for (size_t row = 0; row < nullmap.size(); ++row) {
45+
HyperLogLog hll;
46+
for (size_t value = 0; value <= row; ++value) {
47+
hll.update(row * 100 + value);
48+
}
49+
hll_column->insert_value(std::move(hll));
50+
nullmap_column->insert_value(nullmap[row]);
51+
}
52+
53+
Block block;
54+
block.insert({ColumnNullable::create(std::move(hll_column), std::move(nullmap_column)),
55+
make_nullable(std::make_shared<DataTypeHLL>()), "hll"});
56+
return block;
57+
}
58+
59+
OlapBlockDataConvertor create_hll_convertor() {
60+
TabletColumn hll_column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE,
61+
FieldType::OLAP_FIELD_TYPE_HLL, false);
62+
OlapBlockDataConvertor convertor;
63+
convertor.add_column_data_convertor(hll_column);
64+
return convertor;
65+
}
66+
67+
TEST(OlapColumnDataConvertorHLLTest, ConvertsAllNonNullNullableColumn) {
68+
auto block = create_nullable_hll_block({0, 0, 0});
69+
auto convertor = create_hll_convertor();
70+
convertor.set_source_content(&block, 0, block.rows());
71+
72+
auto [status, accessor] = convertor.convert_column_data(0);
73+
ASSERT_TRUE(status.ok()) << status;
74+
ASSERT_NE(nullptr, accessor->get_nullmap());
75+
76+
const auto* slices = static_cast<const Slice*>(accessor->get_data());
77+
for (size_t row = 0; row < block.rows(); ++row) {
78+
EXPECT_EQ(0, accessor->get_nullmap()[row]);
79+
ASSERT_NE(nullptr, slices[row].data);
80+
ASSERT_GT(slices[row].size, 0);
81+
ASSERT_TRUE(HyperLogLog::is_valid(slices[row]));
82+
HyperLogLog deserialized(slices[row]);
83+
EXPECT_EQ(row + 1, deserialized.estimate_cardinality());
84+
}
85+
}
86+
87+
TEST(OlapColumnDataConvertorHLLTest, ConvertsMixedNullsAtNonZeroRowPosition) {
88+
auto block = create_nullable_hll_block({1, 0, 1, 0, 1});
89+
auto convertor = create_hll_convertor();
90+
constexpr size_t row_pos = 1;
91+
constexpr size_t num_rows = 3;
92+
convertor.set_source_content(&block, row_pos, num_rows);
93+
94+
auto [status, accessor] = convertor.convert_column_data(0);
95+
ASSERT_TRUE(status.ok()) << status;
96+
ASSERT_NE(nullptr, accessor->get_nullmap());
97+
EXPECT_EQ(0, accessor->get_nullmap()[0]);
98+
EXPECT_EQ(1, accessor->get_nullmap()[1]);
99+
EXPECT_EQ(0, accessor->get_nullmap()[2]);
100+
101+
const auto* slices = static_cast<const Slice*>(accessor->get_data());
102+
ASSERT_NE(nullptr, slices[0].data);
103+
ASSERT_GT(slices[0].size, 0);
104+
EXPECT_EQ(nullptr, slices[1].data);
105+
EXPECT_EQ(0, slices[1].size);
106+
ASSERT_NE(nullptr, slices[2].data);
107+
ASSERT_GT(slices[2].size, 0);
108+
EXPECT_EQ(slices[0].data + slices[0].size, slices[2].data);
109+
110+
ASSERT_TRUE(HyperLogLog::is_valid(slices[0]));
111+
ASSERT_TRUE(HyperLogLog::is_valid(slices[2]));
112+
HyperLogLog first(slices[0]);
113+
HyperLogLog third(slices[2]);
114+
EXPECT_EQ(row_pos + 1, first.estimate_cardinality());
115+
EXPECT_EQ(row_pos + 3, third.estimate_cardinality());
116+
}
117+
118+
} // namespace
119+
} // namespace doris

0 commit comments

Comments
 (0)