Skip to content

Commit 522030c

Browse files
committed
[refactor](be) Share the read-time hidden column resolution policy
### What problem does this PR solve? Issue Number: None Related PR: #68125 Problem Summary: `Segment::_get_column_reader_for_read()` and the point-query/row-ID direct reads decided independently whether to synthesize a singleton VERSION/COMMIT_TSO value or read the stored value. The direct-read helper only checked `end_tso != -1`, so a singleton rowset with a non-singleton or unassigned TSO range (for example version [7-7] and TSO [100-200]) returned 200, while `Segment` rejected the same metadata. BINLOG_TSO was likewise handled only in `Segment`. This commit moves the policy into one resolver, `resolve_read_time_hidden_column_value()`, used by both callers. It preserves the pre-publish physical-read case (version [0-0] with TSO [-1--1]), reads materialized values from range rowsets, requires a singleton assigned TSO for a singleton rowset, and keeps BINLOG_TSO on the segment path for direct reads. The direct-read entry point now returns a `Status` so metadata errors reach the caller instead of exposing the on-disk placeholder. ### Release note None ### Check List (For Author) - Test: Unit Test / Regression test - BE UT: `PointQueryHiddenColumnTest.*` and `SegmentIteratorExprZonemapTest.*` passed as part of a 43-test run. - Local regression with `doris-local-regression --network 10.26.20.3/24 --offset 23600`: `point_query_p0/test_point_query_read_time_hidden_columns` and `inverted_index_p0/test_build_index_read_time_hidden_column` passed. - Behavior changed: Yes - Direct reads of a singleton rowset now reject an unassigned or non-singleton TSO range instead of returning the stored placeholder, matching segment scans. Pre-publish and range rowsets keep the existing physical read. - Does this need documentation: No
1 parent 1d85369 commit 522030c

6 files changed

Lines changed: 160 additions & 96 deletions

File tree

‎be/src/service/point_query_executor.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <stdlib.h>
2828

2929
#include <memory>
30+
#include <optional>
3031
#include <unordered_map>
3132
#include <vector>
3233

@@ -622,10 +623,11 @@ Status PointQueryExecutor::_lookup_row_data() {
622623
const auto tablet_schema = _tablet->tablet_schema();
623624
std::vector<int32_t> column_store_reads;
624625
for (int cid : _reusable->column_store_col_uids()) {
625-
if (auto hidden_value = get_read_time_hidden_column_value(
626-
get_read_time_hidden_column_type(*tablet_schema, cid),
627-
rowset->version(), rowset->commit_tso());
628-
hidden_value.has_value()) {
626+
std::optional<Field> hidden_value;
627+
RETURN_IF_ERROR(get_read_time_hidden_column_value(
628+
get_read_time_hidden_column_type(*tablet_schema, cid),
629+
rowset->version(), rowset->commit_tso(), &hidden_value));
630+
if (hidden_value.has_value()) {
629631
result_columns[_reusable->get_col_uid_to_idx().at(cid)]->insert(
630632
*hidden_value);
631633
continue;

‎be/src/storage/read_time_hidden_column.cpp‎

Lines changed: 66 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -50,27 +50,80 @@ bool has_load_placeholder_hidden_column(const TabletSchema& schema) {
5050
return schema.field_index(VERSION_COL) != -1 || schema.field_index(COMMIT_TSO_COL) != -1;
5151
}
5252

53-
std::optional<Field> get_read_time_hidden_column_value(ReadTimeHiddenColumnType column_type,
54-
const Version& version,
55-
const TsoRange& commit_tso) {
56-
if (version.first != version.second) {
57-
return std::nullopt;
53+
namespace {
54+
55+
const std::string& hidden_column_name(ReadTimeHiddenColumnType column_type) {
56+
switch (column_type) {
57+
case ReadTimeHiddenColumnType::VERSION:
58+
return VERSION_COL;
59+
case ReadTimeHiddenColumnType::COMMIT_TSO:
60+
return COMMIT_TSO_COL;
61+
case ReadTimeHiddenColumnType::BINLOG_TSO:
62+
return BINLOG_TSO_COL;
63+
case ReadTimeHiddenColumnType::NONE:
64+
break;
5865
}
66+
__builtin_unreachable();
67+
}
68+
69+
} // namespace
70+
71+
Status resolve_read_time_hidden_column_value(ReadTimeHiddenColumnType column_type,
72+
const Version& version, const TsoRange& commit_tso,
73+
std::optional<Field>* value) {
74+
value->reset();
5975
switch (column_type) {
6076
case ReadTimeHiddenColumnType::VERSION:
61-
return Field::create_field<TYPE_BIGINT>(version.second);
77+
// A singleton rowset exposes its rowset version for every row. Segment writers store a
78+
// placeholder because the publish version is not known when the segment is written.
79+
if (version.first == version.second) {
80+
value->emplace(Field::create_field<TYPE_BIGINT>(version.second));
81+
}
82+
return Status::OK();
6283
case ReadTimeHiddenColumnType::COMMIT_TSO:
63-
if (commit_tso.end_tso() != -1) {
64-
return Field::create_field<TYPE_BIGINT>(commit_tso.end_tso());
84+
case ReadTimeHiddenColumnType::BINLOG_TSO: {
85+
// Version [0-0] identifies a pre-publish physical read, such as segment compaction while a
86+
// RowsetWriter is still open. Its TSO is intentionally [-1--1], so preserve the on-disk
87+
// COMMIT_TSO_COL=0 or BINLOG_TSO_COL=NULL placeholder instead of synthesizing a value.
88+
if (version == Version(0, 0) && commit_tso.start_tso() == -1 &&
89+
commit_tso.end_tso() == -1) {
90+
return Status::OK();
6591
}
66-
return std::nullopt;
67-
case ReadTimeHiddenColumnType::BINLOG_TSO:
68-
// Direct reads leave BINLOG_TSO to the segment reader, which synthesizes it together with
69-
// the other read-time hidden columns.
92+
if (version.first != version.second) {
93+
// Range rowsets already contain materialized TSOs: compacting the initial empty [0-1]
94+
// rowset with [2-2] and [3-3] can leave a valid output with TSO range [-1, T3].
95+
return Status::OK();
96+
}
97+
// A singleton needs its publish TSO to replace the on-disk NULL/0 placeholder.
98+
if (commit_tso.start_tso() < 0 || commit_tso.end_tso() < commit_tso.start_tso()) {
99+
return Status::InternalError(
100+
"reading {} requires a valid commit tso, rowset version={}, commit tso={}",
101+
hidden_column_name(column_type), version.to_string(), commit_tso.to_string());
102+
}
103+
if (commit_tso.start_tso() != commit_tso.end_tso()) {
104+
return Status::InternalError(
105+
"singleton rowset {} requires a singleton commit tso when reading {}, got {}",
106+
version.to_string(), hidden_column_name(column_type), commit_tso.to_string());
107+
}
108+
value->emplace(Field::create_field<TYPE_BIGINT>(commit_tso.end_tso()));
109+
return Status::OK();
110+
}
70111
case ReadTimeHiddenColumnType::NONE:
71-
return std::nullopt;
112+
return Status::OK();
72113
}
73114
__builtin_unreachable();
74115
}
75116

117+
Status get_read_time_hidden_column_value(ReadTimeHiddenColumnType column_type,
118+
const Version& version, const TsoRange& commit_tso,
119+
std::optional<Field>* value) {
120+
if (column_type == ReadTimeHiddenColumnType::BINLOG_TSO) {
121+
// Direct reads leave BINLOG_TSO to the segment reader, which synthesizes it together with
122+
// the other read-time hidden columns.
123+
value->reset();
124+
return Status::OK();
125+
}
126+
return resolve_read_time_hidden_column_value(column_type, version, commit_tso, value);
127+
}
128+
76129
} // namespace doris

‎be/src/storage/read_time_hidden_column.h‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <cstdint>
2121
#include <optional>
2222

23+
#include "common/status.h"
2324
#include "core/field.h"
2425
#include "storage/olap_common.h"
2526

@@ -59,13 +60,20 @@ inline bool row_store_value_may_be_stale(ReadTimeHiddenColumnType column_type) {
5960
// substituting. BINLOG_TSO lives only on row-binlog tablets, whose link path never reaches this.
6061
bool has_load_placeholder_hidden_column(const TabletSchema& schema);
6162

62-
// The logical value every row of a single-version rowset carries for `column_type`, or nullopt
63-
// when the stored value is the one to read: a multi-version rowset, an unassigned commit TSO, or
64-
// BINLOG_TSO. Point-query and row-ID direct reads use this to answer VERSION/COMMIT_TSO without
65-
// opening the segment; every other column, BINLOG_TSO included, goes through Segment, which serves
66-
// a single-version read-time value with a ConstantColumnReader built from the same read options.
67-
std::optional<Field> get_read_time_hidden_column_value(ReadTimeHiddenColumnType column_type,
68-
const Version& version,
69-
const TsoRange& commit_tso);
63+
// The single VERSION/COMMIT_TSO/BINLOG_TSO resolution policy shared by Segment and direct reads.
64+
// On success, `value` receives the logical constant every row of the rowset carries, or nullopt
65+
// when the stored value is the one to read: a multi-version rowset, the pre-publish placeholder
66+
// rowset, or (for BINLOG_TSO) a value direct reads leave to the segment. A singleton rowset whose
67+
// TSO range is unassigned or not a singleton is rejected, exactly as a segment scan rejects it.
68+
Status resolve_read_time_hidden_column_value(ReadTimeHiddenColumnType column_type,
69+
const Version& version, const TsoRange& commit_tso,
70+
std::optional<Field>* value);
71+
72+
// Direct-read entry point for point-query and row-ID reads. VERSION/COMMIT_TSO resolve from rowset
73+
// metadata without opening a segment; BINLOG_TSO keeps its separate semantics and is always left
74+
// to the segment reader.
75+
Status get_read_time_hidden_column_value(ReadTimeHiddenColumnType column_type,
76+
const Version& version, const TsoRange& commit_tso,
77+
std::optional<Field>* value);
7078

7179
} // namespace doris

‎be/src/storage/segment/segment.cpp‎

Lines changed: 17 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <algorithm>
2727
#include <cstring>
2828
#include <memory>
29+
#include <optional>
2930
#include <set>
3031
#include <utility>
3132

@@ -67,6 +68,7 @@
6768
#include "storage/olap_common.h"
6869
#include "storage/predicate/block_column_predicate.h"
6970
#include "storage/predicate/column_predicate.h"
71+
#include "storage/read_time_hidden_column.h"
7072
#include "storage/rowset/rowset_reader_context.h"
7173
#include "storage/schema.h"
7274
#include "storage/segment/column_meta_accessor.h"
@@ -1026,57 +1028,25 @@ Status Segment::_get_column_reader_for_read(const TabletColumn& col,
10261028
RETURN_IF_ERROR(_create_column_meta_once(read_options.stats, &read_options.io_ctx));
10271029
SCOPED_RAW_TIMER(&read_options.stats->segment_create_column_readers_timer_ns);
10281030

1029-
if (col.name() == VERSION_COL) {
1030-
// A singleton rowset exposes its rowset version for every row. Segment writers store a
1031-
// placeholder because the publish version is not known when the segment is written.
1032-
if (read_options.version.first == read_options.version.second) {
1033-
*column_reader = std::make_shared<ConstantColumnReader>(
1034-
Field::create_field<TYPE_BIGINT>(read_options.version.second), col.type());
1031+
// Read-time hidden columns use the resolution policy shared with direct reads; anything else
1032+
// falls through to the stored column below.
1033+
const ReadTimeHiddenColumnType hidden_column_type = get_read_time_hidden_column_type(col);
1034+
if (hidden_column_type != ReadTimeHiddenColumnType::NONE) {
1035+
std::optional<Field> constant_value;
1036+
RETURN_IF_ERROR(
1037+
resolve_read_time_hidden_column_value(hidden_column_type, read_options.version,
1038+
read_options.commit_tso, &constant_value));
1039+
if (constant_value.has_value()) {
1040+
*column_reader =
1041+
std::make_shared<ConstantColumnReader>(std::move(*constant_value), col.type());
10351042
return Status::OK();
10361043
}
10371044
if (!_column_meta_accessor->has_column_uid(col_uid)) {
1038-
return Status::InternalError("could not find version column read version is {}-{}",
1039-
read_options.version.first, read_options.version.second);
1040-
}
1041-
return _column_reader_cache->get_column_reader(col_uid, column_reader, read_options.stats,
1042-
&read_options.io_ctx);
1043-
}
1044-
1045-
if (col.name() == BINLOG_TSO_COL || col.name() == COMMIT_TSO_COL) {
1046-
const int64_t start_tso = read_options.commit_tso.start_tso();
1047-
const int64_t end_tso = read_options.commit_tso.end_tso();
1048-
// Version [0-0] identifies a pre-publish physical read, such as segment compaction while a
1049-
// RowsetWriter is still open. Its TSO is intentionally [-1--1], so preserve the on-disk
1050-
// COMMIT_TSO_COL=0 or BINLOG_TSO_COL=NULL placeholder instead of synthesizing a value.
1051-
if (read_options.version == Version(0, 0) && start_tso == -1 && end_tso == -1) {
1052-
if (!_column_meta_accessor->has_column_uid(col_uid)) {
1053-
return Status::InternalError("could not find {} column", col.name());
1045+
if (hidden_column_type == ReadTimeHiddenColumnType::VERSION) {
1046+
return Status::InternalError("could not find version column read version is {}-{}",
1047+
read_options.version.first,
1048+
read_options.version.second);
10541049
}
1055-
return _column_reader_cache->get_column_reader(
1056-
col_uid, column_reader, read_options.stats, &read_options.io_ctx);
1057-
}
1058-
if (read_options.version.first == read_options.version.second) {
1059-
// A singleton needs its publish TSO to replace the on-disk NULL/0 placeholder.
1060-
// Range rowsets already contain materialized TSOs: compacting the initial empty
1061-
// [0-1] rowset with [2-2] and [3-3] can leave a valid output with TSO range [-1, T3].
1062-
if (start_tso < 0 || end_tso < start_tso) {
1063-
return Status::InternalError(
1064-
"reading {} requires a valid commit tso, rowset version={}, commit tso={}",
1065-
col.name(), read_options.version.to_string(),
1066-
read_options.commit_tso.to_string());
1067-
}
1068-
if (start_tso != end_tso) {
1069-
return Status::InternalError(
1070-
"singleton rowset {} requires a singleton commit tso when reading {}, "
1071-
"got {}",
1072-
read_options.version.to_string(), col.name(),
1073-
read_options.commit_tso.to_string());
1074-
}
1075-
*column_reader = std::make_shared<ConstantColumnReader>(
1076-
Field::create_field<TYPE_BIGINT>(end_tso), col.type());
1077-
return Status::OK();
1078-
}
1079-
if (!_column_meta_accessor->has_column_uid(col_uid)) {
10801050
return Status::InternalError("could not find {} column", col.name());
10811051
}
10821052
return _column_reader_cache->get_column_reader(col_uid, column_reader, read_options.stats,

‎be/test/service/point_query_exector_test.cpp‎

Lines changed: 44 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <gen_cpp/internal_service.pb.h>
2323
#include <gtest/gtest.h>
2424

25+
#include <optional>
2526
#include <unordered_set>
2627
#include <utility>
2728
#include <vector>
@@ -372,27 +373,51 @@ TEST_F(PointQueryHiddenColumnTest, BinlogTsoStaysInJsonbAndKeepsRowCache) {
372373
}
373374

374375
TEST_F(PointQueryHiddenColumnTest, RowsetDerivedValueOnlyForSingletonRowsets) {
376+
std::optional<Field> value;
375377
// A compacted rowset keeps its materialized per-row values.
376-
EXPECT_FALSE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::VERSION, Version(2, 3),
377-
TsoRange(2, 3))
378-
.has_value());
379-
EXPECT_FALSE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO,
380-
Version(2, 3), TsoRange(2, 3))
381-
.has_value());
378+
ASSERT_TRUE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::VERSION, Version(2, 3),
379+
TsoRange(2, 3), &value)
380+
.ok());
381+
EXPECT_FALSE(value.has_value());
382+
ASSERT_TRUE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO,
383+
Version(2, 3), TsoRange(2, 3), &value)
384+
.ok());
385+
EXPECT_FALSE(value.has_value());
382386
// A singleton rowset answers with its own version and assigned commit TSO.
383-
EXPECT_EQ(7, get_read_time_hidden_column_value(ReadTimeHiddenColumnType::VERSION, Version(7, 7),
384-
TsoRange(8, 8))
385-
->get<TYPE_BIGINT>());
386-
EXPECT_EQ(8, get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO,
387-
Version(7, 7), TsoRange(8, 8))
388-
->get<TYPE_BIGINT>());
389-
// An unassigned commit TSO keeps the stored value; BINLOG_TSO is always left to the segment.
390-
EXPECT_FALSE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO,
391-
Version(7, 7), TsoRange())
392-
.has_value());
393-
EXPECT_FALSE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::BINLOG_TSO,
394-
Version(7, 7), TsoRange(8, 8))
395-
.has_value());
387+
ASSERT_TRUE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::VERSION, Version(7, 7),
388+
TsoRange(8, 8), &value)
389+
.ok());
390+
ASSERT_TRUE(value.has_value());
391+
EXPECT_EQ(7, value->get<TYPE_BIGINT>());
392+
ASSERT_TRUE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO,
393+
Version(7, 7), TsoRange(8, 8), &value)
394+
.ok());
395+
ASSERT_TRUE(value.has_value());
396+
EXPECT_EQ(8, value->get<TYPE_BIGINT>());
397+
// The pre-publish placeholder stays a physical read.
398+
ASSERT_TRUE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO,
399+
Version(0, 0), TsoRange(-1, -1), &value)
400+
.ok());
401+
EXPECT_FALSE(value.has_value());
402+
// BINLOG_TSO keeps its separate semantics: direct reads always leave it to the segment.
403+
ASSERT_TRUE(get_read_time_hidden_column_value(ReadTimeHiddenColumnType::BINLOG_TSO,
404+
Version(7, 7), TsoRange(8, 8), &value)
405+
.ok());
406+
EXPECT_FALSE(value.has_value());
407+
408+
// A singleton rowset without a usable TSO is rejected, matching Segment, instead of exposing
409+
// the on-disk placeholder.
410+
auto st = get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO, Version(7, 7),
411+
TsoRange(), &value);
412+
EXPECT_FALSE(st.ok()) << st;
413+
EXPECT_NE(st.to_string().find("requires a valid commit tso"), std::string::npos) << st;
414+
EXPECT_FALSE(value.has_value());
415+
416+
st = get_read_time_hidden_column_value(ReadTimeHiddenColumnType::COMMIT_TSO, Version(7, 7),
417+
TsoRange(100, 200), &value);
418+
EXPECT_FALSE(st.ok()) << st;
419+
EXPECT_NE(st.to_string().find("requires a singleton commit tso"), std::string::npos) << st;
420+
EXPECT_FALSE(value.has_value());
396421
}
397422

398423
// RowCache test class

‎be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <functional>
2323
#include <future>
2424
#include <memory>
25+
#include <optional>
2526
#include <set>
2627
#include <string>
2728
#include <vector>
@@ -1553,11 +1554,16 @@ TEST_F(SegmentIteratorExprZonemapTest, ClassifiesReadTimeHiddenColumnsByName) {
15531554
EXPECT_EQ(column_type, get_read_time_hidden_column_type(_tablet_schema->column(1)));
15541555
EXPECT_EQ(ReadTimeHiddenColumnType::NONE,
15551556
get_read_time_hidden_column_type(_tablet_schema->column(0)));
1556-
EXPECT_EQ(kVersion, get_read_time_hidden_column_value(column_type, Version(kVersion, kVersion),
1557-
TsoRange())
1558-
->get<TYPE_BIGINT>());
1559-
EXPECT_FALSE(get_read_time_hidden_column_value(column_type, Version(2, kVersion), TsoRange())
1560-
.has_value());
1557+
std::optional<Field> value;
1558+
ASSERT_TRUE(get_read_time_hidden_column_value(column_type, Version(kVersion, kVersion),
1559+
TsoRange(), &value)
1560+
.ok());
1561+
ASSERT_TRUE(value.has_value());
1562+
EXPECT_EQ(kVersion, value->get<TYPE_BIGINT>());
1563+
ASSERT_TRUE(
1564+
get_read_time_hidden_column_value(column_type, Version(2, kVersion), TsoRange(), &value)
1565+
.ok());
1566+
EXPECT_FALSE(value.has_value());
15611567
}
15621568

15631569
TEST_F(SegmentIteratorExprZonemapTest, VersionPredicateSkipsPhysicalInvertedIndex) {

0 commit comments

Comments
 (0)