Skip to content

Commit 7c47b53

Browse files
committed
[feature](cloud) Support Cloud Table Stream control plane
### What problem does this PR solve? Issue Number: close apache#65418 Related PR: apache#65810 Problem Summary: Table Stream keeps per-partition consumption offsets in FE for storage-compute integrated deployments. In Cloud mode, target-table transactions are committed by MetaService, so FE-owned offsets cannot be advanced atomically with target data. This change adds the Cloud Table Stream control plane, stores Latest and Versioned offsets in MetaService, reuses the Index and recycler lifecycle, resolves statement-level read snapshots through clone chains, and commits expected-offset CAS updates with target data in the same immediate FDB transaction. It also integrates the Row Binlog commit TSO contract, initializes empty-partition TSO metadata, and waits for pending lazy publish using the actual clone-chain owner before rereading the complete snapshot. ### Release note Add the storage-compute separated Table Stream control plane. ### Check List (For Author) - Test: - Unit Test: Cloud Table Stream 28/28, CloneChainReader 25/25, empty partition TSO initialization 1/1, FE Cloud Stream 26/26 - Behavior changed: Yes. Cloud Table Stream uses MetaService-managed offsets and immediate atomic consumption commits while preserving non-Cloud behavior. - Does this need documentation: Yes. The implementation and test designs have been updated.
1 parent 5f3321b commit 7c47b53

72 files changed

Lines changed: 8512 additions & 268 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cloud/src/common/bvars.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ BvarLatencyRecorderWithTag g_bvar_ms_abort_txn_with_coordinator("ms", "abort_txn
4747
BvarLatencyRecorderWithTag g_bvar_ms_get_prepare_txn_by_coordinator("ms", "get_prepare_txn_by_coordinator");
4848
BvarLatencyRecorderWithTag g_bvar_ms_clean_txn_label("ms", "clean_txn_label");
4949
BvarLatencyRecorderWithTag g_bvar_ms_get_version("ms", "get_version");
50+
BvarLatencyRecorderWithTag g_bvar_ms_get_table_stream_offset(
51+
"ms", "get_table_stream_offset");
5052
BvarLatencyRecorderWithTag g_bvar_ms_batch_get_version("ms", "batch_get_version");
5153
BvarLatencyRecorderWithTag g_bvar_ms_create_tablets("ms", "create_tablets");
5254
BvarLatencyRecorderWithTag g_bvar_ms_update_tablet("ms", "update_tablet");
@@ -321,6 +323,9 @@ BvarStatusWithTag<int64_t> g_bvar_checker_restore_job_cost_many_time("checker",
321323
mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_counter("rpc_kv_get_rowset_get_counter",{"instance_id"});
322324
// get_version
323325
mBvarInt64Adder g_bvar_rpc_kv_get_version_get_counter("rpc_kv_get_version_get_counter",{"instance_id"});
326+
// get_table_stream_offset
327+
mBvarInt64Adder g_bvar_rpc_kv_get_table_stream_offset_get_counter(
328+
"rpc_kv_get_table_stream_offset_get_counter", {"instance_id"});
324329
// get_schema_dict
325330
mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_counter("rpc_kv_get_schema_dict_get_counter",{"instance_id"});
326331
// create_tablets
@@ -528,6 +533,9 @@ mBvarInt64Adder g_bvar_rpc_kv_compact_snapshot_put_counter("rpc_kv_compact_snaps
528533
mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_bytes("rpc_kv_get_rowset_get_bytes",{"instance_id"});
529534
// get_version
530535
mBvarInt64Adder g_bvar_rpc_kv_get_version_get_bytes("rpc_kv_get_version_get_bytes",{"instance_id"});
536+
// get_table_stream_offset
537+
mBvarInt64Adder g_bvar_rpc_kv_get_table_stream_offset_get_bytes(
538+
"rpc_kv_get_table_stream_offset_get_bytes", {"instance_id"});
531539
// get_schema_dict
532540
mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_bytes("rpc_kv_get_schema_dict_get_bytes",{"instance_id"});
533541
// create_tablets

‎cloud/src/common/bvars.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -559,6 +559,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_begin_sub_txn;
559559
extern BvarLatencyRecorderWithTag g_bvar_ms_abort_sub_txn;
560560
extern BvarLatencyRecorderWithTag g_bvar_ms_clean_txn_label;
561561
extern BvarLatencyRecorderWithTag g_bvar_ms_get_version;
562+
extern BvarLatencyRecorderWithTag g_bvar_ms_get_table_stream_offset;
562563
extern BvarLatencyRecorderWithTag g_bvar_ms_batch_get_version;
563564
extern BvarLatencyRecorderWithTag g_bvar_ms_create_tablets;
564565
extern BvarLatencyRecorderWithTag g_bvar_ms_update_tablet;
@@ -787,6 +788,7 @@ extern BvarStatusWithTag<int64_t> g_bvar_checker_restore_job_cost_many_time;
787788
// rpc kv
788789
extern mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_counter;
789790
extern mBvarInt64Adder g_bvar_rpc_kv_get_version_get_counter;
791+
extern mBvarInt64Adder g_bvar_rpc_kv_get_table_stream_offset_get_counter;
790792
extern mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_counter;
791793
extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_counter;
792794
extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_counter;
@@ -929,6 +931,7 @@ extern mBvarInt64Adder g_bvar_rpc_kv_compact_snapshot_put_counter;
929931

930932
extern mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_bytes;
931933
extern mBvarInt64Adder g_bvar_rpc_kv_get_version_get_bytes;
934+
extern mBvarInt64Adder g_bvar_rpc_kv_get_table_stream_offset_get_bytes;
932935
extern mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_bytes;
933936
extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_bytes;
934937
extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_bytes;

‎cloud/src/meta-service/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ add_library(MetaService
3434
meta_service_schema.cpp
3535
meta_service_snapshot.cpp
3636
meta_service_tablet_stats.cpp
37+
meta_service_table_stream.cpp
38+
table_stream_metadata_reader.cpp
3739
meta_service_partition.cpp
3840
meta_service_txn.cpp
3941
meta_service_util.cpp

‎cloud/src/meta-service/http_encode_key.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@ static std::unordered_map<std::string_view,
254254
{"TableVersionKey", {{"instance_id", "db_id", "tbl_id"}, [](param_type& p) { return table_version_key(KeyInfoSetter<TableVersionKeyInfo>{p}.get()); }, parse<VersionPB> , parse_json<VersionPB>}},
255255
{"MetaRowsetKey", {{"instance_id", "tablet_id", "version"}, [](param_type& p) { return meta_rowset_key(KeyInfoSetter<MetaRowsetKeyInfo>{p}.get()); }, parse<doris::RowsetMetaCloudPB> , parse_json<doris::RowsetMetaCloudPB>}},
256256
{"MetaRowsetTmpKey", {{"instance_id", "txn_id", "tablet_id"}, [](param_type& p) { return meta_rowset_tmp_key(KeyInfoSetter<MetaRowsetTmpKeyInfo>{p}.get()); }, parse<doris::RowsetMetaCloudPB> , parse_json<doris::RowsetMetaCloudPB>}},
257+
{"TableStreamOffsetKey", {{"instance_id", "base_db_id", "base_table_id", "stream_db_id", "stream_id", "partition_id"}, [](param_type& p) { return table_stream_offset_key(KeyInfoSetter<TableStreamOffsetKeyInfo>{p}.get()); }, parse<TableStreamOffsetPB>, parse_json<TableStreamOffsetPB>}},
257258
{"MetaTabletKey", {{"instance_id", "table_id", "index_id", "part_id", "tablet_id"}, [](param_type& p) { return meta_tablet_key(KeyInfoSetter<MetaTabletKeyInfo>{p}.get()); }, parse<doris::TabletMetaCloudPB> , parse_json<doris::TabletMetaCloudPB>}},
258259
{"MetaTabletIdxKey", {{"instance_id", "tablet_id"}, [](param_type& p) { return meta_tablet_idx_key(KeyInfoSetter<MetaTabletIdxKeyInfo>{p}.get()); }, parse<TabletIndexPB> , parse_json<TabletIndexPB>}},
259260
{"RecycleIndexKey", {{"instance_id", "index_id"}, [](param_type& p) { return recycle_index_key(KeyInfoSetter<RecycleIndexKeyInfo>{p}.get()); }, parse<RecycleIndexPB> , parse_json<RecycleIndexPB>}},
@@ -304,6 +305,7 @@ static std::unordered_map<std::string_view,
304305
{"VersionedMetaRowsetLoadKey", {{"instance_id", "tablet_id", "version"}, [](param_type& p) { return versioned::meta_rowset_load_key(KeyInfoSetter<versioned::MetaRowsetLoadKeyInfo>{p}.get()); }, true, [](const std::string& s) { return ""; }, parse_json<doris::RowsetMetaCloudPB>}},
305306
{"VersionedMetaRowsetCompactKey", {{"instance_id", "tablet_id", "version"}, [](param_type& p) { return versioned::meta_rowset_compact_key(KeyInfoSetter<versioned::MetaRowsetCompactKeyInfo>{p}.get()); }, true, [](const std::string& s) { return ""; }, parse_json<doris::RowsetMetaCloudPB>}},
306307
{"VersionedMetaDeleteBitmapKey", {{"instance_id", "tablet_id", "rowset_id"}, [](param_type& p) { return versioned::meta_delete_bitmap_key(KeyInfoSetter<versioned::MetaDeleteBitmapInfo>{p}.get()); }, false, parse_to_json<DeleteBitmapStoragePB>, parse_json<DeleteBitmapStoragePB>}},
308+
{"VersionedTableStreamOffsetKey", {{"instance_id", "base_db_id", "base_table_id", "stream_db_id", "stream_id", "partition_id"}, [](param_type& p) { return versioned::table_stream_offset_key(KeyInfoSetter<versioned::TableStreamOffsetKeyInfo>{p}.get()); }, true, parse_to_json<TableStreamOffsetPB>, parse_json<TableStreamOffsetPB>}},
307309
{"VersionedDataRowsetRefCountKey", {{"instance_id", "tablet_id", "rowset_id"}, [](param_type& p) { return versioned::data_rowset_ref_count_key(KeyInfoSetter<versioned::DataRowsetRefCountKeyInfo>{p}.get()); }, false, parse_int64_value, nullptr}},
308310
};
309311
// clang-format on

‎cloud/src/meta-service/meta_service.cpp‎

Lines changed: 45 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -277,8 +277,9 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller,
277277
response->set_version(table_version.version());
278278
} else {
279279
VersionPB partition_version;
280-
TxnErrorCode err =
281-
reader.get_partition_version(partition_id, &partition_version, nullptr);
280+
std::string source_instance;
281+
TxnErrorCode err = reader.get_partition_version(partition_id, &partition_version,
282+
nullptr, false, &source_instance);
282283
if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
283284
msg = "partition version not found";
284285
code = MetaServiceCode::VERSION_NOT_FOUND;
@@ -291,20 +292,17 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller,
291292
}
292293

293294
if (request->wait_for_pending_txn() && partition_version.pending_txn_ids_size() > 0) {
294-
DCHECK_EQ(partition_version.pending_txn_ids_size(), 1)
295-
<< "only support one pending txn id for now, version="
296-
<< partition_version.ShortDebugString();
297-
int64_t first_txn_id = partition_version.pending_txn_ids(0);
298-
std::shared_ptr<TxnLazyCommitTask> task =
299-
txn_lazy_committer_->submit(instance_id, first_txn_id);
300-
std::tie(code, msg) = task->wait();
295+
std::vector<VersionPB> pending_versions {partition_version};
296+
std::tie(code, msg) = wait_for_pending_txns(source_instance, pending_versions);
301297
if (code != MetaServiceCode::OK) {
302-
LOG(WARNING) << "wait for pending txn " << first_txn_id
303-
<< " failed, code=" << code << " msg=" << msg;
304298
return;
305299
}
306-
partition_version.set_version(partition_version.version() + 1);
307-
partition_version.clear_pending_txn_ids();
300+
partition_version = std::move(pending_versions.front());
301+
}
302+
if (!partition_version.has_version()) {
303+
msg = "partition version not found";
304+
code = MetaServiceCode::VERSION_NOT_FOUND;
305+
return;
308306
}
309307

310308
response->set_version(partition_version.version());
@@ -351,27 +349,18 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller,
351349
msg = "malformed version value";
352350
return;
353351
}
354-
if (!version_pb.has_version()) {
355-
msg = "not found";
356-
code = MetaServiceCode::VERSION_NOT_FOUND;
357-
return;
358-
}
359-
360352
if (request->wait_for_pending_txn() && version_pb.pending_txn_ids_size() > 0) {
361-
DCHECK_EQ(version_pb.pending_txn_ids_size(), 1)
362-
<< "only support one pending txn id for now, version="
363-
<< version_pb.ShortDebugString();
364-
int64_t first_txn_id = version_pb.pending_txn_ids(0);
365-
std::shared_ptr<TxnLazyCommitTask> task =
366-
txn_lazy_committer_->submit(instance_id, first_txn_id);
367-
std::tie(code, msg) = task->wait();
353+
std::vector<VersionPB> pending_versions {version_pb};
354+
std::tie(code, msg) = wait_for_pending_txns(instance_id, pending_versions);
368355
if (code != MetaServiceCode::OK) {
369-
LOG(WARNING) << "wait for pending txn " << first_txn_id
370-
<< " failed, code=" << code << " msg=" << msg;
371356
return;
372357
}
373-
version_pb.set_version(version_pb.version() + 1);
374-
version_pb.clear_pending_txn_ids();
358+
version_pb = std::move(pending_versions.front());
359+
}
360+
if (!version_pb.has_version()) {
361+
msg = "not found";
362+
code = MetaServiceCode::VERSION_NOT_FOUND;
363+
return;
375364
}
376365

377366
response->set_version(version_pb.version());
@@ -652,8 +641,9 @@ std::pair<MetaServiceCode, std::string> MetaServiceImpl::batch_get_partition_ver
652641
}
653642
std::unordered_map<int64_t, VersionPB> partition_version_map;
654643
std::unordered_map<int64_t, Versionstamp> versionstamps;
644+
std::unordered_map<int64_t, std::string> partition_source_instances;
655645
err = reader.get_partition_versions(acquired_ids, &partition_version_map,
656-
&versionstamps, true);
646+
&versionstamps, true, &partition_source_instances);
657647
if (err == TxnErrorCode::TXN_TOO_OLD) {
658648
// txn too old, fallback to non-snapshot versions.
659649
LOG(WARNING) << "batch_get_version execution time exceeds the txn mvcc window, "
@@ -666,6 +656,7 @@ std::pair<MetaServiceCode, std::string> MetaServiceImpl::batch_get_partition_ver
666656
}
667657

668658
std::vector<VersionPB> partition_versions;
659+
std::vector<std::string> version_source_instances;
669660
for (auto& acquired_id : acquired_ids) {
670661
auto it = partition_version_map.find(acquired_id);
671662
if (it == partition_version_map.end()) {
@@ -674,15 +665,33 @@ std::pair<MetaServiceCode, std::string> MetaServiceImpl::batch_get_partition_ver
674665
version_pb.set_version(-1);
675666
version_pb.set_update_time_ms(-1);
676667
partition_versions.push_back(std::move(version_pb));
668+
version_source_instances.emplace_back(instance_id);
677669
} else {
678670
partition_versions.push_back(it->second);
671+
DCHECK(partition_source_instances.contains(acquired_id));
672+
version_source_instances.push_back(partition_source_instances.at(acquired_id));
679673
}
680674
}
681675
if (request->wait_for_pending_txn()) {
682-
auto&& [code, msg] =
683-
wait_for_pending_txns(std::string(instance_id), partition_versions);
684-
if (code != MetaServiceCode::OK) {
685-
return {code, msg};
676+
std::unordered_map<std::string, std::vector<size_t>> pending_indexes;
677+
for (size_t j = 0; j < partition_versions.size(); ++j) {
678+
if (!partition_versions[j].pending_txn_ids().empty()) {
679+
pending_indexes[version_source_instances[j]].push_back(j);
680+
}
681+
}
682+
for (const auto& [source_instance, indexes] : pending_indexes) {
683+
std::vector<VersionPB> pending_versions;
684+
pending_versions.reserve(indexes.size());
685+
for (size_t index : indexes) {
686+
pending_versions.push_back(partition_versions[index]);
687+
}
688+
auto&& [code, msg] = wait_for_pending_txns(source_instance, pending_versions);
689+
if (code != MetaServiceCode::OK) {
690+
return {code, msg};
691+
}
692+
for (size_t j = 0; j < indexes.size(); ++j) {
693+
partition_versions[indexes[j]] = std::move(pending_versions[j]);
694+
}
686695
}
687696
}
688697
for (auto& version_pb : partition_versions) {
@@ -5932,7 +5941,7 @@ std::pair<MetaServiceCode, std::string> MetaServiceImpl::wait_for_pending_txns(
59325941
continue;
59335942
}
59345943
version.clear_pending_txn_ids();
5935-
version.set_version(version.version() + 1);
5944+
version.set_version(version.has_version() ? version.version() + 1 : 2);
59365945
}
59375946

59385947
return {MetaServiceCode::OK, ""};

‎cloud/src/meta-service/meta_service.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,11 @@ class MetaServiceImpl : public cloud::MetaService {
163163
const GetVersionRequest* request, GetVersionResponse* response,
164164
::google::protobuf::Closure* done) override;
165165

166+
void get_table_stream_offset(::google::protobuf::RpcController* controller,
167+
const GetTableStreamOffsetRequest* request,
168+
GetTableStreamOffsetResponse* response,
169+
::google::protobuf::Closure* done) override;
170+
166171
void batch_get_version(::google::protobuf::RpcController* controller,
167172
const GetVersionRequest* request, GetVersionResponse* response,
168173
::google::protobuf::Closure* done);
@@ -514,6 +519,10 @@ class MetaServiceImpl : public cloud::MetaService {
514519
const std::vector<int64_t>& partition_ids,
515520
PartitionResponse* response, MetaServiceCode& code,
516521
std::string& msg, KVStats& stats);
522+
void commit_table_stream_partition_internal(Transaction* txn, const PartitionRequest* request,
523+
const std::string& instance_id,
524+
const std::vector<int64_t>& partition_ids,
525+
MetaServiceCode& code, std::string& msg);
517526

518527
// Wait for all pending transactions before returning, and bump up the version to the latest.
519528
std::pair<MetaServiceCode, std::string> wait_for_pending_txns(const std::string& instance_id,
@@ -635,6 +644,14 @@ class MetaServiceProxy final : public MetaService {
635644
call_impl(&cloud::MetaService::get_version, controller, request, response, done);
636645
}
637646

647+
void get_table_stream_offset(::google::protobuf::RpcController* controller,
648+
const GetTableStreamOffsetRequest* request,
649+
GetTableStreamOffsetResponse* response,
650+
::google::protobuf::Closure* done) override {
651+
call_impl(&cloud::MetaService::get_table_stream_offset, controller, request, response,
652+
done);
653+
}
654+
638655
void create_tablets(::google::protobuf::RpcController* controller,
639656
const CreateTabletsRequest* request, CreateTabletsResponse* response,
640657
::google::protobuf::Closure* done) override {

0 commit comments

Comments
 (0)