Skip to content

Commit 15e32e2

Browse files
[feat](doris-catalog) doris cross-cluster insert (#60557)
### What problem does this PR solve? Issue Number: close #60295 Related PR: #xxx Problem Summary: 1. support insert & insert overwrite on remote doris catalog 2. support temporary partitions on remote doris catalog 3. fix remote doris catalog rpc timeout unit 4. **DO NOT** support txn insert & group commit Signed-off-by: TsukiokaKogane <[email protected]>
1 parent 09bc20f commit 15e32e2

29 files changed

Lines changed: 3087 additions & 133 deletions

‎be/src/exec/tablet_info.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,10 @@ VOlapTablePartitionParam::VOlapTablePartitionParam(std::shared_ptr<OlapTableSche
447447
_overwrite_group_id = t_param.overwrite_group_id;
448448
}
449449

450+
if (t_param.__isset.master_address) {
451+
_master_address = std::make_shared<TNetworkAddress>(t_param.master_address);
452+
}
453+
450454
if (_is_auto_partition) {
451455
// the nullable mode depends on partition_exprs. not column slots. so use them.
452456
DCHECK(_partition_function.size() <= _slots.size())

‎be/src/exec/tablet_info.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <butil/fast_rand.h>
2121
#include <gen_cpp/Descriptors_types.h>
22+
#include <gen_cpp/Types_types.h>
2223
#include <gen_cpp/descriptors.pb.h>
2324
#include <gen_cpp/olap_file.pb.h>
2425

@@ -290,6 +291,8 @@ class VOlapTablePartitionParam {
290291
bool is_auto_detect_overwrite() const { return _is_auto_detect_overwrite; }
291292
int64_t get_overwrite_group_id() const { return _overwrite_group_id; }
292293

294+
std::shared_ptr<TNetworkAddress> get_master_address() const { return _master_address; }
295+
293296
std::vector<uint16_t> get_partition_keys() const { return _partition_slot_locs; }
294297

295298
Status add_partitions(const std::vector<TOlapTablePartition>& partitions);
@@ -346,6 +349,7 @@ class VOlapTablePartitionParam {
346349
// "insert overwrite partition(*)", detect which partitions by BE
347350
bool _is_auto_detect_overwrite = false;
348351
int64_t _overwrite_group_id = 0;
352+
std::shared_ptr<TNetworkAddress> _master_address = nullptr;
349353
};
350354

351355
// indicate where's the tablet and all its replications (node-wise)

‎be/src/vec/sink/vrow_distribution.cpp‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -124,14 +124,19 @@ Status VRowDistribution::automatic_create_partition() {
124124

125125
VLOG_NOTICE << "automatic partition rpc begin request " << request;
126126
if (!injected) {
127-
auto* cluster_info = ExecEnv::GetInstance()->cluster_info();
128-
if (cluster_info == nullptr) {
129-
return Status::InternalError("cluster_info is null");
127+
std::shared_ptr<TNetworkAddress> master_addr;
128+
if (_vpartition->get_master_address() == nullptr) {
129+
auto* cluster_info = ExecEnv::GetInstance()->cluster_info();
130+
if (cluster_info == nullptr) {
131+
return Status::InternalError("cluster_info is null");
132+
}
133+
master_addr = std::make_shared<TNetworkAddress>(cluster_info->master_fe_addr);
134+
} else {
135+
master_addr = _vpartition->get_master_address();
130136
}
131-
TNetworkAddress master_addr = cluster_info->master_fe_addr;
132137
int time_out = _state->execution_timeout() * 1000;
133138
RETURN_IF_ERROR(ThriftRpcHelper::rpc<FrontendServiceClient>(
134-
master_addr.hostname, master_addr.port,
139+
master_addr->hostname, master_addr->port,
135140
[&request, &result](FrontendServiceConnection& client) {
136141
client->createPartition(result, request);
137142
},
@@ -220,14 +225,19 @@ Status VRowDistribution::_replace_overwriting_partition() {
220225

221226
VLOG_NOTICE << "auto detect replace partition request: " << request;
222227
if (!injected) {
223-
auto* cluster_info = ExecEnv::GetInstance()->cluster_info();
224-
if (cluster_info == nullptr) {
225-
return Status::InternalError("cluster_info is null");
228+
std::shared_ptr<TNetworkAddress> master_addr;
229+
if (_vpartition->get_master_address() == nullptr) {
230+
auto* cluster_info = ExecEnv::GetInstance()->cluster_info();
231+
if (cluster_info == nullptr) {
232+
return Status::InternalError("cluster_info is null");
233+
}
234+
master_addr = std::make_shared<TNetworkAddress>(cluster_info->master_fe_addr);
235+
} else {
236+
master_addr = _vpartition->get_master_address();
226237
}
227-
TNetworkAddress master_addr = cluster_info->master_fe_addr;
228238
int time_out = _state->execution_timeout() * 1000;
229239
RETURN_IF_ERROR(ThriftRpcHelper::rpc<FrontendServiceClient>(
230-
master_addr.hostname, master_addr.port,
240+
master_addr->hostname, master_addr->port,
231241
[&request, &result](FrontendServiceConnection& client) {
232242
client->replacePartition(result, request);
233243
},

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ public enum OlapTableState {
196196
private DistributionInfo defaultDistributionInfo;
197197

198198
// all info about temporary partitions are save in "tempPartitions"
199+
@Setter
199200
@Getter
200201
@SerializedName(value = "tps", alternate = {"tempPartitions"})
201202
private TempPartitions tempPartitions = new TempPartitions();

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import com.google.common.collect.Sets;
2525
import com.google.gson.annotations.SerializedName;
2626

27+
import java.util.ArrayList;
28+
import java.util.Collection;
2729
import java.util.List;
2830
import java.util.Map;
2931
import java.util.Objects;
@@ -46,6 +48,12 @@ public class TempPartitions implements GsonPostProcessable {
4648
public TempPartitions() {
4749
}
4850

51+
// used by remote doris catalog
52+
public TempPartitions(Map<Long, Partition> idToPartition, Map<String, Partition> nameToPartition) {
53+
this.idToPartition = idToPartition;
54+
this.nameToPartition = nameToPartition;
55+
}
56+
4957
public void addPartition(Partition partition) {
5058
idToPartition.put(partition.getId(), partition);
5159
nameToPartition.put(partition.getName(), partition);
@@ -142,4 +150,12 @@ public boolean equals(Object o) {
142150
public int hashCode() {
143151
return Objects.hash(idToPartition, nameToPartition, partitionInfo);
144152
}
153+
154+
public List<Long> getPartitionIds() {
155+
return new ArrayList<>(idToPartition.keySet());
156+
}
157+
158+
public Collection<Partition> getPartitions() {
159+
return idToPartition.values();
160+
}
145161
}

‎fe/fe-core/src/main/java/org/apache/doris/common/QuotaExceedException.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,8 @@ public QuotaExceedException(String databaseName, long dataQuotaBytes) {
2525
super("Database[" + databaseName + "] data size exceeds quota["
2626
+ DebugUtil.printByteWithUnit(dataQuotaBytes) + "]");
2727
}
28+
29+
public QuotaExceedException(String msg) {
30+
super(msg);
31+
}
2832
}

0 commit comments

Comments
 (0)