Skip to content

Commit 1072377

Browse files
yujun777pull[bot]
authored andcommitted
[fix](dynamic partition) fix create hot partition failed without error response (#20996)
1 parent 4707d8a commit 1072377

5 files changed

Lines changed: 93 additions & 15 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -231,11 +231,24 @@ private static void checkReplicationNum(String val, Database db) throws DdlExcep
231231
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null);
232232
}
233233

234-
private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, Database db) throws DdlException {
234+
private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int hotPartitionNum,
235+
Database db) throws DdlException {
235236
if (replicaAlloc.getTotalReplicaNum() <= 0) {
236237
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO);
237238
}
239+
238240
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null);
241+
if (hotPartitionNum <= 0) {
242+
return;
243+
}
244+
245+
try {
246+
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, TStorageMedium.SSD);
247+
} catch (DdlException e) {
248+
throw new DdlException("Failed to find enough backend for ssd storage medium. When setting "
249+
+ DynamicPartitionProperty.HOT_PARTITION_NUM + " > 0, the hot partitions will store "
250+
+ "in ssd. Please check the replication num,replication tag and storage medium.");
251+
}
239252
}
240253

241254
private static void checkHotPartitionNum(String val) throws DdlException {
@@ -600,28 +613,33 @@ public static Map<String, String> analyzeDynamicPartition(Map<String, String> pr
600613
analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val);
601614
}
602615

616+
int hotPartitionNum = 0;
617+
if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
618+
String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
619+
checkHotPartitionNum(val);
620+
hotPartitionNum = Integer.parseInt(val);
621+
properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
622+
analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val);
623+
}
624+
603625
// check replication_allocation first, then replciation_num
626+
ReplicaAllocation replicaAlloc = null;
604627
if (properties.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION)) {
605-
ReplicaAllocation replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
606-
checkReplicaAllocation(replicaAlloc, db);
628+
replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
607629
properties.remove(DynamicPartitionProperty.REPLICATION_ALLOCATION);
608630
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION, replicaAlloc.toCreateStmt());
609631
} else if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) {
610632
String val = properties.get(DynamicPartitionProperty.REPLICATION_NUM);
611633
checkReplicationNum(val, db);
612634
properties.remove(DynamicPartitionProperty.REPLICATION_NUM);
635+
replicaAlloc = new ReplicaAllocation(Short.valueOf(val));
613636
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
614-
new ReplicaAllocation(Short.valueOf(val)).toCreateStmt());
637+
replicaAlloc.toCreateStmt());
615638
} else {
616-
checkReplicaAllocation(olapTable.getDefaultReplicaAllocation(), db);
639+
replicaAlloc = olapTable.getDefaultReplicaAllocation();
617640
}
641+
checkReplicaAllocation(replicaAlloc, hotPartitionNum, db);
618642

619-
if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
620-
String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
621-
checkHotPartitionNum(val);
622-
properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
623-
analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val);
624-
}
625643
if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS)) {
626644
String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS);
627645
checkReservedHistoryPeriodValidate(reservedHistoryPeriods,

‎fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.doris.common.ExceptionChecker;
2929
import org.apache.doris.common.FeConstants;
3030
import org.apache.doris.qe.ConnectContext;
31+
import org.apache.doris.system.Backend;
3132
import org.apache.doris.thrift.TStorageMedium;
3233
import org.apache.doris.utframe.UtFrameUtils;
3334

@@ -82,6 +83,15 @@ public static void tearDown() {
8283
UtFrameUtils.cleanDorisFeDir(runningDir);
8384
}
8485

86+
private static void changeBeDisk(TStorageMedium storageMedium) {
87+
List<Backend> backends = Env.getCurrentSystemInfo().getAllBackends();
88+
for (Backend be : backends) {
89+
for (DiskInfo diskInfo : be.getDisks().values()) {
90+
diskInfo.setStorageMedium(storageMedium);
91+
}
92+
}
93+
}
94+
8595
private static void createTable(String sql) throws Exception {
8696
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
8797
Env.getCurrentEnv().createTable(createTableStmt);
@@ -970,6 +980,8 @@ public void testHourDynamicPartitionWithIntType() throws Exception {
970980

971981
@Test
972982
public void testHotPartitionNum() throws Exception {
983+
changeBeDisk(TStorageMedium.SSD);
984+
973985
Database testDb =
974986
Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test");
975987
// 1. hour
@@ -1188,9 +1200,11 @@ public void testHotPartitionNum() throws Exception {
11881200
}
11891201

11901202
@Test(expected = DdlException.class)
1191-
public void testHotPartitionNumAbnormal() throws Exception {
1203+
public void testHotPartitionNumAbnormalLT0() throws Exception {
1204+
changeBeDisk(TStorageMedium.SSD);
1205+
11921206
// dynamic_partition.hot_partition_num must larger than 0.
1193-
String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1` (\n"
1207+
String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_lt0` (\n"
11941208
+ " `k1` datetime NULL COMMENT \"\",\n"
11951209
+ " `k2` int NULL COMMENT \"\"\n"
11961210
+ ") ENGINE=OLAP\n"
@@ -1211,6 +1225,32 @@ public void testHotPartitionNumAbnormal() throws Exception {
12111225
createTable(createOlapTblStmt);
12121226
}
12131227

1228+
@Test(expected = DdlException.class)
1229+
public void testHotPartitionNumAbnormalMissSSD() throws Exception {
1230+
changeBeDisk(TStorageMedium.HDD);
1231+
1232+
// when dynamic_partition.hot_partition_num > 0, it require ssd storage medium.
1233+
String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_miss_ssd` (\n"
1234+
+ " `k1` datetime NULL COMMENT \"\",\n"
1235+
+ " `k2` int NULL COMMENT \"\"\n"
1236+
+ ") ENGINE=OLAP\n"
1237+
+ "PARTITION BY RANGE(`k1`)\n"
1238+
+ "()\n"
1239+
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n"
1240+
+ "PROPERTIES (\n"
1241+
+ "\"replication_num\" = \"1\",\n"
1242+
+ "\"dynamic_partition.enable\" = \"true\",\n"
1243+
+ "\"dynamic_partition.start\" = \"-3\",\n"
1244+
+ "\"dynamic_partition.end\" = \"3\",\n"
1245+
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
1246+
+ "\"dynamic_partition.time_unit\" = \"hour\",\n"
1247+
+ "\"dynamic_partition.prefix\" = \"p\",\n"
1248+
+ "\"dynamic_partition.buckets\" = \"1\",\n"
1249+
+ "\"dynamic_partition.hot_partition_num\" = \"1\"\n"
1250+
+ ");";
1251+
createTable(createOlapTblStmt);
1252+
}
1253+
12141254
@Test
12151255
public void testRuntimeInfo() throws Exception {
12161256
DynamicPartitionScheduler scheduler = new DynamicPartitionScheduler("test", 10);

‎regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,6 @@ suite("test_outer_join_with_with_window_function") {
218218
"dynamic_partition.buckets" = "4",
219219
"dynamic_partition.create_history_partition" = "true",
220220
"dynamic_partition.history_partition_num" = "50",
221-
"dynamic_partition.hot_partition_num" = "2",
222221
"dynamic_partition.reserved_history_periods" = "NULL",
223222
"dynamic_partition.start_day_of_month" = "1",
224223
"in_memory" = "false",

‎regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,6 @@ suite("test_outer_join_with_window_function_datev2") {
222222
"dynamic_partition.buckets" = "4",
223223
"dynamic_partition.create_history_partition" = "true",
224224
"dynamic_partition.history_partition_num" = "50",
225-
"dynamic_partition.hot_partition_num" = "2",
226225
"dynamic_partition.reserved_history_periods" = "NULL",
227226
"dynamic_partition.start_day_of_month" = "1",
228227
"in_memory" = "false",

‎regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,4 +149,26 @@ suite("test_dynamic_partition") {
149149
exception "errCode = 2,"
150150
}
151151
sql "drop table if exists dy_par_bad"
152+
test {
153+
sql """
154+
CREATE TABLE IF NOT EXISTS dy_par_bad
155+
( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
156+
AGGREGATE KEY(k1,k2)
157+
PARTITION BY RANGE(k1) ( )
158+
DISTRIBUTED BY HASH(k1) BUCKETS 3
159+
PROPERTIES (
160+
"dynamic_partition.enable"="true",
161+
"dynamic_partition.end"="3",
162+
"dynamic_partition.start"="-3",
163+
"dynamic_partition.prefix"="p",
164+
"dynamic_partition.time_unit"="DAY",
165+
"dynamic_partition.create_history_partition"="true",
166+
"dynamic_partition.hot_partition_num" = "2",
167+
"dynamic_partition.replication_allocation" = "tag.location.default: 1")
168+
"""
169+
// check exception message contains
170+
// dynamic_partition.hot_partition_num > 0 will require ssd storage medium
171+
exception "errCode = 2,"
172+
}
173+
sql "drop table if exists dy_par_bad"
152174
}

0 commit comments

Comments
 (0)