Skip to content

Commit 7cfebd0

Browse files
morningmanmorningman
authored
[fix](hierarchical-storage) Fix bug that storage medium property change back to SSD (#9158)
1. fix bug described in #9159 2. fix a `fill_tuple` bug introduced from #9173
1 parent 62b38d7 commit 7cfebd0

25 files changed

Lines changed: 96 additions & 247 deletions

File tree

‎be/src/exec/broker_scanner.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,7 @@ Status BrokerScanner::_convert_one_row(const Slice& line, Tuple* tuple, MemPool*
467467
RETURN_IF_ERROR(_line_to_src_tuple(line));
468468
if (!_success) {
469469
// If not success, which means we met an invalid row, return.
470+
*fill_tuple = false;
470471
return Status::OK();
471472
}
472473

‎be/test/olap/test_data/header_without_inc_rs.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,5 +141,7 @@
141141
"lo": 10
142142
},
143143
"preferred_rowset_type": "BETA_ROWSET",
144-
"tablet_type": "TABLET_TYPE_DISK"
144+
"tablet_type": "TABLET_TYPE_DISK",
145+
"storage_medium": "HDD",
146+
"remote_storage_name": ""
145147
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,6 @@
295295
import java.util.concurrent.atomic.AtomicBoolean;
296296
import java.util.concurrent.atomic.AtomicLong;
297297
import java.util.stream.Collectors;
298-
299298
import javax.annotation.Nullable;
300299

301300
public class Catalog {
@@ -4997,7 +4996,8 @@ public HashMap<Long, TStorageMedium> getPartitionIdToStorageMediumMap() {
49974996
if (dataProperty.getStorageMedium() == TStorageMedium.SSD
49984997
&& dataProperty.getCooldownTimeMs() < currentTimeMs) {
49994998
// expire. change to HDD.
5000-
partitionInfo.setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD));
4999+
DataProperty hddProperty = new DataProperty(TStorageMedium.HDD);
5000+
partitionInfo.setDataProperty(partition.getId(), hddProperty);
50015001
storageMediumMap.put(partitionId, TStorageMedium.HDD);
50025002
LOG.debug("partition[{}-{}-{}] storage medium changed from SSD to HDD",
50035003
dbId, tableId, partitionId);
@@ -5006,7 +5006,7 @@ public HashMap<Long, TStorageMedium> getPartitionIdToStorageMediumMap() {
50065006
ModifyPartitionInfo info =
50075007
new ModifyPartitionInfo(db.getId(), olapTable.getId(),
50085008
partition.getId(),
5009-
DataProperty.DEFAULT_DATA_PROPERTY,
5009+
hddProperty,
50105010
ReplicaAllocation.NOT_SET,
50115011
partitionInfo.getIsInMemory(partition.getId()));
50125012
editLog.logModifyPartition(info);

‎fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767
import java.util.List;
6868
import java.util.Map;
6969
import java.util.UUID;
70-
7170
import javax.servlet.http.HttpServletRequest;
7271
import javax.servlet.http.HttpServletResponse;
7372

@@ -129,6 +128,9 @@ public Object query_plan(
129128
// status code should conforms to HTTP semantic
130129
resultMap.put("status", e.getCode().code());
131130
resultMap.put("exception", e.getMessage());
131+
} catch (Exception e) {
132+
resultMap.put("status", "1");
133+
resultMap.put("exception", e.getMessage());
132134
}
133135
return ResponseEntityBuilder.ok(resultMap);
134136
}
@@ -252,7 +254,7 @@ private Map<String, Node> assemblePrunedPartitions(List<TScanRangeLocations> sca
252254
for (TScanRangeLocations scanRangeLocations : scanRangeLocationsList) {
253255
// only process palo(doris) scan range
254256
TPaloScanRange scanRange = scanRangeLocations.scan_range.palo_scan_range;
255-
Node tabletRouting = new Node(Long.parseLong(scanRange.version), Integer.parseInt(scanRange.schema_hash));
257+
Node tabletRouting = new Node(Long.parseLong(scanRange.version), 0 /* schema hash is not used */);
256258
for (TNetworkAddress address : scanRange.hosts) {
257259
tabletRouting.addRouting(address.hostname + ":" + address.port);
258260
}

‎fe/fe-core/src/test/java/org/apache/doris/analysis/AlterViewStmtTest.java‎

Lines changed: 0 additions & 182 deletions
This file was deleted.

‎fe/fe-core/src/test/java/org/apache/doris/analysis/ExplainTest.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,6 @@
2222
import org.apache.doris.utframe.UtFrameUtils;
2323

2424
import org.junit.Assert;
25-
import org.junit.Test;
26-
27-
import java.util.UUID;
2825

2926

3027
public class ExplainTest {
@@ -75,20 +72,20 @@ public void after() throws Exception {
7572
Assert.assertEquals(dropDbStmt.toSql(), dropSchemaStmt.toSql());
7673
}
7774

78-
public void testExplainInsertInto() throws Exception {
79-
String sql = "explain insert into test_explain.explain_t1 select * from test_explain.explain_t2";
80-
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
81-
System.out.println(explainString);
82-
Assert.assertTrue(explainString.contains("CAST"));
83-
}
84-
8575
public void testExplainSelect() throws Exception {
8676
String sql = "explain select * from test_explain.explain_t1 where dt = '1001';";
8777
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, false);
8878
System.out.println(explainString);
8979
Assert.assertFalse(explainString.contains("CAST"));
9080
}
9181

82+
public void testExplainInsertInto() throws Exception {
83+
String sql = "explain verbose insert into test_explain.explain_t1 select * from test_explain.explain_t2";
84+
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
85+
System.out.println(explainString);
86+
Assert.assertTrue(explainString.contains("CAST"));
87+
}
88+
9289
public void testExplainVerboseSelect() throws Exception {
9390
String queryStr = "explain verbose select * from test_explain.explain_t1 where dt = '1001';";
9491
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, queryStr, true);

‎fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.catalog;
1919

20+
import org.apache.doris.analysis.AlterViewStmt;
2021
import org.apache.doris.analysis.CreateDbStmt;
2122
import org.apache.doris.analysis.CreateTableStmt;
2223
import org.apache.doris.analysis.CreateViewStmt;
@@ -129,4 +130,22 @@ public void testNestedViews() throws Exception {
129130
System.out.println(explainString);
130131
Assert.assertTrue(explainString.contains("OlapScanNode"));
131132
}
133+
134+
@Test
135+
public void testAlterView() throws Exception {
136+
String originStmt = "select k1 as kc1, sum(k2) as kc2 from test.tbl1 group by kc1";
137+
ExceptionChecker.expectThrowsNoException(
138+
() -> createView("create view test.alter1 as " + originStmt));
139+
Database db = Catalog.getCurrentCatalog().getDbOrDdlException("default_cluster:test");
140+
View alter1 = (View) db.getTableOrDdlException("alter1");
141+
Assert.assertEquals("SELECT `k1` AS `kc1`, sum(`k2`) AS `kc2` FROM `default_cluster:test`.`tbl1` GROUP BY `kc1`", alter1.getInlineViewDef());
142+
143+
String alterStmt = "alter view test.alter1 as with test1_cte (w1, w2) as (select k1, k2 from test.tbl1) select w1 as c1, sum(w2) as c2 from test1_cte where w1 > 10 group by w1 order by w1";
144+
AlterViewStmt alterViewStmt = (AlterViewStmt) UtFrameUtils.parseAndAnalyzeStmt(alterStmt, connectContext);
145+
Catalog.getCurrentCatalog().alterView(alterViewStmt);
146+
147+
alter1 = (View) db.getTableOrDdlException("alter1");
148+
System.out.println(alter1.getInlineViewDef());
149+
Assert.assertEquals("WITH test1_cte(w1, w2) AS (SELECT `k1` AS `k1`, `k2` AS `k2` FROM `default_cluster:test`.`tbl1`) SELECT `w1` AS `c1`, sum(`w2`) AS `c2` FROM `test1_cte` WHERE `w1` > 10 GROUP BY `w1` ORDER BY `w1` ASC", alter1.getInlineViewDef());
150+
}
132151
}

‎fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public void setUp() {
4848
}
4949
@Test
5050
public void testQueryPlanAction() throws IOException, TException {
51-
RequestBody body = RequestBody.create(JSON, "{ \"sql\" : \" select k1,k2 from " + DB_NAME + "." + TABLE_NAME + " \" }");
51+
RequestBody body = RequestBody.create("{ \"sql\" : \" select k1,k2 from " + DB_NAME + "." + TABLE_NAME + " \" }", JSON);
5252
Request request = new Request.Builder()
5353
.post(body)
5454
.addHeader("Authorization", rootAuth)
@@ -67,8 +67,6 @@ public void testQueryPlanAction() throws IOException, TException {
6767
Assert.assertNotNull(tabletObject.get("routings"));
6868
Assert.assertEquals(3, ((JSONArray) tabletObject.get("routings")).size());
6969
Assert.assertEquals(testStartVersion, (long) tabletObject.get("version"));
70-
Assert.assertEquals(testSchemaHash, (long) tabletObject.get("schemaHash"));
71-
7270
}
7371
String queryPlan = (String) ((JSONObject) jsonObject.get("data")).get("opaqued_query_plan");
7472
Assert.assertNotNull(queryPlan);

‎fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1991,8 +1991,8 @@ public void testGroupingSetOutOfBoundError() throws Exception {
19911991
public void testExplainInsertInto() throws Exception {
19921992
ExplainTest explainTest = new ExplainTest();
19931993
explainTest.before(connectContext);
1994-
explainTest.testExplainInsertInto();
19951994
explainTest.testExplainSelect();
1995+
explainTest.testExplainInsertInto();
19961996
explainTest.testExplainVerboseSelect();
19971997
explainTest.testExplainConcatSelect();
19981998
explainTest.testExplainVerboseConcatSelect();
@@ -2097,7 +2097,8 @@ public void testResultExprs() throws Exception {
20972097
" (SELECT 4 AS bid)b ON (a.aid=b.bid)\n";
20982098
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
20992099
Assert.assertFalse(explainString.contains("OUTPUT EXPRS:3 | 4"));
2100-
Assert.assertTrue(explainString.contains("OUTPUT EXPRS:CAST(`a`.`aid` AS INT) | 4"));
2100+
System.out.println(explainString);
2101+
Assert.assertTrue(explainString.contains("OUTPUT EXPRS:`a`.`aid` | 4"));
21012102
}
21022103

21032104
@Test

‎regression-test/conf/regression-conf.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@
2020
// **Note**: default db will be create if not exist
2121
defaultDb = "regression_test"
2222

23-
jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?"
23+
jdbcUrl = "jdbc:mysql://127.0.0.1:9232/?"
2424
jdbcUser = "root"
2525
jdbcPassword = ""
2626

27-
feHttpAddress = "127.0.0.1:8030"
27+
feHttpAddress = "127.0.0.1:8232"
2828
feHttpUser = "root"
2929
feHttpPassword = ""
3030

0 commit comments

Comments
 (0)