Skip to content

Commit da8feed

Browse files
authored
[fix](binlog) Report streams with missing base tables as stale (#68087)
### What problem does this PR solve? Issue Number: None Related PR: #67173 Problem Summary: Dropping a base table leaves its table stream object in metadata, but the table stream metadata function previously reported the retained stream as enabled and non-stale even though its original base table ID could no longer be resolved. This change derives the displayed health from the same base-table lookup used for the identity columns, while preserving the stream configured state so recovering the original table restores its health. ### Release note Table streams whose base table is unavailable are now reported as disabled and stale with an explicit reason. ### Check List (For Author) - Test: Unit Test - DropTableStreamTest - TableStreamManagerCleanupTest - TableStreamBaseTableInfoTest - Behavior changed: Yes. Missing-base streams are reported as disabled and stale. - Does this need documentation: No
1 parent cdcf423 commit da8feed

2 files changed

Lines changed: 77 additions & 4 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/stream/TableStreamManager.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363

6464
public class TableStreamManager extends MasterDaemon implements Writable, GsonPostProcessable {
6565
private static final Logger LOG = LogManager.getLogger(TableStreamManager.class);
66+
private static final String BASE_TABLE_NOT_FOUND_STALE_REASON = "Base table does not exist";
6667
@SerializedName(value = "dbStreamMap")
6768
private Map<Long, Set<Long>> dbStreamMap;
6869
protected MonitoredReentrantReadWriteLock rwLock;
@@ -356,7 +357,8 @@ public void fillTableStreamValuesMetadataResult(List<TRow> dataBatch) {
356357
// STREAM_COMMENT
357358
trow.addToColumnValue(new TCell().setStringVal(stream.getComment()));
358359
TableIf baseTable = stream.getBaseTableNullable();
359-
if (baseTable == null) {
360+
boolean baseTableExists = baseTable != null;
361+
if (!baseTableExists) {
360362
// BASE_TABLE_NAME
361363
trow.addToColumnValue(new TCell().setStringVal("N/A"));
362364
// BASE_TABLE_DB
@@ -377,11 +379,13 @@ public void fillTableStreamValuesMetadataResult(List<TRow> dataBatch) {
377379
trow.addToColumnValue(new TCell().setStringVal(baseTable.getType().name()));
378380
}
379381
// ENABLED
380-
trow.addToColumnValue(new TCell().setBoolVal(!stream.isDisabled()));
382+
trow.addToColumnValue(
383+
new TCell().setBoolVal(baseTableExists && !stream.isDisabled()));
381384
// IS_STALE
382-
trow.addToColumnValue(new TCell().setBoolVal(stream.isStale()));
385+
trow.addToColumnValue(new TCell().setBoolVal(!baseTableExists || stream.isStale()));
383386
// STALE_REASON
384-
trow.addToColumnValue(new TCell().setStringVal(stream.getStaleReason()));
387+
trow.addToColumnValue(new TCell().setStringVal(baseTableExists
388+
? stream.getStaleReason() : BASE_TABLE_NOT_FOUND_STALE_REASON));
385389
dataBatch.add(trow);
386390
} finally {
387391
stream.readUnlock();

‎fe/fe-core/src/test/java/org/apache/doris/catalog/DropTableStreamTest.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.catalog;
1919

20+
import org.apache.doris.catalog.stream.OlapTableStream;
2021
import org.apache.doris.common.Config;
2122
import org.apache.doris.common.DdlException;
2223
import org.apache.doris.common.ExceptionChecker;
@@ -25,11 +26,15 @@
2526
import org.apache.doris.nereids.trees.plans.commands.DropStreamCommand;
2627
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
2728
import org.apache.doris.qe.StmtExecutor;
29+
import org.apache.doris.thrift.TRow;
2830
import org.apache.doris.utframe.TestWithFeService;
2931

3032
import org.junit.jupiter.api.Assertions;
3133
import org.junit.jupiter.api.Test;
3234

35+
import java.util.ArrayList;
36+
import java.util.List;
37+
3338
public class DropTableStreamTest extends TestWithFeService {
3439

3540
@Override
@@ -67,6 +72,33 @@ private void dropStream(String sql) throws Exception {
6772
}
6873
}
6974

75+
private void createBaseTableAndStream(String tableName, String streamName) throws Exception {
76+
createTable("create table test_stream." + tableName + " (k1 int, k2 int) "
77+
+ "unique key(k1) distributed by hash(k1) buckets 1 "
78+
+ "properties('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW', "
79+
+ "'binlog.need_historical_value' = 'true')");
80+
createTable("create stream test_stream." + streamName + " on table test_stream." + tableName
81+
+ " properties('show_initial_rows' = 'true')");
82+
}
83+
84+
private TRow getStreamMetadataRow(String streamName) {
85+
List<TRow> rows = new ArrayList<>();
86+
Env.getCurrentEnv().getTableStreamManager().fillTableStreamValuesMetadataResult(rows);
87+
return rows.stream()
88+
.filter(row -> streamName.equals(row.getColumnValue().get(1).getStringVal()))
89+
.findFirst()
90+
.orElseThrow(AssertionError::new);
91+
}
92+
93+
private void assertStreamMetadataState(String streamName, String baseTableType,
94+
boolean enabled, boolean stale, String staleReason) {
95+
TRow row = getStreamMetadataRow(streamName);
96+
Assertions.assertEquals(baseTableType, row.getColumnValue().get(9).getStringVal());
97+
Assertions.assertEquals(enabled, row.getColumnValue().get(10).isBoolVal());
98+
Assertions.assertEquals(stale, row.getColumnValue().get(11).isBoolVal());
99+
Assertions.assertEquals(staleReason, row.getColumnValue().get(12).getStringVal());
100+
}
101+
70102
@Test
71103
public void testNormalDropStream() throws Exception {
72104
// test drop
@@ -104,6 +136,43 @@ public void testCloudDropRequiresForce() {
104136
}
105137
}
106138

139+
@Test
140+
public void testStreamMetadataFollowsRecoverableBaseTable() throws Exception {
141+
createBaseTableAndStream("tbl_recover", "s_recover");
142+
Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test_stream");
143+
long baseTableId = db.getTableOrMetaException("tbl_recover").getId();
144+
145+
assertStreamMetadataState("s_recover", "OLAP", true, false, "N/A");
146+
147+
dropTableWithSql("drop table test_stream.tbl_recover");
148+
assertStreamMetadataState("s_recover", "N/A", false, true, "Base table does not exist");
149+
150+
recoverTable("recover table test_stream.tbl_recover");
151+
Assertions.assertEquals(baseTableId, db.getTableOrMetaException("tbl_recover").getId());
152+
assertStreamMetadataState("s_recover", "OLAP", true, false, "N/A");
153+
}
154+
155+
@Test
156+
public void testStreamMetadataRejectsSameNameBaseTable() throws Exception {
157+
createBaseTableAndStream("tbl_force", "s_force");
158+
Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test_stream");
159+
long oldBaseTableId = db.getTableOrMetaException("tbl_force").getId();
160+
OlapTableStream stream = (OlapTableStream) db.getTableOrMetaException("s_force");
161+
162+
dropTableWithSql("drop table test_stream.tbl_force force");
163+
Assertions.assertNull(stream.getBaseTableNullable());
164+
assertStreamMetadataState("s_force", "N/A", false, true, "Base table does not exist");
165+
166+
createTable("create table test_stream.tbl_force (k1 int, k2 int) "
167+
+ "unique key(k1) distributed by hash(k1) buckets 1 "
168+
+ "properties('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW', "
169+
+ "'binlog.need_historical_value' = 'true')");
170+
171+
Assertions.assertNotEquals(oldBaseTableId, db.getTableOrMetaException("tbl_force").getId());
172+
Assertions.assertNull(stream.getBaseTableNullable());
173+
assertStreamMetadataState("s_force", "N/A", false, true, "Base table does not exist");
174+
}
175+
107176
@Override
108177
protected void runAfterAll() throws Exception {
109178
dropDatabase("test_stream");

0 commit comments

Comments
 (0)