Skip to content

Commit 25d0c27

Browse files
committed
[fix](stream) Track stream base dependencies by identity
### What problem does this PR solve? Issue Number: close apache#65418 Related PR: apache#66287 Problem Summary: Stream base tables are implicit planning dependencies resolved by stable table ID. Keeping them in the qualifier-keyed relation cache can overwrite an explicitly resolved relation after concurrent renames, while resolving them again during preload or locking can select a different object snapshot. MTMV relation generation also omitted the underlying stream base after it was removed from the relation cache. Store exact implicit dependency objects separately, reuse them for preload and locking, include them in all-level MTMV dependencies, and reject dropped tables during cache-miss recovery resolution. ### Release note Streams continue to track and lock their original base tables by stable identity without corrupting explicit relation bindings. ### Check List (For Author) - Test: Unit Test - 43 FE tests passed across DropTableStreamTest, StatementContextTest, ExplainTableStreamPlanTest, and MTMVRelationTest - StatementContextTest rerun: 11 tests passed - Java compilation and Checkstyle passed - Behavior changed: Yes. Stream base dependencies use a stable per-statement object snapshot for planning locks and MTMV dependency tracking. - Does this need documentation: No
1 parent 9c8100c commit 25d0c27

8 files changed

Lines changed: 159 additions & 51 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/stream/BaseTableStream.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,20 @@ public BaseTableStream(String streamName, List<Column> fullSchema, TableIf baseT
117117

118118
public TableIf getBaseTableNullable() {
119119
TableIf cachedBaseTable = baseTable;
120-
if (cachedBaseTable instanceof Table && ((Table) cachedBaseTable).isDropped) {
121-
baseTable = null;
122-
return null;
120+
if (cachedBaseTable != null) {
121+
if (cachedBaseTable instanceof Table && ((Table) cachedBaseTable).isDropped) {
122+
baseTable = null;
123+
return null;
124+
}
125+
return cachedBaseTable;
123126
}
124-
if (cachedBaseTable == null) {
125-
cachedBaseTable = baseTableInfo.getTableNullable();
126-
baseTable = cachedBaseTable;
127+
TableIf resolvedBaseTable = baseTableInfo.getTableNullable();
128+
// Recovery publishes the table into database maps before clearing its dropped flag.
129+
if (resolvedBaseTable instanceof Table && ((Table) resolvedBaseTable).isDropped) {
130+
return null;
127131
}
128-
return cachedBaseTable;
132+
baseTable = resolvedBaseTable;
133+
return resolvedBaseTable;
129134
}
130135

131136
public void setProperties(Map<String, String> properties) throws org.apache.doris.common.AnalysisException {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,10 @@ public static Pair<Set<TableIf>, Set<TableIf>> getBaseTableFromQuery(String quer
224224
try {
225225
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
226226
planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN);
227-
return Pair.of(Sets.newHashSet(ctx.getStatementContext().getTables().values()),
227+
Set<TableIf> baseTables = Sets.newHashSet(ctx.getStatementContext().getTables().values());
228+
// Implicit dependencies are all-level tables, not relations written at the first query level.
229+
baseTables.addAll(ctx.getStatementContext().getImplicitTableDependencies());
230+
return Pair.of(baseTables,
228231
Sets.newHashSet(ctx.getStatementContext().getOneLevelTables().values()));
229232
} finally {
230233
ctx.setStatementContext(original);
@@ -448,6 +451,8 @@ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map<String,
448451
}
449452

450453
Set<TableIf> baseTables = Sets.newHashSet(statementContext.getTables().values());
454+
// Implicit dependencies are all-level tables, not relations written at the first query level.
455+
baseTables.addAll(statementContext.getImplicitTableDependencies());
451456
Set<TableIf> oneLevelTables = Sets.newHashSet(statementContext.getOneLevelTables().values());
452457
for (TableIf table : baseTables) {
453458
if (table.isTemporary()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 23 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.apache.doris.catalog.Partition;
2828
import org.apache.doris.catalog.TableIf;
2929
import org.apache.doris.catalog.View;
30-
import org.apache.doris.catalog.stream.BaseTableStream;
3130
import org.apache.doris.common.Id;
3231
import org.apache.doris.common.IdGenerator;
3332
import org.apache.doris.common.Pair;
@@ -220,6 +219,9 @@ public enum TableFrom {
220219

221220
// tables in this query directly
222221
private final Map<List<String>, TableIf> tables = Maps.newHashMap();
222+
// Underlying tables resolved while collecting explicit relations. They are not independently named by SQL, so
223+
// keep them out of the qualifier-keyed relation maps and reuse the same snapshot through planning.
224+
private final Set<TableIf> implicitTableDependencies = Sets.newIdentityHashSet();
223225
// onelevel tables in this query directly,
224226
// if
225227
// create v1 as select * from t1
@@ -443,6 +445,14 @@ public Map<List<String>, TableIf> getTables() {
443445
return tables;
444446
}
445447

448+
public void addImplicitTableDependency(TableIf table) {
449+
implicitTableDependencies.add(table);
450+
}
451+
452+
public Set<TableIf> getImplicitTableDependencies() {
453+
return implicitTableDependencies;
454+
}
455+
446456
public Map<List<String>, TableIf> getOneLevelTables() {
447457
return oneLevelTables;
448458
}
@@ -960,16 +970,20 @@ public Map<RelationId, Statistics> getRelationIdToStatisticsMap() {
960970
*/
961971
public synchronized void lock() {
962972
if (!needLockTables
963-
|| (tables.isEmpty() && mtmvRelatedTables.isEmpty() && insertTargetTables.isEmpty())
973+
|| (tables.isEmpty() && mtmvRelatedTables.isEmpty() && insertTargetTables.isEmpty()
974+
&& implicitTableDependencies.isEmpty())
964975
|| !plannerResources.isEmpty()) {
965976
return;
966977
}
978+
// The same object can be both an explicit relation and an implicit dependency; lock it only once.
979+
Set<TableIf> tablesToLock = Sets.newIdentityHashSet();
980+
tablesToLock.addAll(tables.values());
981+
tablesToLock.addAll(mtmvRelatedTables.values());
982+
tablesToLock.addAll(insertTargetTables.values());
983+
tablesToLock.addAll(implicitTableDependencies);
967984
PriorityQueue<TableIf> tableIfs = new PriorityQueue<>(
968-
tables.size() + mtmvRelatedTables.size() + insertTargetTables.size(),
969-
Comparator.comparing(TableIf::getId));
970-
addTablesToLock(tableIfs, tables.values());
971-
addTablesToLock(tableIfs, mtmvRelatedTables.values());
972-
addTablesToLock(tableIfs, insertTargetTables.values());
985+
tablesToLock.size(), Comparator.comparing(TableIf::getId));
986+
tableIfs.addAll(tablesToLock);
973987
while (!tableIfs.isEmpty()) {
974988
TableIf tableIf = tableIfs.poll();
975989
if (!tableIf.needReadLockWhenPlan()) {
@@ -1277,7 +1291,8 @@ public int getExternalTablePreloadCandidateCount() {
12771291
public boolean hasAnyPlanReadLockTable() {
12781292
return containsPlanReadLockTable(tables.values())
12791293
|| containsPlanReadLockTable(mtmvRelatedTables.values())
1280-
|| containsPlanReadLockTable(insertTargetTables.values());
1294+
|| containsPlanReadLockTable(insertTargetTables.values())
1295+
|| containsPlanReadLockTable(implicitTableDependencies);
12811296
}
12821297

12831298
public Optional<ExternalMetadataPreloadResult> getExternalMetadataPreloadResult() {
@@ -1293,31 +1308,10 @@ private boolean containsPlanReadLockTable(Collection<TableIf> tableIfs) {
12931308
if (tableIf.needReadLockWhenPlan()) {
12941309
return true;
12951310
}
1296-
if (tableIf instanceof BaseTableStream) {
1297-
// Mirror addTablesToLock(): a stream needs no plan lock itself, but its stable-ID base may need one.
1298-
TableIf baseTable = ((BaseTableStream) tableIf).getBaseTableNullable();
1299-
if (baseTable != null && baseTable.needReadLockWhenPlan()) {
1300-
return true;
1301-
}
1302-
}
13031311
}
13041312
return false;
13051313
}
13061314

1307-
/**
1308-
* Add explicit relations and stable-ID stream bases to the local ID-ordered lock queue. Stream bases must not be
1309-
* added to the qualifier relation maps because a concurrent rename can make a base qualifier collide with an
1310-
* explicitly resolved relation.
1311-
*/
1312-
private void addTablesToLock(PriorityQueue<TableIf> tableIfs, Collection<TableIf> tables) {
1313-
tableIfs.addAll(tables);
1314-
for (TableIf tableIf : tables) {
1315-
if (tableIf instanceof BaseTableStream) {
1316-
tableIfs.add(((BaseTableStream) tableIf).getBaseTableOrNereidsAnalysisException());
1317-
}
1318-
}
1319-
}
1320-
13211315
private static class CloseableResource implements Closeable {
13221316
public final String resourceName;
13231317
public final String threadName;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ private void collectFromUnboundRelation(CascadesContext cascadesContext,
236236
parseAndCollectFromView(tableQualifier, (View) table, cascadesContext);
237237
}
238238
if (table instanceof BaseTableStream) {
239-
collectFromTableStream((BaseTableStream) table);
239+
collectFromTableStream((BaseTableStream) table, cascadesContext.getStatementContext());
240240
}
241241
}
242242

@@ -312,15 +312,13 @@ protected void parseAndCollectFromView(List<String> tableQualifier, View view, C
312312
parentContext.addPlanProcesses(viewContext.getPlanProcesses());
313313
}
314314

315-
/**
316-
* Validate the stream base by stable ID without caching it under display qualifiers. A concurrent rename can
317-
* make that qualifier belong to an explicitly resolved relation; stream lock dependencies are expanded separately
318-
* in {@link StatementContext#lock()}.
319-
*/
320-
private void collectFromTableStream(BaseTableStream tableStream) {
321-
if (tableStream.getBaseTableNullable() == null) {
315+
private void collectFromTableStream(BaseTableStream tableStream, StatementContext statementContext) {
316+
// Capture the stable-ID result once so preload, locking, and dependency tracking use the same table object.
317+
TableIf baseTable = tableStream.getBaseTableNullable();
318+
if (baseTable == null) {
322319
throw new AnalysisException("Table ["
323320
+ tableStream.getBaseTableFullQualifiers().get(2) + "] does not exist");
324321
}
322+
statementContext.addImplicitTableDependency(baseTable);
325323
}
326324
}

‎fe/fe-core/src/test/java/org/apache/doris/catalog/DropTableStreamTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,15 @@
3333

3434
import org.junit.jupiter.api.Assertions;
3535
import org.junit.jupiter.api.Test;
36+
import org.mockito.Mockito;
3637

3738
import java.util.ArrayList;
3839
import java.util.List;
40+
import java.util.concurrent.CountDownLatch;
41+
import java.util.concurrent.ExecutorService;
42+
import java.util.concurrent.Executors;
43+
import java.util.concurrent.Future;
44+
import java.util.concurrent.TimeUnit;
3945

4046
public class DropTableStreamTest extends TestWithFeService {
4147

@@ -178,6 +184,42 @@ public void testStreamStateFollowsRecoverableBaseTableDrop() throws Exception {
178184
Assertions.assertEquals("N/A", stream.getStaleReason());
179185
}
180186

187+
@Test
188+
public void testRecoveringBaseTableRemainsUnavailableUntilUnmarkedDropped() throws Exception {
189+
createBaseTableAndStream("tbl_recovery_publish", "s_recovery_publish");
190+
Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("test_stream");
191+
OlapTable baseTable = (OlapTable) db.getTableOrMetaException("tbl_recovery_publish");
192+
OlapTableStream stream = (OlapTableStream) db.getTableOrMetaException("s_recovery_publish");
193+
194+
baseTable.markDropped();
195+
db.unregisterTable(baseTable.getId());
196+
Assertions.assertNull(stream.getBaseTableNullable());
197+
198+
OlapTable recoveringBaseTable = Mockito.spy(baseTable);
199+
CountDownLatch tablePublished = new CountDownLatch(1);
200+
CountDownLatch allowUnmarkDropped = new CountDownLatch(1);
201+
Mockito.doAnswer(invocation -> {
202+
tablePublished.countDown();
203+
Assertions.assertTrue(allowUnmarkDropped.await(10, TimeUnit.SECONDS));
204+
return invocation.callRealMethod();
205+
}).when(recoveringBaseTable).unmarkDropped();
206+
207+
ExecutorService executor = Executors.newSingleThreadExecutor();
208+
try {
209+
Future<Boolean> registerFuture = executor.submit(() -> db.registerTable(recoveringBaseTable));
210+
Assertions.assertTrue(tablePublished.await(10, TimeUnit.SECONDS));
211+
212+
Assertions.assertNull(stream.getBaseTableNullable());
213+
214+
allowUnmarkDropped.countDown();
215+
Assertions.assertTrue(registerFuture.get(10, TimeUnit.SECONDS));
216+
Assertions.assertSame(recoveringBaseTable, stream.getBaseTableNullable());
217+
} finally {
218+
allowUnmarkDropped.countDown();
219+
executor.shutdownNow();
220+
}
221+
}
222+
181223
@Test
182224
public void testBaseTableQualifiersFollowRenameAndRecovery() throws Exception {
183225
createBaseTableAndStream("tbl_rename", "s_rename");

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,25 @@
2020
import org.apache.doris.catalog.Database;
2121
import org.apache.doris.catalog.Env;
2222
import org.apache.doris.catalog.MTMV;
23+
import org.apache.doris.catalog.TableIf;
24+
import org.apache.doris.common.Config;
25+
import org.apache.doris.common.Pair;
2326
import org.apache.doris.utframe.TestWithFeService;
2427

2528
import com.google.common.collect.Sets;
2629
import org.junit.jupiter.api.Assertions;
2730
import org.junit.jupiter.api.Test;
2831

32+
import java.util.Set;
33+
2934
public class MTMVRelationTest extends TestWithFeService {
35+
36+
@Override
37+
protected void runBeforeAll() throws Exception {
38+
Config.enable_table_stream = true;
39+
Config.enable_feature_binlog = true;
40+
}
41+
3042
// t1 => v1 => v2
3143
// t2 => mv1
3244
// mv1 join v2 => mv2
@@ -114,4 +126,42 @@ public void testMTMVRelation() throws Exception {
114126
Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(v1));
115127
Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(v2));
116128
}
129+
130+
@Test
131+
public void testMTMVRelationIncludesStreamBaseDependency() throws Exception {
132+
createDatabaseAndUse("stream_mtmv_db");
133+
createTables(
134+
"CREATE TABLE stream_base (k1 int, k2 int)\n"
135+
+ "UNIQUE KEY(k1)\n"
136+
+ "DISTRIBUTED BY HASH(k1) BUCKETS 1\n"
137+
+ "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true',\n"
138+
+ "'binlog.format' = 'ROW', 'binlog.need_historical_value' = 'true')",
139+
"CREATE STREAM stream_source ON TABLE stream_base\n"
140+
+ "PROPERTIES ('show_initial_rows' = 'true')");
141+
createMvByNereids("CREATE MATERIALIZED VIEW stream_mv BUILD DEFERRED\n"
142+
+ "REFRESH COMPLETE ON MANUAL\n"
143+
+ "DISTRIBUTED BY RANDOM BUCKETS 1\n"
144+
+ "PROPERTIES ('replication_num' = '1')\n"
145+
+ "AS SELECT k1, k2 FROM stream_source");
146+
147+
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("stream_mtmv_db");
148+
MTMV mtmv = (MTMV) db.getTableOrAnalysisException("stream_mv");
149+
TableIf stream = db.getTableOrAnalysisException("stream_source");
150+
TableIf baseTable = db.getTableOrAnalysisException("stream_base");
151+
BaseTableInfo streamInfo = new BaseTableInfo(stream);
152+
BaseTableInfo baseTableInfo = new BaseTableInfo(baseTable);
153+
BaseTableInfo mtmvInfo = new BaseTableInfo(mtmv);
154+
155+
Assertions.assertEquals(Sets.newHashSet(streamInfo, baseTableInfo), mtmv.getRelation().getBaseTables());
156+
Assertions.assertEquals(Sets.newHashSet(streamInfo), mtmv.getRelation().getBaseTablesOneLevel());
157+
Assertions.assertEquals(Sets.newHashSet(streamInfo, baseTableInfo),
158+
mtmv.getRelation().getBaseTablesOneLevelAndFromView());
159+
Assertions.assertEquals(Sets.newHashSet(mtmvInfo), Env.getCurrentEnv().getMtmvService().getRelationManager()
160+
.getMtmvsByBaseTable(baseTableInfo));
161+
162+
Pair<Set<TableIf>, Set<TableIf>> queryTables = MTMVPlanUtil.getBaseTableFromQuery(
163+
mtmv.getQuerySql(), connectContext);
164+
Assertions.assertEquals(Sets.newHashSet(stream, baseTable), queryTables.first);
165+
Assertions.assertEquals(Sets.newHashSet(stream), queryTables.second);
166+
}
117167
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,6 @@ public void testLockIncludesStreamBaseTableWithoutReplacingRelationCache() {
184184
Mockito.when(explicitTable.tryReadLock(Mockito.anyLong(), Mockito.any())).thenReturn(true);
185185
Mockito.when(stream.getId()).thenReturn(12L);
186186
Mockito.when(stream.needReadLockWhenPlan()).thenReturn(false);
187-
Mockito.when(stream.getBaseTableOrNereidsAnalysisException()).thenReturn(baseTable);
188187
Mockito.when(baseTable.getId()).thenReturn(13L);
189188
Mockito.when(baseTable.getName()).thenReturn("base");
190189
Mockito.when(baseTable.getNameWithFullQualifiers()).thenReturn("internal.db.base");
@@ -196,12 +195,17 @@ public void testLockIncludesStreamBaseTableWithoutReplacingRelationCache() {
196195
try {
197196
statementContext.getTables().put(ImmutableList.of("internal", "db", "explicit"), explicitTable);
198197
statementContext.getTables().put(ImmutableList.of("internal", "db", "stream"), stream);
198+
statementContext.getTables().put(ImmutableList.of("internal", "db", "base"), baseTable);
199+
statementContext.addImplicitTableDependency(baseTable);
200+
statementContext.addImplicitTableDependency(baseTable);
199201

200202
statementContext.lock();
201203

202-
Mockito.verify(explicitTable).tryReadLock(Mockito.anyLong(), Mockito.any());
203-
Mockito.verify(baseTable).tryReadLock(Mockito.anyLong(), Mockito.any());
204+
InOrder lockOrder = Mockito.inOrder(explicitTable, baseTable);
205+
lockOrder.verify(explicitTable, Mockito.times(1)).tryReadLock(Mockito.anyLong(), Mockito.any());
206+
lockOrder.verify(baseTable, Mockito.times(1)).tryReadLock(Mockito.anyLong(), Mockito.any());
204207
Mockito.verify(stream, Mockito.never()).tryReadLock(Mockito.anyLong(), Mockito.any());
208+
Mockito.verify(stream, Mockito.never()).getBaseTableNullable();
205209
org.junit.jupiter.api.Assertions.assertSame(
206210
explicitTable, statementContext.getTables().get(ImmutableList.of("internal", "db", "explicit")));
207211
} finally {
@@ -210,19 +214,24 @@ public void testLockIncludesStreamBaseTableWithoutReplacingRelationCache() {
210214
}
211215

212216
@Test
213-
public void testPreloadRecognizesStreamBaseTablePlanReadLock() {
217+
public void testPreloadAndLockUseSameStreamBaseSnapshot() {
214218
ConnectContext connectContext = Mockito.mock(ConnectContext.class);
215219
BaseTableStream stream = Mockito.mock(BaseTableStream.class);
216220
TableIf baseTable = Mockito.mock(TableIf.class);
221+
TableIf recoveredBaseTable = Mockito.mock(TableIf.class);
217222
PluginDrivenExternalTable externalTable = Mockito.mock(PluginDrivenExternalTable.class);
218223
SessionVariable sessionVariable = new SessionVariable();
219224
sessionVariable.setEnablePreloadExternalMetadata(true);
220225

221226
Mockito.when(connectContext.getSessionVariable()).thenReturn(sessionVariable);
222227
Mockito.when(connectContext.getQueryIdentifier()).thenReturn("stream-preload");
223228
Mockito.when(stream.needReadLockWhenPlan()).thenReturn(false);
224-
Mockito.when(stream.getBaseTableNullable()).thenReturn(baseTable);
229+
Mockito.when(stream.getBaseTableNullable()).thenReturn(recoveredBaseTable);
230+
Mockito.when(baseTable.getId()).thenReturn(20L);
231+
Mockito.when(baseTable.getName()).thenReturn("base");
232+
Mockito.when(baseTable.getNameWithFullQualifiers()).thenReturn("internal.db.base");
225233
Mockito.when(baseTable.needReadLockWhenPlan()).thenReturn(true);
234+
Mockito.when(baseTable.tryReadLock(Mockito.anyLong(), Mockito.any())).thenReturn(true);
226235
Mockito.when(externalTable.getId()).thenReturn(21L);
227236
Mockito.when(externalTable.supportsExternalMetadataPreload()).thenReturn(true);
228237
Mockito.when(externalTable.getBaseSchema()).thenReturn(Collections.emptyList());
@@ -232,12 +241,17 @@ public void testPreloadRecognizesStreamBaseTablePlanReadLock() {
232241
new OriginStatement("select * from db.stream join ext", 0));
233242
try {
234243
statementContext.getTables().put(ImmutableList.of("internal", "db", "stream"), stream);
244+
statementContext.addImplicitTableDependency(baseTable);
235245
statementContext.registerExternalTableForPreload(externalTable, Optional.empty(), Optional.empty());
236246

237247
ExternalMetadataPreloadResult result = executePreload(statementContext);
248+
statementContext.lock();
238249

239250
org.junit.jupiter.api.Assertions.assertTrue(result.isExecuted());
240251
Mockito.verify(externalTable).getBaseSchema();
252+
Mockito.verify(stream, Mockito.never()).getBaseTableNullable();
253+
Mockito.verify(baseTable, Mockito.times(1)).tryReadLock(Mockito.anyLong(), Mockito.any());
254+
Mockito.verifyNoInteractions(recoveredBaseTable);
241255
} finally {
242256
statementContext.close();
243257
}

0 commit comments

Comments
 (0)