Skip to content

Commit a7850ad

Browse files
committed
[fix](stream) Keep stream lock dependencies out of relation cache
### What problem does this PR solve? Issue Number: close apache#65418 Related PR: apache#66287 Problem Summary: The previous stable-ID locking fix cached a stream base table under its current name qualifier. A pre-lock rename could therefore overwrite an explicit relation already resolved under the same qualifier and make binding scan the wrong table. Keep the relation cache limited to SQL relations and expand each stream base table by stable ID only when constructing the existing ordered lock queue. ### Release note None ### Check List (For Author) - Test: FE unit test and FE build - ./run-fe-ut.sh --run org.apache.doris.nereids.StatementContextTest,org.apache.doris.nereids.trees.plans.ExplainTableStreamPlanTest - ./run-fe-ut.sh --run org.apache.doris.catalog.DropTableStreamTest - env DISABLE_BUILD_UI=ON ./build.sh --fe - Behavior changed: Yes. Explicit relation bindings are preserved while stream base tables are locked by stable ID. - Does this need documentation: No
1 parent 7ea0064 commit a7850ad

4 files changed

Lines changed: 164 additions & 26 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.doris.catalog.Partition;
2828
import org.apache.doris.catalog.TableIf;
2929
import org.apache.doris.catalog.View;
30+
import org.apache.doris.catalog.stream.BaseTableStream;
3031
import org.apache.doris.common.Id;
3132
import org.apache.doris.common.IdGenerator;
3233
import org.apache.doris.common.Pair;
@@ -966,9 +967,9 @@ public synchronized void lock() {
966967
PriorityQueue<TableIf> tableIfs = new PriorityQueue<>(
967968
tables.size() + mtmvRelatedTables.size() + insertTargetTables.size(),
968969
Comparator.comparing(TableIf::getId));
969-
tableIfs.addAll(tables.values());
970-
tableIfs.addAll(mtmvRelatedTables.values());
971-
tableIfs.addAll(insertTargetTables.values());
970+
addTablesToLock(tableIfs, tables.values());
971+
addTablesToLock(tableIfs, mtmvRelatedTables.values());
972+
addTablesToLock(tableIfs, insertTargetTables.values());
972973
while (!tableIfs.isEmpty()) {
973974
TableIf tableIf = tableIfs.poll();
974975
if (!tableIf.needReadLockWhenPlan()) {
@@ -1292,10 +1293,25 @@ private boolean containsPlanReadLockTable(Collection<TableIf> tableIfs) {
12921293
if (tableIf.needReadLockWhenPlan()) {
12931294
return true;
12941295
}
1296+
if (tableIf instanceof BaseTableStream) {
1297+
TableIf baseTable = ((BaseTableStream) tableIf).getBaseTableNullable();
1298+
if (baseTable != null && baseTable.needReadLockWhenPlan()) {
1299+
return true;
1300+
}
1301+
}
12951302
}
12961303
return false;
12971304
}
12981305

1306+
private void addTablesToLock(PriorityQueue<TableIf> tableIfs, Collection<TableIf> tables) {
1307+
tableIfs.addAll(tables);
1308+
for (TableIf tableIf : tables) {
1309+
if (tableIf instanceof BaseTableStream) {
1310+
tableIfs.add(((BaseTableStream) tableIf).getBaseTableOrNereidsAnalysisException());
1311+
}
1312+
}
1313+
}
1314+
12991315
private static class CloseableResource implements Closeable {
13001316
public final String resourceName;
13011317
public final String threadName;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ private void collectFromUnboundRelation(CascadesContext cascadesContext,
237237
}
238238
// we need to collect stream table's base table as well
239239
if (table instanceof BaseTableStream) {
240-
collectFromTableStream((BaseTableStream) table, cascadesContext, tableFrom);
240+
collectFromTableStream((BaseTableStream) table);
241241
}
242242
}
243243

@@ -313,30 +313,10 @@ protected void parseAndCollectFromView(List<String> tableQualifier, View view, C
313313
parentContext.addPlanProcesses(viewContext.getPlanProcesses());
314314
}
315315

316-
private void collectFromTableStream(BaseTableStream tableStream, CascadesContext cascadesContext,
317-
TableFrom tableFrom) {
318-
StatementContext statementContext = cascadesContext.getConnectContext().getStatementContext();
319-
TableIf baseTable = tableStream.getBaseTableNullable();
320-
if (baseTable == null) {
316+
private void collectFromTableStream(BaseTableStream tableStream) {
317+
if (tableStream.getBaseTableNullable() == null) {
321318
throw new AnalysisException("Table ["
322319
+ tableStream.getBaseTableFullQualifiers().get(2) + "] does not exist");
323320
}
324-
325-
// Cache the ID-resolved base table so planner locks the object used during binding.
326-
Map<List<String>, TableIf> tables;
327-
switch (tableFrom) {
328-
case QUERY:
329-
tables = statementContext.getTables();
330-
break;
331-
case INSERT_TARGET:
332-
tables = statementContext.getInsertTargetTables();
333-
break;
334-
case MTMV:
335-
tables = statementContext.getMtmvRelatedTables();
336-
break;
337-
default:
338-
throw new AnalysisException("Unknown table from " + tableFrom);
339-
}
340-
tables.put(baseTable.getFullQualifiers(), baseTable);
341321
}
342322
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.analysis.TableSnapshot;
2222
import org.apache.doris.catalog.DatabaseIf;
2323
import org.apache.doris.catalog.TableIf;
24+
import org.apache.doris.catalog.stream.BaseTableStream;
2425
import org.apache.doris.datasource.CatalogIf;
2526
import org.apache.doris.datasource.mvcc.MvccSnapshot;
2627
import org.apache.doris.datasource.mvcc.PluginDrivenMvccExternalTable;
@@ -170,6 +171,78 @@ public void testSkipPreloadWhenNoInternalTableNeedsPlanReadLock() {
170171
}
171172
}
172173

174+
@Test
175+
public void testLockIncludesStreamBaseTableWithoutReplacingRelationCache() {
176+
ConnectContext connectContext = Mockito.mock(ConnectContext.class);
177+
TableIf explicitTable = Mockito.mock(TableIf.class);
178+
BaseTableStream stream = Mockito.mock(BaseTableStream.class);
179+
TableIf baseTable = Mockito.mock(TableIf.class);
180+
Mockito.when(explicitTable.getId()).thenReturn(11L);
181+
Mockito.when(explicitTable.getName()).thenReturn("explicit");
182+
Mockito.when(explicitTable.getNameWithFullQualifiers()).thenReturn("internal.db.explicit");
183+
Mockito.when(explicitTable.needReadLockWhenPlan()).thenReturn(true);
184+
Mockito.when(explicitTable.tryReadLock(Mockito.anyLong(), Mockito.any())).thenReturn(true);
185+
Mockito.when(stream.getId()).thenReturn(12L);
186+
Mockito.when(stream.needReadLockWhenPlan()).thenReturn(false);
187+
Mockito.when(stream.getBaseTableOrNereidsAnalysisException()).thenReturn(baseTable);
188+
Mockito.when(baseTable.getId()).thenReturn(13L);
189+
Mockito.when(baseTable.getName()).thenReturn("base");
190+
Mockito.when(baseTable.getNameWithFullQualifiers()).thenReturn("internal.db.base");
191+
Mockito.when(baseTable.needReadLockWhenPlan()).thenReturn(true);
192+
Mockito.when(baseTable.tryReadLock(Mockito.anyLong(), Mockito.any())).thenReturn(true);
193+
194+
StatementContext statementContext = new StatementContext(connectContext,
195+
new OriginStatement("select * from db.explicit join db.stream", 0));
196+
try {
197+
statementContext.getTables().put(ImmutableList.of("internal", "db", "explicit"), explicitTable);
198+
statementContext.getTables().put(ImmutableList.of("internal", "db", "stream"), stream);
199+
200+
statementContext.lock();
201+
202+
Mockito.verify(explicitTable).tryReadLock(Mockito.anyLong(), Mockito.any());
203+
Mockito.verify(baseTable).tryReadLock(Mockito.anyLong(), Mockito.any());
204+
Mockito.verify(stream, Mockito.never()).tryReadLock(Mockito.anyLong(), Mockito.any());
205+
org.junit.jupiter.api.Assertions.assertSame(
206+
explicitTable, statementContext.getTables().get(ImmutableList.of("internal", "db", "explicit")));
207+
} finally {
208+
statementContext.close();
209+
}
210+
}
211+
212+
@Test
213+
public void testPreloadRecognizesStreamBaseTablePlanReadLock() {
214+
ConnectContext connectContext = Mockito.mock(ConnectContext.class);
215+
BaseTableStream stream = Mockito.mock(BaseTableStream.class);
216+
TableIf baseTable = Mockito.mock(TableIf.class);
217+
PluginDrivenExternalTable externalTable = Mockito.mock(PluginDrivenExternalTable.class);
218+
SessionVariable sessionVariable = new SessionVariable();
219+
sessionVariable.setEnablePreloadExternalMetadata(true);
220+
221+
Mockito.when(connectContext.getSessionVariable()).thenReturn(sessionVariable);
222+
Mockito.when(connectContext.getQueryIdentifier()).thenReturn("stream-preload");
223+
Mockito.when(stream.needReadLockWhenPlan()).thenReturn(false);
224+
Mockito.when(stream.getBaseTableNullable()).thenReturn(baseTable);
225+
Mockito.when(baseTable.needReadLockWhenPlan()).thenReturn(true);
226+
Mockito.when(externalTable.getId()).thenReturn(21L);
227+
Mockito.when(externalTable.supportsExternalMetadataPreload()).thenReturn(true);
228+
Mockito.when(externalTable.getBaseSchema()).thenReturn(Collections.emptyList());
229+
Mockito.when(externalTable.supportInternalPartitionPruned()).thenReturn(false);
230+
231+
StatementContext statementContext = new StatementContext(connectContext,
232+
new OriginStatement("select * from db.stream join ext", 0));
233+
try {
234+
statementContext.getTables().put(ImmutableList.of("internal", "db", "stream"), stream);
235+
statementContext.registerExternalTableForPreload(externalTable, Optional.empty(), Optional.empty());
236+
237+
ExternalMetadataPreloadResult result = executePreload(statementContext);
238+
239+
org.junit.jupiter.api.Assertions.assertTrue(result.isExecuted());
240+
Mockito.verify(externalTable).getBaseSchema();
241+
} finally {
242+
statementContext.close();
243+
}
244+
}
245+
173246
@Test
174247
public void testPreloadIcebergLatestSnapshotBeforeLock() {
175248
ConnectContext connectContext = Mockito.mock(ConnectContext.class);

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,16 @@
6767

6868
import org.junit.jupiter.api.Assertions;
6969
import org.junit.jupiter.api.Test;
70+
import org.mockito.Mockito;
7071

7172
import java.util.ArrayList;
7273
import java.util.List;
7374
import java.util.Map;
75+
import java.util.concurrent.CountDownLatch;
76+
import java.util.concurrent.ExecutorService;
77+
import java.util.concurrent.Executors;
78+
import java.util.concurrent.Future;
79+
import java.util.concurrent.TimeUnit;
7480

7581
/**
7682
* UTs for table stream query plan, including
@@ -652,6 +658,69 @@ public void testRecordPlanForMvPreRewriteNormalizesStreamScanInsideCte() throws
652658
}
653659
}
654660

661+
@Test
662+
public void testPreLockRenameDoesNotReplaceCachedExplicitTable() throws Exception {
663+
String baseTableName = "stream_cache_base";
664+
String streamName = "stream_cache_stream";
665+
String explicitTableName = "stream_cache_explicit";
666+
createTable("create table test_stream." + baseTableName + " (k1 int, k2 int) "
667+
+ "unique key(k1) distributed by hash(k1) buckets 1 "
668+
+ "properties('replication_num' = '1', 'binlog.enable' = 'true', "
669+
+ "'binlog.format' = 'ROW', 'binlog.need_historical_value' = 'true')");
670+
createTable("create stream test_stream." + streamName + " on table test_stream." + baseTableName
671+
+ " properties('show_initial_rows' = 'true')");
672+
createTable("create table test_stream." + explicitTableName + " (k1 int, k2 int) "
673+
+ "unique key(k1) distributed by hash(k1) buckets 1 "
674+
+ "properties('replication_num' = '1')");
675+
676+
Database db = (Database) Env.getCurrentInternalCatalog().getDbOrMetaException("test_stream");
677+
OlapTable baseTable = (OlapTable) db.getTableOrMetaException(baseTableName);
678+
OlapTable explicitTable = (OlapTable) db.getTableOrMetaException(explicitTableName);
679+
OlapTableStream stream = (OlapTableStream) db.getTableOrMetaException(streamName);
680+
OlapTableStream blockingStream = Mockito.spy(stream);
681+
CountDownLatch streamCollectionStarted = new CountDownLatch(1);
682+
CountDownLatch renamesFinished = new CountDownLatch(1);
683+
Mockito.doAnswer(invocation -> {
684+
streamCollectionStarted.countDown();
685+
Assertions.assertTrue(renamesFinished.await(10, TimeUnit.SECONDS));
686+
return invocation.callRealMethod();
687+
}).when(blockingStream).getBaseTableNullable();
688+
689+
ConnectContext ctx = createDefaultCtx();
690+
ctx.setDatabase("test_stream");
691+
String sql = "select b.k1, s.k1 from test_stream." + explicitTableName + " b join test_stream."
692+
+ streamName + " s on b.k1 = s.k1";
693+
StatementContext statementContext = MemoTestUtils.createStatementContext(ctx, sql);
694+
statementContext.getTables().put(List.of("internal", "test_stream", streamName), blockingStream);
695+
ExecutorService plannerExecutor = Executors.newSingleThreadExecutor();
696+
try {
697+
Future<?> planFuture = plannerExecutor.submit(() -> {
698+
ctx.setThreadLocalInfo();
699+
new NereidsPlanner(statementContext).planWithLock(
700+
(LogicalPlan) parser.parseSingle(sql), PhysicalProperties.ANY);
701+
});
702+
703+
Assertions.assertTrue(streamCollectionStarted.await(10, TimeUnit.SECONDS));
704+
Assertions.assertSame(explicitTable,
705+
statementContext.getTables().get(List.of("internal", "test_stream", explicitTableName)));
706+
707+
connectContext.setThreadLocalInfo();
708+
executeSql("alter table test_stream." + explicitTableName
709+
+ " rename " + explicitTableName + "_renamed");
710+
executeSql("alter table test_stream." + baseTableName + " rename " + explicitTableName);
711+
renamesFinished.countDown();
712+
planFuture.get(30, TimeUnit.SECONDS);
713+
714+
Assertions.assertSame(explicitTable,
715+
statementContext.getTables().get(List.of("internal", "test_stream", explicitTableName)));
716+
Assertions.assertSame(baseTable, blockingStream.getBaseTableNullable());
717+
} finally {
718+
renamesFinished.countDown();
719+
plannerExecutor.shutdownNow();
720+
statementContext.close();
721+
}
722+
}
723+
655724
@Test
656725
public void testMowTimeTravelQualifiedColumnCanBind() {
657726
// MOW time-travel goes through a union whose outputs are rebuilt with empty qualifiers.

0 commit comments

Comments
 (0)