Skip to content

Commit 6c68e50

Browse files
morningmanclaude
andcommitted
[P4-T06d] DDL 远端名解析 — CREATE/DROP TABLE 经 getRemoteName 再发 connector [FIX-DDL-REMOTE]
root cause: 翻闸后 PluginDrivenExternalCatalog.createTable/dropTable 把本地名 原样透传连接器→ODPS SDK,丢失 legacy 的 local→remote 名映射;名映射开启 (lower_case_meta_names / meta_names_mapping)时建错库 / 删错或找不到表(数据正确性回归)。 solution (fe-core only, 2 override): - createTable: getDbNullable 解析 db → db.getRemoteName() 喂 converter 第二参; 表名不解析(legacy parity, mirror MaxComputeMetadataOps.createTableImpl)。 - dropTable: 精确 mirror base ExternalCatalog.dropTable —— db==null 无条件抛、 table==null / handle-absent 才 ifExists;getTableHandle 用 dorisTable 的 remote db/table 名。editlog/cache 仍用本地名(follower-replay 一致)。 tests: PluginDrivenExternalCatalogDdlRoutingTest 17/17;mutation 自证(remote 解析 + db-null 无条件抛 + editlog/getDbForReplay 本地名 各有测试变红);Checkstyle=0。 2 轮对抗 review 收敛(round1 needs-revision 3 test-quality → round2 converged)。 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent fc24781 commit 6c68e50

5 files changed

Lines changed: 380 additions & 24 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/PluginDrivenExternalCatalog.java‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -263,9 +263,20 @@ public Connector getConnector() {
263263
@Override
264264
public boolean createTable(CreateTableInfo createTableInfo) throws UserException {
265265
makeSureInitialized();
266+
// Resolve the local db name to its remote (ODPS) name before handing it to the connector,
267+
// mirroring legacy MaxComputeMetadataOps.createTableImpl (db.getRemoteName()). Without this,
268+
// name-mapped catalogs (lower_case_meta_names / meta_names_mapping, where the local display
269+
// name differs from the remote name) would address the wrong remote schema. The table name
270+
// is intentionally NOT remote-resolved (legacy parity: the table does not exist yet, so
271+
// there is no local->remote mapping for it).
272+
ExternalDatabase<? extends ExternalTable> db = getDbNullable(createTableInfo.getDbName());
273+
if (db == null) {
274+
throw new DdlException("Failed to get database: '" + createTableInfo.getDbName()
275+
+ "' in catalog: " + getName());
276+
}
266277
ConnectorSession session = buildConnectorSession();
267278
ConnectorCreateTableRequest request = CreateTableInfoToConnectorRequestConverter
268-
.convert(createTableInfo, createTableInfo.getDbName());
279+
.convert(createTableInfo, db.getRemoteName());
269280
try {
270281
connector.getMetadata(session).createTable(session, request);
271282
} catch (DorisConnectorException e) {
@@ -280,7 +291,9 @@ public boolean createTable(CreateTableInfo createTableInfo) throws UserException
280291
// Invalidate the FE-side table-name cache so the new table is immediately visible on
281292
// this FE. The legacy metadataOps path did this via afterCreateTable(); since
282293
// PluginDrivenExternalCatalog has no metadataOps, the override must do it here.
283-
getDbForReplay(createTableInfo.getDbName()).ifPresent(db -> db.resetMetaCacheNames());
294+
// (Edit log and cache invalidation deliberately use the LOCAL db/table names for
295+
// follower-replay consistency; only the connector-bound name is remote-resolved.)
296+
getDbForReplay(createTableInfo.getDbName()).ifPresent(d -> d.resetMetaCacheNames());
284297
LOG.info("finished to create table {}.{}.{}", getName(),
285298
createTableInfo.getDbName(), createTableInfo.getTableName());
286299
return false;
@@ -354,9 +367,29 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc
354367
public void dropTable(String dbName, String tableName, boolean isView, boolean isMtmv, boolean isStream,
355368
boolean ifExists, boolean mustTemporary, boolean force) throws DdlException {
356369
makeSureInitialized();
370+
// Resolve the local db/table names to their remote (ODPS) names before handing them to the
371+
// connector, mirroring base ExternalCatalog.dropTable -- the exact path legacy
372+
// MaxComputeMetadataOps.dropTableImpl ran through, which used dorisTable.getRemoteDbName() /
373+
// getRemoteName(). Without this, name-mapped catalogs would locate the wrong remote table
374+
// (IF EXISTS silently no-ops / non-IF-EXISTS wrongly reports "not found"). Matching base:
375+
// a missing db ALWAYS throws (even with IF EXISTS); a missing table honors IF EXISTS.
376+
ExternalDatabase<? extends ExternalTable> db = getDbNullable(dbName);
377+
if (db == null) {
378+
throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + getName());
379+
}
380+
ExternalTable dorisTable = db.getTableNullable(tableName);
381+
if (dorisTable == null) {
382+
if (ifExists) {
383+
return;
384+
}
385+
throw new DdlException("Failed to get table: '" + tableName + "' in database: " + dbName);
386+
}
357387
ConnectorSession session = buildConnectorSession();
358388
ConnectorMetadata metadata = connector.getMetadata(session);
359-
Optional<ConnectorTableHandle> handle = metadata.getTableHandle(session, dbName, tableName);
389+
Optional<ConnectorTableHandle> handle = metadata.getTableHandle(
390+
session, dorisTable.getRemoteDbName(), dorisTable.getRemoteName());
391+
// The table is present in the FE cache but may have been dropped out-of-band on the remote
392+
// side; preserve the existing IF EXISTS handling for that case.
360393
if (!handle.isPresent()) {
361394
if (ifExists) {
362395
return;
@@ -368,8 +401,10 @@ public void dropTable(String dbName, String tableName, boolean isView, boolean i
368401
} catch (DorisConnectorException e) {
369402
throw new DdlException(e.getMessage(), e);
370403
}
404+
// Edit log and cache invalidation deliberately use the LOCAL db/table names for
405+
// follower-replay consistency; only the connector-bound names are remote-resolved.
371406
Env.getCurrentEnv().getEditLog().logDropTable(new DropInfo(getName(), dbName, tableName));
372-
getDbForReplay(dbName).ifPresent(db -> db.unregisterTable(tableName));
407+
getDbForReplay(dbName).ifPresent(d -> d.unregisterTable(tableName));
373408
LOG.info("finished to drop table {}.{}.{}", getName(), dbName, tableName);
374409
}
375410

‎fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenExternalCatalogDdlRoutingTest.java‎

Lines changed: 169 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.junit.jupiter.api.Assertions;
3535
import org.junit.jupiter.api.BeforeEach;
3636
import org.junit.jupiter.api.Test;
37+
import org.mockito.ArgumentCaptor;
3738
import org.mockito.MockedStatic;
3839
import org.mockito.Mockito;
3940

@@ -171,44 +172,132 @@ public void testDropDbWrapsConnectorException() {
171172
}
172173

173174
// ==================== DROP TABLE ====================
175+
// FIX-DDL-REMOTE: dropTable now resolves the local db/table names to their REMOTE (ODPS)
176+
// names (via getDbNullable + db.getTableNullable + getRemoteDbName/getRemoteName) before
177+
// calling the connector, mirroring base ExternalCatalog.dropTable / legacy
178+
// MaxComputeMetadataOps.dropTableImpl. Every drop test therefore stubs dbNullableResult and
179+
// db.getTableNullable; edit log / cache invalidation still use the LOCAL names.
174180

175181
@Test
176-
public void testDropTableResolvesHandleRoutesAndUnregisters() throws Exception {
182+
public void testDropTableResolvesRemoteNamesRoutesAndUnregisters() throws Exception {
183+
// local db1.t1 maps to remote DB1.TBL1 (name mapping enabled).
184+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase(); // resolution db (getDbNullable)
185+
ExternalTable table = Mockito.mock(ExternalTable.class);
186+
Mockito.when(table.getRemoteDbName()).thenReturn("DB1");
187+
Mockito.when(table.getRemoteName()).thenReturn("TBL1");
188+
Mockito.doReturn(table).when(db).getTableNullable("t1");
189+
catalog.dbNullableResult = db;
190+
// Distinct replay db: locks that cache invalidation uses the getDbForReplay lookup, NOT
191+
// the resolution db (a refactor routing unregister through the resolution db must go red).
192+
ExternalDatabase<? extends ExternalTable> replayDb = mockExternalDatabase();
193+
catalog.dbForReplayResult = Optional.of(replayDb);
194+
177195
ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class);
178-
Mockito.when(metadata.getTableHandle(session, "db1", "t1")).thenReturn(Optional.of(handle));
179-
ExternalDatabase<? extends ExternalTable> mockDb = mockExternalDatabase();
180-
catalog.dbForReplayResult = Optional.of(mockDb);
196+
Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.of(handle));
181197

182198
catalog.dropTable("db1", "t1", false, false, false, false, false, false);
183199

200+
// WHY: the connector must receive the REMOTE names so name-mapped catalogs hit the real
201+
// ODPS object; a mutation that passes the local "db1"/"t1" makes this verify red.
202+
Mockito.verify(metadata).getTableHandle(session, "DB1", "TBL1");
184203
Mockito.verify(metadata).dropTable(session, handle);
185-
Mockito.verify(mockEditLog).logDropTable(Mockito.any());
186-
Mockito.verify(mockDb).unregisterTable("t1");
204+
// WHY: edit log + cache invalidation MUST use the LOCAL names -- followers replay the
205+
// persisted DropInfo and the on-FE cache is keyed by local name. A mutation building
206+
// DropInfo / looking up getDbForReplay with the remote names must turn these red.
207+
ArgumentCaptor<org.apache.doris.persist.DropInfo> dropInfo =
208+
ArgumentCaptor.forClass(org.apache.doris.persist.DropInfo.class);
209+
Mockito.verify(mockEditLog).logDropTable(dropInfo.capture());
210+
Assertions.assertEquals("db1", dropInfo.getValue().getDb(),
211+
"edit-log DropInfo must carry the LOCAL db name for follower replay");
212+
Assertions.assertEquals("t1", dropInfo.getValue().getTableName(),
213+
"edit-log DropInfo must carry the LOCAL table name for follower replay");
214+
Assertions.assertEquals("db1", catalog.lastGetDbForReplayArg,
215+
"cache invalidation must look up the LOCAL db name");
216+
Mockito.verify(replayDb).unregisterTable("t1");
217+
Mockito.verify(db, Mockito.never()).unregisterTable(Mockito.anyString());
218+
}
219+
220+
@Test
221+
public void testDropTableMissingDbThrowsEvenWithIfExists() {
222+
catalog.dbNullableResult = null; // db not present
223+
224+
// WHY: mirror base ExternalCatalog.dropTable -- a missing db ALWAYS throws, even with
225+
// IF EXISTS (only a missing TABLE honors IF EXISTS). A mutation that ifExists-gates the
226+
// db==null branch makes this test red.
227+
Assertions.assertThrows(DdlException.class,
228+
() -> catalog.dropTable("missing", "t1", false, false, false, true, false, false));
229+
Mockito.verifyNoInteractions(metadata);
187230
}
188231

189232
@Test
190-
public void testDropTableIfExistsWhenMissingIsNoop() throws Exception {
191-
Mockito.when(metadata.getTableHandle(session, "db1", "missing")).thenReturn(Optional.empty());
233+
public void testDropTableIfExistsWhenMissingTableIsNoop() throws Exception {
234+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase();
235+
Mockito.doReturn(null).when(db).getTableNullable("missing");
236+
catalog.dbNullableResult = db;
192237

193238
catalog.dropTable("db1", "missing", false, false, false, true, false, false);
194239

195-
Mockito.verify(metadata, Mockito.never()).dropTable(Mockito.any(), Mockito.any());
240+
// Table missing + IF EXISTS => no-op; the connector is never even consulted.
241+
Mockito.verifyNoInteractions(metadata);
196242
Mockito.verify(mockEditLog, Mockito.never()).logDropTable(Mockito.any());
197243
}
198244

199245
@Test
200-
public void testDropTableMissingWithoutIfExistsThrows() {
201-
Mockito.when(metadata.getTableHandle(session, "db1", "missing")).thenReturn(Optional.empty());
246+
public void testDropTableMissingTableWithoutIfExistsThrows() {
247+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase();
248+
Mockito.doReturn(null).when(db).getTableNullable("missing");
249+
catalog.dbNullableResult = db;
202250

203251
Assertions.assertThrows(DdlException.class,
204252
() -> catalog.dropTable("db1", "missing", false, false, false, false, false, false));
253+
Mockito.verifyNoInteractions(metadata);
254+
}
255+
256+
@Test
257+
public void testDropTableHandleAbsentAfterLocalResolveIsNoopWithIfExists() throws Exception {
258+
// FE cache has the table (resolves locally), but it was dropped out-of-band remotely:
259+
// getTableHandle returns empty. IF EXISTS must still no-op.
260+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase();
261+
ExternalTable table = Mockito.mock(ExternalTable.class);
262+
Mockito.when(table.getRemoteDbName()).thenReturn("DB1");
263+
Mockito.when(table.getRemoteName()).thenReturn("TBL1");
264+
Mockito.doReturn(table).when(db).getTableNullable("t1");
265+
catalog.dbNullableResult = db;
266+
Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.empty());
267+
268+
catalog.dropTable("db1", "t1", false, false, false, true, false, false);
269+
270+
Mockito.verify(metadata).getTableHandle(session, "DB1", "TBL1");
271+
Mockito.verify(metadata, Mockito.never()).dropTable(Mockito.any(), Mockito.any());
272+
Mockito.verify(mockEditLog, Mockito.never()).logDropTable(Mockito.any());
273+
}
274+
275+
@Test
276+
public void testDropTableHandleAbsentAfterLocalResolveThrowsWithoutIfExists() {
277+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase();
278+
ExternalTable table = Mockito.mock(ExternalTable.class);
279+
Mockito.when(table.getRemoteDbName()).thenReturn("DB1");
280+
Mockito.when(table.getRemoteName()).thenReturn("TBL1");
281+
Mockito.doReturn(table).when(db).getTableNullable("t1");
282+
catalog.dbNullableResult = db;
283+
Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.empty());
284+
285+
Assertions.assertThrows(DdlException.class,
286+
() -> catalog.dropTable("db1", "t1", false, false, false, false, false, false));
205287
Mockito.verify(metadata, Mockito.never()).dropTable(Mockito.any(), Mockito.any());
206288
}
207289

208290
@Test
209291
public void testDropTableWrapsConnectorException() {
292+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase();
293+
ExternalTable table = Mockito.mock(ExternalTable.class);
294+
Mockito.when(table.getRemoteDbName()).thenReturn("DB1");
295+
Mockito.when(table.getRemoteName()).thenReturn("TBL1");
296+
Mockito.doReturn(table).when(db).getTableNullable("t1");
297+
catalog.dbNullableResult = db;
298+
210299
ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class);
211-
Mockito.when(metadata.getTableHandle(session, "db1", "t1")).thenReturn(Optional.of(handle));
300+
Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.of(handle));
212301
Mockito.doThrow(new DorisConnectorException("boom"))
213302
.when(metadata).dropTable(session, handle);
214303

@@ -217,12 +306,58 @@ public void testDropTableWrapsConnectorException() {
217306
Assertions.assertTrue(ex.getMessage().contains("boom"));
218307
}
219308

220-
// ==================== CREATE TABLE (cache-invalidation fix) ====================
309+
// ==================== CREATE TABLE ====================
310+
// FIX-DDL-REMOTE: createTable now resolves the local db name to its REMOTE (ODPS) name (via
311+
// getDbNullable + db.getRemoteName()) and passes THAT to the converter; the table name is
312+
// intentionally NOT remote-resolved (legacy parity). Edit log / cache invalidation still use
313+
// the local names.
314+
315+
@Test
316+
public void testCreateTablePassesRemoteDbNameToConverter() throws UserException {
317+
// local db1 maps to remote DB1.
318+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase();
319+
Mockito.when(db.getRemoteName()).thenReturn("DB1");
320+
catalog.dbNullableResult = db;
321+
catalog.dbForReplayResult = Optional.of(db);
322+
323+
try (MockedStatic<CreateTableInfoToConnectorRequestConverter> conv =
324+
Mockito.mockStatic(CreateTableInfoToConnectorRequestConverter.class)) {
325+
ConnectorCreateTableRequest req = Mockito.mock(ConnectorCreateTableRequest.class);
326+
conv.when(() -> CreateTableInfoToConnectorRequestConverter.convert(Mockito.any(), Mockito.any()))
327+
.thenReturn(req);
328+
CreateTableInfo info = Mockito.mock(CreateTableInfo.class);
329+
Mockito.when(info.getDbName()).thenReturn("db1");
330+
Mockito.when(info.getTableName()).thenReturn("t1");
331+
332+
catalog.createTable(info);
333+
334+
// WHY: the converter (and thus the connector) must receive the REMOTE db name "DB1",
335+
// not the local "db1", so name-mapped catalogs address the real ODPS schema. We assert
336+
// on the SECOND argument actually passed to convert() -- NOT on req.getDbName(), which
337+
// would be vacuous here because the converter is mocked and returns a stub unaffected
338+
// by the dbName argument. A mutation that passes info.getDbName() makes this red.
339+
conv.verify(() -> CreateTableInfoToConnectorRequestConverter.convert(info, "DB1"));
340+
}
341+
}
342+
343+
@Test
344+
public void testCreateTableMissingDbThrows() {
345+
catalog.dbNullableResult = null; // db not present
346+
CreateTableInfo info = Mockito.mock(CreateTableInfo.class);
347+
Mockito.when(info.getDbName()).thenReturn("missing");
348+
349+
Assertions.assertThrows(DdlException.class, () -> catalog.createTable(info));
350+
Mockito.verifyNoInteractions(metadata);
351+
}
221352

222353
@Test
223-
public void testCreateTableInvalidatesDbCache() throws UserException {
224-
ExternalDatabase<? extends ExternalTable> mockDb = mockExternalDatabase();
225-
catalog.dbForReplayResult = Optional.of(mockDb);
354+
public void testCreateTableInvalidatesDbCacheUsingLocalNames() throws UserException {
355+
// remote DB1 != local db1, so the LOCAL-name assertions below are meaningful.
356+
ExternalDatabase<? extends ExternalTable> db = mockExternalDatabase();
357+
Mockito.when(db.getRemoteName()).thenReturn("DB1");
358+
catalog.dbNullableResult = db;
359+
ExternalDatabase<? extends ExternalTable> replayDb = mockExternalDatabase();
360+
catalog.dbForReplayResult = Optional.of(replayDb);
226361

227362
try (MockedStatic<CreateTableInfoToConnectorRequestConverter> conv =
228363
Mockito.mockStatic(CreateTableInfoToConnectorRequestConverter.class)) {
@@ -236,7 +371,20 @@ public void testCreateTableInvalidatesDbCache() throws UserException {
236371
catalog.createTable(info);
237372

238373
Mockito.verify(metadata).createTable(session, req);
239-
Mockito.verify(mockDb).resetMetaCacheNames();
374+
// WHY: edit log MUST carry the LOCAL names (followers replay this persist entry), even
375+
// though the connector got the remote "DB1". A mutation persisting db.getRemoteName()
376+
// must turn these red.
377+
ArgumentCaptor<org.apache.doris.persist.CreateTableInfo> persist =
378+
ArgumentCaptor.forClass(org.apache.doris.persist.CreateTableInfo.class);
379+
Mockito.verify(mockEditLog).logCreateTable(persist.capture());
380+
Assertions.assertEquals("db1", persist.getValue().getDbName(),
381+
"edit-log CreateTableInfo must carry the LOCAL db name for follower replay");
382+
Assertions.assertEquals("t1", persist.getValue().getTblName(),
383+
"edit-log CreateTableInfo must carry the LOCAL table name for follower replay");
384+
// Cache invalidation must look up the LOCAL db name and act on the replay db.
385+
Assertions.assertEquals("db1", catalog.lastGetDbForReplayArg,
386+
"cache invalidation must look up the LOCAL db name");
387+
Mockito.verify(replayDb).resetMetaCacheNames();
240388
}
241389
}
242390

@@ -258,6 +406,9 @@ private static class TestablePluginCatalog extends PluginDrivenExternalCatalog {
258406
Optional<ExternalDatabase<? extends ExternalTable>> dbForReplayResult = Optional.empty();
259407
int resetMetaCacheNamesCount;
260408
String unregisteredDb;
409+
// Records the arg passed to getDbForReplay so tests can assert the cache-invalidation
410+
// lookup uses the LOCAL db name (follower-replay parity), not the remote-resolved one.
411+
String lastGetDbForReplayArg;
261412

262413
TestablePluginCatalog(Connector initial) {
263414
super(1L, "test-catalog", null, testProps(), "", initial);
@@ -281,6 +432,7 @@ public ExternalDatabase<? extends ExternalTable> getDbNullable(String dbName) {
281432

282433
@Override
283434
public Optional<ExternalDatabase<? extends ExternalTable>> getDbForReplay(String dbName) {
435+
lastGetDbForReplayArg = dbName;
284436
return dbForReplayResult;
285437
}
286438

0 commit comments

Comments
 (0)