Skip to content

Commit f6e0f21

Browse files
[fix](lance) address remaining DDL review issues
1 parent d051e49 commit f6e0f21

8 files changed

Lines changed: 107 additions & 53 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1189,13 +1189,16 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc
11891189
try {
11901190
Optional<String> resolvedDbName = metadataOps.dropDbWithResolvedName(dbName, ifExists, force);
11911191
if (!resolvedDbName.isPresent()) {
1192-
// No remote drop happened (for example DROP DATABASE IF EXISTS on a missing
1193-
// database). Do not journal the no-op, otherwise every follower would replay a
1194-
// post-drop hook that retires caches for a database that was never touched.
1195-
LOG.info("skip drop database {}.{} because the database does not exist", getName(), dbName);
1196-
return;
1192+
if (!metadataOps.shouldJournalDropDbNoOp()) {
1193+
// No remote drop happened (for example DROP DATABASE IF EXISTS on a missing
1194+
// database). Do not journal the no-op unless the connector needs follower FEs
1195+
// to replay cache reconciliation.
1196+
LOG.info("skip drop database {}.{} because the database does not exist", getName(), dbName);
1197+
return;
1198+
}
11971199
}
1198-
DropDbInfo info = new DropDbInfo(getName(), resolvedDbName.get(), resolvedDbName.get());
1200+
String journalDbName = resolvedDbName.orElse(dbName);
1201+
DropDbInfo info = new DropDbInfo(getName(), journalDbName, journalDbName);
11991202
Env.getCurrentEnv().getEditLog().logDropDb(info);
12001203
} catch (Exception e) {
12011204
LOG.warn("Failed to drop database {} in catalog {}", dbName, getName(), e);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,10 @@ boolean isRootDatabase(String dbName) {
241241
return namespaceClient.isRootDatabase(dbName);
242242
}
243243

244+
boolean isRootNamespace(String dbName) throws DdlException {
245+
return namespaceClient.isRootNamespace(dbName);
246+
}
247+
244248
boolean databaseExists(String dbName) {
245249
return namespaceClient.databaseExists(dbName);
246250
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataOps.java‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,16 @@ public boolean createDbImpl(String dbName, boolean ifNotExists, Map<String, Stri
8383
if (client.isRootDatabase(dbName)) {
8484
throw new DdlException("Cannot create the configured Lance root database: " + dbName);
8585
}
86-
if (catalog.getDbNullable(dbName) != null) {
87-
if (ifNotExists) {
88-
return true;
86+
ExternalDatabase<?> cachedDb = catalog.getDbNullable(dbName);
87+
if (cachedDb != null) {
88+
if (client.databaseExists(cachedDb.getRemoteName())) {
89+
if (ifNotExists) {
90+
return true;
91+
}
92+
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName);
8993
}
90-
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName);
94+
catalog.unregisterDatabase(cachedDb.getFullName());
95+
catalog.resetMetaCacheNames();
9196
}
9297
if (client.databaseExists(dbName)) {
9398
if (ifNotExists) {
@@ -169,14 +174,26 @@ public void afterDropDb(String dbName) {
169174

170175
@Override
171176
public void afterDropDbNoOp(String dbName) {
177+
catalog.invalidateTableAccessCache();
172178
catalog.retireAllDatabaseObjectsWithoutEngineInvalidation();
173179
}
174180

181+
@Override
182+
public boolean shouldJournalDropDbNoOp() {
183+
return true;
184+
}
185+
175186
@Override
176187
public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserException {
177188
String dbName = createTableInfo.getDbName();
178189
String tableName = createTableInfo.getTableName();
179190
ExternalDatabase<?> db = catalog.getDbNullable(dbName);
191+
boolean filesystemCatalog =
192+
LanceExternalCatalog.LANCE_FILESYSTEM.equals(catalog.getLanceCatalogType());
193+
if (filesystemCatalog && tableName.contains("$")) {
194+
throw new DdlException(
195+
"Lance filesystem table names must not contain the manifest delimiter '$'");
196+
}
180197
if (db == null) {
181198
throw new DdlException("Failed to get database: '" + dbName
182199
+ "' in catalog: " + catalog.getName());
@@ -191,21 +208,30 @@ public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserExcep
191208
}
192209

193210
return execute("Failed to create Lance table " + dbName + "." + tableName, client -> {
211+
if (filesystemCatalog && !client.isRootNamespace(db.getRemoteName())) {
212+
throw new DdlException(
213+
"CREATE TABLE is only supported in the warehouse root namespace "
214+
+ "for Lance filesystem catalogs");
215+
}
194216
if (client.tableExists(db.getRemoteName(), tableName)) {
195217
if (createTableInfo.isIfNotExists()) {
196218
resetTableNameCache(dbName);
197219
return true;
198220
}
199221
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
200222
}
201-
if (db.getTableNullable(tableName) != null) {
202-
resetTableNameCache(dbName);
203-
if (db.getTableNullable(tableName) != null) {
223+
ExternalTable cachedTable = db.getTableNullable(tableName);
224+
if (cachedTable != null) {
225+
if (client.tableExists(db.getRemoteName(), cachedTable.getRemoteName())) {
226+
db.resetMetaCacheNames();
204227
if (createTableInfo.isIfNotExists()) {
205228
return true;
206229
}
207230
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
208231
}
232+
catalog.invalidateTableAccessCache();
233+
db.unregisterTable(cachedTable.getName());
234+
db.resetMetaCacheNames();
209235
}
210236
try {
211237
client.createTable(db.getRemoteName(), tableName, schema, properties);
@@ -425,6 +451,10 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition
425451

426452
static void validateAddColumn(Column column, ColumnPosition position) throws UserException {
427453
validateColumnAttributes(column, "ADD COLUMN");
454+
if (column.getType().getPrimitiveType() == PrimitiveType.DECIMAL256) {
455+
throw new UserException("Lance ADD COLUMN does not support DECIMAL256: "
456+
+ column.getType().toSql());
457+
}
428458
if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) {
429459
throw new UserException("Lance ADD COLUMN does not support default values");
430460
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceClient.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,10 @@ boolean isRootDatabase(String dbName) {
174174
return rootDatabase.equals(dbName);
175175
}
176176

177+
boolean isRootNamespace(String dbName) throws DdlException {
178+
return buildNamespaceId(dbName).isEmpty();
179+
}
180+
177181
boolean databaseExists(String dbName) {
178182
if (isRootDatabase(dbName)) {
179183
return true;
@@ -372,14 +376,20 @@ private void executeTableMutation(String dbName, String tableName, Runnable muta
372376
} catch (TableNotFoundException | NamespaceNotFoundException e) {
373377
throw originalException;
374378
}
379+
String expectedLocation = tableName + ".lance";
375380
try {
376381
// DirectoryNamespace only accepts locations relative to its warehouse root.
377382
namespace.registerTable(new RegisterTableRequest()
378383
.id(tableId)
379-
.location(tableName + ".lance")
384+
.location(expectedLocation)
380385
.mode("Create"));
381386
} catch (TableAlreadyExistsException e) {
382-
// Another mutation registered the external dataset first.
387+
String registeredLocation = describeTable(tableId).getLocation();
388+
if (!expectedLocation.equals(registeredLocation)) {
389+
throw new DdlException("Refusing to mutate Lance table " + tableName
390+
+ " because it was concurrently registered at "
391+
+ registeredLocation + " instead of " + expectedLocation);
392+
}
383393
}
384394
mutation.run();
385395
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,13 @@ default Optional<String> dropDbImplWithResolvedName(String dbName, boolean ifExi
120120
default void afterDropDbNoOp(String dbName) {
121121
}
122122

123+
/**
124+
* Whether a no-op drop still needs to be journaled so follower FEs replay its cache cleanup.
125+
*/
126+
default boolean shouldJournalDropDbNoOp() {
127+
return false;
128+
}
129+
123130
/**
124131
* @param createTableInfo
125132
* @return return false means table does not exist and is created this time

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceMetadataOpsTest.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -251,15 +251,21 @@ public void testCreateTableIgnoresStaleLocalCacheWhenRemoteTableIsGone() throws
251251
Mockito.verify(namespace).createTable(request.capture(), Mockito.any(byte[].class));
252252
Assertions.assertEquals(Arrays.asList("tenant", "analytics", "events"),
253253
request.getValue().getId());
254+
Mockito.verify(database).unregisterTable("events");
254255
Mockito.verify(database, Mockito.atLeastOnce()).resetMetaCacheNames();
255256
Mockito.verify(catalog).invalidateTableAccessCache();
256257
}
257258

258259
@Test
259260
public void testCreateTableRejectsLocalNameConflictAfterRefresh() throws UserException {
260261
LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
261-
Mockito.doThrow(new TableNotFoundException("missing"))
262-
.when(namespace).tableExists(Mockito.any());
262+
Mockito.doAnswer(invocation -> {
263+
if (((org.lance.namespace.model.TableExistsRequest) invocation.getArgument(0))
264+
.getId().get(2).equals("events")) {
265+
throw new TableNotFoundException("missing");
266+
}
267+
return null;
268+
}).when(namespace).tableExists(Mockito.any());
263269
LanceCatalogClient client = newClient(namespace, new RootAllocator(1024 * 1024));
264270
LanceExternalCatalog catalog = catalogWithClient(client);
265271
ExternalDatabase<?> database = Mockito.mock(ExternalDatabase.class);
@@ -297,6 +303,7 @@ public void testIfExistsHandlesConcurrentDropAndRefreshesLocalNames() throws Ddl
297303
Mockito.when(catalog.getDbForReplay("local_db")).thenReturn(Optional.of(database));
298304
Mockito.when(database.getRemoteName()).thenReturn("analytics");
299305
ExternalTable table = table("local_db", "local_table", "analytics", "events");
306+
Mockito.when(database.unregisterTableForReplay("local_table")).thenReturn(true);
300307
LanceMetadataOps ops = new LanceMetadataOps(catalog);
301308

302309
try {
@@ -312,7 +319,7 @@ public void testIfExistsHandlesConcurrentDropAndRefreshesLocalNames() throws Ddl
312319

313320
Mockito.verify(catalog, Mockito.never()).unregisterDatabase(Mockito.anyString());
314321
Mockito.verify(catalog).retireAllDatabaseObjectsWithoutEngineInvalidation();
315-
Mockito.verify(database).unregisterTable("local_table");
322+
Mockito.verify(database).unregisterTableForReplay("local_table");
316323
}
317324

318325
@Test
@@ -357,6 +364,7 @@ public void testDropDatabaseUsesRemoteDatabaseName() throws DdlException {
357364
ArgumentCaptor<DropNamespaceRequest> request = ArgumentCaptor.forClass(DropNamespaceRequest.class);
358365
Mockito.verify(namespace).dropNamespace(request.capture());
359366
Assertions.assertEquals(Arrays.asList("tenant", "Sales"), request.getValue().getId());
367+
Assertions.assertEquals("Cascade", request.getValue().getBehavior());
360368
Mockito.verify(catalog).unregisterDatabase("sales_db");
361369
}
362370

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceNamespaceClientMutationTest.java‎

Lines changed: 24 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,6 @@
3333
import org.lance.namespace.model.DescribeTableResponse;
3434
import org.lance.namespace.model.DropNamespaceRequest;
3535
import org.lance.namespace.model.DropTableRequest;
36-
import org.lance.namespace.model.ListNamespacesRequest;
37-
import org.lance.namespace.model.ListNamespacesResponse;
38-
import org.lance.namespace.model.ListTablesRequest;
39-
import org.lance.namespace.model.ListTablesResponse;
4036
import org.lance.namespace.model.RegisterTableRequest;
4137
import org.lance.namespace.model.TableExistsRequest;
4238
import org.mockito.ArgumentCaptor;
@@ -71,39 +67,17 @@ public void testDatabaseMutationRequestsUseConfiguredParentAndModes() {
7167
}
7268

7369
@Test
74-
public void testForceDropDatabaseDeletesNestedContentsBeforeNamespaces() {
70+
public void testForceDropDatabaseUsesNativeCascade() {
7571
LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
76-
Mockito.when(namespace.listNamespaces(Mockito.any())).thenAnswer(invocation -> {
77-
ListNamespacesRequest request = invocation.getArgument(0);
78-
if (request.getId().equals(Arrays.asList("tenant", "analytics"))) {
79-
return new ListNamespacesResponse().namespaces(Collections.singleton("archive"));
80-
}
81-
return new ListNamespacesResponse().namespaces(Collections.emptySet());
82-
});
83-
Mockito.when(namespace.listTables(Mockito.any())).thenAnswer(invocation -> {
84-
ListTablesRequest request = invocation.getArgument(0);
85-
if (request.getId().equals(Arrays.asList("tenant", "analytics", "archive"))) {
86-
return new ListTablesResponse().tables(Collections.singleton("old_events"));
87-
}
88-
return new ListTablesResponse().tables(Collections.singleton("events"));
89-
});
9072
LanceNamespaceClient client = client(namespace);
9173

9274
client.dropDatabase("analytics", true, true);
9375

94-
InOrder order = Mockito.inOrder(namespace);
95-
order.verify(namespace).dropTable(Mockito.argThat(request ->
96-
request.getId().equals(Arrays.asList("tenant", "analytics", "archive", "old_events"))));
97-
order.verify(namespace).dropNamespace(Mockito.argThat(request ->
98-
request.getId().equals(Arrays.asList("tenant", "analytics", "archive"))
99-
&& "Fail".equals(request.getMode())
100-
&& "Restrict".equals(request.getBehavior())));
101-
order.verify(namespace).dropTable(Mockito.argThat(request ->
102-
request.getId().equals(Arrays.asList("tenant", "analytics", "events"))));
103-
order.verify(namespace).dropNamespace(Mockito.argThat(request ->
76+
Mockito.verify(namespace).dropNamespace(Mockito.argThat(request ->
10477
request.getId().equals(Arrays.asList("tenant", "analytics"))
10578
&& "Skip".equals(request.getMode())
106-
&& "Restrict".equals(request.getBehavior())));
79+
&& "Cascade".equals(request.getBehavior())));
80+
Mockito.verify(namespace, Mockito.never()).dropTable(Mockito.any());
10781
}
10882

10983
@Test
@@ -251,6 +225,8 @@ public void testConcurrentExternalTableRegistrationStillRetriesMutation() {
251225
.when(namespace).alterTableAddColumns(Mockito.any());
252226
Mockito.doThrow(new TableAlreadyExistsException("registered concurrently"))
253227
.when(namespace).registerTable(Mockito.any());
228+
Mockito.when(namespace.describeTable(Mockito.any()))
229+
.thenReturn(new DescribeTableResponse().location("events.lance"));
254230
LanceNamespaceClient client = rootFilesystemClient(namespace);
255231

256232
client.addColumns("default", "events", Collections.singletonList(
@@ -259,6 +235,24 @@ public void testConcurrentExternalTableRegistrationStillRetriesMutation() {
259235
Mockito.verify(namespace, Mockito.times(2)).alterTableAddColumns(Mockito.any());
260236
}
261237

238+
@Test
239+
public void testConcurrentExternalTableRegistrationAtAnotherLocationFails() {
240+
LanceNamespace namespace = Mockito.mock(LanceNamespace.class);
241+
Mockito.doThrow(new TableNotFoundException("not registered"))
242+
.when(namespace).alterTableAddColumns(Mockito.any());
243+
Mockito.doThrow(new TableAlreadyExistsException("registered concurrently"))
244+
.when(namespace).registerTable(Mockito.any());
245+
Mockito.when(namespace.describeTable(Mockito.any()))
246+
.thenReturn(new DescribeTableResponse().location("other.lance"));
247+
LanceNamespaceClient client = rootFilesystemClient(namespace);
248+
249+
Assertions.assertThrows(RuntimeException.class, () -> client.addColumns(
250+
"default", "events", Collections.singletonList(
251+
new AddColumnsEntry().name("score").expression("CAST(NULL AS BIGINT)"))));
252+
253+
Mockito.verify(namespace).alterTableAddColumns(Mockito.any());
254+
}
255+
262256
@Test
263257
public void testMissingRootFilesystemTablePreservesMutationFailure() {
264258
LanceNamespace namespace = Mockito.mock(LanceNamespace.class);

‎regression-test/suites/external_table_p0/lance/test_lance_ddl.groovy‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ suite("test_lance_ddl", "p0,external") {
2525
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
2626
String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
2727
String catalogName = "test_lance_ddl"
28-
String databaseName = "lance_ddl_db"
28+
String databaseName = "default"
2929
String tableName = "events"
3030

3131
sql """DROP CATALOG IF EXISTS `${catalogName}`"""
@@ -43,8 +43,7 @@ suite("test_lance_ddl", "p0,external") {
4343
)
4444
"""
4545

46-
sql """DROP DATABASE IF EXISTS `${catalogName}`.`${databaseName}` FORCE"""
47-
sql """CREATE DATABASE `${catalogName}`.`${databaseName}`"""
46+
sql """DROP TABLE IF EXISTS `${catalogName}`.`${databaseName}`.`${tableName}`"""
4847
sql """
4948
CREATE TABLE `${catalogName}`.`${databaseName}`.`${tableName}` (
5049
id INT NOT NULL COMMENT 'identifier',
@@ -116,7 +115,7 @@ suite("test_lance_ddl", "p0,external") {
116115

117116
String showCreate = sql("""SHOW CREATE TABLE `${catalogName}`.`${databaseName}`.`${tableName}`""")[0][1]
118117
assertTrue(showCreate.contains("ENGINE=LANCE"))
119-
assertTrue(showCreate.contains("COMMENT 'Lance DDL regression table'"))
118+
assertTrue(showCreate.contains("COMMENT \"Lance DDL regression table\""))
120119
assertTrue(showCreate.contains("\"owner\" = \"doris\""))
121120
assertFalse(showCreate.contains("password"))
122121

@@ -126,7 +125,6 @@ suite("test_lance_ddl", "p0,external") {
126125
}
127126

128127
sql """DROP TABLE `${catalogName}`.`${databaseName}`.`${tableName}`"""
129-
sql """DROP DATABASE `${catalogName}`.`${databaseName}`"""
130128
} finally {
131129
sql """DROP CATALOG IF EXISTS `${catalogName}`"""
132130
}

0 commit comments

Comments
 (0)