Skip to content

Commit e4849e2

Browse files
[fix](lance) handle mapped database DDL caches
Signed-off-by: wangzhaobo957-cloud <[email protected]>
1 parent 350b808 commit e4849e2

9 files changed

Lines changed: 548 additions & 73 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4913,7 +4913,17 @@ public static void getDdlStmt(Command command, String dbName, TableIf table, Lis
49134913
}
49144914
if (!properties.isEmpty()) {
49154915
sb.append("\nPROPERTIES (\n");
4916-
sb.append(new PrintableMap<>(properties, " = ", true, true, hidePassword));
4916+
Iterator<Entry<String, String>> iterator = properties.entrySet().iterator();
4917+
while (iterator.hasNext()) {
4918+
Entry<String, String> property = iterator.next();
4919+
String value = hidePassword && PrintableMap.SENSITIVE_KEY.contains(property.getKey())
4920+
? PrintableMap.PASSWORD_MASK : String.valueOf(property.getValue());
4921+
sb.append(" ").append(SqlLiteralUtils.quoteStringLiteral(property.getKey()))
4922+
.append(" = ").append(SqlLiteralUtils.quoteStringLiteral(value));
4923+
if (iterator.hasNext()) {
4924+
sb.append(",\n");
4925+
}
4926+
}
49174927
sb.append("\n)");
49184928
}
49194929
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,8 @@ private byte[] emptyArrowStream(Schema schema) {
295295
ByteArrayOutputStream output = new ByteArrayOutputStream();
296296
ArrowStreamWriter writer = new ArrowStreamWriter(root, null, output)) {
297297
writer.start();
298+
root.setRowCount(0);
299+
writer.writeBatch();
298300
writer.end();
299301
return output.toByteArray();
300302
} catch (IOException e) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataOps.java‎

Lines changed: 65 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,12 @@ public boolean createDbImpl(String dbName, boolean ifNotExists, Map<String, Stri
7272
if (client.isRootDatabase(dbName)) {
7373
throw new DdlException("Cannot create the configured Lance root database: " + dbName);
7474
}
75+
if (catalog.getDbNullable(dbName) != null) {
76+
if (ifNotExists) {
77+
return true;
78+
}
79+
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName);
80+
}
7581
if (client.databaseExists(dbName)) {
7682
if (ifNotExists) {
7783
catalog.resetMetaCacheNames();
@@ -137,7 +143,18 @@ public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws
137143

138144
@Override
139145
public void afterDropDb(String dbName) {
146+
Optional<ExternalDatabase<? extends ExternalTable>> db = catalog.getDbForReplay(dbName);
147+
if (db.isPresent()) {
148+
catalog.unregisterDatabase(db.get().getFullName());
149+
return;
150+
}
140151
catalog.unregisterDatabase(dbName);
152+
catalog.retireAllDatabaseObjectsWithoutEngineInvalidation();
153+
}
154+
155+
@Override
156+
public void afterDropDbNoOp(String dbName) {
157+
catalog.retireAllDatabaseObjectsWithoutEngineInvalidation();
141158
}
142159

143160
@Override
@@ -233,7 +250,12 @@ public void dropTableImpl(ExternalTable dorisTable, boolean ifExists) throws Ddl
233250
public void afterDropTable(String dbName, String tblName) {
234251
catalog.invalidateTableAccessCache();
235252
Optional<ExternalDatabase<?>> db = catalog.getDbForReplay(dbName);
236-
db.ifPresent(externalDatabase -> externalDatabase.unregisterTable(tblName));
253+
if (db.isPresent()) {
254+
boolean invalidated = db.get().unregisterTableForReplay(tblName);
255+
if (!invalidated && !db.get().hasLocalTableName(tblName)) {
256+
db.get().retireAllTableObjectsWithoutEngineInvalidation();
257+
}
258+
}
237259
}
238260

239261
@Override
@@ -334,29 +356,55 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition
334356
throws UserException {
335357
validateModifyColumn(column, position);
336358
Column currentColumn = requireColumn(dorisTable, column.getName());
337-
AlterColumnsEntry alteration = new AlterColumnsEntry().path(currentColumn.getName());
338-
boolean changed = false;
339-
if (!currentColumn.getType().equals(column.getType())) {
340-
alteration.dataType(LanceTypeConverter.toAlterColumnType(column.getType()));
341-
changed = true;
359+
List<AlterColumnsEntry> alterations = new ArrayList<>();
360+
boolean typeChanged = !currentColumn.getType().equals(column.getType());
361+
if (typeChanged && StringUtils.isNotEmpty(currentColumn.getComment())) {
362+
throw new UserException("Lance MODIFY COLUMN does not support changing the type "
363+
+ "of a column with a comment");
364+
}
365+
if (typeChanged) {
366+
alterations.add(new AlterColumnsEntry()
367+
.path(currentColumn.getName())
368+
.dataType(LanceTypeConverter.toAlterColumnType(column.getType())));
342369
}
343370
if (column.isNullableSpecified()
344371
&& currentColumn.isAllowNull() != column.isAllowNull()) {
345-
alteration.nullable(column.isAllowNull());
346-
changed = true;
372+
alterations.add(new AlterColumnsEntry()
373+
.path(currentColumn.getName())
374+
.nullable(column.isAllowNull()));
347375
}
348-
if (!changed) {
376+
if (alterations.isEmpty()) {
349377
return;
350378
}
351379

352-
execute("Failed to modify column " + currentColumn.getName() + " in Lance table "
353-
+ tableName(dorisTable),
354-
client -> {
355-
client.alterColumns(dorisTable.getRemoteDbName(), dorisTable.getRemoteName(),
356-
Collections.singletonList(alteration));
357-
return null;
358-
});
359-
refreshTable(dorisTable, updateTime);
380+
boolean modified = false;
381+
DdlException mutationFailure = null;
382+
try {
383+
for (AlterColumnsEntry alteration : alterations) {
384+
execute("Failed to modify column " + currentColumn.getName() + " in Lance table "
385+
+ tableName(dorisTable),
386+
client -> {
387+
client.alterColumns(dorisTable.getRemoteDbName(), dorisTable.getRemoteName(),
388+
Collections.singletonList(alteration));
389+
return null;
390+
});
391+
modified = true;
392+
}
393+
} catch (DdlException e) {
394+
mutationFailure = e;
395+
throw e;
396+
} finally {
397+
if (modified) {
398+
try {
399+
refreshTable(dorisTable, updateTime);
400+
} catch (RuntimeException refreshFailure) {
401+
if (mutationFailure == null) {
402+
throw refreshFailure;
403+
}
404+
mutationFailure.addSuppressed(refreshFailure);
405+
}
406+
}
407+
}
360408
}
361409

362410
static void validateAddColumn(Column column, ColumnPosition position) throws UserException {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceClient.java‎

Lines changed: 96 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.commons.lang3.StringUtils;
3131
import org.lance.namespace.LanceNamespace;
3232
import org.lance.namespace.errors.NamespaceNotFoundException;
33+
import org.lance.namespace.errors.TableAlreadyExistsException;
3334
import org.lance.namespace.errors.TableNotFoundException;
3435
import org.lance.namespace.errors.TableVersionNotFoundException;
3536
import org.lance.namespace.model.AddColumnsEntry;
@@ -52,6 +53,7 @@
5253
import org.lance.namespace.model.ListTablesRequest;
5354
import org.lance.namespace.model.ListTablesResponse;
5455
import org.lance.namespace.model.NamespaceExistsRequest;
56+
import org.lance.namespace.model.RegisterTableRequest;
5557
import org.lance.namespace.model.TableExistsRequest;
5658
import org.lance.namespace.model.TableVersion;
5759

@@ -77,6 +79,7 @@
7779
final class LanceNamespaceClient {
7880
private static final String DATABASE_NAMESPACE_DELIMITER = ".";
7981
private static final int PAGE_SIZE = 1000;
82+
private static final String LANCE_FILESYSTEM = AbstractLanceProperties.LANCE_FILESYSTEM;
8083
private static final String LANCE_REST = AbstractLanceProperties.LANCE_REST;
8184

8285
private final LanceNamespace namespace;
@@ -204,18 +207,45 @@ void createDatabase(String dbName, Map<String, String> properties) {
204207

205208
void dropDatabase(String dbName, boolean ifExists, boolean force) {
206209
try {
207-
DropNamespaceRequest request = new DropNamespaceRequest()
208-
.id(buildNamespaceId(dbName))
209-
.mode(ifExists ? "Skip" : "Fail")
210-
.behavior(force ? "Cascade" : "Restrict");
210+
List<String> namespaceId = buildNamespaceId(dbName);
211211
synchronized (namespaceLock) {
212-
namespace.dropNamespace(request);
212+
if (force) {
213+
try {
214+
dropNamespaceCascade(namespaceId, ifExists ? "Skip" : "Fail");
215+
} catch (NamespaceNotFoundException e) {
216+
if (!ifExists) {
217+
throw e;
218+
}
219+
}
220+
return;
221+
}
222+
namespace.dropNamespace(new DropNamespaceRequest()
223+
.id(namespaceId)
224+
.mode(ifExists ? "Skip" : "Fail")
225+
.behavior("Restrict"));
213226
}
214227
} catch (DdlException e) {
215228
throw new RuntimeException(e);
216229
}
217230
}
218231

232+
private void dropNamespaceCascade(List<String> namespaceId, String mode) {
233+
for (String child : listChildNamespaces(namespaceId)) {
234+
List<String> childId = new ArrayList<>(namespaceId);
235+
childId.add(child);
236+
dropNamespaceCascade(childId, "Fail");
237+
}
238+
for (String table : listTableNames(namespaceId)) {
239+
List<String> tableId = new ArrayList<>(namespaceId);
240+
tableId.add(table);
241+
namespace.dropTable(new DropTableRequest().id(tableId));
242+
}
243+
namespace.dropNamespace(new DropNamespaceRequest()
244+
.id(namespaceId)
245+
.mode(mode)
246+
.behavior("Restrict"));
247+
}
248+
219249
/**
220250
* Lists all direct child namespace names under the given full Lance namespace.
221251
*
@@ -248,33 +278,36 @@ private List<String> listChildNamespaces(List<String> namespaceId) {
248278

249279
List<String> listTableNames(String dbName) {
250280
try {
251-
List<String> namespaceId = buildNamespaceId(dbName);
252-
List<String> result = new ArrayList<>();
253-
String pageToken = null;
254-
Set<String> consumedTokens = new HashSet<>();
255-
do {
256-
ListTablesRequest request = new ListTablesRequest().id(namespaceId).limit(PAGE_SIZE);
257-
if (pageToken != null) {
258-
request.pageToken(pageToken);
259-
}
260-
ListTablesResponse response;
261-
synchronized (namespaceLock) {
262-
response = namespace.listTables(request);
263-
}
264-
if (response.getTables() != null) {
265-
result.addAll(response.getTables());
266-
}
267-
pageToken = response.getPageToken();
268-
if (StringUtils.isNotEmpty(pageToken) && !consumedTokens.add(pageToken)) {
269-
throw new IllegalStateException("Lance namespace repeated a pagination token");
270-
}
271-
} while (StringUtils.isNotEmpty(pageToken));
272-
return result;
281+
return listTableNames(buildNamespaceId(dbName));
273282
} catch (DdlException e) {
274283
throw new RuntimeException(e);
275284
}
276285
}
277286

287+
private List<String> listTableNames(List<String> namespaceId) {
288+
List<String> result = new ArrayList<>();
289+
String pageToken = null;
290+
Set<String> consumedTokens = new HashSet<>();
291+
do {
292+
ListTablesRequest request = new ListTablesRequest().id(namespaceId).limit(PAGE_SIZE);
293+
if (pageToken != null) {
294+
request.pageToken(pageToken);
295+
}
296+
ListTablesResponse response;
297+
synchronized (namespaceLock) {
298+
response = namespace.listTables(request);
299+
}
300+
if (response.getTables() != null) {
301+
result.addAll(response.getTables());
302+
}
303+
pageToken = response.getPageToken();
304+
if (StringUtils.isNotEmpty(pageToken) && !consumedTokens.add(pageToken)) {
305+
throw new IllegalStateException("Lance namespace repeated a pagination token");
306+
}
307+
} while (StringUtils.isNotEmpty(pageToken));
308+
return result;
309+
}
310+
278311
boolean tableExists(String dbName, String tblName) {
279312
try {
280313
List<String> tableId = buildTableId(dbName, tblName);
@@ -309,9 +342,7 @@ void createTable(String dbName, String tableName, Map<String, String> properties
309342
void dropTable(String dbName, String tableName) {
310343
try {
311344
DropTableRequest request = new DropTableRequest().id(buildTableId(dbName, tableName));
312-
synchronized (namespaceLock) {
313-
namespace.dropTable(request);
314-
}
345+
executeTableMutation(dbName, tableName, () -> namespace.dropTable(request));
315346
} catch (DdlException e) {
316347
throw new RuntimeException(e);
317348
}
@@ -322,9 +353,7 @@ void addColumns(String dbName, String tableName, List<AddColumnsEntry> columns)
322353
AlterTableAddColumnsRequest request = new AlterTableAddColumnsRequest()
323354
.id(buildTableId(dbName, tableName))
324355
.newColumns(columns);
325-
synchronized (namespaceLock) {
326-
namespace.alterTableAddColumns(request);
327-
}
356+
executeTableMutation(dbName, tableName, () -> namespace.alterTableAddColumns(request));
328357
} catch (DdlException e) {
329358
throw new RuntimeException(e);
330359
}
@@ -335,9 +364,7 @@ void alterColumns(String dbName, String tableName, List<AlterColumnsEntry> alter
335364
AlterTableAlterColumnsRequest request = new AlterTableAlterColumnsRequest()
336365
.id(buildTableId(dbName, tableName))
337366
.alterations(alterations);
338-
synchronized (namespaceLock) {
339-
namespace.alterTableAlterColumns(request);
340-
}
367+
executeTableMutation(dbName, tableName, () -> namespace.alterTableAlterColumns(request));
341368
} catch (DdlException e) {
342369
throw new RuntimeException(e);
343370
}
@@ -348,14 +375,44 @@ void dropColumns(String dbName, String tableName, List<String> columns) {
348375
AlterTableDropColumnsRequest request = new AlterTableDropColumnsRequest()
349376
.id(buildTableId(dbName, tableName))
350377
.columns(columns);
351-
synchronized (namespaceLock) {
352-
namespace.alterTableDropColumns(request);
353-
}
378+
executeTableMutation(dbName, tableName, () -> namespace.alterTableDropColumns(request));
354379
} catch (DdlException e) {
355380
throw new RuntimeException(e);
356381
}
357382
}
358383

384+
private void executeTableMutation(String dbName, String tableName, Runnable mutation)
385+
throws DdlException {
386+
List<String> namespaceId = buildNamespaceId(dbName);
387+
List<String> tableId = new ArrayList<>(namespaceId);
388+
tableId.add(tableName);
389+
synchronized (namespaceLock) {
390+
try {
391+
mutation.run();
392+
return;
393+
} catch (TableNotFoundException originalException) {
394+
if (!LANCE_FILESYSTEM.equals(catalogType) || !namespaceId.isEmpty()) {
395+
throw originalException;
396+
}
397+
try {
398+
namespace.tableExists(new TableExistsRequest().id(tableId));
399+
} catch (TableNotFoundException | NamespaceNotFoundException e) {
400+
throw originalException;
401+
}
402+
try {
403+
// DirectoryNamespace only accepts locations relative to its warehouse root.
404+
namespace.registerTable(new RegisterTableRequest()
405+
.id(tableId)
406+
.location(tableName + ".lance")
407+
.mode("Create"));
408+
} catch (TableAlreadyExistsException e) {
409+
// Another mutation registered the external dataset first.
410+
}
411+
mutation.run();
412+
}
413+
}
414+
}
415+
359416
LanceTableAccess resolveTableAccess(String dbName, String tableName) {
360417
List<String> tableId = tableAccessKey(dbName, tableName);
361418
if (tableAccessTtlNanos == 0) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/LanceFileSystemMetastoreProperties.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
/** Properties for a Lance directory namespace backed by a filesystem warehouse. */
3535
public class LanceFileSystemMetastoreProperties extends AbstractLanceProperties {
3636
public static final String WAREHOUSE = "warehouse";
37+
private static final String DIR_LISTING_TO_MANIFEST_MIGRATION_ENABLED =
38+
"dir_listing_to_manifest_migration_enabled";
3739

3840
@ConnectorProperty(
3941
names = {WAREHOUSE},
@@ -56,6 +58,9 @@ public LanceNamespace createNamespace(
5658
BufferAllocator allocator, Map<String, String> javaStorageOptions) {
5759
Map<String, String> namespaceProperties = new HashMap<>();
5860
namespaceProperties.put("root", warehouse);
61+
// Prefer manifest metadata for managed root tables while retaining directory discovery
62+
// for externally created datasets that have not been registered yet.
63+
namespaceProperties.put(DIR_LISTING_TO_MANIFEST_MIGRATION_ENABLED, "true");
5964
javaStorageOptions.forEach(
6065
(key, value) -> namespaceProperties.put("storage." + key, value));
6166
return LanceNamespace.connect("dir", namespaceProperties, allocator);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogClientTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ public void testCreateTableSerializesAnEmptyArrowStream() throws Exception {
218218
try (ArrowStreamReader reader = new ArrowStreamReader(
219219
new ByteArrayInputStream(payload.getValue()), allocator)) {
220220
Assertions.assertEquals(schema, reader.getVectorSchemaRoot().getSchema());
221+
Assertions.assertTrue(reader.loadNextBatch());
222+
Assertions.assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
221223
Assertions.assertFalse(reader.loadNextBatch());
222224
}
223225
} finally {

0 commit comments

Comments
 (0)