Skip to content

Commit a623df2

Browse files
Feature/lance namespace table ddl branch 4.2 (#69384)
### What problem does this PR solve? Issue Number: close #66516 Related PR: #68349 Problem Summary: Lance catalogs are currently read-only from Doris. This PR adds DDL support backed by the Lance Namespace API for both filesystem and REST catalogs. This PR retargets the original branch-4.1 PR to branch-4.2 because branch-4.1 is code frozen. - Add create/drop namespace operations, including FORCE semantics and root-namespace protection. - Add create/drop/rename table operations and ENGINE=LANCE / SHOW CREATE TABLE support. - Convert Doris schemas to Arrow and create empty Lance datasets through Arrow IPC. - Add top-level ADD, MODIFY, RENAME, and DROP COLUMN schema evolution. - Refresh Doris metadata, schema caches, and the upstream Lance table-access cache after successful mutations. - Reject unsupported defaults, positions, comments, complex type changes, nested paths, and multi-clause schema changes explicitly. The current Lance Java SDK does not implement explicit backfill/virtual-column operations for DirectoryNamespace or RestNamespace, so this PR exposes only operations that the pinned SDK can execute reliably and documents that limitation. ### Release note Support namespace, table, and top-level column DDL for Lance external catalogs. ### Test Plan - Compile the FE reactor against JDK 17. - Run focused Lance DDL, type conversion, namespace mutation, metadata cache, and ALTER TABLE command tests. - Run Checkstyle and Markdownlint. - Add a filesystem regression suite covering successful DDL and unsupported operations. ### Test Result - FE reactor compilation: passed. - 65 focused unit tests: passed. - Checkstyle: passed with 0 violations. - Markdownlint for docs/lance-ddl.md: passed. - The Doris + MinIO regression suite was added but not executed locally because this environment does not provide that cluster. Signed-off-by: wangzhaobo957-cloud <[email protected]> Signed-off-by: trae-solo-agent <[email protected]>
1 parent 7872ff4 commit a623df2

27 files changed

Lines changed: 3137 additions & 96 deletions

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,6 +685,7 @@ public void processAlterTable(AlterTableCommand command) throws UserException {
685685
case MAX_COMPUTE_EXTERNAL_TABLE:
686686
case HUDI_EXTERNAL_TABLE:
687687
case TRINO_CONNECTOR_EXTERNAL_TABLE:
688+
case LANCE_EXTERNAL_TABLE:
688689
alterClauses.addAll(command.getOps());
689690
processAlterTableForExternalTable((ExternalTable) tableIf, alterClauses);
690691
return;

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@
111111
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
112112
import org.apache.doris.datasource.iceberg.IcebergSysExternalTable;
113113
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
114+
import org.apache.doris.datasource.lance.LanceExternalTable;
114115
import org.apache.doris.datasource.paimon.PaimonExternalTable;
115116
import org.apache.doris.datasource.paimon.PaimonSysExternalTable;
116117
import org.apache.doris.deploy.DeployManager;
@@ -4589,7 +4590,9 @@ public static void getDdlStmt(Command command, String dbName, TableIf table, Lis
45894590
}
45904591
}
45914592
sb.append("\n) ENGINE=");
4592-
sb.append(table.getType().name());
4593+
sb.append(table.getType() == TableType.LANCE_EXTERNAL_TABLE
4594+
? "LANCE"
4595+
: table.getType().name());
45934596

45944597
if (table instanceof OlapTable) {
45954598
OlapTable olapTable = (OlapTable) table;
@@ -4893,6 +4896,28 @@ public static void getDdlStmt(Command command, String dbName, TableIf table, Lis
48934896
}
48944897
}
48954898
sb.append("\n)");
4899+
} else if (table.getType() == TableType.LANCE_EXTERNAL_TABLE) {
4900+
Map<String, String> properties = new TreeMap<>(
4901+
((LanceExternalTable) table).getTableProperties());
4902+
String tableComment = properties.remove("comment");
4903+
if (StringUtils.isNotBlank(tableComment)) {
4904+
sb.append("\nCOMMENT ").append(SqlLiteralUtils.quoteStringLiteral(tableComment));
4905+
}
4906+
if (!properties.isEmpty()) {
4907+
sb.append("\nPROPERTIES (\n");
4908+
Iterator<Entry<String, String>> iterator = properties.entrySet().iterator();
4909+
while (iterator.hasNext()) {
4910+
Entry<String, String> property = iterator.next();
4911+
String value = hidePassword && PrintableMap.SENSITIVE_KEY.contains(property.getKey())
4912+
? PrintableMap.PASSWORD_MASK : String.valueOf(property.getValue());
4913+
sb.append(" ").append(SqlLiteralUtils.quoteStringLiteral(property.getKey()))
4914+
.append(" = ").append(SqlLiteralUtils.quoteStringLiteral(value));
4915+
if (iterator.hasNext()) {
4916+
sb.append(",\n");
4917+
}
4918+
}
4919+
sb.append("\n)");
4920+
}
48964921
}
48974922

48984923
createTableStmt.add(sb + ";");

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java‎

Lines changed: 47 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,16 @@ public void replayRefreshTable(ExternalObjectLog log) {
231231
invalidateColdReplayDatabase(catalog, log);
232232
return;
233233
}
234+
if (!Strings.isNullOrEmpty(log.getNewTableName())) {
235+
// this is a rename table op
236+
db.get().unregisterTableForReplay(log.getTableName());
237+
if (db.get().getTableForReplay(log.getNewTableName()).isPresent()
238+
|| db.get().hasLocalTableName(log.getNewTableName())) {
239+
db.get().unregisterTableForReplay(log.getNewTableName());
240+
}
241+
db.get().resetMetaCacheNames();
242+
return;
243+
}
234244
Optional<? extends ExternalTable> table;
235245
if (!Strings.isNullOrEmpty(log.getTableName())) {
236246
table = db.get().getTableForReplay(log.getTableName());
@@ -278,55 +288,49 @@ public void replayRefreshTable(ExternalObjectLog log) {
278288
invalidatePaimonCatalogForReplay(catalog);
279289
return;
280290
}
281-
if (!Strings.isNullOrEmpty(log.getNewTableName())) {
282-
// this is a rename table op
283-
db.get().unregisterTable(log.getTableName());
284-
db.get().resetMetaCacheNames();
285-
} else {
286-
List<String> modifiedPartNames = log.getPartitionNames();
287-
List<String> newPartNames = log.getNewPartitionNames();
288-
if (catalog instanceof HMSExternalCatalog
289-
&& ((modifiedPartNames != null && !modifiedPartNames.isEmpty())
290-
|| (newPartNames != null && !newPartNames.isEmpty()))) {
291-
// Partition-level cache invalidation, only for hive catalog. Fence the held table
292-
// before the hive(...) lookup can lazily initialize the group and throw; otherwise
293-
// a failed acquisition skips the row-count fence and the full-table fallback.
291+
List<String> modifiedPartNames = log.getPartitionNames();
292+
List<String> newPartNames = log.getNewPartitionNames();
293+
if (catalog instanceof HMSExternalCatalog
294+
&& ((modifiedPartNames != null && !modifiedPartNames.isEmpty())
295+
|| (newPartNames != null && !newPartNames.isEmpty()))) {
296+
// Partition-level cache invalidation, only for hive catalog. Fence the held table
297+
// before the hive(...) lookup can lazily initialize the group and throw; otherwise
298+
// a failed acquisition skips the row-count fence and the full-table fallback.
299+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get());
300+
try {
301+
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
302+
.hive(catalog.getId());
303+
cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames,
304+
newPartNames);
305+
// The held HMS table also caches the pre-insert table parameters.
306+
table.get().unsetObjectCreated();
307+
// Close the admission window the opening fence left open: a load admitted after
308+
// it can publish the pre-insert value from the still-resident file list.
294309
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get());
310+
} catch (RuntimeException e) {
311+
// The insert is already committed. A partial or failed selective refresh must not
312+
// leave stale Hive partition/file entries, so fall back to the same conservative
313+
// full-table invalidation the leader uses.
314+
LOG.warn("failed to refresh affected partitions when replaying refresh table for {}",
315+
table.get().getNameWithFullQualifiers(), e);
295316
try {
296-
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
297-
.hive(catalog.getId());
298-
cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames,
299-
newPartNames);
300-
// The held HMS table also caches the pre-insert table parameters.
317+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table.get());
318+
} finally {
301319
table.get().unsetObjectCreated();
302-
// Close the admission window the opening fence left open: a load admitted after
303-
// it can publish the pre-insert value from the still-resident file list.
304320
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get());
305-
} catch (RuntimeException e) {
306-
// The insert is already committed. A partial or failed selective refresh must not
307-
// leave stale Hive partition/file entries, so fall back to the same conservative
308-
// full-table invalidation the leader uses.
309-
LOG.warn("failed to refresh affected partitions when replaying refresh table for {}",
310-
table.get().getNameWithFullQualifiers(), e);
311-
try {
312-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table.get());
313-
} finally {
314-
table.get().unsetObjectCreated();
315-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get());
316-
}
317321
}
318-
if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) {
319-
((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime());
320-
}
321-
LOG.info("replay refresh partitions for table {}, "
322-
+ "modified partitions count: {}, "
323-
+ "new partitions count: {}",
324-
table.get().getName(), modifiedPartNames == null ? 0 : modifiedPartNames.size(),
325-
newPartNames == null ? 0 : newPartNames.size());
326-
} else {
327-
// Full table cache invalidation
328-
refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime());
329322
}
323+
if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) {
324+
((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime());
325+
}
326+
LOG.info("replay refresh partitions for table {}, "
327+
+ "modified partitions count: {}, "
328+
+ "new partitions count: {}",
329+
table.get().getName(), modifiedPartNames == null ? 0 : modifiedPartNames.size(),
330+
newPartNames == null ? 0 : newPartNames.size());
331+
} else {
332+
// Full table cache invalidation
333+
refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime());
330334
}
331335
});
332336
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1191,7 +1191,7 @@ public void createDb(String dbName, boolean ifNotExists, Map<String, String> pro
11911191

11921192
public void replayCreateDb(String dbName) {
11931193
if (metadataOps != null) {
1194-
metadataOps.afterCreateDb();
1194+
metadataOps.afterCreateDb(dbName);
11951195
}
11961196
}
11971197

@@ -1204,13 +1204,16 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc
12041204
try {
12051205
Optional<String> resolvedDbName = metadataOps.dropDbWithResolvedName(dbName, ifExists, force);
12061206
if (!resolvedDbName.isPresent()) {
1207-
// No remote drop happened (for example DROP DATABASE IF EXISTS on a missing
1208-
// database). Do not journal the no-op, otherwise every follower would replay a
1209-
// post-drop hook that retires caches for a database that was never touched.
1210-
LOG.info("skip drop database {}.{} because the database does not exist", getName(), dbName);
1211-
return;
1207+
if (!metadataOps.shouldJournalDropDbNoOp()) {
1208+
// No remote drop happened (for example DROP DATABASE IF EXISTS on a missing
1209+
// database). Do not journal the no-op unless the connector needs follower FEs
1210+
// to replay cache reconciliation.
1211+
LOG.info("skip drop database {}.{} because the database does not exist", getName(), dbName);
1212+
return;
1213+
}
12121214
}
1213-
DropDbInfo info = new DropDbInfo(getName(), resolvedDbName.get(), resolvedDbName.get());
1215+
String journalDbName = resolvedDbName.orElse(dbName);
1216+
DropDbInfo info = new DropDbInfo(getName(), journalDbName, journalDbName);
12141217
Env.getCurrentEnv().getEditLog().logDropDb(info);
12151218
} catch (Exception e) {
12161219
LOG.warn("Failed to drop database {} in catalog {}", dbName, getName(), e);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java‎

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838
import com.github.benmanes.caffeine.cache.Ticker;
3939
import org.apache.arrow.memory.BufferAllocator;
4040
import org.apache.arrow.memory.RootAllocator;
41+
import org.apache.arrow.vector.VectorSchemaRoot;
42+
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
4143
import org.apache.arrow.vector.types.pojo.Schema;
4244
import org.apache.commons.lang3.StringUtils;
4345
import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -50,8 +52,13 @@
5052
import org.lance.namespace.errors.TableBranchNotFoundException;
5153
import org.lance.namespace.errors.TableNotFoundException;
5254
import org.lance.namespace.errors.TableVersionNotFoundException;
55+
import org.lance.namespace.model.AddColumnsEntry;
56+
import org.lance.namespace.model.AlterColumnsEntry;
57+
import org.lance.namespace.model.DescribeTableResponse;
5358
import org.lance.namespace.model.TableVersion;
5459

60+
import java.io.ByteArrayOutputStream;
61+
import java.io.IOException;
5562
import java.util.ArrayList;
5663
import java.util.Collections;
5764
import java.util.HashMap;
@@ -101,7 +108,7 @@ static LanceCatalogClient create(AbstractLanceProperties properties,
101108
.indexCacheSizeBytes(INDEX_CACHE_SIZE_BYTES).build();
102109
return new LanceCatalogClient(namespace, allocator, session, properties.getLanceCatalogType(),
103110
properties.getRootDatabase(), parent, storageProperties, namespaceOptions, catalogSecrets,
104-
properties.getTableAccessCacheTtlSeconds());
111+
properties.getTableAccessCacheTtlSeconds(), properties.getNamespaceStorageUri());
105112
} catch (RuntimeException | Error e) {
106113
closeResource(namespace);
107114
closeResource(session);
@@ -123,13 +130,21 @@ static LanceCatalogClient create(AbstractLanceProperties properties,
123130
String catalogType, String rootDatabase, List<String> parentNamespace,
124131
List<StorageProperties> storageProperties, Map<String, String> namespaceStorageOptions,
125132
List<String> catalogSecrets, int tableAccessCacheTtlSeconds) {
133+
this(namespace, allocator, session, catalogType, rootDatabase, parentNamespace,
134+
storageProperties, namespaceStorageOptions, catalogSecrets, tableAccessCacheTtlSeconds, null);
135+
}
136+
137+
LanceCatalogClient(LanceNamespace namespace, BufferAllocator allocator, Session session,
138+
String catalogType, String rootDatabase, List<String> parentNamespace,
139+
List<StorageProperties> storageProperties, Map<String, String> namespaceStorageOptions,
140+
List<String> catalogSecrets, int tableAccessCacheTtlSeconds, String warehouse) {
126141
this.catalogSecrets = Collections.unmodifiableList(new ArrayList<>(catalogSecrets));
127142
this.namespace = namespace;
128143
this.namespaceAllocator = allocator;
129144
this.session = session;
130145
this.namespaceClient = new LanceNamespaceClient(
131146
namespace, catalogType, rootDatabase, parentNamespace, storageProperties,
132-
tableAccessCacheTtlSeconds, Ticker.systemTicker());
147+
namespaceStorageOptions, tableAccessCacheTtlSeconds, Ticker.systemTicker(), warehouse);
133148
this.namespaceStorageOptions = Collections.unmodifiableMap(new HashMap<>(namespaceStorageOptions));
134149
}
135150

@@ -226,6 +241,81 @@ boolean tableExists(String dbName, String tableName) {
226241
return namespaceClient.tableExists(dbName, tableName);
227242
}
228243

244+
boolean isRootDatabase(String dbName) {
245+
return namespaceClient.isRootDatabase(dbName);
246+
}
247+
248+
boolean isRootNamespace(String dbName) throws DdlException {
249+
return namespaceClient.isRootNamespace(dbName);
250+
}
251+
252+
boolean databaseExists(String dbName) {
253+
return namespaceClient.databaseExists(dbName);
254+
}
255+
256+
void createDatabase(String dbName, Map<String, String> properties) {
257+
namespaceClient.createDatabase(dbName, properties);
258+
}
259+
260+
void dropDatabase(String dbName, boolean ifExists, boolean force) {
261+
namespaceClient.dropDatabase(dbName, ifExists, force);
262+
}
263+
264+
void createTable(String dbName, String tableName, Schema schema, Map<String, String> properties) {
265+
namespaceClient.createTable(dbName, tableName, properties, emptyArrowStream(schema));
266+
}
267+
268+
void dropTable(String dbName, String tableName) {
269+
namespaceClient.dropTable(dbName, tableName);
270+
}
271+
272+
void renameTable(String dbName, String oldName, String newName) {
273+
namespaceClient.renameTable(dbName, oldName, newName);
274+
}
275+
276+
void addColumns(String dbName, String tableName, List<AddColumnsEntry> columns) {
277+
namespaceClient.addColumns(dbName, tableName, columns);
278+
}
279+
280+
void alterColumns(String dbName, String tableName, List<AlterColumnsEntry> alterations) {
281+
namespaceClient.alterColumns(dbName, tableName, alterations);
282+
}
283+
284+
void dropColumns(String dbName, String tableName, List<String> columns) {
285+
namespaceClient.dropColumns(dbName, tableName, columns);
286+
}
287+
288+
DescribeTableResponse describeTable(String dbName, String tableName) {
289+
try {
290+
return namespaceClient.describeTable(dbName, tableName, false);
291+
} catch (RuntimeException e) {
292+
throw LanceErrorMessages.failure("Failed to describe Lance table " + dbName + "." + tableName,
293+
e, null, namespaceStorageOptions, catalogSecrets);
294+
}
295+
}
296+
297+
DdlException ddlFailure(String prefix, Throwable error) {
298+
RuntimeException failure = LanceErrorMessages.failure(
299+
prefix, error, null, namespaceStorageOptions, catalogSecrets);
300+
return new DdlException(failure.getMessage(), failure.getCause());
301+
}
302+
303+
private byte[] emptyArrowStream(Schema schema) {
304+
try (BufferAllocator allocator = namespaceAllocator.newChildAllocator(
305+
"lance-create-table", 0, namespaceAllocator.getLimit());
306+
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
307+
ByteArrayOutputStream output = new ByteArrayOutputStream();
308+
ArrowStreamWriter writer = new ArrowStreamWriter(root, null, output)) {
309+
writer.start();
310+
root.setRowCount(0);
311+
writer.writeBatch();
312+
writer.end();
313+
return output.toByteArray();
314+
} catch (IOException e) {
315+
throw new RuntimeException("Failed to serialize Lance table schema", e);
316+
}
317+
}
318+
229319
public LanceTableMetadata loadTableMetadata(String dbName, String tableName) {
230320
return loadTableMetadata(dbName, tableName, Optional.empty());
231321
}

0 commit comments

Comments
 (0)