Skip to content

Commit b2e869c

Browse files
authored
[feature](multi-catalog)support hms catalog create and drop table/db (#30198)
1. rename old create/drop table to add/removeMemoryTable 2. add new create/drop table/db method 3. support hms catalog create/drop table/db
1 parent f2aee04 commit b2e869c

71 files changed

Lines changed: 1478 additions & 362 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2216,6 +2216,21 @@ public class Config extends ConfigBase {
22162216
"Sample size for hive row count estimation."})
22172217
public static int hive_stats_partition_sample_size = 3000;
22182218

2219+
@ConfField(mutable = true, masterOnly = true, description = {
2220+
"Hive创建外部表默认指定的input format",
2221+
"Default hive input format for creating table."})
2222+
public static String hive_default_input_format = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
2223+
2224+
@ConfField(mutable = true, masterOnly = true, description = {
2225+
"Hive创建外部表默认指定的output format",
2226+
"Default hive output format for creating table."})
2227+
public static String hive_default_output_format = "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat";
2228+
2229+
@ConfField(mutable = true, masterOnly = true, description = {
2230+
"Hive创建外部表默认指定的SerDe类",
2231+
"Default hive serde class for creating table."})
2232+
public static String hive_default_serde = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
2233+
22192234
@ConfField
22202235
public static int statistics_sql_parallel_exec_instance_num = 1;
22212236

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -411,8 +411,8 @@ private void processModifyEngineInternal(Database db, Table externalTable,
411411
}
412412
odbcTable.writeLock();
413413
try {
414-
db.dropTable(mysqlTable.getName());
415-
db.createTable(odbcTable);
414+
db.unregisterTable(mysqlTable.getName());
415+
db.registerTable(odbcTable);
416416
if (!isReplay) {
417417
ModifyTableEngineOperationLog log = new ModifyTableEngineOperationLog(db.getId(),
418418
externalTable.getId(), prop);
@@ -591,17 +591,17 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne
591591
String newTblName = newTbl.getName();
592592

593593
// drop origin table and new table
594-
db.dropTable(oldTblName);
595-
db.dropTable(newTblName);
594+
db.unregisterTable(oldTblName);
595+
db.unregisterTable(newTblName);
596596

597597
// rename new table name to origin table name and add it to database
598598
newTbl.checkAndSetName(oldTblName, false);
599-
db.createTable(newTbl);
599+
db.registerTable(newTbl);
600600

601601
if (swapTable) {
602602
// rename origin table name to new table name and add it to database
603603
origTable.checkAndSetName(newTblName, false);
604-
db.createTable(origTable);
604+
db.registerTable(origTable);
605605
} else {
606606
// not swap, the origin table is not used anymore, need to drop all its tablets.
607607
Env.getCurrentEnv().onEraseOlapTable(origTable, isReplay);
@@ -633,8 +633,8 @@ private void modifyViewDef(Database db, View view, String inlineViewDef, long sq
633633
}
634634
view.setNewFullSchema(newFullSchema);
635635
String viewName = view.getName();
636-
db.dropTable(viewName);
637-
db.createTable(view);
636+
db.unregisterTable(viewName);
637+
db.registerTable(view);
638638

639639
AlterViewInfo alterViewInfo = new AlterViewInfo(db.getId(), view.getId(),
640640
inlineViewDef, newFullSchema, sqlMode);
@@ -669,8 +669,8 @@ public void replayModifyViewDef(AlterViewInfo alterViewInfo) throws MetaNotFound
669669
}
670670
view.setNewFullSchema(newFullSchema);
671671

672-
db.dropTable(viewName);
673-
db.createTable(view);
672+
db.unregisterTable(viewName);
673+
db.registerTable(view);
674674

675675
LOG.info("replay modify view[{}] definition to {}", viewName, inlineViewDef);
676676
} finally {

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import org.apache.doris.common.util.ParseUtil;
4040
import org.apache.doris.common.util.PrintableMap;
4141
import org.apache.doris.common.util.PropertyAnalyzer;
42-
import org.apache.doris.common.util.Util;
4342
import org.apache.doris.datasource.es.EsUtil;
4443
import org.apache.doris.mysql.privilege.PrivPredicate;
4544
import org.apache.doris.qe.ConnectContext;
@@ -282,16 +281,13 @@ public void analyze(Analyzer analyzer) throws UserException {
282281
// FIXME: MOW is not supported in cloud mode yet.
283282
properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false");
284283
}
285-
286284
if (Strings.isNullOrEmpty(engineName) || engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
287285
this.properties = maybeRewriteByAutoBucket(distributionDesc, properties);
288286
}
289287

290288
super.analyze(analyzer);
291289
tableName.analyze(analyzer);
292290
FeNameFormat.checkTableName(tableName.getTbl());
293-
// disallow external catalog
294-
Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName());
295291
InternalDatabaseUtil.checkDatabase(tableName.getDb(), ConnectContext.get());
296292
if (!Env.getCurrentEnv().getAccessManager()
297293
.checkTblPriv(ConnectContext.get(), tableName.getDb(), tableName.getTbl(), PrivPredicate.CREATE)) {

‎fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStmt.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.doris.common.ErrorReport;
2323
import org.apache.doris.common.UserException;
2424
import org.apache.doris.common.util.InternalDatabaseUtil;
25-
import org.apache.doris.common.util.Util;
2625
import org.apache.doris.mysql.privilege.PrivPredicate;
2726
import org.apache.doris.qe.ConnectContext;
2827

@@ -84,8 +83,6 @@ public void analyze(Analyzer analyzer) throws UserException {
8483
tableName.setDb(analyzer.getDefaultDb());
8584
}
8685
tableName.analyze(analyzer);
87-
// disallow external catalog
88-
Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName());
8986
InternalDatabaseUtil.checkDatabase(tableName.getDb(), ConnectContext.get());
9087
// check access
9188
if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), tableName.getDb(),

‎fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -905,7 +905,7 @@ private void checkAndPrepareMeta() {
905905
}
906906
tbl.writeLock();
907907
try {
908-
if (!db.createTable(tbl)) {
908+
if (!db.registerTable(tbl)) {
909909
status = new Status(ErrCode.COMMON_ERROR, "Table " + tbl.getName()
910910
+ " already exist in db: " + db.getFullName());
911911
return;
@@ -1286,7 +1286,7 @@ private void replayCheckAndPrepareMeta() {
12861286
db.writeLock();
12871287
restoreTbl.writeLock();
12881288
try {
1289-
db.createTable(restoreTbl);
1289+
db.registerTable(restoreTbl);
12901290
} finally {
12911291
restoreTbl.writeUnlock();
12921292
db.writeUnlock();
@@ -1936,7 +1936,7 @@ public void cancelInternal(boolean isReplay) {
19361936
}
19371937
}
19381938
}
1939-
db.dropTable(restoreTbl.getName());
1939+
db.unregisterTable(restoreTbl.getName());
19401940
} finally {
19411941
restoreTbl.writeUnlock();
19421942
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -644,7 +644,7 @@ private void recoverAllTables(RecycleDatabaseInfo dbInfo) throws DdlException {
644644
}
645645

646646
Table table = tableInfo.getTable();
647-
db.createTable(table);
647+
db.registerTable(table);
648648
LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName());
649649
iterator.remove();
650650
idToRecycleTime.remove(table.getId());
@@ -739,7 +739,7 @@ private synchronized boolean innerRecoverTable(Database db, Table table, String
739739
}
740740
}
741741

742-
db.createTable(table);
742+
db.registerTable(table);
743743
if (isReplay) {
744744
iterator.remove();
745745
} else {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -416,25 +416,26 @@ public Pair<Boolean, Boolean> createTableWithLock(
416416
}
417417
}
418418

419-
public boolean createTable(Table table) {
419+
public boolean registerTable(TableIf table) {
420420
boolean result = true;
421-
table.setQualifiedDbName(fullQualifiedName);
422-
String tableName = table.getName();
421+
Table olapTable = (Table) table;
422+
olapTable.setQualifiedDbName(fullQualifiedName);
423+
String tableName = olapTable.getName();
423424
if (Env.isStoredTableNamesLowerCase()) {
424425
tableName = tableName.toLowerCase();
425426
}
426427
if (isTableExist(tableName)) {
427428
result = false;
428429
} else {
429-
idToTable.put(table.getId(), table);
430-
nameToTable.put(table.getName(), table);
430+
idToTable.put(olapTable.getId(), olapTable);
431+
nameToTable.put(olapTable.getName(), olapTable);
431432
lowerCaseToTableName.put(tableName.toLowerCase(), tableName);
432433
}
433-
table.unmarkDropped();
434+
olapTable.unmarkDropped();
434435
return result;
435436
}
436437

437-
public void dropTable(String tableName) {
438+
public void unregisterTable(String tableName) {
438439
if (Env.isStoredTableNamesLowerCase()) {
439440
tableName = tableName.toLowerCase();
440441
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,18 @@ default OlapTable getOlapTableOrAnalysisException(String tableName) throws Analy
263263
return (OlapTable) table;
264264
}
265265

266-
void dropTable(String tableName);
266+
/**
267+
* register table to memory
268+
* @param table created table
269+
* @return true if add to memory
270+
*/
271+
boolean registerTable(TableIf table);
272+
273+
/**
274+
* unregister table from memory
275+
* @param tableName table name
276+
*/
277+
void unregisterTable(String tableName);
267278

268279
CatalogIf getCatalog();
269280

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2956,7 +2956,7 @@ public Frontend getFeByName(String name) {
29562956

29572957
// The interface which DdlExecutor needs.
29582958
public void createDb(CreateDbStmt stmt) throws DdlException {
2959-
getInternalCatalog().createDb(stmt);
2959+
getCurrentCatalog().createDb(stmt);
29602960
}
29612961

29622962
// For replay edit log, need't lock metadata
@@ -2969,7 +2969,7 @@ public void replayCreateDb(Database db) {
29692969
}
29702970

29712971
public void dropDb(DropDbStmt stmt) throws DdlException {
2972-
getInternalCatalog().dropDb(stmt);
2972+
getCurrentCatalog().dropDb(stmt);
29732973
}
29742974

29752975
public void replayDropDb(String dbName, boolean isForceDrop, Long recycleTime) throws DdlException {
@@ -3041,7 +3041,7 @@ public void replayRenameDatabase(String dbName, String newDbName) {
30413041
* 11. add this table to ColocateGroup if necessary
30423042
*/
30433043
public void createTable(CreateTableStmt stmt) throws UserException {
3044-
getInternalCatalog().createTable(stmt);
3044+
getCurrentCatalog().createTable(stmt);
30453045
}
30463046

30473047
public void createTableLike(CreateTableLikeStmt stmt) throws DdlException {
@@ -3671,7 +3671,7 @@ public void replayAlterExternalTableSchema(String dbName, String tableName, List
36713671

36723672
// Drop table
36733673
public void dropTable(DropTableStmt stmt) throws DdlException {
3674-
getInternalCatalog().dropTable(stmt);
3674+
getCurrentCatalog().dropTable(stmt);
36753675
}
36763676

36773677
public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, boolean isReplay,
@@ -4320,8 +4320,8 @@ public void renameTable(Database db, Table table, String newTableName) throws Dd
43204320
table.setName(newTableName);
43214321
}
43224322

4323-
db.dropTable(oldTableName);
4324-
db.createTable(table);
4323+
db.unregisterTable(oldTableName);
4324+
db.registerTable(table);
43254325

43264326
TableInfo tableInfo = TableInfo.createForTableRename(db.getId(), table.getId(), newTableName);
43274327
editLog.logTableRename(tableInfo);
@@ -4353,9 +4353,9 @@ public void replayRenameTable(TableInfo tableInfo) throws MetaNotFoundException
43534353
table.writeLock();
43544354
try {
43554355
String tableName = table.getName();
4356-
db.dropTable(tableName);
4356+
db.unregisterTable(tableName);
43574357
table.setName(newTableName);
4358-
db.createTable(table);
4358+
db.registerTable(table);
43594359
LOG.info("replay rename table[{}] to {}", tableName, newTableName);
43604360
} finally {
43614361
table.writeUnlock();

‎fe/fe-core/src/main/java/org/apache/doris/catalog/InfoSchemaDb.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,12 @@ public InfoSchemaDb() {
2929
@Override
3030
protected void initTables() {
3131
for (Table table : SchemaTable.TABLE_MAP.values()) {
32-
super.createTable(table);
32+
super.registerTable(table);
3333
}
3434
}
3535

3636
@Override
37-
public boolean createTable(Table table) {
37+
public boolean registerTable(TableIf table) {
3838
return false;
3939
}
4040
}

0 commit comments

Comments
 (0)