Skip to content

Commit a5ac554

Browse files
wsjzHappenLee
authored andcommitted
[feature](insert)support external hive truncate table DDL (apache#36801)
## Proposed changes Issue Number: apache#31442 1. support hive3 'truncate table' 2. forbiden hive2 'truncate table' because of it is not supported by HMS Client API on hive2
1 parent c9a2ce3 commit a5ac554

13 files changed

Lines changed: 210 additions & 6 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.doris.common.ErrorReport;
2424
import org.apache.doris.common.UserException;
2525
import org.apache.doris.common.util.InternalDatabaseUtil;
26-
import org.apache.doris.common.util.Util;
2726
import org.apache.doris.mysql.privilege.PrivPredicate;
2827
import org.apache.doris.qe.ConnectContext;
2928

@@ -44,8 +43,6 @@ public TableRef getTblRef() {
4443
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
4544
super.analyze(analyzer);
4645
tblRef.getName().analyze(analyzer);
47-
// disallow external catalog
48-
Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
4946

5047
if (tblRef.hasExplicitAlias()) {
5148
throw new AnalysisException("Not support truncate table with alias");

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5577,8 +5577,10 @@ public String dumpImage() {
55775577
* otherwise, it will only truncate those specified partitions.
55785578
*
55795579
*/
5580-
public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException {
5581-
getInternalCatalog().truncateTable(truncateTableStmt);
5580+
public void truncateTable(TruncateTableStmt stmt) throws DdlException {
5581+
CatalogIf<?> catalogIf = catalogMgr.getCatalogOrException(stmt.getTblRef().getName().getCtl(),
5582+
catalog -> new DdlException(("Unknown catalog " + catalog)));
5583+
catalogIf.truncateTable(stmt);
55825584
}
55835585

55845586
public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.analysis.DropDbStmt;
2323
import org.apache.doris.analysis.DropTableStmt;
2424
import org.apache.doris.analysis.TableName;
25+
import org.apache.doris.analysis.TruncateTableStmt;
2526
import org.apache.doris.catalog.DatabaseIf;
2627
import org.apache.doris.catalog.Env;
2728
import org.apache.doris.catalog.TableIf;
@@ -194,4 +195,6 @@ default CatalogLog constructEditLog() {
194195
boolean createTable(CreateTableStmt stmt) throws UserException;
195196

196197
void dropTable(DropTableStmt stmt) throws DdlException;
198+
199+
void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException;
197200
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.apache.doris.analysis.DropDbStmt;
2323
import org.apache.doris.analysis.DropTableStmt;
2424
import org.apache.doris.analysis.TableName;
25+
import org.apache.doris.analysis.TableRef;
26+
import org.apache.doris.analysis.TruncateTableStmt;
2527
import org.apache.doris.catalog.DatabaseIf;
2628
import org.apache.doris.catalog.Env;
2729
import org.apache.doris.catalog.InfoSchemaDb;
@@ -832,4 +834,25 @@ public boolean enableAutoAnalyze() {
832834
}
833835
return ret;
834836
}
837+
838+
@Override
839+
public void truncateTable(TruncateTableStmt stmt) throws DdlException {
840+
makeSureInitialized();
841+
if (metadataOps == null) {
842+
throw new UnsupportedOperationException("Truncate table not supported in " + getName());
843+
}
844+
try {
845+
TableRef tableRef = stmt.getTblRef();
846+
TableName tableName = tableRef.getName();
847+
// delete all table data if null
848+
List<String> partitions = null;
849+
if (tableRef.getPartitionNames() != null) {
850+
partitions = tableRef.getPartitionNames().getPartitionNames();
851+
}
852+
metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions);
853+
} catch (Exception e) {
854+
LOG.warn("Failed to drop a table", e);
855+
throw e;
856+
}
857+
}
835858
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ void acquireSharedLock(String queryId, long txnId, String user, TableName tblNam
9494

9595
void dropTable(String dbName, String tableName);
9696

97+
void truncateTable(String dbName, String tblName, List<String> partitions);
98+
9799
void createTable(TableMetadata catalogTable, boolean ignoreIfExists);
98100

99101
void updateTableStatistics(

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,22 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
251251
}
252252
}
253253

254+
@Override
255+
public void truncateTable(String dbName, String tblName, List<String> partitions) throws DdlException {
256+
ExternalDatabase<?> db = catalog.getDbNullable(dbName);
257+
if (db == null) {
258+
throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName());
259+
}
260+
try {
261+
client.truncateTable(dbName, tblName, partitions);
262+
} catch (Exception e) {
263+
throw new DdlException(e.getMessage(), e);
264+
}
265+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(catalog.getId(), dbName, tblName);
266+
db.setLastUpdateTime(System.currentTimeMillis());
267+
db.setUnInitialized(true);
268+
}
269+
254270
@Override
255271
public List<String> listTableNames(String dbName) {
256272
return client.getAllTables(dbName);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,10 @@ public void dropDatabase(String dbName) {
546546
throw new NotImplementedException("PostgreSQL dropDatabase not implemented");
547547
}
548548

549+
public void truncateTable(String dbName, String tblName, List<String> partitions) {
550+
throw new NotImplementedException("PostgreSQL truncateTable not implemented");
551+
}
552+
549553
public void createTable(TableMetadata hiveTable, boolean ignoreIfExists) {
550554
throw new NotImplementedException("PostgreSQL createTable not implemented");
551555
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,23 @@ public void dropTable(String dbName, String tblName) {
244244
}
245245
}
246246

247+
@Override
248+
public void truncateTable(String dbName, String tblName, List<String> partitions) {
249+
try (ThriftHMSClient client = getClient()) {
250+
try {
251+
ugiDoAs(() -> {
252+
client.client.truncateTable(dbName, tblName, partitions);
253+
return null;
254+
});
255+
} catch (Exception e) {
256+
client.setThrowable(e);
257+
throw e;
258+
}
259+
} catch (Exception e) {
260+
throw new HMSClientException("failed to truncate table %s in db %s.", e, tblName, dbName);
261+
}
262+
}
263+
247264
@Override
248265
public boolean tableExists(String dbName, String tblName) {
249266
try (ThriftHMSClient client = getClient()) {
@@ -272,7 +289,7 @@ public List<Partition> listPartitions(String dbName, String tblName) {
272289
throw e;
273290
}
274291
} catch (Exception e) {
275-
throw new HMSClientException("failed to check if table %s in db %s exists", e, tblName, dbName);
292+
throw new HMSClientException("failed to list partitions in table '%s.%s'.", e, dbName, tblName);
276293
}
277294
}
278295

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ public List<String> listDatabaseNames() {
8787
.collect(Collectors.toList());
8888
}
8989

90+
9091
@Override
9192
public List<String> listTableNames(String dbName) {
9293
List<TableIdentifier> tableIdentifiers = catalog.listTables(Namespace.of(dbName));
@@ -178,4 +179,8 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
178179
db.setUnInitialized(true);
179180
}
180181

182+
@Override
183+
public void truncateTable(String dbName, String tblName, List<String> partitions) {
184+
throw new UnsupportedOperationException("Truncate Iceberg table is not supported.");
185+
}
181186
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ public interface ExternalMetadataOps {
6060
*/
6161
void dropTable(DropTableStmt stmt) throws DdlException;
6262

63+
/**
64+
*
65+
* @param dbName
66+
* @param tblName
67+
* @param partitions
68+
*/
69+
void truncateTable(String dbName, String tblName, List<String> partitions) throws DdlException;
70+
6371
/**
6472
*
6573
* @return

0 commit comments

Comments
 (0)