Skip to content

Commit 86ca033

Browse files
wuwenchiHappenLee
authored andcommitted
[testcase](hive)add exception test for hive txn (apache#33278)
Issue apache#31442 apache#32726 1. add LocalDfsFileSystem to manipulate local files. 2. add HMSCachedClientTest to analog HMS services. 3. add test for rollback commit.
1 parent 6ff952f commit 86ca033

10 files changed

Lines changed: 1033 additions & 290 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java‎

Lines changed: 42 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323

2424
import org.apache.doris.backup.Status;
2525
import org.apache.doris.common.Pair;
26+
import org.apache.doris.fs.FileSystem;
2627
import org.apache.doris.fs.remote.RemoteFile;
27-
import org.apache.doris.fs.remote.RemoteFileSystem;
2828
import org.apache.doris.thrift.THivePartitionUpdate;
2929
import org.apache.doris.thrift.TUpdateMode;
3030
import org.apache.doris.transaction.Transaction;
@@ -47,11 +47,13 @@
4747

4848
import java.util.ArrayList;
4949
import java.util.HashMap;
50+
import java.util.HashSet;
5051
import java.util.List;
5152
import java.util.Map;
5253
import java.util.Objects;
5354
import java.util.Optional;
5455
import java.util.Queue;
56+
import java.util.Set;
5557
import java.util.StringJoiner;
5658
import java.util.concurrent.CompletableFuture;
5759
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -63,7 +65,7 @@
6365
public class HMSTransaction implements Transaction {
6466
private static final Logger LOG = LogManager.getLogger(HMSTransaction.class);
6567
private final HiveMetadataOps hiveOps;
66-
private final RemoteFileSystem fs;
68+
private final FileSystem fs;
6769
private String dbName;
6870
private String tbName;
6971

@@ -115,8 +117,8 @@ public void rollback() {
115117
}
116118

117119
public void finishInsertTable(String dbName, String tbName) {
118-
this.tbName = tbName;
119120
this.dbName = dbName;
121+
this.tbName = tbName;
120122
List<THivePartitionUpdate> mergedPUs = mergePartitions(hivePartitionUpdates);
121123
Table table = getTable(dbName, tbName);
122124
List<Pair<THivePartitionUpdate, HivePartitionStatistics>> insertExistsPartitions = new ArrayList<>();
@@ -226,17 +228,10 @@ public void doCommit() {
226228
}
227229
}
228230

229-
hmsCommitter.waitForAsyncFileSystemTasks();
230-
hmsCommitter.doAddPartitionsTask();
231-
hmsCommitter.doUpdateStatisticsTasks();
231+
hmsCommitter.doCommit();
232232
} catch (Throwable t) {
233233
LOG.warn("Failed to commit for {}.{}, abort it.", dbName, tbName);
234-
hmsCommitter.cancelUnStartedAsyncFileSystemTask();
235-
hmsCommitter.undoUpdateStatisticsTasks();
236-
hmsCommitter.undoAddPartitionsTask();
237-
hmsCommitter.waitForAsyncFileSystemTaskSuppressThrowable();
238-
hmsCommitter.runDirectoryClearUpTasksForAbort();
239-
hmsCommitter.runRenameDirTasksForAbort();
234+
hmsCommitter.rollback();
240235
throw t;
241236
} finally {
242237
hmsCommitter.runClearPathsForFinish();
@@ -354,7 +349,7 @@ private static void addSuppressedExceptions(
354349
}
355350
}
356351

357-
private static class UpdateStatisticsTask {
352+
public static class UpdateStatisticsTask {
358353
private final String dbName;
359354
private final String tableName;
360355
private final Optional<String> partitionName;
@@ -442,7 +437,6 @@ public void run(HiveMetadataOps hiveOps) {
442437
throw t;
443438
}
444439
}
445-
partitions.clear();
446440
}
447441

448442
public List<List<String>> rollback(HiveMetadataOps hiveOps) {
@@ -548,7 +542,7 @@ private void recursiveDeleteItems(Path directory, boolean deleteEmptyDir) {
548542

549543
private DeleteRecursivelyResult recursiveDeleteFiles(Path directory, boolean deleteEmptyDir) {
550544
try {
551-
if (!fs.exists(directory.getName()).ok()) {
545+
if (!fs.exists(directory.toString()).ok()) {
552546
return new DeleteRecursivelyResult(true, ImmutableList.of());
553547
}
554548
} catch (Exception e) {
@@ -561,57 +555,53 @@ private DeleteRecursivelyResult recursiveDeleteFiles(Path directory, boolean del
561555
}
562556

563557
private DeleteRecursivelyResult doRecursiveDeleteFiles(Path directory, boolean deleteEmptyDir) {
564-
List<RemoteFile> remoteFiles = new ArrayList<>();
565-
566-
Status status = fs.list(directory.getName(), remoteFiles);
567-
if (!status.ok()) {
558+
List<RemoteFile> allFiles = new ArrayList<>();
559+
Set<String> allDirs = new HashSet<>();
560+
Status statusFile = fs.listFiles(directory.toString(), allFiles);
561+
Status statusDir = fs.listDirectories(directory.toString(), allDirs);
562+
if (!statusFile.ok() || !statusDir.ok()) {
568563
ImmutableList.Builder<String> notDeletedEligibleItems = ImmutableList.builder();
569564
notDeletedEligibleItems.add(directory + "/*");
570565
return new DeleteRecursivelyResult(false, notDeletedEligibleItems.build());
571566
}
572567

573-
boolean isEmptyDir = true;
574-
List<String> notDeletedEligibleItems = new ArrayList<>();
575-
for (RemoteFile file : remoteFiles) {
576-
if (file.isFile()) {
577-
Path filePath = file.getPath();
578-
isEmptyDir = false;
579-
// TODO Check if this file was created by this query
580-
if (!deleteIfExists(filePath)) {
581-
notDeletedEligibleItems.add(filePath.toString());
582-
}
583-
} else if (file.isDirectory()) {
584-
DeleteRecursivelyResult subResult = doRecursiveDeleteFiles(file.getPath(), deleteEmptyDir);
585-
if (!subResult.dirNotExists()) {
586-
isEmptyDir = false;
587-
}
588-
if (!subResult.getNotDeletedEligibleItems().isEmpty()) {
589-
notDeletedEligibleItems.addAll(subResult.getNotDeletedEligibleItems());
590-
}
591-
} else {
592-
isEmptyDir = false;
593-
notDeletedEligibleItems.add(file.getPath().toString());
568+
boolean allDescendentsDeleted = true;
569+
ImmutableList.Builder<String> notDeletedEligibleItems = ImmutableList.builder();
570+
for (RemoteFile file : allFiles) {
571+
String fileName = file.getName();
572+
if (!deleteIfExists(new Path(fileName))) {
573+
allDescendentsDeleted = false;
574+
notDeletedEligibleItems.add(fileName);
594575
}
595576
}
596577

597-
if (isEmptyDir && deleteEmptyDir) {
598-
Verify.verify(notDeletedEligibleItems.isEmpty());
578+
for (String dir : allDirs) {
579+
DeleteRecursivelyResult subResult = doRecursiveDeleteFiles(new Path(dir), deleteEmptyDir);
580+
if (!subResult.dirNotExists()) {
581+
allDescendentsDeleted = false;
582+
}
583+
if (!subResult.getNotDeletedEligibleItems().isEmpty()) {
584+
notDeletedEligibleItems.addAll(subResult.getNotDeletedEligibleItems());
585+
}
586+
}
587+
588+
if (allDescendentsDeleted && deleteEmptyDir) {
589+
Verify.verify(notDeletedEligibleItems.build().isEmpty());
599590
if (!deleteIfExists(directory)) {
600591
return new DeleteRecursivelyResult(false, ImmutableList.of(directory + "/"));
601592
}
602593
// all items of the location have been deleted.
603594
return new DeleteRecursivelyResult(true, ImmutableList.of());
604595
}
605-
606-
return new DeleteRecursivelyResult(false, notDeletedEligibleItems);
596+
return new DeleteRecursivelyResult(false, notDeletedEligibleItems.build());
607597
}
608598

609599
public boolean deleteIfExists(Path path) {
610-
Status status = fs.delete(path.getName());
600+
Status status = fs.delete(path.toString());
611601
if (status.ok()) {
612602
return true;
613603
}
614-
return !fs.exists(path.getName()).ok();
604+
return !fs.exists(path.toString()).ok();
615605
}
616606

617607
public static class DatabaseTableName {
@@ -1039,9 +1029,6 @@ private void undoUpdateStatisticsTasks() {
10391029
}
10401030

10411031
private void undoAddPartitionsTask() {
1042-
if (addPartitionsTask.isEmpty()) {
1043-
return;
1044-
}
10451032

10461033
HivePartition firstPartition = addPartitionsTask.getPartitions().get(0).getPartition();
10471034
String dbName = firstPartition.getDbName();
@@ -1304,10 +1291,16 @@ private void doUpdateStatisticsTasks() {
13041291
}
13051292
}
13061293

1294+
public void doNothing() {
1295+
// do nothing
1296+
// only for regression test and unit test to throw exception
1297+
}
1298+
13071299
public void doCommit() {
13081300
waitForAsyncFileSystemTasks();
13091301
doAddPartitionsTask();
13101302
doUpdateStatisticsTasks();
1303+
doNothing();
13111304
}
13121305

13131306
public void rollback() {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import org.apache.doris.datasource.jdbc.client.JdbcClient;
3535
import org.apache.doris.datasource.jdbc.client.JdbcClientConfig;
3636
import org.apache.doris.datasource.operations.ExternalMetadataOps;
37-
import org.apache.doris.fs.remote.RemoteFileSystem;
37+
import org.apache.doris.fs.FileSystem;
3838
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
3939

4040
import com.google.common.annotations.VisibleForTesting;
@@ -58,7 +58,7 @@ public class HiveMetadataOps implements ExternalMetadataOps {
5858
private static final Logger LOG = LogManager.getLogger(HiveMetadataOps.class);
5959
private static final int MIN_CLIENT_POOL_SIZE = 8;
6060
private final HMSCachedClient client;
61-
private final RemoteFileSystem fs;
61+
private final FileSystem fs;
6262
private final HMSExternalCatalog catalog;
6363

6464
public HiveMetadataOps(HiveConf hiveConf, JdbcClientConfig jdbcClientConfig, HMSExternalCatalog catalog) {
@@ -75,11 +75,19 @@ public HiveMetadataOps(HMSExternalCatalog catalog, HMSCachedClient client) {
7575
this.fs = new DFSFileSystem(catalog.getProperties());
7676
}
7777

78+
// for test
79+
public HiveMetadataOps(HMSExternalCatalog catalog, HMSCachedClient client, FileSystem fs) {
80+
this.catalog = catalog;
81+
this.client = client;
82+
this.fs = fs;
83+
}
84+
85+
7886
public HMSCachedClient getClient() {
7987
return client;
8088
}
8189

82-
public RemoteFileSystem getFs() {
90+
public FileSystem getFs() {
8391
return fs;
8492
}
8593

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@
2525
import org.apache.doris.fs.remote.RemoteFileSystem;
2626

2727
import com.google.common.base.Preconditions;
28+
import com.google.common.base.Strings;
2829
import com.google.common.collect.ImmutableList;
2930
import com.google.common.collect.ImmutableMap;
3031
import org.apache.commons.lang3.StringUtils;
3132
import org.apache.hadoop.hive.common.FileUtils;
33+
import org.apache.hadoop.hive.common.StatsSetupConst;
3234
import org.apache.hadoop.hive.metastore.api.Database;
3335
import org.apache.hadoop.hive.metastore.api.FieldSchema;
3436
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -167,7 +169,9 @@ public static Map<String, Partition> convertToNamePartitionMap(
167169
ImmutableMap.Builder<String, Partition> resultBuilder = ImmutableMap.builder();
168170
for (Map.Entry<String, List<String>> entry : partitionNameToPartitionValues.entrySet()) {
169171
Partition partition = partitionValuesToPartition.get(entry.getValue());
170-
resultBuilder.put(entry.getKey(), partition);
172+
if (partition != null) {
173+
resultBuilder.put(entry.getKey(), partition);
174+
}
171175
}
172176
return resultBuilder.build();
173177
}
@@ -267,4 +271,63 @@ public static Database toHiveDatabase(HiveDatabaseMetadata hiveDb) {
267271
database.setDescription(hiveDb.getComment());
268272
return database;
269273
}
274+
275+
public static Map<String, String> updateStatisticsParameters(
276+
Map<String, String> parameters,
277+
HiveCommonStatistics statistics) {
278+
HashMap<String, String> result = new HashMap<>(parameters);
279+
280+
result.put(StatsSetupConst.NUM_FILES, String.valueOf(statistics.getFileCount()));
281+
result.put(StatsSetupConst.ROW_COUNT, String.valueOf(statistics.getRowCount()));
282+
result.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(statistics.getTotalFileBytes()));
283+
284+
// CDH 5.16 metastore ignores stats unless STATS_GENERATED_VIA_STATS_TASK is set
285+
// https://github.com/cloudera/hive/blob/cdh5.16.2-release/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java#L227-L231
286+
if (!parameters.containsKey("STATS_GENERATED_VIA_STATS_TASK")) {
287+
result.put("STATS_GENERATED_VIA_STATS_TASK", "workaround for potential lack of HIVE-12730");
288+
}
289+
290+
return result;
291+
}
292+
293+
public static HivePartitionStatistics toHivePartitionStatistics(Map<String, String> params) {
294+
long rowCount = Long.parseLong(params.getOrDefault(StatsSetupConst.ROW_COUNT, "-1"));
295+
long totalSize = Long.parseLong(params.getOrDefault(StatsSetupConst.TOTAL_SIZE, "-1"));
296+
long numFiles = Long.parseLong(params.getOrDefault(StatsSetupConst.NUM_FILES, "-1"));
297+
return HivePartitionStatistics.fromCommonStatistics(rowCount, numFiles, totalSize);
298+
}
299+
300+
public static Partition toMetastoreApiPartition(HivePartitionWithStatistics partitionWithStatistics) {
301+
Partition partition =
302+
toMetastoreApiPartition(partitionWithStatistics.getPartition());
303+
partition.setParameters(updateStatisticsParameters(
304+
partition.getParameters(), partitionWithStatistics.getStatistics().getCommonStatistics()));
305+
return partition;
306+
}
307+
308+
public static Partition toMetastoreApiPartition(HivePartition hivePartition) {
309+
Partition result = new Partition();
310+
result.setDbName(hivePartition.getDbName());
311+
result.setTableName(hivePartition.getTblName());
312+
result.setValues(hivePartition.getPartitionValues());
313+
result.setSd(makeStorageDescriptorFromHivePartition(hivePartition));
314+
result.setParameters(hivePartition.getParameters());
315+
return result;
316+
}
317+
318+
public static StorageDescriptor makeStorageDescriptorFromHivePartition(HivePartition partition) {
319+
SerDeInfo serdeInfo = new SerDeInfo();
320+
serdeInfo.setName(partition.getTblName());
321+
serdeInfo.setSerializationLib(partition.getSerde());
322+
323+
StorageDescriptor sd = new StorageDescriptor();
324+
sd.setLocation(Strings.emptyToNull(partition.getPath()));
325+
sd.setCols(partition.getColumns());
326+
sd.setSerdeInfo(serdeInfo);
327+
sd.setInputFormat(partition.getInputFormat());
328+
sd.setOutputFormat(partition.getOutputFormat());
329+
sd.setParameters(ImmutableMap.of());
330+
331+
return sd;
332+
}
270333
}

0 commit comments

Comments
 (0)