Skip to content

Commit bd2e7d0

Browse files
wuwenchiDoris-Extras
authored andcommitted
[feature](hive)support insert overwrite (#32610)
support insert overwrite for unpartitioned table and partitioned table. issue: #31442
1 parent ac8ba43 commit bd2e7d0

4 files changed

Lines changed: 166 additions & 32 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java‎

Lines changed: 91 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ public class HMSCommitter {
8080
private final Queue<DirectoryCleanUpTask> directoryCleanUpTasksForAbort = new ConcurrentLinkedQueue<>();
8181
// when aborted, we need restore directory
8282
private final List<RenameDirectoryTask> renameDirectoryTasksForAbort = new ArrayList<>();
83+
// when finished, we need clear some directories
84+
private final List<String> clearDirsForFinish = new ArrayList<>();
8385
Executor fileSystemExecutor = Executors.newFixedThreadPool(16);
8486

8587
public HMSCommitter(HiveMetadataOps hiveOps, RemoteFileSystem fs, Table table) {
@@ -105,6 +107,8 @@ public void commit(List<THivePartitionUpdate> hivePUs) {
105107
t.addSuppressed(new Exception("Failed to roll back after commit failure", e));
106108
}
107109
throw t;
110+
} finally {
111+
runClearPathsForFinish();
108112
}
109113
}
110114

@@ -250,7 +254,38 @@ public void prepareAppendTable(THivePartitionUpdate pu, HivePartitionStatistics
250254
}
251255

252256
public void prepareOverwriteTable(THivePartitionUpdate pu, HivePartitionStatistics ps) {
253-
257+
String targetPath = pu.getLocation().getTargetPath();
258+
String writePath = pu.getLocation().getWritePath();
259+
if (!targetPath.equals(writePath)) {
260+
Path path = new Path(targetPath);
261+
String oldTablePath = new Path(path.getParent(), "_temp_" + path.getName()).toString();
262+
Status status = fs.renameDir(
263+
targetPath,
264+
oldTablePath,
265+
() -> renameDirectoryTasksForAbort.add(new RenameDirectoryTask(oldTablePath, targetPath)));
266+
if (!status.ok()) {
267+
throw new RuntimeException(
268+
"Error to rename dir from " + targetPath + " to " + oldTablePath + status.getErrMsg());
269+
}
270+
clearDirsForFinish.add(oldTablePath);
271+
272+
status = fs.renameDir(
273+
writePath,
274+
targetPath,
275+
() -> directoryCleanUpTasksForAbort.add(new DirectoryCleanUpTask(targetPath, true)));
276+
if (!status.ok()) {
277+
throw new RuntimeException(
278+
"Error to rename dir from " + writePath + " to " + targetPath + ":" + status.getErrMsg());
279+
}
280+
}
281+
updateStatisticsTasks.add(
282+
new UpdateStatisticsTask(
283+
table.getDbName(),
284+
table.getTableName(),
285+
Optional.empty(),
286+
ps,
287+
false
288+
));
254289
}
255290

256291
public void prepareCreateNewPartition(THivePartitionUpdate pu, HivePartitionStatistics ps) {
@@ -335,7 +370,38 @@ public void prepareInsertExistPartition(List<Pair<THivePartitionUpdate, HivePart
335370

336371

337372
public void prepareOverwritePartition(THivePartitionUpdate pu, HivePartitionStatistics ps) {
373+
String targetPath = pu.getLocation().getTargetPath();
374+
String writePath = pu.getLocation().getWritePath();
375+
if (!targetPath.equals(writePath)) {
376+
Path path = new Path(targetPath);
377+
String oldPartitionPath = new Path(path.getParent(), "_temp_" + path.getName()).toString();
378+
Status status = fs.renameDir(
379+
targetPath,
380+
oldPartitionPath,
381+
() -> renameDirectoryTasksForAbort.add(new RenameDirectoryTask(oldPartitionPath, targetPath)));
382+
if (!status.ok()) {
383+
throw new RuntimeException(
384+
"Error to rename dir from " + targetPath + " to " + oldPartitionPath + ":" + status.getErrMsg());
385+
}
386+
clearDirsForFinish.add(oldPartitionPath);
338387

388+
status = fs.renameDir(
389+
writePath,
390+
targetPath,
391+
() -> directoryCleanUpTasksForAbort.add(new DirectoryCleanUpTask(targetPath, true)));
392+
if (!status.ok()) {
393+
throw new RuntimeException(
394+
"Error to rename dir from " + writePath + " to " + targetPath + ":" + status.getErrMsg());
395+
}
396+
}
397+
updateStatisticsTasks.add(
398+
new UpdateStatisticsTask(
399+
table.getDbName(),
400+
table.getTableName(),
401+
Optional.of(pu.getName()),
402+
ps,
403+
false
404+
));
339405
}
340406

341407

@@ -481,7 +547,7 @@ public void run(HiveMetadataOps hiveOps) {
481547
createdPartitionValues.add(partition.getPartition().getPartitionValues());
482548
}
483549
} catch (Throwable t) {
484-
LOG.error("Failed to add partition", t);
550+
LOG.warn("Failed to add partition", t);
485551
throw t;
486552
}
487553
}
@@ -583,18 +649,38 @@ public String toString() {
583649
}
584650

585651
private void runRenameDirTasksForAbort() {
586-
// TODO abort
652+
Status status;
653+
for (RenameDirectoryTask task : renameDirectoryTasksForAbort) {
654+
status = fs.exists(task.getRenameFrom());
655+
if (status.ok()) {
656+
status = fs.renameDir(task.getRenameFrom(), task.getRenameTo(), () -> {});
657+
if (!status.ok()) {
658+
LOG.warn("Failed to abort rename dir from {} to {}:{}",
659+
task.getRenameFrom(), task.getRenameTo(), status.getErrMsg());
660+
}
661+
}
662+
}
663+
}
664+
665+
private void runClearPathsForFinish() {
666+
Status status;
667+
for (String path : clearDirsForFinish) {
668+
status = fs.delete(path);
669+
if (!status.ok()) {
670+
LOG.warn("Failed to recursively delete path {}:{}", path, status.getErrCode());
671+
}
672+
}
587673
}
588674

589675

590676
private void recursiveDeleteItems(Path directory, boolean deleteEmptyDir) {
591677
DeleteRecursivelyResult deleteResult = recursiveDeleteFiles(directory, deleteEmptyDir);
592678

593679
if (!deleteResult.getNotDeletedEligibleItems().isEmpty()) {
594-
LOG.error("Failed to delete directory {}. Some eligible items can't be deleted: {}.",
680+
LOG.warn("Failed to delete directory {}. Some eligible items can't be deleted: {}.",
595681
directory.toString(), deleteResult.getNotDeletedEligibleItems());
596682
} else if (deleteEmptyDir && !deleteResult.dirNotExists()) {
597-
LOG.error("Failed to delete directory {} due to dir isn't empty", directory.toString());
683+
LOG.warn("Failed to delete directory {} due to dir isn't empty", directory.toString());
598684
}
599685
}
600686

‎fe/fe-core/src/main/java/org/apache/doris/fs/FileSystem.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ public interface FileSystem {
4949

5050
Status rename(String origFilePath, String destFilePath);
5151

52+
default Status renameDir(String origFilePath,
53+
String destFilePath,
54+
Runnable runWhenPathNotExist) {
55+
throw new UnsupportedOperationException("Unsupported operation rename dir on current file system.");
56+
}
57+
5258
default void asyncRename(Executor executor,
5359
List<CompletableFuture<?>> renameFileFutures,
5460
AtomicBoolean cancelled,

‎fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,28 @@ public void asyncRename(
412412
}
413413
}
414414

415+
public Status renameDir(String origFilePath,
416+
String destFilePath,
417+
Runnable runWhenPathNotExist) {
418+
Status status = exists(destFilePath);
419+
if (status.ok()) {
420+
throw new RuntimeException("Destination directory already exists: " + destFilePath);
421+
}
422+
423+
String targetParent = new Path(destFilePath).getParent().toString();
424+
status = exists(targetParent);
425+
if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) {
426+
status = makeDir(targetParent);
427+
}
428+
if (!status.ok()) {
429+
throw new RuntimeException(status.getErrMsg());
430+
}
431+
432+
runWhenPathNotExist.run();
433+
434+
return rename(origFilePath, destFilePath);
435+
}
436+
415437
@Override
416438
public void asyncRenameDir(Executor executor,
417439
List<CompletableFuture<?>> renameFileFutures,
@@ -423,23 +445,7 @@ public void asyncRenameDir(Executor executor,
423445
if (cancelled.get()) {
424446
return;
425447
}
426-
427-
Status status = exists(destFilePath);
428-
if (status.ok()) {
429-
throw new RuntimeException("Destination directory already exists: " + destFilePath);
430-
}
431-
432-
String targetParent = new Path(destFilePath).getParent().toString();
433-
status = exists(targetParent);
434-
if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) {
435-
makeDir(targetParent);
436-
} else if (!status.ok()) {
437-
throw new RuntimeException(status.getErrMsg());
438-
}
439-
440-
runWhenPathNotExist.run();
441-
442-
status = rename(origFilePath, destFilePath);
448+
Status status = renameDir(origFilePath, destFilePath, runWhenPathNotExist);
443449
if (!status.ok()) {
444450
throw new RuntimeException(status.getErrMsg());
445451
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java‎

Lines changed: 46 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -134,20 +134,27 @@ public void testAppendPartitionForUnPartitionedTable() {
134134
pus.add(createRandomAppend(""));
135135
hmsOps.commit(dbName, tbWithoutPartition, pus);
136136
Table table = hmsClient.getTable(dbName, tbWithoutPartition);
137-
Assert.assertEquals(3, Long.parseLong(table.getParameters().get("numRows")));
137+
assertNumRows(3, table);
138138

139139
List<THivePartitionUpdate> pus2 = new ArrayList<>();
140140
pus2.add(createRandomAppend(""));
141141
pus2.add(createRandomAppend(""));
142142
pus2.add(createRandomAppend(""));
143143
hmsOps.commit(dbName, tbWithoutPartition, pus2);
144144
table = hmsClient.getTable(dbName, tbWithoutPartition);
145-
Assert.assertEquals(6, Long.parseLong(table.getParameters().get("numRows")));
145+
assertNumRows(6, table);
146146
}
147147

148148
@Test
149149
public void testOverwritePartitionForUnPartitionedTable() {
150-
// TODO
150+
testAppendPartitionForUnPartitionedTable();
151+
List<THivePartitionUpdate> pus = new ArrayList<>();
152+
pus.add(createRandomOverwrite(""));
153+
pus.add(createRandomOverwrite(""));
154+
pus.add(createRandomOverwrite(""));
155+
hmsOps.commit(dbName, tbWithoutPartition, pus);
156+
Table table = hmsClient.getTable(dbName, tbWithoutPartition);
157+
assertNumRows(3, table);
151158
}
152159

153160
@Test
@@ -162,11 +169,11 @@ public void testNewPartitionForPartitionedTable() {
162169
hmsOps.commit(dbName, tbWithPartition, pus);
163170

164171
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
165-
Assert.assertEquals(3, Long.parseLong(pa.getParameters().get("numRows")));
172+
assertNumRows(3, pa);
166173
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
167-
Assert.assertEquals(2, Long.parseLong(pb.getParameters().get("numRows")));
174+
assertNumRows(2, pb);
168175
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
169-
Assert.assertEquals(1, Long.parseLong(pc.getParameters().get("numRows")));
176+
assertNumRows(1, pc);
170177
}
171178

172179
@Test
@@ -183,11 +190,28 @@ public void testAppendPartitionForPartitionedTable() {
183190
hmsOps.commit(dbName, tbWithPartition, pus);
184191

185192
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
186-
Assert.assertEquals(6, Long.parseLong(pa.getParameters().get("numRows")));
193+
assertNumRows(6, pa);
194+
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
195+
assertNumRows(4, pb);
196+
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
197+
assertNumRows(2, pc);
198+
}
199+
200+
@Test
201+
public void testOverwritePartitionForPartitionedTable() {
202+
testAppendPartitionForPartitionedTable();
203+
List<THivePartitionUpdate> pus = new ArrayList<>();
204+
pus.add(createRandomOverwrite("a"));
205+
pus.add(createRandomOverwrite("b"));
206+
pus.add(createRandomOverwrite("c"));
207+
hmsOps.commit(dbName, tbWithPartition, pus);
208+
209+
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
210+
assertNumRows(1, pa);
187211
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
188-
Assert.assertEquals(4, Long.parseLong(pb.getParameters().get("numRows")));
212+
assertNumRows(1, pb);
189213
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
190-
Assert.assertEquals(2, Long.parseLong(pc.getParameters().get("numRows")));
214+
assertNumRows(1, pc);
191215
}
192216

193217
@Test
@@ -201,7 +225,7 @@ public void testNewManyPartitionForPartitionedTable() {
201225
hmsOps.commit(dbName, tbWithPartition, pus);
202226
for (int i = 0; i < nums; i++) {
203227
Partition p = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("" + i));
204-
Assert.assertEquals(1, Long.parseLong(p.getParameters().get("numRows")));
228+
assertNumRows(1, p);
205229
}
206230

207231
try {
@@ -211,6 +235,14 @@ public void testNewManyPartitionForPartitionedTable() {
211235
}
212236
}
213237

238+
public void assertNumRows(long expected, Partition p) {
239+
Assert.assertEquals(expected, Long.parseLong(p.getParameters().get("numRows")));
240+
}
241+
242+
public void assertNumRows(long expected, Table t) {
243+
Assert.assertEquals(expected, Long.parseLong(t.getParameters().get("numRows")));
244+
}
245+
214246
public THivePartitionUpdate genOnePartitionUpdate(String partitionValue, TUpdateMode mode) {
215247

216248
String uuid = UUID.randomUUID().toString();
@@ -242,4 +274,8 @@ public THivePartitionUpdate createRandomNew(String partition) {
242274
public THivePartitionUpdate createRandomAppend(String partition) {
243275
return genOnePartitionUpdate("c3=" + partition, TUpdateMode.APPEND);
244276
}
277+
278+
public THivePartitionUpdate createRandomOverwrite(String partition) {
279+
return genOnePartitionUpdate("c3=" + partition, TUpdateMode.OVERWRITE);
280+
}
245281
}

0 commit comments

Comments
 (0)