Skip to content

Commit 4078ceb

Browse files
authored
[feature](hive)support insert overwrite (#32610)
support insert overwrite for unpartitioned table and partitioned table. issue: #31442
1 parent df56ab3 commit 4078ceb

4 files changed

Lines changed: 166 additions & 32 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java‎

Lines changed: 91 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ public class HMSCommitter {
8080
private final Queue<DirectoryCleanUpTask> directoryCleanUpTasksForAbort = new ConcurrentLinkedQueue<>();
8181
// when aborted, we need restore directory
8282
private final List<RenameDirectoryTask> renameDirectoryTasksForAbort = new ArrayList<>();
83+
// when finished, we need clear some directories
84+
private final List<String> clearDirsForFinish = new ArrayList<>();
8385
Executor fileSystemExecutor = Executors.newFixedThreadPool(16);
8486

8587
public HMSCommitter(HiveMetadataOps hiveOps, RemoteFileSystem fs, Table table) {
@@ -105,6 +107,8 @@ public void commit(List<THivePartitionUpdate> hivePUs) {
105107
t.addSuppressed(new Exception("Failed to roll back after commit failure", e));
106108
}
107109
throw t;
110+
} finally {
111+
runClearPathsForFinish();
108112
}
109113
}
110114

@@ -250,7 +254,38 @@ public void prepareAppendTable(THivePartitionUpdate pu, HivePartitionStatistics
250254
}
251255

252256
public void prepareOverwriteTable(THivePartitionUpdate pu, HivePartitionStatistics ps) {
253-
257+
String targetPath = pu.getLocation().getTargetPath();
258+
String writePath = pu.getLocation().getWritePath();
259+
if (!targetPath.equals(writePath)) {
260+
Path path = new Path(targetPath);
261+
String oldTablePath = new Path(path.getParent(), "_temp_" + path.getName()).toString();
262+
Status status = fs.renameDir(
263+
targetPath,
264+
oldTablePath,
265+
() -> renameDirectoryTasksForAbort.add(new RenameDirectoryTask(oldTablePath, targetPath)));
266+
if (!status.ok()) {
267+
throw new RuntimeException(
268+
"Error to rename dir from " + targetPath + " to " + oldTablePath + status.getErrMsg());
269+
}
270+
clearDirsForFinish.add(oldTablePath);
271+
272+
status = fs.renameDir(
273+
writePath,
274+
targetPath,
275+
() -> directoryCleanUpTasksForAbort.add(new DirectoryCleanUpTask(targetPath, true)));
276+
if (!status.ok()) {
277+
throw new RuntimeException(
278+
"Error to rename dir from " + writePath + " to " + targetPath + ":" + status.getErrMsg());
279+
}
280+
}
281+
updateStatisticsTasks.add(
282+
new UpdateStatisticsTask(
283+
table.getDbName(),
284+
table.getTableName(),
285+
Optional.empty(),
286+
ps,
287+
false
288+
));
254289
}
255290

256291
public void prepareCreateNewPartition(THivePartitionUpdate pu, HivePartitionStatistics ps) {
@@ -335,7 +370,38 @@ public void prepareInsertExistPartition(List<Pair<THivePartitionUpdate, HivePart
335370

336371

337372
public void prepareOverwritePartition(THivePartitionUpdate pu, HivePartitionStatistics ps) {
373+
String targetPath = pu.getLocation().getTargetPath();
374+
String writePath = pu.getLocation().getWritePath();
375+
if (!targetPath.equals(writePath)) {
376+
Path path = new Path(targetPath);
377+
String oldPartitionPath = new Path(path.getParent(), "_temp_" + path.getName()).toString();
378+
Status status = fs.renameDir(
379+
targetPath,
380+
oldPartitionPath,
381+
() -> renameDirectoryTasksForAbort.add(new RenameDirectoryTask(oldPartitionPath, targetPath)));
382+
if (!status.ok()) {
383+
throw new RuntimeException(
384+
"Error to rename dir from " + targetPath + " to " + oldPartitionPath + ":" + status.getErrMsg());
385+
}
386+
clearDirsForFinish.add(oldPartitionPath);
338387

388+
status = fs.renameDir(
389+
writePath,
390+
targetPath,
391+
() -> directoryCleanUpTasksForAbort.add(new DirectoryCleanUpTask(targetPath, true)));
392+
if (!status.ok()) {
393+
throw new RuntimeException(
394+
"Error to rename dir from " + writePath + " to " + targetPath + ":" + status.getErrMsg());
395+
}
396+
}
397+
updateStatisticsTasks.add(
398+
new UpdateStatisticsTask(
399+
table.getDbName(),
400+
table.getTableName(),
401+
Optional.of(pu.getName()),
402+
ps,
403+
false
404+
));
339405
}
340406

341407

@@ -481,7 +547,7 @@ public void run(HiveMetadataOps hiveOps) {
481547
createdPartitionValues.add(partition.getPartition().getPartitionValues());
482548
}
483549
} catch (Throwable t) {
484-
LOG.error("Failed to add partition", t);
550+
LOG.warn("Failed to add partition", t);
485551
throw t;
486552
}
487553
}
@@ -583,18 +649,38 @@ public String toString() {
583649
}
584650

585651
private void runRenameDirTasksForAbort() {
586-
// TODO abort
652+
Status status;
653+
for (RenameDirectoryTask task : renameDirectoryTasksForAbort) {
654+
status = fs.exists(task.getRenameFrom());
655+
if (status.ok()) {
656+
status = fs.renameDir(task.getRenameFrom(), task.getRenameTo(), () -> {});
657+
if (!status.ok()) {
658+
LOG.warn("Failed to abort rename dir from {} to {}:{}",
659+
task.getRenameFrom(), task.getRenameTo(), status.getErrMsg());
660+
}
661+
}
662+
}
663+
}
664+
665+
private void runClearPathsForFinish() {
666+
Status status;
667+
for (String path : clearDirsForFinish) {
668+
status = fs.delete(path);
669+
if (!status.ok()) {
670+
LOG.warn("Failed to recursively delete path {}:{}", path, status.getErrCode());
671+
}
672+
}
587673
}
588674

589675

590676
private void recursiveDeleteItems(Path directory, boolean deleteEmptyDir) {
591677
DeleteRecursivelyResult deleteResult = recursiveDeleteFiles(directory, deleteEmptyDir);
592678

593679
if (!deleteResult.getNotDeletedEligibleItems().isEmpty()) {
594-
LOG.error("Failed to delete directory {}. Some eligible items can't be deleted: {}.",
680+
LOG.warn("Failed to delete directory {}. Some eligible items can't be deleted: {}.",
595681
directory.toString(), deleteResult.getNotDeletedEligibleItems());
596682
} else if (deleteEmptyDir && !deleteResult.dirNotExists()) {
597-
LOG.error("Failed to delete directory {} due to dir isn't empty", directory.toString());
683+
LOG.warn("Failed to delete directory {} due to dir isn't empty", directory.toString());
598684
}
599685
}
600686

‎fe/fe-core/src/main/java/org/apache/doris/fs/FileSystem.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ public interface FileSystem {
4949

5050
Status rename(String origFilePath, String destFilePath);
5151

52+
default Status renameDir(String origFilePath,
53+
String destFilePath,
54+
Runnable runWhenPathNotExist) {
55+
throw new UnsupportedOperationException("Unsupported operation rename dir on current file system.");
56+
}
57+
5258
default void asyncRename(Executor executor,
5359
List<CompletableFuture<?>> renameFileFutures,
5460
AtomicBoolean cancelled,

‎fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,28 @@ public void asyncRename(
412412
}
413413
}
414414

415+
public Status renameDir(String origFilePath,
416+
String destFilePath,
417+
Runnable runWhenPathNotExist) {
418+
Status status = exists(destFilePath);
419+
if (status.ok()) {
420+
throw new RuntimeException("Destination directory already exists: " + destFilePath);
421+
}
422+
423+
String targetParent = new Path(destFilePath).getParent().toString();
424+
status = exists(targetParent);
425+
if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) {
426+
status = makeDir(targetParent);
427+
}
428+
if (!status.ok()) {
429+
throw new RuntimeException(status.getErrMsg());
430+
}
431+
432+
runWhenPathNotExist.run();
433+
434+
return rename(origFilePath, destFilePath);
435+
}
436+
415437
@Override
416438
public void asyncRenameDir(Executor executor,
417439
List<CompletableFuture<?>> renameFileFutures,
@@ -423,23 +445,7 @@ public void asyncRenameDir(Executor executor,
423445
if (cancelled.get()) {
424446
return;
425447
}
426-
427-
Status status = exists(destFilePath);
428-
if (status.ok()) {
429-
throw new RuntimeException("Destination directory already exists: " + destFilePath);
430-
}
431-
432-
String targetParent = new Path(destFilePath).getParent().toString();
433-
status = exists(targetParent);
434-
if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) {
435-
makeDir(targetParent);
436-
} else if (!status.ok()) {
437-
throw new RuntimeException(status.getErrMsg());
438-
}
439-
440-
runWhenPathNotExist.run();
441-
442-
status = rename(origFilePath, destFilePath);
448+
Status status = renameDir(origFilePath, destFilePath, runWhenPathNotExist);
443449
if (!status.ok()) {
444450
throw new RuntimeException(status.getErrMsg());
445451
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java‎

Lines changed: 46 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -133,20 +133,27 @@ public void testAppendPartitionForUnPartitionedTable() {
133133
pus.add(createRandomAppend(""));
134134
hmsOps.commit(dbName, tbWithoutPartition, pus);
135135
Table table = hmsClient.getTable(dbName, tbWithoutPartition);
136-
Assert.assertEquals(3, Long.parseLong(table.getParameters().get("numRows")));
136+
assertNumRows(3, table);
137137

138138
List<THivePartitionUpdate> pus2 = new ArrayList<>();
139139
pus2.add(createRandomAppend(""));
140140
pus2.add(createRandomAppend(""));
141141
pus2.add(createRandomAppend(""));
142142
hmsOps.commit(dbName, tbWithoutPartition, pus2);
143143
table = hmsClient.getTable(dbName, tbWithoutPartition);
144-
Assert.assertEquals(6, Long.parseLong(table.getParameters().get("numRows")));
144+
assertNumRows(6, table);
145145
}
146146

147147
@Test
148148
public void testOverwritePartitionForUnPartitionedTable() {
149-
// TODO
149+
testAppendPartitionForUnPartitionedTable();
150+
List<THivePartitionUpdate> pus = new ArrayList<>();
151+
pus.add(createRandomOverwrite(""));
152+
pus.add(createRandomOverwrite(""));
153+
pus.add(createRandomOverwrite(""));
154+
hmsOps.commit(dbName, tbWithoutPartition, pus);
155+
Table table = hmsClient.getTable(dbName, tbWithoutPartition);
156+
assertNumRows(3, table);
150157
}
151158

152159
@Test
@@ -161,11 +168,11 @@ public void testNewPartitionForPartitionedTable() {
161168
hmsOps.commit(dbName, tbWithPartition, pus);
162169

163170
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
164-
Assert.assertEquals(3, Long.parseLong(pa.getParameters().get("numRows")));
171+
assertNumRows(3, pa);
165172
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
166-
Assert.assertEquals(2, Long.parseLong(pb.getParameters().get("numRows")));
173+
assertNumRows(2, pb);
167174
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
168-
Assert.assertEquals(1, Long.parseLong(pc.getParameters().get("numRows")));
175+
assertNumRows(1, pc);
169176
}
170177

171178
@Test
@@ -182,11 +189,28 @@ public void testAppendPartitionForPartitionedTable() {
182189
hmsOps.commit(dbName, tbWithPartition, pus);
183190

184191
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
185-
Assert.assertEquals(6, Long.parseLong(pa.getParameters().get("numRows")));
192+
assertNumRows(6, pa);
193+
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
194+
assertNumRows(4, pb);
195+
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
196+
assertNumRows(2, pc);
197+
}
198+
199+
@Test
200+
public void testOverwritePartitionForPartitionedTable() {
201+
testAppendPartitionForPartitionedTable();
202+
List<THivePartitionUpdate> pus = new ArrayList<>();
203+
pus.add(createRandomOverwrite("a"));
204+
pus.add(createRandomOverwrite("b"));
205+
pus.add(createRandomOverwrite("c"));
206+
hmsOps.commit(dbName, tbWithPartition, pus);
207+
208+
Partition pa = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("a"));
209+
assertNumRows(1, pa);
186210
Partition pb = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("b"));
187-
Assert.assertEquals(4, Long.parseLong(pb.getParameters().get("numRows")));
211+
assertNumRows(1, pb);
188212
Partition pc = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("c"));
189-
Assert.assertEquals(2, Long.parseLong(pc.getParameters().get("numRows")));
213+
assertNumRows(1, pc);
190214
}
191215

192216
@Test
@@ -200,7 +224,7 @@ public void testNewManyPartitionForPartitionedTable() {
200224
hmsOps.commit(dbName, tbWithPartition, pus);
201225
for (int i = 0; i < nums; i++) {
202226
Partition p = hmsClient.getPartition(dbName, tbWithPartition, Lists.newArrayList("" + i));
203-
Assert.assertEquals(1, Long.parseLong(p.getParameters().get("numRows")));
227+
assertNumRows(1, p);
204228
}
205229

206230
try {
@@ -210,6 +234,14 @@ public void testNewManyPartitionForPartitionedTable() {
210234
}
211235
}
212236

237+
public void assertNumRows(long expected, Partition p) {
238+
Assert.assertEquals(expected, Long.parseLong(p.getParameters().get("numRows")));
239+
}
240+
241+
public void assertNumRows(long expected, Table t) {
242+
Assert.assertEquals(expected, Long.parseLong(t.getParameters().get("numRows")));
243+
}
244+
213245
public THivePartitionUpdate genOnePartitionUpdate(String partitionValue, TUpdateMode mode) {
214246

215247
String uuid = UUID.randomUUID().toString();
@@ -241,4 +273,8 @@ public THivePartitionUpdate createRandomNew(String partition) {
241273
public THivePartitionUpdate createRandomAppend(String partition) {
242274
return genOnePartitionUpdate("c3=" + partition, TUpdateMode.APPEND);
243275
}
276+
277+
public THivePartitionUpdate createRandomOverwrite(String partition) {
278+
return genOnePartitionUpdate("c3=" + partition, TUpdateMode.OVERWRITE);
279+
}
244280
}

0 commit comments

Comments
 (0)