Skip to content

Commit bdf915a

Browse files
authored
[Enhancement] (image) check image validity as soon as generated (apache#9011)
* load newly generated image file as soon as generated to check if it is valid. * delete the latest invalid image file * fix * fix * get filePath from saveImage() to ensure deleting the correct file while exception happens * fix Co-authored-by: wuhangze <[email protected]>
1 parent 687421b commit bdf915a

3 files changed

Lines changed: 47 additions & 2 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1950,7 +1950,8 @@ public long loadSqlBlockRule(DataInputStream in, long checksum) throws IOExcepti
19501950
}
19511951

19521952
// Only called by checkpoint thread
1953-
public void saveImage() throws IOException {
1953+
// return the latest image file's absolute path
1954+
public String saveImage() throws IOException {
19541955
// Write image.ckpt
19551956
Storage storage = new Storage(this.imageDir);
19561957
File curFile = storage.getImageFile(replayedJournalId.get());
@@ -1963,6 +1964,7 @@ public void saveImage() throws IOException {
19631964
curFile.delete();
19641965
throw new IOException();
19651966
}
1967+
return curFile.getAbsolutePath();
19661968
}
19671969

19681970
public void saveImage(File curFile, long replayedJournalId) throws IOException {

‎fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import org.apache.logging.log4j.LogManager;
3636
import org.apache.logging.log4j.Logger;
3737

38+
import com.google.common.base.Strings;
39+
3840
import java.io.IOException;
3941
import java.io.OutputStream;
4042
import java.net.HttpURLConnection;
@@ -111,6 +113,8 @@ public synchronized void doCheckpoint() {
111113
catalog = Catalog.getCurrentCatalog();
112114
catalog.setEditLog(editLog);
113115
createStaticFieldForCkpt();
116+
boolean exceptionCaught = false;
117+
String latestImageFilePath = null;
114118
try {
115119
catalog.loadImage(imageDir);
116120
catalog.replayJournal(checkPointVersion);
@@ -119,13 +123,25 @@ public synchronized void doCheckpoint() {
119123
checkPointVersion, catalog.getReplayedJournalId()));
120124
}
121125
catalog.fixBugAfterMetadataReplayed(false);
122-
catalog.saveImage();
126+
latestImageFilePath = catalog.saveImage();
123127
replayedJournalId = catalog.getReplayedJournalId();
128+
129+
// destroy checkpoint catalog, reclaim memory
130+
catalog = null;
131+
Catalog.destroyCheckpoint();
132+
destroyStaticFieldForCkpt();
133+
134+
// Load image to verify if the newly generated image file is valid
135+
// If success, do all the following jobs
136+
// If failed, just return
137+
catalog = Catalog.getCurrentCatalog();
138+
catalog.loadImage(imageDir);
124139
if (MetricRepo.isInit) {
125140
MetricRepo.COUNTER_IMAGE_WRITE_SUCCESS.increase(1L);
126141
}
127142
LOG.info("checkpoint finished save image.{}", replayedJournalId);
128143
} catch (Throwable e) {
144+
exceptionCaught = true;
129145
e.printStackTrace();
130146
LOG.error("Exception when generate new image file", e);
131147
if (MetricRepo.isInit) {
@@ -137,6 +153,22 @@ public synchronized void doCheckpoint() {
137153
catalog = null;
138154
Catalog.destroyCheckpoint();
139155
destroyStaticFieldForCkpt();
156+
// if new image generated && exception caught, delete the latest image here
157+
// delete the newest image file, cuz it is invalid
158+
if ((!Strings.isNullOrEmpty(latestImageFilePath)) && exceptionCaught) {
159+
MetaCleaner cleaner = new MetaCleaner(Config.meta_dir + "/image");
160+
try {
161+
cleaner.cleanTheLatestInvalidImageFile(latestImageFilePath);
162+
if (MetricRepo.isInit) {
163+
MetricRepo.COUNTER_IMAGE_CLEAN_SUCCESS.increase(1L);
164+
}
165+
} catch (Throwable ex) {
166+
LOG.error("Master delete latest invalid image file failed.", ex);
167+
if (MetricRepo.isInit) {
168+
MetricRepo.COUNTER_IMAGE_CLEAN_FAILED.increase(1L);
169+
}
170+
}
171+
}
140172
}
141173

142174
// push image file to all the other non master nodes

‎fe/fe-core/src/main/java/org/apache/doris/persist/MetaCleaner.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,17 @@ public void clean() throws IOException {
6767
}
6868
}
6969
}
70+
71+
public void cleanTheLatestInvalidImageFile(String path) throws IOException {
72+
File latestInvalidImage = new File(path);
73+
if (latestInvalidImage.exists()) {
74+
if (latestInvalidImage.delete()) {
75+
LOG.info(latestInvalidImage.getAbsoluteFile() + " deleted.");
76+
} else {
77+
LOG.warn(latestInvalidImage.getAbsoluteFile() + " delete failed.");
78+
}
79+
}
80+
}
7081

7182
private String fileType(File file) throws IOException {
7283
String type = null;

0 commit comments

Comments
 (0)