Skip to content

Commit 687421b

Browse files
authored
keep at least one validated image file (apache#9192)
* rename ImageSeq to LatestImageSeq in Storage * keep at least one validated image file
1 parent 3bdfcde commit 687421b

8 files changed

Lines changed: 46 additions & 37 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1531,7 +1531,7 @@ private boolean getVersionFileFromHelper(Pair<String, Integer> helperNode) throw
15311531
private void getNewImage(Pair<String, Integer> helperNode) throws IOException {
15321532
long localImageVersion = 0;
15331533
Storage storage = new Storage(this.imageDir);
1534-
localImageVersion = storage.getImageSeq();
1534+
localImageVersion = storage.getLatestImageSeq();
15351535

15361536
try {
15371537
URL infoUrl = new URL("http://" + helperNode.first + ":" + Config.http_port + "/info");
@@ -1617,7 +1617,7 @@ public void loadImage(String imageDir) throws IOException, DdlException {
16171617
LOG.info("image does not exist: {}", curFile.getAbsolutePath());
16181618
return;
16191619
}
1620-
replayedJournalId.set(storage.getImageSeq());
1620+
replayedJournalId.set(storage.getLatestImageSeq());
16211621
MetaReader.read(curFile, this);
16221622
}
16231623

‎fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/HaController.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ private void appendImageInfo(Map<String, Object> result) {
134134
Map<String, Object> checkPoint = new HashMap<>();
135135
Storage storage = new Storage(Config.meta_dir + "/image");
136136
checkPoint.put("Name", "Version");
137-
checkPoint.put("Value", storage.getImageSeq());
137+
checkPoint.put("Value", storage.getLatestImageSeq());
138138
list.add(checkPoint);
139139
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
140140
Date date = new Date(lastCheckpointTime);

‎fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ public Object info(HttpServletRequest request, HttpServletResponse response) thr
111111
try {
112112
Storage currentStorageInfo = new Storage(imageDir.getAbsolutePath());
113113
StorageInfo storageInfo = new StorageInfo(currentStorageInfo.getClusterID(),
114-
currentStorageInfo.getImageSeq(), currentStorageInfo.getEditsSeq());
114+
currentStorageInfo.getLatestImageSeq(), currentStorageInfo.getEditsSeq());
115115
return ResponseEntityBuilder.ok(storageInfo);
116116
} catch (IOException e) {
117117
return ResponseEntityBuilder.internalError(e.getMessage());

‎fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ private Map<String, String> getHaInfo() throws IOException {
242242
feInfo.put("is_ready", String.valueOf(Catalog.getCurrentCatalog().isReady()));
243243

244244
Storage storage = new Storage(Config.meta_dir + "/image");
245-
feInfo.put("last_checkpoint_version", String.valueOf(storage.getImageSeq()));
245+
feInfo.put("last_checkpoint_version", String.valueOf(storage.getLatestImageSeq()));
246246
long lastCheckpointTime = storage.getCurrentImageFile().lastModified();
247247
feInfo.put("last_checkpoint_time", String.valueOf(lastCheckpointTime));
248248

‎fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public synchronized void doCheckpoint() {
8383
try {
8484
storage = new Storage(imageDir);
8585
// get max image version
86-
imageVersion = storage.getImageSeq();
86+
imageVersion = storage.getLatestImageSeq();
8787
// get max finalized journal id
8888
checkPointVersion = editLog.getFinalizedJournalId();
8989
LOG.info("last checkpoint journal id: {}, current finalized journal id: {}", imageVersion, checkPointVersion);
@@ -184,7 +184,9 @@ public synchronized void doCheckpoint() {
184184
if (successPushed == otherNodesCount) {
185185
try {
186186
long minOtherNodesJournalId = Long.MAX_VALUE;
187-
long deleteVersion = checkPointVersion;
187+
// Actually, storage.getLatestValidatedImageSeq returns number before this
188+
// checkpoint.
189+
long deleteVersion = storage.getLatestValidatedImageSeq();
188190
if (successPushed > 0) {
189191
for (Frontend fe : allFrontends) {
190192
String host = fe.getHost();
@@ -220,7 +222,7 @@ public synchronized void doCheckpoint() {
220222
}
221223
}
222224
}
223-
deleteVersion = Math.min(minOtherNodesJournalId, checkPointVersion);
225+
deleteVersion = Math.min(minOtherNodesJournalId, deleteVersion);
224226
}
225227

226228
editLog.deleteJournals(deleteVersion + 1);

‎fe/fe-core/src/main/java/org/apache/doris/persist/MetaCleaner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public MetaCleaner(String imageDir) {
3434

3535
public void clean() throws IOException {
3636
Storage storage = new Storage(imageDir);
37-
long currentVersion = storage.getImageSeq();
37+
long currentVersion = storage.getLatestValidatedImageSeq();
3838
long imageDeleteVersion = currentVersion - 1;
3939

4040
File currentImage = storage.getImageFile(currentVersion);

‎fe/fe-core/src/main/java/org/apache/doris/persist/Storage.java‎

Lines changed: 31 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ public class Storage {
6060
private FrontendNodeType role = FrontendNodeType.UNKNOWN;
6161
private String nodeName;
6262
private long editsSeq;
63-
private long imageSeq;
63+
private long latestImageSeq;
64+
private long latestValidatedImageSeq;
6465
private String metaDir;
6566
private List<Long> editsFileSequenceNumbers;
6667

@@ -70,11 +71,11 @@ public Storage(int clusterID, String token, String metaDir) {
7071
this.metaDir = metaDir;
7172
}
7273

73-
public Storage(int clusterID, String token, long imageSeq, long editsSeq, String metaDir) {
74+
public Storage(int clusterID, String token, long latestImageSeq, long editsSeq, String metaDir) {
7475
this.clusterID = clusterID;
7576
this.token = token;
7677
this.editsSeq = editsSeq;
77-
this.imageSeq = imageSeq;
78+
this.latestImageSeq = latestImageSeq;
7879
this.metaDir = metaDir;
7980
}
8081

@@ -114,31 +115,33 @@ public void reload() throws IOException {
114115
nodeName = prop.getProperty(NODE_NAME, null);
115116
}
116117

117-
// Find the latest image
118+
// Find the latest two images
118119
File dir = new File(metaDir);
119120
File[] children = dir.listFiles();
120121
if (children == null) {
121122
return;
122-
} else {
123-
for (File child : children) {
124-
String name = child.getName();
125-
try {
126-
if (!name.equals(EDITS) && !name.equals(IMAGE_NEW)
127-
&& !name.endsWith(".part") && name.contains(".")) {
128-
if (name.startsWith(IMAGE)) {
129-
imageSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), imageSeq);
130-
} else if (name.startsWith(EDITS)) {
131-
// Just record the sequence part of the file name
132-
editsFileSequenceNumbers.add(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)));
133-
editsSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), editsSeq);
123+
}
124+
for (File child : children) {
125+
String name = child.getName();
126+
try {
127+
if (!name.equals(EDITS) && !name.equals(IMAGE_NEW)
128+
&& !name.endsWith(".part") && name.contains(".")) {
129+
if (name.startsWith(IMAGE)) {
130+
long fileSeq = Long.parseLong(name.substring(name.lastIndexOf('.') + 1));
131+
if (latestImageSeq < fileSeq) {
132+
latestValidatedImageSeq = latestImageSeq;
133+
latestImageSeq = fileSeq;
134134
}
135+
} else if (name.startsWith(EDITS)) {
136+
// Just record the sequence part of the file name
137+
editsFileSequenceNumbers.add(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)));
138+
editsSeq = Math.max(Long.parseLong(name.substring(name.lastIndexOf('.') + 1)), editsSeq);
135139
}
136-
} catch (Exception e) {
137-
LOG.warn(name + " is not a validate meta file, ignore it");
138140
}
141+
} catch (Exception e) {
142+
LOG.warn(name + " is not a validate meta file, ignore it");
139143
}
140144
}
141-
142145
}
143146

144147
public int getClusterID() {
@@ -173,12 +176,16 @@ public void setMetaDir(String metaDir) {
173176
this.metaDir = metaDir;
174177
}
175178

176-
public long getImageSeq() {
177-
return imageSeq;
179+
public long getLatestImageSeq() {
180+
return latestImageSeq;
181+
}
182+
183+
public long getLatestValidatedImageSeq() {
184+
return latestValidatedImageSeq;
178185
}
179186

180-
public void setImageSeq(long imageSeq) {
181-
this.imageSeq = imageSeq;
187+
public void setLatestImageSeq(long latestImageSeq) {
188+
this.latestImageSeq = latestImageSeq;
182189
}
183190

184191
public void setEditsSeq(long editsSeq) {
@@ -272,7 +279,7 @@ public static void rename(File from, File to) throws IOException {
272279
}
273280

274281
public File getCurrentImageFile() {
275-
return getImageFile(imageSeq);
282+
return getImageFile(latestImageSeq);
276283
}
277284

278285
public File getImageFile(long version) {

‎fe/fe-core/src/test/java/org/apache/doris/persist/StorageTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ public void testConstruct() {
102102

103103
Storage storage2 = new Storage(1, "token", 2, 3, "test");
104104
Assert.assertEquals(1, storage2.getClusterID());
105-
Assert.assertEquals(2, storage2.getImageSeq());
105+
Assert.assertEquals(2, storage2.getLatestImageSeq());
106106
Assert.assertEquals(3, storage2.getEditsSeq());
107107
Assert.assertEquals("test", storage2.getMetaDir());
108108
}
@@ -116,7 +116,7 @@ public void testStorage() throws Exception {
116116
Assert.assertEquals(966271669, storage.getClusterID());
117117
storage.setClusterID(1234);
118118
Assert.assertEquals(1234, storage.getClusterID());
119-
Assert.assertEquals(0, storage.getImageSeq());
119+
Assert.assertEquals(0, storage.getLatestImageSeq());
120120
Assert.assertEquals(10, Storage.getMetaSeq(new File("storageTestDir/edits.10")));
121121
Assert.assertTrue(Storage.getCurrentEditsFile(new File("storageTestDir"))
122122
.equals(new File("storageTestDir/edits")));
@@ -133,8 +133,8 @@ public void testStorage() throws Exception {
133133

134134
Assert.assertTrue(storage.getVersionFile().equals(new File("storageTestDir/VERSION")));
135135

136-
storage.setImageSeq(100);
137-
Assert.assertEquals(100, storage.getImageSeq());
136+
storage.setLatestImageSeq(100);
137+
Assert.assertEquals(100, storage.getLatestImageSeq());
138138

139139
storage.setEditsSeq(100);
140140
Assert.assertEquals(100, storage.getEditsSeq());

0 commit comments

Comments
 (0)