Skip to content

Commit 39b9474

Browse files
LinaryLinary
authored
Let snapshot saved disk same as rocksdb data path (#1392)
* The goal is to ensure that snapshots can be generated in hard link * Add create and resume snapshot API * Use checkpoint when resume snapshot * Refactor some code in RocksDBStdSessions Change-Id: I88b39003107652f6ff8eae071e88dc888dd21959
1 parent c0dff5b commit 39b9474

File tree

23 files changed

+565
-295
lines changed

23 files changed

+565
-295
lines changed

‎hugegraph-api/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@
171171
</addDefaultSpecificationEntries>
172172
</manifest>
173173
<manifestEntries>
174-
<Implementation-Version>0.59.0.0</Implementation-Version>
174+
<Implementation-Version>0.60.0.0</Implementation-Version>
175175
</manifestEntries>
176176
</archive>
177177
</configuration>

‎hugegraph-api/src/main/java/com/baidu/hugegraph/api/profile/GraphsAPI.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,34 @@ public void clear(@Context GraphManager manager,
138138
g.truncateBackend();
139139
}
140140

141+
@PUT
142+
@Timed
143+
@Path("{name}/snapshot_create")
144+
@Produces(APPLICATION_JSON_WITH_CHARSET)
145+
@RolesAllowed({"admin", "$owner=$name"})
146+
public Object createSnapshot(@Context GraphManager manager,
147+
@PathParam("name") String name) {
148+
LOG.debug("Create snapshot for graph '{}'", name);
149+
150+
HugeGraph g = graph(manager, name);
151+
g.createSnapshot();
152+
return ImmutableMap.of(name, "snapshot_created");
153+
}
154+
155+
@PUT
156+
@Timed
157+
@Path("{name}/snapshot_resume")
158+
@Produces(APPLICATION_JSON_WITH_CHARSET)
159+
@RolesAllowed({"admin", "$owner=$name"})
160+
public Object resumeSnapshot(@Context GraphManager manager,
161+
@PathParam("name") String name) {
162+
LOG.debug("Resume snapshot for graph '{}'", name);
163+
164+
HugeGraph g = graph(manager, name);
165+
g.resumeSnapshot();
166+
return ImmutableMap.of(name, "snapshot_resumed");
167+
}
168+
141169
@PUT
142170
@Timed
143171
@Path("{name}/mode")

‎hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -700,6 +700,18 @@ public void truncateBackend() {
700700
}
701701
}
702702

703+
@Override
704+
public void createSnapshot() {
705+
this.verifyPermission(HugePermission.WRITE, ResourceType.STATUS);
706+
this.hugegraph.createSnapshot();
707+
}
708+
709+
@Override
710+
public void resumeSnapshot() {
711+
this.verifyPermission(HugePermission.WRITE, ResourceType.STATUS);
712+
this.hugegraph.resumeSnapshot();
713+
}
714+
703715
private void verifyAdminPermission() {
704716
verifyPermission(HugePermission.ANY, ResourceType.ROOT);
705717
}

‎hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,11 @@ public final class ApiVersion {
109109
* [0.58] Issue-1173: Supports customized kout/kneighbor,
110110
* multi-node-shortest-path, jaccard-similar and template-paths
111111
* [0.59] Issue-1333: Support graph read mode for olap property
112+
* [0.60] Issue-1392: Support create and resume snapshot
112113
*/
113114

114115
// The second parameter of Version.of() is for IDE running without JAR
115-
public static final Version VERSION = Version.of(ApiVersion.class, "0.59");
116+
public static final Version VERSION = Version.of(ApiVersion.class, "0.60");
116117

117118
public static final void check() {
118119
// Check version of hugegraph-core. Firstly do check from version 0.3

‎hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@ public interface HugeGraph extends Graph {
153153
public void clearBackend();
154154
public void truncateBackend();
155155

156+
public void createSnapshot();
157+
public void resumeSnapshot();
158+
156159
@Override
157160
public HugeFeatures features();
158161

‎hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -360,14 +360,36 @@ public void truncateBackend() {
360360
* When restarting, load the snapshot first and then read backend,
361361
* will not encounter such an intermediate state.
362362
*/
363-
this.storeProvider.writeSnapshot();
363+
this.storeProvider.createSnapshot();
364364
} finally {
365365
LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK);
366366
}
367367

368368
LOG.info("Graph '{}' has been truncated", this.name);
369369
}
370370

371+
@Override
372+
public void createSnapshot() {
373+
LockUtil.lock(this.name, LockUtil.GRAPH_LOCK);
374+
try {
375+
this.storeProvider.createSnapshot();
376+
} finally {
377+
LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK);
378+
}
379+
LOG.info("Graph '{}' has created snapshot", this.name);
380+
}
381+
382+
@Override
383+
public void resumeSnapshot() {
384+
LockUtil.lock(this.name, LockUtil.GRAPH_LOCK);
385+
try {
386+
this.storeProvider.resumeSnapshot();
387+
} finally {
388+
LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK);
389+
}
390+
LOG.info("Graph '{}' has resumed from snapshot", this.name);
391+
}
392+
371393
private SchemaTransaction openSchemaTransaction() throws HugeException {
372394
this.checkGraphNotClosed();
373395
try {
@@ -1424,7 +1446,7 @@ public void invalid2(HugeType type, Object[] ids) {
14241446

14251447
@Override
14261448
public void clear(HugeType type) {
1427-
this.hub.notify(Events.CACHE, Cache.ACTION_CLEAR, type, null);
1449+
this.hub.notify(Events.CACHE, Cache.ACTION_CLEAR, type);
14281450
}
14291451

14301452
@Override

‎hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
import com.baidu.hugegraph.HugeGraph;
2929
import com.baidu.hugegraph.backend.BackendException;
30+
import com.baidu.hugegraph.backend.store.raft.StoreSnapshotFile;
3031
import com.baidu.hugegraph.event.EventHub;
3132
import com.baidu.hugegraph.event.EventListener;
3233
import com.baidu.hugegraph.util.E;
@@ -151,13 +152,19 @@ public void initSystemInfo(HugeGraph graph) {
151152
}
152153

153154
@Override
154-
public void writeSnapshot() {
155-
// TODO: to be implemented
155+
public void createSnapshot() {
156+
String snapshotPrefix = StoreSnapshotFile.SNAPSHOT_DIR;
157+
for (BackendStore store : this.stores.values()) {
158+
store.createSnapshot(snapshotPrefix);
159+
}
156160
}
157161

158162
@Override
159-
public void readSnapshot() {
160-
// TODO: to be implemented
163+
public void resumeSnapshot() {
164+
String snapshotPrefix = StoreSnapshotFile.SNAPSHOT_DIR;
165+
for (BackendStore store : this.stores.values()) {
166+
store.resumeSnapshot(snapshotPrefix, true);
167+
}
161168
}
162169

163170
@Override

‎hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendFeatures.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ public default boolean supportsSharedStorage() {
2929
return true;
3030
}
3131

32+
public default boolean supportsSnapshot() {
33+
return false;
34+
}
35+
3236
public boolean supportsScanToken();
3337

3438
public boolean supportsScanKeyPrefix();

‎hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package com.baidu.hugegraph.backend.store;
2121

2222
import java.util.Iterator;
23+
import java.util.Set;
2324

2425
import com.baidu.hugegraph.backend.id.Id;
2526
import com.baidu.hugegraph.backend.id.IdGenerator;
@@ -116,12 +117,13 @@ public default void setCounterLowest(HugeType type, long lowest) {
116117
// Get current counter for a specific type
117118
public long getCounter(HugeType type);
118119

119-
public default void writeSnapshot(String snapshotPath) {
120-
throw new UnsupportedOperationException("writeSnapshot");
120+
public default Set<String> createSnapshot(String snapshotDir) {
121+
throw new UnsupportedOperationException("createSnapshot");
121122
}
122123

123-
public default void readSnapshot(String snapshotPath) {
124-
throw new UnsupportedOperationException("readSnapshot");
124+
public default void resumeSnapshot(String snapshotDir,
125+
boolean deleteSnapshot) {
126+
throw new UnsupportedOperationException("resumeSnapshot");
125127
}
126128

127129
static enum TxState {

‎hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ public interface BackendStoreProvider {
5454

5555
public void initSystemInfo(HugeGraph graph);
5656

57-
public void writeSnapshot();
57+
public void createSnapshot();
5858

59-
public void readSnapshot();
59+
public void resumeSnapshot();
6060

6161
public void listen(EventListener listener);
6262

0 commit comments

Comments
 (0)