Skip to content

Commit d12f573

Browse files
DanGugeDanGuge
authored
feat(api&core): in oltp apis, add statistics info and support full info about vertices and edges (#2262)
* chore: improve gitignore file * feat: add ApiMeasure to collect runtime data ApiMeasure will count the number of vertices and edges traversed at runtime, and the time the api takes to execute * feat: Add ApiMeasure to JsonSerializer and Modify the Serializer interface * JsonSerializer: return measure information in api response * Serializer: fit the feature that returns complete information about vertices and edges * refactor: format code based on hugegraph-style.xml * feat: Add statistics information in all oltp restful apis response and Support full information about vertices and edges Statistics information: * add vertexIterCounter and edgeIterCounter in HugeTraverser.java to track traversed vertices and edges at run time * modify all oltp restful apis to add statistics information in response Full information about vertices and edges: * add 'with_vertex' and 'with_edge' parameter option in apis * modify oltp apis to support vertex and edge information in api response * add EdgeRecord in HugeTraverser.java to record edges at run time and generate the edge information returned in api response * modify Path and PathSet in HugeTraverser.java to support full edge information storage * modify all traversers to support track of edge information at run time * fix: numeric cast * fix: Jaccard Similarity api test * fix: adjust the code style and naming convention * Empty commit * Empty commit * fix: 1. change System.currentTimeMillis() to System.nanoTime(); 2. modify addCount() * fix: rollback change in .gitignore * fix: rollback ServerOptions.java code style * fix: rollback API.java code style and add exception in else branch * fix: fix code style * fix: name style & code style * rename edgeRecord to edgeResults * fix Request class code style in SameNeighborsAPI.java
1 parent b02c2bd commit d12f573

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2268
-1222
lines changed

‎hugegraph-api/src/main/java/org/apache/hugegraph/api/API.java‎

Lines changed: 67 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -22,41 +22,39 @@
2222
import java.util.concurrent.Callable;
2323
import java.util.function.Consumer;
2424

25-
import jakarta.ws.rs.ForbiddenException;
26-
import jakarta.ws.rs.NotFoundException;
27-
import jakarta.ws.rs.NotSupportedException;
28-
import jakarta.ws.rs.core.MediaType;
29-
25+
import org.apache.commons.lang.mutable.MutableLong;
26+
import org.apache.hugegraph.HugeException;
27+
import org.apache.hugegraph.HugeGraph;
3028
import org.apache.hugegraph.core.GraphManager;
3129
import org.apache.hugegraph.define.Checkable;
3230
import org.apache.hugegraph.metrics.MetricsUtil;
33-
import org.slf4j.Logger;
34-
35-
import org.apache.hugegraph.HugeException;
36-
import org.apache.hugegraph.HugeGraph;
3731
import org.apache.hugegraph.util.E;
32+
import org.apache.hugegraph.util.InsertionOrderUtil;
3833
import org.apache.hugegraph.util.JsonUtil;
3934
import org.apache.hugegraph.util.Log;
35+
import org.slf4j.Logger;
36+
4037
import com.codahale.metrics.Meter;
4138
import com.google.common.collect.ImmutableMap;
4239

43-
public class API {
40+
import jakarta.ws.rs.ForbiddenException;
41+
import jakarta.ws.rs.NotFoundException;
42+
import jakarta.ws.rs.NotSupportedException;
43+
import jakarta.ws.rs.core.MediaType;
4444

45-
protected static final Logger LOG = Log.logger(API.class);
45+
public class API {
4646

4747
public static final String CHARSET = "UTF-8";
48-
4948
public static final String TEXT_PLAIN = MediaType.TEXT_PLAIN;
5049
public static final String APPLICATION_JSON = MediaType.APPLICATION_JSON;
5150
public static final String APPLICATION_JSON_WITH_CHARSET =
5251
APPLICATION_JSON + ";charset=" + CHARSET;
5352
public static final String JSON = MediaType.APPLICATION_JSON_TYPE
5453
.getSubtype();
55-
5654
public static final String ACTION_APPEND = "append";
5755
public static final String ACTION_ELIMINATE = "eliminate";
5856
public static final String ACTION_CLEAR = "clear";
59-
57+
protected static final Logger LOG = Log.logger(API.class);
6058
private static final Meter SUCCEED_METER =
6159
MetricsUtil.registerMeter(API.class, "commit-succeed");
6260
private static final Meter ILLEGAL_ARG_ERROR_METER =
@@ -69,8 +67,7 @@ public class API {
6967
public static HugeGraph graph(GraphManager manager, String graph) {
7068
HugeGraph g = manager.graph(graph);
7169
if (g == null) {
72-
throw new NotFoundException(String.format(
73-
"Graph '%s' does not exist", graph));
70+
throw new NotFoundException(String.format("Graph '%s' does not exist", graph));
7471
}
7572
return g;
7673
}
@@ -140,8 +137,7 @@ protected static void checkUpdatingBody(Checkable body) {
140137
body.checkUpdate();
141138
}
142139

143-
protected static void checkCreatingBody(
144-
Collection<? extends Checkable> bodies) {
140+
protected static void checkCreatingBody(Collection<? extends Checkable> bodies) {
145141
E.checkArgumentNotNull(bodies, "The request body can't be empty");
146142
for (Checkable body : bodies) {
147143
E.checkArgument(body != null,
@@ -150,8 +146,7 @@ protected static void checkCreatingBody(
150146
}
151147
}
152148

153-
protected static void checkUpdatingBody(
154-
Collection<? extends Checkable> bodies) {
149+
protected static void checkUpdatingBody(Collection<? extends Checkable> bodies) {
155150
E.checkArgumentNotNull(bodies, "The request body can't be empty");
156151
for (Checkable body : bodies) {
157152
E.checkArgumentNotNull(body,
@@ -186,8 +181,58 @@ public static boolean checkAndParseAction(String action) {
186181
} else if (action.equals(ACTION_ELIMINATE)) {
187182
return false;
188183
} else {
189-
throw new NotSupportedException(
190-
String.format("Not support action '%s'", action));
184+
throw new NotSupportedException(String.format("Not support action '%s'", action));
185+
}
186+
}
187+
188+
public static class ApiMeasurer {
189+
190+
public static final String EDGE_ITER = "edge_iterations";
191+
public static final String VERTICE_ITER = "vertice_iterations";
192+
public static final String COST = "cost(ns)";
193+
private final long timeStart;
194+
private final Map<String, Object> measures;
195+
196+
public ApiMeasurer() {
197+
this.timeStart = System.nanoTime();
198+
this.measures = InsertionOrderUtil.newMap();
199+
}
200+
201+
public Map<String, Object> measures() {
202+
measures.put(COST, System.nanoTime() - timeStart);
203+
return measures;
204+
}
205+
206+
public void put(String key, String value) {
207+
this.measures.put(key, value);
208+
}
209+
210+
public void put(String key, long value) {
211+
this.measures.put(key, value);
212+
}
213+
214+
public void put(String key, int value) {
215+
this.measures.put(key, value);
216+
}
217+
218+
protected void addCount(String key, long value) {
219+
Object current = measures.get(key);
220+
if (current == null) {
221+
measures.put(key, new MutableLong(value));
222+
} else if (current instanceof MutableLong) {
223+
((MutableLong) measures.computeIfAbsent(key, MutableLong::new)).add(value);
224+
} else if (current instanceof Long) {
225+
Long currentLong = (Long) current;
226+
measures.put(key, new MutableLong(currentLong + value));
227+
} else {
228+
throw new NotSupportedException("addCount() method's 'value' datatype must be " +
229+
"Long or MutableLong");
230+
}
231+
}
232+
233+
public void addIterCount(long verticeIters, long edgeIters) {
234+
this.addCount(EDGE_ITER, edgeIters);
235+
this.addCount(VERTICE_ITER, verticeIters);
191236
}
192237
}
193238
}

‎hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/AllShortestPathsAPI.java‎

Lines changed: 55 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,10 @@
2020
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_CAPACITY;
2121
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
2222

23+
import java.util.HashSet;
24+
import java.util.Iterator;
2325
import java.util.List;
24-
25-
import io.swagger.v3.oas.annotations.tags.Tag;
26-
import jakarta.inject.Singleton;
27-
import jakarta.ws.rs.DefaultValue;
28-
import jakarta.ws.rs.GET;
29-
import jakarta.ws.rs.Path;
30-
import jakarta.ws.rs.PathParam;
31-
import jakarta.ws.rs.Produces;
32-
import jakarta.ws.rs.QueryParam;
33-
import jakarta.ws.rs.core.Context;
34-
35-
import org.slf4j.Logger;
26+
import java.util.Set;
3627

3728
import org.apache.hugegraph.HugeGraph;
3829
import org.apache.hugegraph.api.API;
@@ -44,9 +35,22 @@
4435
import org.apache.hugegraph.traversal.algorithm.ShortestPathTraverser;
4536
import org.apache.hugegraph.type.define.Directions;
4637
import org.apache.hugegraph.util.Log;
38+
import org.apache.tinkerpop.gremlin.structure.Edge;
39+
import org.slf4j.Logger;
40+
4741
import com.codahale.metrics.annotation.Timed;
4842
import com.google.common.collect.ImmutableList;
4943

44+
import io.swagger.v3.oas.annotations.tags.Tag;
45+
import jakarta.inject.Singleton;
46+
import jakarta.ws.rs.DefaultValue;
47+
import jakarta.ws.rs.GET;
48+
import jakarta.ws.rs.Path;
49+
import jakarta.ws.rs.PathParam;
50+
import jakarta.ws.rs.Produces;
51+
import jakarta.ws.rs.QueryParam;
52+
import jakarta.ws.rs.core.Context;
53+
5054
@Path("graphs/{graph}/traversers/allshortestpaths")
5155
@Singleton
5256
@Tag(name = "AllShortestPathsAPI")
@@ -68,13 +72,20 @@ public String get(@Context GraphManager manager,
6872
@DefaultValue(DEFAULT_MAX_DEGREE) long maxDegree,
6973
@QueryParam("skip_degree")
7074
@DefaultValue("0") long skipDegree,
75+
@QueryParam("with_vertex")
76+
@DefaultValue("false") boolean withVertex,
77+
@QueryParam("with_edge")
78+
@DefaultValue("false") boolean withEdge,
7179
@QueryParam("capacity")
7280
@DefaultValue(DEFAULT_CAPACITY) long capacity) {
7381
LOG.debug("Graph [{}] get shortest path from '{}', to '{}' with " +
7482
"direction {}, edge label {}, max depth '{}', " +
75-
"max degree '{}', skipped degree '{}' and capacity '{}'",
83+
"max degree '{}', skipped degree '{}', capacity '{}', " +
84+
"with_vertex '{}' and with_edge '{}'",
7685
graph, source, target, direction, edgeLabel, depth,
77-
maxDegree, skipDegree, capacity);
86+
maxDegree, skipDegree, capacity, withVertex, withEdge);
87+
88+
ApiMeasurer measure = new ApiMeasurer();
7889

7990
Id sourceId = VertexAPI.checkAndParseVertexId(source);
8091
Id targetId = VertexAPI.checkAndParseVertexId(target);
@@ -85,9 +96,35 @@ public String get(@Context GraphManager manager,
8596
ShortestPathTraverser traverser = new ShortestPathTraverser(g);
8697
List<String> edgeLabels = edgeLabel == null ? ImmutableList.of() :
8798
ImmutableList.of(edgeLabel);
88-
HugeTraverser.PathSet paths = traverser.allShortestPaths(
89-
sourceId, targetId, dir, edgeLabels,
90-
depth, maxDegree, skipDegree, capacity);
91-
return manager.serializer(g).writePaths("paths", paths, false);
99+
HugeTraverser.PathSet paths = traverser.allShortestPaths(sourceId, targetId, dir,
100+
edgeLabels, depth, maxDegree,
101+
skipDegree, capacity);
102+
103+
measure.addIterCount(traverser.vertexIterCounter.get(),
104+
traverser.edgeIterCounter.get());
105+
106+
Iterator<?> iterVertex;
107+
Set<Id> vertexIds = new HashSet<>();
108+
for (HugeTraverser.Path path : paths) {
109+
vertexIds.addAll(path.vertices());
110+
}
111+
if (withVertex && !vertexIds.isEmpty()) {
112+
iterVertex = g.vertices(vertexIds.toArray());
113+
measure.addIterCount(vertexIds.size(), 0L);
114+
} else {
115+
iterVertex = vertexIds.iterator();
116+
}
117+
118+
Iterator<?> iterEdge;
119+
Set<Edge> edges = paths.getEdges();
120+
if (withEdge && !edges.isEmpty()) {
121+
iterEdge = edges.iterator();
122+
} else {
123+
iterEdge = HugeTraverser.EdgeRecord.getEdgeIds(edges).iterator();
124+
}
125+
126+
return manager.serializer(g, measure.measures())
127+
.writePaths("paths", paths, false,
128+
iterVertex, iterEdge);
92129
}
93130
}

‎hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/CrosspointsAPI.java‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,6 @@
2121
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;
2222
import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_PATHS_LIMIT;
2323

24-
import io.swagger.v3.oas.annotations.tags.Tag;
25-
import jakarta.inject.Singleton;
26-
import jakarta.ws.rs.DefaultValue;
27-
import jakarta.ws.rs.GET;
28-
import jakarta.ws.rs.Path;
29-
import jakarta.ws.rs.PathParam;
30-
import jakarta.ws.rs.Produces;
31-
import jakarta.ws.rs.QueryParam;
32-
import jakarta.ws.rs.core.Context;
33-
34-
import org.slf4j.Logger;
35-
3624
import org.apache.hugegraph.HugeGraph;
3725
import org.apache.hugegraph.api.API;
3826
import org.apache.hugegraph.api.graph.EdgeAPI;
@@ -43,8 +31,20 @@
4331
import org.apache.hugegraph.traversal.algorithm.PathsTraverser;
4432
import org.apache.hugegraph.type.define.Directions;
4533
import org.apache.hugegraph.util.Log;
34+
import org.slf4j.Logger;
35+
4636
import com.codahale.metrics.annotation.Timed;
4737

38+
import io.swagger.v3.oas.annotations.tags.Tag;
39+
import jakarta.inject.Singleton;
40+
import jakarta.ws.rs.DefaultValue;
41+
import jakarta.ws.rs.GET;
42+
import jakarta.ws.rs.Path;
43+
import jakarta.ws.rs.PathParam;
44+
import jakarta.ws.rs.Produces;
45+
import jakarta.ws.rs.QueryParam;
46+
import jakarta.ws.rs.core.Context;
47+
4848
@Path("graphs/{graph}/traversers/crosspoints")
4949
@Singleton
5050
@Tag(name = "CrosspointsAPI")
@@ -74,6 +74,7 @@ public String get(@Context GraphManager manager,
7474
graph, source, target, direction, edgeLabel,
7575
depth, maxDegree, capacity, limit);
7676

77+
ApiMeasurer measure = new ApiMeasurer();
7778
Id sourceId = VertexAPI.checkAndParseVertexId(source);
7879
Id targetId = VertexAPI.checkAndParseVertexId(target);
7980
Directions dir = Directions.convert(EdgeAPI.parseDirection(direction));
@@ -84,6 +85,9 @@ public String get(@Context GraphManager manager,
8485
dir, edgeLabel, depth,
8586
maxDegree, capacity,
8687
limit);
87-
return manager.serializer(g).writePaths("crosspoints", paths, true);
88+
measure.addIterCount(traverser.vertexIterCounter.get(),
89+
traverser.edgeIterCounter.get());
90+
return manager.serializer(g, measure.measures())
91+
.writePaths("crosspoints", paths, true);
8892
}
8993
}

0 commit comments

Comments
 (0)