Skip to content

Commit f9fdb2d

Browse files
hutiefang76KomachiSion
authored andcommitted
[ISSUE #12867] Add raft group metrics (#15408)
* [ISSUE #12867] Add raft group metrics * fix: preserve raft leader metrics on member events (cherry picked from commit c42ec16)
1 parent fe1a015 commit f9fdb2d

3 files changed

Lines changed: 101 additions & 0 deletions

File tree

‎core/src/main/java/com/alibaba/nacos/core/distributed/raft/JRaftProtocol.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import com.alibaba.nacos.core.cluster.ServerMemberManager;
3636
import com.alibaba.nacos.core.distributed.AbstractConsistencyProtocol;
3737
import com.alibaba.nacos.core.distributed.raft.exception.NoSuchRaftGroupException;
38+
import com.alibaba.nacos.core.monitor.MetricsMonitor;
3839
import com.alibaba.nacos.core.utils.Loggers;
3940
import com.alipay.sofa.jraft.Node;
4041

@@ -143,6 +144,8 @@ public void onEvent(RaftEvent event) {
143144
MapUtil.putIfValNoEmpty(properties, MetadataKey.RAFT_GROUP_MEMBER,
144145
raftClusterInfo);
145146
MapUtil.putIfValNoEmpty(properties, MetadataKey.ERR_MSG, errMsg);
147+
MetricsMonitor.refreshRaftGroupMetrics(groupId, leader, term,
148+
raftConfig.getSelfMember());
146149

147150
value.put(groupId, properties);
148151
metaData.load(value);

‎core/src/main/java/com/alibaba/nacos/core/monitor/MetricsMonitor.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ public final class MetricsMonitor {
5858

5959
private static Map<String, AtomicInteger> moduleConnectionCnt = new ConcurrentHashMap<>();
6060

61+
private static Map<String, AtomicInteger> raftGroupLeaderStatus = new ConcurrentHashMap<>();
62+
63+
private static Map<String, AtomicLong> raftGroupTerm = new ConcurrentHashMap<>();
64+
6165
static {
6266
ImmutableTag immutableTag = new ImmutableTag("module", "core");
6367
List<Tag> tags = new ArrayList<>();
@@ -173,6 +177,52 @@ public static DistributionSummary getRaftFromLeader() {
173177
return RAFT_FROM_LEADER;
174178
}
175179

180+
/**
181+
* Refresh raft group metrics for actuator and prometheus.
182+
*
183+
* @param groupId raft group id
184+
* @param leader current leader endpoint
185+
* @param term current raft term
186+
* @param selfMember local raft endpoint
187+
*/
188+
public static void refreshRaftGroupMetrics(String groupId, String leader, Long term,
189+
String selfMember) {
190+
if (StringUtils.isBlank(groupId)) {
191+
return;
192+
}
193+
if (StringUtils.isNotBlank(leader)) {
194+
AtomicInteger leaderStatus = raftGroupLeaderStatus.computeIfAbsent(groupId,
195+
MetricsMonitor::registerRaftGroupLeaderStatus);
196+
leaderStatus.set(StringUtils.equals(leader, selfMember) ? 1 : 0);
197+
}
198+
if (term != null) {
199+
raftGroupTerm.computeIfAbsent(groupId, MetricsMonitor::registerRaftGroupTerm)
200+
.set(term);
201+
}
202+
}
203+
204+
private static AtomicInteger registerRaftGroupLeaderStatus(String groupId) {
205+
AtomicInteger result = new AtomicInteger();
206+
NacosMeterRegistryCenter.gauge(METER_REGISTRY, "nacos_monitor",
207+
Arrays.asList(
208+
new ImmutableTag("module", "core"),
209+
new ImmutableTag("name", "raftLeaderStatus"),
210+
new ImmutableTag("group", groupId)),
211+
result);
212+
return result;
213+
}
214+
215+
private static AtomicLong registerRaftGroupTerm(String groupId) {
216+
AtomicLong result = new AtomicLong();
217+
NacosMeterRegistryCenter.gauge(METER_REGISTRY, "nacos_monitor",
218+
Arrays.asList(
219+
new ImmutableTag("module", "core"),
220+
new ImmutableTag("name", "raftTerm"),
221+
new ImmutableTag("group", groupId)),
222+
result);
223+
return result;
224+
}
225+
176226
public static GrpcServerExecutorMetric getSdkServerExecutorMetric() {
177227
return sdkServerExecutorMetric;
178228
}

‎core/src/test/java/com/alibaba/nacos/core/monitor/MetricsMonitorTest.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.alibaba.nacos.core.monitor;
1818

1919
import com.alibaba.nacos.sys.utils.ApplicationUtils;
20+
import io.micrometer.core.instrument.Gauge;
2021
import io.micrometer.core.instrument.Timer;
2122
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
2223
import io.micrometer.prometheusmetrics.PrometheusMeterRegistry;
@@ -139,6 +140,53 @@ void testRaftApplyReadTimer() {
139140
assertEquals(30D, raftApplyReadTimer.totalTime(TimeUnit.SECONDS), 0.01);
140141
}
141142

143+
@Test
144+
void testRefreshRaftGroupMetrics() {
145+
String selfMember = "127.0.0.1:7848";
146+
MetricsMonitor.refreshRaftGroupMetrics("naming_persistent_service",
147+
selfMember, 2L, selfMember);
148+
MetricsMonitor.refreshRaftGroupMetrics("naming_instance_metadata",
149+
"127.0.0.2:7848", 3L, selfMember);
150+
151+
Gauge serviceLeaderStatus = NacosMeterRegistryCenter
152+
.getMeterRegistry(NacosMeterRegistryCenter.CORE_STABLE_REGISTRY)
153+
.find("nacos_monitor")
154+
.tags("module", "core", "name", "raftLeaderStatus", "group",
155+
"naming_persistent_service")
156+
.gauge();
157+
Gauge metadataLeaderStatus = NacosMeterRegistryCenter
158+
.getMeterRegistry(NacosMeterRegistryCenter.CORE_STABLE_REGISTRY)
159+
.find("nacos_monitor")
160+
.tags("module", "core", "name", "raftLeaderStatus", "group",
161+
"naming_instance_metadata")
162+
.gauge();
163+
Gauge serviceTerm = NacosMeterRegistryCenter
164+
.getMeterRegistry(NacosMeterRegistryCenter.CORE_STABLE_REGISTRY)
165+
.find("nacos_monitor")
166+
.tags("module", "core", "name", "raftTerm", "group",
167+
"naming_persistent_service")
168+
.gauge();
169+
170+
assertEquals(1D, serviceLeaderStatus.value(), 0.01);
171+
assertEquals(0D, metadataLeaderStatus.value(), 0.01);
172+
assertEquals(2D, serviceTerm.value(), 0.01);
173+
174+
MetricsMonitor.refreshRaftGroupMetrics("naming_persistent_service",
175+
"127.0.0.2:7848", 4L, selfMember);
176+
assertEquals(0D, serviceLeaderStatus.value(), 0.01);
177+
assertEquals(4D, serviceTerm.value(), 0.01);
178+
179+
MetricsMonitor.refreshRaftGroupMetrics("naming_persistent_service",
180+
selfMember, 5L, selfMember);
181+
assertEquals(1D, serviceLeaderStatus.value(), 0.01);
182+
assertEquals(5D, serviceTerm.value(), 0.01);
183+
184+
MetricsMonitor.refreshRaftGroupMetrics("naming_persistent_service",
185+
null, null, selfMember);
186+
assertEquals(1D, serviceLeaderStatus.value(), 0.01);
187+
assertEquals(5D, serviceTerm.value(), 0.01);
188+
}
189+
142190
@Test
143191
void testRecordGrpcRequestEvent() {
144192
MetricsMonitor.recordGrpcRequestEvent("TestRequest", true, 0, "java.lang.RuntimeException",

0 commit comments

Comments
 (0)