Skip to content

Commit fa40654

Browse files
asheshvidyutanalogue
asheshvidyut
and
analogue
authored
[NET-3865] [Supportability] Additional Information in the output of 'consul operator raft list-peers' (#17582)
* init * fix tests * added -detailed in docs * added change log * fix doc * checking for entry in map * fix tests * removed detailed flag * removed detailed flag * revert unwanted changes * removed unwanted changes * updated change log * pr review comment changes * pr comment changes single API instead of two * fix change log * fix tests * fix tests * fix test operator raft endpoint test * Update .changelog/17582.txt Co-authored-by: Semir Patel <[email protected]> * nits * updated docs --------- Co-authored-by: Semir Patel <[email protected]>
1 parent 6a90c23 commit fa40654

File tree

8 files changed

+66
-9
lines changed

8 files changed

+66
-9
lines changed

‎.changelog/17582.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:feature
2+
cli: `consul operator raft list-peers` command shows the number of commits each follower is trailing the leader by to aid in troubleshooting.
3+
```

‎agent/consul/operator_raft_endpoint.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,12 @@ func (op *Operator) RaftGetConfiguration(args *structs.DCSpecificRequest, reply
4848
serverMap[raft.ServerAddress(addr)] = member
4949
}
5050

51+
serverIDLastIndexMap := make(map[raft.ServerID]uint64)
52+
53+
for _, serverState := range op.srv.autopilot.GetState().Servers {
54+
serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex
55+
}
56+
5157
// Fill out the reply.
5258
leader := op.srv.raft.Leader()
5359
reply.Index = future.Index()
@@ -66,6 +72,7 @@ func (op *Operator) RaftGetConfiguration(args *structs.DCSpecificRequest, reply
6672
Leader: server.Address == leader,
6773
Voter: server.Suffrage == raft.Voter,
6874
ProtocolVersion: raftProtocolVersion,
75+
LastIndex: serverIDLastIndexMap[server.ID],
6976
}
7077
reply.Servers = append(reply.Servers, entry)
7178
}

‎agent/consul/operator_raft_endpoint_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@ func TestOperator_RaftGetConfiguration(t *testing.T) {
5050
if len(future.Configuration().Servers) != 1 {
5151
t.Fatalf("bad: %v", future.Configuration().Servers)
5252
}
53+
54+
serverIDLastIndexMap := make(map[raft.ServerID]uint64)
55+
56+
for _, serverState := range s1.autopilot.GetState().Servers {
57+
serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex
58+
}
59+
5360
me := future.Configuration().Servers[0]
5461
expected := structs.RaftConfigurationResponse{
5562
Servers: []*structs.RaftServer{
@@ -60,6 +67,7 @@ func TestOperator_RaftGetConfiguration(t *testing.T) {
6067
Leader: true,
6168
Voter: true,
6269
ProtocolVersion: "3",
70+
LastIndex: serverIDLastIndexMap[me.ID],
6371
},
6472
},
6573
Index: future.Index(),
@@ -113,6 +121,10 @@ func TestOperator_RaftGetConfiguration_ACLDeny(t *testing.T) {
113121
if len(future.Configuration().Servers) != 1 {
114122
t.Fatalf("bad: %v", future.Configuration().Servers)
115123
}
124+
serverIDLastIndexMap := make(map[raft.ServerID]uint64)
125+
for _, serverState := range s1.autopilot.GetState().Servers {
126+
serverIDLastIndexMap[serverState.Server.ID] = serverState.Stats.LastIndex
127+
}
116128
me := future.Configuration().Servers[0]
117129
expected := structs.RaftConfigurationResponse{
118130
Servers: []*structs.RaftServer{
@@ -123,6 +135,7 @@ func TestOperator_RaftGetConfiguration_ACLDeny(t *testing.T) {
123135
Leader: true,
124136
Voter: true,
125137
ProtocolVersion: "3",
138+
LastIndex: serverIDLastIndexMap[me.ID],
126139
},
127140
},
128141
Index: future.Index(),

‎agent/structs/operator.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ type RaftServer struct {
3434
// it's a non-voting server, which will be added in a future release of
3535
// Consul.
3636
Voter bool
37+
38+
// LastIndex is the last log index this server has a record of in its Raft log.
39+
LastIndex uint64
3740
}
3841

3942
// RaftConfigurationResponse is returned when querying for the current Raft

‎api/operator_raft.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ type RaftServer struct {
2828
// it's a non-voting server, which will be added in a future release of
2929
// Consul.
3030
Voter bool
31+
32+
// LastIndex is the last log index this server has a record of in its Raft log.
33+
LastIndex uint64
3134
}
3235

3336
// RaftConfiguration is returned when querying for the current Raft configuration.

‎command/operator/raft/listpeers/operator_raft_list.go‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,24 @@ func raftListPeers(client *api.Client, stale bool) (string, error) {
7070
return "", fmt.Errorf("Failed to retrieve raft configuration: %v", err)
7171
}
7272

73+
leaderLastCommitIndex := uint64(0)
74+
serverIdLastIndexMap := make(map[string]uint64)
75+
76+
for _, raftServer := range reply.Servers {
77+
serverIdLastIndexMap[raftServer.ID] = raftServer.LastIndex
78+
}
79+
80+
for _, s := range reply.Servers {
81+
if s.Leader {
82+
lastIndex, ok := serverIdLastIndexMap[s.ID]
83+
if ok {
84+
leaderLastCommitIndex = lastIndex
85+
}
86+
}
87+
}
88+
7389
// Format it as a nice table.
74-
result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol"}
90+
result := []string{"Node\x1fID\x1fAddress\x1fState\x1fVoter\x1fRaftProtocol\x1fCommit Index\x1fTrails Leader By"}
7591
for _, s := range reply.Servers {
7692
raftProtocol := s.ProtocolVersion
7793

@@ -82,8 +98,20 @@ func raftListPeers(client *api.Client, stale bool) (string, error) {
8298
if s.Leader {
8399
state = "leader"
84100
}
85-
result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s",
86-
s.Node, s.ID, s.Address, state, s.Voter, raftProtocol))
101+
102+
trailsLeaderByText := "-"
103+
serverLastIndex, ok := serverIdLastIndexMap[s.ID]
104+
if ok {
105+
trailsLeaderBy := leaderLastCommitIndex - serverLastIndex
106+
trailsLeaderByText = fmt.Sprintf("%d commits", trailsLeaderBy)
107+
if s.Leader {
108+
trailsLeaderByText = "-"
109+
} else if trailsLeaderBy == 1 {
110+
trailsLeaderByText = fmt.Sprintf("%d commit", trailsLeaderBy)
111+
}
112+
}
113+
result = append(result, fmt.Sprintf("%s\x1f%s\x1f%s\x1f%s\x1f%v\x1f%s\x1f%v\x1f%s",
114+
s.Node, s.ID, s.Address, state, s.Voter, raftProtocol, serverLastIndex, trailsLeaderByText))
87115
}
88116

89117
return columnize.Format(result, &columnize.Config{Delim: string([]byte{0x1f})}), nil

‎command/operator/raft/listpeers/operator_raft_list_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func TestOperatorRaftListPeersCommand(t *testing.T) {
2828
a := agent.NewTestAgent(t, ``)
2929
defer a.Shutdown()
3030

31-
expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3",
31+
expected := fmt.Sprintf("%s %s 127.0.0.1:%d leader true 3 1 -",
3232
a.Config.NodeName, a.Config.NodeID, a.Config.ServerPort)
3333

3434
// Test the list-peers subcommand directly

‎website/content/commands/operator/raft.mdx‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ Usage: `consul operator raft list-peers -stale=[true|false]`
4646
The output looks like this:
4747

4848
```text
49-
Node ID Address State Voter RaftProtocol
50-
alice 127.0.0.1:8300 127.0.0.1:8300 follower true 2
51-
bob 127.0.0.2:8300 127.0.0.2:8300 leader true 3
52-
carol 127.0.0.3:8300 127.0.0.3:8300 follower true 2
49+
Node ID Address State Voter RaftProtocol Commit Index Trails Leader By
50+
alice 127.0.0.1:8300 127.0.0.1:8300 follower true 2 1167 0 commits
51+
bob 127.0.0.2:8300 127.0.0.2:8300 leader true 3 1167 -
52+
carol 127.0.0.3:8300 127.0.0.3:8300 follower true 2 1159 8 commits
5353
```
5454

5555
`Node` is the node name of the server, as known to Consul, or "(unknown)" if
@@ -70,7 +70,7 @@ configuration.
7070

7171
- `-stale` - Enables non-leader servers to provide cluster state information.
7272
If the cluster is in an outage state without a leader,
73-
we recommend setting this option to `true.
73+
we recommend setting this option to `true`.
7474
Default is `false`.
7575

7676
## remove-peer

0 commit comments

Comments
 (0)