- The client connects to the master through Sentinel, and creates a channel to monitor the changes happening under the keyspace "parent*"
- The master goes down
- wait for the failover
- the channel remains open but stops receiving events after the failover.
Expected Behavior
When the sentinel fails over to the replica, the client should continue receiving the events coming from the replica.
Current Behavior
The channel doesn't receive any updates after the failover
Possible Solution
The ping command keeps sticking for a long period of time after the master's shutdown
|
case <-timer.C: |
|
if pingErr := c.pubSub.Ping(ctx); pingErr != nil { |
|
c.pubSub.mu.Lock() |
|
c.pubSub.reconnect(ctx, pingErr) |
|
c.pubSub.mu.Unlock() |
|
} |
Steps to Reproduce
- Run docker compose up provided below
- Run the code from a container inside the network redis
- Shutdown redis-master
- Update the key "parent", the client will not detect the events.
Context (Environment)
The inspection of this issue traefik/traefik#12965 led me to this issue.
Detailed Description
I started with 3 instances
- master instance
- replica instance
- Sentinel
services:
redis-master:
image: redis:8.6
command: redis-server --appendonly yes --notify-keyspace-events KEA
networks:
redis:
aliases:
- redis-master
redis-replica:
image: redis:8.6
command: redis-server --replicaof redis-master 6379 --appendonly yes --notify-keyspace-events KEA
depends_on:
- redis-master
networks:
redis:
aliases:
- redis-replica
sentinel:
image: redis:8.6
entrypoint: []
command: >
sh -c '
cat > /tmp/sentinel.conf <<EOF
port 26379
sentinel monitor mymaster redis-master 6379 1
sentinel down-after-milliseconds mymaster 5000
sentinel failover-timeout mymaster 10000
sentinel resolve-hostnames yes
EOF
redis-sentinel /tmp/sentinel.conf'
depends_on:
- redis-master
- redis-replica
networks:
redis:
aliases:
- sentinel
networks:
redis:Example code
func main() {
ctx := context.Background()
rdb := redis.NewFailoverClient(&redis.FailoverOptions{
MasterName: "mymaster",
SentinelAddrs: []string{"sentinel:26379"},
})
defer rdb.Close()
for {
fmt.Println("subscribing...")
// pubsub := rdb.PSubscribe(ctx, "__keyspace@0__:parent*")
pubsub := rdb.Subscribe(ctx, "__keyspace@0__:parent")
_, err := pubsub.Receive(ctx)
if err != nil {
fmt.Println("####receive subscription error:", err)
_ = pubsub.Close()
time.Sleep(time.Second)
continue
}
ch := pubsub.Channel()
for msg := range ch {
fmt.Printf("####channel=%s payload=%s\n", msg.Channel, msg.Payload)
}
fmt.Println("pubsub channel closed")
_ = pubsub.Close()
time.Sleep(time.Second)
}
}Possible Implementation
I tried to introduce a timeout, it worked, but I'm not sure that it's the best solution
func (c *channel) initHealthCheck() {
c.ping = make(chan struct{}, 1)
go func() {
timer := time.NewTimer(time.Minute)
timer.Stop()
for {
timer.Reset(c.checkInterval)
select {
case <-c.ping:
if !timer.Stop() {
<-timer.C
}
case <-timer.C:
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
if pingErr := c.pubSub.Ping(ctx); pingErr != nil {
reconnectCtx, _ := context.WithTimeout(context.Background(), 10*time.Second)
c.pubSub.mu.Lock()
c.pubSub.reconnect(reconnectCtx, pingErr)
c.pubSub.mu.Unlock()
}
case <-c.pubSub.exit:
return
}
}
}()
}
Expected Behavior
When the sentinel fails over to the replica, the client should continue receiving the events coming from the replica.
Current Behavior
The channel doesn't receive any updates after the failover
Possible Solution
The ping command keeps sticking for a long period of time after the master's shutdown
go-redis/pubsub.go
Lines 696 to 701 in 3dfa4b6
Steps to Reproduce
Context (Environment)
The inspection of this issue traefik/traefik#12965 led me to this issue.
Detailed Description
I started with 3 instances
Example code
Possible Implementation
I tried to introduce a timeout, it worked, but I'm not sure that it's the best solution