Skip to content

Commit 24225c6

Browse files
jsilvelaarmrumnencia
authored
fix(roles): check primary status before each role reconciliation (#9971)
Move the IsPrimary() check from a one-time evaluation at startup into the reconciliation loop so that replicas and designated primaries skip role writes, and a promoted instance starts reconciling without requiring a pod restart. Closes #9800 Signed-off-by: Jaime Silvela <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent 983bc6b commit 24225c6

3 files changed

Lines changed: 156 additions & 9 deletions

File tree

‎internal/management/controller/roles/reconciler.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ func Reconcile(
7878
}
7979

8080
if len(rolesByStatus[apiv1.RoleStatusPendingReconciliation]) != 0 {
81-
// forces runnable to run
8281
instance.TriggerRoleSynchronizer(cluster.Spec.Managed)
8382
contextLogger.Info("Triggered a managed role reconciliation")
8483
}

‎internal/management/controller/roles/runnable.go‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,7 @@ func NewRoleSynchronizer(instance *postgres.Instance, client client.Client) *Rol
8383
func (sr *RoleSynchronizer) Start(ctx context.Context) error {
8484
contextLog := log.FromContext(ctx).WithName("roles_reconciler")
8585
contextLog.Info("starting up the runnable")
86-
isPrimary, err := sr.instance.IsPrimary()
87-
if err != nil {
88-
return err
89-
}
90-
if !isPrimary {
91-
contextLog.Info("skipping the RoleSynchronizer in replicas")
92-
}
86+
9387
go func() {
9488
var config *apiv1.ManagedConfiguration
9589
contextLog.Info("setting up RoleSynchronizer loop")
@@ -112,7 +106,18 @@ func (sr *RoleSynchronizer) Start(ctx context.Context) error {
112106
continue
113107
}
114108

115-
err := sr.reconcile(ctx, config)
109+
// Skip replicas and designated primaries: only true primaries write roles.
110+
isPrimary, err := sr.instance.IsPrimary()
111+
if err != nil {
112+
contextLog.Error(err, "checking primary status")
113+
continue
114+
}
115+
if !isPrimary {
116+
contextLog.Debug("skipping role reconciliation on a non-primary instance")
117+
continue
118+
}
119+
120+
err = sr.reconcile(ctx, config)
116121
if err != nil {
117122
contextLog.Error(err, "synchronizing roles", "config", config)
118123
continue

‎internal/management/controller/roles/runnable_test.go‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"context"
2424
"database/sql"
2525
"fmt"
26+
"sync/atomic"
2627
"time"
2728

2829
"github.com/DATA-DOG/go-sqlmock"
@@ -52,6 +53,148 @@ func (f *fakeInstanceData) GetSuperUserDB() (*sql.DB, error) {
5253
return f.db, nil
5354
}
5455

56+
type mockInstanceForStart struct {
57+
isPrimaryVal atomic.Bool
58+
syncChan chan *apiv1.ManagedConfiguration
59+
isPrimaryChecked chan struct{}
60+
isReadyCalls atomic.Int32
61+
}
62+
63+
func (m *mockInstanceForStart) GetSuperUserDB() (*sql.DB, error) {
64+
return nil, fmt.Errorf("not implemented")
65+
}
66+
67+
func (m *mockInstanceForStart) IsPrimary() (bool, error) {
68+
result := m.isPrimaryVal.Load()
69+
if m.isPrimaryChecked != nil {
70+
select {
71+
case m.isPrimaryChecked <- struct{}{}:
72+
default:
73+
}
74+
}
75+
return result, nil
76+
}
77+
78+
func (m *mockInstanceForStart) RoleSynchronizerChan() <-chan *apiv1.ManagedConfiguration {
79+
return m.syncChan
80+
}
81+
82+
func (m *mockInstanceForStart) IsReady() error {
83+
m.isReadyCalls.Add(1)
84+
return fmt.Errorf("not ready")
85+
}
86+
87+
func (m *mockInstanceForStart) GetClusterName() string {
88+
return "test-cluster"
89+
}
90+
91+
func (m *mockInstanceForStart) GetNamespaceName() string {
92+
return "default"
93+
}
94+
95+
var _ = Describe("RoleSynchronizer Start", func() {
96+
It("should return nil when context is cancelled", func() {
97+
syncChan := make(chan *apiv1.ManagedConfiguration)
98+
instance := &mockInstanceForStart{syncChan: syncChan}
99+
sr := &RoleSynchronizer{instance: instance}
100+
101+
ctx, cancel := context.WithCancel(context.Background())
102+
defer cancel()
103+
104+
errCh := make(chan error, 1)
105+
go func() {
106+
errCh <- sr.Start(ctx)
107+
}()
108+
109+
// Ensure Start is blocking (not returning immediately)
110+
Consistently(errCh, 200*time.Millisecond).ShouldNot(Receive())
111+
112+
cancel()
113+
114+
var startErr error
115+
Eventually(errCh).Should(Receive(&startErr))
116+
Expect(startErr).ToNot(HaveOccurred())
117+
})
118+
119+
It("should skip reconciliation on non-primary instances", func() {
120+
syncChan := make(chan *apiv1.ManagedConfiguration, 1)
121+
isPrimaryChecked := make(chan struct{}, 1)
122+
instance := &mockInstanceForStart{
123+
syncChan: syncChan,
124+
isPrimaryChecked: isPrimaryChecked,
125+
}
126+
sr := &RoleSynchronizer{instance: instance}
127+
128+
ctx, cancel := context.WithCancel(context.Background())
129+
defer cancel()
130+
131+
go sr.Start(ctx) //nolint:errcheck
132+
133+
syncChan <- &apiv1.ManagedConfiguration{
134+
Roles: []apiv1.RoleConfiguration{{Name: "test-role"}},
135+
}
136+
Eventually(isPrimaryChecked).Should(Receive())
137+
Expect(instance.isReadyCalls.Load()).To(BeEquivalentTo(0))
138+
})
139+
140+
It("should attempt reconciliation on primary instances", func() {
141+
syncChan := make(chan *apiv1.ManagedConfiguration, 1)
142+
isPrimaryChecked := make(chan struct{}, 1)
143+
instance := &mockInstanceForStart{
144+
syncChan: syncChan,
145+
isPrimaryChecked: isPrimaryChecked,
146+
}
147+
instance.isPrimaryVal.Store(true)
148+
sr := &RoleSynchronizer{instance: instance}
149+
150+
ctx, cancel := context.WithCancel(context.Background())
151+
defer cancel()
152+
153+
go sr.Start(ctx) //nolint:errcheck
154+
155+
syncChan <- &apiv1.ManagedConfiguration{
156+
Roles: []apiv1.RoleConfiguration{{Name: "test-role"}},
157+
}
158+
Eventually(isPrimaryChecked).Should(Receive())
159+
Eventually(func() int32 {
160+
return instance.isReadyCalls.Load()
161+
}).Should(BeEquivalentTo(1))
162+
})
163+
164+
It("should start reconciling after promotion", func() {
165+
syncChan := make(chan *apiv1.ManagedConfiguration, 1)
166+
isPrimaryChecked := make(chan struct{}, 1)
167+
instance := &mockInstanceForStart{
168+
syncChan: syncChan,
169+
isPrimaryChecked: isPrimaryChecked,
170+
}
171+
sr := &RoleSynchronizer{instance: instance}
172+
173+
ctx, cancel := context.WithCancel(context.Background())
174+
defer cancel()
175+
176+
go sr.Start(ctx) //nolint:errcheck
177+
178+
config := &apiv1.ManagedConfiguration{
179+
Roles: []apiv1.RoleConfiguration{{Name: "test-role"}},
180+
}
181+
182+
// Trigger while still a replica: reconcile must be skipped
183+
syncChan <- config
184+
Eventually(isPrimaryChecked).Should(Receive())
185+
Expect(instance.isReadyCalls.Load()).To(BeEquivalentTo(0))
186+
187+
// Simulate promotion
188+
instance.isPrimaryVal.Store(true)
189+
190+
syncChan <- config
191+
Eventually(isPrimaryChecked).Should(Receive())
192+
Eventually(func() int32 {
193+
return instance.isReadyCalls.Load()
194+
}).Should(BeEquivalentTo(1))
195+
})
196+
})
197+
55198
var _ = Describe("Role synchronizer tests", func() {
56199
var (
57200
db *sql.DB

0 commit comments

Comments
 (0)