Skip to content

Commit 64efac9

Browse files
committed
fix(db-autoscaler): survive a restart mid-stuck-scale rollback
A controller restart or leader failover starts with an empty in-memory state map. lastWritten was already reconstructed from status to keep the ownership back-off durable, but lastConverged and the in-flight anchor were not. So a restart that landed mid-stuck-scale initialized lastConverged to the stuck (larger) current count, making safeRollbackTarget a no-op (target == current), and lost inFlightSince so the convergence deadline never elapsed. The auto-rollback degraded to a permanent StuckScaling freeze at the stuck count instead of rolling back. Restore lastConverged from status.lastConvergedReplicas and, when a scale is still in flight, reconstruct inFlightSince from status.lastScaleTime, so the deadline and rollback are measured across the failover. Covered by a new reconciler test that fails without either restore. Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 97b5ac9 commit 64efac9

2 files changed

Lines changed: 54 additions & 0 deletions

File tree

‎internal/dbautoscaler/reconciler.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
188188
w := *dha.Status.LastAppliedReplicas
189189
st.lastWritten = &w
190190
}
191+
// Likewise restore lastConverged, so a restart mid-stuck-scale can still roll
192+
// back to the last known-good count instead of freezing (it would otherwise
193+
// initialize to the stuck current count and make rollback a no-op).
194+
if st.lastConverged == nil && dha.Status.LastConvergedReplicas != nil {
195+
c := *dha.Status.LastConvergedReplicas
196+
st.lastConverged = &c
197+
}
191198

192199
// WorkloadMonitor: operational + convergence signal.
193200
operational, availableReplicas := r.loadWorkloadMonitor(ctx, dha.Namespace, adapter.ReleaseName(dha.Spec.TargetRef.Name), currentReplicas)
@@ -204,6 +211,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
204211
conv := currentReplicas
205212
st.lastConverged = &conv
206213
}
214+
// A restart also loses the in-flight anchor. When a scale is still in flight,
215+
// reconstruct it from the persisted last-scale time so the convergence deadline
216+
// (and a stuck-scale rollback) is measured across the failover, instead of
217+
// resetting the clock on every reconcile and freezing a stuck scale forever.
218+
if st.inFlightSince == nil && scaleInFlight && dha.Status.LastScaleTime != nil {
219+
t := dha.Status.LastScaleTime.Time
220+
st.inFlightSince = &t
221+
}
207222

208223
// Ownership: detect a competing writer that changed replicas out from under us.
209224
ownershipConflict := st.lastWritten != nil && currentReplicas != *st.lastWritten

‎internal/dbautoscaler/reconciler_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,3 +478,42 @@ func TestReconcileStuckScalingRollback(t *testing.T) {
478478
t.Fatalf("cycle4 expected StuckScaling (backoff), got %s/%s", s, reason)
479479
}
480480
}
481+
482+
// TestStuckScalingRollbackSurvivesRestart guards the restart-durability of the
483+
// stuck-scaling rollback: a fresh controller (empty in-memory state) landing
484+
// mid-stuck-scale must still roll back to the last converged count, not freeze.
485+
// It reconstructs lastConverged from status.lastConvergedReplicas and the
486+
// in-flight anchor from status.lastScaleTime; without either restore the
487+
// rollback degrades to a permanent StuckScaling freeze at the stuck count.
488+
func TestStuckScalingRollbackSurvivesRestart(t *testing.T) {
489+
dha := newDHA("db", "tenant", "Postgres", false)
490+
dl := int32(900)
491+
dha.Spec.Behavior = &autoscalingv1alpha1.Behavior{ConvergenceDeadlineSeconds: &dl}
492+
app := newPostgresApp("db", "tenant", 4, 0) // operator had scaled up to 4...
493+
// ...but the new standby never came up: available stays at 3 (stuck in flight).
494+
env := newTestEnv(t, dha, app, "420", http.StatusOK, workloadMonitor("postgres-db", "tenant", 3, true))
495+
496+
// Seed the persisted status a restart would have left behind: last converged at
497+
// 3, last wrote 4, and the scale-up patch happened well past the deadline ago.
498+
three, four := int32(3), int32(4)
499+
cur := &autoscalingv1alpha1.DatabaseHorizontalAutoscaler{}
500+
if err := env.r.Get(context.TODO(), types.NamespacedName{Namespace: "tenant", Name: "db"}, cur); err != nil {
501+
t.Fatalf("get: %v", err)
502+
}
503+
cur.Status.LastConvergedReplicas = &three
504+
cur.Status.LastAppliedReplicas = &four
505+
cur.Status.LastScaleTime = &metav1.Time{Time: env.now.Add(-1000 * time.Second)}
506+
if err := env.r.Status().Update(context.TODO(), cur); err != nil {
507+
t.Fatalf("seed status: %v", err)
508+
}
509+
510+
// Fresh state map (new leader): no in-memory lastConverged/inFlightSince.
511+
env.r.state = nil
512+
got := env.reconcile(t, dha)
513+
if s, reason := condStatus(got, autoscalingv1alpha1.ConditionAbleToScale); s != string(metav1.ConditionFalse) || reason != autoscalingv1alpha1.ReasonStuckScaling {
514+
t.Fatalf("expected StuckScaling after restart, got %s/%s", s, reason)
515+
}
516+
if r := env.appReplicas(t, "tenant", "db"); r != 3 {
517+
t.Fatalf("expected rollback to last converged (3) after restart, app replicas = %d", r)
518+
}
519+
}

0 commit comments

Comments
 (0)