Skip to content

Commit 97b054b

Browse files
committed
fix(operator): schedule follow-up reconcile when Ready is Unknown within grace
E2E on dev3 caught a dormancy bug that renders the whole recovery driver inactive in practice: when updateStatus copies Ready=Unknown across but the AG's LastTransitionTime is still within stuckGracePeriod, artifactGeneratorStuck returns false and we fall through to the copy path with no RequeueAfter set. If the reconcile happens to fire in the [stuck_start, stuck_start+30s) window — a routine case since operator restarts touch every PackageSource — the controller then sleeps waiting for an AG-status change that will never come (that IS the fluxcd/pkg#934 lost-write race), up to the 10h informer resync. Observed on dev3: `cozystack.networking` stuck at Ready=Unknown for 9+ minutes with zero recovery attempts. Two reconciles fired at stuck_start+23s and +24s — both correctly detected "not yet stuck" — and then nothing until observation ended. Upstream OCIRepository revision was unchanged throughout, ruling out a coincidental source update as the cause. Fix: whenever we copy Ready=Unknown to the PackageSource (or set a synthetic Unknown because no Ready condition exists yet), compute the remaining grace period against the AG's LastTransitionTime (or CreationTimestamp for the missing-Ready branch) and schedule a RequeueAfter at that expiry. A 1s floor prevents a zero-delay tight loop when the anchor is right at or past grace expiry. Extracted the delay computation to `agFollowUpDelay` for direct unit testing. Adds three tests: the follow-up scheduling on Ready=Unknown in-grace, the same on the missing-Ready branch, and a table test on agFollowUpDelay covering the floor edge cases. Signed-off-by: Ivan Okhotnikov <[email protected]>
1 parent ff4f02c commit 97b054b

2 files changed

Lines changed: 146 additions & 2 deletions

File tree

‎internal/operator/packagesource_reconciler.go‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,11 @@ func (r *PackageSourceReconciler) updateStatus(ctx context.Context, packageSourc
460460
Message: "ArtifactGenerator Ready condition not found",
461461
ObservedGeneration: packageSource.Generation,
462462
})
463-
return ctrl.Result{}, r.Status().Update(ctx, packageSource)
463+
// Fresh AG with no Ready yet — if source-watcher doesn't act within
464+
// the grace period, artifactGeneratorStuck will fire on the next
465+
// reconcile. Schedule that reconcile explicitly so we don't sit
466+
// waiting for an AG-status change that may never come.
467+
return ctrl.Result{RequeueAfter: agFollowUpDelay(ag.CreationTimestamp.Time, now)}, r.Status().Update(ctx, packageSource)
464468
}
465469

466470
// Copy Ready condition from ArtifactGenerator to PackageSource
@@ -478,7 +482,34 @@ func (r *PackageSourceReconciler) updateStatus(ctx context.Context, packageSourc
478482
"status", readyCondition.Status,
479483
"reason", readyCondition.Reason)
480484

481-
return ctrl.Result{}, r.Status().Update(ctx, packageSource)
485+
// If we just copied Ready=Unknown across, the artifactGeneratorStuck
486+
// predicate will fire once the grace period elapses. Without an
487+
// explicit RequeueAfter here, this reconciler would wait for a fresh
488+
// AG-status change to re-fire via the Owns() watch — and if
489+
// source-watcher's next write is the one lost to the fluxcd/pkg#934
490+
// split-patch race (i.e., exactly the scenario this workaround exists
491+
// for), no such change will land and we would sit dormant until the
492+
// 10h informer resync. Schedule a follow-up reconcile at grace-period
493+
// expiry so the recovery driver can take over. On Ready=True/False,
494+
// the AG has resolved and no explicit follow-up is needed.
495+
result := ctrl.Result{}
496+
if readyCondition.Status == metav1.ConditionUnknown {
497+
result.RequeueAfter = agFollowUpDelay(readyCondition.LastTransitionTime.Time, now)
498+
}
499+
return result, r.Status().Update(ctx, packageSource)
500+
}
501+
502+
// agFollowUpDelay returns how long to wait before re-reconciling an AG whose
503+
// Ready=Unknown state is still within the grace period. Anchor is the last
504+
// state transition (or AG creation, when no Ready condition exists yet). The
505+
// result is at least one second so we never schedule a zero-delay requeue
506+
// tight loop when the grace period has just elapsed.
507+
func agFollowUpDelay(transitionOrCreation time.Time, now time.Time) time.Duration {
508+
remaining := transitionOrCreation.Add(stuckGracePeriod).Sub(now)
509+
if remaining < time.Second {
510+
return time.Second
511+
}
512+
return remaining
482513
}
483514

484515
// maybeRecoverArtifactGenerator advances the bounded-recovery schedule for an

‎internal/operator/packagesource_reconciler_test.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,119 @@ func TestReconcile_DeletionTimestamp_EarlyReturn(t *testing.T) {
684684
}
685685
}
686686

687+
// TestUpdateStatus_UnknownWithinGrace_SchedulesFollowUp locks in the fix for
688+
// the dormancy bug caught on dev3: when Ready=Unknown but grace hasn't elapsed
689+
// yet, artifactGeneratorStuck returns false, we fall through to the copy path
690+
// and — without an explicit RequeueAfter — the reconciler would sleep until
691+
// the AG's next status change (which may never come if it's exactly the
692+
// split-patch race we exist to fix) or the 10h informer resync. This test
693+
// asserts we schedule a follow-up reconcile at grace-period expiry so the
694+
// recovery driver can take over.
695+
func TestUpdateStatus_UnknownWithinGrace_SchedulesFollowUp(t *testing.T) {
696+
// Ready=Unknown, transitioned 10s ago — grace is 30s, so 20s remaining.
697+
transitionAt := referenceTime.Add(-10 * time.Second)
698+
ps := &cozyv1alpha1.PackageSource{
699+
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
700+
Spec: cozyv1alpha1.PackageSourceSpec{
701+
SourceRef: &cozyv1alpha1.PackageSourceRef{
702+
Name: "src",
703+
Kind: "OCIRepository",
704+
Namespace: "cozy-system",
705+
},
706+
},
707+
}
708+
ag := &sourcewatcherv1beta1.ArtifactGenerator{
709+
ObjectMeta: metav1.ObjectMeta{
710+
Name: "example", Namespace: "cozy-system", Generation: 1,
711+
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
712+
},
713+
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
714+
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
715+
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
716+
},
717+
ObservedSourcesDigest: "sha256:0e7",
718+
Conditions: []metav1.Condition{{
719+
Type: "Ready",
720+
Status: metav1.ConditionUnknown,
721+
Reason: "Progressing",
722+
Message: "Reconciliation in progress",
723+
ObservedGeneration: 1,
724+
LastTransitionTime: metav1.NewTime(transitionAt),
725+
}},
726+
},
727+
}
728+
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
729+
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
730+
731+
res, err := r.updateStatus(context.Background(), ps, referenceTime)
732+
if err != nil {
733+
t.Fatalf("updateStatus: %v", err)
734+
}
735+
// remaining = transitionAt + stuckGracePeriod - now = -10s + 30s = 20s
736+
if res.RequeueAfter != 20*time.Second {
737+
t.Errorf("RequeueAfter = %v, want 20s (remaining grace after 10s elapsed)", res.RequeueAfter)
738+
}
739+
}
740+
741+
// TestUpdateStatus_MissingReadyWithinGrace_SchedulesFollowUp — same dormancy
742+
// fix but for the ready==nil branch: a fresh AG with no Ready condition yet
743+
// must also schedule a follow-up so the operator notices when grace elapses.
744+
func TestUpdateStatus_MissingReadyWithinGrace_SchedulesFollowUp(t *testing.T) {
745+
createdAt := referenceTime.Add(-5 * time.Second) // 25s remaining on grace
746+
ps := &cozyv1alpha1.PackageSource{
747+
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
748+
Spec: cozyv1alpha1.PackageSourceSpec{
749+
SourceRef: &cozyv1alpha1.PackageSourceRef{
750+
Name: "src",
751+
Kind: "OCIRepository",
752+
Namespace: "cozy-system",
753+
},
754+
},
755+
}
756+
ag := &sourcewatcherv1beta1.ArtifactGenerator{
757+
ObjectMeta: metav1.ObjectMeta{
758+
Name: "example", Namespace: "cozy-system", Generation: 1,
759+
CreationTimestamp: metav1.NewTime(createdAt),
760+
},
761+
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{}, // no Ready
762+
}
763+
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
764+
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
765+
766+
res, err := r.updateStatus(context.Background(), ps, referenceTime)
767+
if err != nil {
768+
t.Fatalf("updateStatus: %v", err)
769+
}
770+
if res.RequeueAfter != 25*time.Second {
771+
t.Errorf("RequeueAfter = %v, want 25s", res.RequeueAfter)
772+
}
773+
}
774+
775+
// TestAgFollowUpDelay pins the follow-up-delay computation, including the
776+
// one-second floor that prevents a zero-delay tight loop when the anchor is
777+
// right at or beyond grace expiry.
778+
func TestAgFollowUpDelay(t *testing.T) {
779+
tests := []struct {
780+
name string
781+
anchorOffset time.Duration
782+
want time.Duration
783+
}{
784+
{"anchor 10s ago — grace has 20s left", -10 * time.Second, 20 * time.Second},
785+
{"anchor 5s ago — grace has 25s left", -5 * time.Second, 25 * time.Second},
786+
{"anchor right at grace expiry — floor to 1s", -stuckGracePeriod, time.Second},
787+
{"anchor past grace expiry — floor to 1s", -2 * stuckGracePeriod, time.Second},
788+
{"anchor now — full grace remaining", 0, stuckGracePeriod},
789+
}
790+
for _, tt := range tests {
791+
t.Run(tt.name, func(t *testing.T) {
792+
got := agFollowUpDelay(referenceTime.Add(tt.anchorOffset), referenceTime)
793+
if got != tt.want {
794+
t.Errorf("agFollowUpDelay(offset=%v) = %v, want %v", tt.anchorOffset, got, tt.want)
795+
}
796+
})
797+
}
798+
}
799+
687800
// stuckAG builds an ArtifactGenerator in the fluxcd/pkg#934 stall signature:
688801
// Inventory populated, ObservedSourcesDigest set, Ready=Unknown on the current
689802
// generation, LastTransitionTime old enough to clear the grace period. Used by

0 commit comments

Comments
 (0)