Skip to content

Commit ff4f02c

Browse files
committed
test(operator): pin maybeRecoverArtifactGenerator branches, tighten wait assertions, polish comments
Address the fourth-pass self-review's non-blocking findings: * Add TestMaybeRecoverArtifactGenerator_{ForceBranch, WaitBranch, GiveUpBranch, ResetsCounterOnFreshTransition}. The three switch outcomes of the driver (force / wait / give-up) and the reset-counter guard on a fresh Ready.LastTransitionTime transition were previously only covered indirectly through their pure-function components. Now each outcome is asserted end-to-end against a fake client with a status subresource, verifying the Ready condition, tracking annotations, RequeueAfter, and — for the wait branch — that no AG writes are issued at all. * Tighten TestDecideRecovery's wait-branch assertions from a loose lower bound to exact equality. decideRecovery is deterministic (needed - elapsed) so an exact assertion catches subtle backoff regressions a lower bound would miss (coderabbit's inline nit). * Rename the local var `wantWaitMin` to `wantWait` for the tighter assertion shape. * Fix two stale identifier references in comments (maybeRequeueArtifactGenerator, AwaitingSourceWatcherRequeue) left over from the rename pass. * Add a docstring paragraph on forceArtifactGeneratorDrift explaining why the JSON-merge-patch full-array-replace on `.status.conditions[]` is net-neutral in this workaround's context. Signed-off-by: Ivan Okhotnikov <[email protected]>
1 parent 7b49d91 commit ff4f02c

2 files changed

Lines changed: 250 additions & 36 deletions

File tree

‎internal/operator/packagesource_reconciler.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ func (r *PackageSourceReconciler) Reconcile(ctx context.Context, req ctrl.Reques
101101
// guard the retry driver could keep writing annotations to an
102102
// ArtifactGenerator whose owning PackageSource is about to be garbage
103103
// collected, and Status().Update below would leave a misleading
104-
// "AwaitingSourceWatcherRequeue" condition on a doomed object.
104+
// "AwaitingSourceWatcherRecovery" condition on a doomed object.
105105
// ownerReference cascade takes care of the actual AG cleanup.
106106
if !packageSource.DeletionTimestamp.IsZero() {
107107
return ctrl.Result{}, nil
@@ -384,7 +384,7 @@ func (r *PackageSourceReconciler) createOrUpdate(ctx context.Context, obj client
384384
// ArtifactGenerator). It may return a Result with RequeueAfter set when the
385385
// ArtifactGenerator's upstream Ready condition is stuck and the reconciler is
386386
// driving source-watcher through a bounded requeue schedule; see
387-
// maybeRequeueArtifactGenerator for the strategy. `now` is passed by the
387+
// maybeRecoverArtifactGenerator for the strategy. `now` is passed by the
388388
// caller so every time-sensitive check inside this reconcile agrees on the
389389
// same wall-clock reading.
390390
func (r *PackageSourceReconciler) updateStatus(ctx context.Context, packageSource *cozyv1alpha1.PackageSource, now time.Time) (ctrl.Result, error) {
@@ -699,6 +699,14 @@ func backoffFor(attempt int) time.Duration {
699699
// Ready=False signal from the successful status write, the extra force-drift
700700
// on that retry is benign (source-watcher is already mid-rebuild).
701701
//
702+
// JSON merge patch semantics note: `client.MergeFrom` produces a JSON merge
703+
// patch, which replaces arrays in full (RFC 7396). If source-watcher writes a
704+
// new Reconciling condition between our Get and our status Patch, that
705+
// concurrent addition is overwritten by our patch. This is net-neutral —
706+
// source-watcher's next reconcile (which our force-drift is about to trigger)
707+
// re-emits Reconciling anyway, and any legitimate concurrent add would already
708+
// have been at risk from the same split-write race this workaround exists for.
709+
//
702710
// On success, `ag.Status.Conditions` and `ag.Annotations` reflect the
703711
// persisted state so the caller can re-read the same pointer. On any Patch
704712
// failure the corresponding field is rolled back to its pre-call value.

‎internal/operator/packagesource_reconciler_test.go‎

Lines changed: 240 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -201,47 +201,50 @@ func TestArtifactGeneratorStuck(t *testing.T) {
201201
}
202202
}
203203

204-
// TestDecideRequeue locks in the bounded-retry state machine that drives
205-
// source-watcher out of the stuck window. First attempt bumps immediately; the
206-
// N-th subsequent attempt waits backoffFor(N) since the last bump before firing
207-
// again; after maxRecoveryAttempts fruitless attempts we surface Ready=False.
208-
func TestDecideRequeue(t *testing.T) {
204+
// TestDecideRecovery locks in the bounded-retry state machine that drives
205+
// source-watcher out of the stuck window. First attempt forces immediately;
206+
// the N-th subsequent attempt waits backoffFor(N) since the last force before
207+
// firing again; after maxRecoveryAttempts fruitless attempts we surface
208+
// Ready=False. Wait durations are asserted for exact equality because
209+
// decideRecovery is deterministic (`needed - elapsed`) — a loose bound would
210+
// silently miss a backoff-schedule regression.
211+
func TestDecideRecovery(t *testing.T) {
209212
tests := []struct {
210-
name string
211-
attempts int
212-
lastRequeueAt time.Time
213-
now time.Time
214-
wantAction recoveryAction
215-
wantWaitMin time.Duration // wait must be > 0 and roughly this long; 0 means don't check
213+
name string
214+
attempts int
215+
lastRecoveryAt time.Time
216+
now time.Time
217+
wantAction recoveryAction
218+
wantWait time.Duration // 0 means don't check (Force / GiveUp branches ignore wait)
216219
}{
217220
{
218-
name: "first-ever detection — bump immediately",
221+
name: "first-ever detection — force immediately",
219222
attempts: 0,
220223
now: referenceTime,
221224
wantAction: recoveryActionForce,
222225
},
223226
{
224-
name: "backoff not yet elapsed after attempt 1 — wait",
225-
attempts: 1,
226-
lastRequeueAt: referenceTime.Add(-10 * time.Second),
227-
now: referenceTime,
228-
wantAction: recoveryActionWait,
229-
wantWaitMin: 19 * time.Second, // ~initialBackoff (30s) minus elapsed 10s
227+
name: "backoff not yet elapsed after attempt 1 — wait",
228+
attempts: 1,
229+
lastRecoveryAt: referenceTime.Add(-10 * time.Second),
230+
now: referenceTime,
231+
wantAction: recoveryActionWait,
232+
wantWait: 20 * time.Second, // initialBackoff (30s) minus elapsed 10s
230233
},
231234
{
232-
name: "backoff elapsed after attempt 1 — bump",
233-
attempts: 1,
234-
lastRequeueAt: referenceTime.Add(-45 * time.Second),
235-
now: referenceTime,
236-
wantAction: recoveryActionForce,
235+
name: "backoff elapsed after attempt 1 — force",
236+
attempts: 1,
237+
lastRecoveryAt: referenceTime.Add(-45 * time.Second),
238+
now: referenceTime,
239+
wantAction: recoveryActionForce,
237240
},
238241
{
239-
name: "backoff after attempt 3 (2m) — wait",
240-
attempts: 3,
241-
lastRequeueAt: referenceTime.Add(-30 * time.Second),
242-
now: referenceTime,
243-
wantAction: recoveryActionWait,
244-
wantWaitMin: time.Minute,
242+
name: "backoff after attempt 3 (2m) — wait 90s",
243+
attempts: 3,
244+
lastRecoveryAt: referenceTime.Add(-30 * time.Second),
245+
now: referenceTime,
246+
wantAction: recoveryActionWait,
247+
wantWait: 90 * time.Second, // backoffFor(3)=2m minus elapsed 30s
245248
},
246249
{
247250
name: "attempts exhausted — give up",
@@ -259,12 +262,12 @@ func TestDecideRequeue(t *testing.T) {
259262

260263
for _, tt := range tests {
261264
t.Run(tt.name, func(t *testing.T) {
262-
got := decideRecovery(tt.attempts, tt.lastRequeueAt, tt.now)
265+
got := decideRecovery(tt.attempts, tt.lastRecoveryAt, tt.now)
263266
if got.action != tt.wantAction {
264267
t.Fatalf("action = %v, want %v", got.action, tt.wantAction)
265268
}
266-
if tt.wantWaitMin > 0 && got.wait < tt.wantWaitMin {
267-
t.Errorf("wait = %v, want at least %v", got.wait, tt.wantWaitMin)
269+
if tt.wantWait > 0 && got.wait != tt.wantWait {
270+
t.Errorf("wait = %v, want %v", got.wait, tt.wantWait)
268271
}
269272
})
270273
}
@@ -296,11 +299,11 @@ func TestBackoffFor(t *testing.T) {
296299
}
297300
}
298301

299-
// TestReadRequeueTracking verifies that a corrupted or malformed retry-tracking
302+
// TestReadRecoveryTracking verifies that a corrupted or malformed retry-tracking
300303
// annotation cannot wedge the retry loop by producing a nonsensical counter —
301304
// missing, empty, non-numeric, or negative values must all read back as
302305
// "no prior attempts".
303-
func TestReadRequeueTracking(t *testing.T) {
306+
func TestReadRecoveryTracking(t *testing.T) {
304307
fixed := referenceTime.UTC().Format(time.RFC3339Nano)
305308

306309
tests := []struct {
@@ -680,3 +683,206 @@ func TestReconcile_DeletionTimestamp_EarlyReturn(t *testing.T) {
680683
t.Errorf("Reconcile issued %d writes on a deleted object, want 0", writes)
681684
}
682685
}
686+
687+
// stuckAG builds an ArtifactGenerator in the fluxcd/pkg#934 stall signature:
688+
// Inventory populated, ObservedSourcesDigest set, Ready=Unknown on the current
689+
// generation, LastTransitionTime old enough to clear the grace period. Used by
690+
// the maybeRecoverArtifactGenerator tests below.
691+
func stuckAG(t *testing.T, annotations map[string]string, readyTransition time.Time) *sourcewatcherv1beta1.ArtifactGenerator {
692+
t.Helper()
693+
return &sourcewatcherv1beta1.ArtifactGenerator{
694+
ObjectMeta: metav1.ObjectMeta{
695+
Name: "example",
696+
Namespace: "cozy-system",
697+
Generation: 1,
698+
Annotations: annotations,
699+
},
700+
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
701+
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
702+
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
703+
},
704+
ObservedSourcesDigest: "sha256:0e7",
705+
Conditions: []metav1.Condition{{
706+
Type: "Ready",
707+
Status: metav1.ConditionUnknown,
708+
Reason: "Progressing",
709+
Message: "Reconciliation in progress",
710+
ObservedGeneration: 1,
711+
LastTransitionTime: metav1.NewTime(readyTransition),
712+
}},
713+
},
714+
}
715+
}
716+
717+
// TestMaybeRecoverArtifactGenerator_ForceBranch pins the first-time-stuck path:
718+
// no prior tracking annotations, so decideRecovery returns Force, we patch
719+
// Ready=False on the AG's status, update tracking annotations, and set the
720+
// PackageSource to Unknown/AwaitingSourceWatcherRecovery with a RequeueAfter
721+
// equal to backoffFor(1).
722+
func TestMaybeRecoverArtifactGenerator_ForceBranch(t *testing.T) {
723+
ps := &cozyv1alpha1.PackageSource{
724+
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
725+
}
726+
ag := stuckAG(t, nil, referenceTime.Add(-2*time.Minute))
727+
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
728+
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
729+
730+
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
731+
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
732+
if err != nil {
733+
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
734+
}
735+
if res.RequeueAfter != backoffFor(1) {
736+
t.Errorf("RequeueAfter = %v, want %v", res.RequeueAfter, backoffFor(1))
737+
}
738+
739+
// AG status.Ready must be False in apiserver.
740+
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
741+
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
742+
t.Fatalf("Get AG: %v", err)
743+
}
744+
agReady := meta.FindStatusCondition(persistedAG.Status.Conditions, "Ready")
745+
if agReady == nil || agReady.Status != metav1.ConditionFalse {
746+
t.Errorf("AG Ready in apiserver = %+v, want False", agReady)
747+
}
748+
if persistedAG.Annotations[annotationRecoveryAttempts] != "1" {
749+
t.Errorf("AG %s annotation = %q, want 1", annotationRecoveryAttempts, persistedAG.Annotations[annotationRecoveryAttempts])
750+
}
751+
752+
// PackageSource must be Unknown/AwaitingSourceWatcherRecovery in apiserver.
753+
persistedPS := &cozyv1alpha1.PackageSource{}
754+
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
755+
t.Fatalf("Get PS: %v", err)
756+
}
757+
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
758+
if psReady == nil || psReady.Status != metav1.ConditionUnknown || psReady.Reason != reasonAwaitingRecovery {
759+
t.Errorf("PS Ready = %+v, want Unknown/%s", psReady, reasonAwaitingRecovery)
760+
}
761+
}
762+
763+
// TestMaybeRecoverArtifactGenerator_WaitBranch pins the in-backoff-window path:
764+
// prior attempt exists, backoff not yet elapsed, so decideRecovery returns
765+
// Wait and we do NOT touch the AG's status/annotations — only the PackageSource
766+
// gets Unknown/AwaitingSourceWatcherRecovery and RequeueAfter set to the
767+
// remaining backoff.
768+
func TestMaybeRecoverArtifactGenerator_WaitBranch(t *testing.T) {
769+
priorForceAt := referenceTime.Add(-10 * time.Second) // 20s remaining on initialBackoff=30s
770+
ps := &cozyv1alpha1.PackageSource{
771+
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
772+
}
773+
ag := stuckAG(t, map[string]string{
774+
annotationRecoveryAttempts: "1",
775+
annotationLastRecoveryAt: priorForceAt.UTC().Format(time.RFC3339Nano),
776+
}, referenceTime.Add(-2*time.Minute))
777+
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
778+
agWriteCount := 0
779+
watched := interceptor.NewClient(c, interceptor.Funcs{
780+
Patch: func(ctx context.Context, cli client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
781+
if _, ok := obj.(*sourcewatcherv1beta1.ArtifactGenerator); ok {
782+
agWriteCount++
783+
}
784+
return cli.Patch(ctx, obj, patch, opts...)
785+
},
786+
SubResourcePatch: func(ctx context.Context, cli client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
787+
if _, ok := obj.(*sourcewatcherv1beta1.ArtifactGenerator); ok {
788+
agWriteCount++
789+
}
790+
return cli.SubResource(sub).Patch(ctx, obj, patch, opts...)
791+
},
792+
})
793+
r := &PackageSourceReconciler{Client: watched, Scheme: testScheme(t)}
794+
795+
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
796+
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
797+
if err != nil {
798+
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
799+
}
800+
if res.RequeueAfter != 20*time.Second {
801+
t.Errorf("RequeueAfter = %v, want 20s", res.RequeueAfter)
802+
}
803+
if agWriteCount != 0 {
804+
t.Errorf("wait branch wrote %d times to AG, want 0", agWriteCount)
805+
}
806+
persistedPS := &cozyv1alpha1.PackageSource{}
807+
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
808+
t.Fatalf("Get PS: %v", err)
809+
}
810+
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
811+
if psReady == nil || psReady.Reason != reasonAwaitingRecovery {
812+
t.Errorf("PS Ready = %+v, want reason %s", psReady, reasonAwaitingRecovery)
813+
}
814+
}
815+
816+
// TestMaybeRecoverArtifactGenerator_GiveUpBranch pins the exhaustion path:
817+
// attempts equals maxRecoveryAttempts, so we surface the failure as
818+
// PackageSource Ready=False/SourceWatcherStalled and schedule no follow-up
819+
// reconcile (an operator must intervene). The AG is left untouched.
820+
func TestMaybeRecoverArtifactGenerator_GiveUpBranch(t *testing.T) {
821+
ps := &cozyv1alpha1.PackageSource{
822+
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
823+
}
824+
priorForceAt := referenceTime.Add(-time.Hour).UTC().Format(time.RFC3339Nano)
825+
ag := stuckAG(t, map[string]string{
826+
annotationRecoveryAttempts: strconv.Itoa(maxRecoveryAttempts),
827+
annotationLastRecoveryAt: priorForceAt,
828+
}, referenceTime.Add(-time.Hour)) // last transition BEFORE lastRecoveryAt so reset-counter doesn't fire
829+
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
830+
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
831+
832+
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
833+
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
834+
if err != nil {
835+
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
836+
}
837+
if res.RequeueAfter != 0 {
838+
t.Errorf("RequeueAfter = %v, want 0 (give-up path stops rescheduling)", res.RequeueAfter)
839+
}
840+
persistedPS := &cozyv1alpha1.PackageSource{}
841+
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
842+
t.Fatalf("Get PS: %v", err)
843+
}
844+
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
845+
if psReady == nil || psReady.Status != metav1.ConditionFalse || psReady.Reason != reasonSourceWatcherBad {
846+
t.Errorf("PS Ready = %+v, want False/%s", psReady, reasonSourceWatcherBad)
847+
}
848+
}
849+
850+
// TestMaybeRecoverArtifactGenerator_ResetsCounterOnFreshTransition pins the
851+
// fresh-episode escape hatch: attempts=maxRecoveryAttempts BUT the AG's Ready
852+
// condition has a LastTransitionTime newer than lastRecoveryAt. That means
853+
// source-watcher touched the condition after our last give-up (whether the AG
854+
// briefly recovered and re-stalled or a concurrent operator moved on), so this
855+
// is a fresh stall episode and deserves a fresh budget rather than an instant
856+
// give-up. Expected: counter resets to 0 → Force branch → attempts land at 1.
857+
func TestMaybeRecoverArtifactGenerator_ResetsCounterOnFreshTransition(t *testing.T) {
858+
priorForceAt := referenceTime.Add(-time.Hour)
859+
freshTransitionAt := referenceTime.Add(-2 * time.Minute) // AFTER priorForceAt
860+
ps := &cozyv1alpha1.PackageSource{
861+
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
862+
}
863+
ag := stuckAG(t, map[string]string{
864+
annotationRecoveryAttempts: strconv.Itoa(maxRecoveryAttempts),
865+
annotationLastRecoveryAt: priorForceAt.UTC().Format(time.RFC3339Nano),
866+
}, freshTransitionAt)
867+
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
868+
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
869+
870+
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
871+
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
872+
if err != nil {
873+
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
874+
}
875+
// If reset worked, we take the Force branch — RequeueAfter is backoffFor(1),
876+
// not zero (which give-up would produce).
877+
if res.RequeueAfter != backoffFor(1) {
878+
t.Errorf("RequeueAfter = %v, want backoffFor(1)=%v — reset-counter branch did not fire", res.RequeueAfter, backoffFor(1))
879+
}
880+
// Attempts counter must now be 1 in the apiserver (reset to 0, incremented on Force).
881+
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
882+
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
883+
t.Fatalf("Get AG: %v", err)
884+
}
885+
if got := persistedAG.Annotations[annotationRecoveryAttempts]; got != "1" {
886+
t.Errorf("AG %s = %q after reset, want 1", annotationRecoveryAttempts, got)
887+
}
888+
}

0 commit comments

Comments
 (0)