@@ -201,47 +201,50 @@ func TestArtifactGeneratorStuck(t *testing.T) {
201201 }
202202}
203203
204- // TestDecideRequeue locks in the bounded-retry state machine that drives
205- // source-watcher out of the stuck window. First attempt bumps immediately; the
206- // N-th subsequent attempt waits backoffFor(N) since the last bump before firing
207- // again; after maxRecoveryAttempts fruitless attempts we surface Ready=False.
208- func TestDecideRequeue (t * testing.T ) {
204+ // TestDecideRecovery locks in the bounded-retry state machine that drives
205+ // source-watcher out of the stuck window. First attempt forces immediately;
206+ // the N-th subsequent attempt waits backoffFor(N) since the last force before
207+ // firing again; after maxRecoveryAttempts fruitless attempts we surface
208+ // Ready=False. Wait durations are asserted for exact equality because
209+ // decideRecovery is deterministic (`needed - elapsed`) — a loose bound would
210+ // silently miss a backoff-schedule regression.
211+ func TestDecideRecovery (t * testing.T ) {
209212 tests := []struct {
210- name string
211- attempts int
212- lastRequeueAt time.Time
213- now time.Time
214- wantAction recoveryAction
215- wantWaitMin time.Duration // wait must be > 0 and roughly this long; 0 means don't check
213+ name string
214+ attempts int
215+ lastRecoveryAt time.Time
216+ now time.Time
217+ wantAction recoveryAction
218+ wantWait time.Duration // 0 means don't check (Force / GiveUp branches ignore wait)
216219 }{
217220 {
218- name : "first-ever detection — bump immediately" ,
221+ name : "first-ever detection — force immediately" ,
219222 attempts : 0 ,
220223 now : referenceTime ,
221224 wantAction : recoveryActionForce ,
222225 },
223226 {
224- name : "backoff not yet elapsed after attempt 1 — wait" ,
225- attempts : 1 ,
226- lastRequeueAt : referenceTime .Add (- 10 * time .Second ),
227- now : referenceTime ,
228- wantAction : recoveryActionWait ,
229- wantWaitMin : 19 * time .Second , // ~ initialBackoff (30s) minus elapsed 10s
227+ name : "backoff not yet elapsed after attempt 1 — wait" ,
228+ attempts : 1 ,
229+ lastRecoveryAt : referenceTime .Add (- 10 * time .Second ),
230+ now : referenceTime ,
231+ wantAction : recoveryActionWait ,
232+ wantWait : 20 * time .Second , // initialBackoff (30s) minus elapsed 10s
230233 },
231234 {
232- name : "backoff elapsed after attempt 1 — bump " ,
233- attempts : 1 ,
234- lastRequeueAt : referenceTime .Add (- 45 * time .Second ),
235- now : referenceTime ,
236- wantAction : recoveryActionForce ,
235+ name : "backoff elapsed after attempt 1 — force " ,
236+ attempts : 1 ,
237+ lastRecoveryAt : referenceTime .Add (- 45 * time .Second ),
238+ now : referenceTime ,
239+ wantAction : recoveryActionForce ,
237240 },
238241 {
239- name : "backoff after attempt 3 (2m) — wait" ,
240- attempts : 3 ,
241- lastRequeueAt : referenceTime .Add (- 30 * time .Second ),
242- now : referenceTime ,
243- wantAction : recoveryActionWait ,
244- wantWaitMin : time .Minute ,
242+ name : "backoff after attempt 3 (2m) — wait 90s " ,
243+ attempts : 3 ,
244+ lastRecoveryAt : referenceTime .Add (- 30 * time .Second ),
245+ now : referenceTime ,
246+ wantAction : recoveryActionWait ,
247+ wantWait : 90 * time .Second , // backoffFor(3)=2m minus elapsed 30s
245248 },
246249 {
247250 name : "attempts exhausted — give up" ,
@@ -259,12 +262,12 @@ func TestDecideRequeue(t *testing.T) {
259262
260263 for _ , tt := range tests {
261264 t .Run (tt .name , func (t * testing.T ) {
262- got := decideRecovery (tt .attempts , tt .lastRequeueAt , tt .now )
265+ got := decideRecovery (tt .attempts , tt .lastRecoveryAt , tt .now )
263266 if got .action != tt .wantAction {
264267 t .Fatalf ("action = %v, want %v" , got .action , tt .wantAction )
265268 }
266- if tt .wantWaitMin > 0 && got .wait < tt .wantWaitMin {
267- t .Errorf ("wait = %v, want at least %v" , got .wait , tt .wantWaitMin )
269+ if tt .wantWait > 0 && got .wait != tt .wantWait {
270+ t .Errorf ("wait = %v, want %v" , got .wait , tt .wantWait )
268271 }
269272 })
270273 }
@@ -296,11 +299,11 @@ func TestBackoffFor(t *testing.T) {
296299 }
297300}
298301
299- // TestReadRequeueTracking verifies that a corrupted or malformed retry-tracking
302+ // TestReadRecoveryTracking verifies that a corrupted or malformed retry-tracking
300303// annotation cannot wedge the retry loop by producing a nonsensical counter —
301304// missing, empty, non-numeric, or negative values must all read back as
302305// "no prior attempts".
303- func TestReadRequeueTracking (t * testing.T ) {
306+ func TestReadRecoveryTracking (t * testing.T ) {
304307 fixed := referenceTime .UTC ().Format (time .RFC3339Nano )
305308
306309 tests := []struct {
@@ -680,3 +683,206 @@ func TestReconcile_DeletionTimestamp_EarlyReturn(t *testing.T) {
680683 t .Errorf ("Reconcile issued %d writes on a deleted object, want 0" , writes )
681684 }
682685}
686+
687+ // stuckAG builds an ArtifactGenerator in the fluxcd/pkg#934 stall signature:
688+ // Inventory populated, ObservedSourcesDigest set, Ready=Unknown on the current
689+ // generation, LastTransitionTime old enough to clear the grace period. Used by
690+ // the maybeRecoverArtifactGenerator tests below.
691+ func stuckAG (t * testing.T , annotations map [string ]string , readyTransition time.Time ) * sourcewatcherv1beta1.ArtifactGenerator {
692+ t .Helper ()
693+ return & sourcewatcherv1beta1.ArtifactGenerator {
694+ ObjectMeta : metav1.ObjectMeta {
695+ Name : "example" ,
696+ Namespace : "cozy-system" ,
697+ Generation : 1 ,
698+ Annotations : annotations ,
699+ },
700+ Status : sourcewatcherv1beta1.ArtifactGeneratorStatus {
701+ Inventory : []sourcewatcherv1beta1.ExternalArtifactReference {
702+ {Name : "one" , Namespace : "cozy-system" , Digest : "sha256:aaa" },
703+ },
704+ ObservedSourcesDigest : "sha256:0e7" ,
705+ Conditions : []metav1.Condition {{
706+ Type : "Ready" ,
707+ Status : metav1 .ConditionUnknown ,
708+ Reason : "Progressing" ,
709+ Message : "Reconciliation in progress" ,
710+ ObservedGeneration : 1 ,
711+ LastTransitionTime : metav1 .NewTime (readyTransition ),
712+ }},
713+ },
714+ }
715+ }
716+
717+ // TestMaybeRecoverArtifactGenerator_ForceBranch pins the first-time-stuck path:
718+ // no prior tracking annotations, so decideRecovery returns Force, we patch
719+ // Ready=False on the AG's status, update tracking annotations, and set the
720+ // PackageSource to Unknown/AwaitingSourceWatcherRecovery with a RequeueAfter
721+ // equal to backoffFor(1).
722+ func TestMaybeRecoverArtifactGenerator_ForceBranch (t * testing.T ) {
723+ ps := & cozyv1alpha1.PackageSource {
724+ ObjectMeta : metav1.ObjectMeta {Name : "example" , Namespace : "cozy-system" , Generation : 1 },
725+ }
726+ ag := stuckAG (t , nil , referenceTime .Add (- 2 * time .Minute ))
727+ c := fake .NewClientBuilder ().WithScheme (testScheme (t )).WithStatusSubresource (ps , ag ).WithObjects (ps , ag ).Build ()
728+ r := & PackageSourceReconciler {Client : c , Scheme : testScheme (t )}
729+
730+ ready := meta .FindStatusCondition (ag .Status .Conditions , "Ready" )
731+ res , err := r .maybeRecoverArtifactGenerator (context .Background (), ps , ag , ready , referenceTime )
732+ if err != nil {
733+ t .Fatalf ("maybeRecoverArtifactGenerator: %v" , err )
734+ }
735+ if res .RequeueAfter != backoffFor (1 ) {
736+ t .Errorf ("RequeueAfter = %v, want %v" , res .RequeueAfter , backoffFor (1 ))
737+ }
738+
739+ // AG status.Ready must be False in apiserver.
740+ persistedAG := & sourcewatcherv1beta1.ArtifactGenerator {}
741+ if err := c .Get (context .Background (), client .ObjectKeyFromObject (ag ), persistedAG ); err != nil {
742+ t .Fatalf ("Get AG: %v" , err )
743+ }
744+ agReady := meta .FindStatusCondition (persistedAG .Status .Conditions , "Ready" )
745+ if agReady == nil || agReady .Status != metav1 .ConditionFalse {
746+ t .Errorf ("AG Ready in apiserver = %+v, want False" , agReady )
747+ }
748+ if persistedAG .Annotations [annotationRecoveryAttempts ] != "1" {
749+ t .Errorf ("AG %s annotation = %q, want 1" , annotationRecoveryAttempts , persistedAG .Annotations [annotationRecoveryAttempts ])
750+ }
751+
752+ // PackageSource must be Unknown/AwaitingSourceWatcherRecovery in apiserver.
753+ persistedPS := & cozyv1alpha1.PackageSource {}
754+ if err := c .Get (context .Background (), client .ObjectKeyFromObject (ps ), persistedPS ); err != nil {
755+ t .Fatalf ("Get PS: %v" , err )
756+ }
757+ psReady := meta .FindStatusCondition (persistedPS .Status .Conditions , "Ready" )
758+ if psReady == nil || psReady .Status != metav1 .ConditionUnknown || psReady .Reason != reasonAwaitingRecovery {
759+ t .Errorf ("PS Ready = %+v, want Unknown/%s" , psReady , reasonAwaitingRecovery )
760+ }
761+ }
762+
763+ // TestMaybeRecoverArtifactGenerator_WaitBranch pins the in-backoff-window path:
764+ // prior attempt exists, backoff not yet elapsed, so decideRecovery returns
765+ // Wait and we do NOT touch the AG's status/annotations — only the PackageSource
766+ // gets Unknown/AwaitingSourceWatcherRecovery and RequeueAfter set to the
767+ // remaining backoff.
768+ func TestMaybeRecoverArtifactGenerator_WaitBranch (t * testing.T ) {
769+ priorForceAt := referenceTime .Add (- 10 * time .Second ) // 20s remaining on initialBackoff=30s
770+ ps := & cozyv1alpha1.PackageSource {
771+ ObjectMeta : metav1.ObjectMeta {Name : "example" , Namespace : "cozy-system" , Generation : 1 },
772+ }
773+ ag := stuckAG (t , map [string ]string {
774+ annotationRecoveryAttempts : "1" ,
775+ annotationLastRecoveryAt : priorForceAt .UTC ().Format (time .RFC3339Nano ),
776+ }, referenceTime .Add (- 2 * time .Minute ))
777+ c := fake .NewClientBuilder ().WithScheme (testScheme (t )).WithStatusSubresource (ps , ag ).WithObjects (ps , ag ).Build ()
778+ agWriteCount := 0
779+ watched := interceptor .NewClient (c , interceptor.Funcs {
780+ Patch : func (ctx context.Context , cli client.WithWatch , obj client.Object , patch client.Patch , opts ... client.PatchOption ) error {
781+ if _ , ok := obj .(* sourcewatcherv1beta1.ArtifactGenerator ); ok {
782+ agWriteCount ++
783+ }
784+ return cli .Patch (ctx , obj , patch , opts ... )
785+ },
786+ SubResourcePatch : func (ctx context.Context , cli client.Client , sub string , obj client.Object , patch client.Patch , opts ... client.SubResourcePatchOption ) error {
787+ if _ , ok := obj .(* sourcewatcherv1beta1.ArtifactGenerator ); ok {
788+ agWriteCount ++
789+ }
790+ return cli .SubResource (sub ).Patch (ctx , obj , patch , opts ... )
791+ },
792+ })
793+ r := & PackageSourceReconciler {Client : watched , Scheme : testScheme (t )}
794+
795+ ready := meta .FindStatusCondition (ag .Status .Conditions , "Ready" )
796+ res , err := r .maybeRecoverArtifactGenerator (context .Background (), ps , ag , ready , referenceTime )
797+ if err != nil {
798+ t .Fatalf ("maybeRecoverArtifactGenerator: %v" , err )
799+ }
800+ if res .RequeueAfter != 20 * time .Second {
801+ t .Errorf ("RequeueAfter = %v, want 20s" , res .RequeueAfter )
802+ }
803+ if agWriteCount != 0 {
804+ t .Errorf ("wait branch wrote %d times to AG, want 0" , agWriteCount )
805+ }
806+ persistedPS := & cozyv1alpha1.PackageSource {}
807+ if err := c .Get (context .Background (), client .ObjectKeyFromObject (ps ), persistedPS ); err != nil {
808+ t .Fatalf ("Get PS: %v" , err )
809+ }
810+ psReady := meta .FindStatusCondition (persistedPS .Status .Conditions , "Ready" )
811+ if psReady == nil || psReady .Reason != reasonAwaitingRecovery {
812+ t .Errorf ("PS Ready = %+v, want reason %s" , psReady , reasonAwaitingRecovery )
813+ }
814+ }
815+
816+ // TestMaybeRecoverArtifactGenerator_GiveUpBranch pins the exhaustion path:
817+ // attempts equals maxRecoveryAttempts, so we surface the failure as
818+ // PackageSource Ready=False/SourceWatcherStalled and schedule no follow-up
819+ // reconcile (an operator must intervene). The AG is left untouched.
820+ func TestMaybeRecoverArtifactGenerator_GiveUpBranch (t * testing.T ) {
821+ ps := & cozyv1alpha1.PackageSource {
822+ ObjectMeta : metav1.ObjectMeta {Name : "example" , Namespace : "cozy-system" , Generation : 1 },
823+ }
824+ priorForceAt := referenceTime .Add (- time .Hour ).UTC ().Format (time .RFC3339Nano )
825+ ag := stuckAG (t , map [string ]string {
826+ annotationRecoveryAttempts : strconv .Itoa (maxRecoveryAttempts ),
827+ annotationLastRecoveryAt : priorForceAt ,
828+ }, referenceTime .Add (- time .Hour )) // last transition BEFORE lastRecoveryAt so reset-counter doesn't fire
829+ c := fake .NewClientBuilder ().WithScheme (testScheme (t )).WithStatusSubresource (ps , ag ).WithObjects (ps , ag ).Build ()
830+ r := & PackageSourceReconciler {Client : c , Scheme : testScheme (t )}
831+
832+ ready := meta .FindStatusCondition (ag .Status .Conditions , "Ready" )
833+ res , err := r .maybeRecoverArtifactGenerator (context .Background (), ps , ag , ready , referenceTime )
834+ if err != nil {
835+ t .Fatalf ("maybeRecoverArtifactGenerator: %v" , err )
836+ }
837+ if res .RequeueAfter != 0 {
838+ t .Errorf ("RequeueAfter = %v, want 0 (give-up path stops rescheduling)" , res .RequeueAfter )
839+ }
840+ persistedPS := & cozyv1alpha1.PackageSource {}
841+ if err := c .Get (context .Background (), client .ObjectKeyFromObject (ps ), persistedPS ); err != nil {
842+ t .Fatalf ("Get PS: %v" , err )
843+ }
844+ psReady := meta .FindStatusCondition (persistedPS .Status .Conditions , "Ready" )
845+ if psReady == nil || psReady .Status != metav1 .ConditionFalse || psReady .Reason != reasonSourceWatcherBad {
846+ t .Errorf ("PS Ready = %+v, want False/%s" , psReady , reasonSourceWatcherBad )
847+ }
848+ }
849+
850+ // TestMaybeRecoverArtifactGenerator_ResetsCounterOnFreshTransition pins the
851+ // fresh-episode escape hatch: attempts=maxRecoveryAttempts BUT the AG's Ready
852+ // condition has a LastTransitionTime newer than lastRecoveryAt. That means
853+ // source-watcher touched the condition after our last give-up (whether the AG
854+ // briefly recovered and re-stalled or a concurrent operator moved on), so this
855+ // is a fresh stall episode and deserves a fresh budget rather than an instant
856+ // give-up. Expected: counter resets to 0 → Force branch → attempts land at 1.
857+ func TestMaybeRecoverArtifactGenerator_ResetsCounterOnFreshTransition (t * testing.T ) {
858+ priorForceAt := referenceTime .Add (- time .Hour )
859+ freshTransitionAt := referenceTime .Add (- 2 * time .Minute ) // AFTER priorForceAt
860+ ps := & cozyv1alpha1.PackageSource {
861+ ObjectMeta : metav1.ObjectMeta {Name : "example" , Namespace : "cozy-system" , Generation : 1 },
862+ }
863+ ag := stuckAG (t , map [string ]string {
864+ annotationRecoveryAttempts : strconv .Itoa (maxRecoveryAttempts ),
865+ annotationLastRecoveryAt : priorForceAt .UTC ().Format (time .RFC3339Nano ),
866+ }, freshTransitionAt )
867+ c := fake .NewClientBuilder ().WithScheme (testScheme (t )).WithStatusSubresource (ps , ag ).WithObjects (ps , ag ).Build ()
868+ r := & PackageSourceReconciler {Client : c , Scheme : testScheme (t )}
869+
870+ ready := meta .FindStatusCondition (ag .Status .Conditions , "Ready" )
871+ res , err := r .maybeRecoverArtifactGenerator (context .Background (), ps , ag , ready , referenceTime )
872+ if err != nil {
873+ t .Fatalf ("maybeRecoverArtifactGenerator: %v" , err )
874+ }
875+ // If reset worked, we take the Force branch — RequeueAfter is backoffFor(1),
876+ // not zero (which give-up would produce).
877+ if res .RequeueAfter != backoffFor (1 ) {
878+ t .Errorf ("RequeueAfter = %v, want backoffFor(1)=%v — reset-counter branch did not fire" , res .RequeueAfter , backoffFor (1 ))
879+ }
880+ // Attempts counter must now be 1 in the apiserver (reset to 0, incremented on Force).
881+ persistedAG := & sourcewatcherv1beta1.ArtifactGenerator {}
882+ if err := c .Get (context .Background (), client .ObjectKeyFromObject (ag ), persistedAG ); err != nil {
883+ t .Fatalf ("Get AG: %v" , err )
884+ }
885+ if got := persistedAG .Annotations [annotationRecoveryAttempts ]; got != "1" {
886+ t .Errorf ("AG %s = %q after reset, want 1" , annotationRecoveryAttempts , got )
887+ }
888+ }
0 commit comments