Skip to content

Commit 4c8f094

Browse files
committed
fix(controller): requeue monitors reconciled while the DataVolume watch syncs
Starting the DataVolume source replays every DataVolume to its handler at once, and the handler queues the matching monitors, while the reader is installed only after the source has synced. A monitor reconciled in that window saw no reader, kept its last verdict and returned with nothing left to queue it again, so a disk that failed or finished while the controller was down could keep a wrong WorkloadsReady until the next unrelated event. Such a reconcile still publishes the replica counts and now asks to be requeued after a few seconds, until the reader is in place or the sync attempt gives up. It is a requeue rather than an error because the window is expected: an error would count as a reconcile failure and log a line for every monitor in the cluster while it lasts. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent ed9d620 commit 4c8f094

2 files changed

Lines changed: 115 additions & 27 deletions

File tree

‎internal/controller/workloadmonitor_controller.go‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,11 @@ type WorkloadMonitorReconciler struct {
8080
// served, and until then DataVolumes do not count towards Operational.
8181
DataVolumeReader client.Reader
8282
dataVolumeMu sync.RWMutex
83+
// dataVolumeWatchSyncing is set while the DataVolume watch is started and
84+
// its reader is not installed yet. The source replays every DataVolume to
85+
// the handler in that window, and a reconcile it queues must be requeued:
86+
// nothing queues it again once the reader is in place.
87+
dataVolumeWatchSyncing bool
8388
// reconciled is set by the first Reconcile. controller-runtime starts the
8489
// workers only after the controller has started its sources
8590
// (pkg/internal/controller/controller.go, Controller.Start), so from then on
@@ -95,12 +100,26 @@ const (
95100
// A reconcile context has no deadline, so a reader installed on an informer
96101
// that never syncs would park every WorkloadMonitor reconcile for good.
97102
dataVolumeSyncTimeout = time.Minute
103+
// dataVolumeSyncRequeue is how soon a monitor reconciled while the
104+
// DataVolume watch syncs is looked at again.
105+
dataVolumeSyncRequeue = 5 * time.Second
98106
)
99107

100-
func (r *WorkloadMonitorReconciler) dataVolumeReader() client.Reader {
108+
var errDataVolumeWatchSyncing = errors.New("the DataVolume watch has not synced yet")
109+
110+
func (r *WorkloadMonitorReconciler) dataVolumeReader() (client.Reader, error) {
101111
r.dataVolumeMu.RLock()
102112
defer r.dataVolumeMu.RUnlock()
103-
return r.DataVolumeReader
113+
if r.DataVolumeReader == nil && r.dataVolumeWatchSyncing {
114+
return nil, errDataVolumeWatchSyncing
115+
}
116+
return r.DataVolumeReader, nil
117+
}
118+
119+
func (r *WorkloadMonitorReconciler) setDataVolumeWatchSyncing(syncing bool) {
120+
r.dataVolumeMu.Lock()
121+
defer r.dataVolumeMu.Unlock()
122+
r.dataVolumeWatchSyncing = syncing
104123
}
105124

106125
var dataVolumeGVK = schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}
@@ -176,6 +195,8 @@ func (r *WorkloadMonitorReconciler) tryStartDataVolumeWatch(ctx context.Context,
176195
dv.SetGroupVersionKind(dataVolumeGVK)
177196
src := source.Kind[client.Object](informers, dv,
178197
handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(client.Object(dv), r.Client)))
198+
r.setDataVolumeWatchSyncing(true)
199+
defer r.setDataVolumeWatchSyncing(false)
179200
if err := c.Watch(src); err != nil {
180201
return false, fmt.Errorf("watching DataVolumes: %w", err)
181202
}
@@ -198,9 +219,9 @@ func (r *WorkloadMonitorReconciler) tryStartDataVolumeWatch(ctx context.Context,
198219
// reader counts as no DataVolumes. read is false when the DataVolumes could not
199220
// be read: no reader yet, or a failed List.
200221
func (r *WorkloadMonitorReconciler) dataVolumesMessage(ctx context.Context, monitor *cozyv1alpha1.WorkloadMonitor) (message string, read bool, err error) {
201-
reader := r.dataVolumeReader()
222+
reader, err := r.dataVolumeReader()
202223
if reader == nil {
203-
return "", false, nil
224+
return "", false, err
204225
}
205226
list := &unstructured.UnstructuredList{}
206227
list.SetGroupVersionKind(dataVolumeGVK.GroupVersion().WithKind(dataVolumeGVK.Kind + "List"))
@@ -875,8 +896,14 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
875896
}
876897

877898
dataVolumesMessage, dataVolumesRead, dataVolumesErr := r.dataVolumesMessage(ctx, monitor)
899+
// The sync window is expected and short; an error here would count as a
900+
// reconcile failure for every monitor in the cluster while it lasts.
901+
dataVolumeWatchSyncing := errors.Is(dataVolumesErr, errDataVolumeWatchSyncing)
902+
if dataVolumeWatchSyncing {
903+
dataVolumesErr = nil
904+
}
878905
if dataVolumesErr != nil {
879-
logger.Error(dataVolumesErr, "Unable to list DataVolumes for WorkloadMonitor, keeping the last DataVolume verdict", "monitor", monitor.Name)
906+
logger.Error(dataVolumesErr, "Unable to read DataVolumes for WorkloadMonitor, keeping the last DataVolume verdict", "monitor", monitor.Name)
880907
}
881908

882909
// Update WorkloadMonitor status based on observed pods
@@ -926,6 +953,10 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
926953
return ctrl.Result{}, errors.Join(bucketMetricsErr, dataVolumesErr)
927954
}
928955

956+
if dataVolumeWatchSyncing {
957+
return ctrl.Result{RequeueAfter: dataVolumeSyncRequeue}, nil
958+
}
959+
929960
// Requeue periodically if there are BucketClaims to keep sizes up to date.
930961
// Bucket sizes come from Prometheus metrics that update every 60s.
931962
if len(bucketClaimList.Items) > 0 {

‎internal/controller/workloadmonitor_controller_test.go‎

Lines changed: 79 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1380,8 +1380,9 @@ func TestReconcile_DataVolumeKindNotServedIsNoDataVolumes(t *testing.T) {
13801380
}
13811381

13821382
// reconcileWithUnreadableDataVolumes reconciles a monitor that selects one
1383-
// ready pod while every DataVolume List fails, or while withReader is false.
1384-
func reconcileWithUnreadableDataVolumes(t *testing.T, withReader bool, status cozyv1alpha1.WorkloadMonitorStatus) (*cozyv1alpha1.WorkloadMonitor, error) {
1383+
// ready pod while every DataVolume List fails, or while withReader is false;
1384+
// syncing marks the DataVolume watch as started and not yet synced.
1385+
func reconcileWithUnreadableDataVolumes(t *testing.T, withReader, syncing bool, status cozyv1alpha1.WorkloadMonitorStatus) (*cozyv1alpha1.WorkloadMonitor, reconcile.Result, error) {
13851386
t.Helper()
13861387
s := newTestScheme()
13871388
selector := map[string]string{"app.kubernetes.io/instance": "m"}
@@ -1413,17 +1414,18 @@ func reconcileWithUnreadableDataVolumes(t *testing.T, withReader bool, status co
14131414
if withReader {
14141415
reconciler.DataVolumeReader = fakeClient
14151416
}
1417+
reconciler.dataVolumeWatchSyncing = syncing
14161418
req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "m", Namespace: "default"}}
1417-
_, err := reconciler.Reconcile(context.TODO(), req)
1419+
result, err := reconciler.Reconcile(context.TODO(), req)
14181420
updated := &cozyv1alpha1.WorkloadMonitor{}
14191421
if gerr := fakeClient.Get(context.TODO(), req.NamespacedName, updated); gerr != nil {
14201422
t.Fatalf("Failed to get updated WorkloadMonitor: %v", gerr)
14211423
}
1422-
return updated, err
1424+
return updated, result, err
14231425
}
14241426

14251427
func TestReconcile_DataVolumeListErrorStillPublishesStatus(t *testing.T) {
1426-
got, err := reconcileWithUnreadableDataVolumes(t, true, cozyv1alpha1.WorkloadMonitorStatus{})
1428+
got, _, err := reconcileWithUnreadableDataVolumes(t, true, false, cozyv1alpha1.WorkloadMonitorStatus{})
14271429
if err == nil {
14281430
t.Error("Reconcile returned nil on a DataVolume list failure, want the error so the request is retried")
14291431
}
@@ -1439,7 +1441,7 @@ func TestReconcile_UnreadDataVolumesKeepTheLastVerdict(t *testing.T) {
14391441
const stuck = "DataVolume m is ImportInProgress"
14401442
for _, withReader := range []bool{true, false} {
14411443
t.Run(fmt.Sprintf("reader=%v", withReader), func(t *testing.T) {
1442-
got, _ := reconcileWithUnreadableDataVolumes(t, withReader, cozyv1alpha1.WorkloadMonitorStatus{
1444+
got, _, _ := reconcileWithUnreadableDataVolumes(t, withReader, false, cozyv1alpha1.WorkloadMonitorStatus{
14431445
Operational: ptr.To(false),
14441446
Message: stuck,
14451447
Reason: cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady,
@@ -1454,6 +1456,24 @@ func TestReconcile_UnreadDataVolumesKeepTheLastVerdict(t *testing.T) {
14541456
}
14551457
}
14561458

1459+
func TestReconcile_SyncingDataVolumeWatchPublishesAndRequeues(t *testing.T) {
1460+
const stuck = "DataVolume m is Failed"
1461+
got, result, err := reconcileWithUnreadableDataVolumes(t, false, true, cozyv1alpha1.WorkloadMonitorStatus{
1462+
Operational: ptr.To(false),
1463+
Message: stuck,
1464+
Reason: cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady,
1465+
})
1466+
if err != nil || result.RequeueAfter <= 0 {
1467+
t.Errorf("Reconcile = %+v, %v while the DataVolume watch syncs, want a requeue and no error: the window is expected, not a failure", result, err)
1468+
}
1469+
if got.Status.ObservedReplicas != 1 || got.Status.AvailableReplicas != 1 {
1470+
t.Errorf("replicas observed=%d available=%d, want 1 and 1", got.Status.ObservedReplicas, got.Status.AvailableReplicas)
1471+
}
1472+
if got.Status.Operational == nil || *got.Status.Operational || got.Status.Message != stuck {
1473+
t.Errorf("Operational=%v Message=%q, want false and %q", got.Status.Operational, got.Status.Message, stuck)
1474+
}
1475+
}
1476+
14571477
func TestReconcile_NoDataVolumeReaderSkipsDataVolumes(t *testing.T) {
14581478
busy := "ImportInProgress"
14591479
got := reconcileDataVolumeMonitor(t, false, nil,
@@ -1497,9 +1517,14 @@ func TestDataVolumeAPIServed_DiscoveryErrorIsReturned(t *testing.T) {
14971517
type startingWatcher struct {
14981518
sources []source.Source
14991519
err error
1520+
// onWatch runs when Watch is called, before the source starts.
1521+
onWatch func()
15001522
}
15011523

15021524
func (w *startingWatcher) Watch(src source.Source) error {
1525+
if w.onWatch != nil {
1526+
w.onWatch()
1527+
}
15031528
if w.err != nil {
15041529
return w.err
15051530
}
@@ -1508,6 +1533,12 @@ func (w *startingWatcher) Watch(src source.Source) error {
15081533
return src.Start(context.Background(), queue)
15091534
}
15101535

1536+
func readerOf(r *WorkloadMonitorReconciler) client.Reader {
1537+
r.dataVolumeMu.RLock()
1538+
defer r.dataVolumeMu.RUnlock()
1539+
return r.DataVolumeReader
1540+
}
1541+
15111542
func servedMapper() meta.RESTMapper {
15121543
mapper := meta.NewDefaultRESTMapper(nil)
15131544
mapper.Add(dataVolumeGVK, meta.RESTScopeNamespace)
@@ -1536,17 +1567,17 @@ func TestTryStartDataVolumeWatch_WaitsUntilTheKindIsServed(t *testing.T) {
15361567
if err != nil || done {
15371568
t.Fatalf("before CDI: done=%v err=%v, want false, nil", done, err)
15381569
}
1539-
if len(w.sources) != 0 || r.dataVolumeReader() != nil {
1540-
t.Fatalf("before CDI: %d watches, reader %v; want none", len(w.sources), r.dataVolumeReader())
1570+
if len(w.sources) != 0 || readerOf(r) != nil {
1571+
t.Fatalf("before CDI: %d watches, reader %v; want none", len(w.sources), readerOf(r))
15411572
}
15421573

15431574
mapper.Add(dataVolumeGVK, meta.RESTScopeNamespace)
15441575
done, err = r.tryStartDataVolumeWatch(context.Background(), mapper, w, informers, time.Second)
15451576
if err != nil || !done {
15461577
t.Fatalf("after CDI: done=%v err=%v, want true, nil", done, err)
15471578
}
1548-
if len(w.sources) != 1 || r.dataVolumeReader() == nil {
1549-
t.Fatalf("after CDI: %d watches, reader %v; want one watch and a reader", len(w.sources), r.dataVolumeReader())
1579+
if len(w.sources) != 1 || readerOf(r) == nil {
1580+
t.Fatalf("after CDI: %d watches, reader %v; want one watch and a reader", len(w.sources), readerOf(r))
15501581
}
15511582
}
15521583

@@ -1556,8 +1587,8 @@ func TestTryStartDataVolumeWatch_WaitsForTheFirstReconcile(t *testing.T) {
15561587
r := &WorkloadMonitorReconciler{}
15571588
w := &startingWatcher{}
15581589
done, err := r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, &informertest.FakeInformers{}, time.Second)
1559-
if err != nil || done || len(w.sources) != 0 || r.dataVolumeReader() != nil {
1560-
t.Fatalf("done=%v err=%v watches=%d reader=%v, want nothing registered before the first reconcile", done, err, len(w.sources), r.dataVolumeReader())
1590+
if err != nil || done || len(w.sources) != 0 || readerOf(r) != nil {
1591+
t.Fatalf("done=%v err=%v watches=%d reader=%v, want nothing registered before the first reconcile", done, err, len(w.sources), readerOf(r))
15611592
}
15621593
}
15631594

@@ -1570,17 +1601,17 @@ func TestTryStartDataVolumeWatch_UnsyncedSourceLeavesNoReaderAndRetries(t *testi
15701601
fi.Synced = false
15711602

15721603
done, err := r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, informers, 50*time.Millisecond)
1573-
if err == nil || done || r.dataVolumeReader() != nil {
1574-
t.Fatalf("informer never synced: done=%v err=%v reader=%v, want an error, not done, no reader", done, err, r.dataVolumeReader())
1604+
if err == nil || done || readerOf(r) != nil {
1605+
t.Fatalf("informer never synced: done=%v err=%v reader=%v, want an error, not done, no reader", done, err, readerOf(r))
15751606
}
15761607

15771608
fi.SyncedLock.Lock()
15781609
fi.Synced = true
15791610
fi.SyncedLock.Unlock()
15801611

15811612
done, err = r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, informers, time.Second)
1582-
if err != nil || !done || r.dataVolumeReader() == nil {
1583-
t.Fatalf("retry after sync: done=%v err=%v reader=%v, want done and a reader", done, err, r.dataVolumeReader())
1613+
if err != nil || !done || readerOf(r) == nil {
1614+
t.Fatalf("retry after sync: done=%v err=%v reader=%v, want done and a reader", done, err, readerOf(r))
15841615
}
15851616
if len(w.sources) != 2 {
15861617
t.Fatalf("got %d watches, want a fresh source on the retry", len(w.sources))
@@ -1597,8 +1628,34 @@ func TestTryStartDataVolumeWatch_StoppingManagerLeavesNoReader(t *testing.T) {
15971628
ctx, cancel := context.WithCancel(context.Background())
15981629
cancel()
15991630
done, _ := r.tryStartDataVolumeWatch(ctx, servedMapper(), &startingWatcher{}, informers, time.Second)
1600-
if done || r.dataVolumeReader() != nil {
1601-
t.Fatalf("done=%v reader=%v, want no reader when the manager is stopping", done, r.dataVolumeReader())
1631+
if done || readerOf(r) != nil {
1632+
t.Fatalf("done=%v reader=%v, want no reader when the manager is stopping", done, readerOf(r))
1633+
}
1634+
}
1635+
1636+
// The source replays every DataVolume to the handler as soon as Watch starts
1637+
// it, before the reader is installed. A reconcile in that window must be
1638+
// retried, or the events that queued it are lost.
1639+
func TestTryStartDataVolumeWatch_ReconcileBeforeTheReaderIsRetried(t *testing.T) {
1640+
for _, synced := range []bool{true, false} {
1641+
t.Run(fmt.Sprintf("synced=%v", synced), func(t *testing.T) {
1642+
r := &WorkloadMonitorReconciler{}
1643+
r.reconciled.Store(true)
1644+
informers := &informertest.FakeInformers{}
1645+
fakeDataVolumeInformer(t, informers).Synced = synced
1646+
monitor := &cozyv1alpha1.WorkloadMonitor{ObjectMeta: metav1.ObjectMeta{Name: "m", Namespace: "default"}}
1647+
var errDuringWatch error
1648+
w := &startingWatcher{onWatch: func() {
1649+
_, _, errDuringWatch = r.dataVolumesMessage(context.Background(), monitor)
1650+
}}
1651+
_, _ = r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, informers, 50*time.Millisecond)
1652+
if errDuringWatch == nil {
1653+
t.Error("reading DataVolumes while the watch syncs returned no error, want one so the reconcile is retried")
1654+
}
1655+
if _, _, err := r.dataVolumesMessage(context.Background(), monitor); err != nil && !synced {
1656+
t.Errorf("after a sync timeout: %v, want no error so reconciles stop failing until the next attempt", err)
1657+
}
1658+
})
16021659
}
16031660
}
16041661

@@ -1607,8 +1664,8 @@ func TestTryStartDataVolumeWatch_DiscoveryErrorRetries(t *testing.T) {
16071664
r.reconciled.Store(true)
16081665
w := &startingWatcher{}
16091666
done, err := r.tryStartDataVolumeWatch(context.Background(), failingRESTMapper{meta.NewDefaultRESTMapper(nil)}, w, &informertest.FakeInformers{}, time.Second)
1610-
if err == nil || done || len(w.sources) != 0 || r.dataVolumeReader() != nil {
1611-
t.Fatalf("done=%v err=%v watches=%d reader=%v, want an error and nothing registered", done, err, len(w.sources), r.dataVolumeReader())
1667+
if err == nil || done || len(w.sources) != 0 || readerOf(r) != nil {
1668+
t.Fatalf("done=%v err=%v watches=%d reader=%v, want an error and nothing registered", done, err, len(w.sources), readerOf(r))
16121669
}
16131670
}
16141671

@@ -1617,7 +1674,7 @@ func TestTryStartDataVolumeWatch_FailedWatchLeavesNoReader(t *testing.T) {
16171674
r.reconciled.Store(true)
16181675
w := &startingWatcher{err: fmt.Errorf("watch failed")}
16191676
done, err := r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, &informertest.FakeInformers{}, time.Second)
1620-
if err == nil || done || r.dataVolumeReader() != nil {
1621-
t.Fatalf("done=%v err=%v reader=%v, want an error, not done, no reader", done, err, r.dataVolumeReader())
1677+
if err == nil || done || readerOf(r) != nil {
1678+
t.Fatalf("done=%v err=%v reader=%v, want an error, not done, no reader", done, err, readerOf(r))
16221679
}
16231680
}

0 commit comments

Comments
 (0)