Skip to content

Commit f29ef2e

Browse files
committed
fix(controller): requeue monitors reconciled while the DataVolume watch syncs
Starting the DataVolume source replays every DataVolume to its handler at once, and the handler queues the matching monitors, while the reader is installed only after the source has synced. A monitor reconciled in that window saw no reader, kept its last verdict and returned with nothing left to queue it again, so a disk that failed or finished while the controller was down could keep a wrong WorkloadsReady until the next unrelated event. Such a reconcile still publishes the replica counts and now asks to be requeued after a few seconds, until the reader is in place or the sync attempt gives up. It is a requeue rather than an error because the window is expected: an error would count as a reconcile failure and log a line for every monitor in the cluster while it lasts. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 146ed47 commit f29ef2e

2 files changed

Lines changed: 114 additions & 27 deletions

File tree

‎internal/controller/workloadmonitor_controller.go‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,11 @@ type WorkloadMonitorReconciler struct {
7979
// served, and until then DataVolumes do not count towards Operational.
8080
DataVolumeReader client.Reader
8181
dataVolumeMu sync.RWMutex
82+
// dataVolumeWatchSyncing is set while the DataVolume watch is started and
83+
// its reader is not installed yet. The source replays every DataVolume to
84+
// the handler in that window, and a reconcile it queues must be requeued:
85+
// nothing queues it again once the reader is in place.
86+
dataVolumeWatchSyncing bool
8287
// reconciled is closed by the first Reconcile. controller-runtime starts the
8388
// workers only after the controller has started its sources
8489
// (pkg/internal/controller/controller.go, Controller.Start), so from then on
@@ -95,12 +100,20 @@ const (
95100
// A reconcile context has no deadline, so a reader installed on an informer
96101
// that never syncs would park every WorkloadMonitor reconcile for good.
97102
dataVolumeSyncTimeout = time.Minute
103+
// dataVolumeSyncRequeue is how soon a monitor reconciled while the
104+
// DataVolume watch syncs is looked at again.
105+
dataVolumeSyncRequeue = 5 * time.Second
98106
)
99107

100-
func (r *WorkloadMonitorReconciler) dataVolumeReader() client.Reader {
108+
var errDataVolumeWatchSyncing = errors.New("the DataVolume watch has not synced yet")
109+
110+
func (r *WorkloadMonitorReconciler) dataVolumeReader() (client.Reader, error) {
101111
r.dataVolumeMu.RLock()
102112
defer r.dataVolumeMu.RUnlock()
103-
return r.DataVolumeReader
113+
if r.DataVolumeReader == nil && r.dataVolumeWatchSyncing {
114+
return nil, errDataVolumeWatchSyncing
115+
}
116+
return r.DataVolumeReader, nil
104117
}
105118

106119
func (r *WorkloadMonitorReconciler) reconciledCh() chan struct{} {
@@ -112,6 +125,12 @@ func (r *WorkloadMonitorReconciler) reconciledCh() chan struct{} {
112125
return r.reconciled
113126
}
114127

128+
func (r *WorkloadMonitorReconciler) setDataVolumeWatchSyncing(syncing bool) {
129+
r.dataVolumeMu.Lock()
130+
defer r.dataVolumeMu.Unlock()
131+
r.dataVolumeWatchSyncing = syncing
132+
}
133+
115134
var dataVolumeGVK = schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}
116135

117136
// dataVolumeInFlightPhases are the DataVolumePhase values in which CDI still has
@@ -186,6 +205,8 @@ func (r *WorkloadMonitorReconciler) tryStartDataVolumeWatch(ctx context.Context,
186205
dv.SetGroupVersionKind(dataVolumeGVK)
187206
src := source.Kind[client.Object](informers, dv,
188207
handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(client.Object(dv), r.Client)))
208+
r.setDataVolumeWatchSyncing(true)
209+
defer r.setDataVolumeWatchSyncing(false)
189210
if err := c.Watch(src); err != nil {
190211
return false, fmt.Errorf("watching DataVolumes: %w", err)
191212
}
@@ -233,9 +254,9 @@ func (r *WorkloadMonitorReconciler) watchDataVolumes(ctx context.Context, mapper
233254
// reader counts as no DataVolumes. read is false when the DataVolumes could not
234255
// be read: no reader yet, or a failed List.
235256
func (r *WorkloadMonitorReconciler) dataVolumesMessage(ctx context.Context, monitor *cozyv1alpha1.WorkloadMonitor) (message string, read bool, err error) {
236-
reader := r.dataVolumeReader()
257+
reader, err := r.dataVolumeReader()
237258
if reader == nil {
238-
return "", false, nil
259+
return "", false, err
239260
}
240261
list := &unstructured.UnstructuredList{}
241262
list.SetGroupVersionKind(dataVolumeGVK.GroupVersion().WithKind(dataVolumeGVK.Kind + "List"))
@@ -910,8 +931,14 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
910931
}
911932

912933
dataVolumesMessage, dataVolumesRead, dataVolumesErr := r.dataVolumesMessage(ctx, monitor)
934+
// The sync window is expected and short; an error here would count as a
935+
// reconcile failure for every monitor in the cluster while it lasts.
936+
dataVolumeWatchSyncing := errors.Is(dataVolumesErr, errDataVolumeWatchSyncing)
937+
if dataVolumeWatchSyncing {
938+
dataVolumesErr = nil
939+
}
913940
if dataVolumesErr != nil {
914-
logger.Error(dataVolumesErr, "Unable to list DataVolumes for WorkloadMonitor, keeping the last DataVolume verdict", "monitor", monitor.Name)
941+
logger.Error(dataVolumesErr, "Unable to read DataVolumes for WorkloadMonitor, keeping the last DataVolume verdict", "monitor", monitor.Name)
915942
}
916943

917944
// Update WorkloadMonitor status based on observed pods
@@ -961,6 +988,10 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
961988
return ctrl.Result{}, errors.Join(bucketMetricsErr, dataVolumesErr)
962989
}
963990

991+
if dataVolumeWatchSyncing {
992+
return ctrl.Result{RequeueAfter: dataVolumeSyncRequeue}, nil
993+
}
994+
964995
// Requeue periodically if there are BucketClaims to keep sizes up to date.
965996
// Bucket sizes come from Prometheus metrics that update every 60s.
966997
if len(bucketClaimList.Items) > 0 {

‎internal/controller/workloadmonitor_controller_test.go‎

Lines changed: 78 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1418,8 +1418,9 @@ func TestReconcile_DataVolumeKindNotServedIsNoDataVolumes(t *testing.T) {
14181418
}
14191419

14201420
// reconcileWithUnreadableDataVolumes reconciles a monitor that selects one
1421-
// ready pod while every DataVolume List fails, or while withReader is false.
1422-
func reconcileWithUnreadableDataVolumes(t *testing.T, withReader bool, status cozyv1alpha1.WorkloadMonitorStatus) (*cozyv1alpha1.WorkloadMonitor, error) {
1421+
// ready pod while every DataVolume List fails, or while withReader is false;
1422+
// syncing marks the DataVolume watch as started and not yet synced.
1423+
func reconcileWithUnreadableDataVolumes(t *testing.T, withReader, syncing bool, status cozyv1alpha1.WorkloadMonitorStatus) (*cozyv1alpha1.WorkloadMonitor, reconcile.Result, error) {
14231424
t.Helper()
14241425
s := newTestScheme()
14251426
selector := map[string]string{"app.kubernetes.io/instance": "m"}
@@ -1451,17 +1452,18 @@ func reconcileWithUnreadableDataVolumes(t *testing.T, withReader bool, status co
14511452
if withReader {
14521453
reconciler.DataVolumeReader = fakeClient
14531454
}
1455+
reconciler.dataVolumeWatchSyncing = syncing
14541456
req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "m", Namespace: "default"}}
1455-
_, err := reconciler.Reconcile(context.TODO(), req)
1457+
result, err := reconciler.Reconcile(context.TODO(), req)
14561458
updated := &cozyv1alpha1.WorkloadMonitor{}
14571459
if gerr := fakeClient.Get(context.TODO(), req.NamespacedName, updated); gerr != nil {
14581460
t.Fatalf("Failed to get updated WorkloadMonitor: %v", gerr)
14591461
}
1460-
return updated, err
1462+
return updated, result, err
14611463
}
14621464

14631465
func TestReconcile_DataVolumeListErrorStillPublishesStatus(t *testing.T) {
1464-
got, err := reconcileWithUnreadableDataVolumes(t, true, cozyv1alpha1.WorkloadMonitorStatus{})
1466+
got, _, err := reconcileWithUnreadableDataVolumes(t, true, false, cozyv1alpha1.WorkloadMonitorStatus{})
14651467
if err == nil {
14661468
t.Error("Reconcile returned nil on a DataVolume list failure, want the error so the request is retried")
14671469
}
@@ -1477,7 +1479,7 @@ func TestReconcile_UnreadDataVolumesKeepTheLastVerdict(t *testing.T) {
14771479
const stuck = "DataVolume m is ImportInProgress"
14781480
for _, withReader := range []bool{true, false} {
14791481
t.Run(fmt.Sprintf("reader=%v", withReader), func(t *testing.T) {
1480-
got, _ := reconcileWithUnreadableDataVolumes(t, withReader, cozyv1alpha1.WorkloadMonitorStatus{
1482+
got, _, _ := reconcileWithUnreadableDataVolumes(t, withReader, false, cozyv1alpha1.WorkloadMonitorStatus{
14811483
Operational: ptr.To(false),
14821484
Message: stuck,
14831485
Reason: cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady,
@@ -1492,6 +1494,24 @@ func TestReconcile_UnreadDataVolumesKeepTheLastVerdict(t *testing.T) {
14921494
}
14931495
}
14941496

1497+
func TestReconcile_SyncingDataVolumeWatchPublishesAndRequeues(t *testing.T) {
1498+
const stuck = "DataVolume m is Failed"
1499+
got, result, err := reconcileWithUnreadableDataVolumes(t, false, true, cozyv1alpha1.WorkloadMonitorStatus{
1500+
Operational: ptr.To(false),
1501+
Message: stuck,
1502+
Reason: cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady,
1503+
})
1504+
if err != nil || result.RequeueAfter <= 0 {
1505+
t.Errorf("Reconcile = %+v, %v while the DataVolume watch syncs, want a requeue and no error: the window is expected, not a failure", result, err)
1506+
}
1507+
if got.Status.ObservedReplicas != 1 || got.Status.AvailableReplicas != 1 {
1508+
t.Errorf("replicas observed=%d available=%d, want 1 and 1", got.Status.ObservedReplicas, got.Status.AvailableReplicas)
1509+
}
1510+
if got.Status.Operational == nil || *got.Status.Operational || got.Status.Message != stuck {
1511+
t.Errorf("Operational=%v Message=%q, want false and %q", got.Status.Operational, got.Status.Message, stuck)
1512+
}
1513+
}
1514+
14951515
func TestReconcile_NoDataVolumeReaderSkipsDataVolumes(t *testing.T) {
14961516
busy := "ImportInProgress"
14971517
got := reconcileDataVolumeMonitor(t, false, nil,
@@ -1535,9 +1555,14 @@ func TestDataVolumeAPIServed_DiscoveryErrorIsReturned(t *testing.T) {
15351555
type startingWatcher struct {
15361556
sources []source.Source
15371557
err error
1558+
// onWatch runs when Watch is called, before the source starts.
1559+
onWatch func()
15381560
}
15391561

15401562
func (w *startingWatcher) Watch(src source.Source) error {
1563+
if w.onWatch != nil {
1564+
w.onWatch()
1565+
}
15411566
if w.err != nil {
15421567
return w.err
15431568
}
@@ -1546,6 +1571,12 @@ func (w *startingWatcher) Watch(src source.Source) error {
15461571
return src.Start(context.Background(), queue)
15471572
}
15481573

1574+
func readerOf(r *WorkloadMonitorReconciler) client.Reader {
1575+
r.dataVolumeMu.RLock()
1576+
defer r.dataVolumeMu.RUnlock()
1577+
return r.DataVolumeReader
1578+
}
1579+
15491580
func servedMapper() meta.RESTMapper {
15501581
mapper := meta.NewDefaultRESTMapper(nil)
15511582
mapper.Add(dataVolumeGVK, meta.RESTScopeNamespace)
@@ -1573,17 +1604,17 @@ func TestTryStartDataVolumeWatch_WaitsUntilTheKindIsServed(t *testing.T) {
15731604
if err != nil || done {
15741605
t.Fatalf("before CDI: done=%v err=%v, want false, nil", done, err)
15751606
}
1576-
if len(w.sources) != 0 || r.dataVolumeReader() != nil {
1577-
t.Fatalf("before CDI: %d watches, reader %v; want none", len(w.sources), r.dataVolumeReader())
1607+
if len(w.sources) != 0 || readerOf(r) != nil {
1608+
t.Fatalf("before CDI: %d watches, reader %v; want none", len(w.sources), readerOf(r))
15781609
}
15791610

15801611
mapper.Add(dataVolumeGVK, meta.RESTScopeNamespace)
15811612
done, err = r.tryStartDataVolumeWatch(context.Background(), mapper, w, informers, time.Second)
15821613
if err != nil || !done {
15831614
t.Fatalf("after CDI: done=%v err=%v, want true, nil", done, err)
15841615
}
1585-
if len(w.sources) != 1 || r.dataVolumeReader() == nil {
1586-
t.Fatalf("after CDI: %d watches, reader %v; want one watch and a reader", len(w.sources), r.dataVolumeReader())
1616+
if len(w.sources) != 1 || readerOf(r) == nil {
1617+
t.Fatalf("after CDI: %d watches, reader %v; want one watch and a reader", len(w.sources), readerOf(r))
15871618
}
15881619
}
15891620

@@ -1607,14 +1638,14 @@ func TestWatchDataVolumes_StartsOnTheFirstReconcile(t *testing.T) {
16071638
go r.watchDataVolumes(ctx, servedMapper(), &startingWatcher{}, informers, time.Hour)
16081639

16091640
time.Sleep(100 * time.Millisecond)
1610-
if r.dataVolumeReader() != nil {
1641+
if readerOf(r) != nil {
16111642
t.Fatal("the DataVolume reader was installed before the first reconcile")
16121643
}
16131644
if _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "m", Namespace: "default"}}); err != nil {
16141645
t.Fatalf("Reconcile: %v", err)
16151646
}
16161647
err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 5*time.Second, true, func(context.Context) (bool, error) {
1617-
return r.dataVolumeReader() != nil, nil
1648+
return readerOf(r) != nil, nil
16181649
})
16191650
if err != nil {
16201651
t.Fatal("the DataVolume reader was not installed after the first reconcile, within a fraction of the poll interval")
@@ -1629,17 +1660,17 @@ func TestTryStartDataVolumeWatch_UnsyncedSourceLeavesNoReaderAndRetries(t *testi
16291660
fi.Synced = false
16301661

16311662
done, err := r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, informers, 50*time.Millisecond)
1632-
if err == nil || done || r.dataVolumeReader() != nil {
1633-
t.Fatalf("informer never synced: done=%v err=%v reader=%v, want an error, not done, no reader", done, err, r.dataVolumeReader())
1663+
if err == nil || done || readerOf(r) != nil {
1664+
t.Fatalf("informer never synced: done=%v err=%v reader=%v, want an error, not done, no reader", done, err, readerOf(r))
16341665
}
16351666

16361667
fi.SyncedLock.Lock()
16371668
fi.Synced = true
16381669
fi.SyncedLock.Unlock()
16391670

16401671
done, err = r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, informers, time.Second)
1641-
if err != nil || !done || r.dataVolumeReader() == nil {
1642-
t.Fatalf("retry after sync: done=%v err=%v reader=%v, want done and a reader", done, err, r.dataVolumeReader())
1672+
if err != nil || !done || readerOf(r) == nil {
1673+
t.Fatalf("retry after sync: done=%v err=%v reader=%v, want done and a reader", done, err, readerOf(r))
16431674
}
16441675
if len(w.sources) != 2 {
16451676
t.Fatalf("got %d watches, want a fresh source on the retry", len(w.sources))
@@ -1655,28 +1686,53 @@ func TestTryStartDataVolumeWatch_StoppingManagerLeavesNoReader(t *testing.T) {
16551686
ctx, cancel := context.WithCancel(context.Background())
16561687
cancel()
16571688
done, err := r.tryStartDataVolumeWatch(ctx, servedMapper(), &startingWatcher{}, informers, time.Second)
1658-
if done || r.dataVolumeReader() != nil {
1659-
t.Fatalf("done=%v reader=%v, want no reader when the manager is stopping", done, r.dataVolumeReader())
1689+
if done || readerOf(r) != nil {
1690+
t.Fatalf("done=%v reader=%v, want no reader when the manager is stopping", done, readerOf(r))
16601691
}
16611692
if !errors.Is(err, context.Canceled) || strings.Contains(err.Error(), "%!") {
16621693
t.Errorf("err = %q, want context.Canceled and nothing formatted from a nil error", err)
16631694
}
16641695
}
16651696

1697+
// The source replays every DataVolume to the handler as soon as Watch starts
1698+
// it, before the reader is installed. A reconcile in that window must be
1699+
// retried, or the events that queued it are lost.
1700+
func TestTryStartDataVolumeWatch_ReconcileBeforeTheReaderIsRetried(t *testing.T) {
1701+
for _, synced := range []bool{true, false} {
1702+
t.Run(fmt.Sprintf("synced=%v", synced), func(t *testing.T) {
1703+
r := &WorkloadMonitorReconciler{}
1704+
informers := &informertest.FakeInformers{}
1705+
fakeDataVolumeInformer(t, informers).Synced = synced
1706+
monitor := &cozyv1alpha1.WorkloadMonitor{ObjectMeta: metav1.ObjectMeta{Name: "m", Namespace: "default"}}
1707+
var errDuringWatch error
1708+
w := &startingWatcher{onWatch: func() {
1709+
_, _, errDuringWatch = r.dataVolumesMessage(context.Background(), monitor)
1710+
}}
1711+
_, _ = r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, informers, 50*time.Millisecond)
1712+
if errDuringWatch == nil {
1713+
t.Error("reading DataVolumes while the watch syncs returned no error, want one so the reconcile is retried")
1714+
}
1715+
if _, _, err := r.dataVolumesMessage(context.Background(), monitor); err != nil && !synced {
1716+
t.Errorf("after a sync timeout: %v, want no error so reconciles stop failing until the next attempt", err)
1717+
}
1718+
})
1719+
}
1720+
}
1721+
16661722
func TestTryStartDataVolumeWatch_DiscoveryErrorRetries(t *testing.T) {
16671723
r := &WorkloadMonitorReconciler{}
16681724
w := &startingWatcher{}
16691725
done, err := r.tryStartDataVolumeWatch(context.Background(), failingRESTMapper{meta.NewDefaultRESTMapper(nil)}, w, &informertest.FakeInformers{}, time.Second)
1670-
if err == nil || done || len(w.sources) != 0 || r.dataVolumeReader() != nil {
1671-
t.Fatalf("done=%v err=%v watches=%d reader=%v, want an error and nothing registered", done, err, len(w.sources), r.dataVolumeReader())
1726+
if err == nil || done || len(w.sources) != 0 || readerOf(r) != nil {
1727+
t.Fatalf("done=%v err=%v watches=%d reader=%v, want an error and nothing registered", done, err, len(w.sources), readerOf(r))
16721728
}
16731729
}
16741730

16751731
func TestTryStartDataVolumeWatch_FailedWatchLeavesNoReader(t *testing.T) {
16761732
r := &WorkloadMonitorReconciler{}
16771733
w := &startingWatcher{err: fmt.Errorf("watch failed")}
16781734
done, err := r.tryStartDataVolumeWatch(context.Background(), servedMapper(), w, &informertest.FakeInformers{}, time.Second)
1679-
if err == nil || done || r.dataVolumeReader() != nil {
1680-
t.Fatalf("done=%v err=%v reader=%v, want an error, not done, no reader", done, err, r.dataVolumeReader())
1735+
if err == nil || done || readerOf(r) != nil {
1736+
t.Fatalf("done=%v err=%v reader=%v, want an error, not done, no reader", done, err, readerOf(r))
16811737
}
16821738
}

0 commit comments

Comments
 (0)