@@ -11,19 +11,27 @@ import (
1111 "sort"
1212 "strconv"
1313 "strings"
14+ "sync"
1415 "time"
1516
1617 apierrors "k8s.io/apimachinery/pkg/api/errors"
18+ "k8s.io/apimachinery/pkg/api/meta"
19+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1720 "k8s.io/apimachinery/pkg/runtime"
21+ "k8s.io/apimachinery/pkg/runtime/schema"
1822 "k8s.io/apimachinery/pkg/types"
23+ "k8s.io/apimachinery/pkg/util/wait"
1924 "k8s.io/client-go/tools/record"
2025 "k8s.io/client-go/util/retry"
2126 "k8s.io/utils/pointer"
2227 ctrl "sigs.k8s.io/controller-runtime"
28+ "sigs.k8s.io/controller-runtime/pkg/cache"
2329 "sigs.k8s.io/controller-runtime/pkg/client"
2430 "sigs.k8s.io/controller-runtime/pkg/handler"
2531 "sigs.k8s.io/controller-runtime/pkg/log"
32+ "sigs.k8s.io/controller-runtime/pkg/manager"
2633 "sigs.k8s.io/controller-runtime/pkg/reconcile"
34+ "sigs.k8s.io/controller-runtime/pkg/source"
2735
2836 corev1 "k8s.io/api/core/v1"
2937 "k8s.io/apimachinery/pkg/api/resource"
@@ -67,6 +75,186 @@ type WorkloadMonitorReconciler struct {
6775 // namespace.cozystack.io/monitoring label. Used when SeaweedFS and the
6876 // stack that scrapes it run outside this cluster.
6977 SeaweedfsMetricsEndpoint string
78+ // DataVolumeReader reads CDI DataVolumes. Nil until the DataVolume kind is
79+ // served, and until then DataVolumes do not count towards Operational.
80+ DataVolumeReader client.Reader
81+ dataVolumeMu sync.RWMutex
82+ // reconciled is closed by the first Reconcile. controller-runtime starts the
83+ // workers only after the controller has started its sources
84+ // (pkg/internal/controller/controller.go, Controller.Start), so from then on
85+ // Watch starts a new source itself instead of queueing it for Start.
86+ reconciled chan struct {}
87+ reconciledOnce sync.Once
88+ }
89+
90+ const (
91+ // dataVolumeAPIPollInterval is how often the controller looks for the
92+ // DataVolume kind until CDI is installed.
93+ dataVolumeAPIPollInterval = 30 * time .Second
94+ // dataVolumeSyncTimeout bounds the wait for the DataVolume informer to sync.
95+ // A reconcile context has no deadline, so a reader installed on an informer
96+ // that never syncs would park every WorkloadMonitor reconcile for good.
97+ dataVolumeSyncTimeout = time .Minute
98+ )
99+
100+ func (r * WorkloadMonitorReconciler ) dataVolumeReader () client.Reader {
101+ r .dataVolumeMu .RLock ()
102+ defer r .dataVolumeMu .RUnlock ()
103+ return r .DataVolumeReader
104+ }
105+
106+ func (r * WorkloadMonitorReconciler ) reconciledCh () chan struct {} {
107+ r .dataVolumeMu .Lock ()
108+ defer r .dataVolumeMu .Unlock ()
109+ if r .reconciled == nil {
110+ r .reconciled = make (chan struct {})
111+ }
112+ return r .reconciled
113+ }
114+
115+ var dataVolumeGVK = schema.GroupVersionKind {Group : "cdi.kubevirt.io" , Version : "v1beta1" , Kind : "DataVolume" }
116+
117+ // dataVolumeInFlightPhases are the DataVolumePhase values in which CDI still has
118+ // work to do on the disk (containerized-data-importer-api pkg/apis/core/v1beta1).
119+ // A phase outside this set counts as settled, so a CDI release that adds or
120+ // renames a phase cannot turn working disks non-operational; the price is that
121+ // a renamed in-flight phase reads as settled until this list follows it.
122+ // PendingPopulation and WaitForFirstConsumer are where a disk on a
123+ // WaitForFirstConsumer class rests until a VM consumes it, so neither is in
124+ // flight. UploadReady is in flight although it lasts until someone uploads:
125+ // CDI holds it from the moment the upload server is ready until the transfer
126+ // succeeds (pkg/controller/datavolume/upload-controller.go), so it covers an
127+ // empty disk and a half-uploaded one alike.
128+ // Paused is where a multi-stage (checkpoint) import waits for its next
129+ // checkpoint, which only an outside actor supplies; the
130+ // vm-disk chart renders no checkpoint source, so no disk of it gets there.
131+ var dataVolumeInFlightPhases = map [string ]bool {
132+ "Pending" : true ,
133+ "PVCBound" : true ,
134+ "ImportScheduled" : true ,
135+ "ImportInProgress" : true ,
136+ "CloneScheduled" : true ,
137+ "CloneInProgress" : true ,
138+ "SnapshotForSmartCloneInProgress" : true ,
139+ "CloneFromSnapshotSourceInProgress" : true ,
140+ "SmartClonePVCInProgress" : true ,
141+ "CSICloneInProgress" : true ,
142+ "PrepClaimInProgress" : true ,
143+ "RebindInProgress" : true ,
144+ "ExpansionInProgress" : true ,
145+ "NamespaceTransferInProgress" : true ,
146+ "UploadScheduled" : true ,
147+ "UploadReady" : true ,
148+ }
149+
150+ // isDataVolumeReady reports whether CDI is done with the disk. Failed and
151+ // Unknown are not ready, and neither is a DataVolume without a phase: CDI
152+ // leaves one there when it could not render the PVC spec.
153+ func isDataVolumeReady (dv * unstructured.Unstructured ) bool {
154+ phase , _ , _ := unstructured .NestedString (dv .Object , "status" , "phase" )
155+ switch phase {
156+ case "" , "Failed" , "Unknown" :
157+ return false
158+ }
159+ return ! dataVolumeInFlightPhases [phase ]
160+ }
161+
162+ // dataVolumeAPIServed treats a NoMatch as not served and returns any other
163+ // discovery error, so the caller retries instead of concluding CDI is absent.
164+ func dataVolumeAPIServed (mapper meta.RESTMapper ) (bool , error ) {
165+ _ , err := mapper .RESTMapping (dataVolumeGVK .GroupKind (), dataVolumeGVK .Version )
166+ if meta .IsNoMatchError (err ) {
167+ return false , nil
168+ }
169+ return err == nil , err
170+ }
171+
172+ type sourceWatcher interface {
173+ Watch (src source.Source ) error
174+ }
175+
176+ // tryStartDataVolumeWatch registers the DataVolume watch and reader once the
177+ // kind is served, and reports whether it did. The reader is set only after the
178+ // source has synced: the cache blocks a List on an unsynced informer until the
179+ // caller's context ends. A source that does not sync in time is cancelled by
180+ // WaitForSync, and the next call starts a fresh one.
181+ func (r * WorkloadMonitorReconciler ) tryStartDataVolumeWatch (ctx context.Context , mapper meta.RESTMapper , c sourceWatcher , informers cache.Cache , syncTimeout time.Duration ) (bool , error ) {
182+ served , err := dataVolumeAPIServed (mapper )
183+ if err != nil || ! served {
184+ return false , err
185+ }
186+ dv := & unstructured.Unstructured {}
187+ dv .SetGroupVersionKind (dataVolumeGVK )
188+ src := source .Kind [client.Object ](informers , dv ,
189+ handler .EnqueueRequestsFromMapFunc (mapObjectToMonitor (client .Object (dv ), r .Client )))
190+ if err := c .Watch (src ); err != nil {
191+ return false , fmt .Errorf ("watching DataVolumes: %w" , err )
192+ }
193+ syncCtx , cancel := context .WithTimeout (ctx , syncTimeout )
194+ defer cancel ()
195+ if err := src .WaitForSync (syncCtx ); err != nil {
196+ return false , fmt .Errorf ("waiting for the DataVolume informer to sync: %w" , err )
197+ }
198+ // WaitForSync returns nil when ctx itself is cancelled.
199+ if err := ctx .Err (); err != nil {
200+ return false , err
201+ }
202+ r .dataVolumeMu .Lock ()
203+ defer r .dataVolumeMu .Unlock ()
204+ // The manager's client reads unstructured objects straight from the API
205+ // server; the cache shares the informer the watch above starts.
206+ r .DataVolumeReader = informers
207+ return true , nil
208+ }
209+
210+ // watchDataVolumes adds the DataVolume watch once the first reconcile has run
211+ // and the kind is served, checking discovery again every interval until it is.
212+ // Starting on the first reconcile rather than on the next tick keeps a disk
213+ // created right after a controller restart from reading populated until then.
214+ func (r * WorkloadMonitorReconciler ) watchDataVolumes (ctx context.Context , mapper meta.RESTMapper , c sourceWatcher , informers cache.Cache , interval time.Duration ) {
215+ select {
216+ case <- ctx .Done ():
217+ return
218+ case <- r .reconciledCh ():
219+ }
220+ logger := log .FromContext (ctx )
221+ // The condition never returns an error, so the poll ends only when the
222+ // watch is added or the manager stops.
223+ _ = wait .PollUntilContextCancel (ctx , interval , true , func (ctx context.Context ) (bool , error ) {
224+ done , err := r .tryStartDataVolumeWatch (ctx , mapper , c , informers , dataVolumeSyncTimeout )
225+ if err != nil {
226+ logger .Error (err , "Unable to start the DataVolume watch, retrying" )
227+ }
228+ return done , nil
229+ })
230+ }
231+
232+ // dataVolumesReady reports whether every DataVolume the monitor selects is
233+ // ready. A NoMatch from the reader counts as no DataVolumes.
234+ func (r * WorkloadMonitorReconciler ) dataVolumesReady (ctx context.Context , monitor * cozyv1alpha1.WorkloadMonitor ) (bool , error ) {
235+ reader := r .dataVolumeReader ()
236+ if reader == nil {
237+ return true , nil
238+ }
239+ list := & unstructured.UnstructuredList {}
240+ list .SetGroupVersionKind (dataVolumeGVK .GroupVersion ().WithKind (dataVolumeGVK .Kind + "List" ))
241+ if err := reader .List (
242+ ctx ,
243+ list ,
244+ client .InNamespace (monitor .Namespace ),
245+ client .MatchingLabels (monitor .Spec .Selector ),
246+ ); err != nil {
247+ if meta .IsNoMatchError (err ) {
248+ return true , nil
249+ }
250+ return false , err
251+ }
252+ for i := range list .Items {
253+ if ! isDataVolumeReady (& list .Items [i ]) {
254+ return false , nil
255+ }
256+ }
257+ return true , nil
70258}
71259
72260// +kubebuilder:rbac:groups=cozystack.io,resources=workloadmonitors,verbs=get;list;watch;create;update;patch;delete
@@ -78,6 +266,7 @@ type WorkloadMonitorReconciler struct {
78266// +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get
79267// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
80268// +kubebuilder:rbac:groups=objectstorage.k8s.io,resources=bucketclaims,verbs=get;list;watch
269+ // +kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch
81270
82271// isBucketClaimReady checks if the BucketClaim has been provisioned.
83272func (r * WorkloadMonitorReconciler ) isBucketClaimReady (bc * cosiv1alpha1.BucketClaim ) bool {
@@ -568,6 +757,7 @@ func (r *WorkloadMonitorReconciler) reconcilePodForMonitor(
568757// 1. It reconciles WorkloadMonitor objects themselves (create/update/delete).
569758// 2. It also reconciles Pod events mapped to WorkloadMonitor via label selector.
570759func (r * WorkloadMonitorReconciler ) Reconcile (ctx context.Context , req ctrl.Request ) (ctrl.Result , error ) {
760+ r .reconciledOnce .Do (func () { close (r .reconciledCh ()) })
571761 logger := log .FromContext (ctx )
572762
573763 // Fetch the WorkloadMonitor object if it exists
@@ -691,6 +881,12 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
691881 }
692882 }
693883
884+ dataVolumesReady , err := r .dataVolumesReady (ctx , monitor )
885+ if err != nil {
886+ logger .Error (err , "Unable to list DataVolumes for WorkloadMonitor" , "monitor" , monitor .Name )
887+ return ctrl.Result {}, err
888+ }
889+
694890 // Update WorkloadMonitor status based on observed pods
695891 monitor .Status .ObservedReplicas = observedReplicas
696892 monitor .Status .AvailableReplicas = availableReplicas
@@ -711,6 +907,9 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
711907 if fresh .Spec .MinReplicas != nil && availableReplicas < * fresh .Spec .MinReplicas {
712908 fresh .Status .Operational = pointer .Bool (false )
713909 }
910+ if ! dataVolumesReady {
911+ fresh .Status .Operational = pointer .Bool (false )
912+ }
714913 return r .Status ().Update (ctx , fresh )
715914 })
716915 if err != nil {
@@ -736,7 +935,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
736935
737936// SetupWithManager registers our controller with the Manager and sets up watches.
738937func (r * WorkloadMonitorReconciler ) SetupWithManager (mgr ctrl.Manager ) error {
739- return ctrl .NewControllerManagedBy (mgr ).
938+ builder := ctrl .NewControllerManagedBy (mgr ).
740939 // Watch WorkloadMonitor objects
741940 For (& cozyv1alpha1.WorkloadMonitor {}).
742941 // Also watch Pod objects and map them back to WorkloadMonitor if labels match
@@ -755,8 +954,20 @@ func (r *WorkloadMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error {
755954 handler .EnqueueRequestsFromMapFunc (mapObjectToMonitor (& cosiv1alpha1.BucketClaim {}, r .Client )),
756955 ).
757956 // Watch for changes to Workload objects we create (owned by WorkloadMonitor)
758- Owns (& cozyv1alpha1.Workload {}).
759- Complete (r )
957+ Owns (& cozyv1alpha1.Workload {})
958+ c , err := builder .Build (r )
959+ if err != nil {
960+ return err
961+ }
962+
963+ // A watch on a kind whose CRD is absent never syncs, and controller-runtime
964+ // does not start a controller with unsynced caches. CDI ships with the IaaS
965+ // bundle only and can be installed while this controller is running, so
966+ // the DataVolume watch is added once the kind appears rather than at setup.
967+ return mgr .Add (manager .RunnableFunc (func (ctx context.Context ) error {
968+ r .watchDataVolumes (ctx , mgr .GetRESTMapper (), c , mgr .GetCache (), dataVolumeAPIPollInterval )
969+ return nil
970+ }))
760971}
761972
762973func mapObjectToMonitor [T client.Object ](_ T , c client.Client ) func (ctx context.Context , obj client.Object ) []reconcile.Request {
0 commit comments