Skip to content

Commit 9e561ef

Browse files
committed
fix(controller): count DataVolume readiness in WorkloadMonitor
A vm-disk application reported its workloads operational whatever state its DataVolume was in, so an import or clone that never finished left the disk looking healthy while it was unusable. Its monitor has minReplicas 0 and only pods fed Operational. The monitor now also lists the DataVolumes its selector matches and reports not operational while any of them is in flight, Failed, Unknown or without a phase. UploadReady counts as in flight: CDI holds it from the moment the upload server is ready until the transfer succeeds, so it covers an empty disk and a half-uploaded one alike. Waiting for the first consumer counts as ready, since that is where such a disk rests, and so does a phase this controller does not know, so a CDI release that renames one cannot turn working disks red. This is deliberately not a Helm wait: a readiness gate holds a helm-controller worker for its whole window, and a disk that never populates would keep taking it back. The signal lands in the application's WorkloadsReady condition instead, and Ready keeps following the HelmRelease. CDI ships with the IaaS bundle only and can be installed while this controller is running. A watch on a kind whose CRD is absent keeps the controller from starting, so the DataVolume watch is added from a runnable that polls discovery until the kind is served. It starts on the first reconcile, the earliest point at which a new watch is started at once, rather than on the next tick, so a disk created just after a restart is not read as populated for a poll interval. The reader is installed only once that watch has synced within a bound: a reconcile context has no deadline, and a List on an informer that never syncs would park every WorkloadMonitor reconcile for good. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 88b61c9 commit 9e561ef

2 files changed

Lines changed: 580 additions & 3 deletions

File tree

‎internal/controller/workloadmonitor_controller.go‎

Lines changed: 214 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,27 @@ import (
1111
"sort"
1212
"strconv"
1313
"strings"
14+
"sync"
1415
"time"
1516

1617
apierrors "k8s.io/apimachinery/pkg/api/errors"
18+
"k8s.io/apimachinery/pkg/api/meta"
19+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1720
"k8s.io/apimachinery/pkg/runtime"
21+
"k8s.io/apimachinery/pkg/runtime/schema"
1822
"k8s.io/apimachinery/pkg/types"
23+
"k8s.io/apimachinery/pkg/util/wait"
1924
"k8s.io/client-go/tools/record"
2025
"k8s.io/client-go/util/retry"
2126
"k8s.io/utils/pointer"
2227
ctrl "sigs.k8s.io/controller-runtime"
28+
"sigs.k8s.io/controller-runtime/pkg/cache"
2329
"sigs.k8s.io/controller-runtime/pkg/client"
2430
"sigs.k8s.io/controller-runtime/pkg/handler"
2531
"sigs.k8s.io/controller-runtime/pkg/log"
32+
"sigs.k8s.io/controller-runtime/pkg/manager"
2633
"sigs.k8s.io/controller-runtime/pkg/reconcile"
34+
"sigs.k8s.io/controller-runtime/pkg/source"
2735

2836
corev1 "k8s.io/api/core/v1"
2937
"k8s.io/apimachinery/pkg/api/resource"
@@ -67,6 +75,186 @@ type WorkloadMonitorReconciler struct {
6775
// namespace.cozystack.io/monitoring label. Used when SeaweedFS and the
6876
// stack that scrapes it run outside this cluster.
6977
SeaweedfsMetricsEndpoint string
78+
// DataVolumeReader reads CDI DataVolumes. Nil until the DataVolume kind is
79+
// served, and until then DataVolumes do not count towards Operational.
80+
DataVolumeReader client.Reader
81+
dataVolumeMu sync.RWMutex
82+
// reconciled is closed by the first Reconcile. controller-runtime starts the
83+
// workers only after the controller has started its sources
84+
// (pkg/internal/controller/controller.go, Controller.Start), so from then on
85+
// Watch starts a new source itself instead of queueing it for Start.
86+
reconciled chan struct{}
87+
reconciledOnce sync.Once
88+
}
89+
90+
const (
91+
// dataVolumeAPIPollInterval is how often the controller looks for the
92+
// DataVolume kind until CDI is installed.
93+
dataVolumeAPIPollInterval = 30 * time.Second
94+
// dataVolumeSyncTimeout bounds the wait for the DataVolume informer to sync.
95+
// A reconcile context has no deadline, so a reader installed on an informer
96+
// that never syncs would park every WorkloadMonitor reconcile for good.
97+
dataVolumeSyncTimeout = time.Minute
98+
)
99+
100+
func (r *WorkloadMonitorReconciler) dataVolumeReader() client.Reader {
101+
r.dataVolumeMu.RLock()
102+
defer r.dataVolumeMu.RUnlock()
103+
return r.DataVolumeReader
104+
}
105+
106+
func (r *WorkloadMonitorReconciler) reconciledCh() chan struct{} {
107+
r.dataVolumeMu.Lock()
108+
defer r.dataVolumeMu.Unlock()
109+
if r.reconciled == nil {
110+
r.reconciled = make(chan struct{})
111+
}
112+
return r.reconciled
113+
}
114+
115+
var dataVolumeGVK = schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}
116+
117+
// dataVolumeInFlightPhases are the DataVolumePhase values in which CDI still has
118+
// work to do on the disk (containerized-data-importer-api pkg/apis/core/v1beta1).
119+
// A phase outside this set counts as settled, so a CDI release that adds or
120+
// renames a phase cannot turn working disks non-operational; the price is that
121+
// a renamed in-flight phase reads as settled until this list follows it.
122+
// PendingPopulation and WaitForFirstConsumer are where a disk on a
123+
// WaitForFirstConsumer class rests until a VM consumes it, so neither is in
124+
// flight. UploadReady is in flight although it lasts until someone uploads:
125+
// CDI holds it from the moment the upload server is ready until the transfer
126+
// succeeds (pkg/controller/datavolume/upload-controller.go), so it covers an
127+
// empty disk and a half-uploaded one alike.
128+
// Paused is where a multi-stage (checkpoint) import waits for its next
129+
// checkpoint, which only an outside actor supplies; the
130+
// vm-disk chart renders no checkpoint source, so no disk of it gets there.
131+
var dataVolumeInFlightPhases = map[string]bool{
132+
"Pending": true,
133+
"PVCBound": true,
134+
"ImportScheduled": true,
135+
"ImportInProgress": true,
136+
"CloneScheduled": true,
137+
"CloneInProgress": true,
138+
"SnapshotForSmartCloneInProgress": true,
139+
"CloneFromSnapshotSourceInProgress": true,
140+
"SmartClonePVCInProgress": true,
141+
"CSICloneInProgress": true,
142+
"PrepClaimInProgress": true,
143+
"RebindInProgress": true,
144+
"ExpansionInProgress": true,
145+
"NamespaceTransferInProgress": true,
146+
"UploadScheduled": true,
147+
"UploadReady": true,
148+
}
149+
150+
// isDataVolumeReady reports whether CDI is done with the disk. Failed and
151+
// Unknown are not ready, and neither is a DataVolume without a phase: CDI
152+
// leaves one there when it could not render the PVC spec.
153+
func isDataVolumeReady(dv *unstructured.Unstructured) bool {
154+
phase, _, _ := unstructured.NestedString(dv.Object, "status", "phase")
155+
switch phase {
156+
case "", "Failed", "Unknown":
157+
return false
158+
}
159+
return !dataVolumeInFlightPhases[phase]
160+
}
161+
162+
// dataVolumeAPIServed treats a NoMatch as not served and returns any other
163+
// discovery error, so the caller retries instead of concluding CDI is absent.
164+
func dataVolumeAPIServed(mapper meta.RESTMapper) (bool, error) {
165+
_, err := mapper.RESTMapping(dataVolumeGVK.GroupKind(), dataVolumeGVK.Version)
166+
if meta.IsNoMatchError(err) {
167+
return false, nil
168+
}
169+
return err == nil, err
170+
}
171+
172+
type sourceWatcher interface {
173+
Watch(src source.Source) error
174+
}
175+
176+
// tryStartDataVolumeWatch registers the DataVolume watch and reader once the
177+
// kind is served, and reports whether it did. The reader is set only after the
178+
// source has synced: the cache blocks a List on an unsynced informer until the
179+
// caller's context ends. A source that does not sync in time is cancelled by
180+
// WaitForSync, and the next call starts a fresh one.
181+
func (r *WorkloadMonitorReconciler) tryStartDataVolumeWatch(ctx context.Context, mapper meta.RESTMapper, c sourceWatcher, informers cache.Cache, syncTimeout time.Duration) (bool, error) {
182+
served, err := dataVolumeAPIServed(mapper)
183+
if err != nil || !served {
184+
return false, err
185+
}
186+
dv := &unstructured.Unstructured{}
187+
dv.SetGroupVersionKind(dataVolumeGVK)
188+
src := source.Kind[client.Object](informers, dv,
189+
handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(client.Object(dv), r.Client)))
190+
if err := c.Watch(src); err != nil {
191+
return false, fmt.Errorf("watching DataVolumes: %w", err)
192+
}
193+
syncCtx, cancel := context.WithTimeout(ctx, syncTimeout)
194+
defer cancel()
195+
if err := src.WaitForSync(syncCtx); err != nil {
196+
return false, fmt.Errorf("waiting for the DataVolume informer to sync: %w", err)
197+
}
198+
// WaitForSync returns nil when ctx itself is cancelled.
199+
if err := ctx.Err(); err != nil {
200+
return false, err
201+
}
202+
r.dataVolumeMu.Lock()
203+
defer r.dataVolumeMu.Unlock()
204+
// The manager's client reads unstructured objects straight from the API
205+
// server; the cache shares the informer the watch above starts.
206+
r.DataVolumeReader = informers
207+
return true, nil
208+
}
209+
210+
// watchDataVolumes adds the DataVolume watch once the first reconcile has run
211+
// and the kind is served, checking discovery again every interval until it is.
212+
// Starting on the first reconcile rather than on the next tick keeps a disk
213+
// created right after a controller restart from reading populated until then.
214+
func (r *WorkloadMonitorReconciler) watchDataVolumes(ctx context.Context, mapper meta.RESTMapper, c sourceWatcher, informers cache.Cache, interval time.Duration) {
215+
select {
216+
case <-ctx.Done():
217+
return
218+
case <-r.reconciledCh():
219+
}
220+
logger := log.FromContext(ctx)
221+
// The condition never returns an error, so the poll ends only when the
222+
// watch is added or the manager stops.
223+
_ = wait.PollUntilContextCancel(ctx, interval, true, func(ctx context.Context) (bool, error) {
224+
done, err := r.tryStartDataVolumeWatch(ctx, mapper, c, informers, dataVolumeSyncTimeout)
225+
if err != nil {
226+
logger.Error(err, "Unable to start the DataVolume watch, retrying")
227+
}
228+
return done, nil
229+
})
230+
}
231+
232+
// dataVolumesReady reports whether every DataVolume the monitor selects is
233+
// ready. A NoMatch from the reader counts as no DataVolumes.
234+
func (r *WorkloadMonitorReconciler) dataVolumesReady(ctx context.Context, monitor *cozyv1alpha1.WorkloadMonitor) (bool, error) {
235+
reader := r.dataVolumeReader()
236+
if reader == nil {
237+
return true, nil
238+
}
239+
list := &unstructured.UnstructuredList{}
240+
list.SetGroupVersionKind(dataVolumeGVK.GroupVersion().WithKind(dataVolumeGVK.Kind + "List"))
241+
if err := reader.List(
242+
ctx,
243+
list,
244+
client.InNamespace(monitor.Namespace),
245+
client.MatchingLabels(monitor.Spec.Selector),
246+
); err != nil {
247+
if meta.IsNoMatchError(err) {
248+
return true, nil
249+
}
250+
return false, err
251+
}
252+
for i := range list.Items {
253+
if !isDataVolumeReady(&list.Items[i]) {
254+
return false, nil
255+
}
256+
}
257+
return true, nil
70258
}
71259

72260
// +kubebuilder:rbac:groups=cozystack.io,resources=workloadmonitors,verbs=get;list;watch;create;update;patch;delete
@@ -78,6 +266,7 @@ type WorkloadMonitorReconciler struct {
78266
// +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get
79267
// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
80268
// +kubebuilder:rbac:groups=objectstorage.k8s.io,resources=bucketclaims,verbs=get;list;watch
269+
// +kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch
81270

82271
// isBucketClaimReady checks if the BucketClaim has been provisioned.
83272
func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketClaim) bool {
@@ -568,6 +757,7 @@ func (r *WorkloadMonitorReconciler) reconcilePodForMonitor(
568757
// 1. It reconciles WorkloadMonitor objects themselves (create/update/delete).
569758
// 2. It also reconciles Pod events mapped to WorkloadMonitor via label selector.
570759
func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
760+
r.reconciledOnce.Do(func() { close(r.reconciledCh()) })
571761
logger := log.FromContext(ctx)
572762

573763
// Fetch the WorkloadMonitor object if it exists
@@ -691,6 +881,12 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
691881
}
692882
}
693883

884+
dataVolumesReady, err := r.dataVolumesReady(ctx, monitor)
885+
if err != nil {
886+
logger.Error(err, "Unable to list DataVolumes for WorkloadMonitor", "monitor", monitor.Name)
887+
return ctrl.Result{}, err
888+
}
889+
694890
// Update WorkloadMonitor status based on observed pods
695891
monitor.Status.ObservedReplicas = observedReplicas
696892
monitor.Status.AvailableReplicas = availableReplicas
@@ -711,6 +907,9 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
711907
if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas {
712908
fresh.Status.Operational = pointer.Bool(false)
713909
}
910+
if !dataVolumesReady {
911+
fresh.Status.Operational = pointer.Bool(false)
912+
}
714913
return r.Status().Update(ctx, fresh)
715914
})
716915
if err != nil {
@@ -736,7 +935,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
736935

737936
// SetupWithManager registers our controller with the Manager and sets up watches.
738937
func (r *WorkloadMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error {
739-
return ctrl.NewControllerManagedBy(mgr).
938+
builder := ctrl.NewControllerManagedBy(mgr).
740939
// Watch WorkloadMonitor objects
741940
For(&cozyv1alpha1.WorkloadMonitor{}).
742941
// Also watch Pod objects and map them back to WorkloadMonitor if labels match
@@ -755,8 +954,20 @@ func (r *WorkloadMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error {
755954
handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&cosiv1alpha1.BucketClaim{}, r.Client)),
756955
).
757956
// Watch for changes to Workload objects we create (owned by WorkloadMonitor)
758-
Owns(&cozyv1alpha1.Workload{}).
759-
Complete(r)
957+
Owns(&cozyv1alpha1.Workload{})
958+
c, err := builder.Build(r)
959+
if err != nil {
960+
return err
961+
}
962+
963+
// A watch on a kind whose CRD is absent never syncs, and controller-runtime
964+
// does not start a controller with unsynced caches. CDI ships with the IaaS
965+
// bundle only and can be installed while this controller is running, so
966+
// the DataVolume watch is added once the kind appears rather than at setup.
967+
return mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
968+
r.watchDataVolumes(ctx, mgr.GetRESTMapper(), c, mgr.GetCache(), dataVolumeAPIPollInterval)
969+
return nil
970+
}))
760971
}
761972

762973
func mapObjectToMonitor[T client.Object](_ T, c client.Client) func(ctx context.Context, obj client.Object) []reconcile.Request {

0 commit comments

Comments
 (0)