|
| 1 | +diff --git a/opensearch-operator/controllers/opensearchController.go b/opensearch-operator/controllers/opensearchController.go |
| 2 | +index 6bf9f9c..5ef3e39 100644 |
| 3 | +--- a/opensearch-operator/controllers/opensearchController.go |
| 4 | ++++ b/opensearch-operator/controllers/opensearchController.go |
| 5 | +@@ -201,6 +201,10 @@ func (r *OpenSearchClusterReconciler) deleteExternalResources(ctx context.Contex |
| 6 | + r.Instance, |
| 7 | + ) |
| 8 | + |
| 9 | ++ // Not wrapped: every DeleteResources at this version returns immediately |
| 10 | ++ // (tls.go:578, securityconfig.go:254, configuration.go:182, cluster.go:384, |
| 11 | ++ // dashboards.go:188), so a watchdog here could only ever spawn goroutines |
| 12 | ++ // for a call that cannot block. |
| 13 | + componentReconcilers := []reconcilers.ComponentReconciler{ |
| 14 | + tls.DeleteResources, |
| 15 | + securityconfig.DeleteResources, |
| 16 | +@@ -234,6 +238,15 @@ func (r *OpenSearchClusterReconciler) reconcilePhasePending(ctx context.Context) |
| 17 | + return ctrl.Result{Requeue: true}, nil |
| 18 | + } |
| 19 | + |
| 20 | ++// A pass through the components takes seconds on a healthy cluster: the |
| 21 | ++// controller requeues itself every 30s, and on the e2e runs that prompted this |
| 22 | ++// the reconciles of a settled cluster came about 10s apart, driven by watch |
| 23 | ++// events on top of that requeue. A minute is past anything normal and still |
| 24 | ++// early enough to name a stall while whoever is watching is still watching, |
| 25 | ++// and it is both how long a call may run before it is reported and how often |
| 26 | ++// the report then repeats. |
| 27 | ++const componentStallReportEvery = time.Minute |
| 28 | ++ |
| 29 | + func (r *OpenSearchClusterReconciler) reconcilePhaseRunning(ctx context.Context) (ctrl.Result, error) { |
| 30 | + // Update initialized status first |
| 31 | + if !r.Instance.Status.Initialized { |
| 32 | +@@ -313,16 +326,19 @@ func (r *OpenSearchClusterReconciler) reconcilePhaseRunning(ctx context.Context) |
| 33 | + r.Instance, |
| 34 | + ) |
| 35 | + |
| 36 | ++ // Wrapped so a component that stops returning is named in the log. Without |
| 37 | ++ // it the loop goes quiet mid-pass and the object is left unreconciled with |
| 38 | ++ // nothing written about it. |
| 39 | + componentReconcilers := []reconcilers.ComponentReconciler{ |
| 40 | +- tls.Reconcile, |
| 41 | +- securityconfig.Reconcile, |
| 42 | +- config.Reconcile, |
| 43 | +- cluster.Reconcile, |
| 44 | +- scaler.Reconcile, |
| 45 | +- dashboards.Reconcile, |
| 46 | +- upgrade.Reconcile, |
| 47 | +- restart.Reconcile, |
| 48 | +- snapshotrepository.Reconcile, |
| 49 | ++ reconcilers.WithStallWatchdog(r.Logger, "tls", componentStallReportEvery, componentStallReportEvery, tls.Reconcile), |
| 50 | ++ reconcilers.WithStallWatchdog(r.Logger, "securityconfig", componentStallReportEvery, componentStallReportEvery, securityconfig.Reconcile), |
| 51 | ++ reconcilers.WithStallWatchdog(r.Logger, "config", componentStallReportEvery, componentStallReportEvery, config.Reconcile), |
| 52 | ++ reconcilers.WithStallWatchdog(r.Logger, "cluster", componentStallReportEvery, componentStallReportEvery, cluster.Reconcile), |
| 53 | ++ reconcilers.WithStallWatchdog(r.Logger, "scaler", componentStallReportEvery, componentStallReportEvery, scaler.Reconcile), |
| 54 | ++ reconcilers.WithStallWatchdog(r.Logger, "dashboards", componentStallReportEvery, componentStallReportEvery, dashboards.Reconcile), |
| 55 | ++ reconcilers.WithStallWatchdog(r.Logger, "upgrade", componentStallReportEvery, componentStallReportEvery, upgrade.Reconcile), |
| 56 | ++ reconcilers.WithStallWatchdog(r.Logger, "restart", componentStallReportEvery, componentStallReportEvery, restart.Reconcile), |
| 57 | ++ reconcilers.WithStallWatchdog(r.Logger, "snapshotrepository", componentStallReportEvery, componentStallReportEvery, snapshotrepository.Reconcile), |
| 58 | + } |
| 59 | + for _, rec := range componentReconcilers { |
| 60 | + result, err := rec() |
| 61 | +diff --git a/opensearch-operator/pkg/reconcilers/stallwatchdog.go b/opensearch-operator/pkg/reconcilers/stallwatchdog.go |
| 62 | +new file mode 100644 |
| 63 | +index 0000000..75e2e49 |
| 64 | +--- /dev/null |
| 65 | ++++ b/opensearch-operator/pkg/reconcilers/stallwatchdog.go |
| 66 | +@@ -0,0 +1,75 @@ |
| 67 | ++package reconcilers |
| 68 | ++ |
| 69 | ++import ( |
| 70 | ++ "time" |
| 71 | ++ |
| 72 | ++ "github.com/go-logr/logr" |
| 73 | ++ "sigs.k8s.io/controller-runtime/pkg/reconcile" |
| 74 | ++) |
| 75 | ++ |
| 76 | ++// WithStallWatchdog wraps a component reconciler so a call that stops returning |
| 77 | ++// keeps saying so, under the name of the component it is stuck in. |
| 78 | ++// |
| 79 | ++// None of the components the cluster controller runs in sequence logs |
| 80 | ++// unconditionally when it is entered. The TLS one has a line above its work, |
| 81 | ++// but it sits on the early return for a cluster with no security section, which |
| 82 | ++// the chart in this repository never produces, and the rolling restart one |
| 83 | ++// opens at V(1), off at the verbosity the operator ships with. So a call that |
| 84 | ++// never comes back leaves nothing behind at all: the last line in the operator |
| 85 | ++// log stays whatever was written before the stall, the object stops being |
| 86 | ++// reconciled, and the two look identical from the outside. Logging every entry |
| 87 | ++// instead would put nine lines per cluster into the log every reconcile, so |
| 88 | ++// nothing is written while a component behaves. |
| 89 | ++// |
| 90 | ++// The report repeats rather than firing once, so a log read at any point during |
| 91 | ++// a stall says which component is in flight and for how long. A single line |
| 92 | ++// dated at the start of the stall does not answer the question a reader has |
| 93 | ++// five minutes in, which is whether the call is still there. |
| 94 | ++func WithStallWatchdog(logger logr.Logger, name string, warnAfter, repeatEvery time.Duration, rec ComponentReconciler) ComponentReconciler { |
| 95 | ++ return func() (reconcile.Result, error) { |
| 96 | ++ done := make(chan struct{}) |
| 97 | ++ defer close(done) |
| 98 | ++ |
| 99 | ++ // Taken here rather than inside the goroutine: on a host loaded enough to |
| 100 | ++ // be worth a stall report, the two differ by however long the scheduler |
| 101 | ++ // took, which is measured into the one number the report carries. |
| 102 | ++ started := time.Now() |
| 103 | ++ |
| 104 | ++ go func() { |
| 105 | ++ timer := time.NewTimer(warnAfter) |
| 106 | ++ defer timer.Stop() |
| 107 | ++ select { |
| 108 | ++ case <-done: |
| 109 | ++ return |
| 110 | ++ case <-timer.C: |
| 111 | ++ } |
| 112 | ++ |
| 113 | ++ ticker := time.NewTicker(repeatEvery) |
| 114 | ++ defer ticker.Stop() |
| 115 | ++ for { |
| 116 | ++ // Narrows the window on a false report, does not close it: a |
| 117 | ++ // tick landing on the return would otherwise write its line |
| 118 | ++ // half the time, because a select with both cases ready picks |
| 119 | ++ // at random. A preemption between this check and the write |
| 120 | ++ // still gets one line out. |
| 121 | ++ select { |
| 122 | ++ case <-done: |
| 123 | ++ return |
| 124 | ++ default: |
| 125 | ++ } |
| 126 | ++ |
| 127 | ++ logger.Info("component reconciler has not returned", |
| 128 | ++ "component", name, |
| 129 | ++ "elapsed", time.Since(started).String()) |
| 130 | ++ |
| 131 | ++ select { |
| 132 | ++ case <-done: |
| 133 | ++ return |
| 134 | ++ case <-ticker.C: |
| 135 | ++ } |
| 136 | ++ } |
| 137 | ++ }() |
| 138 | ++ |
| 139 | ++ return rec() |
| 140 | ++ } |
| 141 | ++} |
| 142 | +diff --git a/opensearch-operator/pkg/reconcilers/stallwatchdog_test.go b/opensearch-operator/pkg/reconcilers/stallwatchdog_test.go |
| 143 | +new file mode 100644 |
| 144 | +index 0000000..0092f4e |
| 145 | +--- /dev/null |
| 146 | ++++ b/opensearch-operator/pkg/reconcilers/stallwatchdog_test.go |
| 147 | +@@ -0,0 +1,160 @@ |
| 148 | ++package reconcilers |
| 149 | ++ |
| 150 | ++import ( |
| 151 | ++ "errors" |
| 152 | ++ "regexp" |
| 153 | ++ "sync" |
| 154 | ++ "time" |
| 155 | ++ |
| 156 | ++ "github.com/go-logr/logr/funcr" |
| 157 | ++ . "github.com/onsi/ginkgo/v2" |
| 158 | ++ . "github.com/onsi/gomega" |
| 159 | ++ "sigs.k8s.io/controller-runtime/pkg/reconcile" |
| 160 | ++) |
| 161 | ++ |
| 162 | ++type recordingSink struct { |
| 163 | ++ mu sync.Mutex |
| 164 | ++ lines []string |
| 165 | ++} |
| 166 | ++ |
| 167 | ++func (r *recordingSink) write(prefix, args string) { |
| 168 | ++ r.mu.Lock() |
| 169 | ++ defer r.mu.Unlock() |
| 170 | ++ r.lines = append(r.lines, prefix+args) |
| 171 | ++} |
| 172 | ++ |
| 173 | ++func (r *recordingSink) snapshot() []string { |
| 174 | ++ r.mu.Lock() |
| 175 | ++ defer r.mu.Unlock() |
| 176 | ++ out := make([]string, len(r.lines)) |
| 177 | ++ copy(out, r.lines) |
| 178 | ++ return out |
| 179 | ++} |
| 180 | ++ |
| 181 | ++var errStallWatchdogTest = errors.New("component failed") |
| 182 | ++ |
| 183 | ++var elapsedPattern = regexp.MustCompile(`"elapsed"="([^"]+)"`) |
| 184 | ++ |
| 185 | ++var _ = Describe("component reconciler stall watchdog", func() { |
| 186 | ++ var sink *recordingSink |
| 187 | ++ |
| 188 | ++ BeforeEach(func() { |
| 189 | ++ sink = &recordingSink{} |
| 190 | ++ }) |
| 191 | ++ |
| 192 | ++ It("passes the wrapped result and error through untouched", func() { |
| 193 | ++ want := reconcile.Result{Requeue: true, RequeueAfter: 7 * time.Second} |
| 194 | ++ wrapped := WithStallWatchdog(funcr.New(sink.write, funcr.Options{}), "cluster", time.Hour, time.Hour, func() (reconcile.Result, error) { |
| 195 | ++ return want, errStallWatchdogTest |
| 196 | ++ }) |
| 197 | ++ |
| 198 | ++ got, err := wrapped() |
| 199 | ++ |
| 200 | ++ Expect(got).To(Equal(want)) |
| 201 | ++ Expect(err).To(MatchError(errStallWatchdogTest)) |
| 202 | ++ }) |
| 203 | ++ |
| 204 | ++ It("stays silent while the component returns promptly", func() { |
| 205 | ++ // The call takes long enough for a watchdog that ignored warnAfter to |
| 206 | ++ // get its report out, and warnAfter is far enough away that a loaded CI |
| 207 | ++ // host cannot reach it: what is pinned is the threshold, not the speed |
| 208 | ++ // of the scheduler. |
| 209 | ++ wrapped := WithStallWatchdog(funcr.New(sink.write, funcr.Options{}), "cluster", 10*time.Second, 10*time.Second, func() (reconcile.Result, error) { |
| 210 | ++ time.Sleep(150 * time.Millisecond) |
| 211 | ++ return reconcile.Result{}, nil |
| 212 | ++ }) |
| 213 | ++ |
| 214 | ++ _, err := wrapped() |
| 215 | ++ Expect(err).ToNot(HaveOccurred()) |
| 216 | ++ |
| 217 | ++ Consistently(sink.snapshot, 200*time.Millisecond, 20*time.Millisecond).Should(BeEmpty()) |
| 218 | ++ }) |
| 219 | ++ |
| 220 | ++ It("drops the watchdog when the call returns before the threshold", func() { |
| 221 | ++ wrapped := WithStallWatchdog(funcr.New(sink.write, funcr.Options{}), "cluster", 120*time.Millisecond, 120*time.Millisecond, func() (reconcile.Result, error) { |
| 222 | ++ return reconcile.Result{}, nil |
| 223 | ++ }) |
| 224 | ++ |
| 225 | ++ _, err := wrapped() |
| 226 | ++ Expect(err).ToNot(HaveOccurred()) |
| 227 | ++ |
| 228 | ++ // Well past the threshold: a watchdog that does not notice the call |
| 229 | ++ // returned would report here, once per wrapped call on every reconcile |
| 230 | ++ // of every cluster, which is the cost this design claims not to have. |
| 231 | ++ Consistently(sink.snapshot, 400*time.Millisecond, 40*time.Millisecond).Should(BeEmpty()) |
| 232 | ++ }) |
| 233 | ++ |
| 234 | ++ It("keeps reporting while the call stays blocked", func() { |
| 235 | ++ release := make(chan struct{}) |
| 236 | ++ returned := make(chan struct{}) |
| 237 | ++ wrapped := WithStallWatchdog(funcr.New(sink.write, funcr.Options{}), "restart", 10*time.Millisecond, 10*time.Millisecond, func() (reconcile.Result, error) { |
| 238 | ++ <-release |
| 239 | ++ return reconcile.Result{}, nil |
| 240 | ++ }) |
| 241 | ++ |
| 242 | ++ go func() { |
| 243 | ++ defer GinkgoRecover() |
| 244 | ++ defer close(returned) |
| 245 | ++ _, err := wrapped() |
| 246 | ++ Expect(err).ToNot(HaveOccurred()) |
| 247 | ++ }() |
| 248 | ++ |
| 249 | ++ // A single report would satisfy the spec above and say nothing about |
| 250 | ++ // whether the call is still in flight by the time anyone reads the |
| 251 | ++ // log: what a reader needs is a line dated now, not one dated at the |
| 252 | ++ // moment the stall began. |
| 253 | ++ Eventually(func() int { return len(sink.snapshot()) }).Should(BeNumerically(">=", 3)) |
| 254 | ++ |
| 255 | ++ // The report carries how long the call has been in flight, and that |
| 256 | ++ // number has to move: a frozen or zero elapsed says nothing about |
| 257 | ++ // whether the stall started a second or an hour ago. One snapshot |
| 258 | ++ // feeds both halves of the assertion, since the ticker appends |
| 259 | ++ // between two reads and they then disagree on the count. |
| 260 | ++ lines := sink.snapshot() |
| 261 | ++ var elapsed []string |
| 262 | ++ for _, line := range lines { |
| 263 | ++ if m := elapsedPattern.FindStringSubmatch(line); m != nil { |
| 264 | ++ elapsed = append(elapsed, m[1]) |
| 265 | ++ } |
| 266 | ++ } |
| 267 | ++ Expect(elapsed).To(HaveLen(len(lines))) |
| 268 | ++ Expect(elapsed[0]).ToNot(Equal("0s")) |
| 269 | ++ Expect(elapsed[1]).ToNot(Equal(elapsed[0])) |
| 270 | ++ |
| 271 | ++ close(release) |
| 272 | ++ Eventually(returned).Should(BeClosed()) |
| 273 | ++ }) |
| 274 | ++ |
| 275 | ++ It("names the component while its call is still in flight", func() { |
| 276 | ++ release := make(chan struct{}) |
| 277 | ++ returned := make(chan struct{}) |
| 278 | ++ wrapped := WithStallWatchdog(funcr.New(sink.write, funcr.Options{}), "securityconfig", 10*time.Millisecond, 10*time.Millisecond, func() (reconcile.Result, error) { |
| 279 | ++ <-release |
| 280 | ++ return reconcile.Result{}, nil |
| 281 | ++ }) |
| 282 | ++ |
| 283 | ++ go func() { |
| 284 | ++ defer GinkgoRecover() |
| 285 | ++ defer close(returned) |
| 286 | ++ _, err := wrapped() |
| 287 | ++ Expect(err).ToNot(HaveOccurred()) |
| 288 | ++ }() |
| 289 | ++ |
| 290 | ++ Eventually(sink.snapshot).Should(ContainElement(ContainSubstring("securityconfig"))) |
| 291 | ++ |
| 292 | ++ close(release) |
| 293 | ++ Eventually(returned).Should(BeClosed()) |
| 294 | ++ |
| 295 | ++ // The goroutine has to stop with the call, or every stalled reconcile |
| 296 | ++ // leaves one behind writing for the life of the process. A report |
| 297 | ++ // already in flight when the call returned is allowed to land, so what |
| 298 | ++ // is pinned is that the count settles at all rather than the moment it |
| 299 | ++ // does: a goroutine still running adds a line every interval and never |
| 300 | ++ // lets two readings five intervals apart agree. |
| 301 | ++ Eventually(func() bool { |
| 302 | ++ before := len(sink.snapshot()) |
| 303 | ++ time.Sleep(50 * time.Millisecond) |
| 304 | ++ return len(sink.snapshot()) == before |
| 305 | ++ }, 2*time.Second, 10*time.Millisecond).Should(BeTrue()) |
| 306 | ++ }) |
| 307 | ++}) |
0 commit comments