|
32 | 32 | apiVersion: opensearch.opster.io/v1 |
33 | 33 | kind: OpenSearchCluster |
34 | 34 | name: opensearch-test |
| 35 | + # The roll's own fields — updateRevision, currentRevision, generation, |
| 36 | + # updatedReplicas — are what separate a template that was never rewritten from |
| 37 | + # one the operator has not restarted the pod onto (#4259). The nodePool |
| 38 | + # StatefulSet is OnDelete (opensearch-k8s-operator v2.8.0, builders/cluster.go |
| 39 | + # sets UpdateStrategy.Type OnDelete), so the roll is the operator's |
| 40 | + # rolling-restart reconciler's to drive — it deletes the pod itself — not the |
| 41 | + # StatefulSet controller's. describe omits those fields, so the StatefulSet is |
| 42 | + # captured both ways: get -o yaml carries the revisions and the rendered volume |
| 43 | + # list, describe carries the pod template and the controller's events. |
| 44 | + - get: |
| 45 | + apiVersion: apps/v1 |
| 46 | + kind: StatefulSet |
| 47 | + format: yaml |
| 48 | + - describe: |
| 49 | + apiVersion: apps/v1 |
| 50 | + kind: StatefulSet |
| 51 | + name: opensearch-test-nodes |
35 | 52 | # cert-manager records a refusal to issue in the Certificate's conditions. |
36 | 53 | # Without this an unissued chain shows up only as an absent Secret, and the |
37 | 54 | # CR then names a Secret that is never going to appear. |
|
46 | 63 | - podLogs: |
47 | 64 | selector: opster.io/opensearch-cluster=opensearch-test |
48 | 65 | tail: 100 |
| 66 | + # The restart is the operator's rolling-restart reconciler's to start (see the |
| 67 | + # StatefulSet catch note above). It logs "Starting rolling restart of the |
| 68 | + # StatefulSet" at rollingRestart.go:155 (opensearch-k8s-operator v2.8.0) once |
| 69 | + # the early gates pass — cluster Initialized (:123), every replica ready (:94), |
| 70 | + # an update actually pending (:150-154). Just before that line, at :138, it |
| 71 | + # builds its OpenSearch client, which pings the cluster |
| 72 | + # (NewOsClusterClientFromConfig, os_client.go:104-112) and returns the error |
| 73 | + # from the reconcile at :139-141 when the cluster is unreachable. So the line's |
| 74 | + # absence reads two ways, not one: an early gate held silently, or that client |
| 75 | + # ping failed — loudly, as a reconcile error — before the line was reached. The |
| 76 | + # line present and followed by "Couldn't proceed with rolling restart ... |
| 77 | + # because <reason>" (:182) means the roll reached the health/pod-delete stage, |
| 78 | + # where CheckClusterStatusForRestart (:177) holds it until cluster health is |
| 79 | + # green (or yellow under the .opensearch-observability exception) and <reason> |
| 80 | + # names the hold — "waiting for health to be green" for the mount-plan incident. |
| 81 | + # That "Couldn't proceed ... because" line is the health hold's signal, one per |
| 82 | + # ~10s requeue and what the tail below is sized for, and |
| 83 | + # CheckClusterStatusForRestart is not the last gate: PreparePodForDelete (:195) |
| 84 | + # follows it. The verdict is in the operator, not the cluster pods above, and it |
| 85 | + # runs in its own namespace. |
| 86 | + - podLogs: |
| 87 | + namespace: cozy-opensearch-operator |
| 88 | + selector: control-plane=controller-manager |
| 89 | + # tail spans the whole assert window. The manager logs ~4-5 lines per ~10s |
| 90 | + # requeue, not one, so 100 lines reach back only about a third of a 12m roll |
| 91 | + # and can lose a one-shot cause. 2000 is effectively everything the container |
| 92 | + # has logged, not a sized window — on run 105251129282 it returned all 457 |
| 93 | + # lines since start. |
| 94 | + tail: 2000 |
49 | 95 | # The external Service is a LoadBalancer, so an address pool with nothing left |
50 | 96 | # in it holds the HelmRelease short of Ready with nothing in the HelmRelease |
51 | 97 | # itself naming the cause. |
@@ -291,13 +337,54 @@ spec: |
291 | 337 | # same reason — the replacement pod pulls and starts OpenSearch from scratch. |
292 | 338 | - name: verify-statefulset-rolled-onto-the-new-mount-plan |
293 | 339 | try: |
| 340 | + # Two asserts, not one, so a timeout names which half did not happen: the |
| 341 | + # template rewrite and the roll onto it are separate outcomes with separate |
| 342 | + # owners (#4259). |
| 343 | + # |
| 344 | + # First half: the operator rewrote the template onto the subPath mount plan. |
| 345 | + # That is the operator's to do the moment the CR names the cert-manager |
| 346 | + # Secrets, which verify-cr-dropped-generate and the certificate steps already |
| 347 | + # waited for, so it lands within a reconcile or two rather than a pod |
| 348 | + # lifetime. A shorter budget than the roll below, and a failure here is the |
| 349 | + # operator not writing the volumes — not a pod that will not start. |
| 350 | + - assert: |
| 351 | + timeout: 6m |
| 352 | + resource: |
| 353 | + apiVersion: apps/v1 |
| 354 | + kind: StatefulSet |
| 355 | + metadata: |
| 356 | + name: opensearch-test-nodes |
| 357 | + (spec.template.spec.volumes[?name == 'http-cert'].secret.secretName): |
| 358 | + - opensearch-test-http-server |
| 359 | + (spec.template.spec.volumes[?name == 'http-key'].secret.secretName): |
| 360 | + - opensearch-test-http-server |
| 361 | + (spec.template.spec.volumes[?name == 'http-ca'].secret.secretName): |
| 362 | + - opensearch-test.http-ca |
| 363 | + # Second half: the operator restarted the pod onto that template and the |
| 364 | + # StatefulSet finished rolling. currentRevision only advances here once the |
| 365 | + # operator's rolling-restart reconciler patches it (see the StatefulSet catch |
| 366 | + # note above), so equal revisions plus a ready updated replica is that |
| 367 | + # reconciler having run to the end. This half also asserts readyReplicas, so a |
| 368 | + # failure with the assert above green is the roll not completing — either the |
| 369 | + # operator never restarted the pod (the reconciler holds on the health gate, |
| 370 | + # per the podLogs note) or a replacement was deleted but never came Ready — not |
| 371 | + # a template that was never written. |
| 372 | + # |
| 373 | + # The three volume projections are carried in here too, so the roll is pinned |
| 374 | + # to the template the first half checked and not to whatever revision happens |
| 375 | + # to be current when the status converges. The CR moves only |
| 376 | + # once in this suite, but the split otherwise lets a rewrite landing after the |
| 377 | + # first assert satisfy this half on its own. observedGeneration == generation |
| 378 | + # is compared at the resource root because a status-scoped expression cannot |
| 379 | + # reach metadata from its own node. |
294 | 380 | - assert: |
295 | 381 | timeout: 12m |
296 | 382 | resource: |
297 | 383 | apiVersion: apps/v1 |
298 | 384 | kind: StatefulSet |
299 | 385 | metadata: |
300 | 386 | name: opensearch-test-nodes |
| 387 | + (status.observedGeneration == metadata.generation): true |
301 | 388 | (spec.template.spec.volumes[?name == 'http-cert'].secret.secretName): |
302 | 389 | - opensearch-test-http-server |
303 | 390 | (spec.template.spec.volumes[?name == 'http-key'].secret.secretName): |
|
0 commit comments