Skip to content

Commit e774fc1

Browse files
committed
fix(opensearch-operator): keep a single-replica cluster electable after bootstrap
The operator deletes the bootstrap pod as soon as initialization completes and never looks at the voting configuration first. With one cluster_manager replica there are exactly two eligible nodes, OpenSearch keeps the configuration at an odd size of one, and the voter it keeps is the bootstrap node that formed the cluster. Deleting that pod takes away the only vote there was, and the survivor can never hold an election again, because changing the configuration itself needs a quorum. The cluster is unusable about a minute after it installs, and a node with no cluster-manager still answers the readiness probe, so nothing says so until something restarts the pod. Build the operator in-tree, the way kamaji and redis-operator already are, and carry a patch that excludes the bootstrap node from the voting configuration and waits for that to commit before deleting the pod. Reaching the cluster is a precondition rather than a nicety: every failure keeps the pod and requeues, because a bootstrap pod that outlives its purpose costs a pod, while a cluster that cannot elect does not recover without a human. Upstream carries the same defect, tracked as opensearch-project/opensearch-k8s-operator#1448. Assisted-by: LLM Signed-off-by: IvanHunters <[email protected]>
1 parent 37e22dc commit e774fc1

4 files changed

Lines changed: 273 additions & 1 deletion

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ build: build-deps
3535
make -C packages/system/securitygroup-controller image
3636
make -C packages/system/grafana-operator image
3737
make -C packages/system/redis-operator image
38+
make -C packages/system/opensearch-operator image
3839
make -C packages/core/testing image
3940
make -C packages/core/talos image
4041
make -C packages/core/platform image
Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,32 @@
1+
# Single source for the version: the image's ARG VERSION. The chart and the
2+
# binary have to move together, because the chart ships the CRDs the binary is
3+
# compiled against.
4+
OPENSEARCH_OPERATOR_TAG=$(shell grep -F 'ARG VERSION=' images/opensearch-operator/Dockerfile | cut -f2 -d=)
5+
16
export NAME=opensearch-operator
27
export NAMESPACE=cozy-$(NAME)
38

9+
include ../../../hack/common-envs.mk
410
include ../../../hack/package.mk
511

612
update:
713
rm -rf charts
814
helm repo add opensearch-operator https://opensearch-project.github.io/opensearch-k8s-operator/
915
helm repo update opensearch-operator
10-
helm pull opensearch-operator/opensearch-operator --version 2.8.0 --untar --untardir charts
16+
helm pull opensearch-operator/opensearch-operator --version $(patsubst v%,%,$(OPENSEARCH_OPERATOR_TAG)) --untar --untardir charts
1117
patch --no-backup-if-mismatch -p1 < patches/leaderElection.diff
1218

19+
image:
20+
docker buildx build images/opensearch-operator \
21+
$(call image-tags,opensearch-operator,$(OPENSEARCH_OPERATOR_TAG)) \
22+
$(call cache-args,opensearch-operator) \
23+
--metadata-file images/opensearch-operator.json \
24+
$(BUILDX_ARGS)
25+
REPOSITORY="$(REGISTRY)/opensearch-operator" \
26+
yq -i '.opensearch-operator.manager.image.repository = strenv(REPOSITORY)' values.yaml
27+
TAG=$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/opensearch-operator.json -o json -r) \
28+
yq -i '.opensearch-operator.manager.image.tag = strenv(TAG)' values.yaml
29+
rm -f images/opensearch-operator.json
30+
1331
test:
1432
helm unittest .
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Build the manager binary from the upstream sources with the Cozystack patch
2+
# applied. The patch keeps a single-replica cluster electable when the operator
3+
# retires its bootstrap pod; see patches/bootstrap-voting-exclusion.diff and
4+
# https://github.com/opensearch-project/opensearch-k8s-operator/issues/1448.
5+
FROM golang:1.26@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a AS builder
6+
7+
# Keep in step with the chart version pulled by `make update`: the chart's
8+
# appVersion is what picks the operator image upstream, so a mismatch here would
9+
# run one version's binary against another's CRDs.
10+
ARG VERSION=v2.8.0
11+
12+
ARG TARGETOS
13+
ARG TARGETARCH
14+
15+
WORKDIR /workspace
16+
17+
RUN curl -sSL https://github.com/opensearch-project/opensearch-k8s-operator/archive/refs/tags/${VERSION}.tar.gz | tar -xzvf- --strip=1
18+
19+
COPY patches /patches
20+
RUN git apply /patches/*.diff
21+
22+
WORKDIR /workspace/opensearch-operator
23+
24+
RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -o manager main.go
25+
26+
FROM gcr.io/distroless/static:nonroot@sha256:d29e660cc75a5b6b1334e03c5c81ccf9bc0884a002c6000dbf0fb96034814478
27+
WORKDIR /
28+
COPY --from=builder /workspace/opensearch-operator/manager .
29+
USER 65532:65532
30+
31+
ENTRYPOINT ["/manager"]
Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
diff --git a/opensearch-operator/opensearch-gateway/services/os_client.go b/opensearch-operator/opensearch-gateway/services/os_client.go
2+
index 2ec30c5..6e3fd2e 100644
3+
--- a/opensearch-operator/opensearch-gateway/services/os_client.go
4+
+++ b/opensearch-operator/opensearch-gateway/services/os_client.go
5+
@@ -4,6 +4,7 @@ import (
6+
"context"
7+
"crypto/tls"
8+
"encoding/json"
9+
+ "fmt"
10+
"github.com/Opster/opensearch-k8s-operator/opensearch-operator/pkg/helpers"
11+
"io"
12+
"k8s.io/utils/ptr"
13+
@@ -535,3 +536,89 @@ func generateAPIPathSnapshotUpdatePolicies(resource, name string, seqno, primary
14+
path.WriteString(strconv.Itoa(primaryterm))
15+
return path
16+
}
17+
+
18+
+// votingConfigState is the slice of the cluster state that says who may vote.
19+
+// The coordination metadata names voters by node id, so the node list travels
20+
+// in the same response to resolve an id back to a name.
21+
+type votingConfigState struct {
22+
+ Nodes map[string]struct {
23+
+ Name string `json:"name"`
24+
+ } `json:"nodes"`
25+
+ Metadata struct {
26+
+ ClusterCoordination struct {
27+
+ LastCommittedConfig []string `json:"last_committed_config"`
28+
+ } `json:"cluster_coordination"`
29+
+ } `json:"metadata"`
30+
+}
31+
+
32+
+// ExcludeNodeFromVotingConfig commits a voting configuration that leaves
33+
+// nodeName out, which is what makes the node safe to remove: a node still in
34+
+// the configuration is a vote the cluster keeps counting on.
35+
+func (client *OsClusterClient) ExcludeNodeFromVotingConfig(ctx context.Context, nodeName string) error {
36+
+ path := generateAPIPathVotingConfigExclusions(nodeName)
37+
+ res, err := doHTTPPost(ctx, client.client, path, nil)
38+
+ if err != nil {
39+
+ return err
40+
+ }
41+
+ defer helpers.SafeClose(res.Body)
42+
+ if res.IsError() {
43+
+ return fmt.Errorf("excluding %s from the voting configuration returned %s", nodeName, res.Status())
44+
+ }
45+
+ return nil
46+
+}
47+
+
48+
+// ClearVotingConfigExclusions empties the exclusion list, which is capped and
49+
+// would otherwise keep entries for nodes that are long gone. wait_for_removal
50+
+// stays at its default: clearing without it can re-admit a node that is still
51+
+// running, and the caller has already removed the one it excluded.
52+
+func (client *OsClusterClient) ClearVotingConfigExclusions(ctx context.Context) error {
53+
+ var path strings.Builder
54+
+ path.WriteString("/_cluster/voting_config_exclusions")
55+
+ res, err := doHTTPDelete(ctx, client.client, path)
56+
+ if err != nil {
57+
+ return err
58+
+ }
59+
+ defer helpers.SafeClose(res.Body)
60+
+ if res.IsError() {
61+
+ return fmt.Errorf("clearing the voting configuration exclusions returned %s", res.Status())
62+
+ }
63+
+ return nil
64+
+}
65+
+
66+
+// IsInVotingConfig reports whether nodeName is still one of the nodes whose
67+
+// vote counts. An exclusion is committed through the cluster state rather than
68+
+// applied by the time the request returns, so this is what tells a caller the
69+
+// exclusion has actually taken.
70+
+func (client *OsClusterClient) IsInVotingConfig(ctx context.Context, nodeName string) (bool, error) {
71+
+ var path strings.Builder
72+
+ path.WriteString("/_cluster/state/nodes,metadata")
73+
+ path.WriteString("?filter_path=nodes.*.name,metadata.cluster_coordination.last_committed_config")
74+
+ res, err := doHTTPGet(ctx, client.client, path)
75+
+ if err != nil {
76+
+ return false, err
77+
+ }
78+
+ defer helpers.SafeClose(res.Body)
79+
+ if res.IsError() {
80+
+ return false, fmt.Errorf("reading the voting configuration returned %s", res.Status())
81+
+ }
82+
+
83+
+ var state votingConfigState
84+
+ if err := json.NewDecoder(res.Body).Decode(&state); err != nil {
85+
+ return false, err
86+
+ }
87+
+
88+
+ for _, id := range state.Metadata.ClusterCoordination.LastCommittedConfig {
89+
+ if node, ok := state.Nodes[id]; ok && node.Name == nodeName {
90+
+ return true, nil
91+
+ }
92+
+ }
93+
+ return false, nil
94+
+}
95+
+
96+
+func generateAPIPathVotingConfigExclusions(nodeName string) strings.Builder {
97+
+ var path strings.Builder
98+
+ path.Grow(len("/_cluster/voting_config_exclusions?node_names=") + len(nodeName))
99+
+ path.WriteString("/_cluster/voting_config_exclusions?node_names=")
100+
+ path.WriteString(nodeName)
101+
+ return path
102+
+}
103+
diff --git a/opensearch-operator/pkg/reconcilers/cluster.go b/opensearch-operator/pkg/reconcilers/cluster.go
104+
index 710f585..9750774 100644
105+
--- a/opensearch-operator/pkg/reconcilers/cluster.go
106+
+++ b/opensearch-operator/pkg/reconcilers/cluster.go
107+
@@ -4,6 +4,7 @@ import (
108+
"context"
109+
"fmt"
110+
"strings"
111+
+ "time"
112+
113+
"github.com/Opster/opensearch-k8s-operator/opensearch-operator/pkg/reconcilers/k8s"
114+
"github.com/Opster/opensearch-k8s-operator/opensearch-operator/pkg/reconcilers/util"
115+
@@ -17,6 +18,8 @@ import (
116+
"github.com/go-logr/logr"
117+
"github.com/samber/lo"
118+
appsv1 "k8s.io/api/apps/v1"
119+
+ corev1 "k8s.io/api/core/v1"
120+
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
121+
"k8s.io/apimachinery/pkg/api/resource"
122+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
123+
"k8s.io/client-go/tools/record"
124+
@@ -25,6 +28,12 @@ import (
125+
"sigs.k8s.io/controller-runtime/pkg/log"
126+
)
127+
128+
+// How long to wait before trying again when the cluster cannot be reached or
129+
+// has not yet committed the exclusion. Short enough that a healthy cluster
130+
+// sheds its bootstrap pod promptly, long enough not to spin on an API that is
131+
+// still coming up.
132+
+const bootstrapRetireRetry = 10 * time.Second
133+
+
134+
type ClusterReconciler struct {
135+
client k8s.K8sClient
136+
ctx context.Context
137+
@@ -98,7 +107,7 @@ func (r *ClusterReconciler) Reconcile() (ctrl.Result, error) {
138+
bootstrapPod := builders.NewBootstrapPod(r.instance, r.reconcilerContext.Volumes, r.reconcilerContext.VolumeMounts)
139+
result.CombineErr(ctrl.SetControllerReference(r.instance, bootstrapPod, r.client.Scheme()))
140+
if r.instance.Status.Initialized {
141+
- result.Combine(r.client.ReconcileResource(bootstrapPod, reconciler.StateAbsent))
142+
+ result.Combine(r.retireBootstrapPod(bootstrapPod))
143+
} else {
144+
result.Combine(r.client.ReconcileResource(bootstrapPod, reconciler.StatePresent))
145+
}
146+
@@ -130,6 +139,76 @@ func (r *ClusterReconciler) Reconcile() (ctrl.Result, error) {
147+
return result.Result, result.Err
148+
}
149+
150+
+// retireBootstrapPod takes the bootstrap node out of the voting configuration
151+
+// before deleting its pod.
152+
+//
153+
+// The bootstrap node is the one that formed the cluster, so it is the voter the
154+
+// cluster keeps while the voting configuration holds a single node, which is
155+
+// what it holds whenever exactly two nodes are cluster_manager-eligible: the
156+
+// bootstrap pod and one node pool replica. Deleting the pod then takes away the
157+
+// only vote there was, and the survivor can never hold an election again,
158+
+// because changing the voting configuration itself needs a quorum. Excluding
159+
+// the node first moves the configuration onto the surviving node, which is what
160+
+// makes the deletion safe.
161+
+//
162+
+// Every failure to reach the cluster leaves the pod alone and requeues. A
163+
+// bootstrap pod that outlives its purpose costs a pod; a cluster that cannot
164+
+// elect a cluster-manager does not recover without a human.
165+
+func (r *ClusterReconciler) retireBootstrapPod(bootstrapPod *corev1.Pod) (*ctrl.Result, error) {
166+
+ if _, err := r.client.GetPod(bootstrapPod.Name, bootstrapPod.Namespace); err != nil {
167+
+ if k8serrors.IsNotFound(err) {
168+
+ return &ctrl.Result{}, nil
169+
+ }
170+
+ return &ctrl.Result{}, err
171+
+ }
172+
+
173+
+ annotations := map[string]string{"cluster-name": r.instance.GetName()}
174+
+ nodeName := builders.BootstrapPodName(r.instance)
175+
+
176+
+ clusterClient, err := util.CreateClientForCluster(r.client, r.ctx, r.instance, nil)
177+
+ if err != nil {
178+
+ r.logger.Error(err, "Failed to create os client to retire the bootstrap node", "node", nodeName)
179+
+ r.recorder.AnnotatedEventf(r.instance, annotations, "Warning", "Bootstrap",
180+
+ "Cannot reach the cluster to exclude %s from voting; keeping the bootstrap pod", nodeName)
181+
+ return &ctrl.Result{RequeueAfter: bootstrapRetireRetry}, nil
182+
+ }
183+
+
184+
+ if err := clusterClient.ExcludeNodeFromVotingConfig(r.ctx, nodeName); err != nil {
185+
+ r.logger.Error(err, "Failed to exclude the bootstrap node from the voting configuration", "node", nodeName)
186+
+ r.recorder.AnnotatedEventf(r.instance, annotations, "Warning", "Bootstrap",
187+
+ "Could not exclude %s from voting; keeping the bootstrap pod", nodeName)
188+
+ return &ctrl.Result{RequeueAfter: bootstrapRetireRetry}, nil
189+
+ }
190+
+
191+
+ // The exclusion travels through the cluster state, so it is committed a
192+
+ // moment after the request returns rather than by the time it does.
193+
+ stillVoting, err := clusterClient.IsInVotingConfig(r.ctx, nodeName)
194+
+ if err != nil {
195+
+ r.logger.Error(err, "Failed to read the voting configuration", "node", nodeName)
196+
+ return &ctrl.Result{RequeueAfter: bootstrapRetireRetry}, nil
197+
+ }
198+
+ if stillVoting {
199+
+ r.logger.Info("Waiting for the bootstrap node to leave the voting configuration", "node", nodeName)
200+
+ return &ctrl.Result{RequeueAfter: bootstrapRetireRetry}, nil
201+
+ }
202+
+
203+
+ result, err := r.client.ReconcileResource(bootstrapPod, reconciler.StateAbsent)
204+
+ if err != nil {
205+
+ return result, err
206+
+ }
207+
+
208+
+ // The pod is gone, so a leftover entry cannot cost the cluster a vote. It
209+
+ // would only fill the exclusion list, which is capped, so report it and
210+
+ // carry on rather than failing the reconcile over it.
211+
+ if err := clusterClient.ClearVotingConfigExclusions(r.ctx); err != nil {
212+
+ r.logger.Error(err, "Failed to clear the voting configuration exclusions", "node", nodeName)
213+
+ }
214+
+
215+
+ r.recorder.AnnotatedEventf(r.instance, annotations, "Normal", "Bootstrap",
216+
+ "Excluded %s from voting and removed the bootstrap pod", nodeName)
217+
+ return result, nil
218+
+}
219+
+
220+
func (r *ClusterReconciler) reconcileNodeStatefulSet(nodePool opsterv1.NodePool, username string) (*ctrl.Result, error) {
221+
found, nodePoolConfig := r.reconcilerContext.fetchNodePoolHash(nodePool.Component)
222+

0 commit comments

Comments
 (0)