Skip to content

Commit 28195d3

Browse files
committed
fix(opensearch-operator): stop excluding a node whose drain cannot finish
The yellow restart gate carried in the previous patch lets a permanently yellow cluster through, and with three or more nodes packages/apps/opensearch turns drainDataNodes on. The restart then reaches PreparePodForDelete, which excludes the candidate from allocation before it can know whether the drain will finish. On the cluster that made the gate wait it never finishes: a shard on that node has nowhere else to go, which is the same condition holding the cluster yellow. The exclusion is withdrawn only after a pod is really deleted, so a data node stays out of allocation for as long as the restart is stuck, shedding its shards onto the rest of the cluster and buying nothing. Ask the allocator before excluding anything, and only while nothing is moving, so a drain that is making progress costs nothing. A started shard answering can_move_to_other_node: no means the drain cannot finish, and then no exclusion is written and one left by an earlier pass is withdrawn. The two-data-node path keeps its behaviour, because it never waits for a full drain. Checking after the fact instead would flap: the next reconcile, ten seconds later, would exclude the node again. Upstream as opensearch-project/opensearch-k8s-operator#1585, fixed there by #1586; the patch goes away with the first release carrying it. Assisted-by: LLM Signed-off-by: IvanHunters <[email protected]>
1 parent 301a7fe commit 28195d3

2 files changed

Lines changed: 315 additions & 0 deletions

File tree

‎packages/system/opensearch-operator/images/opensearch-operator/Dockerfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,21 @@
1313
# data nodes is left out, because packages/apps/opensearch renders the plain
1414
# data role only; carry it as well if that ever changes. Drop the patch on the
1515
# first operator release that includes the change.
16+
#
17+
# patches/skip-unusable-drain-exclusion.diff is the other half of that change,
18+
# for a release with three or more nodes, where packages/apps/opensearch turns
19+
# drainDataNodes on. The gate above now lets a permanently yellow cluster reach
20+
# PreparePodForDelete, which excludes the candidate from allocation before it can
21+
# know whether the drain will finish. On the cluster that made the gate wait, it
22+
# never does, and the exclusion is only withdrawn after a pod is really deleted,
23+
# so a data node stays out of allocation for as long as the restart is stuck. The
24+
# patch asks the allocator first, and only while nothing is moving. Tracked
25+
# upstream as opensearch-project/opensearch-k8s-operator#1585, fixed by #1586;
26+
# drop it on the first release carrying that.
27+
#
28+
# The patches are applied in the order the glob expands, which is alphabetical,
29+
# and skip-unusable-drain-exclusion builds on restart-on-permanently-yellow. A
30+
# rename that reorders them fails the build rather than producing a wrong binary.
1631
FROM golang:1.26@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a AS builder
1732

1833
# Keep in step with the chart version pulled by `make update`: the chart's
Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
diff --git a/opensearch-operator/opensearch-gateway/responses/AllocationExplainResponse.go b/opensearch-operator/opensearch-gateway/responses/AllocationExplainResponse.go
2+
new file mode 100644
3+
--- /dev/null
4+
+++ b/opensearch-operator/opensearch-gateway/responses/AllocationExplainResponse.go
5+
@@ -0,0 +1,13 @@
6+
+package responses
7+
+
8+
+// AllocationExplainResponse carries the part of _cluster/allocation/explain this
9+
+// operator reads. The endpoint returns a great deal more; everything else is
10+
+// left out on purpose, so a field the operator never consults cannot break
11+
+// decoding when OpenSearch renames it.
12+
+type AllocationExplainResponse struct {
13+
+ Index string `json:"index"`
14+
+ Shard int `json:"shard"`
15+
+ // CanMoveToOtherNode is "yes", "no" or "throttled", and is only present for a
16+
+ // shard that is currently assigned.
17+
+ CanMoveToOtherNode string `json:"can_move_to_other_node,omitempty"`
18+
+}
19+
diff --git a/opensearch-operator/opensearch-gateway/services/drain_progress_test.go b/opensearch-operator/opensearch-gateway/services/drain_progress_test.go
20+
new file mode 100644
21+
--- /dev/null
22+
+++ b/opensearch-operator/opensearch-gateway/services/drain_progress_test.go
23+
@@ -0,0 +1,165 @@
24+
+package services
25+
+
26+
+import (
27+
+ "encoding/json"
28+
+ "io"
29+
+ "net/http"
30+
+ "strings"
31+
+ "testing"
32+
+
33+
+ "github.com/go-logr/logr"
34+
+ "github.com/jarcoal/httpmock"
35+
+)
36+
+
37+
+// drainMockClient serves the endpoints a drain decision reads: health, cluster
38+
+// settings (with the exclude list), the shard table, and one allocation
39+
+// explanation per shard, keyed by "<index>/<shard>".
40+
+func drainMockClient(t *testing.T, health, excluded, catShards string, explains map[string]string) (*OsClusterClient, *httpmock.MockTransport, *[]string) {
41+
+ t.Helper()
42+
+ tr := httpmock.NewMockTransport()
43+
+ for _, m := range []string{http.MethodGet, http.MethodHead} {
44+
+ tr.RegisterResponder(m, `=~^http://os.test:9200/?$`, httpmock.NewStringResponder(200, `{"version":{"number":"2.0.0"}}`))
45+
+ }
46+
+ tr.RegisterResponder(http.MethodGet, `=~/_cluster/health`, httpmock.NewStringResponder(200, health))
47+
+ tr.RegisterResponder(http.MethodGet, `=~/_cluster/settings`, httpmock.NewStringResponder(200,
48+
+ `{"transient":{"cluster":{"routing":{"allocation":{"enable":"all","exclude":{"_name":"`+excluded+`"}}}}},"persistent":{}}`))
49+
+ puts := &[]string{}
50+
+ tr.RegisterResponder(http.MethodPut, `=~/_cluster/settings`, func(r *http.Request) (*http.Response, error) {
51+
+ b, _ := io.ReadAll(r.Body)
52+
+ *puts = append(*puts, string(b))
53+
+ return httpmock.NewStringResponse(200, `{}`), nil
54+
+ })
55+
+ tr.RegisterResponder(http.MethodGet, `=~/_cat/shards`, httpmock.NewStringResponder(200, catShards))
56+
+ tr.RegisterResponder(http.MethodGet, `=~/_cat/indices`, httpmock.NewStringResponder(200, `[]`))
57+
+ tr.RegisterResponder(http.MethodPost, `=~/_cluster/allocation/explain`, func(r *http.Request) (*http.Response, error) {
58+
+ var req struct {
59+
+ Index string `json:"index"`
60+
+ Shard int `json:"shard"`
61+
+ }
62+
+ b, _ := io.ReadAll(r.Body)
63+
+ if err := json.Unmarshal(b, &req); err != nil {
64+
+ return httpmock.NewStringResponse(400, `{}`), nil
65+
+ }
66+
+ body, ok := explains[req.Index]
67+
+ if !ok {
68+
+ return httpmock.NewStringResponse(404, `{}`), nil
69+
+ }
70+
+ return httpmock.NewStringResponse(200, body), nil
71+
+ })
72+
+ c, err := NewOsClusterClient("http://os.test:9200", "u", "p", WithTransport(tr))
73+
+ if err != nil {
74+
+ t.Fatal(err)
75+
+ }
76+
+ return c, tr, puts
77+
+}
78+
+
79+
+func explainBody(canMove string) string {
80+
+ return `{"index":"a","shard":0,"primary":true,"current_state":"started","can_remain_on_current_node":"no","can_move_to_other_node":"` + canMove + `","node_allocation_decisions":[]}`
81+
+}
82+
+
83+
+func TestDrainCanProceed(t *testing.T) {
84+
+ onNode := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"}]`
85+
+ tests := []struct {
86+
+ name string
87+
+ shards string
88+
+ explains map[string]string
89+
+ want bool
90+
+ }{
91+
+ {"a shard that can move keeps the drain alive", onNode, map[string]string{"a": explainBody("yes")}, true},
92+
+ {"a shard with nowhere to go stops it", onNode, map[string]string{"a": explainBody("no")}, false},
93+
+ {
94+
+ "one immovable shard is enough to stop it",
95+
+ `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"b","shard":"0","prirep":"p","state":"STARTED","node":"n0"}]`,
96+
+ map[string]string{"a": explainBody("yes"), "b": explainBody("no")},
97+
+ false,
98+
+ },
99+
+ {"an empty node has nothing to block it", `[]`, nil, true},
100+
+ {"shards on other nodes are not this node's problem",
101+
+ `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n1"}]`,
102+
+ map[string]string{"a": explainBody("no")}, true},
103+
+ {"a relocating shard is already moving",
104+
+ `[{"index":"a","shard":"0","prirep":"p","state":"RELOCATING","node":"n0 -> 10.0.0.2 id n2"}]`,
105+
+ map[string]string{"a": explainBody("no")}, true},
106+
+ }
107+
+ for _, tt := range tests {
108+
+ t.Run(tt.name, func(t *testing.T) {
109+
+ c, _, _ := drainMockClient(t, `{"status":"yellow"}`, "", tt.shards, tt.explains)
110+
+ got, _, err := drainCanProceed(c, "n0")
111+
+ if err != nil {
112+
+ t.Fatal(err)
113+
+ }
114+
+ if got != tt.want {
115+
+ t.Errorf("got %v, want %v", got, tt.want)
116+
+ }
117+
+ })
118+
+ }
119+
+}
120+
+
121+
+// The node holds a shard nothing else can take, so the drain will not finish.
122+
+// The exclusion has to be released rather than held for as long as the restart
123+
+// is stuck, and it must not be re-applied on the next pass either.
124+
+func TestPreparePodForDeleteReleasesAnUnproductiveExclusion(t *testing.T) {
125+
+ stuck := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"a","shard":"0","prirep":"r","state":"UNASSIGNED","node":null}]`
126+
+ quietYellow := `{"status":"yellow","number_of_data_nodes":3,"unassigned_shards":1}`
127+
+
128+
+ t.Run("an exclusion already in place is withdrawn", func(t *testing.T) {
129+
+ c, _, puts := drainMockClient(t, quietYellow, "n0", stuck, map[string]string{"a": explainBody("no")})
130+
+ ready, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3)
131+
+ if err != nil {
132+
+ t.Fatal(err)
133+
+ }
134+
+ if ready {
135+
+ t.Fatal("the pod is not safe to delete, it still holds a shard")
136+
+ }
137+
+ if len(*puts) != 1 {
138+
+ t.Fatalf("expected exactly one settings write, got %d: %v", len(*puts), *puts)
139+
+ }
140+
+ if strings.Contains((*puts)[0], `"n0"`) {
141+
+ t.Errorf("the write should clear the exclusion, got %s", (*puts)[0])
142+
+ }
143+
+ })
144+
+
145+
+ t.Run("no exclusion is applied in the first place", func(t *testing.T) {
146+
+ c, _, puts := drainMockClient(t, quietYellow, "", stuck, map[string]string{"a": explainBody("no")})
147+
+ ready, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3)
148+
+ if err != nil {
149+
+ t.Fatal(err)
150+
+ }
151+
+ if ready {
152+
+ t.Fatal("the pod is not safe to delete, it still holds a shard")
153+
+ }
154+
+ if len(*puts) != 0 {
155+
+ t.Errorf("nothing should be written when the node was never excluded, got %v", *puts)
156+
+ }
157+
+ })
158+
+}
159+
+
160+
+// A drain that can still make progress keeps the behaviour it had: exclude the
161+
+// node and wait for it to empty.
162+
+func TestPreparePodForDeleteStillDrainsWhenItCan(t *testing.T) {
163+
+ movable := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"a","shard":"0","prirep":"r","state":"STARTED","node":"n1"}]`
164+
+ c, _, puts := drainMockClient(t, `{"status":"green","number_of_data_nodes":3}`, "", movable, map[string]string{"a": explainBody("yes")})
165+
+ ready, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3)
166+
+ if err != nil {
167+
+ t.Fatal(err)
168+
+ }
169+
+ if ready {
170+
+ t.Fatal("the node is not empty yet")
171+
+ }
172+
+ if len(*puts) != 1 || !strings.Contains((*puts)[0], `n0`) {
173+
+ t.Fatalf("expected the node to be excluded, writes were %v", *puts)
174+
+ }
175+
+}
176+
+
177+
+// While shards are actually moving there is nothing to decide, and the
178+
+// allocation explanation must not be asked for at all.
179+
+func TestPreparePodForDeleteDoesNotExplainWhileShardsMove(t *testing.T) {
180+
+ moving := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"a","shard":"0","prirep":"r","state":"UNASSIGNED","node":null}]`
181+
+ c, tr, _ := drainMockClient(t, `{"status":"yellow","number_of_data_nodes":3,"relocating_shards":1}`, "n0", moving, map[string]string{"a": explainBody("no")})
182+
+ if _, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3); err != nil {
183+
+ t.Fatal(err)
184+
+ }
185+
+ if n := tr.GetCallCountInfo()["POST =~/_cluster/allocation/explain"]; n != 0 {
186+
+ t.Errorf("explain must not be called while the allocator is busy, got %d calls", n)
187+
+ }
188+
+}
189+
diff --git a/opensearch-operator/opensearch-gateway/services/os_client.go b/opensearch-operator/opensearch-gateway/services/os_client.go
190+
--- a/opensearch-operator/opensearch-gateway/services/os_client.go
191+
+++ b/opensearch-operator/opensearch-gateway/services/os_client.go
192+
@@ -171,6 +171,26 @@ func (client *OsClusterClient) CatIndices() ([]responses.CatIndicesResponse, err
193+
return response, err
194+
}
195+
196+
+func (client *OsClusterClient) GetAllocationExplain(index string, shard int, primary bool) (responses.AllocationExplainResponse, error) {
197+
+ body := map[string]interface{}{
198+
+ "index": index,
199+
+ "shard": shard,
200+
+ "primary": primary,
201+
+ }
202+
+ req := opensearchapi.ClusterAllocationExplainRequest{Body: opensearchutil.NewJSONReader(body)}
203+
+ explainRes, err := req.Do(context.Background(), client.client)
204+
+ var response responses.AllocationExplainResponse
205+
+ if err != nil {
206+
+ return response, err
207+
+ }
208+
+ defer helpers.SafeClose(explainRes.Body)
209+
+ if explainRes.IsError() {
210+
+ return response, fmt.Errorf("failed to get allocation explanation: %s", explainRes.String())
211+
+ }
212+
+ err = json.NewDecoder(explainRes.Body).Decode(&response)
213+
+ return response, err
214+
+}
215+
+
216+
func (client *OsClusterClient) CatShards(headers []string) ([]responses.CatShardsResponse, error) {
217+
req := opensearchapi.CatShardsRequest{Format: "json", H: headers}
218+
indicesRes, err := req.Do(context.Background(), client.client)
219+
diff --git a/opensearch-operator/opensearch-gateway/services/os_data_service.go b/opensearch-operator/opensearch-gateway/services/os_data_service.go
220+
--- a/opensearch-operator/opensearch-gateway/services/os_data_service.go
221+
+++ b/opensearch-operator/opensearch-gateway/services/os_data_service.go
222+
@@ -4,6 +4,7 @@ import (
223+
"context"
224+
"encoding/json"
225+
"fmt"
226+
+ "strconv"
227+
"strings"
228+
229+
"github.com/google/go-cmp/cmp"
230+
@@ -249,8 +250,70 @@ func ReactivateShardAllocation(service *OsClusterClient) error {
231+
return nil
232+
}
233+
234+
+// drainCanProceed reports whether the allocator still has somewhere to put what
235+
+// is left on nodeName. A drain waits for the node to empty, so one shard that
236+
+// can be placed nowhere else is what turns that wait into a permanent one; the
237+
+// reason for the first such shard is returned for the caller's log line.
238+
+//
239+
+// Only STARTED copies are asked about: a relocating one is already on its way
240+
+// out, and an unassigned one is not on this node to begin with.
241+
+func drainCanProceed(service *OsClusterClient, nodeName string) (bool, string, error) {
242+
+ shards, err := service.CatShards([]string{"index", "shard", "prirep", "state", "node"})
243+
+ if err != nil {
244+
+ return false, "", err
245+
+ }
246+
+ for _, shard := range shardsOnNodeFromResponse(shards, nodeName) {
247+
+ if shard.State != "STARTED" {
248+
+ continue
249+
+ }
250+
+ shardNum, err := strconv.Atoi(shard.Shard)
251+
+ if err != nil {
252+
+ continue
253+
+ }
254+
+ explain, err := service.GetAllocationExplain(shard.Index, shardNum, shard.PrimaryOrReplica == "p")
255+
+ if err != nil {
256+
+ return false, "", err
257+
+ }
258+
+ if strings.EqualFold(explain.CanMoveToOtherNode, "no") {
259+
+ return false, fmt.Sprintf("%s[%s] has no other node to move to", shard.Index, shard.Shard), nil
260+
+ }
261+
+ }
262+
+ return true, "", nil
263+
+}
264+
+
265+
func PreparePodForDelete(service *OsClusterClient, lg logr.Logger, podName string, drainNode bool, nodeCount int32) (bool, error) {
266+
if drainNode {
267+
+ // Excluding a node is only worth anything while the allocator can move its
268+
+ // shards elsewhere. When it cannot, the node never empties, the restart
269+
+ // waits for something that will not happen, and the exclusion keeps a data
270+
+ // node out of allocation for the whole of it. Two data nodes are exempt:
271+
+ // that path never waits for a full drain, only for system-index primaries.
272+
+ //
273+
+ // Asked only while nothing is moving, so a drain in progress pays nothing:
274+
+ // the cost is one allocation explanation per started shard on the node, and
275+
+ // it stops at the first shard that cannot move.
276+
+ if nodeCount != 2 {
277+
+ health, err := service.GetHealth()
278+
+ if err != nil {
279+
+ return false, err
280+
+ }
281+
+ if !hasShardActivity(health) {
282+
+ canDrain, reason, err := drainCanProceed(service, podName)
283+
+ if err != nil {
284+
+ return false, err
285+
+ }
286+
+ if !canDrain {
287+
+ // Withdraw an exclusion left by an earlier pass. With an empty
288+
+ // exclude list this writes nothing.
289+
+ if _, err := RemoveExcludeNodeHost(service, podName); err != nil {
290+
+ return false, err
291+
+ }
292+
+ lg.Info(fmt.Sprintf("Not draining %s: %s. Leaving allocation as it is until that changes", podName, reason))
293+
+ return false, nil
294+
+ }
295+
+ }
296+
+ }
297+
+
298+
// If we are draining nodes then drain the working node
299+
_, err := AppendExcludeNodeHost(service, podName)
300+
if err != nil {

0 commit comments

Comments
 (0)