|
| 1 | +diff --git a/opensearch-operator/opensearch-gateway/responses/AllocationExplainResponse.go b/opensearch-operator/opensearch-gateway/responses/AllocationExplainResponse.go |
| 2 | +new file mode 100644 |
| 3 | +--- /dev/null |
| 4 | ++++ b/opensearch-operator/opensearch-gateway/responses/AllocationExplainResponse.go |
| 5 | +@@ -0,0 +1,13 @@ |
| 6 | ++package responses |
| 7 | ++ |
| 8 | ++// AllocationExplainResponse carries the part of _cluster/allocation/explain this |
| 9 | ++// operator reads. The endpoint returns a great deal more; everything else is |
| 10 | ++// left out on purpose, so a field the operator never consults cannot break |
| 11 | ++// decoding when OpenSearch renames it. |
| 12 | ++type AllocationExplainResponse struct { |
| 13 | ++ Index string `json:"index"` |
| 14 | ++ Shard int `json:"shard"` |
| 15 | ++ // CanMoveToOtherNode is "yes", "no" or "throttled", and is only present for a |
| 16 | ++ // shard that is currently assigned. |
| 17 | ++ CanMoveToOtherNode string `json:"can_move_to_other_node,omitempty"` |
| 18 | ++} |
| 19 | +diff --git a/opensearch-operator/opensearch-gateway/services/drain_progress_test.go b/opensearch-operator/opensearch-gateway/services/drain_progress_test.go |
| 20 | +new file mode 100644 |
| 21 | +--- /dev/null |
| 22 | ++++ b/opensearch-operator/opensearch-gateway/services/drain_progress_test.go |
| 23 | +@@ -0,0 +1,165 @@ |
| 24 | ++package services |
| 25 | ++ |
| 26 | ++import ( |
| 27 | ++ "encoding/json" |
| 28 | ++ "io" |
| 29 | ++ "net/http" |
| 30 | ++ "strings" |
| 31 | ++ "testing" |
| 32 | ++ |
| 33 | ++ "github.com/go-logr/logr" |
| 34 | ++ "github.com/jarcoal/httpmock" |
| 35 | ++) |
| 36 | ++ |
| 37 | ++// drainMockClient serves the endpoints a drain decision reads: health, cluster |
| 38 | ++// settings (with the exclude list), the shard table, and one allocation |
| 39 | ++// explanation per shard, keyed by "<index>/<shard>". |
| 40 | ++func drainMockClient(t *testing.T, health, excluded, catShards string, explains map[string]string) (*OsClusterClient, *httpmock.MockTransport, *[]string) { |
| 41 | ++ t.Helper() |
| 42 | ++ tr := httpmock.NewMockTransport() |
| 43 | ++ for _, m := range []string{http.MethodGet, http.MethodHead} { |
| 44 | ++ tr.RegisterResponder(m, `=~^http://os.test:9200/?$`, httpmock.NewStringResponder(200, `{"version":{"number":"2.0.0"}}`)) |
| 45 | ++ } |
| 46 | ++ tr.RegisterResponder(http.MethodGet, `=~/_cluster/health`, httpmock.NewStringResponder(200, health)) |
| 47 | ++ tr.RegisterResponder(http.MethodGet, `=~/_cluster/settings`, httpmock.NewStringResponder(200, |
| 48 | ++ `{"transient":{"cluster":{"routing":{"allocation":{"enable":"all","exclude":{"_name":"`+excluded+`"}}}}},"persistent":{}}`)) |
| 49 | ++ puts := &[]string{} |
| 50 | ++ tr.RegisterResponder(http.MethodPut, `=~/_cluster/settings`, func(r *http.Request) (*http.Response, error) { |
| 51 | ++ b, _ := io.ReadAll(r.Body) |
| 52 | ++ *puts = append(*puts, string(b)) |
| 53 | ++ return httpmock.NewStringResponse(200, `{}`), nil |
| 54 | ++ }) |
| 55 | ++ tr.RegisterResponder(http.MethodGet, `=~/_cat/shards`, httpmock.NewStringResponder(200, catShards)) |
| 56 | ++ tr.RegisterResponder(http.MethodGet, `=~/_cat/indices`, httpmock.NewStringResponder(200, `[]`)) |
| 57 | ++ tr.RegisterResponder(http.MethodPost, `=~/_cluster/allocation/explain`, func(r *http.Request) (*http.Response, error) { |
| 58 | ++ var req struct { |
| 59 | ++ Index string `json:"index"` |
| 60 | ++ Shard int `json:"shard"` |
| 61 | ++ } |
| 62 | ++ b, _ := io.ReadAll(r.Body) |
| 63 | ++ if err := json.Unmarshal(b, &req); err != nil { |
| 64 | ++ return httpmock.NewStringResponse(400, `{}`), nil |
| 65 | ++ } |
| 66 | ++ body, ok := explains[req.Index] |
| 67 | ++ if !ok { |
| 68 | ++ return httpmock.NewStringResponse(404, `{}`), nil |
| 69 | ++ } |
| 70 | ++ return httpmock.NewStringResponse(200, body), nil |
| 71 | ++ }) |
| 72 | ++ c, err := NewOsClusterClient("http://os.test:9200", "u", "p", WithTransport(tr)) |
| 73 | ++ if err != nil { |
| 74 | ++ t.Fatal(err) |
| 75 | ++ } |
| 76 | ++ return c, tr, puts |
| 77 | ++} |
| 78 | ++ |
| 79 | ++func explainBody(canMove string) string { |
| 80 | ++ return `{"index":"a","shard":0,"primary":true,"current_state":"started","can_remain_on_current_node":"no","can_move_to_other_node":"` + canMove + `","node_allocation_decisions":[]}` |
| 81 | ++} |
| 82 | ++ |
| 83 | ++func TestDrainCanProceed(t *testing.T) { |
| 84 | ++ onNode := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"}]` |
| 85 | ++ tests := []struct { |
| 86 | ++ name string |
| 87 | ++ shards string |
| 88 | ++ explains map[string]string |
| 89 | ++ want bool |
| 90 | ++ }{ |
| 91 | ++ {"a shard that can move keeps the drain alive", onNode, map[string]string{"a": explainBody("yes")}, true}, |
| 92 | ++ {"a shard with nowhere to go stops it", onNode, map[string]string{"a": explainBody("no")}, false}, |
| 93 | ++ { |
| 94 | ++ "one immovable shard is enough to stop it", |
| 95 | ++ `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"b","shard":"0","prirep":"p","state":"STARTED","node":"n0"}]`, |
| 96 | ++ map[string]string{"a": explainBody("yes"), "b": explainBody("no")}, |
| 97 | ++ false, |
| 98 | ++ }, |
| 99 | ++ {"an empty node has nothing to block it", `[]`, nil, true}, |
| 100 | ++ {"shards on other nodes are not this node's problem", |
| 101 | ++ `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n1"}]`, |
| 102 | ++ map[string]string{"a": explainBody("no")}, true}, |
| 103 | ++ {"a relocating shard is already moving", |
| 104 | ++ `[{"index":"a","shard":"0","prirep":"p","state":"RELOCATING","node":"n0 -> 10.0.0.2 id n2"}]`, |
| 105 | ++ map[string]string{"a": explainBody("no")}, true}, |
| 106 | ++ } |
| 107 | ++ for _, tt := range tests { |
| 108 | ++ t.Run(tt.name, func(t *testing.T) { |
| 109 | ++ c, _, _ := drainMockClient(t, `{"status":"yellow"}`, "", tt.shards, tt.explains) |
| 110 | ++ got, _, err := drainCanProceed(c, "n0") |
| 111 | ++ if err != nil { |
| 112 | ++ t.Fatal(err) |
| 113 | ++ } |
| 114 | ++ if got != tt.want { |
| 115 | ++ t.Errorf("got %v, want %v", got, tt.want) |
| 116 | ++ } |
| 117 | ++ }) |
| 118 | ++ } |
| 119 | ++} |
| 120 | ++ |
| 121 | ++// The node holds a shard nothing else can take, so the drain will not finish. |
| 122 | ++// The exclusion has to be released rather than held for as long as the restart |
| 123 | ++// is stuck, and it must not be re-applied on the next pass either. |
| 124 | ++func TestPreparePodForDeleteReleasesAnUnproductiveExclusion(t *testing.T) { |
| 125 | ++ stuck := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"a","shard":"0","prirep":"r","state":"UNASSIGNED","node":null}]` |
| 126 | ++ quietYellow := `{"status":"yellow","number_of_data_nodes":3,"unassigned_shards":1}` |
| 127 | ++ |
| 128 | ++ t.Run("an exclusion already in place is withdrawn", func(t *testing.T) { |
| 129 | ++ c, _, puts := drainMockClient(t, quietYellow, "n0", stuck, map[string]string{"a": explainBody("no")}) |
| 130 | ++ ready, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3) |
| 131 | ++ if err != nil { |
| 132 | ++ t.Fatal(err) |
| 133 | ++ } |
| 134 | ++ if ready { |
| 135 | ++ t.Fatal("the pod is not safe to delete, it still holds a shard") |
| 136 | ++ } |
| 137 | ++ if len(*puts) != 1 { |
| 138 | ++ t.Fatalf("expected exactly one settings write, got %d: %v", len(*puts), *puts) |
| 139 | ++ } |
| 140 | ++ if strings.Contains((*puts)[0], `"n0"`) { |
| 141 | ++ t.Errorf("the write should clear the exclusion, got %s", (*puts)[0]) |
| 142 | ++ } |
| 143 | ++ }) |
| 144 | ++ |
| 145 | ++ t.Run("no exclusion is applied in the first place", func(t *testing.T) { |
| 146 | ++ c, _, puts := drainMockClient(t, quietYellow, "", stuck, map[string]string{"a": explainBody("no")}) |
| 147 | ++ ready, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3) |
| 148 | ++ if err != nil { |
| 149 | ++ t.Fatal(err) |
| 150 | ++ } |
| 151 | ++ if ready { |
| 152 | ++ t.Fatal("the pod is not safe to delete, it still holds a shard") |
| 153 | ++ } |
| 154 | ++ if len(*puts) != 0 { |
| 155 | ++ t.Errorf("nothing should be written when the node was never excluded, got %v", *puts) |
| 156 | ++ } |
| 157 | ++ }) |
| 158 | ++} |
| 159 | ++ |
| 160 | ++// A drain that can still make progress keeps the behaviour it had: exclude the |
| 161 | ++// node and wait for it to empty. |
| 162 | ++func TestPreparePodForDeleteStillDrainsWhenItCan(t *testing.T) { |
| 163 | ++ movable := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"a","shard":"0","prirep":"r","state":"STARTED","node":"n1"}]` |
| 164 | ++ c, _, puts := drainMockClient(t, `{"status":"green","number_of_data_nodes":3}`, "", movable, map[string]string{"a": explainBody("yes")}) |
| 165 | ++ ready, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3) |
| 166 | ++ if err != nil { |
| 167 | ++ t.Fatal(err) |
| 168 | ++ } |
| 169 | ++ if ready { |
| 170 | ++ t.Fatal("the node is not empty yet") |
| 171 | ++ } |
| 172 | ++ if len(*puts) != 1 || !strings.Contains((*puts)[0], `n0`) { |
| 173 | ++ t.Fatalf("expected the node to be excluded, writes were %v", *puts) |
| 174 | ++ } |
| 175 | ++} |
| 176 | ++ |
| 177 | ++// While shards are actually moving there is nothing to decide, and the |
| 178 | ++// allocation explanation must not be asked for at all. |
| 179 | ++func TestPreparePodForDeleteDoesNotExplainWhileShardsMove(t *testing.T) { |
| 180 | ++ moving := `[{"index":"a","shard":"0","prirep":"p","state":"STARTED","node":"n0"},{"index":"a","shard":"0","prirep":"r","state":"UNASSIGNED","node":null}]` |
| 181 | ++ c, tr, _ := drainMockClient(t, `{"status":"yellow","number_of_data_nodes":3,"relocating_shards":1}`, "n0", moving, map[string]string{"a": explainBody("no")}) |
| 182 | ++ if _, err := PreparePodForDelete(c, logr.Discard(), "n0", true, 3); err != nil { |
| 183 | ++ t.Fatal(err) |
| 184 | ++ } |
| 185 | ++ if n := tr.GetCallCountInfo()["POST =~/_cluster/allocation/explain"]; n != 0 { |
| 186 | ++ t.Errorf("explain must not be called while the allocator is busy, got %d calls", n) |
| 187 | ++ } |
| 188 | ++} |
| 189 | +diff --git a/opensearch-operator/opensearch-gateway/services/os_client.go b/opensearch-operator/opensearch-gateway/services/os_client.go |
| 190 | +--- a/opensearch-operator/opensearch-gateway/services/os_client.go |
| 191 | ++++ b/opensearch-operator/opensearch-gateway/services/os_client.go |
| 192 | +@@ -171,6 +171,26 @@ func (client *OsClusterClient) CatIndices() ([]responses.CatIndicesResponse, err |
| 193 | + return response, err |
| 194 | + } |
| 195 | + |
| 196 | ++func (client *OsClusterClient) GetAllocationExplain(index string, shard int, primary bool) (responses.AllocationExplainResponse, error) { |
| 197 | ++ body := map[string]interface{}{ |
| 198 | ++ "index": index, |
| 199 | ++ "shard": shard, |
| 200 | ++ "primary": primary, |
| 201 | ++ } |
| 202 | ++ req := opensearchapi.ClusterAllocationExplainRequest{Body: opensearchutil.NewJSONReader(body)} |
| 203 | ++ explainRes, err := req.Do(context.Background(), client.client) |
| 204 | ++ var response responses.AllocationExplainResponse |
| 205 | ++ if err != nil { |
| 206 | ++ return response, err |
| 207 | ++ } |
| 208 | ++ defer helpers.SafeClose(explainRes.Body) |
| 209 | ++ if explainRes.IsError() { |
| 210 | ++ return response, fmt.Errorf("failed to get allocation explanation: %s", explainRes.String()) |
| 211 | ++ } |
| 212 | ++ err = json.NewDecoder(explainRes.Body).Decode(&response) |
| 213 | ++ return response, err |
| 214 | ++} |
| 215 | ++ |
| 216 | + func (client *OsClusterClient) CatShards(headers []string) ([]responses.CatShardsResponse, error) { |
| 217 | + req := opensearchapi.CatShardsRequest{Format: "json", H: headers} |
| 218 | + indicesRes, err := req.Do(context.Background(), client.client) |
| 219 | +diff --git a/opensearch-operator/opensearch-gateway/services/os_data_service.go b/opensearch-operator/opensearch-gateway/services/os_data_service.go |
| 220 | +--- a/opensearch-operator/opensearch-gateway/services/os_data_service.go |
| 221 | ++++ b/opensearch-operator/opensearch-gateway/services/os_data_service.go |
| 222 | +@@ -4,6 +4,7 @@ import ( |
| 223 | + "context" |
| 224 | + "encoding/json" |
| 225 | + "fmt" |
| 226 | ++ "strconv" |
| 227 | + "strings" |
| 228 | + |
| 229 | + "github.com/google/go-cmp/cmp" |
| 230 | +@@ -249,8 +250,70 @@ func ReactivateShardAllocation(service *OsClusterClient) error { |
| 231 | + return nil |
| 232 | + } |
| 233 | + |
| 234 | ++// drainCanProceed reports whether the allocator still has somewhere to put what |
| 235 | ++// is left on nodeName. A drain waits for the node to empty, so one shard that |
| 236 | ++// can be placed nowhere else is what turns that wait into a permanent one; the |
| 237 | ++// reason for the first such shard is returned for the caller's log line. |
| 238 | ++// |
| 239 | ++// Only STARTED copies are asked about: a relocating one is already on its way |
| 240 | ++// out, and an unassigned one is not on this node to begin with. |
| 241 | ++func drainCanProceed(service *OsClusterClient, nodeName string) (bool, string, error) { |
| 242 | ++ shards, err := service.CatShards([]string{"index", "shard", "prirep", "state", "node"}) |
| 243 | ++ if err != nil { |
| 244 | ++ return false, "", err |
| 245 | ++ } |
| 246 | ++ for _, shard := range shardsOnNodeFromResponse(shards, nodeName) { |
| 247 | ++ if shard.State != "STARTED" { |
| 248 | ++ continue |
| 249 | ++ } |
| 250 | ++ shardNum, err := strconv.Atoi(shard.Shard) |
| 251 | ++ if err != nil { |
| 252 | ++ continue |
| 253 | ++ } |
| 254 | ++ explain, err := service.GetAllocationExplain(shard.Index, shardNum, shard.PrimaryOrReplica == "p") |
| 255 | ++ if err != nil { |
| 256 | ++ return false, "", err |
| 257 | ++ } |
| 258 | ++ if strings.EqualFold(explain.CanMoveToOtherNode, "no") { |
| 259 | ++ return false, fmt.Sprintf("%s[%s] has no other node to move to", shard.Index, shard.Shard), nil |
| 260 | ++ } |
| 261 | ++ } |
| 262 | ++ return true, "", nil |
| 263 | ++} |
| 264 | ++ |
| 265 | + func PreparePodForDelete(service *OsClusterClient, lg logr.Logger, podName string, drainNode bool, nodeCount int32) (bool, error) { |
| 266 | + if drainNode { |
| 267 | ++ // Excluding a node is only worth anything while the allocator can move its |
| 268 | ++ // shards elsewhere. When it cannot, the node never empties, the restart |
| 269 | ++ // waits for something that will not happen, and the exclusion keeps a data |
| 270 | ++ // node out of allocation for the whole of it. Two data nodes are exempt: |
| 271 | ++ // that path never waits for a full drain, only for system-index primaries. |
| 272 | ++ // |
| 273 | ++ // Asked only while nothing is moving, so a drain in progress pays nothing: |
| 274 | ++ // the cost is one allocation explanation per started shard on the node, and |
| 275 | ++ // it stops at the first shard that cannot move. |
| 276 | ++ if nodeCount != 2 { |
| 277 | ++ health, err := service.GetHealth() |
| 278 | ++ if err != nil { |
| 279 | ++ return false, err |
| 280 | ++ } |
| 281 | ++ if !hasShardActivity(health) { |
| 282 | ++ canDrain, reason, err := drainCanProceed(service, podName) |
| 283 | ++ if err != nil { |
| 284 | ++ return false, err |
| 285 | ++ } |
| 286 | ++ if !canDrain { |
| 287 | ++ // Withdraw an exclusion left by an earlier pass. With an empty |
| 288 | ++ // exclude list this writes nothing. |
| 289 | ++ if _, err := RemoveExcludeNodeHost(service, podName); err != nil { |
| 290 | ++ return false, err |
| 291 | ++ } |
| 292 | ++ lg.Info(fmt.Sprintf("Not draining %s: %s. Leaving allocation as it is until that changes", podName, reason)) |
| 293 | ++ return false, nil |
| 294 | ++ } |
| 295 | ++ } |
| 296 | ++ } |
| 297 | ++ |
| 298 | + // If we are draining nodes then drain the working node |
| 299 | + _, err := AppendExcludeNodeHost(service, podName) |
| 300 | + if err != nil { |
0 commit comments