|
| 1 | +package server_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "bytes" |
| 5 | + "context" |
| 6 | + "encoding/json" |
| 7 | + "fmt" |
| 8 | + "io" |
| 9 | + "net/http" |
| 10 | + "slices" |
| 11 | + "sync" |
| 12 | + "testing" |
| 13 | + |
| 14 | + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" |
| 15 | + helmv2 "github.com/fluxcd/helm-controller/api/v2" |
| 16 | + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" |
| 17 | + apierrors "k8s.io/apimachinery/pkg/api/errors" |
| 18 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 19 | + "k8s.io/apiserver/pkg/admission" |
| 20 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 21 | + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" |
| 22 | + |
| 23 | + fluxshard "github.com/cozystack/cozystack/internal/fluxshardoperator" |
| 24 | + "github.com/cozystack/cozystack/pkg/config" |
| 25 | +) |
| 26 | + |
| 27 | +func TestApplicationApplyCreate(t *testing.T) { |
| 28 | + for _, uid := range []string{"", "nonexistent"} { |
| 29 | + for _, dryRun := range []bool{false, true} { |
| 30 | + for _, force := range []bool{false, true} { |
| 31 | + t.Run(fmt.Sprintf("uid=%s/dryRun=%v/force=%v", uid, dryRun, force), func(t *testing.T) { |
| 32 | + ts, backing := startAppsServer(t, &config.ResourceConfig{Resources: []config.Resource{redisConfig}}) |
| 33 | + payload := fmt.Sprintf(`{"apiVersion":"apps.cozystack.io/v1alpha1","kind":"Redis","metadata":{"name":"new-app","namespace":"tenant-a","uid":%q},"spec":{"replicas":1}}`, uid) |
| 34 | + url := ts.URL + fmt.Sprintf("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/redises/new-app?fieldManager=create-test&force=%v", force) |
| 35 | + if dryRun { |
| 36 | + url += "&dryRun=All" |
| 37 | + } |
| 38 | + req, err := http.NewRequest(http.MethodPatch, url, bytes.NewBufferString(payload)) |
| 39 | + if err != nil { |
| 40 | + t.Fatal(err) |
| 41 | + } |
| 42 | + req.Header.Set("Content-Type", "application/apply-patch+yaml") |
| 43 | + resp, err := http.DefaultClient.Do(req) |
| 44 | + if err != nil { |
| 45 | + t.Fatal(err) |
| 46 | + } |
| 47 | + response, err := io.ReadAll(resp.Body) |
| 48 | + if closeErr := resp.Body.Close(); closeErr != nil { |
| 49 | + t.Fatal(closeErr) |
| 50 | + } |
| 51 | + if err != nil { |
| 52 | + t.Fatal(err) |
| 53 | + } |
| 54 | + want := http.StatusCreated |
| 55 | + if uid != "" { |
| 56 | + want = http.StatusConflict |
| 57 | + } |
| 58 | + if resp.StatusCode != want { |
| 59 | + t.Fatalf("HTTP %d, want %d: %s", resp.StatusCode, want, response) |
| 60 | + } |
| 61 | + live := &helmv2.HelmRelease{} |
| 62 | + err = backing.Get(context.Background(), client.ObjectKey{Namespace: "tenant-a", Name: "redis-new-app"}, live) |
| 63 | + if dryRun || uid != "" { |
| 64 | + if !apierrors.IsNotFound(err) { |
| 65 | + t.Fatalf("unexpected persisted object: err=%v live=%#v", err, live) |
| 66 | + } |
| 67 | + return |
| 68 | + } |
| 69 | + if err != nil { |
| 70 | + t.Fatal(err) |
| 71 | + } |
| 72 | + if !bytes.Contains(live.Spec.Values.Raw, []byte(`"replicas":1`)) { |
| 73 | + t.Fatalf("lost apply spec: %s", live.Spec.Values.Raw) |
| 74 | + } |
| 75 | + }) |
| 76 | + } |
| 77 | + } |
| 78 | + } |
| 79 | +} |
| 80 | + |
| 81 | +type updateAdmissionProbe struct { |
| 82 | + *admission.Handler |
| 83 | + mu sync.Mutex |
| 84 | + mutated []string |
| 85 | + validated []string |
| 86 | + rejectFresh bool |
| 87 | + dryRun bool |
| 88 | +} |
| 89 | + |
| 90 | +func (p *updateAdmissionProbe) Admit(_ context.Context, attrs admission.Attributes, _ admission.ObjectInterfaces) error { |
| 91 | + if attrs.IsDryRun() != p.dryRun { |
| 92 | + return fmt.Errorf("mutating admission lost dry-run option") |
| 93 | + } |
| 94 | + old := attrs.GetOldObject().(*appsv1alpha1.Application) |
| 95 | + p.mu.Lock() |
| 96 | + p.mutated = append(p.mutated, old.ResourceVersion) |
| 97 | + p.mu.Unlock() |
| 98 | + app := attrs.GetObject().(*appsv1alpha1.Application) |
| 99 | + if app.Annotations == nil { |
| 100 | + app.Annotations = map[string]string{} |
| 101 | + } |
| 102 | + app.Annotations["admitted-version"] = old.ResourceVersion |
| 103 | + return nil |
| 104 | +} |
| 105 | + |
| 106 | +func (p *updateAdmissionProbe) Validate(_ context.Context, attrs admission.Attributes, _ admission.ObjectInterfaces) error { |
| 107 | + if attrs.IsDryRun() != p.dryRun { |
| 108 | + return fmt.Errorf("validating admission lost dry-run option") |
| 109 | + } |
| 110 | + old := attrs.GetOldObject().(*appsv1alpha1.Application) |
| 111 | + p.mu.Lock() |
| 112 | + p.validated = append(p.validated, old.ResourceVersion) |
| 113 | + reject := p.rejectFresh && len(p.mutated) > 0 && old.ResourceVersion != p.mutated[0] |
| 114 | + p.mu.Unlock() |
| 115 | + if reject { |
| 116 | + return apierrors.NewForbidden(attrs.GetResource().GroupResource(), attrs.GetName(), fmt.Errorf("fresh admission snapshot rejected")) |
| 117 | + } |
| 118 | + return nil |
| 119 | +} |
| 120 | + |
| 121 | +func (p *updateAdmissionProbe) versions() ([]string, []string) { |
| 122 | + p.mu.Lock() |
| 123 | + defer p.mu.Unlock() |
| 124 | + return append([]string(nil), p.mutated...), append([]string(nil), p.validated...) |
| 125 | +} |
| 126 | + |
| 127 | +func TestApplicationUpdateHTTPConcurrency(t *testing.T) { |
| 128 | + for _, method := range []string{"put", "json", "merge", "strategic", "apply"} { |
| 129 | + for _, version := range []string{"omitted", "current", "stale", "zero", "test", "matchingUID", "emptyUID", "wrongUID", "uncontended", "admissionReject"} { |
| 130 | + if version == "test" && method != "json" { |
| 131 | + continue |
| 132 | + } |
| 133 | + for _, dryRun := range []bool{false, true} { |
| 134 | + for _, force := range []bool{false, true} { |
| 135 | + if force && method != "apply" { |
| 136 | + continue |
| 137 | + } |
| 138 | + t.Run(fmt.Sprintf("%s/%s/dryRun=%v/force=%v", method, version, dryRun, force), func(t *testing.T) { |
| 139 | + var mu sync.Mutex |
| 140 | + writes := 0 |
| 141 | + funcs := interceptor.Funcs{Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { |
| 142 | + mu.Lock() |
| 143 | + defer mu.Unlock() |
| 144 | + writes++ |
| 145 | + writeOptions := (&client.UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions() |
| 146 | + if writeOptions.FieldManager != "concurrency-test" || writeOptions.FieldValidation != metav1.FieldValidationStrict || slices.Contains(writeOptions.DryRun, metav1.DryRunAll) != dryRun { |
| 147 | + return fmt.Errorf("write options changed: %#v", writeOptions) |
| 148 | + } |
| 149 | + if writes == 1 && version != "uncontended" { |
| 150 | + live := &helmv2.HelmRelease{} |
| 151 | + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil { |
| 152 | + return err |
| 153 | + } |
| 154 | + live.Spec.Values = &apiextv1.JSON{Raw: []byte(`{"retained":2,"numeric":1e1,"_internal":true}`)} |
| 155 | + live.Spec.Suspend = true |
| 156 | + live.Labels["apps.cozystack.io-concurrent"] = "kept" |
| 157 | + live.Labels["foreign.example/controller"] = "kept" |
| 158 | + live.Labels[fluxshard.ShardKeyLabel] = "concurrent-shard" |
| 159 | + live.Annotations = map[string]string{"foreign.example/controller": "kept", "apps.cozystack.io-concurrent": "kept"} |
| 160 | + live.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "owner", UID: "owner"}} |
| 161 | + live.Finalizers = []string{"finalizers.fluxcd.io", metav1.FinalizerDeleteDependents} |
| 162 | + if err := c.Update(ctx, live); err != nil { |
| 163 | + return err |
| 164 | + } |
| 165 | + if err := c.Delete(ctx, live); err != nil { |
| 166 | + return err |
| 167 | + } |
| 168 | + } |
| 169 | + // The fake client skips resourceVersion validation for dry-run writes. |
| 170 | + live := &helmv2.HelmRelease{} |
| 171 | + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil { |
| 172 | + return err |
| 173 | + } |
| 174 | + if obj.GetResourceVersion() != live.ResourceVersion || obj.GetUID() != live.UID { |
| 175 | + return apierrors.NewConflict(helmv2.GroupVersion.WithResource("helmreleases").GroupResource(), obj.GetName(), fmt.Errorf("object changed")) |
| 176 | + } |
| 177 | + return c.Update(ctx, obj, opts...) |
| 178 | + }} |
| 179 | + probe := &updateAdmissionProbe{Handler: admission.NewHandler(admission.Update), dryRun: dryRun, rejectFresh: version == "admissionReject"} |
| 180 | + ts, backing := startAppsServerWithInterceptors(t, &config.ResourceConfig{Resources: []config.Resource{redisConfig}}, funcs, probe) |
| 181 | + ctx := context.Background() |
| 182 | + hr := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "redis-app", Namespace: "tenant-a", UID: "original", Labels: map[string]string{ |
| 183 | + "apps.cozystack.io/application.kind": "Redis", "apps.cozystack.io/application.group": "apps.cozystack.io", "apps.cozystack.io/application.name": "app", |
| 184 | + }}, Spec: helmv2.HelmReleaseSpec{Values: &apiextv1.JSON{Raw: []byte(`{"retained":1,"numeric":10.0}`)}}} |
| 185 | + if err := backing.Create(ctx, hr); err != nil { |
| 186 | + t.Fatal(err) |
| 187 | + } |
| 188 | + url := ts.URL + "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/redises/app" |
| 189 | + metadata := map[string]any{"name": "app", "namespace": "tenant-a"} |
| 190 | + switch version { |
| 191 | + case "current", "test", "uncontended": |
| 192 | + metadata["resourceVersion"] = hr.ResourceVersion |
| 193 | + case "stale": |
| 194 | + metadata["resourceVersion"] = "999" |
| 195 | + case "zero": |
| 196 | + metadata["resourceVersion"] = "0" |
| 197 | + case "matchingUID": |
| 198 | + metadata["uid"] = "original" |
| 199 | + case "emptyUID": |
| 200 | + metadata["uid"] = "" |
| 201 | + case "wrongUID": |
| 202 | + metadata["uid"] = "different" |
| 203 | + } |
| 204 | + payload := map[string]any{"apiVersion": "apps.cozystack.io/v1alpha1", "kind": "Redis", "metadata": metadata, "spec": map[string]any{"requested": true}} |
| 205 | + verb, contentType := http.MethodPatch, "application/merge-patch+json" |
| 206 | + var body []byte |
| 207 | + switch method { |
| 208 | + case "put": |
| 209 | + verb, contentType = http.MethodPut, "application/json" |
| 210 | + case "strategic": |
| 211 | + contentType = "application/strategic-merge-patch+json" |
| 212 | + case "apply": |
| 213 | + contentType = "application/apply-patch+yaml" |
| 214 | + case "json": |
| 215 | + contentType = "application/json-patch+json" |
| 216 | + ops := []map[string]any{{"op": "add", "path": "/spec/requested", "value": true}} |
| 217 | + if rv, ok := metadata["resourceVersion"]; ok { |
| 218 | + op := "add" |
| 219 | + if version == "test" { |
| 220 | + op = "test" |
| 221 | + } |
| 222 | + ops = append(ops, map[string]any{"op": op, "path": "/metadata/resourceVersion", "value": rv}) |
| 223 | + } |
| 224 | + if uid, ok := metadata["uid"]; ok { |
| 225 | + ops = append(ops, map[string]any{"op": "add", "path": "/metadata/uid", "value": uid}) |
| 226 | + } |
| 227 | + body, _ = json.Marshal(ops) |
| 228 | + } |
| 229 | + if body == nil { |
| 230 | + body, _ = json.Marshal(payload) |
| 231 | + } |
| 232 | + url += "?fieldManager=concurrency-test&fieldValidation=Strict" |
| 233 | + if dryRun { |
| 234 | + url += "&dryRun=All" |
| 235 | + } |
| 236 | + if method == "apply" { |
| 237 | + url += fmt.Sprintf("&force=%v", force) |
| 238 | + } |
| 239 | + req, err := http.NewRequestWithContext(ctx, verb, url, bytes.NewReader(body)) |
| 240 | + if err != nil { |
| 241 | + t.Fatal(err) |
| 242 | + } |
| 243 | + req.Header.Set("Content-Type", contentType) |
| 244 | + resp, err := http.DefaultClient.Do(req) |
| 245 | + if err != nil { |
| 246 | + t.Fatal(err) |
| 247 | + } |
| 248 | + response, err := io.ReadAll(resp.Body) |
| 249 | + if closeErr := resp.Body.Close(); closeErr != nil { |
| 250 | + t.Fatal(closeErr) |
| 251 | + } |
| 252 | + if err != nil { |
| 253 | + t.Fatal(err) |
| 254 | + } |
| 255 | + want := http.StatusOK |
| 256 | + if version == "current" || version == "stale" { |
| 257 | + want = http.StatusConflict |
| 258 | + } |
| 259 | + if version == "test" { |
| 260 | + want = http.StatusUnprocessableEntity |
| 261 | + } |
| 262 | + if version == "wrongUID" { |
| 263 | + want = http.StatusUnprocessableEntity |
| 264 | + if method == "put" { |
| 265 | + want = http.StatusConflict |
| 266 | + } |
| 267 | + } |
| 268 | + if version == "admissionReject" { |
| 269 | + want = http.StatusForbidden |
| 270 | + } |
| 271 | + if resp.StatusCode != want { |
| 272 | + t.Fatalf("HTTP %d, want %d: %s", resp.StatusCode, want, response) |
| 273 | + } |
| 274 | + mutatedVersions, validatedVersions := probe.versions() |
| 275 | + if want == http.StatusOK || want == http.StatusForbidden { |
| 276 | + wantCalls := 2 |
| 277 | + if version == "uncontended" { |
| 278 | + wantCalls = 1 |
| 279 | + } |
| 280 | + if len(mutatedVersions) != wantCalls || len(validatedVersions) != wantCalls { |
| 281 | + t.Fatalf("admission mutations=%v validations=%v, want %d calls", mutatedVersions, validatedVersions, wantCalls) |
| 282 | + } |
| 283 | + for i := range mutatedVersions { |
| 284 | + if mutatedVersions[i] != validatedVersions[i] { |
| 285 | + t.Fatalf("admission snapshot mismatch: mutations=%v validations=%v", mutatedVersions, validatedVersions) |
| 286 | + } |
| 287 | + } |
| 288 | + if wantCalls == 2 && mutatedVersions[0] == mutatedVersions[1] { |
| 289 | + t.Fatalf("admission used stale snapshot: %v", mutatedVersions) |
| 290 | + } |
| 291 | + } |
| 292 | + if want == http.StatusOK { |
| 293 | + var typed struct { |
| 294 | + Metadata metav1.ObjectMeta `json:"metadata"` |
| 295 | + Spec struct { |
| 296 | + Numeric int64 `json:"numeric"` |
| 297 | + Requested bool `json:"requested"` |
| 298 | + } `json:"spec"` |
| 299 | + } |
| 300 | + if err := json.Unmarshal(response, &typed); err != nil { |
| 301 | + t.Fatalf("typed readback: %v; response=%s", err, response) |
| 302 | + } |
| 303 | + if !typed.Spec.Requested || typed.Metadata.Annotations["admitted-version"] != mutatedVersions[len(mutatedVersions)-1] { |
| 304 | + t.Fatalf("proposed update missing from response: %s", response) |
| 305 | + } |
| 306 | + if method != "put" && (typed.Spec.Numeric != 10 || bytes.Contains(response, []byte("_internal"))) { |
| 307 | + t.Fatalf("public projection changed: %s", response) |
| 308 | + } |
| 309 | + } |
| 310 | + live := &helmv2.HelmRelease{} |
| 311 | + if err := backing.Get(ctx, client.ObjectKeyFromObject(hr), live); err != nil { |
| 312 | + t.Fatal(err) |
| 313 | + } |
| 314 | + if dryRun && (live.Annotations["apps.cozystack.io-admitted-version"] != "" || len(live.ManagedFields) != 0) { |
| 315 | + t.Fatalf("dry-run admission or field management persisted: %#v", live.ObjectMeta) |
| 316 | + } |
| 317 | + if want == http.StatusOK && !dryRun && live.Annotations["apps.cozystack.io-admitted-version"] != mutatedVersions[len(mutatedVersions)-1] { |
| 318 | + t.Fatalf("persisted stale admission output: annotations=%v versions=%v", live.Annotations, mutatedVersions) |
| 319 | + } |
| 320 | + mu.Lock() |
| 321 | + attempts := writes |
| 322 | + mu.Unlock() |
| 323 | + if want == http.StatusForbidden && attempts != 1 { |
| 324 | + t.Fatalf("denied fresh admission reached backend: attempts=%d", attempts) |
| 325 | + } |
| 326 | + if version == "stale" || version == "wrongUID" { |
| 327 | + if attempts != 0 { |
| 328 | + t.Fatalf("failed identity constraint reached backend: %d", attempts) |
| 329 | + } |
| 330 | + return |
| 331 | + } |
| 332 | + if version == "uncontended" { |
| 333 | + if attempts != 1 { |
| 334 | + t.Fatalf("uncontended write attempts=%d", attempts) |
| 335 | + } |
| 336 | + if bytes.Contains(live.Spec.Values.Raw, []byte("requested")) == dryRun { |
| 337 | + t.Fatalf("dryRun=%v persisted=%s", dryRun, live.Spec.Values.Raw) |
| 338 | + } |
| 339 | + return |
| 340 | + } |
| 341 | + if live.Labels["foreign.example/controller"] != "kept" || live.Annotations["foreign.example/controller"] != "kept" || live.Labels[fluxshard.ShardKeyLabel] != "concurrent-shard" { |
| 342 | + t.Fatalf("concurrent controller metadata lost: %#v", live.ObjectMeta) |
| 343 | + } |
| 344 | + if len(live.OwnerReferences) != 1 || live.OwnerReferences[0].UID != "owner" || live.DeletionTimestamp == nil { |
| 345 | + t.Fatalf("concurrent lifecycle metadata lost: %#v", live.ObjectMeta) |
| 346 | + } |
| 347 | + wantFinalizers := 2 |
| 348 | + if method == "put" && want == http.StatusOK && !dryRun { |
| 349 | + wantFinalizers = 1 |
| 350 | + } |
| 351 | + if !live.Spec.Suspend || len(live.Finalizers) != wantFinalizers || !slices.Contains(live.Finalizers, "finalizers.fluxcd.io") || slices.Contains(live.Finalizers, metav1.FinalizerDeleteDependents) != (wantFinalizers == 2) { |
| 352 | + t.Fatalf("concurrent suspension/finalizers lost: %#v", live) |
| 353 | + } |
| 354 | + if want != http.StatusOK || dryRun { |
| 355 | + if bytes.Contains(live.Spec.Values.Raw, []byte("requested")) { |
| 356 | + t.Fatalf("failed/dry-run request persisted: %s", live.Spec.Values.Raw) |
| 357 | + } |
| 358 | + return |
| 359 | + } |
| 360 | + if attempts != 2 { |
| 361 | + t.Fatalf("write attempts=%d, want 2", attempts) |
| 362 | + } |
| 363 | + if !bytes.Contains(live.Spec.Values.Raw, []byte(`"requested":true`)) { |
| 364 | + t.Fatalf("request lost: %s", live.Spec.Values.Raw) |
| 365 | + } |
| 366 | + if method != "put" && (!bytes.Contains(live.Spec.Values.Raw, []byte(`"retained":2`)) || live.Labels["apps.cozystack.io-concurrent"] != "kept" || live.Annotations["apps.cozystack.io-concurrent"] != "kept") { |
| 367 | + t.Fatalf("patch lost concurrent fields: values=%s labels=%v", live.Spec.Values.Raw, live.Labels) |
| 368 | + } |
| 369 | + }) |
| 370 | + } |
| 371 | + } |
| 372 | + } |
| 373 | + } |
| 374 | +} |
0 commit comments