Skip to content

Commit 9dd8da4

Browse files
committed
fix(api): preserve Application update concurrency constraints
Admitting one version and writing a later one can overwrite concurrent changes. A retry must not weaken the caller's UID or resourceVersion constraints. Keep omitted resourceVersion updates unconditional for compatibility; numeric zero follows the same Kubernetes versioner semantics. Fixes: 728f676 ("fix(api): retry HelmRelease update on resourceVersion conflict") Fixes: e6ffb4f ("[cozystack-api] Fix resourceVersion error") Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 4087218 commit 9dd8da4

13 files changed

Lines changed: 987 additions & 109 deletions
Lines changed: 374 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,374 @@
1+
package server_test
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"encoding/json"
7+
"fmt"
8+
"io"
9+
"net/http"
10+
"slices"
11+
"sync"
12+
"testing"
13+
14+
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
15+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
16+
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
17+
apierrors "k8s.io/apimachinery/pkg/api/errors"
18+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19+
"k8s.io/apiserver/pkg/admission"
20+
"sigs.k8s.io/controller-runtime/pkg/client"
21+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
22+
23+
fluxshard "github.com/cozystack/cozystack/internal/fluxshardoperator"
24+
"github.com/cozystack/cozystack/pkg/config"
25+
)
26+
27+
func TestApplicationApplyCreate(t *testing.T) {
28+
for _, uid := range []string{"", "nonexistent"} {
29+
for _, dryRun := range []bool{false, true} {
30+
for _, force := range []bool{false, true} {
31+
t.Run(fmt.Sprintf("uid=%s/dryRun=%v/force=%v", uid, dryRun, force), func(t *testing.T) {
32+
ts, backing := startAppsServer(t, &config.ResourceConfig{Resources: []config.Resource{redisConfig}})
33+
payload := fmt.Sprintf(`{"apiVersion":"apps.cozystack.io/v1alpha1","kind":"Redis","metadata":{"name":"new-app","namespace":"tenant-a","uid":%q},"spec":{"replicas":1}}`, uid)
34+
url := ts.URL + fmt.Sprintf("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/redises/new-app?fieldManager=create-test&force=%v", force)
35+
if dryRun {
36+
url += "&dryRun=All"
37+
}
38+
req, err := http.NewRequest(http.MethodPatch, url, bytes.NewBufferString(payload))
39+
if err != nil {
40+
t.Fatal(err)
41+
}
42+
req.Header.Set("Content-Type", "application/apply-patch+yaml")
43+
resp, err := http.DefaultClient.Do(req)
44+
if err != nil {
45+
t.Fatal(err)
46+
}
47+
response, err := io.ReadAll(resp.Body)
48+
if closeErr := resp.Body.Close(); closeErr != nil {
49+
t.Fatal(closeErr)
50+
}
51+
if err != nil {
52+
t.Fatal(err)
53+
}
54+
want := http.StatusCreated
55+
if uid != "" {
56+
want = http.StatusConflict
57+
}
58+
if resp.StatusCode != want {
59+
t.Fatalf("HTTP %d, want %d: %s", resp.StatusCode, want, response)
60+
}
61+
live := &helmv2.HelmRelease{}
62+
err = backing.Get(context.Background(), client.ObjectKey{Namespace: "tenant-a", Name: "redis-new-app"}, live)
63+
if dryRun || uid != "" {
64+
if !apierrors.IsNotFound(err) {
65+
t.Fatalf("unexpected persisted object: err=%v live=%#v", err, live)
66+
}
67+
return
68+
}
69+
if err != nil {
70+
t.Fatal(err)
71+
}
72+
if !bytes.Contains(live.Spec.Values.Raw, []byte(`"replicas":1`)) {
73+
t.Fatalf("lost apply spec: %s", live.Spec.Values.Raw)
74+
}
75+
})
76+
}
77+
}
78+
}
79+
}
80+
81+
type updateAdmissionProbe struct {
82+
*admission.Handler
83+
mu sync.Mutex
84+
mutated []string
85+
validated []string
86+
rejectFresh bool
87+
dryRun bool
88+
}
89+
90+
func (p *updateAdmissionProbe) Admit(_ context.Context, attrs admission.Attributes, _ admission.ObjectInterfaces) error {
91+
if attrs.IsDryRun() != p.dryRun {
92+
return fmt.Errorf("mutating admission lost dry-run option")
93+
}
94+
old := attrs.GetOldObject().(*appsv1alpha1.Application)
95+
p.mu.Lock()
96+
p.mutated = append(p.mutated, old.ResourceVersion)
97+
p.mu.Unlock()
98+
app := attrs.GetObject().(*appsv1alpha1.Application)
99+
if app.Annotations == nil {
100+
app.Annotations = map[string]string{}
101+
}
102+
app.Annotations["admitted-version"] = old.ResourceVersion
103+
return nil
104+
}
105+
106+
func (p *updateAdmissionProbe) Validate(_ context.Context, attrs admission.Attributes, _ admission.ObjectInterfaces) error {
107+
if attrs.IsDryRun() != p.dryRun {
108+
return fmt.Errorf("validating admission lost dry-run option")
109+
}
110+
old := attrs.GetOldObject().(*appsv1alpha1.Application)
111+
p.mu.Lock()
112+
p.validated = append(p.validated, old.ResourceVersion)
113+
reject := p.rejectFresh && len(p.mutated) > 0 && old.ResourceVersion != p.mutated[0]
114+
p.mu.Unlock()
115+
if reject {
116+
return apierrors.NewForbidden(attrs.GetResource().GroupResource(), attrs.GetName(), fmt.Errorf("fresh admission snapshot rejected"))
117+
}
118+
return nil
119+
}
120+
121+
func (p *updateAdmissionProbe) versions() ([]string, []string) {
122+
p.mu.Lock()
123+
defer p.mu.Unlock()
124+
return append([]string(nil), p.mutated...), append([]string(nil), p.validated...)
125+
}
126+
127+
func TestApplicationUpdateHTTPConcurrency(t *testing.T) {
128+
for _, method := range []string{"put", "json", "merge", "strategic", "apply"} {
129+
for _, version := range []string{"omitted", "current", "stale", "zero", "test", "matchingUID", "emptyUID", "wrongUID", "uncontended", "admissionReject"} {
130+
if version == "test" && method != "json" {
131+
continue
132+
}
133+
for _, dryRun := range []bool{false, true} {
134+
for _, force := range []bool{false, true} {
135+
if force && method != "apply" {
136+
continue
137+
}
138+
t.Run(fmt.Sprintf("%s/%s/dryRun=%v/force=%v", method, version, dryRun, force), func(t *testing.T) {
139+
var mu sync.Mutex
140+
writes := 0
141+
funcs := interceptor.Funcs{Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
142+
mu.Lock()
143+
defer mu.Unlock()
144+
writes++
145+
writeOptions := (&client.UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions()
146+
if writeOptions.FieldManager != "concurrency-test" || writeOptions.FieldValidation != metav1.FieldValidationStrict || slices.Contains(writeOptions.DryRun, metav1.DryRunAll) != dryRun {
147+
return fmt.Errorf("write options changed: %#v", writeOptions)
148+
}
149+
if writes == 1 && version != "uncontended" {
150+
live := &helmv2.HelmRelease{}
151+
if err := c.Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil {
152+
return err
153+
}
154+
live.Spec.Values = &apiextv1.JSON{Raw: []byte(`{"retained":2,"numeric":1e1,"_internal":true}`)}
155+
live.Spec.Suspend = true
156+
live.Labels["apps.cozystack.io-concurrent"] = "kept"
157+
live.Labels["foreign.example/controller"] = "kept"
158+
live.Labels[fluxshard.ShardKeyLabel] = "concurrent-shard"
159+
live.Annotations = map[string]string{"foreign.example/controller": "kept", "apps.cozystack.io-concurrent": "kept"}
160+
live.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "owner", UID: "owner"}}
161+
live.Finalizers = []string{"finalizers.fluxcd.io", metav1.FinalizerDeleteDependents}
162+
if err := c.Update(ctx, live); err != nil {
163+
return err
164+
}
165+
if err := c.Delete(ctx, live); err != nil {
166+
return err
167+
}
168+
}
169+
// The fake client skips resourceVersion validation for dry-run writes.
170+
live := &helmv2.HelmRelease{}
171+
if err := c.Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil {
172+
return err
173+
}
174+
if obj.GetResourceVersion() != live.ResourceVersion || obj.GetUID() != live.UID {
175+
return apierrors.NewConflict(helmv2.GroupVersion.WithResource("helmreleases").GroupResource(), obj.GetName(), fmt.Errorf("object changed"))
176+
}
177+
return c.Update(ctx, obj, opts...)
178+
}}
179+
probe := &updateAdmissionProbe{Handler: admission.NewHandler(admission.Update), dryRun: dryRun, rejectFresh: version == "admissionReject"}
180+
ts, backing := startAppsServerWithInterceptors(t, &config.ResourceConfig{Resources: []config.Resource{redisConfig}}, funcs, probe)
181+
ctx := context.Background()
182+
hr := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "redis-app", Namespace: "tenant-a", UID: "original", Labels: map[string]string{
183+
"apps.cozystack.io/application.kind": "Redis", "apps.cozystack.io/application.group": "apps.cozystack.io", "apps.cozystack.io/application.name": "app",
184+
}}, Spec: helmv2.HelmReleaseSpec{Values: &apiextv1.JSON{Raw: []byte(`{"retained":1,"numeric":10.0}`)}}}
185+
if err := backing.Create(ctx, hr); err != nil {
186+
t.Fatal(err)
187+
}
188+
url := ts.URL + "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/redises/app"
189+
metadata := map[string]any{"name": "app", "namespace": "tenant-a"}
190+
switch version {
191+
case "current", "test", "uncontended":
192+
metadata["resourceVersion"] = hr.ResourceVersion
193+
case "stale":
194+
metadata["resourceVersion"] = "999"
195+
case "zero":
196+
metadata["resourceVersion"] = "0"
197+
case "matchingUID":
198+
metadata["uid"] = "original"
199+
case "emptyUID":
200+
metadata["uid"] = ""
201+
case "wrongUID":
202+
metadata["uid"] = "different"
203+
}
204+
payload := map[string]any{"apiVersion": "apps.cozystack.io/v1alpha1", "kind": "Redis", "metadata": metadata, "spec": map[string]any{"requested": true}}
205+
verb, contentType := http.MethodPatch, "application/merge-patch+json"
206+
var body []byte
207+
switch method {
208+
case "put":
209+
verb, contentType = http.MethodPut, "application/json"
210+
case "strategic":
211+
contentType = "application/strategic-merge-patch+json"
212+
case "apply":
213+
contentType = "application/apply-patch+yaml"
214+
case "json":
215+
contentType = "application/json-patch+json"
216+
ops := []map[string]any{{"op": "add", "path": "/spec/requested", "value": true}}
217+
if rv, ok := metadata["resourceVersion"]; ok {
218+
op := "add"
219+
if version == "test" {
220+
op = "test"
221+
}
222+
ops = append(ops, map[string]any{"op": op, "path": "/metadata/resourceVersion", "value": rv})
223+
}
224+
if uid, ok := metadata["uid"]; ok {
225+
ops = append(ops, map[string]any{"op": "add", "path": "/metadata/uid", "value": uid})
226+
}
227+
body, _ = json.Marshal(ops)
228+
}
229+
if body == nil {
230+
body, _ = json.Marshal(payload)
231+
}
232+
url += "?fieldManager=concurrency-test&fieldValidation=Strict"
233+
if dryRun {
234+
url += "&dryRun=All"
235+
}
236+
if method == "apply" {
237+
url += fmt.Sprintf("&force=%v", force)
238+
}
239+
req, err := http.NewRequestWithContext(ctx, verb, url, bytes.NewReader(body))
240+
if err != nil {
241+
t.Fatal(err)
242+
}
243+
req.Header.Set("Content-Type", contentType)
244+
resp, err := http.DefaultClient.Do(req)
245+
if err != nil {
246+
t.Fatal(err)
247+
}
248+
response, err := io.ReadAll(resp.Body)
249+
if closeErr := resp.Body.Close(); closeErr != nil {
250+
t.Fatal(closeErr)
251+
}
252+
if err != nil {
253+
t.Fatal(err)
254+
}
255+
want := http.StatusOK
256+
if version == "current" || version == "stale" {
257+
want = http.StatusConflict
258+
}
259+
if version == "test" {
260+
want = http.StatusUnprocessableEntity
261+
}
262+
if version == "wrongUID" {
263+
want = http.StatusUnprocessableEntity
264+
if method == "put" {
265+
want = http.StatusConflict
266+
}
267+
}
268+
if version == "admissionReject" {
269+
want = http.StatusForbidden
270+
}
271+
if resp.StatusCode != want {
272+
t.Fatalf("HTTP %d, want %d: %s", resp.StatusCode, want, response)
273+
}
274+
mutatedVersions, validatedVersions := probe.versions()
275+
if want == http.StatusOK || want == http.StatusForbidden {
276+
wantCalls := 2
277+
if version == "uncontended" {
278+
wantCalls = 1
279+
}
280+
if len(mutatedVersions) != wantCalls || len(validatedVersions) != wantCalls {
281+
t.Fatalf("admission mutations=%v validations=%v, want %d calls", mutatedVersions, validatedVersions, wantCalls)
282+
}
283+
for i := range mutatedVersions {
284+
if mutatedVersions[i] != validatedVersions[i] {
285+
t.Fatalf("admission snapshot mismatch: mutations=%v validations=%v", mutatedVersions, validatedVersions)
286+
}
287+
}
288+
if wantCalls == 2 && mutatedVersions[0] == mutatedVersions[1] {
289+
t.Fatalf("admission used stale snapshot: %v", mutatedVersions)
290+
}
291+
}
292+
if want == http.StatusOK {
293+
var typed struct {
294+
Metadata metav1.ObjectMeta `json:"metadata"`
295+
Spec struct {
296+
Numeric int64 `json:"numeric"`
297+
Requested bool `json:"requested"`
298+
} `json:"spec"`
299+
}
300+
if err := json.Unmarshal(response, &typed); err != nil {
301+
t.Fatalf("typed readback: %v; response=%s", err, response)
302+
}
303+
if !typed.Spec.Requested || typed.Metadata.Annotations["admitted-version"] != mutatedVersions[len(mutatedVersions)-1] {
304+
t.Fatalf("proposed update missing from response: %s", response)
305+
}
306+
if method != "put" && (typed.Spec.Numeric != 10 || bytes.Contains(response, []byte("_internal"))) {
307+
t.Fatalf("public projection changed: %s", response)
308+
}
309+
}
310+
live := &helmv2.HelmRelease{}
311+
if err := backing.Get(ctx, client.ObjectKeyFromObject(hr), live); err != nil {
312+
t.Fatal(err)
313+
}
314+
if dryRun && (live.Annotations["apps.cozystack.io-admitted-version"] != "" || len(live.ManagedFields) != 0) {
315+
t.Fatalf("dry-run admission or field management persisted: %#v", live.ObjectMeta)
316+
}
317+
if want == http.StatusOK && !dryRun && live.Annotations["apps.cozystack.io-admitted-version"] != mutatedVersions[len(mutatedVersions)-1] {
318+
t.Fatalf("persisted stale admission output: annotations=%v versions=%v", live.Annotations, mutatedVersions)
319+
}
320+
mu.Lock()
321+
attempts := writes
322+
mu.Unlock()
323+
if want == http.StatusForbidden && attempts != 1 {
324+
t.Fatalf("denied fresh admission reached backend: attempts=%d", attempts)
325+
}
326+
if version == "stale" || version == "wrongUID" {
327+
if attempts != 0 {
328+
t.Fatalf("failed identity constraint reached backend: %d", attempts)
329+
}
330+
return
331+
}
332+
if version == "uncontended" {
333+
if attempts != 1 {
334+
t.Fatalf("uncontended write attempts=%d", attempts)
335+
}
336+
if bytes.Contains(live.Spec.Values.Raw, []byte("requested")) == dryRun {
337+
t.Fatalf("dryRun=%v persisted=%s", dryRun, live.Spec.Values.Raw)
338+
}
339+
return
340+
}
341+
if live.Labels["foreign.example/controller"] != "kept" || live.Annotations["foreign.example/controller"] != "kept" || live.Labels[fluxshard.ShardKeyLabel] != "concurrent-shard" {
342+
t.Fatalf("concurrent controller metadata lost: %#v", live.ObjectMeta)
343+
}
344+
if len(live.OwnerReferences) != 1 || live.OwnerReferences[0].UID != "owner" || live.DeletionTimestamp == nil {
345+
t.Fatalf("concurrent lifecycle metadata lost: %#v", live.ObjectMeta)
346+
}
347+
wantFinalizers := 2
348+
if method == "put" && want == http.StatusOK && !dryRun {
349+
wantFinalizers = 1
350+
}
351+
if !live.Spec.Suspend || len(live.Finalizers) != wantFinalizers || !slices.Contains(live.Finalizers, "finalizers.fluxcd.io") || slices.Contains(live.Finalizers, metav1.FinalizerDeleteDependents) != (wantFinalizers == 2) {
352+
t.Fatalf("concurrent suspension/finalizers lost: %#v", live)
353+
}
354+
if want != http.StatusOK || dryRun {
355+
if bytes.Contains(live.Spec.Values.Raw, []byte("requested")) {
356+
t.Fatalf("failed/dry-run request persisted: %s", live.Spec.Values.Raw)
357+
}
358+
return
359+
}
360+
if attempts != 2 {
361+
t.Fatalf("write attempts=%d, want 2", attempts)
362+
}
363+
if !bytes.Contains(live.Spec.Values.Raw, []byte(`"requested":true`)) {
364+
t.Fatalf("request lost: %s", live.Spec.Values.Raw)
365+
}
366+
if method != "put" && (!bytes.Contains(live.Spec.Values.Raw, []byte(`"retained":2`)) || live.Labels["apps.cozystack.io-concurrent"] != "kept" || live.Annotations["apps.cozystack.io-concurrent"] != "kept") {
367+
t.Fatalf("patch lost concurrent fields: values=%s labels=%v", live.Spec.Values.Raw, live.Labels)
368+
}
369+
})
370+
}
371+
}
372+
}
373+
}
374+
}

0 commit comments

Comments
 (0)