Skip to content

Commit f095f57

Browse files
[lineage] Cache owner GETs and short-circuit re-admission
The admission webhook walks the owner chain (Pod → STS → HelmRelease → app CR) using uncached dynamic GETs against the apiserver. Each level costs a full RT, so under apiserver load a single Pod admission spent 300-500ms p99 in the lineage handler — making it the top admission-latency contributor in clusters with many tenants. Two changes: * New TTL cache in pkg/lineage/cache.go, threaded through WalkOwnershipGraph via a *walkState wrapper. Both positive and negative results are cached so repeated admissions for siblings of the same parent chain collapse to one RT per TTL window. The cache is opt-in (nil is a valid no-op), so existing callers and the legacy variadic visited-map form keep working. Default TTL of 60s for the webhook strikes a balance between staleness on owner DELETE and apiserver pressure under load. * Handle() returns early if the object already carries every lineage label. The objectSelector on the MutatingWebhookConfiguration already excludes managed objects, but a defensive check costs nothing and protects against the selector being relaxed or labels being applied by a concurrent controller mid-admission. Tests cover nil-safety, TTL expiry, negative caching, key discrimination, and backwards compatibility of the variadic memory parameter. Signed-off-by: mattia-eleuteri <[email protected]>
1 parent e16d987 commit f095f57

5 files changed

Lines changed: 405 additions & 43 deletions

File tree

‎internal/lineagecontrollerwebhook/types.go‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,30 @@ package lineagecontrollerwebhook
33
import (
44
"sync"
55
"sync/atomic"
6+
"time"
67

8+
"github.com/cozystack/cozystack/pkg/lineage"
79
"k8s.io/apimachinery/pkg/api/meta"
810
"k8s.io/apimachinery/pkg/runtime"
911
"k8s.io/client-go/dynamic"
1012
"sigs.k8s.io/controller-runtime/pkg/client"
1113
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
1214
)
1315

16+
// ownerCacheTTL controls how long a successful or negative dynamic-client GET
17+
// result is reused across admission requests. The labels we derive are based on
18+
// stable identity (kind/name/owner refs), so even a generous TTL is safe; the
19+
// dominant cost we are saving is the apiserver round-trip on every admission.
20+
const ownerCacheTTL = 60 * time.Second
21+
1422
// +kubebuilder:webhook:path=/mutate-lineage,mutating=true,failurePolicy=Fail,sideEffects=None,groups="",resources=pods,secrets,services,persistentvolumeclaims,verbs=create;update,versions=v1,name=mlineage.cozystack.io,admissionReviewVersions={v1}
1523
type LineageControllerWebhook struct {
1624
client.Client
17-
Scheme *runtime.Scheme
18-
decoder admission.Decoder
19-
dynClient dynamic.Interface
20-
mapper meta.RESTMapper
21-
config atomic.Value
22-
initOnce sync.Once
25+
Scheme *runtime.Scheme
26+
decoder admission.Decoder
27+
dynClient dynamic.Interface
28+
mapper meta.RESTMapper
29+
config atomic.Value
30+
initOnce sync.Once
31+
ownerCache *lineage.ObjectCache
2332
}

‎internal/lineagecontrollerwebhook/webhook.go‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,13 @@ func (h *LineageControllerWebhook) SetupWithManagerAsWebhook(mgr ctrl.Manager) e
6767
return err
6868
}
6969

70+
// Owner lookups dominate admission latency: each Pod admission walks the
71+
// owner chain (Pod → STS → HelmRelease → app CR) with sequential dynamic
72+
// GETs to the apiserver. The cache collapses repeat lookups across
73+
// concurrent admissions for siblings of the same parent chain. TTL is short
74+
// enough that DELETE/recreate of an owner is reflected within a minute.
75+
h.ownerCache = lineage.NewObjectCache(ownerCacheTTL)
76+
7077
h.initConfig()
7178
// Register HTTP path -> handler.
7279
mgr.GetWebhookServer().Register("/mutate-lineage", &admission.Webhook{Handler: h})
@@ -95,6 +102,18 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req
95102
return admission.Errored(400, fmt.Errorf("decode object: %w", err))
96103
}
97104

105+
// Fast-path: the objectSelector on the MutatingWebhookConfiguration already
106+
// skips admission for objects that carry the managed-by-cozystack label, so
107+
// in steady state this branch only triggers for races (a controller
108+
// re-applying labels mid-flight) and for safety if the selector is ever
109+
// loosened. Returning an empty patch is significantly cheaper than walking
110+
// the owner chain again, and the labels we'd recompute are guaranteed
111+
// identical (they're derived from stable identity).
112+
if hasAllLineageLabels(obj) {
113+
logger.V(1).Info("object already carries lineage labels, skipping owner walk")
114+
return admission.Allowed("lineage labels already present")
115+
}
116+
98117
labels, err := h.computeLabels(ctx, obj)
99118
for {
100119
if err != nil && errors.Is(err, NoAncestors) {
@@ -124,11 +143,11 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req
124143
}
125144

126145
func (h *LineageControllerWebhook) computeLabels(ctx context.Context, o *unstructured.Unstructured) (map[string]string, error) {
127-
owners := lineage.WalkOwnershipGraph(ctx, h.dynClient, h.mapper, h, o)
146+
owners := lineage.WalkOwnershipGraphWithCache(ctx, h.dynClient, h.mapper, h, h.ownerCache, o)
128147
if len(owners) == 0 {
129148
return map[string]string{ManagedObjectKey: "false"}, NoAncestors
130149
}
131-
obj, err := owners[0].GetUnstructured(ctx, h.dynClient, h.mapper)
150+
obj, err := owners[0].GetUnstructuredCached(ctx, h.dynClient, h.mapper, h.ownerCache)
132151
if err != nil {
133152
return nil, err
134153
}
@@ -185,6 +204,26 @@ func (h *LineageControllerWebhook) applyLabels(o *unstructured.Unstructured, lab
185204
o.SetLabels(existing)
186205
}
187206

207+
// hasAllLineageLabels reports whether `o` already carries every label the
208+
// webhook would set. We treat ManagedObjectKey alone as insufficient because
209+
// the "unmanaged" path writes it with value "false" without setting the rest;
210+
// only the full set proves a previous successful walk.
211+
func hasAllLineageLabels(o *unstructured.Unstructured) bool {
212+
labels := o.GetLabels()
213+
if labels == nil {
214+
return false
215+
}
216+
if labels[ManagedObjectKey] != "true" {
217+
return false
218+
}
219+
for _, k := range []string{ManagerGroupKey, ManagerKindKey, ManagerNameKey} {
220+
if _, ok := labels[k]; !ok {
221+
return false
222+
}
223+
}
224+
return true
225+
}
226+
188227
func (h *LineageControllerWebhook) decodeUnstructured(req admission.Request, out *unstructured.Unstructured) error {
189228
if h.decoder != nil {
190229
if err := h.decoder.Decode(req, out); err == nil {

‎pkg/lineage/cache.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package lineage
2+
3+
import (
4+
"sync"
5+
"time"
6+
7+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
8+
)
9+
10+
// ObjectCache is a small TTL cache shared across WalkOwnershipGraph calls so the
11+
// admission webhook can avoid repeatedly issuing the same uncached dynamic GET to
12+
// the apiserver. A child Pod's owner (StatefulSet, HelmRelease, application CR)
13+
// is looked up on every admission of every sibling — the cache collapses those
14+
// per-process lookups to a single round-trip per TTL window.
15+
//
16+
// The cache stores both successful lookups and negative results (NotFound or
17+
// other transient errors) so a tight stream of admissions for the same parent
18+
// chain does not amplify pressure on the apiserver. Entries expire after `ttl`
19+
// from the time of insertion; staleness is acceptable because the labels we
20+
// compute are derived from immutable identity fields (kind, name, namespace,
21+
// owner refs) — not from object state that mutates frequently.
22+
//
23+
// A nil ObjectCache is a valid no-op cache: Get always misses, Set is a no-op.
24+
type ObjectCache struct {
25+
ttl time.Duration
26+
mu sync.RWMutex
27+
items map[cacheKey]cacheEntry
28+
}
29+
30+
type cacheKey struct {
31+
apiVersion string
32+
kind string
33+
namespace string
34+
name string
35+
}
36+
37+
type cacheEntry struct {
38+
obj *unstructured.Unstructured
39+
err error
40+
expiresAt time.Time
41+
}
42+
43+
// NewObjectCache returns a cache with the given TTL. A ttl of zero or less
44+
// disables caching by returning nil; callers can pass the result straight to
45+
// WalkOwnershipGraph without nil-checking.
46+
func NewObjectCache(ttl time.Duration) *ObjectCache {
47+
if ttl <= 0 {
48+
return nil
49+
}
50+
return &ObjectCache{
51+
ttl: ttl,
52+
items: make(map[cacheKey]cacheEntry),
53+
}
54+
}
55+
56+
// Get returns a cached lookup, if any. The second return value reports whether
57+
// the entry was present and unexpired. A nil receiver is safe.
58+
func (c *ObjectCache) Get(apiVersion, kind, namespace, name string) (*unstructured.Unstructured, error, bool) {
59+
if c == nil {
60+
return nil, nil, false
61+
}
62+
k := cacheKey{apiVersion, kind, namespace, name}
63+
c.mu.RLock()
64+
entry, ok := c.items[k]
65+
c.mu.RUnlock()
66+
if !ok || time.Now().After(entry.expiresAt) {
67+
return nil, nil, false
68+
}
69+
return entry.obj, entry.err, true
70+
}
71+
72+
// Set stores a lookup result. Both successful objects and errors are cached so
73+
// that repeated misses (NotFound, NoKindMatch) do not retry every admission.
74+
// A nil receiver is safe.
75+
func (c *ObjectCache) Set(apiVersion, kind, namespace, name string, obj *unstructured.Unstructured, err error) {
76+
if c == nil {
77+
return
78+
}
79+
k := cacheKey{apiVersion, kind, namespace, name}
80+
entry := cacheEntry{
81+
obj: obj,
82+
err: err,
83+
expiresAt: time.Now().Add(c.ttl),
84+
}
85+
c.mu.Lock()
86+
c.items[k] = entry
87+
// Opportunistically evict expired entries while we hold the write lock to
88+
// keep the map bounded in the steady state without a separate sweeper.
89+
if len(c.items) > evictionThreshold {
90+
now := time.Now()
91+
for key, e := range c.items {
92+
if now.After(e.expiresAt) {
93+
delete(c.items, key)
94+
}
95+
}
96+
}
97+
c.mu.Unlock()
98+
}
99+
100+
// evictionThreshold is the size at which Set opportunistically GCs expired
101+
// entries. Set high enough that the GC scan stays infrequent under steady
102+
// load, low enough that the map cannot grow unbounded under pathological
103+
// admission patterns.
104+
const evictionThreshold = 4096

‎pkg/lineage/cache_test.go‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
package lineage
2+
3+
import (
4+
"errors"
5+
"testing"
6+
"time"
7+
8+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
9+
)
10+
11+
func TestObjectCache_NilSafe(t *testing.T) {
12+
var c *ObjectCache // explicitly nil
13+
14+
// All operations must be no-ops without panicking.
15+
if obj, err, ok := c.Get("v1", "Pod", "ns", "n"); ok || obj != nil || err != nil {
16+
t.Fatalf("nil cache must always miss, got ok=%v obj=%v err=%v", ok, obj, err)
17+
}
18+
c.Set("v1", "Pod", "ns", "n", &unstructured.Unstructured{}, nil) // must not panic
19+
}
20+
21+
func TestObjectCache_DisabledByZeroTTL(t *testing.T) {
22+
if got := NewObjectCache(0); got != nil {
23+
t.Fatalf("NewObjectCache(0) should return nil to disable caching, got %v", got)
24+
}
25+
if got := NewObjectCache(-time.Second); got != nil {
26+
t.Fatalf("NewObjectCache(<0) should return nil to disable caching, got %v", got)
27+
}
28+
}
29+
30+
func TestObjectCache_HitMiss(t *testing.T) {
31+
c := NewObjectCache(time.Minute)
32+
33+
if _, _, ok := c.Get("v1", "Pod", "ns", "n"); ok {
34+
t.Fatal("expected miss before Set")
35+
}
36+
37+
obj := &unstructured.Unstructured{}
38+
obj.SetName("foo")
39+
c.Set("v1", "Pod", "ns", "n", obj, nil)
40+
41+
got, err, ok := c.Get("v1", "Pod", "ns", "n")
42+
if !ok {
43+
t.Fatal("expected hit after Set")
44+
}
45+
if err != nil {
46+
t.Fatalf("expected nil err, got %v", err)
47+
}
48+
if got != obj {
49+
t.Fatalf("expected cached object pointer to match, got %v", got)
50+
}
51+
}
52+
53+
func TestObjectCache_NegativeResultIsCached(t *testing.T) {
54+
c := NewObjectCache(time.Minute)
55+
want := errors.New("not found")
56+
c.Set("v1", "Pod", "ns", "n", nil, want)
57+
58+
got, err, ok := c.Get("v1", "Pod", "ns", "n")
59+
if !ok {
60+
t.Fatal("expected hit for negative cached entry")
61+
}
62+
if got != nil {
63+
t.Fatalf("expected nil object for negative entry, got %v", got)
64+
}
65+
if !errors.Is(err, want) {
66+
t.Fatalf("expected cached err to be %v, got %v", want, err)
67+
}
68+
}
69+
70+
func TestObjectCache_Expires(t *testing.T) {
71+
c := NewObjectCache(20 * time.Millisecond)
72+
c.Set("v1", "Pod", "ns", "n", &unstructured.Unstructured{}, nil)
73+
74+
if _, _, ok := c.Get("v1", "Pod", "ns", "n"); !ok {
75+
t.Fatal("expected hit immediately after Set")
76+
}
77+
78+
time.Sleep(40 * time.Millisecond)
79+
80+
if _, _, ok := c.Get("v1", "Pod", "ns", "n"); ok {
81+
t.Fatal("expected miss after TTL elapsed")
82+
}
83+
}
84+
85+
func TestObjectCache_DistinctKeys(t *testing.T) {
86+
// Sanity: cacheKey discriminates on every identity field so unrelated lookups
87+
// don't collide.
88+
c := NewObjectCache(time.Minute)
89+
c.Set("v1", "Pod", "ns", "n", mustU("a"), nil)
90+
c.Set("v1", "Pod", "ns-other", "n", mustU("b"), nil)
91+
c.Set("v1", "Service", "ns", "n", mustU("c"), nil)
92+
c.Set("apps/v1", "Pod", "ns", "n", mustU("d"), nil)
93+
94+
cases := []struct {
95+
apiVersion, kind, namespace, name string
96+
wantName string
97+
}{
98+
{"v1", "Pod", "ns", "n", "a"},
99+
{"v1", "Pod", "ns-other", "n", "b"},
100+
{"v1", "Service", "ns", "n", "c"},
101+
{"apps/v1", "Pod", "ns", "n", "d"},
102+
}
103+
for _, tc := range cases {
104+
got, _, ok := c.Get(tc.apiVersion, tc.kind, tc.namespace, tc.name)
105+
if !ok {
106+
t.Fatalf("expected hit for %s/%s in %s/%s", tc.apiVersion, tc.kind, tc.namespace, tc.name)
107+
}
108+
if got.GetName() != tc.wantName {
109+
t.Fatalf("expected obj.name=%s, got %s for %s/%s in %s/%s", tc.wantName, got.GetName(), tc.apiVersion, tc.kind, tc.namespace, tc.name)
110+
}
111+
}
112+
}
113+
114+
func TestParseWalkMemory(t *testing.T) {
115+
// fresh state from no args
116+
st, err := parseWalkMemory(nil)
117+
if err != nil || st == nil || st.visited == nil {
118+
t.Fatalf("expected fresh state, got st=%v err=%v", st, err)
119+
}
120+
121+
// legacy map[ObjectID]bool form
122+
legacy := map[ObjectID]bool{{Name: "x"}: true}
123+
st, err = parseWalkMemory([]interface{}{legacy})
124+
if err != nil {
125+
t.Fatalf("legacy form should accept map[ObjectID]bool, got err=%v", err)
126+
}
127+
if !st.visited[ObjectID{Name: "x"}] {
128+
t.Fatal("legacy visited map should be reused as-is")
129+
}
130+
131+
// preferred *walkState form
132+
cache := NewObjectCache(time.Minute)
133+
original := &walkState{visited: map[ObjectID]bool{{Name: "y"}: true}, cache: cache}
134+
st, err = parseWalkMemory([]interface{}{original})
135+
if err != nil {
136+
t.Fatalf("walkState form should be accepted, got err=%v", err)
137+
}
138+
if st != original {
139+
t.Fatal("walkState should be reused by reference so recursion shares cache")
140+
}
141+
142+
// invalid type returns fresh state + error
143+
st, err = parseWalkMemory([]interface{}{"oops"})
144+
if err == nil {
145+
t.Fatal("expected error for unsupported memory type")
146+
}
147+
if st == nil || st.visited == nil {
148+
t.Fatal("even on error, returned state must be usable to avoid nil-map panic in callers")
149+
}
150+
}
151+
152+
func mustU(name string) *unstructured.Unstructured {
153+
u := &unstructured.Unstructured{}
154+
u.SetName(name)
155+
return u
156+
}

0 commit comments

Comments
 (0)