Skip to content

Commit 3f96365

Browse files
perf(lineage): cache owner GETs and short-circuit re-admission
Every admission walks the owner chain (Pod, StatefulSet, HelmRelease, application CR) with one uncached GET per level, and sibling Pods repeat the same lookups. On a cluster with many tenants that put the lineage webhook at 300-500ms p99, the largest share of apiserver admission latency. Owners are now reused for 60 seconds, successful lookups only. A failed lookup, whether RESTMapping, NotFound or a timeout, makes the walk find no owner, so the object is labeled unmanaged, and the webhook's objectSelector never admits a labeled object again. A cached failure would therefore spread to every object admitted within the TTL, and the aggregated cozystack-api makes such failures routine: discovery of apps.cozystack.io fails while it rolls. Non-Pod objects that already carry every lineage label skip the walk, since the objectSelector still matches an UPDATE whose old object lacks the label. Pods go through it regardless, as each Pod admission also sets the scheduling class. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 0a2054e commit 3f96365

6 files changed

Lines changed: 506 additions & 43 deletions

File tree

‎internal/lineagecontrollerwebhook/types.go‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,30 @@ package lineagecontrollerwebhook
33
import (
44
"sync"
55
"sync/atomic"
6+
"time"
67

8+
"github.com/cozystack/cozystack/pkg/lineage"
79
"k8s.io/apimachinery/pkg/api/meta"
810
"k8s.io/apimachinery/pkg/runtime"
911
"k8s.io/client-go/dynamic"
1012
"sigs.k8s.io/controller-runtime/pkg/client"
1113
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
1214
)
1315

16+
// ownerCacheTTL is how long an owner object is reused across admissions. The
17+
// lineage labels depend only on owner kinds and names, but the scheduling class
18+
// is read from the application's spec, so a change to it reaches new Pods up to
19+
// this long after it is made.
20+
const ownerCacheTTL = 60 * time.Second
21+
1422
// +kubebuilder:webhook:path=/mutate-lineage,mutating=true,failurePolicy=Fail,sideEffects=None,groups="",resources=pods,secrets,services,persistentvolumeclaims,verbs=create;update,versions=v1,name=mlineage.cozystack.io,admissionReviewVersions={v1}
1523
type LineageControllerWebhook struct {
1624
client.Client
17-
Scheme *runtime.Scheme
18-
decoder admission.Decoder
19-
dynClient dynamic.Interface
20-
mapper meta.RESTMapper
21-
config atomic.Value
22-
initOnce sync.Once
25+
Scheme *runtime.Scheme
26+
decoder admission.Decoder
27+
dynClient dynamic.Interface
28+
mapper meta.RESTMapper
29+
config atomic.Value
30+
initOnce sync.Once
31+
ownerCache *lineage.ObjectCache
2332
}

‎internal/lineagecontrollerwebhook/webhook.go‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ func (h *LineageControllerWebhook) SetupWithManagerAsWebhook(mgr ctrl.Manager) e
7575
return err
7676
}
7777

78+
h.ownerCache = lineage.NewObjectCache(ownerCacheTTL)
79+
7880
h.initConfig()
7981
// Register HTTP path -> handler.
8082
mgr.GetWebhookServer().Register("/mutate-lineage", &admission.Webhook{Handler: h})
@@ -103,6 +105,14 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req
103105
return admission.Errored(400, fmt.Errorf("decode object: %w", err))
104106
}
105107

108+
// The objectSelector skips objects carrying the managed label, but on UPDATE
109+
// it matches if either the old or the new object does. Pods go on, since
110+
// every Pod admission also needs applySchedulingClass.
111+
if obj.GetKind() != "Pod" && hasAllLineageLabels(obj) {
112+
logger.V(1).Info("object already carries lineage labels, skipping owner walk")
113+
return admission.Allowed("lineage labels already present")
114+
}
115+
106116
owner, err := h.getOwner(ctx, obj)
107117
switch {
108118
case err != nil && errors.Is(err, AncestryAmbiguous):
@@ -135,11 +145,11 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req
135145
}
136146

137147
func (h *LineageControllerWebhook) getOwner(ctx context.Context, o *unstructured.Unstructured) (*unstructured.Unstructured, error) {
138-
owners := lineage.WalkOwnershipGraph(ctx, h.dynClient, h.mapper, h, o)
148+
owners := lineage.WalkOwnershipGraphWithCache(ctx, h.dynClient, h.mapper, h, h.ownerCache, o)
139149
if len(owners) == 0 {
140150
return nil, NoAncestors
141151
}
142-
obj, err := owners[0].GetUnstructured(ctx, h.dynClient, h.mapper)
152+
obj, err := owners[0].GetUnstructuredCached(ctx, h.dynClient, h.mapper, h.ownerCache)
143153
if err != nil {
144154
return nil, err
145155
}
@@ -201,6 +211,24 @@ func (h *LineageControllerWebhook) applyLabels(o *unstructured.Unstructured, lab
201211
o.SetLabels(existing)
202212
}
203213

214+
// hasAllLineageLabels needs more than ManagedObjectKey, which an unmanaged
215+
// object also carries, set to "false".
216+
func hasAllLineageLabels(o *unstructured.Unstructured) bool {
217+
labels := o.GetLabels()
218+
if labels == nil {
219+
return false
220+
}
221+
if labels[ManagedObjectKey] != "true" {
222+
return false
223+
}
224+
for _, k := range []string{ManagerGroupKey, ManagerKindKey, ManagerNameKey} {
225+
if _, ok := labels[k]; !ok {
226+
return false
227+
}
228+
}
229+
return true
230+
}
231+
204232
// applySchedulingClass injects schedulerName and scheduling-class annotation
205233
// into Pods whose namespace carries the scheduler.cozystack.io/scheduling-class label.
206234
// If the referenced SchedulingClass CR does not exist (e.g. the scheduler

‎pkg/lineage/cache.go‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package lineage
2+
3+
import (
4+
"sync"
5+
"time"
6+
7+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
8+
)
9+
10+
// ObjectCache holds owner objects fetched during ownership walks, so sibling
11+
// admissions under the same owner chain share one GET per TTL window.
12+
//
13+
// Only successful lookups are stored. A failed lookup makes the webhook label
14+
// the object unmanaged, and its objectSelector never admits a labeled object
15+
// again, so a cached failure would outlive the TTL on every object admitted
16+
// meanwhile.
17+
//
18+
// A nil ObjectCache is valid: Get always misses and Set does nothing.
19+
type ObjectCache struct {
20+
ttl time.Duration
21+
mu sync.RWMutex
22+
items map[cacheKey]cacheEntry
23+
}
24+
25+
type cacheKey struct {
26+
apiVersion string
27+
kind string
28+
namespace string
29+
name string
30+
}
31+
32+
type cacheEntry struct {
33+
obj *unstructured.Unstructured
34+
expiresAt time.Time
35+
}
36+
37+
// NewObjectCache returns nil, a disabled cache, when ttl is not positive.
38+
func NewObjectCache(ttl time.Duration) *ObjectCache {
39+
if ttl <= 0 {
40+
return nil
41+
}
42+
return &ObjectCache{
43+
ttl: ttl,
44+
items: make(map[cacheKey]cacheEntry),
45+
}
46+
}
47+
48+
// Get returns the stored pointer, shared with concurrent admissions: callers
49+
// must DeepCopy before mutating it.
50+
func (c *ObjectCache) Get(apiVersion, kind, namespace, name string) (*unstructured.Unstructured, bool) {
51+
if c == nil {
52+
return nil, false
53+
}
54+
k := cacheKey{apiVersion, kind, namespace, name}
55+
c.mu.RLock()
56+
entry, ok := c.items[k]
57+
c.mu.RUnlock()
58+
if !ok || time.Now().After(entry.expiresAt) {
59+
return nil, false
60+
}
61+
return entry.obj, true
62+
}
63+
64+
func (c *ObjectCache) Set(apiVersion, kind, namespace, name string, obj *unstructured.Unstructured) {
65+
if c == nil {
66+
return
67+
}
68+
k := cacheKey{apiVersion, kind, namespace, name}
69+
entry := cacheEntry{
70+
obj: obj,
71+
expiresAt: time.Now().Add(c.ttl),
72+
}
73+
c.mu.Lock()
74+
c.items[k] = entry
75+
if len(c.items) > evictionThreshold {
76+
c.evictExpiredSampleLocked(time.Now(), evictionSampleSize)
77+
}
78+
c.mu.Unlock()
79+
}
80+
81+
// evictExpiredSampleLocked removes the expired entries among the first n that
82+
// map iteration yields. Go randomises that order, so repeated calls cover the
83+
// whole map without a full scan under the write lock. The caller holds c.mu.
84+
func (c *ObjectCache) evictExpiredSampleLocked(now time.Time, n int) {
85+
if n <= 0 {
86+
return
87+
}
88+
i := 0
89+
for key, e := range c.items {
90+
if now.After(e.expiresAt) {
91+
delete(c.items, key)
92+
}
93+
i++
94+
if i >= n {
95+
return
96+
}
97+
}
98+
}
99+
100+
const (
101+
// Nothing is evicted below evictionThreshold entries, and Get does not
102+
// drop expired ones, so up to this many owner objects (HelmReleases and
103+
// application CRs, values included) stay in memory for the life of the
104+
// process, expired or not.
105+
evictionThreshold = 4096
106+
// evictionSampleSize caps the entries one Set inspects above the
107+
// threshold, which bounds the write-lock hold time.
108+
evictionSampleSize = 64
109+
)

‎pkg/lineage/cache_test.go‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
package lineage
2+
3+
import (
4+
"strconv"
5+
"testing"
6+
"time"
7+
8+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
9+
)
10+
11+
func TestObjectCache_NilSafe(t *testing.T) {
12+
var c *ObjectCache
13+
14+
if obj, ok := c.Get("v1", "Pod", "ns", "n"); ok || obj != nil {
15+
t.Fatalf("nil cache must always miss, got ok=%v obj=%v", ok, obj)
16+
}
17+
c.Set("v1", "Pod", "ns", "n", &unstructured.Unstructured{}) // must not panic
18+
}
19+
20+
func TestObjectCache_DisabledByZeroTTL(t *testing.T) {
21+
if got := NewObjectCache(0); got != nil {
22+
t.Fatalf("NewObjectCache(0) should return nil to disable caching, got %v", got)
23+
}
24+
if got := NewObjectCache(-time.Second); got != nil {
25+
t.Fatalf("NewObjectCache(<0) should return nil to disable caching, got %v", got)
26+
}
27+
}
28+
29+
func TestObjectCache_HitMiss(t *testing.T) {
30+
c := NewObjectCache(time.Minute)
31+
32+
if _, ok := c.Get("v1", "Pod", "ns", "n"); ok {
33+
t.Fatal("expected miss before Set")
34+
}
35+
36+
obj := &unstructured.Unstructured{}
37+
obj.SetName("foo")
38+
c.Set("v1", "Pod", "ns", "n", obj)
39+
40+
got, ok := c.Get("v1", "Pod", "ns", "n")
41+
if !ok {
42+
t.Fatal("expected hit after Set")
43+
}
44+
if got != obj {
45+
t.Fatalf("expected cached object pointer to match, got %v", got)
46+
}
47+
}
48+
49+
func TestObjectCache_Expires(t *testing.T) {
50+
c := NewObjectCache(20 * time.Millisecond)
51+
c.Set("v1", "Pod", "ns", "n", &unstructured.Unstructured{})
52+
53+
if _, ok := c.Get("v1", "Pod", "ns", "n"); !ok {
54+
t.Fatal("expected hit immediately after Set")
55+
}
56+
57+
time.Sleep(40 * time.Millisecond)
58+
59+
if _, ok := c.Get("v1", "Pod", "ns", "n"); ok {
60+
t.Fatal("expected miss after TTL elapsed")
61+
}
62+
}
63+
64+
func TestObjectCache_DistinctKeys(t *testing.T) {
65+
c := NewObjectCache(time.Minute)
66+
c.Set("v1", "Pod", "ns", "n", mustU("a"))
67+
c.Set("v1", "Pod", "ns-other", "n", mustU("b"))
68+
c.Set("v1", "Service", "ns", "n", mustU("c"))
69+
c.Set("apps/v1", "Pod", "ns", "n", mustU("d"))
70+
71+
cases := []struct {
72+
apiVersion, kind, namespace, name string
73+
wantName string
74+
}{
75+
{"v1", "Pod", "ns", "n", "a"},
76+
{"v1", "Pod", "ns-other", "n", "b"},
77+
{"v1", "Service", "ns", "n", "c"},
78+
{"apps/v1", "Pod", "ns", "n", "d"},
79+
}
80+
for _, tc := range cases {
81+
got, ok := c.Get(tc.apiVersion, tc.kind, tc.namespace, tc.name)
82+
if !ok {
83+
t.Fatalf("expected hit for %s/%s in %s/%s", tc.apiVersion, tc.kind, tc.namespace, tc.name)
84+
}
85+
if got.GetName() != tc.wantName {
86+
t.Fatalf("expected obj.name=%s, got %s for %s/%s in %s/%s", tc.wantName, got.GetName(), tc.apiVersion, tc.kind, tc.namespace, tc.name)
87+
}
88+
}
89+
}
90+
91+
func TestParseWalkMemory(t *testing.T) {
92+
st, err := parseWalkMemory(nil)
93+
if err != nil || st == nil || st.visited == nil {
94+
t.Fatalf("expected fresh state, got st=%v err=%v", st, err)
95+
}
96+
97+
legacy := map[ObjectID]bool{{Name: "x"}: true}
98+
st, err = parseWalkMemory([]interface{}{legacy})
99+
if err != nil {
100+
t.Fatalf("legacy form should accept map[ObjectID]bool, got err=%v", err)
101+
}
102+
if !st.visited[ObjectID{Name: "x"}] {
103+
t.Fatal("legacy visited map should be reused as-is")
104+
}
105+
106+
var nilLegacy map[ObjectID]bool
107+
st, err = parseWalkMemory([]interface{}{nilLegacy})
108+
if err != nil {
109+
t.Fatalf("nil legacy map should be tolerated, got err=%v", err)
110+
}
111+
if st.visited == nil {
112+
t.Fatal("nil legacy map should be replaced with a usable, non-nil map")
113+
}
114+
st.visited[ObjectID{Name: "z"}] = true // must not panic
115+
116+
cache := NewObjectCache(time.Minute)
117+
original := &walkState{visited: map[ObjectID]bool{{Name: "y"}: true}, cache: cache}
118+
st, err = parseWalkMemory([]interface{}{original})
119+
if err != nil {
120+
t.Fatalf("walkState form should be accepted, got err=%v", err)
121+
}
122+
if st != original {
123+
t.Fatal("walkState should be reused by reference so recursion shares cache")
124+
}
125+
126+
st, err = parseWalkMemory([]interface{}{"oops"})
127+
if err == nil {
128+
t.Fatal("expected error for unsupported memory type")
129+
}
130+
if st == nil || st.visited == nil {
131+
t.Fatal("even on error, returned state must be usable to avoid nil-map panic in callers")
132+
}
133+
}
134+
135+
func TestObjectCache_EvictionSampleIsBounded(t *testing.T) {
136+
c := NewObjectCache(time.Hour)
137+
138+
for i := 0; i < evictionThreshold+1000; i++ {
139+
key := cacheKey{apiVersion: "v1", kind: "Pod", namespace: "ns", name: strconv.Itoa(i)}
140+
c.items[key] = cacheEntry{
141+
obj: nil,
142+
expiresAt: time.Now().Add(-time.Minute),
143+
}
144+
}
145+
beforeLen := len(c.items)
146+
147+
c.Set("v1", "Pod", "ns", "trigger", mustU("t"))
148+
149+
delta := beforeLen + 1 - len(c.items)
150+
if delta < 0 {
151+
t.Fatalf("expected map to shrink or stay equal after eviction, got delta=%d", delta)
152+
}
153+
if delta > evictionSampleSize {
154+
t.Fatalf("eviction sample is unbounded: removed %d entries in one Set, expected at most %d", delta, evictionSampleSize)
155+
}
156+
}
157+
158+
func mustU(name string) *unstructured.Unstructured {
159+
u := &unstructured.Unstructured{}
160+
u.SetName(name)
161+
return u
162+
}

0 commit comments

Comments
 (0)