Skip to content

Commit dab250d

Browse files
committed
fix(kamaji): resolve Gateway API availability once at startup
Kamaji checked for the Gateway API on every TenantControlPlane reconcile. That discovery call ignores the reconcile context, so it is bounded only by the client's 32s default timeout, which is longer than the 30s reconcile budget. A slow discovery endpoint could therefore use up the whole reconcile. The result could also disagree with the watches, which were set up once at startup. The patch carries the fix proposed upstream in clastix/kamaji#1327, not yet merged there, onto the Kamaji tag cozystack ships. Availability is resolved once when the manager starts, and the same value drives the watches, the indexer, the reconciled resources and the admission webhook's Gateway API check. A discovery error at startup now stops the manager, so it is no longer read as "Gateway API not installed". A Gateway API installed after Kamaji starts needs a Kamaji restart. On this platform Kamaji depends on cert-manager, which depends on gateway-api-crds, so the CRDs are already present when Kamaji starts. The patch keeps the upstream tests so that it matches the upstream commit. The docs hunk is left out because it is not part of the image. Drop this patch once Kamaji ships the change. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent b113cc7 commit dab250d

1 file changed

Lines changed: 375 additions & 0 deletions

File tree

Lines changed: 375 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,375 @@
1+
diff --git a/cmd/manager/cmd.go b/cmd/manager/cmd.go
2+
index b4e8f20..ec346cd 100644
3+
--- a/cmd/manager/cmd.go
4+
+++ b/cmd/manager/cmd.go
5+
@@ -155,6 +155,16 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
6+
return err
7+
}
8+
9+
+ // Resolved once for the process lifetime, so a discovery error must not be read as "not installed".
10+
+ gatewayAPIAvailable, err := utilities.GatewayAPIResourcesAvailable(ctx, discoveryClient)
11+
+ if err != nil {
12+
+ setupLog.Error(err, "unable to discover Gateway API availability")
13+
+
14+
+ return err
15+
+ }
16+
+
17+
+ setupLog.Info("Gateway API availability resolved", "available", gatewayAPIAvailable)
18+
+
19+
reconciler := &controllers.TenantControlPlaneReconciler{
20+
Client: mgr.GetClient(),
21+
APIReader: mgr.GetAPIReader(),
22+
@@ -174,7 +184,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
23+
KamajiMigrateImage: migrateJobImage,
24+
KamajiMigrateCABundle: webhookCABundle,
25+
MaxConcurrentReconciles: maxConcurrentReconciles,
26+
- DiscoveryClient: discoveryClient,
27+
+ GatewayAPIAvailable: gatewayAPIAvailable,
28+
}
29+
30+
if err = reconciler.SetupWithManager(ctx, mgr); err != nil {
31+
@@ -228,7 +238,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
32+
}
33+
34+
// Only requires to look for the core api group.
35+
- if utilities.AreGatewayResourcesAvailable(ctx, mgr.GetClient(), discoveryClient) {
36+
+ if gatewayAPIAvailable {
37+
if err = (&kamajiv1alpha1.GatewayListener{}).SetupWithManager(ctx, mgr); err != nil {
38+
setupLog.Error(err, "unable to create indexer", "indexer", "GatewayListener")
39+
40+
@@ -265,8 +275,9 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
41+
},
42+
},
43+
handlers.TenantControlPlaneGatewayValidation{
44+
- Client: mgr.GetClient(),
45+
- DiscoveryClient: discoveryClient,
46+
+ Client: mgr.GetClient(),
47+
+ DiscoveryClient: discoveryClient,
48+
+ GatewayAPIAvailable: gatewayAPIAvailable,
49+
},
50+
},
51+
routes.TenantControlPlaneTelemetry{}: {
52+
diff --git a/controllers/resources.go b/controllers/resources.go
53+
index fa375df..b12f085 100644
54+
--- a/controllers/resources.go
55+
+++ b/controllers/resources.go
56+
@@ -11,7 +11,6 @@ import (
57+
"github.com/go-logr/logr"
58+
"github.com/google/uuid"
59+
k8stypes "k8s.io/apimachinery/pkg/types"
60+
- "k8s.io/client-go/discovery"
61+
"sigs.k8s.io/controller-runtime/pkg/client"
62+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
63+
64+
@@ -22,7 +21,6 @@ import (
65+
"github.com/clastix/kamaji/internal/resources"
66+
ds "github.com/clastix/kamaji/internal/resources/datastore"
67+
"github.com/clastix/kamaji/internal/resources/konnectivity"
68+
- "github.com/clastix/kamaji/internal/utilities"
69+
)
70+
71+
type GroupResourceBuilderConfiguration struct {
72+
@@ -40,7 +38,7 @@ type GroupResourceBuilderConfiguration struct {
73+
KamajiService string
74+
KamajiMigrateImage string
75+
KamajiMigrateCABundle []byte
76+
- DiscoveryClient discovery.DiscoveryInterface
77+
+ GatewayAPIAvailable bool
78+
}
79+
80+
type GroupDeletableResourceBuilderConfiguration struct {
81+
@@ -72,8 +70,7 @@ func GetResources(ctx context.Context, config GroupResourceBuilderConfiguration)
82+
resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...)
83+
resources = append(resources, getKubernetesIngressResources(config.client)...)
84+
85+
- // Conditionally add Gateway resources
86+
- if utilities.AreGatewayResourcesAvailable(ctx, config.client, config.DiscoveryClient) {
87+
+ if config.GatewayAPIAvailable {
88+
resources = append(resources, getKubernetesGatewayResources(config.client)...)
89+
resources = append(resources, getKonnectivityGatewayResources(config.client)...)
90+
}
91+
diff --git a/controllers/resources_test.go b/controllers/resources_test.go
92+
new file mode 100644
93+
index 0000000..623bfdd
94+
--- /dev/null
95+
+++ b/controllers/resources_test.go
96+
@@ -0,0 +1,124 @@
97+
+// Copyright 2022 Clastix Labs
98+
+// SPDX-License-Identifier: Apache-2.0
99+
+
100+
+package controllers
101+
+
102+
+import (
103+
+ "context"
104+
+ "testing"
105+
+
106+
+ "k8s.io/apimachinery/pkg/api/meta"
107+
+ "k8s.io/apimachinery/pkg/runtime/schema"
108+
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
109+
+
110+
+ "github.com/clastix/kamaji/internal/datastore"
111+
+ "github.com/clastix/kamaji/internal/resources"
112+
+ "github.com/clastix/kamaji/internal/resources/konnectivity"
113+
+)
114+
+
115+
+// countingRESTMapper counts RESTMapper lookups, the route by which a controller-runtime client reaches discovery.
116+
+type countingRESTMapper struct {
117+
+ meta.RESTMapper
118+
+
119+
+ calls int
120+
+}
121+
+
122+
+func (m *countingRESTMapper) KindFor(r schema.GroupVersionResource) (schema.GroupVersionKind, error) {
123+
+ m.calls++
124+
+
125+
+ return m.RESTMapper.KindFor(r)
126+
+}
127+
+
128+
+func (m *countingRESTMapper) KindsFor(r schema.GroupVersionResource) ([]schema.GroupVersionKind, error) {
129+
+ m.calls++
130+
+
131+
+ return m.RESTMapper.KindsFor(r)
132+
+}
133+
+
134+
+func (m *countingRESTMapper) ResourceFor(r schema.GroupVersionResource) (schema.GroupVersionResource, error) {
135+
+ m.calls++
136+
+
137+
+ return m.RESTMapper.ResourceFor(r)
138+
+}
139+
+
140+
+func (m *countingRESTMapper) ResourcesFor(r schema.GroupVersionResource) ([]schema.GroupVersionResource, error) {
141+
+ m.calls++
142+
+
143+
+ return m.RESTMapper.ResourcesFor(r)
144+
+}
145+
+
146+
+func (m *countingRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) {
147+
+ m.calls++
148+
+
149+
+ return m.RESTMapper.RESTMapping(gk, versions...)
150+
+}
151+
+
152+
+func (m *countingRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) {
153+
+ m.calls++
154+
+
155+
+ return m.RESTMapper.RESTMappings(gk, versions...)
156+
+}
157+
+
158+
+type stubConnection struct {
159+
+ datastore.Connection
160+
+}
161+
+
162+
+func (stubConnection) GetConnectionString() string { return "" }
163+
+
164+
+func TestGetResources_ConsultsNoRESTMapper(t *testing.T) {
165+
+ t.Parallel()
166+
+
167+
+ for _, available := range []bool{true, false} {
168+
+ mapper := &countingRESTMapper{RESTMapper: meta.NewDefaultRESTMapper(nil)}
169+
+ config := GroupResourceBuilderConfiguration{
170+
+ client: fake.NewClientBuilder().WithRESTMapper(mapper).Build(),
171+
+ Connection: stubConnection{},
172+
+ GatewayAPIAvailable: available,
173+
+ }
174+
+
175+
+ GetResources(context.Background(), config)
176+
+
177+
+ if mapper.calls != 0 {
178+
+ t.Fatalf("GatewayAPIAvailable=%t: GetResources runs inside the reconcile timeout and must not consult the RESTMapper, got %d lookups", available, mapper.calls)
179+
+ }
180+
+ }
181+
+}
182+
+
183+
+func TestGetResources_GatewayResourcesFollowAvailability(t *testing.T) {
184+
+ t.Parallel()
185+
+
186+
+ tests := []struct {
187+
+ name string
188+
+ available bool
189+
+ }{
190+
+ {name: "gateway api available", available: true},
191+
+ {name: "gateway api not available", available: false},
192+
+ }
193+
+
194+
+ for _, tt := range tests {
195+
+ t.Run(tt.name, func(t *testing.T) {
196+
+ t.Parallel()
197+
+
198+
+ config := GroupResourceBuilderConfiguration{
199+
+ client: fake.NewClientBuilder().Build(),
200+
+ Connection: stubConnection{},
201+
+ GatewayAPIAvailable: tt.available,
202+
+ }
203+
+
204+
+ var kubernetesGateway, konnectivityGateway bool
205+
+
206+
+ for _, resource := range GetResources(context.Background(), config) {
207+
+ switch resource.(type) {
208+
+ case *resources.KubernetesGatewayResource:
209+
+ kubernetesGateway = true
210+
+ case *konnectivity.KubernetesKonnectivityGatewayResource:
211+
+ konnectivityGateway = true
212+
+ }
213+
+ }
214+
+
215+
+ if kubernetesGateway != tt.available || konnectivityGateway != tt.available {
216+
+ t.Fatalf("want gateway resources present=%t, got KubernetesGatewayResource=%t KubernetesKonnectivityGatewayResource=%t", tt.available, kubernetesGateway, konnectivityGateway)
217+
+ }
218+
+ })
219+
+ }
220+
+}
221+
diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go
222+
index 7eb378e..5aebcf7 100644
223+
--- a/controllers/tenantcontrolplane_controller.go
224+
+++ b/controllers/tenantcontrolplane_controller.go
225+
@@ -19,7 +19,6 @@ import (
226+
networkingv1 "k8s.io/api/networking/v1"
227+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
228+
k8stypes "k8s.io/apimachinery/pkg/types"
229+
- "k8s.io/client-go/discovery"
230+
"k8s.io/client-go/util/retry"
231+
"k8s.io/client-go/util/workqueue"
232+
"k8s.io/utils/clock"
233+
@@ -45,7 +44,6 @@ import (
234+
kamajierrors "github.com/clastix/kamaji/internal/errors"
235+
"github.com/clastix/kamaji/internal/metrics"
236+
"github.com/clastix/kamaji/internal/resources"
237+
- "github.com/clastix/kamaji/internal/utilities"
238+
)
239+
240+
// TenantControlPlaneReconciler reconciles a TenantControlPlane object.
241+
@@ -62,7 +60,9 @@ type TenantControlPlaneReconciler struct {
242+
KamajiMigrateCABundle []byte
243+
MaxConcurrentReconciles int
244+
ReconcileTimeout time.Duration
245+
- DiscoveryClient discovery.DiscoveryInterface
246+
+ // GatewayAPIAvailable is resolved once at startup: it selects both the Gateway API watches and the
247+
+ // Gateway resources reconciled, so the two cannot disagree.
248+
+ GatewayAPIAvailable bool
249+
// CertificateChan is the channel used by the CertificateLifecycleController that is checking for
250+
// certificates and kubeconfig user certs validity: a generic event for the given TCP will be triggered
251+
// once the validity threshold for the given certificate is reached.
252+
@@ -240,7 +240,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
253+
KamajiService: r.KamajiService,
254+
KamajiMigrateImage: r.KamajiMigrateImage,
255+
KamajiMigrateCABundle: r.KamajiMigrateCABundle,
256+
- DiscoveryClient: r.DiscoveryClient,
257+
+ GatewayAPIAvailable: r.GatewayAPIAvailable,
258+
}
259+
registeredResources := GetResources(ctx, groupResourceBuilderConfiguration)
260+
261+
@@ -384,8 +384,7 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr
262+
return ok && v == "migrate"
263+
})))
264+
265+
- // Conditionally add Gateway API ownership if available
266+
- if utilities.AreGatewayResourcesAvailable(ctx, r.Client, r.DiscoveryClient) {
267+
+ if r.GatewayAPIAvailable {
268+
controllerBuilder = controllerBuilder.
269+
Owns(&gatewayv1.HTTPRoute{}).
270+
Owns(&gatewayv1.GRPCRoute{}).
271+
diff --git a/internal/webhook/handlers/tcp_gateway_validate.go b/internal/webhook/handlers/tcp_gateway_validate.go
272+
index 4efcbcd..7c3bf1b 100644
273+
--- a/internal/webhook/handlers/tcp_gateway_validate.go
274+
+++ b/internal/webhook/handlers/tcp_gateway_validate.go
275+
@@ -21,6 +21,9 @@ import (
276+
type TenantControlPlaneGatewayValidation struct {
277+
Client client.Client
278+
DiscoveryClient discovery.DiscoveryInterface
279+
+ // GatewayAPIAvailable is the value the controller resolved at startup: admitting a Gateway spec
280+
+ // it will not reconcile would fail silently.
281+
+ GatewayAPIAvailable bool
282+
}
283+
284+
func (t TenantControlPlaneGatewayValidation) OnCreate(object runtime.Object) AdmissionResponse {
285+
@@ -64,8 +67,8 @@ func (t TenantControlPlaneGatewayValidation) OnDelete(object runtime.Object) Adm
286+
}
287+
288+
func (t TenantControlPlaneGatewayValidation) validateGatewayAPIAvailability(ctx context.Context) error {
289+
- if !utilities.AreGatewayResourcesAvailable(ctx, t.Client, t.DiscoveryClient) {
290+
- return fmt.Errorf("the Gateway API is not available in this cluster, cannot use gatewayRoute configuration")
291+
+ if !t.GatewayAPIAvailable {
292+
+ return fmt.Errorf("the Gateway API was not available when Kamaji started, cannot use gatewayRoute configuration: install the Gateway API CRDs and restart Kamaji")
293+
}
294+
295+
// Additional check for TLSRoute specifically
296+
diff --git a/internal/webhook/handlers/tcp_gateway_validate_test.go b/internal/webhook/handlers/tcp_gateway_validate_test.go
297+
index 2614fbb..c709f51 100644
298+
--- a/internal/webhook/handlers/tcp_gateway_validate_test.go
299+
+++ b/internal/webhook/handlers/tcp_gateway_validate_test.go
300+
@@ -104,6 +104,7 @@ var _ = Describe("TCP Gateway Validation Webhook", func() {
301+
302+
Context("and Gateway APIs are available", func() {
303+
BeforeEach(func() {
304+
+ handler.GatewayAPIAvailable = true
305+
mockDiscovery.serverGroups = &metav1.APIGroupList{
306+
Groups: []metav1.APIGroup{
307+
{Name: "gateway.networking.k8s.io"},
308+
@@ -138,19 +139,20 @@ var _ = Describe("TCP Gateway Validation Webhook", func() {
309+
It("should deny creation with clear error message", func() {
310+
_, err := handler.OnCreate(tcp)(ctx, admission.Request{})
311+
Expect(err).To(HaveOccurred())
312+
- Expect(err.Error()).To(ContainSubstring("Gateway API is not available in this cluster"))
313+
+ Expect(err.Error()).To(ContainSubstring("Gateway API was not available when Kamaji started"))
314+
})
315+
316+
It("should deny updates with clear error message", func() {
317+
oldTCP := tcp.DeepCopy()
318+
_, err := handler.OnUpdate(tcp, oldTCP)(ctx, admission.Request{})
319+
Expect(err).To(HaveOccurred())
320+
- Expect(err.Error()).To(ContainSubstring("Gateway API is not available in this cluster"))
321+
+ Expect(err.Error()).To(ContainSubstring("Gateway API was not available when Kamaji started"))
322+
})
323+
})
324+
325+
Context("and Gateway API group exists but TLSRoute is not available", func() {
326+
BeforeEach(func() {
327+
+ handler.GatewayAPIAvailable = true
328+
mockDiscovery.serverGroups = &metav1.APIGroupList{
329+
Groups: []metav1.APIGroup{
330+
{Name: "gateway.networking.k8s.io"},
331+
@@ -170,6 +172,35 @@ var _ = Describe("TCP Gateway Validation Webhook", func() {
332+
Expect(err.Error()).To(ContainSubstring("TLSRoute resource is not available"))
333+
})
334+
})
335+
+
336+
+ Context("and Gateway APIs were installed after Kamaji started", func() {
337+
+ BeforeEach(func() {
338+
+ handler.GatewayAPIAvailable = false
339+
+ mockDiscovery.serverGroups = &metav1.APIGroupList{
340+
+ Groups: []metav1.APIGroup{
341+
+ {Name: "gateway.networking.k8s.io"},
342+
+ },
343+
+ }
344+
+ mockDiscovery.serverResources["gateway.networking.k8s.io/v1"] = &metav1.APIResourceList{
345+
+ APIResources: []metav1.APIResource{
346+
+ {Kind: "TLSRoute"},
347+
+ },
348+
+ }
349+
+ })
350+
+
351+
+ It("should deny creation because the controller does not reconcile Gateway resources until restarted", func() {
352+
+ _, err := handler.OnCreate(tcp)(ctx, admission.Request{})
353+
+ Expect(err).To(HaveOccurred())
354+
+ Expect(err.Error()).To(ContainSubstring("restart Kamaji"))
355+
+ })
356+
+
357+
+ It("should deny updates because the controller does not reconcile Gateway resources until restarted", func() {
358+
+ oldTCP := tcp.DeepCopy()
359+
+ _, err := handler.OnUpdate(tcp, oldTCP)(ctx, admission.Request{})
360+
+ Expect(err).To(HaveOccurred())
361+
+ Expect(err.Error()).To(ContainSubstring("restart Kamaji"))
362+
+ })
363+
+ })
364+
})
365+
366+
Context("when Gateway configuration is added in update", func() {
367+
@@ -186,7 +217,7 @@ var _ = Describe("TCP Gateway Validation Webhook", func() {
368+
369+
_, err := handler.OnUpdate(tcp, oldTCP)(ctx, admission.Request{})
370+
Expect(err).To(HaveOccurred())
371+
- Expect(err.Error()).To(ContainSubstring("Gateway API is not available"))
372+
+ Expect(err.Error()).To(ContainSubstring("Gateway API was not available when Kamaji started"))
373+
})
374+
375+
It("should allow removing Gateway configuration", func() {

0 commit comments

Comments
 (0)