Skip to content

Commit 8cc3489

Browse files
committed
[api] Use shared informer cache
This patch changes all clients in the Cozystack API server to typed ones from the controller runtime. This should improve the performance of the API server and simplifies the code by removing work with unstructured objects and dynamic clients. ```release-note [api] Use typed and cache-backed k8s clients in the Cozystack API to improve performance. Get rid of operations on unstructured objects and use of dynamic clients. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 21de4f7 commit 8cc3489

6 files changed

Lines changed: 255 additions & 236 deletions

File tree

‎pkg/apiserver/apiserver.go‎

Lines changed: 53 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,22 @@ limitations under the License.
1717
package apiserver
1818

1919
import (
20+
"context"
2021
"fmt"
22+
"time"
2123

2224
helmv2 "github.com/fluxcd/helm-controller/api/v2"
25+
corev1 "k8s.io/api/core/v1"
26+
rbacv1 "k8s.io/api/rbac/v1"
2327
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2428
"k8s.io/apimachinery/pkg/runtime"
2529
"k8s.io/apimachinery/pkg/runtime/schema"
2630
"k8s.io/apimachinery/pkg/runtime/serializer"
2731
"k8s.io/apiserver/pkg/registry/rest"
2832
genericapiserver "k8s.io/apiserver/pkg/server"
29-
"k8s.io/client-go/dynamic"
30-
"k8s.io/client-go/kubernetes"
31-
restclient "k8s.io/client-go/rest"
33+
ctrl "sigs.k8s.io/controller-runtime"
34+
"sigs.k8s.io/controller-runtime/pkg/cache"
35+
"sigs.k8s.io/controller-runtime/pkg/client"
3236

3337
"github.com/cozystack/cozystack/pkg/apis/apps"
3438
appsinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
@@ -50,6 +54,7 @@ var (
5054
// versions and content types.
5155
Codecs = serializer.NewCodecFactory(Scheme)
5256
CozyComponentName = "cozy"
57+
syncPeriod = 5 * time.Minute
5358
)
5459

5560
func init() {
@@ -58,9 +63,15 @@ func init() {
5863

5964
// Register HelmRelease types.
6065
if err := helmv2.AddToScheme(Scheme); err != nil {
61-
panic(fmt.Sprintf("Failed to add HelmRelease types to scheme: %v", err))
66+
panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
6267
}
6368

69+
if err := corev1.AddToScheme(Scheme); err != nil {
70+
panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
71+
}
72+
if err := rbacv1.AddToScheme(Scheme); err != nil {
73+
panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
74+
}
6475
// Add unversioned types.
6576
metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
6677

@@ -118,43 +129,51 @@ func (c completedConfig) New() (*CozyServer, error) {
118129
}
119130

120131
// Create a dynamic client for HelmRelease using InClusterConfig.
121-
inClusterConfig, err := restclient.InClusterConfig()
132+
cfg, err := ctrl.GetConfig()
122133
if err != nil {
123-
return nil, fmt.Errorf("unable to get in-cluster config: %v", err)
134+
return nil, fmt.Errorf("failed to get kubeconfig: %w", err)
124135
}
125136

126-
dynamicClient, err := dynamic.NewForConfig(inClusterConfig)
137+
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
138+
Scheme: Scheme,
139+
Cache: cache.Options{SyncPeriod: &syncPeriod},
140+
})
127141
if err != nil {
128-
return nil, fmt.Errorf("unable to create dynamic client: %v", err)
142+
return nil, fmt.Errorf("failed to build manager: %w", err)
129143
}
130144

131-
clientset, err := kubernetes.NewForConfig(inClusterConfig)
132-
if err != nil {
133-
return nil, fmt.Errorf("create kube clientset: %v", err)
145+
ctx := context.Background()
146+
147+
if err = mustGetInformers(ctx, mgr,
148+
&helmv2.HelmRelease{},
149+
&corev1.Secret{},
150+
&corev1.Namespace{},
151+
&corev1.Service{},
152+
&rbacv1.RoleBinding{},
153+
); err != nil {
154+
return nil, fmt.Errorf("failed to get informers: %w", err)
134155
}
135156

157+
go mgr.Start(ctrl.SetupSignalHandler())
158+
159+
cli := mgr.GetClient()
160+
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: Scheme})
161+
if err != nil {
162+
return nil, fmt.Errorf("failed to build watch client: %w", err)
163+
}
136164
// --- static, cluster-scoped resource for core group ---
137165
coreV1alpha1Storage := map[string]rest.Storage{}
138166
coreV1alpha1Storage["tenantnamespaces"] = cozyregistry.RESTInPeace(
139-
tenantnamespacestorage.NewREST(
140-
clientset.CoreV1(),
141-
clientset.RbacV1(),
142-
),
167+
tenantnamespacestorage.NewREST(cli, watchCli),
143168
)
144169
coreV1alpha1Storage["tenantsecrets"] = cozyregistry.RESTInPeace(
145-
tenantsecretstorage.NewREST(
146-
clientset.CoreV1(),
147-
),
170+
tenantsecretstorage.NewREST(cli, watchCli),
148171
)
149172
coreV1alpha1Storage["tenantsecretstables"] = cozyregistry.RESTInPeace(
150-
tenantsecretstablestorage.NewREST(
151-
clientset.CoreV1(),
152-
),
173+
tenantsecretstablestorage.NewREST(cli, watchCli),
153174
)
154175
coreV1alpha1Storage["tenantmodules"] = cozyregistry.RESTInPeace(
155-
tenantmodulestorage.NewREST(
156-
dynamicClient,
157-
),
176+
tenantmodulestorage.NewREST(cli, watchCli),
158177
)
159178

160179
coreApiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(core.GroupName, Scheme, metav1.ParameterCodec, Codecs)
@@ -166,7 +185,7 @@ func (c completedConfig) New() (*CozyServer, error) {
166185
// --- dynamically-configured, per-tenant resources ---
167186
appsV1alpha1Storage := map[string]rest.Storage{}
168187
for _, resConfig := range c.ResourceConfig.Resources {
169-
storage := applicationstorage.NewREST(dynamicClient, &resConfig)
188+
storage := applicationstorage.NewREST(cli, watchCli, &resConfig)
170189
appsV1alpha1Storage[resConfig.Application.Plural] = cozyregistry.RESTInPeace(storage)
171190
}
172191
appsApiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apps.GroupName, Scheme, metav1.ParameterCodec, Codecs)
@@ -177,3 +196,12 @@ func (c completedConfig) New() (*CozyServer, error) {
177196

178197
return s, nil
179198
}
199+
200+
func mustGetInformers(ctx context.Context, mgr ctrl.Manager, types ...client.Object) error {
201+
for i := range types {
202+
if _, err := mgr.GetCache().GetInformer(ctx, types[i]); err != nil {
203+
return fmt.Errorf("failed to get informer for %T: %w", types[i], err)
204+
}
205+
}
206+
return nil
207+
}

0 commit comments

Comments
 (0)