Skip to content

Commit 8de39f6

Browse files
committed
[api] Fix representation of dynamic list kinds
## What this PR does This patch fixes an issue with the Cozystack API server that causes it to respond with the first registered dynamic list kind when listing dynamic resources. E.g., when running `k get postgreses`, the raw JSON response from the cozystack API would be ```json { "apiVersion":"apps.cozystack.io/v1alpha1", "kind":"BootBoxList", "items": [ { "apiVersion":"apps.cozystack.io/v1alpha1", "kind":"Postgres", ... }, ... ], ... } ``` The root cause is the way the `Typer` interface is implemented for the `runtime.Scheme` where the dynamic types are registered. Since the base type for all dynamic types is a `&cozyv1alpha1.Application{}`, all types are registered under the same key and the `Typer` defaults to the first `GroupVersionKind` that was registered. Only when a correctly formed `&unstructured.Unstructured{}` is returned by the API, is this resolving logic circumvented and the `GroupVersionKind` is instead inferred from the fields of the returned object. Even an `UnstructuredList` is not acceptable as a return type, instead the `items` key should be directly set on the underlying `map[string]interface{}`. This patch implements the changes detailed above. Additionally, the following features, fixes, and improvements are added: * Makefile targets to build and run the Cozystack API locally, against a Kubernetes server in the environment's KUBECONFIG. Debugging with Delve is also supported. * CI tests are added to verify the new changes. * A bug in the registration of the corev1alpha1 types is fixed. * Updated the `ConvertToTable` function to properly handle list kinds which might be of the `&unstructured.Unstructured{}` concrete type (not an `UnstructuredList`). * The scheme used by the API server's Client and WatchClient is separated from the scheme used to serve dynamic types. * The client config for reading the runtime configuration now uses the controller-runtime, which handles flags and environment variables properly, unlike `clientcmd`. ### Release note ```release-note [api] Fix incorrect list kind for list requests to the Cozystack API for dynamic resources. Add Makefile targets for local testing. Minor schema building improvements. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 9d6af84 commit 8de39f6

7 files changed

Lines changed: 109 additions & 21 deletions

File tree

‎hack/e2e-test-openapi.bats‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,26 @@
1919
curl -sS --fail 'http://localhost:21234/openapi/v2?timeout=32s' -H 'Accept: application/[email protected]+protobuf' > /dev/null
2020
)
2121
}
22+
23+
@test "Test kinds" {
24+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.kind')
25+
if [ "$val" != "TenantList" ]; then
26+
echo "Expected kind to be TenantList, got $val"
27+
exit 1
28+
fi
29+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.items[0].kind')
30+
if [ "$val" != "Tenant" ]; then
31+
echo "Expected kind to be Tenant, got $val"
32+
exit 1
33+
fi
34+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.kind')
35+
if [ "$val" != "IngressList" ]; then
36+
echo "Expected kind to be IngressList, got $val"
37+
exit 1
38+
fi
39+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.items[0].kind')
40+
if [ "$val" != "Ingress" ]; then
41+
echo "Expected kind to be Ingress, got $val"
42+
exit 1
43+
fi
44+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
apiserver.local.config/

‎packages/system/cozystack-api/Makefile‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,18 @@ NAMESPACE=cozy-system
44
include ../../../scripts/common-envs.mk
55
include ../../../scripts/package.mk
66

7+
run-local:
8+
openssl req -nodes -new -x509 -keyout /tmp/ca.key -out /tmp/ca.crt -subj "/CN=kube-ca"
9+
openssl req -out /tmp/client.csr -new -newkey rsa:2048 -nodes -keyout /tmp/client.key -subj "/C=US/ST=SomeState/L=L/OU=Dev/CN=development/O=system:masters"
10+
openssl x509 -req -days 365 -in /tmp/client.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/client.crt
11+
openssl req -out /tmp/apiserver.csr -new -newkey rsa:2048 -nodes -keyout /tmp/apiserver.key -subj "/CN=cozystack-api" -config cozystack-api-openssl.cnf
12+
openssl x509 -req -days 365 -in /tmp/apiserver.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/apiserver.crt -extensions v3_req -extfile cozystack-api-openssl.cnf
13+
CGO_ENABLED=0 go build -o /tmp/cozystack-api ../../../cmd/cozystack-api/main.go
14+
/tmp/cozystack-api --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
15+
16+
debug:
17+
dlv debug ../../../cmd/cozystack-api/main.go -- --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
18+
719
image: image-cozystack-api
820

921
image-cozystack-api:
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[ req ]
2+
distinguished_name = req_distinguished_name
3+
req_extensions = v3_req
4+
prompt = no
5+
6+
[ req_distinguished_name ]
7+
CN = cozystack-api
8+
9+
[ v3_req ]
10+
subjectAltName = @alt_names
11+
12+
[ alt_names ]
13+
IP.1 = 127.0.0.1

‎pkg/apiserver/apiserver.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,16 @@ import (
3030
"k8s.io/apimachinery/pkg/runtime/serializer"
3131
"k8s.io/apiserver/pkg/registry/rest"
3232
genericapiserver "k8s.io/apiserver/pkg/server"
33+
"k8s.io/klog/v2"
3334
ctrl "sigs.k8s.io/controller-runtime"
3435
"sigs.k8s.io/controller-runtime/pkg/cache"
3536
"sigs.k8s.io/controller-runtime/pkg/client"
37+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3638

3739
"github.com/cozystack/cozystack/pkg/apis/apps"
3840
appsinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
39-
coreinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
4041
"github.com/cozystack/cozystack/pkg/apis/core"
42+
coreinstall "github.com/cozystack/cozystack/pkg/apis/core/install"
4143
"github.com/cozystack/cozystack/pkg/config"
4244
cozyregistry "github.com/cozystack/cozystack/pkg/registry"
4345
applicationstorage "github.com/cozystack/cozystack/pkg/registry/apps/application"
@@ -48,7 +50,8 @@ import (
4850

4951
var (
5052
// Scheme defines methods for serializing and deserializing API objects.
51-
Scheme = runtime.NewScheme()
53+
Scheme = runtime.NewScheme()
54+
mgrScheme = runtime.NewScheme()
5255
// Codecs provides methods for retrieving codecs and serializers for specific
5356
// versions and content types.
5457
Codecs = serializer.NewCodecFactory(Scheme)
@@ -57,18 +60,23 @@ var (
5760
)
5861

5962
func init() {
63+
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{
64+
Development: true,
65+
// any other zap.Options tweaks
66+
})))
67+
klog.SetLogger(ctrl.Log.WithName("klog"))
6068
appsinstall.Install(Scheme)
6169
coreinstall.Install(Scheme)
6270

6371
// Register HelmRelease types.
64-
if err := helmv2.AddToScheme(Scheme); err != nil {
72+
if err := helmv2.AddToScheme(mgrScheme); err != nil {
6573
panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
6674
}
6775

68-
if err := corev1.AddToScheme(Scheme); err != nil {
76+
if err := corev1.AddToScheme(mgrScheme); err != nil {
6977
panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
7078
}
71-
if err := rbacv1.AddToScheme(Scheme); err != nil {
79+
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
7280
panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
7381
}
7482
// Add unversioned types.
@@ -134,7 +142,7 @@ func (c completedConfig) New() (*CozyServer, error) {
134142
}
135143

136144
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
137-
Scheme: Scheme,
145+
Scheme: mgrScheme,
138146
Cache: cache.Options{SyncPeriod: &syncPeriod},
139147
})
140148
if err != nil {
@@ -164,7 +172,7 @@ func (c completedConfig) New() (*CozyServer, error) {
164172
}
165173

166174
cli := mgr.GetClient()
167-
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: Scheme})
175+
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: mgrScheme})
168176
if err != nil {
169177
return nil, fmt.Errorf("failed to build watch client: %w", err)
170178
}

‎pkg/cmd/server/start.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ import (
4242
genericoptions "k8s.io/apiserver/pkg/server/options"
4343
utilfeature "k8s.io/apiserver/pkg/util/feature"
4444
utilversionpkg "k8s.io/apiserver/pkg/util/version"
45-
"k8s.io/client-go/tools/clientcmd"
4645
"k8s.io/component-base/featuregate"
4746
baseversion "k8s.io/component-base/version"
4847
netutils "k8s.io/utils/net"
4948
"sigs.k8s.io/controller-runtime/pkg/client"
49+
k8sconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
5050
)
5151

5252
// CozyServerOptions holds the state for the Cozy API server
@@ -150,7 +150,7 @@ func (o *CozyServerOptions) Complete() error {
150150
return fmt.Errorf("failed to register types: %w", err)
151151
}
152152

153-
cfg, err := clientcmd.BuildConfigFromFlags("", "")
153+
cfg, err := k8sconfig.GetConfig()
154154
if err != nil {
155155
return fmt.Errorf("failed to get kubeconfig: %w", err)
156156
}

‎pkg/registry/apps/application/rest.go‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -389,11 +389,9 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
389389
}
390390

391391
// Explicitly set apiVersion and kind in unstructured object
392-
appList := &unstructured.UnstructuredList{}
393-
appList.SetAPIVersion("apps.cozystack.io/v1alpha1")
394-
appList.SetKind(r.kindName + "List")
392+
appList := r.NewList().(*unstructured.Unstructured)
395393
appList.SetResourceVersion(hrList.GetResourceVersion())
396-
appList.Items = items
394+
appList.Object["items"] = items
397395

398396
klog.V(6).Infof("Successfully listed %d Application resources in namespace %s", len(items), namespace)
399397
return appList, nil
@@ -516,14 +514,12 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
516514
klog.Errorf("Failed to convert Application to unstructured for resource %s: %v", convertedApp.GetName(), err)
517515
return nil, false, fmt.Errorf("failed to convert Application to unstructured: %v", err)
518516
}
519-
520-
// Explicitly set apiVersion and kind in unstructured object
521-
unstructuredApp["apiVersion"] = "apps.cozystack.io/v1alpha1"
522-
unstructuredApp["kind"] = r.kindName
517+
obj := &unstructured.Unstructured{Object: unstructuredApp}
518+
obj.SetGroupVersionKind(r.gvk)
523519

524520
klog.V(6).Infof("Returning patched Application object: %+v", unstructuredApp)
525521

526-
return &unstructured.Unstructured{Object: unstructuredApp}, false, nil
522+
return obj, false, nil
527523
}
528524

529525
// Delete removes an Application by deleting the corresponding HelmRelease
@@ -723,11 +719,13 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio
723719
klog.Errorf("Failed to convert Application to unstructured: %v", err)
724720
continue
725721
}
722+
obj := &unstructured.Unstructured{Object: unstructuredApp}
723+
obj.SetGroupVersionKind(r.gvk)
726724

727725
// Create watch event with Application object
728726
appEvent := watch.Event{
729727
Type: event.Type,
730-
Object: &unstructured.Unstructured{Object: unstructuredApp},
728+
Object: obj,
731729
}
732730

733731
// Send event to custom watcher
@@ -1060,6 +1058,34 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO
10601058
table = r.buildTableFromApplications(apps)
10611059
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
10621060
case *unstructured.Unstructured:
1061+
var apps []appsv1alpha1.Application
1062+
for {
1063+
var items interface{}
1064+
var ok bool
1065+
var objects []unstructured.Unstructured
1066+
if items, ok = obj.Object["items"]; !ok {
1067+
break
1068+
}
1069+
if objects, ok = items.([]unstructured.Unstructured); !ok {
1070+
break
1071+
}
1072+
apps = make([]appsv1alpha1.Application, 0, len(objects))
1073+
var a appsv1alpha1.Application
1074+
for i := range objects {
1075+
err := runtime.DefaultUnstructuredConverter.FromUnstructured(objects[i].Object, &a)
1076+
if err != nil {
1077+
klog.Errorf("Failed to convert Unstructured to Application: %v", err)
1078+
continue
1079+
}
1080+
apps = append(apps, a)
1081+
}
1082+
break
1083+
}
1084+
if apps != nil {
1085+
table = r.buildTableFromApplications(apps)
1086+
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
1087+
break
1088+
}
10631089
var app appsv1alpha1.Application
10641090
err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &app)
10651091
if err != nil {
@@ -1196,12 +1222,17 @@ func (r *REST) Destroy() {
11961222

11971223
// New creates a new instance of Application
11981224
func (r *REST) New() runtime.Object {
1199-
return &appsv1alpha1.Application{}
1225+
obj := &unstructured.Unstructured{}
1226+
obj.SetGroupVersionKind(r.gvk)
1227+
return obj
12001228
}
12011229

12021230
// NewList returns an empty list of Application objects
12031231
func (r *REST) NewList() runtime.Object {
1204-
return &appsv1alpha1.ApplicationList{}
1232+
obj := &unstructured.Unstructured{}
1233+
obj.SetGroupVersionKind(r.gvk.GroupVersion().WithKind(r.kindName + "List"))
1234+
obj.Object["items"] = make([]interface{}, 0)
1235+
return obj
12051236
}
12061237

12071238
// Kind returns the resource kind used for API discovery

0 commit comments

Comments
 (0)