Skip to content

Commit 4e8c8d6

Browse files
committed
[api] Fix representation of dynamic list kinds
## What this PR does This patch fixes an issue with the Cozystack API server that causes it to respond with the first registered dynamic list kind when listing dynamic resources. E.g., when running `k get postgreses`, the raw JSON response from the cozystack API would be ```json { "apiVersion":"apps.cozystack.io/v1alpha1", "kind":"BootBoxList", "items": [ { "apiVersion":"apps.cozystack.io/v1alpha1", "kind":"Postgres", ... }, ... ], ... } ``` The root cause is the way the `Typer` interface is implemented for the `runtime.Scheme` where the dynamic types are registered. Since the base type for all dynamic types is a `&cozyv1alpha1.Application{}`, all types are registered under the same key and the `Typer` defaults to the first `GroupVersionKind` that was registered. Only when a correctly formed `&unstructured.Unstructured{}` is returned by the API, is this resolving logic circumvented and the `GroupVersionKind` is instead inferred from the fields of the returned object. Even an `UnstructuredList` is not acceptable as a return type, instead the `items` key should be directly set on the underlying `map[string]interface{}`. This patch implements the changes detailed above. Additionally, the following features, fixes, and improvements are added: * Makefile targets to build and run the Cozystack API locally, against a Kubernetes server in the environment's KUBECONFIG. Debugging with Delve is also supported. * CI tests are added to verify the new changes. * A bug in the registration of the corev1alpha1 types is fixed. * Updated the `ConvertToTable` function to properly handle list kinds which might be of the `&unstructured.Unstructured{}` concrete type (not an `UnstructuredList`). * The scheme used by the API server's Client and WatchClient is separated from the scheme used to serve dynamic types. * The client config for reading the runtime configuration now uses the controller-runtime, which handles flags and environment variables properly, unlike `clientcmd`. ### Release note ```release-note [api] Fix incorrect list kind for list requests to the Cozystack API for dynamic resources. Add Makefile targets for local testing. Minor schema building improvements. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 9d6af84 commit 4e8c8d6

7 files changed

Lines changed: 118 additions & 23 deletions

File tree

‎hack/e2e-test-openapi.bats‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,26 @@
1919
curl -sS --fail 'http://localhost:21234/openapi/v2?timeout=32s' -H 'Accept: application/[email protected]+protobuf' > /dev/null
2020
)
2121
}
22+
23+
@test "Test kinds" {
24+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.kind')
25+
if [ "$val" != "TenantList" ]; then
26+
echo "Expected kind to be TenantList, got $val"
27+
exit 1
28+
fi
29+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.items[0].kind')
30+
if [ "$val" != "Tenant" ]; then
31+
echo "Expected kind to be Tenant, got $val"
32+
exit 1
33+
fi
34+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.kind')
35+
if [ "$val" != "IngressList" ]; then
36+
echo "Expected kind to be IngressList, got $val"
37+
exit 1
38+
fi
39+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.items[0].kind')
40+
if [ "$val" != "Ingress" ]; then
41+
echo "Expected kind to be Ingress, got $val"
42+
exit 1
43+
fi
44+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
apiserver.local.config/

‎packages/system/cozystack-api/Makefile‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,18 @@ NAMESPACE=cozy-system
44
include ../../../scripts/common-envs.mk
55
include ../../../scripts/package.mk
66

7+
run-local:
8+
openssl req -nodes -new -x509 -keyout /tmp/ca.key -out /tmp/ca.crt -subj "/CN=kube-ca"
9+
openssl req -out /tmp/client.csr -new -newkey rsa:2048 -nodes -keyout /tmp/client.key -subj "/C=US/ST=SomeState/L=L/OU=Dev/CN=development/O=system:masters"
10+
openssl x509 -req -days 365 -in /tmp/client.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/client.crt
11+
openssl req -out /tmp/apiserver.csr -new -newkey rsa:2048 -nodes -keyout /tmp/apiserver.key -subj "/CN=cozystack-api" -config cozystack-api-openssl.cnf
12+
openssl x509 -req -days 365 -in /tmp/apiserver.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/apiserver.crt -extensions v3_req -extfile cozystack-api-openssl.cnf
13+
CGO_ENABLED=0 go build -o /tmp/cozystack-api ../../../cmd/cozystack-api/main.go
14+
/tmp/cozystack-api --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
15+
16+
debug:
17+
dlv debug ../../../cmd/cozystack-api/main.go -- --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
18+
719
image: image-cozystack-api
820

921
image-cozystack-api:
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[ req ]
2+
distinguished_name = req_distinguished_name
3+
req_extensions = v3_req
4+
prompt = no
5+
6+
[ req_distinguished_name ]
7+
CN = cozystack-api
8+
9+
[ v3_req ]
10+
subjectAltName = @alt_names
11+
12+
[ alt_names ]
13+
IP.1 = 127.0.0.1

‎pkg/apiserver/apiserver.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,16 @@ import (
3030
"k8s.io/apimachinery/pkg/runtime/serializer"
3131
"k8s.io/apiserver/pkg/registry/rest"
3232
genericapiserver "k8s.io/apiserver/pkg/server"
33+
"k8s.io/klog/v2"
3334
ctrl "sigs.k8s.io/controller-runtime"
3435
"sigs.k8s.io/controller-runtime/pkg/cache"
3536
"sigs.k8s.io/controller-runtime/pkg/client"
37+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3638

3739
"github.com/cozystack/cozystack/pkg/apis/apps"
3840
appsinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
39-
coreinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
4041
"github.com/cozystack/cozystack/pkg/apis/core"
42+
coreinstall "github.com/cozystack/cozystack/pkg/apis/core/install"
4143
"github.com/cozystack/cozystack/pkg/config"
4244
cozyregistry "github.com/cozystack/cozystack/pkg/registry"
4345
applicationstorage "github.com/cozystack/cozystack/pkg/registry/apps/application"
@@ -48,7 +50,8 @@ import (
4850

4951
var (
5052
// Scheme defines methods for serializing and deserializing API objects.
51-
Scheme = runtime.NewScheme()
53+
Scheme = runtime.NewScheme()
54+
mgrScheme = runtime.NewScheme()
5255
// Codecs provides methods for retrieving codecs and serializers for specific
5356
// versions and content types.
5457
Codecs = serializer.NewCodecFactory(Scheme)
@@ -57,18 +60,23 @@ var (
5760
)
5861

5962
func init() {
63+
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{
64+
Development: true,
65+
// any other zap.Options tweaks
66+
})))
67+
klog.SetLogger(ctrl.Log.WithName("klog"))
6068
appsinstall.Install(Scheme)
6169
coreinstall.Install(Scheme)
6270

6371
// Register HelmRelease types.
64-
if err := helmv2.AddToScheme(Scheme); err != nil {
72+
if err := helmv2.AddToScheme(mgrScheme); err != nil {
6573
panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
6674
}
6775

68-
if err := corev1.AddToScheme(Scheme); err != nil {
76+
if err := corev1.AddToScheme(mgrScheme); err != nil {
6977
panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
7078
}
71-
if err := rbacv1.AddToScheme(Scheme); err != nil {
79+
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
7280
panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
7381
}
7482
// Add unversioned types.
@@ -134,7 +142,7 @@ func (c completedConfig) New() (*CozyServer, error) {
134142
}
135143

136144
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
137-
Scheme: Scheme,
145+
Scheme: mgrScheme,
138146
Cache: cache.Options{SyncPeriod: &syncPeriod},
139147
})
140148
if err != nil {
@@ -164,7 +172,7 @@ func (c completedConfig) New() (*CozyServer, error) {
164172
}
165173

166174
cli := mgr.GetClient()
167-
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: Scheme})
175+
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: mgrScheme})
168176
if err != nil {
169177
return nil, fmt.Errorf("failed to build watch client: %w", err)
170178
}

‎pkg/cmd/server/start.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ import (
4242
genericoptions "k8s.io/apiserver/pkg/server/options"
4343
utilfeature "k8s.io/apiserver/pkg/util/feature"
4444
utilversionpkg "k8s.io/apiserver/pkg/util/version"
45-
"k8s.io/client-go/tools/clientcmd"
4645
"k8s.io/component-base/featuregate"
4746
baseversion "k8s.io/component-base/version"
4847
netutils "k8s.io/utils/net"
4948
"sigs.k8s.io/controller-runtime/pkg/client"
49+
k8sconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
5050
)
5151

5252
// CozyServerOptions holds the state for the Cozy API server
@@ -150,7 +150,7 @@ func (o *CozyServerOptions) Complete() error {
150150
return fmt.Errorf("failed to register types: %w", err)
151151
}
152152

153-
cfg, err := clientcmd.BuildConfigFromFlags("", "")
153+
cfg, err := k8sconfig.GetConfig()
154154
if err != nil {
155155
return fmt.Errorf("failed to get kubeconfig: %w", err)
156156
}

‎pkg/registry/apps/application/rest.go‎

Lines changed: 52 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -389,11 +389,9 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
389389
}
390390

391391
// Explicitly set apiVersion and kind in unstructured object
392-
appList := &unstructured.UnstructuredList{}
393-
appList.SetAPIVersion("apps.cozystack.io/v1alpha1")
394-
appList.SetKind(r.kindName + "List")
392+
appList := r.NewList().(*unstructured.Unstructured)
395393
appList.SetResourceVersion(hrList.GetResourceVersion())
396-
appList.Items = items
394+
appList.Object["items"] = items
397395

398396
klog.V(6).Infof("Successfully listed %d Application resources in namespace %s", len(items), namespace)
399397
return appList, nil
@@ -441,9 +439,16 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
441439
}
442440

443441
// Assert the new object is of type Application
444-
app, ok := newObj.(*appsv1alpha1.Application)
442+
us, ok := newObj.(*unstructured.Unstructured)
445443
if !ok {
446-
errMsg := fmt.Sprintf("expected Application object, got %T", newObj)
444+
errMsg := fmt.Sprintf("expected unstructured.Unstructured object, got %T", newObj)
445+
klog.Errorf(errMsg)
446+
return nil, false, fmt.Errorf(errMsg)
447+
}
448+
app := &appsv1alpha1.Application{}
449+
450+
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, app); err != nil {
451+
errMsg := fmt.Sprintf("returned unstructured.Unstructured object was not an Application")
447452
klog.Errorf(errMsg)
448453
return nil, false, fmt.Errorf(errMsg)
449454
}
@@ -516,14 +521,12 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
516521
klog.Errorf("Failed to convert Application to unstructured for resource %s: %v", convertedApp.GetName(), err)
517522
return nil, false, fmt.Errorf("failed to convert Application to unstructured: %v", err)
518523
}
519-
520-
// Explicitly set apiVersion and kind in unstructured object
521-
unstructuredApp["apiVersion"] = "apps.cozystack.io/v1alpha1"
522-
unstructuredApp["kind"] = r.kindName
524+
obj := &unstructured.Unstructured{Object: unstructuredApp}
525+
obj.SetGroupVersionKind(r.gvk)
523526

524527
klog.V(6).Infof("Returning patched Application object: %+v", unstructuredApp)
525528

526-
return &unstructured.Unstructured{Object: unstructuredApp}, false, nil
529+
return obj, false, nil
527530
}
528531

529532
// Delete removes an Application by deleting the corresponding HelmRelease
@@ -723,11 +726,13 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio
723726
klog.Errorf("Failed to convert Application to unstructured: %v", err)
724727
continue
725728
}
729+
obj := &unstructured.Unstructured{Object: unstructuredApp}
730+
obj.SetGroupVersionKind(r.gvk)
726731

727732
// Create watch event with Application object
728733
appEvent := watch.Event{
729734
Type: event.Type,
730-
Object: &unstructured.Unstructured{Object: unstructuredApp},
735+
Object: obj,
731736
}
732737

733738
// Send event to custom watcher
@@ -1060,6 +1065,34 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO
10601065
table = r.buildTableFromApplications(apps)
10611066
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
10621067
case *unstructured.Unstructured:
1068+
var apps []appsv1alpha1.Application
1069+
for {
1070+
var items interface{}
1071+
var ok bool
1072+
var objects []unstructured.Unstructured
1073+
if items, ok = obj.Object["items"]; !ok {
1074+
break
1075+
}
1076+
if objects, ok = items.([]unstructured.Unstructured); !ok {
1077+
break
1078+
}
1079+
apps = make([]appsv1alpha1.Application, 0, len(objects))
1080+
var a appsv1alpha1.Application
1081+
for i := range objects {
1082+
err := runtime.DefaultUnstructuredConverter.FromUnstructured(objects[i].Object, &a)
1083+
if err != nil {
1084+
klog.Errorf("Failed to convert Unstructured to Application: %v", err)
1085+
continue
1086+
}
1087+
apps = append(apps, a)
1088+
}
1089+
break
1090+
}
1091+
if apps != nil {
1092+
table = r.buildTableFromApplications(apps)
1093+
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
1094+
break
1095+
}
10631096
var app appsv1alpha1.Application
10641097
err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &app)
10651098
if err != nil {
@@ -1196,12 +1229,17 @@ func (r *REST) Destroy() {
11961229

11971230
// New creates a new instance of Application
11981231
func (r *REST) New() runtime.Object {
1199-
return &appsv1alpha1.Application{}
1232+
obj := &unstructured.Unstructured{}
1233+
obj.SetGroupVersionKind(r.gvk)
1234+
return obj
12001235
}
12011236

12021237
// NewList returns an empty list of Application objects
12031238
func (r *REST) NewList() runtime.Object {
1204-
return &appsv1alpha1.ApplicationList{}
1239+
obj := &unstructured.Unstructured{}
1240+
obj.SetGroupVersionKind(r.gvk.GroupVersion().WithKind(r.kindName + "List"))
1241+
obj.Object["items"] = make([]interface{}, 0)
1242+
return obj
12051243
}
12061244

12071245
// Kind returns the resource kind used for API discovery

0 commit comments

Comments
 (0)