Skip to content

Commit 5414d37

Browse files
committed
[api] Fix representation of dynamic list kinds
## What this PR does This patch fixes an issue with the Cozystack API server that causes it to respond with the first registered dynamic list kind when listing dynamic resources. E.g., when running `k get postgreses`, the raw JSON response from the cozystack API would be ```json { "apiVersion":"apps.cozystack.io/v1alpha1", "kind":"BootBoxList", "items": [ { "apiVersion":"apps.cozystack.io/v1alpha1", "kind":"Postgres", ... }, ... ], ... } ``` The root cause is the way the `Typer` interface is implemented for the `runtime.Scheme` where the dynamic types are registered. Since the base type for all dynamic types is a `&cozyv1alpha1.Application{}`, all types are registered under the same key and the `Typer` defaults to the first `GroupVersionKind` that was registered. Only when a correctly formed `&unstructured.Unstructured{}` is returned by the API, is this resolving logic circumvented and the `GroupVersionKind` is instead inferred from the fields of the returned object. Even an `UnstructuredList` is not acceptable as a return type, instead the `items` key should be directly set on the underlying `map[string]interface{}`. This patch implements the changes detailed above. Additionally, the following features, fixes, and improvements are added: * Makefile targets to build and run the Cozystack API locally, against a Kubernetes server in the environment's KUBECONFIG. Debugging with Delve is also supported. * CI tests are added to verify the new changes. * A bug in the registration of the corev1alpha1 types is fixed. * Updated the `ConvertToTable` function to properly handle list kinds which might be of the `&unstructured.Unstructured{}` concrete type (not an `UnstructuredList`). * The scheme used by the API server's Client and WatchClient is separated from the scheme used to serve dynamic types. * The client config for reading the runtime configuration now uses the controller-runtime, which handles flags and environment variables properly, unlike `clientcmd`. ### Release note ```release-note [api] Fix incorrect list kind for list requests to the Cozystack API for dynamic resources. Add Makefile targets for local testing. Minor schema building improvements. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 9d6af84 commit 5414d37

7 files changed

Lines changed: 128 additions & 25 deletions

File tree

‎hack/e2e-test-openapi.bats‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,26 @@
1919
curl -sS --fail 'http://localhost:21234/openapi/v2?timeout=32s' -H 'Accept: application/[email protected]+protobuf' > /dev/null
2020
)
2121
}
22+
23+
@test "Test kinds" {
24+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.kind')
25+
if [ "$val" != "TenantList" ]; then
26+
echo "Expected kind to be TenantList, got $val"
27+
exit 1
28+
fi
29+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/tenants | jq -r '.items[0].kind')
30+
if [ "$val" != "Tenant" ]; then
31+
echo "Expected kind to be Tenant, got $val"
32+
exit 1
33+
fi
34+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.kind')
35+
if [ "$val" != "IngressList" ]; then
36+
echo "Expected kind to be IngressList, got $val"
37+
exit 1
38+
fi
39+
val=$(kubectl get --raw /apis/apps.cozystack.io/v1alpha1/ingresses | jq -r '.items[0].kind')
40+
if [ "$val" != "Ingress" ]; then
41+
echo "Expected kind to be Ingress, got $val"
42+
exit 1
43+
fi
44+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
apiserver.local.config/

‎packages/system/cozystack-api/Makefile‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,18 @@ NAMESPACE=cozy-system
44
include ../../../scripts/common-envs.mk
55
include ../../../scripts/package.mk
66

7+
run-local:
8+
openssl req -nodes -new -x509 -keyout /tmp/ca.key -out /tmp/ca.crt -subj "/CN=kube-ca"
9+
openssl req -out /tmp/client.csr -new -newkey rsa:2048 -nodes -keyout /tmp/client.key -subj "/C=US/ST=SomeState/L=L/OU=Dev/CN=development/O=system:masters"
10+
openssl x509 -req -days 365 -in /tmp/client.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/client.crt
11+
openssl req -out /tmp/apiserver.csr -new -newkey rsa:2048 -nodes -keyout /tmp/apiserver.key -subj "/CN=cozystack-api" -config cozystack-api-openssl.cnf
12+
openssl x509 -req -days 365 -in /tmp/apiserver.csr -CA /tmp/ca.crt -CAkey /tmp/ca.key -set_serial 01 -sha256 -out /tmp/apiserver.crt -extensions v3_req -extfile cozystack-api-openssl.cnf
13+
CGO_ENABLED=0 go build -o /tmp/cozystack-api ../../../cmd/cozystack-api/main.go
14+
/tmp/cozystack-api --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
15+
16+
debug:
17+
dlv debug ../../../cmd/cozystack-api/main.go -- --client-ca-file /tmp/ca.crt --tls-cert-file /tmp/apiserver.crt --tls-private-key-file /tmp/apiserver.key --secure-port 6443 --kubeconfig $(KUBECONFIG) --authorization-kubeconfig $(KUBECONFIG) --authentication-kubeconfig $(KUBECONFIG)
18+
719
image: image-cozystack-api
820

921
image-cozystack-api:
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[ req ]
2+
distinguished_name = req_distinguished_name
3+
req_extensions = v3_req
4+
prompt = no
5+
6+
[ req_distinguished_name ]
7+
CN = cozystack-api
8+
9+
[ v3_req ]
10+
subjectAltName = @alt_names
11+
12+
[ alt_names ]
13+
IP.1 = 127.0.0.1

‎pkg/apiserver/apiserver.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,16 @@ import (
3030
"k8s.io/apimachinery/pkg/runtime/serializer"
3131
"k8s.io/apiserver/pkg/registry/rest"
3232
genericapiserver "k8s.io/apiserver/pkg/server"
33+
"k8s.io/klog/v2"
3334
ctrl "sigs.k8s.io/controller-runtime"
3435
"sigs.k8s.io/controller-runtime/pkg/cache"
3536
"sigs.k8s.io/controller-runtime/pkg/client"
37+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3638

3739
"github.com/cozystack/cozystack/pkg/apis/apps"
3840
appsinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
39-
coreinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
4041
"github.com/cozystack/cozystack/pkg/apis/core"
42+
coreinstall "github.com/cozystack/cozystack/pkg/apis/core/install"
4143
"github.com/cozystack/cozystack/pkg/config"
4244
cozyregistry "github.com/cozystack/cozystack/pkg/registry"
4345
applicationstorage "github.com/cozystack/cozystack/pkg/registry/apps/application"
@@ -48,7 +50,8 @@ import (
4850

4951
var (
5052
// Scheme defines methods for serializing and deserializing API objects.
51-
Scheme = runtime.NewScheme()
53+
Scheme = runtime.NewScheme()
54+
mgrScheme = runtime.NewScheme()
5255
// Codecs provides methods for retrieving codecs and serializers for specific
5356
// versions and content types.
5457
Codecs = serializer.NewCodecFactory(Scheme)
@@ -57,18 +60,23 @@ var (
5760
)
5861

5962
func init() {
63+
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{
64+
Development: true,
65+
// any other zap.Options tweaks
66+
})))
67+
klog.SetLogger(ctrl.Log.WithName("klog"))
6068
appsinstall.Install(Scheme)
6169
coreinstall.Install(Scheme)
6270

6371
// Register HelmRelease types.
64-
if err := helmv2.AddToScheme(Scheme); err != nil {
72+
if err := helmv2.AddToScheme(mgrScheme); err != nil {
6573
panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
6674
}
6775

68-
if err := corev1.AddToScheme(Scheme); err != nil {
76+
if err := corev1.AddToScheme(mgrScheme); err != nil {
6977
panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
7078
}
71-
if err := rbacv1.AddToScheme(Scheme); err != nil {
79+
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
7280
panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
7381
}
7482
// Add unversioned types.
@@ -134,7 +142,7 @@ func (c completedConfig) New() (*CozyServer, error) {
134142
}
135143

136144
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
137-
Scheme: Scheme,
145+
Scheme: mgrScheme,
138146
Cache: cache.Options{SyncPeriod: &syncPeriod},
139147
})
140148
if err != nil {
@@ -164,7 +172,7 @@ func (c completedConfig) New() (*CozyServer, error) {
164172
}
165173

166174
cli := mgr.GetClient()
167-
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: Scheme})
175+
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: mgrScheme})
168176
if err != nil {
169177
return nil, fmt.Errorf("failed to build watch client: %w", err)
170178
}

‎pkg/cmd/server/start.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ import (
4242
genericoptions "k8s.io/apiserver/pkg/server/options"
4343
utilfeature "k8s.io/apiserver/pkg/util/feature"
4444
utilversionpkg "k8s.io/apiserver/pkg/util/version"
45-
"k8s.io/client-go/tools/clientcmd"
4645
"k8s.io/component-base/featuregate"
4746
baseversion "k8s.io/component-base/version"
4847
netutils "k8s.io/utils/net"
4948
"sigs.k8s.io/controller-runtime/pkg/client"
49+
k8sconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
5050
)
5151

5252
// CozyServerOptions holds the state for the Cozy API server
@@ -150,7 +150,7 @@ func (o *CozyServerOptions) Complete() error {
150150
return fmt.Errorf("failed to register types: %w", err)
151151
}
152152

153-
cfg, err := clientcmd.BuildConfigFromFlags("", "")
153+
cfg, err := k8sconfig.GetConfig()
154154
if err != nil {
155155
return fmt.Errorf("failed to get kubeconfig: %w", err)
156156
}

‎pkg/registry/apps/application/rest.go‎

Lines changed: 62 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,17 @@ func (r *REST) GetSingularName() string {
142142
// Create handles the creation of a new Application by converting it to a HelmRelease
143143
func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
144144
// Assert the object is of type Application
145-
app, ok := obj.(*appsv1alpha1.Application)
145+
us, ok := obj.(*unstructured.Unstructured)
146146
if !ok {
147-
return nil, fmt.Errorf("expected Application object, got %T", obj)
147+
return nil, fmt.Errorf("expected unstructured.Unstructured object, got %T", obj)
148+
}
149+
150+
app := &appsv1alpha1.Application{}
151+
152+
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, app); err != nil {
153+
errMsg := fmt.Sprintf("returned unstructured.Unstructured object was not an Application")
154+
klog.Errorf(errMsg)
155+
return nil, fmt.Errorf(errMsg)
148156
}
149157

150158
// Convert Application to HelmRelease
@@ -389,11 +397,9 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
389397
}
390398

391399
// Explicitly set apiVersion and kind in unstructured object
392-
appList := &unstructured.UnstructuredList{}
393-
appList.SetAPIVersion("apps.cozystack.io/v1alpha1")
394-
appList.SetKind(r.kindName + "List")
400+
appList := r.NewList().(*unstructured.Unstructured)
395401
appList.SetResourceVersion(hrList.GetResourceVersion())
396-
appList.Items = items
402+
appList.Object["items"] = items
397403

398404
klog.V(6).Infof("Successfully listed %d Application resources in namespace %s", len(items), namespace)
399405
return appList, nil
@@ -441,9 +447,16 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
441447
}
442448

443449
// Assert the new object is of type Application
444-
app, ok := newObj.(*appsv1alpha1.Application)
450+
us, ok := newObj.(*unstructured.Unstructured)
445451
if !ok {
446-
errMsg := fmt.Sprintf("expected Application object, got %T", newObj)
452+
errMsg := fmt.Sprintf("expected unstructured.Unstructured object, got %T", newObj)
453+
klog.Errorf(errMsg)
454+
return nil, false, fmt.Errorf(errMsg)
455+
}
456+
app := &appsv1alpha1.Application{}
457+
458+
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, app); err != nil {
459+
errMsg := fmt.Sprintf("returned unstructured.Unstructured object was not an Application")
447460
klog.Errorf(errMsg)
448461
return nil, false, fmt.Errorf(errMsg)
449462
}
@@ -516,14 +529,12 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
516529
klog.Errorf("Failed to convert Application to unstructured for resource %s: %v", convertedApp.GetName(), err)
517530
return nil, false, fmt.Errorf("failed to convert Application to unstructured: %v", err)
518531
}
519-
520-
// Explicitly set apiVersion and kind in unstructured object
521-
unstructuredApp["apiVersion"] = "apps.cozystack.io/v1alpha1"
522-
unstructuredApp["kind"] = r.kindName
532+
obj := &unstructured.Unstructured{Object: unstructuredApp}
533+
obj.SetGroupVersionKind(r.gvk)
523534

524535
klog.V(6).Infof("Returning patched Application object: %+v", unstructuredApp)
525536

526-
return &unstructured.Unstructured{Object: unstructuredApp}, false, nil
537+
return obj, false, nil
527538
}
528539

529540
// Delete removes an Application by deleting the corresponding HelmRelease
@@ -723,11 +734,13 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio
723734
klog.Errorf("Failed to convert Application to unstructured: %v", err)
724735
continue
725736
}
737+
obj := &unstructured.Unstructured{Object: unstructuredApp}
738+
obj.SetGroupVersionKind(r.gvk)
726739

727740
// Create watch event with Application object
728741
appEvent := watch.Event{
729742
Type: event.Type,
730-
Object: &unstructured.Unstructured{Object: unstructuredApp},
743+
Object: obj,
731744
}
732745

733746
// Send event to custom watcher
@@ -1060,6 +1073,34 @@ func (r *REST) ConvertToTable(ctx context.Context, object runtime.Object, tableO
10601073
table = r.buildTableFromApplications(apps)
10611074
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
10621075
case *unstructured.Unstructured:
1076+
var apps []appsv1alpha1.Application
1077+
for {
1078+
var items interface{}
1079+
var ok bool
1080+
var objects []unstructured.Unstructured
1081+
if items, ok = obj.Object["items"]; !ok {
1082+
break
1083+
}
1084+
if objects, ok = items.([]unstructured.Unstructured); !ok {
1085+
break
1086+
}
1087+
apps = make([]appsv1alpha1.Application, 0, len(objects))
1088+
var a appsv1alpha1.Application
1089+
for i := range objects {
1090+
err := runtime.DefaultUnstructuredConverter.FromUnstructured(objects[i].Object, &a)
1091+
if err != nil {
1092+
klog.Errorf("Failed to convert Unstructured to Application: %v", err)
1093+
continue
1094+
}
1095+
apps = append(apps, a)
1096+
}
1097+
break
1098+
}
1099+
if apps != nil {
1100+
table = r.buildTableFromApplications(apps)
1101+
table.ListMeta.ResourceVersion = obj.GetResourceVersion()
1102+
break
1103+
}
10631104
var app appsv1alpha1.Application
10641105
err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &app)
10651106
if err != nil {
@@ -1196,12 +1237,17 @@ func (r *REST) Destroy() {
11961237

11971238
// New creates a new instance of Application
11981239
func (r *REST) New() runtime.Object {
1199-
return &appsv1alpha1.Application{}
1240+
obj := &unstructured.Unstructured{}
1241+
obj.SetGroupVersionKind(r.gvk)
1242+
return obj
12001243
}
12011244

12021245
// NewList returns an empty list of Application objects
12031246
func (r *REST) NewList() runtime.Object {
1204-
return &appsv1alpha1.ApplicationList{}
1247+
obj := &unstructured.Unstructured{}
1248+
obj.SetGroupVersionKind(r.gvk.GroupVersion().WithKind(r.kindName + "List"))
1249+
obj.Object["items"] = make([]interface{}, 0)
1250+
return obj
12051251
}
12061252

12071253
// Kind returns the resource kind used for API discovery

0 commit comments

Comments
 (0)