Skip to content

Commit d558ef7

Browse files
lokeshranginenilokeshrangineni
authored
feat: Operator E2E test to validate FeatureStore custom resource using remote registry (#4822)
* Added new e2e test case to do the remote registry deployment. abstracted the code to do the featurestore testing. Signed-off-by: lrangine <[email protected]> * abstracted the code even further. Now the custom resource CR will execute only once for all the test cases. Signed-off-by: lrangine <[email protected]> * fixing the operator e2e test. this was commented before by mistake. Signed-off-by: lrangine <[email protected]> * increasing the remote registry deployment timeout to see if it solves the github CI timeout issue. Signed-off-by: lrangine <[email protected]> * Trying to increase the timeout and also adding the debugging actions when there is a failure. Signed-off-by: lrangine <[email protected]> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <[email protected]> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <[email protected]> * increased the go test timeout to 30m to fix the github operator e2e test action. Signed-off-by: lrangine <[email protected]> * incorporating the code review comments. Signed-off-by: lrangine <[email protected]> * incorporating the code review comments by using the feastRef. Signed-off-by: lrangine <[email protected]> * incorporating the code review comments by marshaling json to the go struct object rather than map. Signed-off-by: lrangine <[email protected]> * Fixing the lint error. Signed-off-by: lrangine <[email protected]> --------- Signed-off-by: lrangine <[email protected]>
1 parent 3d6bf42 commit d558ef7

5 files changed

Lines changed: 230 additions & 116 deletions

File tree

‎.github/workflows/operator-e2e-integration-tests.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ on:
1313

1414
jobs:
1515
operator-e2e-tests:
16+
timeout-minutes: 40
1617
if:
1718
((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
1819
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
@@ -38,7 +39,7 @@ jobs:
3839

3940
- name: Create KIND cluster
4041
run: |
41-
kind create cluster --name $KIND_CLUSTER --wait 5m
42+
kind create cluster --name $KIND_CLUSTER --wait 10m
4243
4344
- name: Set up kubernetes context
4445
run: |
@@ -51,8 +52,16 @@ jobs:
5152
cd infra/feast-operator/
5253
make test-e2e
5354
55+
- name: Debug KIND Cluster when there is a failure
56+
if: failure()
57+
run: |
58+
kubectl get pods --all-namespaces
59+
kubectl describe nodes
60+
5461
- name: Clean up
5562
if: always()
5663
run: |
5764
# Delete the KIND cluster after tests
5865
kind delete cluster --name kind-$KIND_CLUSTER
66+
67+

‎infra/feast-operator/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ test: build-installer fmt vet lint envtest ## Run tests.
117117
# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
118118
.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up.
119119
test-e2e:
120-
go test ./test/e2e/ -v -ginkgo.v
120+
go test -timeout 30m ./test/e2e/ -v -ginkgo.v
121121

122122
.PHONY: lint
123123
lint: golangci-lint ## Run golangci-lint linter & yamllint

‎infra/feast-operator/test/e2e/e2e_test.go‎

Lines changed: 157 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -28,145 +28,188 @@ import (
2828
)
2929

3030
const feastControllerNamespace = "feast-operator-system"
31+
const timeout = 2 * time.Minute
32+
const controllerDeploymentName = "feast-operator-controller-manager"
3133

3234
var _ = Describe("controller", Ordered, func() {
3335
BeforeAll(func() {
3436
By("creating manager namespace")
3537
cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace)
3638
_, _ = utils.Run(cmd)
39+
var err error
40+
// projectimage stores the name of the image used in the example
41+
var projectimage = "localhost/feast-operator:v0.0.1"
42+
43+
By("building the manager(Operator) image")
44+
cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
45+
_, err = utils.Run(cmd)
46+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
47+
48+
By("loading the the manager(Operator) image on Kind")
49+
err = utils.LoadImageToKindClusterWithName(projectimage)
50+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
51+
52+
By("building the feast image")
53+
cmd = exec.Command("make", "feast-ci-dev-docker-img")
54+
_, err = utils.Run(cmd)
55+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
56+
// this image will be built in above make target.
57+
var feastImage = "feastdev/feature-server:dev"
58+
var feastLocalImage = "localhost/feastdev/feature-server:dev"
59+
60+
By("Tag the local feast image for the integration tests")
61+
cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage)
62+
_, err = utils.Run(cmd)
63+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
64+
65+
By("loading the the feast image on Kind cluster")
66+
err = utils.LoadImageToKindClusterWithName(feastLocalImage)
67+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
68+
69+
By("installing CRDs")
70+
cmd = exec.Command("make", "install")
71+
_, err = utils.Run(cmd)
72+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
73+
74+
By("deploying the controller-manager")
75+
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
76+
_, err = utils.Run(cmd)
77+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
78+
79+
By("Validating that the controller-manager deployment is in available state")
80+
err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout)
81+
Expect(err).To(BeNil(), fmt.Sprintf(
82+
"Deployment %s is not available but expected to be available. \nError: %v\n",
83+
controllerDeploymentName, err,
84+
))
85+
fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName)
3786
})
3887

3988
AfterAll(func() {
4089
//Add any post clean up code here.
90+
By("Uninstalling the feast CRD")
91+
cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace)
92+
_, err := utils.Run(cmd)
93+
ExpectWithOffset(1, err).NotTo(HaveOccurred())
4194
})
4295

43-
Context("Operator", func() {
96+
Context("Operator E2E Tests", func() {
4497
It("Should be able to deploy and run a default feature store CR successfully", func() {
45-
//var controllerPodName string
46-
var err error
47-
48-
// projectimage stores the name of the image used in the example
49-
var projectimage = "localhost/feast-operator:v0.0.1"
50-
51-
By("building the manager(Operator) image")
52-
cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
53-
_, err = utils.Run(cmd)
54-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
55-
56-
By("loading the the manager(Operator) image on Kind")
57-
err = utils.LoadImageToKindClusterWithName(projectimage)
58-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
59-
60-
By("building the feast image")
61-
cmd = exec.Command("make", "feast-ci-dev-docker-img")
62-
_, err = utils.Run(cmd)
63-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
64-
// this image will be built in above make target.
65-
var feastImage = "feastdev/feature-server:dev"
66-
var feastLocalImage = "localhost/feastdev/feature-server:dev"
67-
68-
By("Tag the local feast image for the integration tests")
69-
cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage)
70-
_, err = utils.Run(cmd)
71-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
72-
73-
By("loading the the feast image on Kind cluster")
74-
err = utils.LoadImageToKindClusterWithName(feastLocalImage)
75-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
76-
77-
By("installing CRDs")
78-
cmd = exec.Command("make", "install")
79-
_, err = utils.Run(cmd)
80-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
81-
82-
By("deploying the controller-manager")
83-
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
84-
_, err = utils.Run(cmd)
85-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
86-
87-
timeout := 2 * time.Minute
88-
89-
controllerDeploymentName := "feast-operator-controller-manager"
90-
By("Validating that the controller-manager deployment is in available state")
91-
err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout)
92-
Expect(err).To(BeNil(), fmt.Sprintf(
93-
"Deployment %s is not available but expected to be available. \nError: %v\n",
94-
controllerDeploymentName, err,
95-
))
96-
fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName)
97-
9898
By("deploying the Simple Feast Custom Resource to Kubernetes")
99-
cmd = exec.Command("kubectl", "apply", "-f",
100-
"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml")
99+
namespace := "default"
100+
cmd := exec.Command("kubectl", "apply", "-f",
101+
"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
101102
_, cmdOutputerr := utils.Run(cmd)
102103
ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
103104

104-
namespace := "default"
105-
106-
deploymentNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online",
107-
"feast-simple-feast-setup-offline"}
108-
for _, deploymentName := range deploymentNames {
109-
By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName))
110-
err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout)
111-
Expect(err).To(BeNil(), fmt.Sprintf(
112-
"Deployment %s is not available but expected to be available. \nError: %v\n",
113-
deploymentName, err,
114-
))
115-
fmt.Printf("Feast Deployment %s is available\n", deploymentName)
116-
}
117-
118-
By("Check if the feast client - kubernetes config map exists.")
119-
configMapName := "feast-simple-feast-setup-client"
120-
err = checkIfConfigMapExists(namespace, configMapName)
121-
Expect(err).To(BeNil(), fmt.Sprintf(
122-
"config map %s is not available but expected to be available. \nError: %v\n",
123-
configMapName, err,
124-
))
125-
fmt.Printf("Feast Deployment %s is available\n", configMapName)
126-
127-
serviceAccountNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online",
128-
"feast-simple-feast-setup-offline"}
129-
for _, serviceAccountName := range serviceAccountNames {
130-
By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName))
131-
err = checkIfServiceAccountExists(namespace, serviceAccountName)
132-
Expect(err).To(BeNil(), fmt.Sprintf(
133-
"Service account %s does not exist in namespace %s. Error: %v",
134-
serviceAccountName, namespace, err,
135-
))
136-
fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace)
137-
}
138-
139-
serviceNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online",
140-
"feast-simple-feast-setup-offline"}
141-
for _, serviceName := range serviceNames {
142-
By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName))
143-
err = checkIfKubernetesServiceExists(namespace, serviceName)
144-
Expect(err).To(BeNil(), fmt.Sprintf(
145-
"kubernetes service %s is not available but expected to be available. \nError: %v\n",
146-
serviceName, err,
147-
))
148-
fmt.Printf("kubernetes service %s is available\n", serviceName)
149-
}
150-
151-
By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", "simple-feast-setup"))
152-
err = checkIfFeatureStoreCustomResourceConditionsInReady("simple-feast-setup", namespace)
153-
Expect(err).To(BeNil(), fmt.Sprintf(
154-
"FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n",
155-
"simple-feast-setup", err,
156-
))
157-
fmt.Printf("FeatureStore customer resource %s conditions are in Ready State\n", "simple-feast-setup")
105+
featureStoreName := "simple-feast-setup"
106+
validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout)
158107

159108
By("deleting the feast deployment")
160109
cmd = exec.Command("kubectl", "delete", "-f",
161110
"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml")
162111
_, cmdOutputerr = utils.Run(cmd)
163112
ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
113+
})
114+
115+
It("Should be able to deploy and run a feature store with remote registry CR successfully", func() {
116+
By("deploying the Simple Feast Custom Resource to Kubernetes")
117+
namespace := "default"
118+
cmd := exec.Command("kubectl", "apply", "-f",
119+
"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
120+
_, cmdOutputerr := utils.Run(cmd)
121+
ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
122+
123+
featureStoreName := "simple-feast-setup"
124+
validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout)
125+
126+
var remoteRegistryNs = "remote-registry"
127+
cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs)
128+
_, _ = utils.Run(cmd)
129+
130+
By("deploying the Simple Feast remote registry Custom Resource to Kubernetes")
131+
cmd = exec.Command("kubectl", "apply", "-f",
132+
"test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs)
133+
_, cmdOutputerr = utils.Run(cmd)
134+
ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
164135

165-
By("Uninstalling the feast CRD")
166-
cmd = exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace)
167-
_, err = utils.Run(cmd)
168-
ExpectWithOffset(1, err).NotTo(HaveOccurred())
136+
remoteFeatureStoreName := "simple-feast-remote-setup"
137+
138+
validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout)
139+
140+
By("deleting the feast remote registry deployment")
141+
cmd = exec.Command("kubectl", "delete", "-f",
142+
"test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs)
143+
_, cmdOutputerr = utils.Run(cmd)
144+
ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
169145

146+
By("deleting the feast deployment")
147+
cmd = exec.Command("kubectl", "delete", "-f",
148+
"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
149+
_, cmdOutputerr = utils.Run(cmd)
150+
ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
170151
})
171152
})
172153
})
154+
155+
func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) {
156+
hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName)
157+
Expect(err).To(BeNil(), fmt.Sprintf(
158+
"Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n",
159+
featureStoreName, err))
160+
161+
k8ResourceNames := []string{fmt.Sprintf("feast-%s-online", featureStoreName),
162+
fmt.Sprintf("feast-%s-offline", featureStoreName),
163+
}
164+
165+
if !hasRemoteRegistry {
166+
k8ResourceNames = append(k8ResourceNames, fmt.Sprintf("feast-%s-registry", featureStoreName))
167+
}
168+
169+
for _, deploymentName := range k8ResourceNames {
170+
By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName))
171+
err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout)
172+
Expect(err).To(BeNil(), fmt.Sprintf(
173+
"Deployment %s is not available but expected to be available. \nError: %v\n",
174+
deploymentName, err,
175+
))
176+
fmt.Printf("Feast Deployment %s is available\n", deploymentName)
177+
}
178+
179+
By("Check if the feast client - kubernetes config map exists.")
180+
configMapName := fmt.Sprintf("feast-%s-client", featureStoreName)
181+
err = checkIfConfigMapExists(namespace, configMapName)
182+
Expect(err).To(BeNil(), fmt.Sprintf(
183+
"config map %s is not available but expected to be available. \nError: %v\n",
184+
configMapName, err,
185+
))
186+
fmt.Printf("Feast Deployment client config map %s is available\n", configMapName)
187+
188+
for _, serviceAccountName := range k8ResourceNames {
189+
By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName))
190+
err = checkIfServiceAccountExists(namespace, serviceAccountName)
191+
Expect(err).To(BeNil(), fmt.Sprintf(
192+
"Service account %s does not exist in namespace %s. Error: %v",
193+
serviceAccountName, namespace, err,
194+
))
195+
fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace)
196+
}
197+
198+
for _, serviceName := range k8ResourceNames {
199+
By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName))
200+
err = checkIfKubernetesServiceExists(namespace, serviceName)
201+
Expect(err).To(BeNil(), fmt.Sprintf(
202+
"kubernetes service %s is not available but expected to be available. \nError: %v\n",
203+
serviceName, err,
204+
))
205+
fmt.Printf("kubernetes service %s is available\n", serviceName)
206+
}
207+
208+
By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName))
209+
err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace)
210+
Expect(err).To(BeNil(), fmt.Sprintf(
211+
"FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n",
212+
featureStoreName, err,
213+
))
214+
fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName)
215+
}

0 commit comments

Comments
 (0)