apache · thelabdude · Oct 29, 2021 · Oct 26, 2021 · Oct 26, 2021 · Oct 26, 2021
diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go
@@ -1459,4 +1459,10 @@ type SolrSecurityOptions struct {
 	// endpoints with credentials sourced from an env var instead of HTTP directly.
 	// +optional
 	ProbesRequireAuth bool `json:"probesRequireAuth,omitempty"`
+
+	// Configure a user-provided security.json from a secret to allow for advanced security config.
+	// If not specified, the operator bootstraps a security.json with basic auth enabled.
+	// This is a bootstrapping config only; once Solr is initialized, the security config should be managed by the security API.
+	// +optional
+	BootstrapSecurityJson *corev1.SecretKeySelector `json:"bootstrapSecurityJson,omitempty"`
 }
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/solr.apache.org_solrclouds.yaml b/config/crd/bases/solr.apache.org_solrclouds.yaml
@@ -5895,6 +5895,21 @@ spec:
                   basicAuthSecret:
                     description: "Secret (kubernetes.io/basic-auth) containing credentials the operator should use for API requests to secure Solr pods. If you provide this secret, then the operator assumes you've also configured your own security.json file and uploaded it to Solr. If you change the password for this user using the Solr security API, then you *must* update the secret with the new password or the operator will be  locked out of Solr and API requests will fail, ultimately causing a CrashBackoffLoop for all pods if probe endpoints are secured (see 'probesRequireAuth' setting). \n If you don't supply this secret, then the operator creates a kubernetes.io/basic-auth secret containing the password for the \"k8s-oper\" user. All API requests from the operator are made as the \"k8s-oper\" user, which is configured with read-only access to a minimal set of endpoints. In addition, the operator bootstraps a default security.json file and credentials for two additional users: admin and solr. The 'solr' user has basic read access to Solr resources. Once the security.json is bootstrapped, the operator will not update it! You're expected to use the 'admin' user to access the Security API to make further changes. It's strictly a bootstrapping operation."
                     type: string
+                  bootstrapSecurityJson:
+                    description: Configure a user-provided security.json from a secret to allow for advanced security config. If not specified, the operator bootstraps a security.json with basic auth enabled. This is a bootstrapping config only; once Solr is initialized, the security config should be managed by the security API.
+                    properties:
+                      key:
+                        description: The key of the secret to select from.  Must be a valid secret key.
+                        type: string
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                        type: string
+                      optional:
+                        description: Specify whether the Secret or its key must be defined
+                        type: boolean
+                    required:
+                    - key
+                    type: object
                   probesRequireAuth:
                     description: Flag to indicate if the configured HTTP endpoint(s) used for the probes require authentication; defaults to false. If you set to true, then probes will use a local command on the main container to hit the secured endpoints with credentials sourced from an env var instead of HTTP directly.
                     type: boolean

diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go
@@ -26,7 +26,6 @@ import (
 	"github.com/apache/solr-operator/controllers/util"
 	"github.com/go-logr/logr"
 	batchv1 "k8s.io/api/batch/v1"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -160,13 +159,12 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac
 		return nil, collectionBackupsFinished, actionTaken, err
 	}
 
-	var httpHeaders map[string]string
+	// Add any additional values needed to Authn to Solr to the Context used when invoking the API
 	if solrCloud.Spec.SolrSecurity != nil {
-		basicAuthSecret := &corev1.Secret{}
-		if err := r.Get(ctx, types.NamespacedName{Name: solrCloud.BasicAuthSecretName(), Namespace: solrCloud.Namespace}, basicAuthSecret); err != nil {
+		ctx, err = util.AddAuthToContext(ctx, &r.Client, solrCloud.Spec.SolrSecurity, solrCloud.Namespace)
+		if err != nil {
 			return nil, collectionBackupsFinished, actionTaken, err
 		}
-		httpHeaders = map[string]string{"Authorization": util.BasicAuthHeader(basicAuthSecret)}
 	}
 
 	// First check if the collection backups have been completed
@@ -208,7 +206,7 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac
 
 	// Go through each collection specified and reconcile the backup.
 	for _, collection := range backup.Spec.Collections {
-		_, err = reconcileSolrCollectionBackup(backup, solrCloud, backupRepository, collection, httpHeaders, logger)
+		_, err = reconcileSolrCollectionBackup(ctx, backup, solrCloud, backupRepository, collection, logger)
 	}
 
 	// First check if the collection backups have been completed
@@ -217,7 +215,7 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac
 	return solrCloud, collectionBackupsFinished, actionTaken, err
 }
 
-func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *solrv1beta1.SolrCloud, backupRepository *solrv1beta1.SolrBackupRepository, collection string, httpHeaders map[string]string, logger logr.Logger) (finished bool, err error) {
+func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, solrCloud *solrv1beta1.SolrCloud, backupRepository *solrv1beta1.SolrBackupRepository, collection string, logger logr.Logger) (finished bool, err error) {
 	now := metav1.Now()
 	collectionBackupStatus := solrv1beta1.CollectionBackupStatus{}
 	collectionBackupStatus.Collection = collection
@@ -233,7 +231,7 @@ func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *so
 	// If the collection backup hasn't started, start it
 	if !collectionBackupStatus.InProgress && !collectionBackupStatus.Finished {
 		// Start the backup by calling solr
-		started, err := util.StartBackupForCollection(solrCloud, backupRepository, backup, collection, httpHeaders, logger)
+		started, err := util.StartBackupForCollection(ctx, solrCloud, backupRepository, backup, collection, logger)
 		if err != nil {
 			return true, err
 		}
@@ -244,7 +242,7 @@ func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *so
 		collectionBackupStatus.BackupName = util.FullCollectionBackupName(collection, backup.Name)
 	} else if collectionBackupStatus.InProgress {
 		// Check the state of the backup, when it is in progress, and update the state accordingly
-		finished, successful, asyncStatus, err := util.CheckBackupForCollection(solrCloud, collection, backup.Name, httpHeaders, logger)
+		finished, successful, asyncStatus, err := util.CheckBackupForCollection(ctx, solrCloud, collection, backup.Name, logger)
 		if err != nil {
 			return false, err
 		}
@@ -259,7 +257,7 @@ func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *so
 				collectionBackupStatus.FinishTime = &now
 			}
 
-			err = util.DeleteAsyncInfoForBackup(solrCloud, collection, backup.Name, httpHeaders, logger)
+			err = util.DeleteAsyncInfoForBackup(ctx, solrCloud, collection, backup.Name, logger)
 		} else {
 			collectionBackupStatus.AsyncBackupStatus = asyncStatus
 		}

diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go
@@ -408,14 +408,17 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		}
 
 		// If authn enabled on Solr, we need to pass the auth header
-		var authHeader map[string]string
 		if security != nil {
-			authHeader = security.AuthHeader()
+			ctx, err = security.AddAuthToContext(ctx)
+			if err != nil {
+				updateLogger.Error(err, "failed to create Authorization header when reconciling", "SolrCloud", instance.Name)
+				return requeueOrNot, err
+			}
 		}
 
 		// Pick which pods should be deleted for an update.
 		// Don't exit on an error, which would only occur because of an HTTP Exception. Requeue later instead.
-		additionalPodsToUpdate, retryLater := util.DeterminePodsSafeToUpdate(instance, outOfDatePods, int(newStatus.ReadyReplicas), availableUpdatedPodCount, len(outOfDatePodsNotStarted), updateLogger, authHeader)
+		additionalPodsToUpdate, retryLater := util.DeterminePodsSafeToUpdate(ctx, instance, outOfDatePods, int(newStatus.ReadyReplicas), availableUpdatedPodCount, len(outOfDatePodsNotStarted), updateLogger)
 		podsToUpdate = append(podsToUpdate, additionalPodsToUpdate...)
 
 		for _, pod := range podsToUpdate {

diff --git a/controllers/solrcloud_controller_basic_auth_test.go b/controllers/solrcloud_controller_basic_auth_test.go
@@ -27,6 +27,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 var _ = FDescribe("SolrCloud controller - Basic Auth", func() {
@@ -82,6 +83,34 @@ var _ = FDescribe("SolrCloud controller - Basic Auth", func() {
 		})
 	})
 
+	FContext("Boostrap Security JSON with Custom Probe Paths", func() {
+		BeforeEach(func() {
+			customHandler := corev1.Handler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Scheme: corev1.URISchemeHTTP,
+					Path:   "/solr/readyz",
+					Port:   intstr.FromInt(8983),
+				},
+			}
+
+			// verify users can vary the probe path and the secure probe exec command uses them
+			solrCloud.Spec.CustomSolrKubeOptions = solrv1beta1.CustomSolrKubeOptions{
+				PodOptions: &solrv1beta1.PodOptions{
+					LivenessProbe:  &corev1.Probe{Handler: customHandler},
+					ReadinessProbe: &corev1.Probe{Handler: customHandler},
+				},
+			}
+
+			solrCloud.Spec.SolrSecurity = &solrv1beta1.SolrSecurityOptions{
+				AuthenticationType: solrv1beta1.Basic,
+				ProbesRequireAuth:  true,
+			}
+		})
+		FIt("has the correct resources", func() {
+			expectStatefulSetBasicAuthConfig(ctx, solrCloud, true)
+		})
+	})
+
 	FContext("Boostrap Security JSON with ZK ACLs", func() {
 		BeforeEach(func() {
 			solrCloud.Spec.SolrSecurity = &solrv1beta1.SolrSecurityOptions{
@@ -144,6 +173,34 @@ var _ = FDescribe("SolrCloud controller - Basic Auth", func() {
 			expectStatefulSetBasicAuthConfig(ctx, solrCloud, false)
 		})
 	})
+
+	FContext("User Provided Credentials and security.json secret", func() {
+		BeforeEach(func() {
+			basicAuthSecretName := "my-basic-auth-secret"
+			solrCloud.Spec.SolrSecurity = &solrv1beta1.SolrSecurityOptions{
+				AuthenticationType: solrv1beta1.Basic,
+				BasicAuthSecret:    basicAuthSecretName,
+				BootstrapSecurityJson: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{Name: "my-security-json"},
+					Key:                  util.SecurityJsonFile,
+				},
+			}
+		})
+		FIt("has the correct resources", func() {
+			By("Making sure that no statefulSet exists until the BasicAuth Secret is created")
+			expectNoStatefulSet(ctx, solrCloud, solrCloud.StatefulSetName())
+
+			By("Create the basicAuth secret")
+			basicAuthSecret := createBasicAuthSecret(solrCloud.Spec.SolrSecurity.BasicAuthSecret, solrv1beta1.DefaultBasicAuthUsername, solrCloud.Namespace)
+			Expect(k8sClient.Create(ctx, basicAuthSecret)).To(Succeed(), "Could not create the necessary basicAuth secret")
+
+			By("Create the security.json Secret")
+			createMockSecurityJsonSecret(ctx, "my-security-json", solrCloud.Namespace)
+
+			By("Make sure the StatefulSet is created and configured correctly")
+			expectStatefulSetBasicAuthConfig(ctx, solrCloud, false)
+		})
+	})
 })
 
 var boostrapedSecretKeys = []string{
@@ -155,8 +212,13 @@ var boostrapedSecretKeys = []string{
 func expectStatefulSetBasicAuthConfig(ctx context.Context, sc *solrv1beta1.SolrCloud, expectBootstrapSecret bool) *appsv1.StatefulSet {
 	Expect(sc.Spec.SolrSecurity).To(Not(BeNil()), "solrSecurity is not configured for this SolrCloud instance!")
 
+	expProbePath := "/solr/admin/info/system"
+	if sc.Spec.CustomSolrKubeOptions.PodOptions != nil && sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe != nil {
+		expProbePath = sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
+	}
+
 	stateful := expectStatefulSetWithChecks(ctx, sc, sc.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
-		expectBasicAuthConfigOnPodTemplateWithGomega(g, sc, &found.Spec.Template, expectBootstrapSecret)
+		expectBasicAuthConfigOnPodTemplateWithGomega(g, sc, &found.Spec.Template, expectBootstrapSecret, expProbePath)
 	})
 
 	expectSecretWithChecks(ctx, sc, sc.BasicAuthSecretName(), func(innerG Gomega, found *corev1.Secret) {
@@ -192,12 +254,7 @@ func expectStatefulSetBasicAuthConfig(ctx context.Context, sc *solrv1beta1.SolrC
 }
 
 // Ensures config is setup for basic-auth enabled Solr pods
-func expectBasicAuthConfigOnPodTemplate(solrCloud *solrv1beta1.SolrCloud, podTemplate *corev1.PodTemplateSpec, expectBootstrapSecret bool) *corev1.Container {
-	return expectBasicAuthConfigOnPodTemplateWithGomega(Default, solrCloud, podTemplate, expectBootstrapSecret)
-}
-
-// Ensures config is setup for basic-auth enabled Solr pods
-func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1beta1.SolrCloud, podTemplate *corev1.PodTemplateSpec, expectBootstrapSecret bool) *corev1.Container {
+func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1beta1.SolrCloud, podTemplate *corev1.PodTemplateSpec, expectBootstrapSecret bool, expProbePath string) *corev1.Container {
 	// check the env vars needed for the probes to work with auth
 	g.Expect(podTemplate.Spec.Containers).To(Not(BeEmpty()), "Solr Pod requires containers")
 	mainContainer := podTemplate.Spec.Containers[0]
@@ -233,8 +290,8 @@ func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1bet
 			"-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory "+
 			"-Dsolr.install.dir=\"/opt/solr\" -Dlog4j.configurationFile=\"/opt/solr/server/resources/log4j2-console.xml\" "+
 			"-classpath \"/opt/solr/server/solr-webapp/webapp/WEB-INF/lib/*:/opt/solr/server/lib/ext/*:/opt/solr/server/lib/*\" "+
-			"org.apache.solr.util.SolrCLI api -get http://localhost:8983/solr/admin/info/system",
-			solrCloud.Name, solrCloud.Name)
+			"org.apache.solr.util.SolrCLI api -get http://localhost:8983%s",
+			solrCloud.Name, solrCloud.Name, expProbePath)
 		g.Expect(mainContainer.LivenessProbe).To(Not(BeNil()), "main container should have a liveness probe defined")
 		g.Expect(mainContainer.LivenessProbe.Exec).To(Not(BeNil()), "liveness probe should have an exec when auth is enabled")
 		g.Expect(mainContainer.LivenessProbe.Exec.Command).To(Not(BeEmpty()), "liveness probe command cannot be empty")
@@ -248,7 +305,7 @@ func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1bet
 	}
 
 	// if no user-provided auth secret, then check that security.json gets bootstrapped correctly
-	if solrCloud.Spec.SolrSecurity.BasicAuthSecret == "" {
+	if solrCloud.Spec.SolrSecurity.BasicAuthSecret == "" || solrCloud.Spec.SolrSecurity.BootstrapSecurityJson != nil {
 		// initContainers
 		g.Expect(podTemplate.Spec.InitContainers).To(Not(BeEmpty()), "The Solr Pod template requires an init container to bootstrap the security.json")
 		var expInitContainer *corev1.Container = nil
@@ -259,7 +316,7 @@ func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1bet
 			}
 		}
 
-		if expectBootstrapSecret {
+		if expectBootstrapSecret || solrCloud.Spec.SolrSecurity.BootstrapSecurityJson != nil {
 			// if the zookeeperRef has ACLs set, verify the env vars were set correctly for this initContainer
 			allACL, _ := solrCloud.Spec.ZookeeperRef.GetACLs()
 			if allACL != nil {
@@ -269,12 +326,7 @@ func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1bet
 				testACLEnvVarsWithGomega(g, expInitContainer.Env[3:len(expInitContainer.Env)-2], true)
 			} // else this ref not using ACLs
 
-			g.Expect(expInitContainer).To(Not(BeNil()), "Didn't find the setup-zk InitContainer in the sts!")
-			expCmd := "ZK_SECURITY_JSON=$(/opt/solr/server/scripts/cloud-scripts/zkcli.sh -zkhost ${ZK_HOST} -cmd get /security.json); " +
-				"if [ ${#ZK_SECURITY_JSON} -lt 3 ]; then " +
-				"echo $SECURITY_JSON > /tmp/security.json; " +
-				"/opt/solr/server/scripts/cloud-scripts/zkcli.sh -zkhost ${ZK_HOST} -cmd putfile /security.json /tmp/security.json; echo \"put security.json in ZK\"; fi"
-			g.Expect(expInitContainer.Command[2]).To(ContainSubstring(expCmd), "setup-zk initContainer not configured to bootstrap security.json!")
+			expectPutSecurityJsonInZkCmd(g, expInitContainer)
 		} else {
 			g.Expect(expInitContainer).To(Or(
 				BeNil(),
@@ -292,3 +344,20 @@ func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1bet
 
 	return &mainContainer // return as a convenience in case tests want to do more checking on the main container
 }
+
+func expectPutSecurityJsonInZkCmd(g Gomega, expInitContainer *corev1.Container) {
+	g.Expect(expInitContainer).To(Not(BeNil()), "Didn't find the setup-zk InitContainer in the sts!")
+	expCmd := "ZK_SECURITY_JSON=$(/opt/solr/server/scripts/cloud-scripts/zkcli.sh -zkhost ${ZK_HOST} -cmd get /security.json); " +
+		"if [ ${#ZK_SECURITY_JSON} -lt 3 ]; then " +
+		"echo $SECURITY_JSON > /tmp/security.json; " +
+		"/opt/solr/server/scripts/cloud-scripts/zkcli.sh -zkhost ${ZK_HOST} -cmd putfile /security.json /tmp/security.json; echo \"put security.json in ZK\"; fi"
+	g.Expect(expInitContainer.Command[2]).To(ContainSubstring(expCmd), "setup-zk initContainer not configured to bootstrap security.json!")
+}
+
+func createMockSecurityJsonSecret(ctx context.Context, name string, ns string) corev1.Secret {
+	secData := map[string]string{}
+	secData[util.SecurityJsonFile] = "{}"
+	sec := corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, StringData: secData}
+	Expect(k8sClient.Create(ctx, &sec)).To(Succeed(), "Could not create mock security.json secret")
+	return sec
+}