Enables TLS on Prometheus endpoint. (mongodb#897)

Author: Rodrigo Valin

Diff for: .dockerignore

@@ -4,5 +4,10 @@ zz_*
 vendor/
 scripts/
 .git/
+bin/
+testbin/
+.mypy_cache/
+main
+__debug_bin
 # allow agent LICENSE
 !scripts/dev/templates/agent/LICENSE

Diff for: api/v1/mongodbcommunity_types.go

@@ -146,6 +146,11 @@ type Prometheus struct {
 	// Indicates path to the metrics endpoint.
 	// +kubebuilder:validation:Pattern=^\/[a-z0-9]+$
 	MetricsPath string `json:"metricsPath,omitempty"`
+
+	// Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the
+	// Prometheus endpoint.
+	// +optional
+	TLSSecretRef SecretKeyReference `json:"tlsSecretKeyRef,omitempty"`
 }
 
 func (p Prometheus) GetPasswordKey() string {
@@ -708,12 +713,23 @@ func (m MongoDBCommunity) TLSSecretNamespacedName() types.NamespacedName {
 	return types.NamespacedName{Name: m.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: m.Namespace}
 }
 
+// PrometheusTLSSecretNamespacedName will get the namespaced name of the Secret containing the server certificate and key
+func (m MongoDBCommunity) PrometheusTLSSecretNamespacedName() types.NamespacedName {
+	return types.NamespacedName{Name: m.Spec.Prometheus.TLSSecretRef.Name, Namespace: m.Namespace}
+}
+
 // TLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator
 // containing the combined certificate and key.
 func (m MongoDBCommunity) TLSOperatorSecretNamespacedName() types.NamespacedName {
 	return types.NamespacedName{Name: m.Name + "-server-certificate-key", Namespace: m.Namespace}
 }
 
+// PrometheusTLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator
+// containing the combined certificate and key.
+func (m MongoDBCommunity) PrometheusTLSOperatorSecretNamespacedName() types.NamespacedName {
+	return types.NamespacedName{Name: m.Name + "-prometheus-certificate-key", Namespace: m.Namespace}
+}
+
 func (m MongoDBCommunity) NamespacedName() types.NamespacedName {
 	return types.NamespacedName{Name: m.Name, Namespace: m.Namespace}
 }

Diff for: config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml

@@ -113,6 +113,21 @@ spec:
                     description: Port where metrics endpoint will bind to. Defaults
                       to 9216.
                     type: integer
+                  tlsSecretKeyRef:
+                    description: Name of a Secret (type kubernetes.io/tls) holding
+                      the certificates to use in the Prometheus endpoint.
+                    properties:
+                      key:
+                        description: Key is the key in the secret storing this password.
+                          Defaults to "password"
+                        type: string
+                      name:
+                        description: Name is the name of the secret storing this user's
+                          password
+                        type: string
+                    required:
+                    - name
+                    type: object
                   username:
                     description: HTTP Basic Auth Username for metrics endpoint.
                     type: string

Diff for: controllers/mongodb_tls.go

@@ -82,7 +82,7 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo
 
 	// validate whether the secret contains "tls.crt" and "tls.key", or it contains "tls.pem"
 	// if it contains all three, then the pem entry should be equal to the concatenation of crt and key
-	_, err = getPemOrConcatenatedCrtAndKey(r.client, mdb)
+	_, err = getPemOrConcatenatedCrtAndKey(r.client, mdb, mdb.TLSSecretNamespacedName())
 	if err != nil {
 		r.log.Warnf(err.Error())
 		return false, nil
@@ -102,7 +102,7 @@ func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv
 		return automationconfig.NOOP(), nil
 	}
 
-	certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb)
+	certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.TLSSecretNamespacedName())
 	if err != nil {
 		return automationconfig.NOOP(), err
 	}
@@ -111,13 +111,13 @@ func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv
 }
 
 // getCertAndKey will fetch the certificate and key from the user-provided Secret.
-func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity) string {
-	cert, err := secret.ReadKey(getter, tlsSecretCertName, mdb.TLSSecretNamespacedName())
+func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string {
+	cert, err := secret.ReadKey(getter, tlsSecretCertName, secretName)
 	if err != nil {
 		return ""
 	}
 
-	key, err := secret.ReadKey(getter, tlsSecretKeyName, mdb.TLSSecretNamespacedName())
+	key, err := secret.ReadKey(getter, tlsSecretKeyName, secretName)
 	if err != nil {
 		return ""
 	}
@@ -126,8 +126,8 @@ func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity) string {
 }
 
 // getPem will fetch the pem from the user-provided secret
-func getPem(getter secret.Getter, mdb mdbv1.MongoDBCommunity) string {
-	pem, err := secret.ReadKey(getter, tlsSecretPemName, mdb.TLSSecretNamespacedName())
+func getPem(getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) string {
+	pem, err := secret.ReadKey(getter, tlsSecretPemName, secretName)
 	if err != nil {
 		return ""
 	}
@@ -144,9 +144,9 @@ func combineCertificateAndKey(cert, key string) string {
 // This is either the tls.pem entry in the given secret, or the concatenation
 // of tls.crt and tls.key
 // It performs a basic validation on the entries.
-func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) {
-	certKey := getCertAndKey(getter, mdb)
-	pem := getPem(getter, mdb)
+func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommunity, secretName types.NamespacedName) (string, error) {
+	certKey := getCertAndKey(getter, mdb, secretName)
+	pem := getPem(getter, mdb, secretName)
 	if certKey == "" && pem == "" {
 		return "", fmt.Errorf(`Neither "%s" nor the pair "%s"/"%s" were present in the TLS secret`, tlsSecretPemName, tlsSecretCertName, tlsSecretKeyName)
 	}
@@ -165,7 +165,7 @@ func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommun
 // ensureTLSSecret will create or update the operator-managed Secret containing
 // the concatenated certificate and key from the user-provided Secret.
 func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error {
-	certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb)
+	certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.TLSSecretNamespacedName())
 	if err != nil {
 		return err
 	}
@@ -182,6 +182,26 @@ func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDB
 	return secret.CreateOrUpdate(getUpdateCreator, operatorSecret)
 }
 
+// ensurePrometheusTLSSecret will create or update the operator-managed Secret containing
+// the concatenated certificate and key from the user-provided Secret.
+func ensurePrometheusTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error {
+	certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.DeepCopy().PrometheusTLSSecretNamespacedName())
+	if err != nil {
+		return err
+	}
+	// Calculate file name from certificate and key
+	fileName := tlsOperatorSecretFileName(certKey)
+
+	operatorSecret := secret.Builder().
+		SetName(mdb.PrometheusTLSOperatorSecretNamespacedName().Name).
+		SetNamespace(mdb.PrometheusTLSOperatorSecretNamespacedName().Namespace).
+		SetField(fileName, certKey).
+		SetOwnerReferences(mdb.GetOwnerReferences()).
+		Build()
+
+	return secret.CreateOrUpdate(getUpdateCreator, operatorSecret)
+}
+
 // tlsOperatorSecretFileName calculates the file name to use for the mounted
 // certificate-key file. The name is based on the hash of the combined cert and key.
 // If the certificate or key changes, the file path changes as well which will trigger
@@ -250,3 +270,27 @@ func buildTLSPodSpecModification(mdb mdbv1.MongoDBCommunity) podtemplatespec.Mod
 		podtemplatespec.WithVolumeMounts(construct.MongodbName, tlsSecretVolumeMount, caVolumeMount),
 	)
 }
+
+// buildTLSPrometheus adds the TLS mounts for Prometheus.
+func buildTLSPrometheus(mdb mdbv1.MongoDBCommunity) podtemplatespec.Modification {
+	if mdb.Spec.Prometheus == nil || mdb.Spec.Prometheus.TLSSecretRef.Name == "" {
+		return podtemplatespec.NOOP()
+	}
+
+	// Configure a volume which mounts the secret holding the server key and certificate
+	// The same key-certificate pair is used for all servers
+	tlsSecretVolume := statefulset.CreateVolumeFromSecret("prom-tls-secret", mdb.PrometheusTLSOperatorSecretNamespacedName().Name)
+
+	// TODO: Is it ok to use the same `tlsOperatorSecretMountPath`
+	tlsSecretVolumeMount := statefulset.CreateVolumeMount(tlsSecretVolume.Name, tlsOperatorSecretMountPath, statefulset.WithReadOnly(true))
+
+	// MongoDB expects both key and certificate to be provided in a single PEM file
+	// We are using a secret format where they are stored in separate fields, tls.crt and tls.key
+	// Because of this we need to use an init container which reads the two files mounted from the secret and combines them into one
+	return podtemplatespec.Apply(
+		// podtemplatespec.WithVolume(caVolume),
+		podtemplatespec.WithVolume(tlsSecretVolume),
+		podtemplatespec.WithVolumeMounts(construct.AgentName, tlsSecretVolumeMount),
+		podtemplatespec.WithVolumeMounts(construct.MongodbName, tlsSecretVolumeMount),
+	)
+}

Diff for: controllers/prometheus.go

@@ -0,0 +1,62 @@
+package controllers
+
+import (
+	"fmt"
+
+	mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1"
+	"github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig"
+	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret"
+	"github.com/pkg/errors"
+
+	"k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	listenAddress = "0.0.0.0"
+)
+
+// PrometheusModification adds Prometheus configuration to AutomationConfig.
+func getPrometheusModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) {
+	if mdb.Spec.Prometheus == nil {
+		return automationconfig.NOOP(), nil
+	}
+
+	secretNamespacedName := types.NamespacedName{Name: mdb.Spec.Prometheus.PasswordSecretRef.Name, Namespace: mdb.Namespace}
+	password, err := secret.ReadKey(getUpdateCreator, mdb.Spec.Prometheus.GetPasswordKey(), secretNamespacedName)
+	if err != nil {
+		return automationconfig.NOOP(), errors.Errorf("could not configure Prometheus modification: %s", err)
+	}
+
+	var certKey string
+	var tlsPEMPath string
+	var scheme string
+
+	if mdb.Spec.Prometheus.TLSSecretRef.Name != "" {
+		certKey, err = getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.PrometheusTLSSecretNamespacedName())
+		if err != nil {
+			return automationconfig.NOOP(), err
+		}
+		tlsPEMPath = tlsOperatorSecretMountPath + tlsOperatorSecretFileName(certKey)
+		scheme = "https"
+	} else {
+		scheme = "http"
+	}
+
+	return func(config *automationconfig.AutomationConfig) {
+		promConfig := automationconfig.NewDefaultPrometheus(mdb.Spec.Prometheus.Username)
+
+		promConfig.TLSPemPath = tlsPEMPath
+		promConfig.Scheme = scheme
+		promConfig.Password = password
+
+		if mdb.Spec.Prometheus.Port > 0 {
+			promConfig.ListenAddress = fmt.Sprintf("%s:%d", listenAddress, mdb.Spec.Prometheus.Port)
+		}
+
+		if mdb.Spec.Prometheus.MetricsPath != "" {
+			promConfig.MetricsPath = mdb.Spec.Prometheus.MetricsPath
+		}
+
+		config.Prometheus = &promConfig
+	}, nil
+}

Diff for: controllers/replica_set_controller.go

@@ -15,8 +15,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container"
-	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret"
-	"github.com/mongodb/mongodb-kubernetes-operator/pkg/monitoring"
 
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/functions"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge"
@@ -187,6 +185,14 @@ func (r ReplicaSetReconciler) Reconcile(ctx context.Context, request reconcile.R
 		)
 	}
 
+	if err := r.ensurePrometheusTLSResources(mdb); err != nil {
+		return status.Update(r.client.Status(), &mdb,
+			statusOptions().
+				withMessage(Error, fmt.Sprintf("Error ensuring TLS resources: %s", err)).
+				withFailedPhase(),
+		)
+	}
+
 	if err := r.ensureUserResources(mdb); err != nil {
 		return status.Update(r.client.Status(), &mdb,
 			statusOptions().
@@ -299,6 +305,23 @@ func (r *ReplicaSetReconciler) ensureTLSResources(mdb mdbv1.MongoDBCommunity) er
 	return nil
 }
 
+// ensurePrometheusTLSResources creates any required TLS resources that the MongoDBCommunity
+// requires for TLS configuration.
+func (r *ReplicaSetReconciler) ensurePrometheusTLSResources(mdb mdbv1.MongoDBCommunity) error {
+	if mdb.Spec.Prometheus == nil || mdb.Spec.Prometheus.TLSSecretRef.Name == "" {
+		return nil
+	}
+
+	// the TLS secret needs to be created beforehand, as both the StatefulSet and AutomationConfig
+	// require the contents.
+	r.log.Infof("Prometheus TLS is enabled, creating/updating TLS secret")
+	if err := ensurePrometheusTLSSecret(r.client, mdb); err != nil {
+		return errors.Errorf("could not ensure TLS secret: %s", err)
+	}
+
+	return nil
+}
+
 // deployStatefulSet deploys the backing StatefulSet of the MongoDBCommunity resource.
 //
 // When `Spec.Arbiters` > 0, a second StatefulSet will be created, with the amount
@@ -585,15 +608,9 @@ func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity)
 		secretNamespacedName := types.NamespacedName{Name: mdb.Spec.Prometheus.PasswordSecretRef.Name, Namespace: mdb.Namespace}
 		r.secretWatcher.Watch(secretNamespacedName, mdb.NamespacedName())
 
-		password, err := secret.ReadKey(r.client, mdb.Spec.Prometheus.GetPasswordKey(), secretNamespacedName)
+		prometheusModification, err = getPrometheusModification(r.client, mdb)
 		if err != nil {
-			if apiErrors.IsNotFound(err) {
-				r.log.Infof("Could not read Secret %s. Prometheus will not be configured during this reconciliation, %s", mdb.Spec.Prometheus.PasswordSecretRef.Name, err)
-			} else {
-				return automationconfig.AutomationConfig{}, errors.Errorf("could not configure Prometheus modification: %s", err)
-			}
-		} else {
-			prometheusModification = monitoring.PrometheusModification(mdb, password)
+			return automationconfig.AutomationConfig{}, errors.Errorf("could not enable TLS on Prometheus endpoint: %s", err)
 		}
 	}
 
@@ -661,6 +678,7 @@ func buildStatefulSetModificationFunction(mdb mdbv1.MongoDBCommunity) statefulse
 		statefulset.WithPodSpecTemplate(
 			podtemplatespec.Apply(
 				buildTLSPodSpecModification(mdb),
+				buildTLSPrometheus(mdb),
 			),
 		),