feat: Expose ExternalDNS operand metrics via kube-rbac-proxy sidecar

ExternalDNS operand pods bind metrics to 127.0.0.1, making them
inaccessible from outside the pod. Prometheus cannot scrape them.

This adds a kube-rbac-proxy sidecar to each operand deployment that
proxies the localhost metrics over HTTPS, along with a Service
(annotated for OpenShift serving cert) and ServiceMonitor for
automatic Prometheus scraping.

Resolves: OCPBUGS-58102

Assisted with Claude

Author: Thealisyed

bundle/manifests/external-dns-operator.clusterserviceversion.yaml

@@ -470,6 +470,7 @@ spec:
                 - --operator-namespace=$(OPERATOR_NAMESPACE)
                 - --operand-namespace=$(OPERATOR_NAMESPACE)
                 - --externaldns-image=$(RELATED_IMAGE_EXTERNAL_DNS)
+                - --kube-rbac-proxy-image=$(RELATED_IMAGE_KUBE_RBAC_PROXY)
                 - --trusted-ca-configmap=$(TRUSTED_CA_CONFIGMAP_NAME)
                 - --leader-elect
                 - --webhook-disable-http2
@@ -480,6 +481,8 @@ spec:
                       fieldPath: metadata.namespace
                 - name: RELATED_IMAGE_EXTERNAL_DNS
                   value: quay.io/external-dns-operator/external-dns:latest
+                - name: RELATED_IMAGE_KUBE_RBAC_PROXY
+                  value: quay.io/openshift/origin-kube-rbac-proxy:latest
                 - name: TRUSTED_CA_CONFIGMAP_NAME
                 image: quay.io/openshift/origin-external-dns-operator:latest
                 name: external-dns-operator

config/rbac/operand_role.yaml

@@ -31,3 +31,15 @@ rules:
       - get
       - watch
       - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create

main.go

@@ -42,6 +42,7 @@ func main() {
 	flag.StringVar(&opCfg.OperatorNamespace, "operator-namespace", operatorconfig.DefaultOperatorNamespace, "The namespace that the operator is running in.")
 	flag.StringVar(&opCfg.OperandNamespace, "operand-namespace", operatorconfig.DefaultOperandNamespace, "The namespace that ExternalDNS containers should run in.")
 	flag.StringVar(&opCfg.ExternalDNSImage, "externaldns-image", operatorconfig.DefaultExternalDNSImage, "The container image used for running ExternalDNS.")
+	flag.StringVar(&opCfg.KubeRBACProxyImage, "kube-rbac-proxy-image", operatorconfig.DefaultKubeRBACProxyImage, "The container image used for the kube-rbac-proxy metrics sidecar.")
 	flag.StringVar(&opCfg.CertDir, "cert-dir", operatorconfig.DefaultCertDir, "The directory for keys and certificates for serving the webhook.")
 	flag.StringVar(&opCfg.TrustedCAConfigMapName, "trusted-ca-configmap", operatorconfig.DefaultTrustedCAConfigMapName, "The name of the config map containing TLS CA(s) which should be trusted by ExternalDNS containers. PEM encoded file under \"ca-bundle.crt\" key is expected.")
 	flag.BoolVar(&opCfg.EnableWebhook, "enable-webhook", operatorconfig.DefaultEnableWebhook, "Enable the validating webhook server. Defaults to true.")

pkg/operator/config/config.go

@@ -35,6 +35,7 @@ import (
 
 const (
 	DefaultExternalDNSImage        = "quay.io/external-dns-operator/external-dns:latest"
+	DefaultKubeRBACProxyImage      = "quay.io/openshift/origin-kube-rbac-proxy:latest"
 	DefaultMetricsAddr             = "127.0.0.1:8080"
 	DefaultOperatorNamespace       = "external-dns-operator"
 	DefaultOperandNamespace        = "external-dns"
@@ -59,6 +60,10 @@ type Config struct {
 	// by the operator.
 	ExternalDNSImage string
 
+	// KubeRBACProxyImage is the kube-rbac-proxy image for the metrics sidecar container
+	// in the ExternalDNS operand deployment.
+	KubeRBACProxyImage string
+
 	// MetricsBindAddress is the TCP address that the operator should bind to for
 	// serving prometheus metrics. It can be set to "0" to disable the metrics serving.
 	MetricsBindAddress string

pkg/operator/controller/externaldns/controller.go

@@ -50,6 +50,8 @@ type Config struct {
 	Namespace string
 	// Image is the ExternalDNS image to use.
 	Image string
+	// KubeRBACProxyImage is the kube-rbac-proxy image for the metrics sidecar.
+	KubeRBACProxyImage string
 	// OperatorNamespace is the namespace in which this operator is deployed.
 	OperatorNamespace string
 	// IsOpenShift is the flag which instructs the operator that it runs in OpenShift.
@@ -102,6 +104,10 @@ func New(mgr manager.Manager, cfg Config) (controller.Controller, error) {
 		return nil, err
 	}
 
+	if err := c.Watch(source.Kind[client.Object](operatorCache, &corev1.Service{}, handler.EnqueueRequestForOwner(operatorScheme, operatorRESTMapper, &operatorv1beta1.ExternalDNS{}, handler.OnlyControllerOwner()))); err != nil {
+		return nil, err
+	}
+
 	// secret replicated by the credentials controller
 	// needs to trigger the reconciliation of the corresponding ExternalDNS
 	// because of the annotation with the secret's hash in the operand deployment
@@ -207,11 +213,33 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		trustCAConfigMap = configMap
 	}
 
-	_, currentDeployment, err := r.ensureExternalDNSDeployment(ctx, r.config.Namespace, r.config.Image, sa, credSecret, trustCAConfigMap, externalDNS)
+	_, currentDeployment, err := r.ensureExternalDNSDeployment(ctx, r.config.Namespace, r.config.Image, r.config.KubeRBACProxyImage, sa, credSecret, trustCAConfigMap, externalDNS)
 	if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to ensure externalDNS deployment: %w", err)
 	}
 
+	// Ensure metrics service and service monitor for Prometheus scraping.
+	// If kube-rbac-proxy image is not configured, clean up any existing metrics resources.
+	if r.config.KubeRBACProxyImage != "" {
+		if err := r.ensureExternalDNSMetricsService(ctx, r.config.Namespace, externalDNS); err != nil {
+			return reconcile.Result{}, fmt.Errorf("failed to ensure externalDNS metrics service: %w", err)
+		}
+		if r.config.IsOpenShift {
+			if err := r.ensureExternalDNSServiceMonitor(ctx, r.config.Namespace, externalDNS); err != nil {
+				return reconcile.Result{}, fmt.Errorf("failed to ensure externalDNS service monitor: %w", err)
+			}
+		}
+	} else {
+		if err := r.deleteExternalDNSMetricsService(ctx, r.config.Namespace, externalDNS); err != nil {
+			return reconcile.Result{}, fmt.Errorf("failed to delete externalDNS metrics service: %w", err)
+		}
+		if r.config.IsOpenShift {
+			if err := r.deleteExternalDNSServiceMonitor(ctx, r.config.Namespace, externalDNS); err != nil {
+				return reconcile.Result{}, fmt.Errorf("failed to delete externalDNS service monitor: %w", err)
+			}
+		}
+	}
+
 	if err := r.updateExternalDNSStatus(ctx, externalDNS, currentDeployment, true); err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to update externalDNS custom resource %s: %w", externalDNS.Name, err)
 	}

pkg/operator/controller/externaldns/deployment.go

@@ -77,6 +77,7 @@ var sourceStringTable = map[operatorv1beta1.ExternalDNSSourceType]string{
 type deploymentConfig struct {
 	namespace              string
 	image                  string
+	kubeRBACProxyImage     string
 	serviceAccount         *corev1.ServiceAccount
 	externalDNS            *operatorv1beta1.ExternalDNS
 	isOpenShift            bool
@@ -89,7 +90,7 @@ type deploymentConfig struct {
 
 // ensureExternalDNSDeployment ensures that the externalDNS deployment exists.
 // Returns a Boolean value indicating whether the deployment exists, a pointer to the deployment, and an error when relevant.
-func (r *reconciler) ensureExternalDNSDeployment(ctx context.Context, namespace, image string, serviceAccount *corev1.ServiceAccount, credSecret *corev1.Secret, trustCAConfigMap *corev1.ConfigMap, externalDNS *operatorv1beta1.ExternalDNS) (bool, *appsv1.Deployment, error) {
+func (r *reconciler) ensureExternalDNSDeployment(ctx context.Context, namespace, image, kubeRBACProxyImage string, serviceAccount *corev1.ServiceAccount, credSecret *corev1.Secret, trustCAConfigMap *corev1.ConfigMap, externalDNS *operatorv1beta1.ExternalDNS) (bool, *appsv1.Deployment, error) {
 	nsName := types.NamespacedName{Namespace: namespace, Name: controller.ExternalDNSResourceName(externalDNS)}
 
 	// build credentials secret's hash
@@ -109,16 +110,17 @@ func (r *reconciler) ensureExternalDNSDeployment(ctx context.Context, namespace,
 	}
 
 	desired, err := desiredExternalDNSDeployment(&deploymentConfig{
-		namespace,
-		image,
-		serviceAccount,
-		externalDNS,
-		r.config.IsOpenShift,
-		r.config.PlatformStatus,
-		credSecret.Name,
-		credSecretHash,
-		trustCAConfigMapName,
-		trustCAConfigMapHash,
+		namespace:              namespace,
+		image:                  image,
+		kubeRBACProxyImage:     kubeRBACProxyImage,
+		serviceAccount:         serviceAccount,
+		externalDNS:            externalDNS,
+		isOpenShift:            r.config.IsOpenShift,
+		platformStatus:         r.config.PlatformStatus,
+		secret:                 credSecret.Name,
+		secretHash:             credSecretHash,
+		trustedCAConfigMapName: trustCAConfigMapName,
+		trustedCAConfigMapHash: trustCAConfigMapHash,
 	})
 	if err != nil {
 		return false, nil, fmt.Errorf("failed to build externalDNS deployment: %w", err)
@@ -296,6 +298,17 @@ func desiredExternalDNSDeployment(cfg *deploymentConfig) (*appsv1.Deployment, er
 			depl.Spec.Template.Spec.Containers = append(depl.Spec.Template.Spec.Containers, *container)
 		}
 	}
+	// Add kube-rbac-proxy sidecar(s) and metrics cert volume for secure metrics exposure.
+	// One sidecar per zone container, each proxying the corresponding metrics port.
+	if cfg.kubeRBACProxyImage != "" {
+		for i := 0; i < cbld.counter; i++ {
+			proxyContainer := kubeRBACProxyContainer(cfg.kubeRBACProxyImage, i)
+			depl.Spec.Template.Spec.Containers = append(depl.Spec.Template.Spec.Containers, proxyContainer)
+		}
+		certVolume := metricsCertVolume(controller.ExternalDNSMetricsSecretName(cfg.externalDNS))
+		depl.Spec.Template.Spec.Volumes = append(depl.Spec.Template.Spec.Volumes, certVolume)
+	}
+
 	return depl, nil
 }
 
@@ -418,6 +431,10 @@ func externalDNSContainersChanged(current, expected, updated *appsv1.Deployment)
 				updated.Spec.Template.Spec.Containers[currCont.Index].SecurityContext = updatedContext
 				changed = true
 			}
+			if !equalContainerPorts(currCont.Ports, expCont.Ports) {
+				updated.Spec.Template.Spec.Containers[currCont.Index].Ports = expCont.Ports
+				changed = true
+			}
 		} else {
 			// expected container is not present - add it
 			updated.Spec.Template.Spec.Containers = append(updated.Spec.Template.Spec.Containers, expCont.Container)
@@ -701,6 +718,27 @@ func securityContextChanged(current, updated, desired *corev1.SecurityContext) (
 	return changed, updated
 }
 
+// equalContainerPorts returns true if 2 container port slices have the same content.
+func equalContainerPorts(current, expected []corev1.ContainerPort) bool {
+	if len(current) != len(expected) {
+		return false
+	}
+	currentMap := map[string]corev1.ContainerPort{}
+	for _, p := range current {
+		currentMap[p.Name] = p
+	}
+	for _, ep := range expected {
+		cp, found := currentMap[ep.Name]
+		if !found {
+			return false
+		}
+		if cp.ContainerPort != ep.ContainerPort || cp.Protocol != ep.Protocol {
+			return false
+		}
+	}
+	return true
+}
+
 func equalBoolPtr(current, desired *bool) bool {
 	if desired == nil {
 		return true

pkg/operator/controller/externaldns/deployment_test.go

@@ -3845,15 +3845,15 @@ func TestDesiredExternalDNSDeployment(t *testing.T) {
 				}
 			}()
 			depl, err := desiredExternalDNSDeployment(&deploymentConfig{
-				test.OperandNamespace,
-				test.OperandImage,
-				serviceAccount,
-				tc.inputExternalDNS,
-				tc.inputIsOpenShift,
-				tc.inputPlatformStatus,
-				tc.inputSecretName,
-				testSecretHash,
-				tc.inputTrustedCAConfigMapName, "",
+				namespace:              test.OperandNamespace,
+				image:                  test.OperandImage,
+				serviceAccount:         serviceAccount,
+				externalDNS:            tc.inputExternalDNS,
+				isOpenShift:            tc.inputIsOpenShift,
+				platformStatus:         tc.inputPlatformStatus,
+				secret:                 tc.inputSecretName,
+				secretHash:             testSecretHash,
+				trustedCAConfigMapName: tc.inputTrustedCAConfigMapName,
 			})
 			if err != nil {
 				t.Errorf("expected no error from calling desiredExternalDNSDeployment, but received %v", err)
@@ -5999,7 +5999,7 @@ func TestEnsureExternalDNSDeployment(t *testing.T) {
 				log:    zap.New(zap.UseDevMode(true)),
 			}
 
-			gotExist, gotDepl, err := r.ensureExternalDNSDeployment(context.TODO(), test.OperandNamespace, test.OperandImage, serviceAccount, tc.credSecret, tc.trustCAConfigMap, &tc.extDNS)
+			gotExist, gotDepl, err := r.ensureExternalDNSDeployment(context.TODO(), test.OperandNamespace, test.OperandImage, "", serviceAccount, tc.credSecret, tc.trustCAConfigMap, &tc.extDNS)
 			if err != nil {
 				if !tc.errExpected {
 					t.Fatalf("unexpected error received: %v", err)

pkg/operator/controller/externaldns/pod.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 
@@ -57,6 +58,16 @@ const (
 	// all capabilities in the container security context
 	allCapabilities = "ALL"
 	//
+	// kube-rbac-proxy metrics sidecar
+	//
+	kubeRBACProxyContainerName = "kube-rbac-proxy"
+	kubeRBACProxySecurePort    = 8443
+	kubeRBACProxyPortName      = "https"
+	metricsCertVolumeName      = "metrics-cert"
+	metricsCertMountPath       = "/var/run/secrets/serving-cert"
+	metricsCertTLSCertFile     = metricsCertMountPath + "/tls.crt"
+	metricsCertTLSKeyFile      = metricsCertMountPath + "/tls.key"
+	//
 	// AWS
 	//
 	awsCredentialEnvVarName       = "AWS_SHARED_CREDENTIALS_FILE"
@@ -686,3 +697,94 @@ func (b *externalDNSVolumeBuilder) bluecatVolumes() []corev1.Volume {
 func addTXTPrefixFlag(args []string) []string {
 	return append(args, fmt.Sprintf("--txt-prefix=%s", defaultTXTRecordPrefix))
 }
+
+// numMetricsPorts returns the number of metrics ports needed for the given ExternalDNS instance.
+// This mirrors the zone container creation logic in desiredExternalDNSDeployment.
+func numMetricsPorts(externalDNS *operatorv1beta1.ExternalDNS) int {
+	if len(externalDNS.Spec.Zones) == 0 {
+		if externalDNS.Spec.Provider.Type == operatorv1beta1.ProviderTypeAzure {
+			return 2
+		}
+		return 1
+	}
+	return len(externalDNS.Spec.Zones)
+}
+
+// kubeRBACProxyPortNameForSeq returns the port name for the kube-rbac-proxy sidecar
+// at the given sequence index.
+func kubeRBACProxyPortNameForSeq(seq int) string {
+	if seq == 0 {
+		return kubeRBACProxyPortName
+	}
+	return fmt.Sprintf("%s-%d", kubeRBACProxyPortName, seq)
+}
+
+// kubeRBACProxyContainer returns the kube-rbac-proxy sidecar container definition
+// that proxies metrics from the ExternalDNS container's localhost metrics port at the given sequence.
+func kubeRBACProxyContainer(image string, seq int) corev1.Container {
+	securePort := kubeRBACProxySecurePort + seq
+	upstreamPort := defaultMetricsStartPort + seq
+	portName := kubeRBACProxyPortNameForSeq(seq)
+	containerName := kubeRBACProxyContainerName
+	if seq > 0 {
+		containerName = fmt.Sprintf("%s-%d", kubeRBACProxyContainerName, seq)
+	}
+	return corev1.Container{
+		Name:  containerName,
+		Image: image,
+		Args: []string{
+			fmt.Sprintf("--secure-listen-address=0.0.0.0:%d", securePort),
+			fmt.Sprintf("--upstream=http://127.0.0.1:%d/", upstreamPort),
+			"--logtostderr=true",
+			"--v=10",
+			fmt.Sprintf("--tls-cert-file=%s", metricsCertTLSCertFile),
+			fmt.Sprintf("--tls-private-key-file=%s", metricsCertTLSKeyFile),
+			"--http2-disable",
+		},
+		Ports: []corev1.ContainerPort{
+			{
+				Name:          portName,
+				ContainerPort: int32(securePort),
+				Protocol:      corev1.ProtocolTCP,
+			},
+		},
+		Resources: corev1.ResourceRequirements{
+			Requests: corev1.ResourceList{
+				corev1.ResourceCPU:    resource.MustParse("100m"),
+				corev1.ResourceMemory: resource.MustParse("20Mi"),
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      metricsCertVolumeName,
+				MountPath: metricsCertMountPath,
+				ReadOnly:  true,
+			},
+		},
+		TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
+		SecurityContext: &corev1.SecurityContext{
+			Capabilities: &corev1.Capabilities{
+				Drop: []corev1.Capability{allCapabilities},
+			},
+			Privileged:               ptr.To[bool](false),
+			RunAsNonRoot:             ptr.To[bool](true),
+			AllowPrivilegeEscalation: ptr.To[bool](false),
+			SeccompProfile: &corev1.SeccompProfile{
+				Type: corev1.SeccompProfileTypeRuntimeDefault,
+			},
+		},
+	}
+}
+
+// metricsCertVolume returns the volume for the metrics serving certificate secret.
+func metricsCertVolume(secretName string) corev1.Volume {
+	return corev1.Volume{
+		Name: metricsCertVolumeName,
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: secretName,
+			},
+		},
+	}
+}
+