OCPBUGS-59763: enforce client side auth requirement for metrics endpoint

cmd/manager/main.go

@@ -2,14 +2,15 @@ package main
 
 import (
 	"context"
+	"crypto/x509"
 	"flag"
 	"fmt"
 	"net/http"
 	"os"
 	"runtime"
 	"time"
 
-	"github.com/operator-framework/operator-marketplace/pkg/certificateauthority"
+	ca "github.com/operator-framework/operator-marketplace/pkg/certificateauthority"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/fields"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -24,6 +25,7 @@ import (
 	configv1 "github.com/operator-framework/operator-marketplace/pkg/apis/config/v1"
 	apiutils "github.com/operator-framework/operator-marketplace/pkg/apis/operators/shared"
 	"github.com/operator-framework/operator-marketplace/pkg/controller"
+	"github.com/operator-framework/operator-marketplace/pkg/controller/configmap"
 	"github.com/operator-framework/operator-marketplace/pkg/controller/options"
 	"github.com/operator-framework/operator-marketplace/pkg/defaults"
 	"github.com/operator-framework/operator-marketplace/pkg/metrics"
@@ -112,9 +114,10 @@ func main() {
 		os.Exit(0)
 	}
 
+	clientCAStore := ca.NewClientCAStore(x509.NewCertPool())
 	// set TLS to serve metrics over a secure channel if cert is provided
 	// cert is provided by default by the marketplace-trusted-ca volume mounted as part of the marketplace-operator deployment
-	if err := metrics.ServePrometheus(tlsCertPath, tlsKeyPath); err != nil {
+	if err := metrics.ServePrometheus(tlsCertPath, tlsKeyPath, clientCAStore); err != nil {
 		logger.Fatalf("failed to serve prometheus metrics: %s", err)
 	}
 
@@ -153,10 +156,18 @@ func main() {
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
 				&corev1.ConfigMap{}: {
-					Field: fields.SelectorFromSet(fields.Set{
-						"metadata.namespace": namespace,
-						"metadata.name":      certificateauthority.TrustedCaConfigMapName,
-					}),
+					Namespaces: map[string]cache.Config{
+						namespace: {
+							FieldSelector: fields.SelectorFromSet(fields.Set{
+								"metadata.name": ca.TrustedCaConfigMapName,
+							}),
+						},
+						configmap.ClientCANamespace: {
+							FieldSelector: fields.SelectorFromSet(fields.Set{
+								"metadata.name": configmap.ClientCAConfigMap,
+							}),
+						},
+					},
 				},
 			},
 		},
@@ -192,7 +203,7 @@ func main() {
 		}
 
 		logger.Info("setting up controllers")
-		if err := controller.AddToManager(mgr, options.ControllerOptions{}); err != nil {
+		if err := controller.AddToManager(mgr, options.ControllerOptions{ClientCAStore: clientCAStore}); err != nil {
 			logger.Fatal(err)
 		}
 

manifests/06_role_binding.yaml

@@ -36,3 +36,23 @@ roleRef:
   kind: Role
   name: marketplace-operator
   apiGroup: rbac.authorization.k8s.io
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: marketplace-operator-auth-reader
+  namespace: kube-system
+  annotations:
+    include.release.openshift.io/hypershift: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+    capability.openshift.io/name: "marketplace"
+subjects:
+- kind: ServiceAccount
+  name: marketplace-operator
+  namespace: openshift-marketplace
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader

pkg/certificateauthority/clientcaconfigmap.go

@@ -0,0 +1,25 @@
+package certificateauthority
+
+import "crypto/x509"
+
+type ClientCAStore struct {
+	clientCA *x509.CertPool
+}
+
+func NewClientCAStore(certpool *x509.CertPool) *ClientCAStore {
+	if certpool == nil {
+		certpool = x509.NewCertPool()
+	}
+	return &ClientCAStore{clientCA: certpool}
+}
+
+func (c *ClientCAStore) Update(newCAPEM []byte) {
+	if newCAPEM == nil {
+		return
+	}
+	c.clientCA.AppendCertsFromPEM(newCAPEM)
+}
+
+func (c *ClientCAStore) GetCA() *x509.CertPool {
+	return c.clientCA
+}

pkg/controller/configmap/configmap_controller.go

@@ -3,6 +3,7 @@ package configmap
 import (
 	"context"
 	"os"
+
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 
 	mktconfig "github.com/operator-framework/operator-marketplace/pkg/apis/config/v1"
@@ -18,18 +19,28 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const (
+	// the rootCA for authenticating client certs is made available at the
+	// kube-system/extension-apiserver-authentication configMap, under the key
+	// client-ca-file.
+	ClientCANamespace = "kube-system"
+	ClientCAConfigMap = "extension-apiserver-authentication"
+	ClientCAKey = "client-ca-file"
+)
+
 // Add creates a new ConfigMap Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, _ options.ControllerOptions) error {
-	return add(mgr, newReconciler(mgr))
+func Add(mgr manager.Manager, o options.ControllerOptions) error {
+	return add(mgr, NewReconciler(mgr, o.ClientCAStore))
 }
 
 // newReconciler returns a new ReconcileConfigMap.
-func newReconciler(mgr manager.Manager) *ReconcileConfigMap {
+func NewReconciler(mgr manager.Manager, clientCAStore *ca.ClientCAStore) *ReconcileConfigMap {
 	client := mgr.GetClient()
 	return &ReconcileConfigMap{
 		client:  client,
 		handler: ca.NewHandler(client),
+		clientCAStore: clientCAStore,
 	}
 }
 
@@ -54,13 +65,21 @@ func getPredicateFunctions() predicate.Funcs {
 	return predicate.Funcs{
 		CreateFunc: func(e event.CreateEvent) bool {
 			// If the ConfigMap is created we should kick off an event.
+			if e.Object.GetName() == ClientCAConfigMap && 
+				e.Object.GetNamespace() == ClientCANamespace {
+					return true
+			}
 			if e.Object.GetName() == ca.TrustedCaConfigMapName {
 				return true
 			}
 			return false
 		},
 		UpdateFunc: func(e event.UpdateEvent) bool {
 			// If the ConfigMap is updated we should kick off an event.
+			if e.ObjectOld.GetName() == ClientCAConfigMap && 
+				e.ObjectOld.GetNamespace() == ClientCANamespace {
+					return true
+			}
 			if e.ObjectOld.GetName() == ca.TrustedCaConfigMapName {
 				return true
 			}
@@ -81,13 +100,17 @@ var _ reconcile.Reconciler = &ReconcileConfigMap{}
 type ReconcileConfigMap struct {
 	client  client.Client
 	handler ca.Handler
+	clientCAStore *ca.ClientCAStore
 }
 
 // Reconcile will restart the marketplace operator if the Certificate Authority ConfigMap is
 // not in sync with the Certificate Authority bundle on disk..
 func (r *ReconcileConfigMap) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
 	log.Printf("Reconciling ConfigMap %s/%s", request.Namespace, request.Name)
 
+	if request.Name == ClientCAConfigMap && request.Namespace == ClientCANamespace {
+		return r.updateClientCA(ctx, request)
+	}
 	// Check if the CA ConfigMap is in the same namespace that Marketplace is deployed in.
 	isConfigMapInOtherNamespace, err := shared.IsObjectInOtherNamespace(request.Namespace)
 	if err != nil {
@@ -114,3 +137,19 @@ func isRunningOnPod() bool {
 	_, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 	return !os.IsNotExist(err)
 }
+
+func (r *ReconcileConfigMap) updateClientCA(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	// Get configMap object
+	clientCAConfigMap := &corev1.ConfigMap{}
+	if err := r.client.Get(ctx, request.NamespacedName, clientCAConfigMap); err != nil {
+		// Requested object was not found, could have been deleted after reconcile request.
+		// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
+		// Return and don't requeue
+		return reconcile.Result{}, client.IgnoreNotFound(err)
+	}
+	if newCA, ok := clientCAConfigMap.Data[ClientCAKey]; ok {
+		r.clientCAStore.Update([]byte(newCA))
+	}
+
+	return reconcile.Result{}, nil
+}