feat(ta): allowNamespaces and denyNamespaces

When the target allocator is configured to watch Prometheus custom
resources in the cluster to discover targets, it is currently hard-coded
to require a ClusterRole with a policy rule of listing namespaces. This
prevents usage in environments where configuring ClusterRoles is not
permitted i.e. in namespace-as-a-service setups where only Roles can be
created.

This change introduces two fields in the prometheusCR specification to
allow configuring the namespaces that can be interacted with by the
target allocator.

- allowNamespaces is a list of namespaces for the target allocator to
  watch. If set to an empty list, it will list all list all namespaces
  in the cluster. This is mutually exclusive with denyNamespaces.
  Defaults to an empty list.
- denyNamespaces is a list of namespaces for the target allocator to not
  watch. If set to an empty list, it will not exclude any namespaces.
  This is mutually exclusive with allowNamespaces.  Defaults to an empty
  list.

A new end to end test was created,
e2e-targetallocator/targetallocator-namespace, to test setting
allowNamespaces and denyNamespaces.

The first part of the test sets up a a collector and TA in a namespace suffixed with
'-top-secret'. The TA in this namespace is configured with the
allowNamespaces to by this '-top-secret' namespace. Then a check is
performed to make sure that it can discover the service monitor that is
created by the otel-operator. This setup also doesn't make use of any
ClusterRole as it only creates a Role to perform its task of discovering
targets in its own namespace.

The second part of the test sets up another collector and TA are created
in a separate namespace without the '-top-secret' namespace with a
ClusterRole. The denyNamespaces is set to the namespace suffixed with
the '-top-secret' namespace, and later we check that we can't discover
the ServiceMonitor that is in the 'top-secret' namespace. We also check
that it can still discover the ServiceMonitor that is in its own
namespace.

Documentation was added to the otel-allocator README on how to set
allowNamespaces to configure the scope that the TA is allowed to
interact with.

Fixes: #3086

Signed-off-by: Charlie Le <[email protected]>

Author: CharlieTLe

.chloggen/namespace-ta.yaml

@@ -0,0 +1,19 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
+component: target allocator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: |
+  Add support for setting the allowNamespaces and denyNamespaces in the target allocator.
+
+# One or more tracking issues related to the change
+issues: [3086]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  allowNamespaces can be set to an empty list to watch all namespaces (default) or to list of namespaces to watch.
+  denyNamespaces can be set to an empty list to deny watching any namespaces (default) or to a list of namespaces to deny watching.

apis/v1alpha1/targetallocator_webhook.go

@@ -116,6 +116,11 @@ func (w TargetAllocatorWebhook) validate(ctx context.Context, ta *TargetAllocato
 		if err != nil || len(warnings) > 0 {
 			return warnings, err
 		}
+
+		// Check to see that allowNamespaces and denyNamespaces are not both set at the same time
+		if len(ta.Spec.PrometheusCR.AllowNamespaces) > 0 && len(ta.Spec.PrometheusCR.DenyNamespaces) > 0 {
+			return warnings, fmt.Errorf("allowNamespaces and denyNamespaces are mutually exclusive")
+		}
 	}
 
 	return warnings, nil

apis/v1alpha1/targetallocator_webhook_test.go

@@ -301,6 +301,19 @@ func TestTargetAllocatorValidatingWebhook(t *testing.T) {
 			},
 			expectedErr: "the OpenTelemetry Spec Ports configuration is incorrect",
 		},
+		{
+			name: "allowNamespaces and denyNamespaces can't both be set",
+			targetallocator: TargetAllocator{
+				Spec: TargetAllocatorSpec{
+					PrometheusCR: v1beta1.TargetAllocatorPrometheusCR{
+						Enabled:         true,
+						AllowNamespaces: []string{"ns1"},
+						DenyNamespaces:  []string{"ns2"},
+					},
+				},
+			},
+			expectedErr: "allowNamespaces and denyNamespaces are mutually exclusive",
+		},
 	}
 
 	for _, test := range tests {

apis/v1beta1/targetallocator_types.go

@@ -12,6 +12,12 @@ type TargetAllocatorPrometheusCR struct {
 	// Enabled indicates whether to use a PrometheusOperator custom resources as targets or not.
 	// +optional
 	Enabled bool `json:"enabled,omitempty"`
+	// AllowNamespaces Namespaces to scope the interaction of the Target Allocator and the apiserver (allow list). This is mutually exclusive with DenyNamespaces.
+	// +optional
+	AllowNamespaces []string `json:"allowNamespaces,omitempty"`
+	// DenyNamespaces Namespaces to scope the interaction of the Target Allocator and the apiserver (deny list). This is mutually exclusive with AllowNamespaces.
+	// +optional
+	DenyNamespaces []string `json:"denyNamespaces,omitempty"`
 	// Default interval between consecutive scrapes. Intervals set in ServiceMonitors and PodMonitors override it.
 	//Equivalent to the same setting on the Prometheus CR.
 	//

apis/v1beta1/zz_generated.deepcopy.go

@@ -7889,6 +7889,14 @@ spec:
                     type: object
                   prometheusCR:
                     properties:
+                      allowNamespaces:
+                        items:
+                          type: string
+                        type: array
+                      denyNamespaces:
+                        items:
+                          type: string
+                        type: array
                       enabled:
                         type: boolean
                       podMonitorSelector:

bundle/community/manifests/opentelemetry.io_opentelemetrycollectors.yaml

@@ -2257,6 +2257,14 @@ spec:
                 type: string
               prometheusCR:
                 properties:
+                  allowNamespaces:
+                    items:
+                      type: string
+                    type: array
+                  denyNamespaces:
+                    items:
+                      type: string
+                    type: array
                   enabled:
                     type: boolean
                   podMonitorSelector:

bundle/community/manifests/opentelemetry.io_targetallocators.yaml

@@ -7889,6 +7889,14 @@ spec:
                     type: object
                   prometheusCR:
                     properties:
+                      allowNamespaces:
+                        items:
+                          type: string
+                        type: array
+                      denyNamespaces:
+                        items:
+                          type: string
+                        type: array
                       enabled:
                         type: boolean
                       podMonitorSelector:

bundle/openshift/manifests/opentelemetry.io_opentelemetrycollectors.yaml

@@ -2257,6 +2257,14 @@ spec:
                 type: string
               prometheusCR:
                 properties:
+                  allowNamespaces:
+                    items:
+                      type: string
+                    type: array
+                  denyNamespaces:
+                    items:
+                      type: string
+                    type: array
                   enabled:
                     type: boolean
                   podMonitorSelector:

bundle/openshift/manifests/opentelemetry.io_targetallocators.yaml

@@ -191,9 +191,11 @@ Upstream documentation here: [PrometheusReceiver](https://github.com/open-teleme
 
 ### RBAC
 
-Before the TargetAllocator can start scraping, you need to set up Kubernetes RBAC (role-based access controls) resources. This means that you need to have a `ServiceAccount` and corresponding cluster roles so that the TargetAllocator has access to all of the necessary resources to pull metrics from.
+Before the TargetAllocator can start scraping, you need to set up Kubernetes RBAC (role-based access controls) resources. This means that you need to have a `ServiceAccount` and corresponding ClusterRoles/Roles so that the TargetAllocator has access to all the necessary resources to pull metrics from.
 
-You can create your own `ServiceAccount`, and reference it in `spec.targetAllocator.serviceAccount` in your `OpenTelemetryCollector` CR. You’ll then need to configure the `ClusterRole` and `ClusterRoleBinding` for this `ServiceAccount`, as per below.
+You can create your own `ServiceAccount`, and reference it in `spec.targetAllocator.serviceAccount` in your `OpenTelemetryCollector` CR. You’ll then need to configure the `ClusterRole` and `ClusterRoleBinding` or `Role` and `RoleBinding` for this `ServiceAccount`, as per below.
+
+#### Cluster-scoped RBAC
 
 ```yaml
   targetAllocator:
@@ -204,11 +206,11 @@ You can create your own `ServiceAccount`, and reference it in `spec.targetAlloca
 ```
 
 > 🚨 **Note**: The Collector part of this same CR *also* has a serviceAccount key which only affects the collector and *not*
-the TargetAllocator.
+> the TargetAllocator.
 
-If you omit the `ServiceAccount` name, the TargetAllocator creates a `ServiceAccount` for you. The `ServiceAccount`’s default name is a concatenation of the Collector name and the `-targetallocator` suffix. By default, this `ServiceAccount` has no defined policy, so you’ll need to create your own `ClusterRole` and `ClusterRoleBinding` for it, as per below.
+If you omit the `ServiceAccount` name, the TargetAllocator creates a `ServiceAccount` for you. The `ServiceAccount`’s default name is a concatenation of the Collector name and the `-targetallocator` suffix. By default, this `ServiceAccount` has no defined policy, so you’ll need to create your own `ClusterRole` and `ClusterRoleBinding` or `Role` and `RoleBinding` for it, as per below.
 
-The role below will provide the minimum access required for the Target Allocator to query all the targets it needs based on any Prometheus configurations:
+The ClusterRole below will provide the minimum access required for the Target Allocator to query all the targets it needs based on any Prometheus configurations:
 
 ```yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -242,7 +244,7 @@ rules:
   verbs: ["get"]
 ```
 
-If you enable the the `prometheusCR` (set `spec.targetAllocator.prometheusCR.enabled` to `true`) in the `OpenTelemetryCollector` CR, you will also need to define the following roles. These give the TargetAllocator access to the `PodMonitor` and `ServiceMonitor` CRs. It also gives namespace access to the `PodMonitor` and `ServiceMonitor`.
+If you enable the `prometheusCR` (set `spec.targetAllocator.prometheusCR.enabled` to `true`) in the `OpenTelemetryCollector` CR, you will also need to define the following ClusterRoles. These give the TargetAllocator access to the `PodMonitor` and `ServiceMonitor` CRs. It also gives namespace access to the `PodMonitor` and `ServiceMonitor`.
 
 ```yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -263,8 +265,82 @@ rules:
   verbs: ["get", "list", "watch"]
 ```
 
-> ✨ The above roles can be combined into a single role.
+> ✨ The above ClusterRoles can be combined into a single ClusterRole.
+ 
+#### Namespace-scoped RBAC
+
+If you want to have the TargetAllocator watch a specific namespace, you can set the allowNamespaces field 
+in the TargetAllocator's prometheusCR configuration. This is useful if you want to restrict the TargetAllocator to only watch Prometheus
+CRs in a specific namespace, and not have cluster-wide access.
+
+```yaml
+  targetAllocator:
+    enabled: true
+    serviceAccount: opentelemetry-targetallocator-sa
+    prometheusCR:
+      enabled: true
+      allowNamespaces: 
+      - foo
+```
+
+In this case, you will need to create a Role and RoleBinding instead of a ClusterRole and ClusterRoleBinding. The Role
+and RoleBinding should be created in the namespace specified by the allowNamespaces field.
 
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: opentelemetry-targetallocator-role
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - services
+      - endpoints
+      - configmaps
+      - secrets
+      - namespaces
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - servicemonitors
+      - podmonitors
+      - scrapeconfigs
+      - probes
+    verbs:
+      - get
+      - watch
+      - list
+```
 
 ### Service / Pod monitor endpoint credentials
 
@@ -420,4 +496,3 @@ Shards the received targets based on the discovered Collector instances
 
 ### Collector
 Client to watch for deployed Collector instances which will then provided to the Allocator. 
-

cmd/otel-allocator/README.md

@@ -53,6 +53,8 @@ type Config struct {
 
 type PrometheusCRConfig struct {
 	Enabled                         bool                  `yaml:"enabled,omitempty"`
+	AllowNamespaces                 []string              `yaml:"allow_namespaces,omitempty"`
+	DenyNamespaces                  []string              `yaml:"deny_namespaces,omitempty"`
 	PodMonitorSelector              *metav1.LabelSelector `yaml:"pod_monitor_selector,omitempty"`
 	PodMonitorNamespaceSelector     *metav1.LabelSelector `yaml:"pod_monitor_namespace_selector,omitempty"`
 	ServiceMonitorSelector          *metav1.LabelSelector `yaml:"service_monitor_selector,omitempty"`

cmd/otel-allocator/internal/config/config.go

@@ -53,7 +53,30 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 		return nil, err
 	}
 
-	factory := informers.NewMonitoringInformerFactories(map[string]struct{}{v1.NamespaceAll: {}}, map[string]struct{}{}, mClient, allocatorconfig.DefaultResyncTime, nil) //TODO decide what strategy to use regarding namespaces
+	if len(cfg.PrometheusCR.AllowNamespaces) != 0 && len(cfg.PrometheusCR.DenyNamespaces) != 0 {
+		return nil, fmt.Errorf("both allow and deny namespaces are set, only one should be set")
+	}
+
+	allowList := map[string]struct{}{}
+	if len(cfg.PrometheusCR.AllowNamespaces) != 0 {
+		logger.Info("watching namespace(s)", "namespaces", cfg.PrometheusCR.AllowNamespaces)
+		for _, ns := range cfg.PrometheusCR.AllowNamespaces {
+			allowList[ns] = struct{}{}
+		}
+	} else {
+		logger.Info("cfg.PrometheusCR.AllowNamespaces is unset, watching all namespaces")
+		allowList = map[string]struct{}{v1.NamespaceAll: {}}
+	}
+
+	denyList := map[string]struct{}{}
+	if len(cfg.PrometheusCR.DenyNamespaces) != 0 {
+		logger.Info("excluding namespace(s)", "namespaces", cfg.PrometheusCR.DenyNamespaces)
+		for _, ns := range cfg.PrometheusCR.DenyNamespaces {
+			denyList[ns] = struct{}{}
+		}
+	}
+
+	factory := informers.NewMonitoringInformerFactories(allowList, denyList, mClient, allocatorconfig.DefaultResyncTime, nil)
 
 	monitoringInformers, err := getInformers(factory)
 	if err != nil {
@@ -99,7 +122,7 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 			logger.Error(err, "Retrying namespace informer creation in promOperator CRD watcher")
 			return true
 		}, func() error {
-			nsMonInf, err = getNamespaceInformer(ctx, map[string]struct{}{v1.NamespaceAll: {}}, promLogger, clientset, operatorMetrics)
+			nsMonInf, err = getNamespaceInformer(ctx, allowList, denyList, promLogger, clientset, operatorMetrics)
 			return err
 		})
 	if getNamespaceInformerErr != nil {
@@ -149,7 +172,7 @@ type PrometheusCRWatcher struct {
 	store                           *assets.StoreBuilder
 }
 
-func getNamespaceInformer(ctx context.Context, allowList map[string]struct{}, promOperatorLogger *slog.Logger, clientset kubernetes.Interface, operatorMetrics *operator.Metrics) (cache.SharedIndexInformer, error) {
+func getNamespaceInformer(ctx context.Context, allowList, denyList map[string]struct{}, promOperatorLogger *slog.Logger, clientset kubernetes.Interface, operatorMetrics *operator.Metrics) (cache.SharedIndexInformer, error) {
 	kubernetesVersion, err := clientset.Discovery().ServerVersion()
 	if err != nil {
 		return nil, err
@@ -165,7 +188,7 @@ func getNamespaceInformer(ctx context.Context, allowList map[string]struct{}, pr
 		clientset.CoreV1(),
 		clientset.AuthorizationV1().SelfSubjectAccessReviews(),
 		allowList,
-		map[string]struct{}{},
+		denyList,
 	)
 	if err != nil {
 		return nil, err

cmd/otel-allocator/internal/watcher/promOperator.go

@@ -7875,6 +7875,14 @@ spec:
                     type: object
                   prometheusCR:
                     properties:
+                      allowNamespaces:
+                        items:
+                          type: string
+                        type: array
+                      denyNamespaces:
+                        items:
+                          type: string
+                        type: array
                       enabled:
                         type: boolean
                       podMonitorSelector:

config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml

@@ -2254,6 +2254,14 @@ spec:
                 type: string
               prometheusCR:
                 properties:
+                  allowNamespaces:
+                    items:
+                      type: string
+                    type: array
+                  denyNamespaces:
+                    items:
+                      type: string
+                    type: array
                   enabled:
                     type: boolean
                   podMonitorSelector: