feat: Limit caching of ConfigMaps and Secrets

feat: Toggle caching of ConfigMaps and Secrets with CommonLabels

Author: Baarsgaard

deploy/helm/grafana-operator/README.md

@@ -104,5 +104,6 @@ It's easier to just manage this configuration outside of the operator.
 | serviceMonitor.targetLabels | list | `[]` | Set of labels to transfer from the Kubernetes Service onto the target |
 | serviceMonitor.telemetryPath | string | `"/metrics"` | Set path to metrics path |
 | tolerations | list | `[]` | pod tolerations |
+| watchLabeledReferencesOnly | bool | `false` | Sets the `WATCH_LABELED_REFERENCES_ONLY` environment variable, it enables the caching of ConfigMaps and Secrets labeled with `"app.kubernetes.io/managed-by": "grafana-operator"` to reduce requests to the api. By default, ConfigMaps and Secrets are not cached to reduce the memory usage of the operator in large clusters. WARNING This will hide unlabeled ConfigMaps and Secrets from the Operator |
 | watchNamespaceSelector | string | `""` | Sets the `WATCH_NAMESPACE_SELECTOR` environment variable, it defines which namespaces the operator should be listening for based on a namespace label (e.g. `"environment: dev"`). By default, the operator watches all namespaces. To make it watch only its own namespace, check out `namespaceScope` option instead. |
 | watchNamespaces | string | `""` | Sets the `WATCH_NAMESPACE` environment variable, it defines which namespaces the operator should be listening for (e.g. `"grafana, foo"`). By default, the operator watches all namespaces. To make it watch only its own namespace, check out `namespaceScope` option instead. |

deploy/helm/grafana-operator/templates/deployment.yaml

@@ -51,6 +51,12 @@ spec:
               {{ else }}
               value: {{quote .Values.watchNamespaceSelector }}
               {{- end }}
+            - name: WATCH_LABELED_REFERENCES_ONLY
+              {{- if .Values.watchLabeledReferencesOnly }}
+              value: "true"
+              {{ else }}
+              value: ""
+              {{- end }}
             {{- with .Values.env }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

deploy/helm/grafana-operator/values.yaml

@@ -15,6 +15,12 @@ watchNamespaces: ""
 # By default, the operator watches all namespaces. To make it watch only its own namespace, check out `namespaceScope` option instead.
 watchNamespaceSelector: ""
 
+# -- Sets the `WATCH_LABELED_REFERENCES_ONLY` environment variable,
+# it enables the caching of ConfigMaps and Secrets labeled with `"app.kubernetes.io/managed-by": "grafana-operator"` to reduce requests to the api.
+# By default, ConfigMaps and Secrets are not cached to reduce the memory usage of the operator in large clusters.
+# WARNING This will hide unlabeled ConfigMaps and Secrets from the Operator
+watchLabeledReferencesOnly: false
+
 # -- Determines if the target cluster is OpenShift. Additional rbac permissions for routes will be added on OpenShift
 isOpenShift: false
 

main.go

@@ -67,6 +67,10 @@ const (
 	// eg: "environment: dev"
 	// If empty or undefined, the operator will run in cluster scope.
 	watchNamespaceEnvSelector = "WATCH_NAMESPACE_SELECTOR"
+	// Enable caching of ConfigMaps and Secrets to reduce API read requests
+	// If empty or undefined, the operator will disable caching
+	// This will hide all referenced ConfigMaps and Secrets not labeled with: app.kubernetes.io/managed-by: grafana-operator
+	watchLabeledReferencesOnlyEnvVar = "WATCH_LABELED_REFERENCES_ONLY"
 )
 
 var (
@@ -105,8 +109,10 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Detect environment variables
 	watchNamespace, _ := os.LookupEnv(watchNamespaceEnvVar)
 	watchNamespaceSelector, _ := os.LookupEnv(watchNamespaceEnvSelector)
+	_, watchLabeledReferencesOnly := os.LookupEnv(watchLabeledReferencesOnlyEnvVar)
 
 	// Platform detection
 	restConfig := ctrl.GetConfigOrDie()
@@ -131,23 +137,28 @@ func main() {
 		LeaderElectionID:       "f75f3bba.integreatly.org",
 		PprofBindAddress:       pprofAddr,
 		// Limit caching to reduce heap usage with CommonLabels as selector
-		// ConfigMap and Secret are omitted here to prevent interference with Get and List in reconcilers, see TODO below
 		Cache: cache.Options{ByObject: map[client.Object]cache.ByObject{
 			&v1.Deployment{}:                cacheLabels,
 			&corev1.Service{}:               cacheLabels,
 			&corev1.ServiceAccount{}:        cacheLabels,
 			&networkingv1.Ingress{}:         cacheLabels,
 			&corev1.PersistentVolumeClaim{}: cacheLabels,
+			&corev1.ConfigMap{}:             cacheLabels, // Matching just labeled ConfigMaps and Secrets greatly reduces cache size
+			&corev1.Secret{}:                cacheLabels, // Omitting labels or supporting custom labels would require changes in Grafana Reconciler
 		}},
 	}
 	if isOpenShift {
 		controllerOptions.Cache.ByObject[&routev1.Route{}] = cacheLabels
 	}
-
-	// TODO Add a config option to limit ConfigMaps and Secrets in Cache
-	// Likely similar to how namespace scope is handled
-	// controllerOptions.Cache.ByObject[&corev1.ConfigMap{}] = cacheLabels
-	// controllerOptions.Cache.ByObject[&corev1.Secret{}] = cacheLabels
+	// Disable ConfigMap and Secret cache lookups per default
+	// all reads will hit the api
+	if !watchLabeledReferencesOnly {
+		controllerOptions.Client = client.Options{
+			Cache: &client.CacheOptions{
+				DisableFor: []client.Object{&corev1.ConfigMap{}, &corev1.Secret{}},
+			},
+		}
+	}
 
 	// Determine Operator scope
 	switch {