diff --git a/apis/v1alpha1/config_types.go b/apis/v1alpha1/config_types.go index 5db892e6e..d34f648be 100644 --- a/apis/v1alpha1/config_types.go +++ b/apis/v1alpha1/config_types.go @@ -57,6 +57,17 @@ type ConfigSpec struct { // Namespace holds the namespace where bpfman-operator resources shall be // deployed. Namespace string `json:"namespace,omitempty"` + + // overrides is list of overides for components that are managed by + // the operator. Marking a component unmanaged will prevent + // the operator from creating or updating the object. + // +listType=map + // +listMapKey=kind + // +listMapKey=group + // +listMapKey=namespace + // +listMapKey=name + // +optional + Overrides []ComponentOverride `json:"overrides,omitempty"` } // AgentSpec defines the desired state of the bpfman agent. @@ -94,3 +105,28 @@ type ConfigList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []Config `json:"items"` } + +// ComponentOverride allows overriding the operator's behavior for a component. +// +k8s:deepcopy-gen=true +type ComponentOverride struct { + // kind indentifies which object to override. + // +required + Kind string `json:"kind"` + // group identifies the API group that the kind is in. + // +required + Group string `json:"group"` + + // namespace is the component's namespace. If the resource is cluster + // scoped, the namespace should be empty. + // +required + Namespace string `json:"namespace"` + // name is the component's name. + // +required + Name string `json:"name"` + + // unmanaged controls if cluster version operator should stop managing the + // resources in this cluster. + // Default: false + // +required + Unmanaged bool `json:"unmanaged"` +} diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 454bbc333..bb570a9cf 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -1529,12 +1529,27 @@ func (in *ClusterBpfApplicationStateList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ComponentOverride) DeepCopyInto(out *ComponentOverride) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentOverride. +func (in *ComponentOverride) DeepCopy() *ComponentOverride { + if in == nil { + return nil + } + out := new(ComponentOverride) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Config) DeepCopyInto(out *Config) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -1592,6 +1607,11 @@ func (in *ConfigList) DeepCopyObject() runtime.Object { func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = *in out.Agent = in.Agent + if in.Overrides != nil { + in, out := &in.Overrides, &out.Overrides + *out = make([]ComponentOverride, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. diff --git a/bundle/manifests/bpfman-operator.clusterserviceversion.yaml b/bundle/manifests/bpfman-operator.clusterserviceversion.yaml index 0a7f0c16f..8a3c76a40 100644 --- a/bundle/manifests/bpfman-operator.clusterserviceversion.yaml +++ b/bundle/manifests/bpfman-operator.clusterserviceversion.yaml @@ -1012,7 +1012,7 @@ metadata: capabilities: Basic Install categories: OpenShift Optional containerImage: quay.io/bpfman/bpfman-operator:latest - createdAt: "2025-09-26T11:09:36Z" + createdAt: "2025-09-26T15:23:48Z" description: The bpfman Operator is designed to manage eBPF programs for applications. features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "false" diff --git a/bundle/manifests/bpfman.io_configs.yaml b/bundle/manifests/bpfman.io_configs.yaml index ac0c836a8..364008bbb 100644 --- a/bundle/manifests/bpfman.io_configs.yaml +++ b/bundle/manifests/bpfman.io_configs.yaml @@ -79,6 +79,50 @@ spec: Namespace holds the namespace where bpfman-operator resources shall be deployed. type: string + overrides: + description: |- + overrides is list of overides for components that are managed by + the operator. Marking a component unmanaged will prevent + the operator from creating or updating the object. + items: + description: ComponentOverride allows overriding the operator's + behavior for a component. + properties: + group: + description: group identifies the API group that the kind is + in. + type: string + kind: + description: kind indentifies which object to override. + type: string + name: + description: name is the component's name. + type: string + namespace: + description: |- + namespace is the component's namespace. If the resource is cluster + scoped, the namespace should be empty. + type: string + unmanaged: + description: |- + unmanaged controls if cluster version operator should stop managing the + resources in this cluster. + Default: false + type: boolean + required: + - group + - kind + - name + - namespace + - unmanaged + type: object + type: array + x-kubernetes-list-map-keys: + - kind + - group + - namespace + - name + x-kubernetes-list-type: map required: - configuration - image diff --git a/config/crd/bases/bpfman.io_configs.yaml b/config/crd/bases/bpfman.io_configs.yaml index f94557cd9..e7c741052 100644 --- a/config/crd/bases/bpfman.io_configs.yaml +++ b/config/crd/bases/bpfman.io_configs.yaml @@ -79,6 +79,50 @@ spec: Namespace holds the namespace where bpfman-operator resources shall be deployed. type: string + overrides: + description: |- + overrides is list of overides for components that are managed by + the operator. Marking a component unmanaged will prevent + the operator from creating or updating the object. + items: + description: ComponentOverride allows overriding the operator's + behavior for a component. + properties: + group: + description: group identifies the API group that the kind is + in. + type: string + kind: + description: kind indentifies which object to override. + type: string + name: + description: name is the component's name. + type: string + namespace: + description: |- + namespace is the component's namespace. If the resource is cluster + scoped, the namespace should be empty. + type: string + unmanaged: + description: |- + unmanaged controls if cluster version operator should stop managing the + resources in this cluster. + Default: false + type: boolean + required: + - group + - kind + - name + - namespace + - unmanaged + type: object + type: array + x-kubernetes-list-map-keys: + - kind + - group + - namespace + - name + x-kubernetes-list-type: map required: - configuration - image diff --git a/controllers/bpfman-operator/config.go b/controllers/bpfman-operator/config.go index b082d6a7f..a48be6cb5 100644 --- a/controllers/bpfman-operator/config.go +++ b/controllers/bpfman-operator/config.go @@ -149,15 +149,22 @@ func (r *BpfmanConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request } func (r *BpfmanConfigReconciler) reconcileCM(ctx context.Context, bpfmanConfig *v1alpha1.Config) error { - cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: internal.BpfmanCmName, - Namespace: bpfmanConfig.Spec.Namespace}, + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: internal.BpfmanCmName, + Namespace: bpfmanConfig.Spec.Namespace, + }, Data: map[string]string{ internal.BpfmanTOML: bpfmanConfig.Spec.Configuration, internal.BpfmanAgentLogLevel: bpfmanConfig.Spec.Agent.LogLevel, internal.BpfmanLogLevel: bpfmanConfig.Spec.LogLevel, }, } + return assureResource(ctx, r, bpfmanConfig, cm, func(existing, desired *corev1.ConfigMap) bool { return !equality.Semantic.DeepEqual(existing.Data, desired.Data) }) @@ -373,6 +380,12 @@ func load[T client.Object](t T, path, name string) (T, error) { // Creates the resource if it doesn't exist, otherwise updates it to match the desired state. func assureResource[T client.Object](ctx context.Context, r *BpfmanConfigReconciler, bpfmanConfig *v1alpha1.Config, resource T, needsUpdate func(existing T, desired T) bool) error { + if isOverridden(resource, bpfmanConfig.Spec.Overrides) { + r.Logger.Info("Ignoring object (override in place)", + "type", resource.GetObjectKind(), "namespace", resource.GetNamespace(), "name", resource.GetName()) + return nil + } + if err := ctrl.SetControllerReference(bpfmanConfig, resource, r.Scheme); err != nil { return err } @@ -411,6 +424,20 @@ func assureResource[T client.Object](ctx context.Context, r *BpfmanConfigReconci return nil } +// isOverridden determines if a given object shall be unmanaged. +func isOverridden(resource client.Object, overrides []v1alpha1.ComponentOverride) bool { + for _, override := range overrides { + if override.Unmanaged && + override.Kind == resource.GetObjectKind().GroupVersionKind().Kind && + override.Group == resource.GetObjectKind().GroupVersionKind().Group && + override.Namespace == resource.GetNamespace() && + override.Name == resource.GetName() { + return true + } + } + return false +} + // handleDeletion manages the safe deletion of Config resources by // removing the finalizer to allow Kubernetes garbage collection to // proceed via owner references. diff --git a/controllers/bpfman-operator/config_test.go b/controllers/bpfman-operator/config_test.go index 35005e751..1883c16ad 100644 --- a/controllers/bpfman-operator/config_test.go +++ b/controllers/bpfman-operator/config_test.go @@ -48,6 +48,14 @@ const ( logLevel = "FAKE" ) +type clusterObjects struct { + cm *corev1.ConfigMap + csiDriver *storagev1.CSIDriver + ds *appsv1.DaemonSet + metricsDs *appsv1.DaemonSet + scc *osv1.SecurityContextConstraints +} + // TestReconcile tests the BpfmanConfigReconciler's ability to create, update, and restore // Kubernetes resources (ConfigMap, CSIDriver, DaemonSets, and OpenShift SCC). // It verifies proper owner reference setup for cascading deletion - the Kubernetes API @@ -142,7 +150,7 @@ func TestReconcile(t *testing.T) { } t.Log("Making invalid changes to objects") - if err := modifyObjects(ctx, cl, tc.isOpenShift); err != nil { + if _, err := modifyObjects(ctx, cl, tc.isOpenShift); err != nil { t.Fatalf("failed to modify objects for restoration test: %v", err) } @@ -157,6 +165,29 @@ func TestReconcile(t *testing.T) { if err != nil { t.Fatalf("objects not properly restored after modification: %v", err) } + + t.Log("Setting overrides") + if err := setOverrides(ctx, cl); err != nil { + t.Fatalf("failed to modify objects for restoration test: %v", err) + } + + t.Log("Making invalid changes to objects") + cos, err := modifyObjects(ctx, cl, tc.isOpenShift) + if err != nil { + t.Fatalf("failed to modify objects for restoration test: %v", err) + } + + t.Log("Running reconcile - should not change anything (overrides in place)") + _, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile failed after modifying objects: %v", err) + } + + t.Log("Making sure that all objects are unchanged") + err = testObjectsUnchanged(ctx, cl, tc.isOpenShift, cos) + if err != nil { + t.Fatalf("objects not as expected (should be unchanged) after reconcile: %v", err) + } }) } } @@ -467,29 +498,30 @@ func testAllObjectsPresent(ctx context.Context, cl client.Client, bpfmanConfig * } // modifyObjects intentionally corrupts various Kubernetes objects with invalid data -// to test that the reconciler properly restores them to the expected state. +// to test that the reconciler properly restores them to the expected state (or not in the case of overrides). // Modifies ConfigMap data, DaemonSet security context, container images, and OpenShift SCC settings. -func modifyObjects(ctx context.Context, cl client.Client, isOpenShift bool) error { +// Returns the modified objects. +func modifyObjects(ctx context.Context, cl client.Client, isOpenShift bool) (clusterObjects, error) { + var co clusterObjects + // ConfigMap. bpfmanCM := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: internal.BpfmanCmName, Namespace: internal.BpfmanNamespace, }, - Data: map[string]string{ - "this was": "all deleted", - }, } if err := cl.Get(ctx, types.NamespacedName{Name: bpfmanCM.Name, Namespace: bpfmanCM.Namespace}, bpfmanCM); err != nil { - return err + return co, err } bpfmanCM.Data = map[string]string{ "this was": "all deleted", } if err := cl.Update(ctx, bpfmanCM); err != nil { - return err + return co, err } + co.cm = bpfmanCM // CSI Driver. csiDriver := &storagev1.CSIDriver{ @@ -498,7 +530,7 @@ func modifyObjects(ctx context.Context, cl client.Client, isOpenShift bool) erro }, } if err := cl.Get(ctx, types.NamespacedName{Name: internal.BpfmanCsiDriverName}, csiDriver); err != nil { - return err + return co, err } if csiDriver.Spec.AttachRequired == nil || !*csiDriver.Spec.AttachRequired { csiDriver.Spec.AttachRequired = ptr.To(true) @@ -506,8 +538,9 @@ func modifyObjects(ctx context.Context, cl client.Client, isOpenShift bool) erro csiDriver.Spec.AttachRequired = ptr.To(false) } if err := cl.Update(ctx, csiDriver); err != nil { - return err + return co, err } + co.csiDriver = csiDriver // Bpfman DaemonSet. bpfmanDs := &appsv1.DaemonSet{ @@ -518,13 +551,14 @@ func modifyObjects(ctx context.Context, cl client.Client, isOpenShift bool) erro } if err := cl.Get(ctx, types.NamespacedName{Name: bpfmanDs.Name, Namespace: bpfmanDs.Namespace}, bpfmanDs); err != nil { - return err + return co, err } // Set invalid supplemental group ID to test reconciler restoration bpfmanDs.Spec.Template.Spec.SecurityContext.SupplementalGroups = []int64{12345} if err := cl.Update(ctx, bpfmanDs); err != nil { - return err + return co, err } + co.ds = bpfmanDs // Modify the metrics proxy daemonset for restoration testing. metricsDs := &appsv1.DaemonSet{ @@ -535,16 +569,17 @@ func modifyObjects(ctx context.Context, cl client.Client, isOpenShift bool) erro } if err := cl.Get(ctx, types.NamespacedName{Name: metricsDs.Name, Namespace: metricsDs.Namespace}, metricsDs); err != nil { - return err + return co, err } if len(metricsDs.Spec.Template.Spec.Containers) == 0 { - return fmt.Errorf("metrics DaemonSet has no containers configured") + return co, fmt.Errorf("metrics DaemonSet has no containers configured") } // Set invalid container image to test reconciler restoration metricsDs.Spec.Template.Spec.Containers[0].Image = "invalid image" if err := cl.Update(ctx, metricsDs); err != nil { - return err + return co, err } + co.metricsDs = metricsDs if isOpenShift { // Check the SCC was created with the correct configuration. @@ -555,14 +590,15 @@ func modifyObjects(ctx context.Context, cl client.Client, isOpenShift bool) erro } if err := cl.Get(ctx, types.NamespacedName{Name: internal.BpfmanRestrictedSccName, Namespace: corev1.NamespaceAll}, restrictedSCC); err != nil { - return err + return co, err } restrictedSCC.AllowHostPorts = false if err := cl.Update(ctx, restrictedSCC); err != nil { - return err + return co, err } + co.scc = restrictedSCC } - return nil + return co, nil } func verifyCM(cm *corev1.ConfigMap, requiredFields map[string]*string) error { @@ -578,3 +614,136 @@ func verifyCM(cm *corev1.ConfigMap, requiredFields map[string]*string) error { } return nil } + +func setOverrides(ctx context.Context, cl client.Client) error { + bpfmanConfig := &v1alpha1.Config{} + if err := cl.Get(ctx, types.NamespacedName{Name: internal.BpfmanConfigName}, bpfmanConfig); err != nil { + return err + } + + bpfmanConfig.Spec.Overrides = []v1alpha1.ComponentOverride{ + { + Kind: "ConfigMap", + Group: "", + Namespace: "bpfman", + Name: "bpfman-config", + Unmanaged: true, + }, + { + Kind: "DaemonSet", + Group: "apps", + Namespace: "bpfman", + Name: "bpfman-daemon", + Unmanaged: true, + }, + { + Kind: "DaemonSet", + Group: "apps", + Namespace: "bpfman", + Name: "bpfman-metrics-proxy", + Unmanaged: true, + }, + { + Kind: "CSIDriver", + Group: "storage.k8s.io", + Namespace: "", + Name: "csi.bpfman.io", + Unmanaged: true, + }, + { + Kind: "SecurityContextConstraints", + Group: "security.openshift.io", + Namespace: "", + Name: "bpfman-restricted", + Unmanaged: true, + }, + } + + if err := cl.Update(ctx, bpfmanConfig); err != nil { + return err + } + return nil +} + +// testObjectsUnchanged verifies that objects marked as unmanaged in overrides +// remain unchanged after reconciliation by comparing their current state +// against their previously captured state. +func testObjectsUnchanged(ctx context.Context, cl client.Client, isOpenShift bool, cos clusterObjects) error { + bpfmanCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: internal.BpfmanCmName, + Namespace: internal.BpfmanNamespace, + }, + Data: map[string]string{ + "this was": "all deleted", + }, + } + if err := cl.Get(ctx, types.NamespacedName{Name: bpfmanCM.Name, Namespace: bpfmanCM.Namespace}, + bpfmanCM); err != nil { + return err + } + if !reflect.DeepEqual(bpfmanCM, cos.cm) { + return fmt.Errorf("deep equal failed for CM, got: %+v, expected: %+v", bpfmanCM, cos.cm) + } + + // CSI Driver. + csiDriver := &storagev1.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: internal.BpfmanCsiDriverName, + }, + } + if err := cl.Get(ctx, types.NamespacedName{Name: internal.BpfmanCsiDriverName}, csiDriver); err != nil { + return err + } + if !reflect.DeepEqual(csiDriver, cos.csiDriver) { + return fmt.Errorf("deep equal failed for CSI Driver, got: %+v, expected: %+v", csiDriver, cos.csiDriver) + } + + // Bpfman DaemonSet. + bpfmanDs := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: internal.BpfmanDsName, + Namespace: internal.BpfmanNamespace, + }, + } + if err := cl.Get(ctx, types.NamespacedName{Name: bpfmanDs.Name, Namespace: bpfmanDs.Namespace}, + bpfmanDs); err != nil { + return err + } + if !reflect.DeepEqual(bpfmanDs, cos.ds) { + return fmt.Errorf("deep equal failed for Bpfman DS, got: %+v, expected: %+v", bpfmanDs, cos.ds) + } + + // Metrics proxy DaemonSet. + metricsDs := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: internal.BpfmanMetricsProxyDsName, + Namespace: internal.BpfmanNamespace, + }, + } + if err := cl.Get(ctx, types.NamespacedName{Name: metricsDs.Name, Namespace: metricsDs.Namespace}, + metricsDs); err != nil { + return err + } + if !reflect.DeepEqual(metricsDs, cos.metricsDs) { + return fmt.Errorf("deep equal failed for Metrics DS, got: %+v, expected: %+v", metricsDs, cos.metricsDs) + } + + if isOpenShift { + // SCC. + restrictedSCC := &osv1.SecurityContextConstraints{ + ObjectMeta: metav1.ObjectMeta{ + Name: internal.BpfmanRestrictedSccName, + }, + } + if err := cl.Get(ctx, types.NamespacedName{Name: internal.BpfmanRestrictedSccName, Namespace: corev1.NamespaceAll}, + restrictedSCC); err != nil { + return err + } + if !reflect.DeepEqual(restrictedSCC, cos.scc) { + return fmt.Errorf("deep equal failed for SCC, got: %+v, expected: %+v", restrictedSCC, cos.scc) + } + } + + return nil +}