fix(clusterchecksrunner): preserve packaged conf.d assets (e.g. SNMP … (#3374)

fix(clusterchecksrunner): preserve packaged conf.d assets (e.g. SNMP profiles) on CLC runners (CONS-8516)

test(common): cover remove-corechecks volume helpers

The patch-coverage gate computes per-package coverage (go test ./...
without -coverpkg), so GetVolumeMountForRmCorechecksInit and the edited
remove-corechecks volume helpers in the common package registered as 0%
even though a clusterchecksrunner test exercises them. Add a common-package
test that covers these helpers directly.

fix(clusterchecksrunner): copy check assets without preserving ownership

cp -a (= --preserve=all) tries to preserve the root ownership of the
image's packaged conf.d. When the runner is forced to run as a non-root
UID (OpenShift restricted SCC, or a securityContext.runAsUser override),
that fails with "Operation not permitted" and, under set -e, blocks the
pod from starting. Use cp -RL to copy recursively without preserving
privileged attributes so the assets are owned by the runtime user.

test(golden): regenerate golden manifests for cp -RL asset copy

Merge branch 'main' into fspano/CONS-8516-clc-preserve-check-assets

Co-authored-by: frank.spano <frank.spano@datadoghq.com>

Author: frank-spano

internal/controller/datadogagent/common/const.go

@@ -83,6 +83,15 @@ const (
 	ChecksdVolumeName = "checksd"
 	ChecksdVolumePath = "/checks.d"
 
+	// RmCorechecksVolumeName is the volume that overlays the agent's conf.d
+	// directory on the cluster checks runner to prevent default core checks
+	// from running there.
+	RmCorechecksVolumeName = "remove-corechecks"
+	// RmCorechecksConfdInitPath is the scratch mount path used by the init
+	// container that seeds the remove-corechecks overlay with the packaged
+	// conf.d assets (e.g. SNMP profiles) from the agent image.
+	RmCorechecksConfdInitPath = "/opt/conf.d-overlay"
+
 	HostRootVolumeName = "hostroot"
 	HostRootHostPath   = "/"
 	HostRootMountPath  = "/host/root"

internal/controller/datadogagent/common/volumes.go

@@ -49,7 +49,7 @@ func GetVolumeForChecksd() corev1.Volume {
 // GetVolumeForRmCorechecks return the volume that overwrites the corecheck directory
 func GetVolumeForRmCorechecks() corev1.Volume {
 	return corev1.Volume{
-		Name: "remove-corechecks",
+		Name: RmCorechecksVolumeName,
 		VolumeSource: corev1.VolumeSource{
 			EmptyDir: &corev1.EmptyDirVolumeSource{},
 		},
@@ -171,11 +171,22 @@ func GetVolumeMountForChecksd() corev1.VolumeMount {
 // GetVolumeMountForRmCorechecks return the VolumeMount that overwrites the corechecks directory
 func GetVolumeMountForRmCorechecks() corev1.VolumeMount {
 	return corev1.VolumeMount{
-		Name:      "remove-corechecks",
+		Name:      RmCorechecksVolumeName,
 		MountPath: fmt.Sprintf("%s/%s", ConfigVolumePath, "conf.d"),
 	}
 }
 
+// GetVolumeMountForRmCorechecksInit return the VolumeMount used by the init
+// container that seeds the remove-corechecks overlay. It is mounted at a scratch
+// path (not over the agent conf.d) so the init container can read the packaged
+// conf.d assets from the agent image and copy them into the overlay.
+func GetVolumeMountForRmCorechecksInit() corev1.VolumeMount {
+	return corev1.VolumeMount{
+		Name:      RmCorechecksVolumeName,
+		MountPath: RmCorechecksConfdInitPath,
+	}
+}
+
 // GetVolumeMountForAuth returns the VolumeMount that contains the authentication information
 func GetVolumeMountForAuth(readOnly bool) corev1.VolumeMount {
 	return corev1.VolumeMount{

internal/controller/datadogagent/common/volumes_test.go

@@ -0,0 +1,58 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+package common
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestGetVolumeForRmCorechecks(t *testing.T) {
+	vol := GetVolumeForRmCorechecks()
+
+	assert.Equal(t, RmCorechecksVolumeName, vol.Name)
+	assert.NotNil(t, vol.EmptyDir, "remove-corechecks must be backed by an emptyDir so the init container can seed it")
+}
+
+func TestGetVolumeMountForRmCorechecks(t *testing.T) {
+	vm := GetVolumeMountForRmCorechecks()
+
+	assert.Equal(t, RmCorechecksVolumeName, vm.Name)
+	// This mount overlays the agent conf.d so default core checks do not run.
+	assert.Equal(t, fmt.Sprintf("%s/%s", ConfigVolumePath, "conf.d"), vm.MountPath)
+}
+
+func TestGetVolumeMountForRmCorechecksInit(t *testing.T) {
+	vm := GetVolumeMountForRmCorechecksInit()
+
+	// Shares the remove-corechecks volume ...
+	assert.Equal(t, RmCorechecksVolumeName, vm.Name)
+	// ... but mounts it at a scratch path, NOT over the agent conf.d, so the
+	// init container can still read the image's packaged conf.d assets.
+	assert.Equal(t, RmCorechecksConfdInitPath, vm.MountPath)
+	assert.NotEqual(t, fmt.Sprintf("%s/%s", ConfigVolumePath, "conf.d"), vm.MountPath)
+}
+
+// TestRmCorechecksInitAndOverlayShareVolume documents the contract the init
+// container relies on: the seeding mount and the runtime overlay mount refer to
+// the same volume at different paths, so assets copied by the init container are
+// visible under the agent conf.d overlay at runtime.
+func TestRmCorechecksInitAndOverlayShareVolume(t *testing.T) {
+	initMount := GetVolumeMountForRmCorechecksInit()
+	overlayMount := GetVolumeMountForRmCorechecks()
+
+	assert.Equal(t, initMount.Name, overlayMount.Name)
+	assert.NotEqual(t, initMount.MountPath, overlayMount.MountPath)
+
+	// Both mounts must reference the volume produced by GetVolumeForRmCorechecks.
+	vol := GetVolumeForRmCorechecks()
+	for _, vm := range []corev1.VolumeMount{initMount, overlayMount} {
+		assert.Equal(t, vol.Name, vm.Name)
+	}
+}

internal/controller/datadogagent/component/clusterchecksrunner/const.go

@@ -5,6 +5,35 @@
 
 package clusterchecksrunner
 
+import (
+	"github.com/DataDog/datadog-operator/internal/controller/datadogagent/common"
+)
+
 const (
 	pdbMaxUnavailableInstances = 1
+
+	// initCopyCheckAssetsContainerName is the name of the init container that
+	// seeds the conf.d overlay with packaged check assets from the agent image.
+	initCopyCheckAssetsContainerName = "init-copy-check-assets"
+
+	// copyCheckAssetsCmd copies the packaged conf.d assets from the agent image
+	// into the remove-corechecks overlay, then drops the default check
+	// configurations ("*.yaml.default") so default core checks still do not run
+	// on the cluster checks runner. This preserves packaged assets that live
+	// under "conf.d/<check>.d/" subdirectories (for example SNMP profiles under
+	// "snmp.d/default_profiles"), mirroring the Helm chart behaviour and letting
+	// cluster checks such as the SNMP check autodetect profiles.
+	//
+	// The copy uses "cp -RL" rather than "cp -a": the packaged conf.d is owned by
+	// root in the image, and "cp -a" (= --preserve=all) would try to preserve
+	// that ownership. When the runner is forced to run as a non-root UID (e.g. an
+	// OpenShift restricted SCC or a securityContext.runAsUser override) preserving
+	// ownership fails with "Operation not permitted", and with "set -e" that would
+	// block the pod from starting. Copying recursively without preserving
+	// privileged attributes lets the assets be owned by the runtime user instead.
+	copyCheckAssetsCmd = `set -euo pipefail
+if [ -d /etc/datadog-agent/conf.d ]; then
+  cp -RL /etc/datadog-agent/conf.d/. ` + common.RmCorechecksConfdInitPath + `/
+  find ` + common.RmCorechecksConfdInitPath + ` -type f -name '*.yaml.default' -delete
+fi`
 )

internal/controller/datadogagent/component/clusterchecksrunner/default.go

@@ -161,6 +161,28 @@ func defaultPodSpec(dda metav1.Object, volumes []corev1.Volume, volumeMounts []c
 				},
 				VolumeMounts: volumeMountsForInitConfig(),
 			},
+			{
+				// Seed the conf.d overlay with the packaged check assets from
+				// the agent image (e.g. SNMP profiles) while dropping default
+				// check configs, so cluster checks that rely on those assets
+				// (such as SNMP profile autodetection) work on the runner. This
+				// container intentionally does not mount the config or
+				// remove-corechecks volumes over /etc/datadog-agent/conf.d so it
+				// can read the image's packaged conf.d.
+				Name:    initCopyCheckAssetsContainerName,
+				Image:   clusterChecksRunnerImage(),
+				Command: []string{"bash", "-c"},
+				Args: []string{
+					copyCheckAssetsCmd,
+				},
+				VolumeMounts: []corev1.VolumeMount{
+					common.GetVolumeMountForRmCorechecksInit(),
+				},
+				SecurityContext: &corev1.SecurityContext{
+					ReadOnlyRootFilesystem:   new(true),
+					AllowPrivilegeEscalation: new(false),
+				},
+			},
 		},
 		Containers: []corev1.Container{
 			{

internal/controller/datadogagent/component/clusterchecksrunner/default_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,6 +43,52 @@ func Test_getPodDisruptionBudget(t *testing.T) {
 	assert.Nil(t, testpdb.Spec.MinAvailable)
 }
 
+func Test_copyCheckAssetsInitContainer(t *testing.T) {
+	dda := v2alpha1.DatadogAgent{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-datadog-agent",
+			Namespace: "some-namespace",
+		},
+	}
+
+	podSpec := NewDefaultClusterChecksRunnerPodTemplateSpec(dda.GetObjectMeta(), &dda.Spec).Spec
+
+	// Locate the init container that seeds the conf.d overlay with packaged
+	// check assets (SNMP profiles, etc.) so cluster checks can use them.
+	var initContainer *corev1.Container
+	for i := range podSpec.InitContainers {
+		if podSpec.InitContainers[i].Name == initCopyCheckAssetsContainerName {
+			initContainer = &podSpec.InitContainers[i]
+			break
+		}
+	}
+
+	if !assert.NotNil(t, initContainer, "expected a %q init container", initCopyCheckAssetsContainerName) {
+		return
+	}
+
+	// It must mount remove-corechecks at the scratch path and must NOT overlay
+	// /etc/datadog-agent or its conf.d, otherwise it could not read the image's
+	// packaged conf.d assets.
+	assert.Len(t, initContainer.VolumeMounts, 1)
+	assert.Equal(t, common.RmCorechecksVolumeName, initContainer.VolumeMounts[0].Name)
+	assert.Equal(t, common.RmCorechecksConfdInitPath, initContainer.VolumeMounts[0].MountPath)
+	for _, vm := range initContainer.VolumeMounts {
+		assert.NotEqual(t, common.ConfigVolumePath, vm.MountPath)
+		assert.NotEqual(t, common.ConfigVolumePath+"/conf.d", vm.MountPath)
+	}
+
+	// The command copies the image conf.d into the overlay and drops default
+	// check configs while preserving packaged sub-directory assets. It must copy
+	// without preserving ownership (no "cp -a") so it does not fail when the
+	// runner is forced to run as a non-root UID.
+	require.Len(t, initContainer.Args, 1)
+	assert.Contains(t, initContainer.Args[0], "cp -RL /etc/datadog-agent/conf.d/.")
+	assert.NotContains(t, initContainer.Args[0], "cp -a", "cp -a preserves root ownership and breaks non-root runners")
+	assert.Contains(t, initContainer.Args[0], common.RmCorechecksConfdInitPath)
+	assert.Contains(t, initContainer.Args[0], "*.yaml.default")
+}
+
 func TestDefaultEnvVarsJMXUseContainerSupport(t *testing.T) {
 	tests := []struct {
 		name string

internal/controller/testutils/renderer/testdata/golden/comprehensive-aks.golden.yaml

@@ -2168,7 +2168,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    agent.datadoghq.com/agentspechash: 9f9cc18615463016b51e37676cc4b72a
+    agent.datadoghq.com/agentspechash: e5182e5df9f9aa5754b67d69dde750fe
   labels:
     agent.datadoghq.com/component: cluster-checks-runner
     agent.datadoghq.com/name: datadog-agent
@@ -2362,6 +2362,25 @@ spec:
         - mountPath: /checks.d
           name: checksd
           readOnly: true
+      - args:
+        - |-
+          set -euo pipefail
+          if [ -d /etc/datadog-agent/conf.d ]; then
+            cp -RL /etc/datadog-agent/conf.d/. /opt/conf.d-overlay/
+            find /opt/conf.d-overlay -type f -name '*.yaml.default' -delete
+          fi
+        command:
+        - bash
+        - -c
+        image: registry.datadoghq.com/agent:7.81.1
+        name: init-copy-check-assets
+        resources: {}
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /opt/conf.d-overlay
+          name: remove-corechecks
       serviceAccountName: datadog-agent-cluster-checks-runner
       volumes:
       - configMap:

internal/controller/testutils/renderer/testdata/golden/comprehensive-autopilot.golden.yaml

@@ -2133,7 +2133,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    agent.datadoghq.com/agentspechash: 9bb3d4e42d99fde0ef482717a4d139d7
+    agent.datadoghq.com/agentspechash: 977561eccc4bc79683f1bbbbf100d060
   labels:
     agent.datadoghq.com/component: cluster-checks-runner
     agent.datadoghq.com/name: datadog-agent
@@ -2331,6 +2331,25 @@ spec:
         - mountPath: /checks.d
           name: checksd
           readOnly: true
+      - args:
+        - |-
+          set -euo pipefail
+          if [ -d /etc/datadog-agent/conf.d ]; then
+            cp -RL /etc/datadog-agent/conf.d/. /opt/conf.d-overlay/
+            find /opt/conf.d-overlay -type f -name '*.yaml.default' -delete
+          fi
+        command:
+        - bash
+        - -c
+        image: gcr.io/datadoghq/agent:7.81.1
+        name: init-copy-check-assets
+        resources: {}
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /opt/conf.d-overlay
+          name: remove-corechecks
       serviceAccountName: datadog-agent-cluster-checks-runner
       volumes:
       - configMap:

internal/controller/testutils/renderer/testdata/golden/comprehensive-baseline.golden.yaml

@@ -2165,7 +2165,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    agent.datadoghq.com/agentspechash: 9f9cc18615463016b51e37676cc4b72a
+    agent.datadoghq.com/agentspechash: e5182e5df9f9aa5754b67d69dde750fe
   labels:
     agent.datadoghq.com/component: cluster-checks-runner
     agent.datadoghq.com/name: datadog-agent
@@ -2359,6 +2359,25 @@ spec:
         - mountPath: /checks.d
           name: checksd
           readOnly: true
+      - args:
+        - |-
+          set -euo pipefail
+          if [ -d /etc/datadog-agent/conf.d ]; then
+            cp -RL /etc/datadog-agent/conf.d/. /opt/conf.d-overlay/
+            find /opt/conf.d-overlay -type f -name '*.yaml.default' -delete
+          fi
+        command:
+        - bash
+        - -c
+        image: registry.datadoghq.com/agent:7.81.1
+        name: init-copy-check-assets
+        resources: {}
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /opt/conf.d-overlay
+          name: remove-corechecks
       serviceAccountName: datadog-agent-cluster-checks-runner
       volumes:
       - configMap:

internal/controller/testutils/renderer/testdata/golden/comprehensive-eks-hostname-from-file.golden.yaml

@@ -2187,7 +2187,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    agent.datadoghq.com/agentspechash: 9f9cc18615463016b51e37676cc4b72a
+    agent.datadoghq.com/agentspechash: e5182e5df9f9aa5754b67d69dde750fe
   labels:
     agent.datadoghq.com/component: cluster-checks-runner
     agent.datadoghq.com/name: datadog-agent
@@ -2381,6 +2381,25 @@ spec:
         - mountPath: /checks.d
           name: checksd
           readOnly: true
+      - args:
+        - |-
+          set -euo pipefail
+          if [ -d /etc/datadog-agent/conf.d ]; then
+            cp -RL /etc/datadog-agent/conf.d/. /opt/conf.d-overlay/
+            find /opt/conf.d-overlay -type f -name '*.yaml.default' -delete
+          fi
+        command:
+        - bash
+        - -c
+        image: registry.datadoghq.com/agent:7.81.1
+        name: init-copy-check-assets
+        resources: {}
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /opt/conf.d-overlay
+          name: remove-corechecks
       serviceAccountName: datadog-agent-cluster-checks-runner
       volumes:
       - configMap: