add featuregate for k8s 1.28 native sidecar container

.chloggen/native_sidecar.yaml

@@ -0,0 +1,21 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
+component: collector
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add native sidecar injection behind a feature gate which is disabled by default.
+
+# One or more tracking issues related to the change
+issues: [2376]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Native sidecars are supported since Kubernetes version `1.28` and are availabe by default since `1.29`.
+  To use native sidecars on Kubernetes v1.28 make sure the "SidecarContainers" feature gate on kubernetes is enabled.
+  If native sidecars are available, the operator can be advised to use them by adding adding
+  the `--feature-gates=operator.sidecarcontainers.native` to the Operator args.
+  In the future this may will become availabe as deployment mode on the Collector CR. See [#3356](https://github.com/open-telemetry/opentelemetry-operator/issues/3356)

.github/workflows/e2e.yaml

@@ -46,6 +46,9 @@ jobs:
            setup: "add-operator-arg OPERATOR_ARG='--feature-gates=operator.targetallocator.mtls' add-certmanager-permissions prepare-e2e"
          - group: e2e-automatic-rbac
            setup: "add-rbac-permissions-to-operator prepare-e2e"
+         - group: e2e-native-sidecar
+           setup: "add-operator-arg OPERATOR_ARG='--feature-gates=operator.sidecarcontainers.native' prepare-e2e"
+           kube-version: "1.31"
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v4

Makefile

@@ -267,6 +267,13 @@ generate: controller-gen
 e2e: chainsaw
 	$(CHAINSAW) test --test-dir ./tests/e2e
 
+# e2e-native-sidecar
+# NOTE: make sure the k8s featuregate "SidecarContainers" is set to true.
+# NOTE: make sure the operator featuregate "operator.sidecarcontainers.native" is enabled.
+.PHONY: e2e-native-sidecar
+e2e-native-sidecar: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-native-sidecar
+
 # end-to-end-test for testing automatic RBAC creation
 .PHONY: e2e-automatic-rbac
 e2e-automatic-rbac: chainsaw

pkg/featuregate/featuregate.go

@@ -25,6 +25,18 @@ const (
 )
 
 var (
+	// EnableNativeSidecarContainers is the feature gate that controls whether a
+	// sidecar should be injected as a native sidecar or the classic way.
+	// Native sidecar containers have been available since kubernetes v1.28 in
+	// alpha and v1.29 in beta.
+	// It needs to be enabled with +featureGate=SidecarContainers.
+	// See:
+	// https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features
+	EnableNativeSidecarContainers = featuregate.GlobalRegistry().MustRegister(
+		"operator.sidecarcontainers.native",
+		featuregate.StageAlpha,
+		featuregate.WithRegisterDescription("controls whether the operator supports sidecar containers as init containers"),
+	)
 	// PrometheusOperatorIsAvailable is the feature gate that enables features associated to the Prometheus Operator.
 	PrometheusOperatorIsAvailable = featuregate.GlobalRegistry().MustRegister(
 		"operator.observability.prometheus",

pkg/sidecar/pod.go

@@ -17,6 +17,7 @@ package sidecar
 
 import (
 	"fmt"
+	"slices"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
@@ -25,6 +26,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-operator/internal/config"
 	"github.com/open-telemetry/opentelemetry-operator/internal/manifests/collector"
 	"github.com/open-telemetry/opentelemetry-operator/internal/naming"
+	"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
 )
 
 const (
@@ -47,7 +49,17 @@ func add(cfg config.Config, logger logr.Logger, otelcol v1beta1.OpenTelemetryCol
 		container.Env = append(container.Env, attributes...)
 	}
 	pod.Spec.InitContainers = append(pod.Spec.InitContainers, otelcol.Spec.InitContainers...)
-	pod.Spec.Containers = append(pod.Spec.Containers, container)
+
+	if featuregate.EnableNativeSidecarContainers.IsEnabled() {
+		policy := corev1.ContainerRestartPolicyAlways
+		container.RestartPolicy = &policy
+		// NOTE: Use ReadinessProbe as startup probe.
+		// See https://github.com/open-telemetry/opentelemetry-operator/pull/2801#discussion_r1547571121
+		container.StartupProbe = container.ReadinessProbe
+		pod.Spec.InitContainers = append(pod.Spec.InitContainers, container)
+	} else {
+		pod.Spec.Containers = append(pod.Spec.Containers, container)
+	}
 	pod.Spec.Volumes = append(pod.Spec.Volumes, otelcol.Spec.Volumes...)
 
 	if pod.Labels == nil {
@@ -58,26 +70,34 @@ func add(cfg config.Config, logger logr.Logger, otelcol v1beta1.OpenTelemetryCol
 	return pod, nil
 }
 
+func isOtelColContainer(c corev1.Container) bool { return c.Name == naming.Container() }
+
 // remove the sidecar container from the given pod.
 func remove(pod corev1.Pod) corev1.Pod {
 	if !existsIn(pod) {
 		return pod
 	}
 
-	var containers []corev1.Container
-	for _, container := range pod.Spec.Containers {
-		if container.Name != naming.Container() {
-			containers = append(containers, container)
-		}
+	pod.Spec.Containers = slices.DeleteFunc(pod.Spec.Containers, isOtelColContainer)
+
+	if featuregate.EnableNativeSidecarContainers.IsEnabled() {
+		// NOTE: we also remove init containers (native sidecars) since k8s 1.28.
+		// This should have no side effects.
+		pod.Spec.InitContainers = slices.DeleteFunc(pod.Spec.InitContainers, isOtelColContainer)
 	}
-	pod.Spec.Containers = containers
 	return pod
 }
 
 // existsIn checks whether a sidecar container exists in the given pod.
 func existsIn(pod corev1.Pod) bool {
-	for _, container := range pod.Spec.Containers {
-		if container.Name == naming.Container() {
+	if slices.ContainsFunc(pod.Spec.Containers, isOtelColContainer) {
+		return true
+	}
+
+	if featuregate.EnableNativeSidecarContainers.IsEnabled() {
+		// NOTE: we also check init containers (native sidecars) since k8s 1.28.
+		// This should have no side effects.
+		if slices.ContainsFunc(pod.Spec.InitContainers, isOtelColContainer) {
 			return true
 		}
 	}

pkg/sidecar/pod_test.go

@@ -19,17 +19,107 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	colfeaturegate "go.opentelemetry.io/collector/featuregate"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/open-telemetry/opentelemetry-operator/apis/v1beta1"
 	"github.com/open-telemetry/opentelemetry-operator/internal/config"
 	"github.com/open-telemetry/opentelemetry-operator/internal/naming"
+	"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
 )
 
 var logger = logf.Log.WithName("unit-tests")
 
+func enableSidecarFeatureGate(t *testing.T) {
+	originalVal := featuregate.EnableNativeSidecarContainers.IsEnabled()
+	t.Logf("original is: %+v", originalVal)
+	require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNativeSidecarContainers.ID(), true))
+	t.Cleanup(func() {
+		require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNativeSidecarContainers.ID(), originalVal))
+	})
+}
+
+func TestAddNativeSidecar(t *testing.T) {
+	enableSidecarFeatureGate(t)
+	// prepare
+	pod := corev1.Pod{
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{Name: "my-app"},
+			},
+			InitContainers: []corev1.Container{
+				{
+					Name: "my-init",
+				},
+			},
+			// cross-test: the pod has a volume already, make sure we don't remove it
+			Volumes: []corev1.Volume{{}},
+		},
+	}
+
+	otelcol := v1beta1.OpenTelemetryCollector{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "otelcol-native-sidecar",
+			Namespace: "some-app",
+		},
+		Spec: v1beta1.OpenTelemetryCollectorSpec{
+			Mode: v1beta1.ModeSidecar,
+			OpenTelemetryCommonFields: v1beta1.OpenTelemetryCommonFields{
+				InitContainers: []corev1.Container{
+					{
+						Name: "test",
+					},
+				},
+			},
+		},
+	}
+
+	otelcolYaml, err := otelcol.Spec.Config.Yaml()
+	require.NoError(t, err)
+	cfg := config.New(config.WithCollectorImage("some-default-image"))
+
+	// test
+	changed, err := add(cfg, logger, otelcol, pod, nil)
+
+	// verify
+	assert.NoError(t, err)
+	require.Len(t, changed.Spec.Containers, 1)
+	require.Len(t, changed.Spec.InitContainers, 3)
+	require.Len(t, changed.Spec.Volumes, 1)
+	assert.Equal(t, "some-app.otelcol-native-sidecar",
+		changed.Labels["sidecar.opentelemetry.io/injected"])
+	expectedPolicy := corev1.ContainerRestartPolicyAlways
+	assert.Equal(t, corev1.Container{
+		Name:          "otc-container",
+		Image:         "some-default-image",
+		Args:          []string{"--config=env:OTEL_CONFIG"},
+		RestartPolicy: &expectedPolicy,
+		Env: []corev1.EnvVar{
+			{
+				Name: "POD_NAME",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "metadata.name",
+					},
+				},
+			},
+			{
+				Name:  "OTEL_CONFIG",
+				Value: string(otelcolYaml),
+			},
+		},
+		Ports: []corev1.ContainerPort{
+			{
+				Name:          "metrics",
+				ContainerPort: 8888,
+				Protocol:      corev1.ProtocolTCP,
+			},
+		},
+	}, changed.Spec.InitContainers[2])
+}
+
 func TestAddSidecarWhenNoSidecarExists(t *testing.T) {
 	// prepare
 	pod := corev1.Pod{
@@ -146,6 +236,11 @@ func TestRemoveSidecar(t *testing.T) {
 				{Name: naming.Container()},
 				{Name: naming.Container()}, // two sidecars! should remove both
 			},
+			InitContainers: []corev1.Container{
+				{Name: "something"},
+				{Name: naming.Container()}, // NOTE: native sidecar since k8s 1.28.
+				{Name: naming.Container()}, // two sidecars! should remove both
+			},
 		},
 	}
 
@@ -174,6 +269,8 @@ func TestRemoveNonExistingSidecar(t *testing.T) {
 }
 
 func TestExistsIn(t *testing.T) {
+	enableSidecarFeatureGate(t)
+
 	for _, tt := range []struct {
 		desc     string
 		pod      corev1.Pod
@@ -190,6 +287,19 @@ func TestExistsIn(t *testing.T) {
 			},
 			true},
 
+		{"does-have-native-sidecar",
+			corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{Name: "my-app"},
+					},
+					InitContainers: []corev1.Container{
+						{Name: naming.Container()},
+					},
+				},
+			},
+			true},
+
 		{"does-not-have-sidecar",
 			corev1.Pod{
 				Spec: corev1.PodSpec{

tests/e2e-native-sidecar/00-assert.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    sidecar.opentelemetry.io/inject: "true"
+  name: myapp
+spec:
+  containers:
+  - name: myapp
+  initContainers:
+  - name: otc-container
+    restartPolicy: Always
+status:
+  containerStatuses:
+  - name: myapp
+    ready: true
+    started: true
+  initContainerStatuses:
+  - name: otc-container
+    ready: true
+    started: true

tests/e2e-native-sidecar/00-install.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: a-sidecar
+spec:
+  mode: sidecar
+  resources:
+    limits:
+      cpu: 500m
+      memory: 128Mi
+    requests:
+      cpu: 5m
+      memory: 64Mi
+
+  config:
+    receivers:
+      otlp:
+        protocols:
+          http: {}
+    exporters:
+      debug: {}
+    service:
+      pipelines:
+        metrics:
+          receivers: [otlp]
+          exporters: [debug]
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp
+  annotations:
+    sidecar.opentelemetry.io/inject: "true"
+spec:
+  containers:
+  - name: myapp
+    image: ghcr.io/open-telemetry/opentelemetry-operator/e2e-test-app-python:main
+    ports:
+      - containerPort: 8080
+        protocol: TCP

tests/e2e-native-sidecar/chainsaw-test.yaml

@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: native-sidecar
+spec:
+  steps:
+  - name: step-00
+    try:
+    - apply:
+        file: 00-install.yaml
+    - assert:
+        file: 00-assert.yaml