Skip to content

Commit 9a1b585

Browse files
frzifusfrzifus
authored
add featuregate for k8s 1.28 native sidecar container (#2801)
* add featuregate for k8s 1.28 native sidecar container Signed-off-by: Benedikt Bongartz <[email protected]> * use slices methods to deal with sidecars Signed-off-by: Benedikt Bongartz <[email protected]> * apply recommendations Signed-off-by: Benedikt Bongartz <[email protected]> * test with k8s 1.29 Signed-off-by: Benedikt Bongartz <[email protected]> --------- Signed-off-by: Benedikt Bongartz <[email protected]>
1 parent 8de23cc commit 9a1b585

File tree

9 files changed

+259
-9
lines changed

9 files changed

+259
-9
lines changed

‎.chloggen/native_sidecar.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: enhancement
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
5+
component: collector
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: Add native sidecar injection behind a feature gate which is disabled by default.
9+
10+
# One or more tracking issues related to the change
11+
issues: [2376]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext: |
17+
Native sidecars are supported since Kubernetes version `1.28` and are availabe by default since `1.29`.
18+
To use native sidecars on Kubernetes v1.28 make sure the "SidecarContainers" feature gate on kubernetes is enabled.
19+
If native sidecars are available, the operator can be advised to use them by adding adding
20+
the `--feature-gates=operator.sidecarcontainers.native` to the Operator args.
21+
In the future this may will become availabe as deployment mode on the Collector CR. See [#3356](https://github.com/open-telemetry/opentelemetry-operator/issues/3356)

‎.github/workflows/e2e.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ jobs:
4646
setup: "add-operator-arg OPERATOR_ARG='--feature-gates=operator.targetallocator.mtls' add-certmanager-permissions prepare-e2e"
4747
- group: e2e-automatic-rbac
4848
setup: "add-rbac-permissions-to-operator prepare-e2e"
49+
- group: e2e-native-sidecar
50+
setup: "add-operator-arg OPERATOR_ARG='--feature-gates=operator.sidecarcontainers.native' prepare-e2e"
51+
kube-version: "1.29"
4952
steps:
5053
- name: Check out code into the Go module directory
5154
uses: actions/checkout@v4

‎Makefile

+7
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,13 @@ generate: controller-gen
267267
e2e: chainsaw
268268
$(CHAINSAW) test --test-dir ./tests/e2e
269269

270+
# e2e-native-sidecar
271+
# NOTE: make sure the k8s featuregate "SidecarContainers" is set to true.
272+
# NOTE: make sure the operator featuregate "operator.sidecarcontainers.native" is enabled.
273+
.PHONY: e2e-native-sidecar
274+
e2e-native-sidecar: chainsaw
275+
$(CHAINSAW) test --test-dir ./tests/e2e-native-sidecar
276+
270277
# end-to-end-test for testing automatic RBAC creation
271278
.PHONY: e2e-automatic-rbac
272279
e2e-automatic-rbac: chainsaw

‎pkg/featuregate/featuregate.go

+12
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,18 @@ const (
2525
)
2626

2727
var (
28+
// EnableNativeSidecarContainers is the feature gate that controls whether a
29+
// sidecar should be injected as a native sidecar or the classic way.
30+
// Native sidecar containers have been available since kubernetes v1.28 in
31+
// alpha and v1.29 in beta.
32+
// It needs to be enabled with +featureGate=SidecarContainers.
33+
// See:
34+
// https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features
35+
EnableNativeSidecarContainers = featuregate.GlobalRegistry().MustRegister(
36+
"operator.sidecarcontainers.native",
37+
featuregate.StageAlpha,
38+
featuregate.WithRegisterDescription("controls whether the operator supports sidecar containers as init containers. Should only be enabled on k8s v1.29+"),
39+
)
2840
// PrometheusOperatorIsAvailable is the feature gate that enables features associated to the Prometheus Operator.
2941
PrometheusOperatorIsAvailable = featuregate.GlobalRegistry().MustRegister(
3042
"operator.observability.prometheus",

‎pkg/sidecar/pod.go

+29-9
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package sidecar
1717

1818
import (
1919
"fmt"
20+
"slices"
2021

2122
"github.com/go-logr/logr"
2223
corev1 "k8s.io/api/core/v1"
@@ -25,6 +26,7 @@ import (
2526
"github.com/open-telemetry/opentelemetry-operator/internal/config"
2627
"github.com/open-telemetry/opentelemetry-operator/internal/manifests/collector"
2728
"github.com/open-telemetry/opentelemetry-operator/internal/naming"
29+
"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
2830
)
2931

3032
const (
@@ -47,7 +49,17 @@ func add(cfg config.Config, logger logr.Logger, otelcol v1beta1.OpenTelemetryCol
4749
container.Env = append(container.Env, attributes...)
4850
}
4951
pod.Spec.InitContainers = append(pod.Spec.InitContainers, otelcol.Spec.InitContainers...)
50-
pod.Spec.Containers = append(pod.Spec.Containers, container)
52+
53+
if featuregate.EnableNativeSidecarContainers.IsEnabled() {
54+
policy := corev1.ContainerRestartPolicyAlways
55+
container.RestartPolicy = &policy
56+
// NOTE: Use ReadinessProbe as startup probe.
57+
// See https://github.com/open-telemetry/opentelemetry-operator/pull/2801#discussion_r1547571121
58+
container.StartupProbe = container.ReadinessProbe
59+
pod.Spec.InitContainers = append(pod.Spec.InitContainers, container)
60+
} else {
61+
pod.Spec.Containers = append(pod.Spec.Containers, container)
62+
}
5163
pod.Spec.Volumes = append(pod.Spec.Volumes, otelcol.Spec.Volumes...)
5264

5365
if pod.Labels == nil {
@@ -58,26 +70,34 @@ func add(cfg config.Config, logger logr.Logger, otelcol v1beta1.OpenTelemetryCol
5870
return pod, nil
5971
}
6072

73+
func isOtelColContainer(c corev1.Container) bool { return c.Name == naming.Container() }
74+
6175
// remove the sidecar container from the given pod.
6276
func remove(pod corev1.Pod) corev1.Pod {
6377
if !existsIn(pod) {
6478
return pod
6579
}
6680

67-
var containers []corev1.Container
68-
for _, container := range pod.Spec.Containers {
69-
if container.Name != naming.Container() {
70-
containers = append(containers, container)
71-
}
81+
pod.Spec.Containers = slices.DeleteFunc(pod.Spec.Containers, isOtelColContainer)
82+
83+
if featuregate.EnableNativeSidecarContainers.IsEnabled() {
84+
// NOTE: we also remove init containers (native sidecars) since k8s 1.28.
85+
// This should have no side effects.
86+
pod.Spec.InitContainers = slices.DeleteFunc(pod.Spec.InitContainers, isOtelColContainer)
7287
}
73-
pod.Spec.Containers = containers
7488
return pod
7589
}
7690

7791
// existsIn checks whether a sidecar container exists in the given pod.
7892
func existsIn(pod corev1.Pod) bool {
79-
for _, container := range pod.Spec.Containers {
80-
if container.Name == naming.Container() {
93+
if slices.ContainsFunc(pod.Spec.Containers, isOtelColContainer) {
94+
return true
95+
}
96+
97+
if featuregate.EnableNativeSidecarContainers.IsEnabled() {
98+
// NOTE: we also check init containers (native sidecars) since k8s 1.28.
99+
// This should have no side effects.
100+
if slices.ContainsFunc(pod.Spec.InitContainers, isOtelColContainer) {
81101
return true
82102
}
83103
}

‎pkg/sidecar/pod_test.go

+110
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,107 @@ import (
1919

2020
"github.com/stretchr/testify/assert"
2121
"github.com/stretchr/testify/require"
22+
colfeaturegate "go.opentelemetry.io/collector/featuregate"
2223
corev1 "k8s.io/api/core/v1"
2324
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2425
logf "sigs.k8s.io/controller-runtime/pkg/log"
2526

2627
"github.com/open-telemetry/opentelemetry-operator/apis/v1beta1"
2728
"github.com/open-telemetry/opentelemetry-operator/internal/config"
2829
"github.com/open-telemetry/opentelemetry-operator/internal/naming"
30+
"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
2931
)
3032

3133
var logger = logf.Log.WithName("unit-tests")
3234

35+
func enableSidecarFeatureGate(t *testing.T) {
36+
originalVal := featuregate.EnableNativeSidecarContainers.IsEnabled()
37+
t.Logf("original is: %+v", originalVal)
38+
require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNativeSidecarContainers.ID(), true))
39+
t.Cleanup(func() {
40+
require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNativeSidecarContainers.ID(), originalVal))
41+
})
42+
}
43+
44+
func TestAddNativeSidecar(t *testing.T) {
45+
enableSidecarFeatureGate(t)
46+
// prepare
47+
pod := corev1.Pod{
48+
Spec: corev1.PodSpec{
49+
Containers: []corev1.Container{
50+
{Name: "my-app"},
51+
},
52+
InitContainers: []corev1.Container{
53+
{
54+
Name: "my-init",
55+
},
56+
},
57+
// cross-test: the pod has a volume already, make sure we don't remove it
58+
Volumes: []corev1.Volume{{}},
59+
},
60+
}
61+
62+
otelcol := v1beta1.OpenTelemetryCollector{
63+
ObjectMeta: metav1.ObjectMeta{
64+
Name: "otelcol-native-sidecar",
65+
Namespace: "some-app",
66+
},
67+
Spec: v1beta1.OpenTelemetryCollectorSpec{
68+
Mode: v1beta1.ModeSidecar,
69+
OpenTelemetryCommonFields: v1beta1.OpenTelemetryCommonFields{
70+
InitContainers: []corev1.Container{
71+
{
72+
Name: "test",
73+
},
74+
},
75+
},
76+
},
77+
}
78+
79+
otelcolYaml, err := otelcol.Spec.Config.Yaml()
80+
require.NoError(t, err)
81+
cfg := config.New(config.WithCollectorImage("some-default-image"))
82+
83+
// test
84+
changed, err := add(cfg, logger, otelcol, pod, nil)
85+
86+
// verify
87+
assert.NoError(t, err)
88+
require.Len(t, changed.Spec.Containers, 1)
89+
require.Len(t, changed.Spec.InitContainers, 3)
90+
require.Len(t, changed.Spec.Volumes, 1)
91+
assert.Equal(t, "some-app.otelcol-native-sidecar",
92+
changed.Labels["sidecar.opentelemetry.io/injected"])
93+
expectedPolicy := corev1.ContainerRestartPolicyAlways
94+
assert.Equal(t, corev1.Container{
95+
Name: "otc-container",
96+
Image: "some-default-image",
97+
Args: []string{"--config=env:OTEL_CONFIG"},
98+
RestartPolicy: &expectedPolicy,
99+
Env: []corev1.EnvVar{
100+
{
101+
Name: "POD_NAME",
102+
ValueFrom: &corev1.EnvVarSource{
103+
FieldRef: &corev1.ObjectFieldSelector{
104+
FieldPath: "metadata.name",
105+
},
106+
},
107+
},
108+
{
109+
Name: "OTEL_CONFIG",
110+
Value: string(otelcolYaml),
111+
},
112+
},
113+
Ports: []corev1.ContainerPort{
114+
{
115+
Name: "metrics",
116+
ContainerPort: 8888,
117+
Protocol: corev1.ProtocolTCP,
118+
},
119+
},
120+
}, changed.Spec.InitContainers[2])
121+
}
122+
33123
func TestAddSidecarWhenNoSidecarExists(t *testing.T) {
34124
// prepare
35125
pod := corev1.Pod{
@@ -146,6 +236,11 @@ func TestRemoveSidecar(t *testing.T) {
146236
{Name: naming.Container()},
147237
{Name: naming.Container()}, // two sidecars! should remove both
148238
},
239+
InitContainers: []corev1.Container{
240+
{Name: "something"},
241+
{Name: naming.Container()}, // NOTE: native sidecar since k8s 1.28.
242+
{Name: naming.Container()}, // two sidecars! should remove both
243+
},
149244
},
150245
}
151246

@@ -174,6 +269,8 @@ func TestRemoveNonExistingSidecar(t *testing.T) {
174269
}
175270

176271
func TestExistsIn(t *testing.T) {
272+
enableSidecarFeatureGate(t)
273+
177274
for _, tt := range []struct {
178275
desc string
179276
pod corev1.Pod
@@ -190,6 +287,19 @@ func TestExistsIn(t *testing.T) {
190287
},
191288
true},
192289

290+
{"does-have-native-sidecar",
291+
corev1.Pod{
292+
Spec: corev1.PodSpec{
293+
Containers: []corev1.Container{
294+
{Name: "my-app"},
295+
},
296+
InitContainers: []corev1.Container{
297+
{Name: naming.Container()},
298+
},
299+
},
300+
},
301+
true},
302+
193303
{"does-not-have-sidecar",
194304
corev1.Pod{
195305
Spec: corev1.PodSpec{

‎tests/e2e-native-sidecar/00-assert.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
apiVersion: v1
3+
kind: Pod
4+
metadata:
5+
annotations:
6+
sidecar.opentelemetry.io/inject: "true"
7+
name: myapp
8+
spec:
9+
containers:
10+
- name: myapp
11+
initContainers:
12+
- name: otc-container
13+
restartPolicy: Always
14+
status:
15+
containerStatuses:
16+
- name: myapp
17+
ready: true
18+
started: true
19+
initContainerStatuses:
20+
- name: otc-container
21+
ready: true
22+
started: true

‎tests/e2e-native-sidecar/00-install.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
---
2+
apiVersion: opentelemetry.io/v1beta1
3+
kind: OpenTelemetryCollector
4+
metadata:
5+
name: a-sidecar
6+
spec:
7+
mode: sidecar
8+
resources:
9+
limits:
10+
cpu: 500m
11+
memory: 128Mi
12+
requests:
13+
cpu: 5m
14+
memory: 64Mi
15+
16+
config:
17+
receivers:
18+
otlp:
19+
protocols:
20+
http: {}
21+
exporters:
22+
debug: {}
23+
service:
24+
pipelines:
25+
metrics:
26+
receivers: [otlp]
27+
exporters: [debug]
28+
---
29+
apiVersion: v1
30+
kind: Pod
31+
metadata:
32+
name: myapp
33+
annotations:
34+
sidecar.opentelemetry.io/inject: "true"
35+
spec:
36+
containers:
37+
- name: myapp
38+
image: ghcr.io/open-telemetry/opentelemetry-operator/e2e-test-app-python:main
39+
ports:
40+
- containerPort: 8080
41+
protocol: TCP

‎tests/e2e-native-sidecar/chainsaw-test.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
2+
apiVersion: chainsaw.kyverno.io/v1alpha1
3+
kind: Test
4+
metadata:
5+
creationTimestamp: null
6+
name: native-sidecar
7+
spec:
8+
steps:
9+
- name: step-00
10+
try:
11+
- apply:
12+
file: 00-install.yaml
13+
- assert:
14+
file: 00-assert.yaml

0 commit comments

Comments
 (0)