Skip to content

Commit f141dcb

Browse files
frzifusfrzifus
committed
add featuregate for k8s 1.28 native sidecar container
Signed-off-by: Benedikt Bongartz <[email protected]>
1 parent b038590 commit f141dcb

File tree

8 files changed

+243
-1
lines changed

8 files changed

+243
-1
lines changed

‎.chloggen/native_sidecar.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: enhancement
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
5+
component: pkg/sidecar
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: Add native sidecar injection behind a feature gate which is disabled by default.
9+
10+
# One or more tracking issues related to the change
11+
issues: [2376]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext:

‎Makefile

+7
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,13 @@ generate: controller-gen
267267
e2e: chainsaw
268268
$(CHAINSAW) test --test-dir ./tests/e2e
269269

270+
# e2e-native-sidecar
271+
# NOTE: make sure the k8s featuregate "SidecarContainers" is set to true.
272+
# NOTE: make sure the operator featuregate "operator.sidecarcontainers.native" is enabled.
273+
.PHONY: e2e-native-sidecar
274+
e2e-native-sidecar: chainsaw
275+
$(CHAINSAW) test --test-dir ./tests/e2e-native-sidecar
276+
270277
# end-to-end-test for testing automatic RBAC creation
271278
.PHONY: e2e-automatic-rbac
272279
e2e-automatic-rbac: chainsaw

‎pkg/featuregate/featuregate.go

+13
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,19 @@ const (
2525
)
2626

2727
var (
28+
// EnableNativeSidecarContainers is the feature gate that controls whether a
29+
// sidecar should be injected as a native sidecar or the classic way.
30+
// Native sidecar containers have been available since kubernetes v1.28 in
31+
// alpha and v1.29 in beta.
32+
// It needs to be enabled with +featureGate=SidecarContainers.
33+
// See:
34+
// https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features
35+
EnableNativeSidecarContainers = featuregate.GlobalRegistry().MustRegister(
36+
"operator.sidecarcontainers.native",
37+
featuregate.StageAlpha,
38+
featuregate.WithRegisterDescription("controls whether the operator supports sidecar containers as init containers"),
39+
featuregate.WithRegisterFromVersion("v0.105.0"),
40+
)
2841
// PrometheusOperatorIsAvailable is the feature gate that enables features associated to the Prometheus Operator.
2942
PrometheusOperatorIsAvailable = featuregate.GlobalRegistry().MustRegister(
3043
"operator.observability.prometheus",

‎pkg/sidecar/pod.go

+29-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/open-telemetry/opentelemetry-operator/internal/config"
2626
"github.com/open-telemetry/opentelemetry-operator/internal/manifests/collector"
2727
"github.com/open-telemetry/opentelemetry-operator/internal/naming"
28+
"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
2829
)
2930

3031
const (
@@ -47,7 +48,17 @@ func add(cfg config.Config, logger logr.Logger, otelcol v1beta1.OpenTelemetryCol
4748
container.Env = append(container.Env, attributes...)
4849
}
4950
pod.Spec.InitContainers = append(pod.Spec.InitContainers, otelcol.Spec.InitContainers...)
50-
pod.Spec.Containers = append(pod.Spec.Containers, container)
51+
52+
if featuregate.EnableNativeSidecarContainers.IsEnabled() {
53+
policy := corev1.ContainerRestartPolicyAlways
54+
container.RestartPolicy = &policy
55+
// NOTE: Use ReadinessProbe as startup probe.
56+
// See https://github.com/open-telemetry/opentelemetry-operator/pull/2801#discussion_r1547571121
57+
container.StartupProbe = container.StartupProbe
58+
pod.Spec.InitContainers = append(pod.Spec.InitContainers, container)
59+
} else {
60+
pod.Spec.Containers = append(pod.Spec.Containers, container)
61+
}
5162
pod.Spec.Volumes = append(pod.Spec.Volumes, otelcol.Spec.Volumes...)
5263

5364
if pod.Labels == nil {
@@ -71,6 +82,16 @@ func remove(pod corev1.Pod) corev1.Pod {
7182
}
7283
}
7384
pod.Spec.Containers = containers
85+
86+
// NOTE: we also remove init containers (native sidecars) since k8s 1.28.
87+
// This should have no side effects.
88+
var initContainers []corev1.Container
89+
for _, initContainer := range pod.Spec.InitContainers {
90+
if initContainer.Name != naming.Container() {
91+
initContainers = append(initContainers, initContainer)
92+
}
93+
}
94+
pod.Spec.InitContainers = initContainers
7495
return pod
7596
}
7697

@@ -81,5 +102,12 @@ func existsIn(pod corev1.Pod) bool {
81102
return true
82103
}
83104
}
105+
// NOTE: we also check init containers (native sidecars) since k8s 1.28.
106+
// This should have no side effects.
107+
for _, container := range pod.Spec.InitContainers {
108+
if container.Name == naming.Container() {
109+
return true
110+
}
111+
}
84112
return false
85113
}

‎pkg/sidecar/pod_test.go

+108
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,107 @@ import (
1919

2020
"github.com/stretchr/testify/assert"
2121
"github.com/stretchr/testify/require"
22+
colfeaturegate "go.opentelemetry.io/collector/featuregate"
2223
corev1 "k8s.io/api/core/v1"
2324
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2425
logf "sigs.k8s.io/controller-runtime/pkg/log"
2526

2627
"github.com/open-telemetry/opentelemetry-operator/apis/v1beta1"
2728
"github.com/open-telemetry/opentelemetry-operator/internal/config"
2829
"github.com/open-telemetry/opentelemetry-operator/internal/naming"
30+
"github.com/open-telemetry/opentelemetry-operator/pkg/featuregate"
2931
)
3032

3133
var logger = logf.Log.WithName("unit-tests")
3234

35+
func sidecarFeatureGate(t *testing.T) {
36+
originalVal := featuregate.EnableNativeSidecarContainers.IsEnabled()
37+
t.Logf("original is: %+v", originalVal)
38+
require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNativeSidecarContainers.ID(), true))
39+
t.Cleanup(func() {
40+
require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNativeSidecarContainers.ID(), originalVal))
41+
})
42+
}
43+
44+
func TestAddNativeSidecar(t *testing.T) {
45+
sidecarFeatureGate(t)
46+
// prepare
47+
pod := corev1.Pod{
48+
Spec: corev1.PodSpec{
49+
Containers: []corev1.Container{
50+
{Name: "my-app"},
51+
},
52+
InitContainers: []corev1.Container{
53+
{
54+
Name: "my-init",
55+
},
56+
},
57+
// cross-test: the pod has a volume already, make sure we don't remove it
58+
Volumes: []corev1.Volume{{}},
59+
},
60+
}
61+
62+
otelcol := v1beta1.OpenTelemetryCollector{
63+
ObjectMeta: metav1.ObjectMeta{
64+
Name: "otelcol-native-sidecar",
65+
Namespace: "some-app",
66+
},
67+
Spec: v1beta1.OpenTelemetryCollectorSpec{
68+
Mode: v1beta1.ModeSidecar,
69+
OpenTelemetryCommonFields: v1beta1.OpenTelemetryCommonFields{
70+
InitContainers: []corev1.Container{
71+
{
72+
Name: "test",
73+
},
74+
},
75+
},
76+
},
77+
}
78+
79+
otelcolYaml, err := otelcol.Spec.Config.Yaml()
80+
require.NoError(t, err)
81+
cfg := config.New(config.WithCollectorImage("some-default-image"))
82+
83+
// test
84+
changed, err := add(cfg, logger, otelcol, pod, nil)
85+
86+
// verify
87+
assert.NoError(t, err)
88+
require.Len(t, changed.Spec.Containers, 1)
89+
require.Len(t, changed.Spec.InitContainers, 3)
90+
require.Len(t, changed.Spec.Volumes, 1)
91+
assert.Equal(t, "some-app.otelcol-native-sidecar",
92+
changed.Labels["sidecar.opentelemetry.io/injected"])
93+
expectedPolicy := corev1.ContainerRestartPolicyAlways
94+
assert.Equal(t, corev1.Container{
95+
Name: "otc-container",
96+
Image: "some-default-image",
97+
Args: []string{"--config=env:OTEL_CONFIG"},
98+
RestartPolicy: &expectedPolicy,
99+
Env: []corev1.EnvVar{
100+
{
101+
Name: "POD_NAME",
102+
ValueFrom: &corev1.EnvVarSource{
103+
FieldRef: &corev1.ObjectFieldSelector{
104+
FieldPath: "metadata.name",
105+
},
106+
},
107+
},
108+
{
109+
Name: "OTEL_CONFIG",
110+
Value: string(otelcolYaml),
111+
},
112+
},
113+
Ports: []corev1.ContainerPort{
114+
{
115+
Name: "metrics",
116+
ContainerPort: 8888,
117+
Protocol: corev1.ProtocolTCP,
118+
},
119+
},
120+
}, changed.Spec.InitContainers[2])
121+
}
122+
33123
func TestAddSidecarWhenNoSidecarExists(t *testing.T) {
34124
// prepare
35125
pod := corev1.Pod{
@@ -146,6 +236,11 @@ func TestRemoveSidecar(t *testing.T) {
146236
{Name: naming.Container()},
147237
{Name: naming.Container()}, // two sidecars! should remove both
148238
},
239+
InitContainers: []corev1.Container{
240+
{Name: "something"},
241+
{Name: naming.Container()}, // NOTE: native sidecar since k8s 1.28.
242+
{Name: naming.Container()}, // two sidecars! should remove both
243+
},
149244
},
150245
}
151246

@@ -190,6 +285,19 @@ func TestExistsIn(t *testing.T) {
190285
},
191286
true},
192287

288+
{"does-have-native-sidecar",
289+
corev1.Pod{
290+
Spec: corev1.PodSpec{
291+
Containers: []corev1.Container{
292+
{Name: "my-app"},
293+
},
294+
InitContainers: []corev1.Container{
295+
{Name: naming.Container()},
296+
},
297+
},
298+
},
299+
true},
300+
193301
{"does-not-have-sidecar",
194302
corev1.Pod{
195303
Spec: corev1.PodSpec{

‎tests/e2e-native-sidecar/00-assert.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
apiVersion: v1
3+
kind: Pod
4+
metadata:
5+
sidecar.opentelemetry.io/inject: "true"
6+
name: myapp
7+
status:
8+
containerStatuses:
9+
- name: myapp
10+
ready: true
11+
started: true
12+
initContainerStatuses:
13+
- name: otc-container
14+
ready: true
15+
started: true

‎tests/e2e-native-sidecar/00-install.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
---
2+
apiVersion: opentelemetry.io/v1beta1
3+
kind: OpenTelemetryCollector
4+
metadata:
5+
name: a-sidecar
6+
spec:
7+
mode: sidecar
8+
resources:
9+
limits:
10+
cpu: 500m
11+
memory: 128Mi
12+
requests:
13+
cpu: 5m
14+
memory: 64Mi
15+
16+
config:
17+
receivers:
18+
otlp:
19+
protocols:
20+
http: {}
21+
exporters:
22+
debug: {}
23+
service:
24+
pipelines:
25+
metrics:
26+
receivers: [otlp]
27+
exporters: [debug]
28+
---
29+
apiVersion: v1
30+
kind: Pod
31+
metadata:
32+
name: myapp
33+
annotations:
34+
sidecar.opentelemetry.io/inject: "true"
35+
spec:
36+
containers:
37+
- name: myapp
38+
image: jaegertracing/vertx-create-span:operator-e2e-tests
39+
ports:
40+
- containerPort: 8080
41+
protocol: TCP

‎tests/e2e-native-sidecar/chainsaw-test.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
2+
apiVersion: chainsaw.kyverno.io/v1alpha1
3+
kind: Test
4+
metadata:
5+
creationTimestamp: null
6+
name: native-sidecar
7+
spec:
8+
steps:
9+
- name: step-00
10+
try:
11+
- apply:
12+
file: 00-install.yaml
13+
- assert:
14+
file: 00-assert.yaml

0 commit comments

Comments
 (0)