Skip to content

Commit 506a78c

Browse files
oldiumoldium
committed
Add handling of environment variables in Secrets
Load also environment variables defined in Secret to make them visible to OpenTelemetry Operator. Signed-off-by: Oldřich Jedlička <[email protected]>
1 parent 24c4e31 commit 506a78c

File tree

3 files changed

+131
-63
lines changed

3 files changed

+131
-63
lines changed

‎.chloggen/enhancement-envs-from-secret.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: enhancement
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
5+
component: auto-instrumentation
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: Add support for detecting and reading environment variables from POD's Secret resources.
9+
10+
# One or more tracking issues related to the change
11+
issues: [1393, 1814]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext: |
17+
Now the auto-instrumentation will see the environment variables defined via POD's `env.valueFrom.secretKeyRef` and
18+
`envFrom.secretRef` fields, so the auto-instrumentation will be able to prevent overriding them, or it will be able to
19+
extend the existing definition.

‎pkg/instrumentation/container.go

+103-56
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ type Container struct {
3333
index int
3434
inheritedEnv map[string]string
3535
configMaps map[string]*corev1.ConfigMap
36+
secrets map[string]*corev1.Secret
3637
}
3738

3839
func NewContainer(client client.Reader, ctx context.Context, logger logr.Logger, namespace string, pod *corev1.Pod, index int) (Container, error) {
@@ -42,22 +43,17 @@ func NewContainer(client client.Reader, ctx context.Context, logger logr.Logger,
4243
container := &pod.Spec.Containers[index]
4344

4445
configMaps := make(map[string]*corev1.ConfigMap)
46+
secrets := make(map[string]*corev1.Secret)
4547
inheritedEnv := make(map[string]string)
4648
for _, envsFrom := range container.EnvFrom {
4749
if envsFrom.ConfigMapRef != nil {
48-
prefix := envsFrom.Prefix
49-
name := envsFrom.ConfigMapRef.Name
50-
if cm, err := getOrLoadResource(client, ctx, namespace, configMaps, name); err == nil {
51-
for k, v := range cm.Data {
52-
// Safely overwrite the value, last one from EnvFrom wins in Kubernetes, with the direct value
53-
// from the container itself taking precedence
54-
inheritedEnv[prefix+k] = v
55-
}
56-
} else if envsFrom.ConfigMapRef.Optional == nil || !*envsFrom.ConfigMapRef.Optional {
57-
return Container{}, fmt.Errorf("failed to load environment variables: %w", err)
50+
if err := loadAllEnvVars(client, ctx, namespace, configMaps, envsFrom.ConfigMapRef.Name, envsFrom.Prefix, envsFrom.ConfigMapRef.Optional, inheritedEnv); err != nil {
51+
return Container{}, err
5852
}
5953
} else if envsFrom.SecretRef != nil {
60-
logger.V(2).Info("ignoring SecretRef in EnvFrom", "container", container.Name, "secret", envsFrom.SecretRef.Name)
54+
if err := loadAllEnvVars(client, ctx, namespace, secrets, envsFrom.SecretRef.Name, envsFrom.Prefix, envsFrom.SecretRef.Optional, inheritedEnv); err != nil {
55+
return Container{}, err
56+
}
6157
}
6258
}
6359

@@ -73,39 +69,10 @@ func NewContainer(client client.Reader, ctx context.Context, logger logr.Logger,
7369
index: index,
7470
inheritedEnv: inheritedEnv,
7571
configMaps: configMaps,
72+
secrets: secrets,
7673
}, nil
7774
}
7875

79-
func getOrLoadResource[T any, PT interface {
80-
client.Object
81-
*T
82-
}](client client.Reader, ctx context.Context, namespace string, cache map[string]*T, name string) (*T, error) {
83-
var obj T
84-
if cached, ok := cache[name]; ok {
85-
if cached != nil {
86-
return cached, nil
87-
} else {
88-
return nil, fmt.Errorf("failed to get %s %s/%s", reflect.TypeOf(obj).Name(), namespace, name)
89-
}
90-
}
91-
92-
if client == nil || ctx == nil {
93-
// Cache error value
94-
cache[name] = nil
95-
return nil, fmt.Errorf("client or context is nil, cannot load %s %s/%s", reflect.TypeOf(obj).Name(), namespace, name)
96-
}
97-
98-
err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, PT(&obj))
99-
if err != nil {
100-
// Cache error value
101-
cache[name] = nil
102-
return nil, fmt.Errorf("failed to get %s %s/%s: %w", reflect.TypeOf(obj).Name(), namespace, name, err)
103-
}
104-
105-
cache[name] = &obj
106-
return &obj, nil
107-
}
108-
10976
func (c *Container) validate(pod *corev1.Pod, envsToBeValidated ...string) error {
11077
// Try if the value is resolvable
11178
for _, envToBeValidated := range envsToBeValidated {
@@ -169,21 +136,9 @@ func (c *Container) getOrMakeEnvVar(pod *corev1.Pod, name string) (corev1.EnvVar
169136
func (c *Container) resolveEnvVar(envVar corev1.EnvVar) (corev1.EnvVar, error) {
170137
if envVar.Value == "" && envVar.ValueFrom != nil {
171138
if envVar.ValueFrom.ConfigMapKeyRef != nil {
172-
configMapName := envVar.ValueFrom.ConfigMapKeyRef.Name
173-
configMapKey := envVar.ValueFrom.ConfigMapKeyRef.Key
174-
if cm, err := getOrLoadResource(c.client, c.ctx, c.namespace, c.configMaps, configMapName); err == nil {
175-
if value, ok := cm.Data[configMapKey]; ok {
176-
return corev1.EnvVar{Name: envVar.Name, Value: value}, nil
177-
} else if envVar.ValueFrom.ConfigMapKeyRef.Optional == nil || !*envVar.ValueFrom.ConfigMapKeyRef.Optional {
178-
return corev1.EnvVar{}, fmt.Errorf("failed to resolve environment variable %s, key %s not found in ConfigMap %s/%s", envVar.Name, configMapKey, c.namespace, configMapName)
179-
} else {
180-
return corev1.EnvVar{Name: envVar.Name, Value: ""}, nil
181-
}
182-
} else if envVar.ValueFrom.ConfigMapKeyRef.Optional == nil || !*envVar.ValueFrom.ConfigMapKeyRef.Optional {
183-
return corev1.EnvVar{}, fmt.Errorf("failed to resolve environment variable %s: %w", envVar.Name, err)
184-
} else {
185-
return corev1.EnvVar{Name: envVar.Name, Value: ""}, nil
186-
}
139+
return loadEnvVar(c.client, c.ctx, c.namespace, c.configMaps, envVar.Name, envVar.ValueFrom.ConfigMapKeyRef.Name, envVar.ValueFrom.ConfigMapKeyRef.Key, envVar.ValueFrom.ConfigMapKeyRef.Optional)
140+
} else if envVar.ValueFrom.SecretKeyRef != nil {
141+
return loadEnvVar(c.client, c.ctx, c.namespace, c.secrets, envVar.Name, envVar.ValueFrom.SecretKeyRef.Name, envVar.ValueFrom.SecretKeyRef.Key, envVar.ValueFrom.SecretKeyRef.Optional)
187142
} else {
188143
v := reflect.ValueOf(*envVar.ValueFrom)
189144
for i := 0; i < v.NumField(); i++ {
@@ -197,6 +152,98 @@ func (c *Container) resolveEnvVar(envVar corev1.EnvVar) (corev1.EnvVar, error) {
197152
return envVar, nil
198153
}
199154

155+
func getOrLoadResource[T any, PT interface {
156+
client.Object
157+
*T
158+
}](client client.Reader, ctx context.Context, namespace string, cache map[string]*T, name string) (*T, error) {
159+
var obj T
160+
if cached, ok := cache[name]; ok {
161+
if cached != nil {
162+
return cached, nil
163+
} else {
164+
return nil, fmt.Errorf("failed to get %s %s/%s", reflect.TypeOf(obj).Name(), namespace, name)
165+
}
166+
}
167+
168+
if client == nil || ctx == nil {
169+
// Cache error value
170+
cache[name] = nil
171+
return nil, fmt.Errorf("client or context is nil, cannot load %s %s/%s", reflect.TypeOf(obj).Name(), namespace, name)
172+
}
173+
174+
err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, PT(&obj))
175+
if err != nil {
176+
// Cache error value
177+
cache[name] = nil
178+
return nil, fmt.Errorf("failed to get %s %s/%s: %w", reflect.TypeOf(obj).Name(), namespace, name, err)
179+
}
180+
181+
cache[name] = &obj
182+
return &obj, nil
183+
}
184+
185+
func loadAllEnvVars[T any, PT interface {
186+
client.Object
187+
*T
188+
}](client client.Reader, ctx context.Context, namespace string, cache map[string]*T, name string, prefix string, optional *bool, inheritedEnv map[string]string) error {
189+
if obj, err := getOrLoadResource[T, PT](client, ctx, namespace, cache, name); err == nil {
190+
return fillEnvVars(obj, prefix, inheritedEnv)
191+
} else if optional == nil || !*optional {
192+
return fmt.Errorf("failed to load environment variables: %w", err)
193+
} else {
194+
return nil
195+
}
196+
}
197+
198+
func loadEnvVar[T any, PT interface {
199+
client.Object
200+
*T
201+
}](client client.Reader, ctx context.Context, namespace string, cache map[string]*T, variable string, name string, key string, optional *bool) (corev1.EnvVar, error) {
202+
if resource, err := getOrLoadResource[T, PT](client, ctx, namespace, cache, name); err == nil {
203+
if value, ok := getResourceValue(resource, key); ok {
204+
return corev1.EnvVar{Name: variable, Value: value}, nil
205+
} else if optional == nil || !*optional {
206+
return corev1.EnvVar{}, fmt.Errorf("failed to resolve environment variable %s, key %s not found in %s %s/%s", variable, key, reflect.TypeOf(*resource).Name(), namespace, name)
207+
} else {
208+
return corev1.EnvVar{Name: variable, Value: ""}, nil
209+
}
210+
} else if optional == nil || !*optional {
211+
return corev1.EnvVar{}, fmt.Errorf("failed to resolve environment variable %s: %w", variable, err)
212+
} else {
213+
return corev1.EnvVar{Name: variable, Value: ""}, nil
214+
}
215+
}
216+
217+
func fillEnvVars(obj any, prefix string, inheritedEnv map[string]string) error {
218+
switch o := obj.(type) {
219+
case *corev1.ConfigMap:
220+
for k, v := range o.Data {
221+
inheritedEnv[prefix+k] = v
222+
}
223+
return nil
224+
case *corev1.Secret:
225+
for k, v := range o.Data {
226+
inheritedEnv[prefix+k] = string(v)
227+
}
228+
return nil
229+
default:
230+
return fmt.Errorf("unsupported type %T", obj)
231+
}
232+
}
233+
234+
func getResourceValue(obj interface{}, key string) (string, bool) {
235+
switch o := obj.(type) {
236+
case *corev1.ConfigMap:
237+
val, ok := o.Data[key]
238+
return val, ok
239+
case *corev1.Secret:
240+
val, ok := o.Data[key]
241+
return string(val), ok
242+
default:
243+
return "", false
244+
}
245+
}
246+
200247
func existsEnvVarInEnv(env []corev1.EnvVar, name string) bool {
201248
for i := range env {
202249
if env[i].Name == name {

‎pkg/instrumentation/container_test.go

+9-7
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ func TestInheritedEnv(t *testing.T) {
377377
err: "failed to get ConfigMap inheritedenv-notfoundexplicit/my-config",
378378
},
379379
{
380-
name: "SecretRef not supported",
380+
name: "Simple SecretRef usage",
381381
ns: corev1.Namespace{
382382
ObjectMeta: metav1.ObjectMeta{
383383
Name: "inheritedenv-secretref",
@@ -410,7 +410,9 @@ func TestInheritedEnv(t *testing.T) {
410410
},
411411
},
412412
},
413-
log: "ignoring SecretRef in EnvFrom",
413+
expected: map[string]string{
414+
"OTEL_ENVFROM_SECRET_VALUE1": "my-secret-value1",
415+
},
414416
},
415417
}
416418

@@ -1011,7 +1013,7 @@ func TestResolving(t *testing.T) {
10111013
Namespace: "validations",
10121014
},
10131015
Data: map[string][]byte{
1014-
"secret-ref-value1": []byte("my-valuefrom-value1"),
1016+
"secret-ref-value1": []byte("my-secret-valuefrom-value1"),
10151017
},
10161018
},
10171019
}
@@ -1151,10 +1153,10 @@ func TestResolving(t *testing.T) {
11511153
expectedResolve: "",
11521154
},
11531155
{
1154-
name: "Test unsupported existing Secret variable",
1155-
variable: "OTEL_ENV_VALUEFROM_SECRET1",
1156-
expectedExists: true,
1157-
err: "the container defines env var value via ValueFrom.SecretKeyRef, envVar: OTEL_ENV_VALUEFROM_SECRET1",
1156+
name: "Test unsupported existing Secret variable",
1157+
variable: "OTEL_ENV_VALUEFROM_SECRET1",
1158+
expectedExists: true,
1159+
expectedResolve: "my-secret-valuefrom-value1",
11581160
},
11591161
{
11601162
name: "Test unsupported existing FieldRef variable",

0 commit comments

Comments
 (0)