Skip to content

Commit 4334932

Browse files
committed
[WIP] LD_PRELOAD based injection
1 parent 922e7a1 commit 4334932

38 files changed

+2577
-14
lines changed

Makefile

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ OTELCOL_VERSION ?= "$(shell grep -v '\#' versions.txt | grep opentelemetry-colle
66
OPERATOR_VERSION ?= "$(shell grep -v '\#' versions.txt | grep operator= | awk -F= '{print $$2}')"
77
TARGETALLOCATOR_VERSION ?= $(shell grep -v '\#' versions.txt | grep targetallocator | awk -F= '{print $$2}')
88
OPERATOR_OPAMP_BRIDGE_VERSION ?= "$(shell grep -v '\#' versions.txt | grep operator-opamp-bridge | awk -F= '{print $$2}')"
9+
AUTO_INSTRUMENTATION_INJECTOR_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-injector | awk -F= '{print $$2}')"
910
AUTO_INSTRUMENTATION_JAVA_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-java | awk -F= '{print $$2}')"
1011
AUTO_INSTRUMENTATION_NODEJS_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-nodejs | awk -F= '{print $$2}')"
1112
AUTO_INSTRUMENTATION_PYTHON_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-python | awk -F= '{print $$2}')"
@@ -14,7 +15,7 @@ AUTO_INSTRUMENTATION_GO_VERSION ?= "$(shell grep -v '\#' versions.txt | grep aut
1415
AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-apache-httpd | awk -F= '{print $$2}')"
1516
AUTO_INSTRUMENTATION_NGINX_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-nginx | awk -F= '{print $$2}')"
1617
COMMON_LDFLAGS ?= -s -w
17-
OPERATOR_LDFLAGS ?= -X ${VERSION_PKG}.version=${VERSION} -X ${VERSION_PKG}.buildDate=${VERSION_DATE} -X ${VERSION_PKG}.otelCol=${OTELCOL_VERSION} -X ${VERSION_PKG}.targetAllocator=${TARGETALLOCATOR_VERSION} -X ${VERSION_PKG}.operatorOpAMPBridge=${OPERATOR_OPAMP_BRIDGE_VERSION} -X ${VERSION_PKG}.autoInstrumentationJava=${AUTO_INSTRUMENTATION_JAVA_VERSION} -X ${VERSION_PKG}.autoInstrumentationNodeJS=${AUTO_INSTRUMENTATION_NODEJS_VERSION} -X ${VERSION_PKG}.autoInstrumentationPython=${AUTO_INSTRUMENTATION_PYTHON_VERSION} -X ${VERSION_PKG}.autoInstrumentationDotNet=${AUTO_INSTRUMENTATION_DOTNET_VERSION} -X ${VERSION_PKG}.autoInstrumentationGo=${AUTO_INSTRUMENTATION_GO_VERSION} -X ${VERSION_PKG}.autoInstrumentationApacheHttpd=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION} -X ${VERSION_PKG}.autoInstrumentationNginx=${AUTO_INSTRUMENTATION_NGINX_VERSION}
18+
OPERATOR_LDFLAGS ?= -X ${VERSION_PKG}.version=${VERSION} -X ${VERSION_PKG}.buildDate=${VERSION_DATE} -X ${VERSION_PKG}.otelCol=${OTELCOL_VERSION} -X ${VERSION_PKG}.targetAllocator=${TARGETALLOCATOR_VERSION} -X ${VERSION_PKG}.operatorOpAMPBridge=${OPERATOR_OPAMP_BRIDGE_VERSION} -X ${VERSION_PKG}.autoInstrumentationJava=${AUTO_INSTRUMENTATION_JAVA_VERSION} -X ${VERSION_PKG}.autoInstrumentationInjector=${AUTO_INSTRUMENTATION_INJECTOR_VERSION} -X ${VERSION_PKG}.autoInstrumentationNodeJS=${AUTO_INSTRUMENTATION_NODEJS_VERSION} -X ${VERSION_PKG}.autoInstrumentationPython=${AUTO_INSTRUMENTATION_PYTHON_VERSION} -X ${VERSION_PKG}.autoInstrumentationDotNet=${AUTO_INSTRUMENTATION_DOTNET_VERSION} -X ${VERSION_PKG}.autoInstrumentationGo=${AUTO_INSTRUMENTATION_GO_VERSION} -X ${VERSION_PKG}.autoInstrumentationApacheHttpd=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION} -X ${VERSION_PKG}.autoInstrumentationNginx=${AUTO_INSTRUMENTATION_NGINX_VERSION}
1819
ARCH ?= $(shell go env GOARCH)
1920
ifeq ($(shell uname), Darwin)
2021
SED_INPLACE := sed -i ''

README.md

+1
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,7 @@ Language support can be disabled by passing the flag with a value of `false`.
597597
| ApacheHttpD | `enable-apache-httpd-instrumentation` | `true` |
598598
| Go | `enable-go-instrumentation` | `false` |
599599
| Nginx | `enable-nginx-instrumentation` | `false` |
600+
| Injector | `enable-injector-instrumentation` | `false` |
600601

601602

602603
OpenTelemetry Operator allows to instrument multiple containers using multiple language specific instrumentations.

apis/v1alpha1/instrumentation_types.go

+28
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ type InstrumentationSpec struct {
3838
// +optional
3939
Env []corev1.EnvVar `json:"env,omitempty"`
4040

41+
// Injector defines configuration for the injector-based auto-instrumentation.
42+
// +optional
43+
Injector Injector `json:"injector,omitempty"`
44+
4145
// Java defines configuration for java auto-instrumentation.
4246
// +optional
4347
Java Java `json:"java,omitempty"`
@@ -144,6 +148,30 @@ type Defaults struct {
144148
UseLabelsForResourceAttributes bool `json:"useLabelsForResourceAttributes,omitempty"`
145149
}
146150

151+
type Injector struct {
152+
// Image is a container image with auto-instrumentation injector.
153+
// +optional
154+
Image string `json:"image,omitempty"`
155+
156+
// VolumeClaimTemplate defines a ephemeral volume used for auto-instrumentation.
157+
// If omitted, an emptyDir is used with size limit VolumeSizeLimit
158+
VolumeClaimTemplate corev1.PersistentVolumeClaimTemplate `json:"volumeClaimTemplate,omitempty"`
159+
160+
// VolumeSizeLimit defines size limit for volume used for auto-instrumentation.
161+
// The default size is 200Mi.
162+
VolumeSizeLimit *resource.Quantity `json:"volumeLimitSize,omitempty"`
163+
164+
// Env defines instrumentation specific env vars. There are four layers for env vars' definitions and
165+
// the precedence order is: `original container env vars` > `language specific env vars` > `common env vars` > `instrument spec configs' vars`.
166+
// If the former var had been defined, then the other vars would be ignored.
167+
// +optional
168+
Env []corev1.EnvVar `json:"env,omitempty"`
169+
170+
// Resources describes the compute resource requirements.
171+
// +optional
172+
Resources corev1.ResourceRequirements `json:"resources,omitempty"`
173+
}
174+
147175
// Java defines Java SDK and instrumentation configuration.
148176
type Java struct {
149177
// Image is a container image with javaagent auto-instrumentation JAR.

apis/v1alpha1/instrumentation_webhook.go

+20
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,21 @@ func (w InstrumentationWebhook) defaulter(r *Instrumentation) error {
8181
if r.Labels == nil {
8282
r.Labels = map[string]string{}
8383
}
84+
if r.Spec.Injector.Image == "" {
85+
r.Spec.Injector.Image = w.cfg.AutoInstrumentationInjectorImage()
86+
}
87+
if r.Spec.Injector.Resources.Limits == nil {
88+
r.Spec.Injector.Resources.Limits = corev1.ResourceList{
89+
corev1.ResourceCPU: resource.MustParse("500m"),
90+
corev1.ResourceMemory: resource.MustParse("64Mi"),
91+
}
92+
}
93+
if r.Spec.Injector.Resources.Requests == nil {
94+
r.Spec.Injector.Resources.Requests = corev1.ResourceList{
95+
corev1.ResourceCPU: resource.MustParse("50m"),
96+
corev1.ResourceMemory: resource.MustParse("64Mi"),
97+
}
98+
}
8499
if r.Spec.Java.Image == "" {
85100
r.Spec.Java.Image = w.cfg.AutoInstrumentationJavaImage()
86101
}
@@ -187,6 +202,7 @@ func (w InstrumentationWebhook) defaulter(r *Instrumentation) error {
187202
if r.Annotations == nil {
188203
r.Annotations = map[string]string{}
189204
}
205+
r.Annotations[constants.AnnotationDefaultAutoInstrumentationInjector] = w.cfg.AutoInstrumentationInjectorImage()
190206
r.Annotations[constants.AnnotationDefaultAutoInstrumentationJava] = w.cfg.AutoInstrumentationJavaImage()
191207
r.Annotations[constants.AnnotationDefaultAutoInstrumentationNodeJS] = w.cfg.AutoInstrumentationNodeJSImage()
192208
r.Annotations[constants.AnnotationDefaultAutoInstrumentationPython] = w.cfg.AutoInstrumentationPythonImage()
@@ -244,6 +260,10 @@ func (w InstrumentationWebhook) validate(r *Instrumentation) (admission.Warnings
244260
if err != nil {
245261
return warnings, fmt.Errorf("spec.java.volumeClaimTemplate and spec.java.volumeSizeLimit cannot both be defined: %w", err)
246262
}
263+
err = validateInstrVolume(r.Spec.Injector.VolumeClaimTemplate, r.Spec.Injector.VolumeSizeLimit)
264+
if err != nil {
265+
return warnings, fmt.Errorf("spec.injector.volumeClaimTemplate and spec.injector.volumeSizeLimit cannot both be defined: %w", err)
266+
}
247267
err = validateInstrVolume(r.Spec.Nginx.VolumeClaimTemplate, r.Spec.Nginx.VolumeSizeLimit)
248268
if err != nil {
249269
return warnings, fmt.Errorf("spec.nginx.volumeClaimTemplate and spec.nginx.volumeSizeLimit cannot both be defined: %w", err)

apis/v1alpha1/instrumentation_webhook_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ func TestInstrumentationDefaultingWebhook(t *testing.T) {
2121
inst := &Instrumentation{}
2222
err := InstrumentationWebhook{
2323
cfg: config.New(
24+
config.WithAutoInstrumentationInjectorImage("injector-img:1"),
2425
config.WithAutoInstrumentationJavaImage("java-img:1"),
2526
config.WithAutoInstrumentationNodeJSImage("nodejs-img:1"),
2627
config.WithAutoInstrumentationPythonImage("python-img:1"),
@@ -30,6 +31,7 @@ func TestInstrumentationDefaultingWebhook(t *testing.T) {
3031
),
3132
}.Default(context.Background(), inst)
3233
assert.NoError(t, err)
34+
assert.Equal(t, "injector-img:1", inst.Spec.Injector.Image)
3335
assert.Equal(t, "java-img:1", inst.Spec.Java.Image)
3436
assert.Equal(t, "nodejs-img:1", inst.Spec.NodeJS.Image)
3537
assert.Equal(t, "python-img:1", inst.Spec.Python.Image)

apis/v1alpha1/zz_generated.deepcopy.go

+30
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
### Injector + Instrumentations image
2+
#
3+
# The injector image container both the LD_PRELOAD object, as well as
4+
# all the supported instrumentations (as it needs to copy them into
5+
# the emptyDir volume)
6+
7+
# Java agent
8+
FROM busybox AS java-agent
9+
10+
ARG javaagentversion
11+
12+
ADD https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v$javaagentversion/opentelemetry-javaagent.jar /javaagent.jar
13+
14+
RUN chmod -R go+r /javaagent.jar
15+
16+
# build injector
17+
FROM ubuntu:24.04 AS build-injector
18+
19+
RUN apt-get update && apt-get install --no-install-recommends build-essential -y
20+
21+
COPY ./src /injector/src
22+
WORKDIR /injector
23+
RUN gcc \
24+
-shared \
25+
-nostdlib \
26+
-fPIC \
27+
-Wl,--version-script=src/injector.exports.map \
28+
/injector/src/injector.c \
29+
-o /injector/injector.so
30+
31+
# injector_build_end - do not remove this line (see images/instrumentation/injector/test/scripts/test-all.sh)
32+
33+
# build final image
34+
FROM busybox
35+
WORKDIR /
36+
37+
COPY copy-instrumentation.sh /
38+
39+
COPY --from=build-injector /injector/injector.so /instrumentation/injector.so
40+
COPY --from=java-agent /javaagent.jar /instrumentation/jvm/javaagent.jar
41+
42+
CMD ["/copy-instrumentation.sh"]

autoinstrumentation/injector/TODOs

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Open questions
2+
3+
* How can the injector "inherit" features of other instrumentations, e.g.:
4+
* Specific versions
5+
* JavaAgent extensions
6+
* Detect libc (Python, .NET) -> Will need likely a zig injector
7+
8+
$ TODOs
9+
10+
* C Lint
11+
* Swap default off
12+
* Injector Image CI build
13+
* Documentation
14+
* OTep
15+
* Versioning scheme
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/sh
2+
3+
# SPDX-FileCopyrightText: Copyright 2025 The OpenTelemetry authors
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
set -eu
7+
8+
if [ "${OTEL_LOG_LEVEL:-}" = "debug" ]; then
9+
set -x
10+
fi
11+
12+
cd -P -- "$(dirname -- "$0")"
13+
14+
if [ -z "${INSTRUMENTATION_FOLDER_SOURCE:-}" ]; then
15+
INSTRUMENTATION_FOLDER_SOURCE=/instrumentation
16+
fi
17+
if [ ! -d "${INSTRUMENTATION_FOLDER_SOURCE}" ]; then
18+
>&2 echo "[OTel injector] Instrumentation source directory ${INSTRUMENTATION_FOLDER_SOURCE} does not exist."
19+
exit 1
20+
fi
21+
22+
if [ -z "${INSTRUMENTATION_FOLDER_DESTINATION:-}" ]; then
23+
INSTRUMENTATION_FOLDER_DESTINATION=/
24+
fi
25+
26+
# We deliberately do not create the base directory for $INSTRUMENTATION_FOLDER_DESTINATION via mkdir, it needs
27+
# be an existing mount point provided externally.
28+
cp -R "${INSTRUMENTATION_FOLDER_SOURCE}"/ "${INSTRUMENTATION_FOLDER_DESTINATION}"
29+
30+
if [ "${OTEL_DEBUG_DEBUG:-}" = "debug" ]; then
31+
>&2 echo "[OTel injector] Status of '${INSTRUMENTATION_FOLDER_DESTINATION}' after copying instrumentation files: $(find ${INSTRUMENTATION_FOLDER_DESTINATION})"
32+
fi
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
2.13.1
+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#include <stdint.h>
2+
#include <unistd.h>
3+
4+
#define ALIGN (sizeof(size_t))
5+
#define UCHAR_MAX 255
6+
#define ONES ((size_t)-1 / UCHAR_MAX)
7+
#define HIGHS (ONES * (UCHAR_MAX / 2 + 1))
8+
#define HASZERO(x) ((x) - ONES & ~(x) & HIGHS)
9+
10+
#define JAVA_TOOL_OPTIONS_ENV_VAR_NAME "JAVA_TOOL_OPTIONS"
11+
#define JAVA_TOOL_OPTIONS_REQUIRE \
12+
"-javaagent:" \
13+
"/otel-auto-instrumentation-injector/instrumentation/jvm/javaagent.jar"
14+
15+
extern char **__environ;
16+
17+
size_t __strlen(const char *s) {
18+
const char *a = s;
19+
const size_t *w;
20+
for (; (uintptr_t)s % ALIGN; s++)
21+
if (!*s)
22+
return s - a;
23+
for (w = (const void *)s; !HASZERO(*w); w++)
24+
;
25+
for (s = (const void *)w; *s; s++)
26+
;
27+
return s - a;
28+
}
29+
30+
char *__strchrnul(const char *s, int c) {
31+
size_t *w, k;
32+
33+
c = (unsigned char)c;
34+
if (!c)
35+
return (char *)s + __strlen(s);
36+
37+
for (; (uintptr_t)s % ALIGN; s++)
38+
if (!*s || *(unsigned char *)s == c)
39+
return (char *)s;
40+
k = ONES * c;
41+
for (w = (void *)s; !HASZERO(*w) && !HASZERO(*w ^ k); w++)
42+
;
43+
for (s = (void *)w; *s && *(unsigned char *)s != c; s++)
44+
;
45+
return (char *)s;
46+
}
47+
48+
char *__strcpy(char *restrict dest, const char *restrict src) {
49+
const unsigned char *s = src;
50+
unsigned char *d = dest;
51+
while ((*d++ = *s++))
52+
;
53+
return dest;
54+
}
55+
56+
char *__strcat(char *restrict dest, const char *restrict src) {
57+
__strcpy(dest + __strlen(dest), src);
58+
return dest;
59+
}
60+
61+
int __strcmp(const char *l, const char *r) {
62+
for (; *l == *r && *l; l++, r++)
63+
;
64+
return *(unsigned char *)l - *(unsigned char *)r;
65+
}
66+
67+
int __strncmp(const char *_l, const char *_r, size_t n) {
68+
const unsigned char *l = (void *)_l, *r = (void *)_r;
69+
if (!n--)
70+
return 0;
71+
for (; *l && *r && n && *l == *r; l++, r++, n--)
72+
;
73+
return *l - *r;
74+
}
75+
76+
char *__getenv(const char *name) {
77+
size_t l = __strchrnul(name, '=') - name;
78+
if (l && !name[l] && __environ)
79+
for (char **e = __environ; *e; e++)
80+
if (!__strncmp(name, *e, l) && l[*e] == '=')
81+
return *e + l + 1;
82+
return 0;
83+
}
84+
85+
/*
86+
* Buffers of statically-allocated memory that we can use to safely return to
87+
* the program manipulated values of env vars without dynamic allocations.
88+
*/
89+
char cachedModifiedRuntimeOptionsValue[1012];
90+
91+
char *getenv(const char *name) {
92+
char *origValue = __getenv(name);
93+
int l = __strlen(name);
94+
95+
char *javaToolOptionsVarName = JAVA_TOOL_OPTIONS_ENV_VAR_NAME;
96+
if (__strcmp(name, javaToolOptionsVarName) == 0) {
97+
if (__strlen(cachedModifiedRuntimeOptionsValue) == 0) {
98+
// No runtime environment variable has been requested before,
99+
// calculate the modified value and cache it.
100+
101+
// Prepend our --require as the first item to the JAVA_TOOL_OPTIONS
102+
// string.
103+
char *javaToolOptionsRequire = JAVA_TOOL_OPTIONS_REQUIRE;
104+
__strcat(cachedModifiedRuntimeOptionsValue, javaToolOptionsRequire);
105+
}
106+
107+
return cachedModifiedRuntimeOptionsValue;
108+
}
109+
110+
return origValue;
111+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
global:
3+
getenv;
4+
local:
5+
*;
6+
};

0 commit comments

Comments
 (0)