OpenTelemetry Auto-Instrumentation

Starefossen · Starefossen · commit 51254f59f740 · 2024-02-23T15:50:34.000+01:00
diff --git a/charts/naiserator/values.yaml b/charts/naiserator/values.yaml
@@ -59,6 +59,9 @@ naiserator:
   observability:
     otel:
       enabled: false
+      auto-instrumentation:
+        enabled: false
+        app-config: "nais-system/apps"
       collector:
         labels:
           - "app.kubernetes.io/name=opentelemetry-collector"
diff --git a/pkg/naiserator/config/config.go b/pkg/naiserator/config/config.go
@@ -71,8 +71,14 @@ type Observability struct {
 }
 
 type Otel struct {
-	Enabled   bool          `json:"enabled"`
-	Collector OtelCollector `json:"collector"`
+	Enabled             bool                `json:"enabled"`
+	Collector           OtelCollector       `json:"collector"`
+	AutoInstrumentation AutoInstrumentation `json:"auto-instrumentation"`
+}
+
+type AutoInstrumentation struct {
+	Enabled   bool   `json:"enabled"`
+	AppConfig string `json:"app-config"`
 }
 
 type OtelCollector struct {
@@ -174,67 +180,69 @@ type Config struct {
 }
 
 const (
-	AivenProject                        = "aiven-project"
-	AivenRange                          = "aiven-range"
-	ApiServerIp                         = "api-server-ip"
-	Bind                                = "bind"
-	HealthProbeBindAddress              = "health-probe-bind-address"
-	ClusterName                         = "cluster-name"
-	DryRun                              = "dry-run"
-	NaisNamespace                       = "nais-namespace"
-	FeaturesAccessPolicyNotAllowedCIDRs = "features.access-policy-not-allowed-cidrs"
-	FeaturesAzurerator                  = "features.azurerator"
-	FeaturesGCP                         = "features.gcp"
-	FeaturesIDPorten                    = "features.idporten"
-	FeaturesJwker                       = "features.jwker"
-	FeaturesCNRM                        = "features.cnrm"
-	FeaturesKafkarator                  = "features.kafkarator"
-	FeaturesLinkerd                     = "features.linkerd"
-	FeaturesMaskinporten                = "features.maskinporten"
-	FeaturesNetworkPolicy               = "features.network-policy"
-	FeaturesPrometheusOperator          = "features.prometheus-operator"
-	FeaturesVault                       = "features.vault"
-	FeaturesWebhook                     = "features.webhook"
-	FeaturesWonderwall                  = "features.wonderwall"
-	FeaturesLegacyGCP                   = "features.legacy-gcp"
-	FQDNPolicyEnabled                   = "fqdn-policy.enabled"
-	GoogleCloudSQLProxyContainerImage   = "google-cloud-sql-proxy-container-image"
-	GoogleProjectId                     = "google-project-id"
-	InformerFullSynchronizationInterval = "informer.full-sync-interval"
-	KafkaBrokers                        = "kafka.brokers"
-	KafkaEnabled                        = "kafka.enabled"
-	KafkaLogVerbosity                   = "kafka.log-verbosity"
-	KafkaTLSCAPath                      = "kafka.tls.ca-path"
-	KafkaTLSCertificatePath             = "kafka.tls.certificate-path"
-	KafkaTLSEnabled                     = "kafka.tls.enabled"
-	KafkaTLSInsecure                    = "kafka.tls.insecure"
-	KafkaTLSPrivateKeyPath              = "kafka.tls.private-key-path"
-	KafkaTopic                          = "kafka.topic"
-	KubeConfig                          = "kubeconfig"
-	LeaderElectionImage                 = "leader-election.image"
-	MaxConcurrentReconciles             = "max-concurrent-reconciles"
-	ObservabilityLoggingDestinations    = "observability.logging.destinations"
-	ObservabilityOtelCollectorLabels    = "observability.otel.collector.labels"
-	ObservabilityOtelCollectorNamespace = "observability.otel.collector.namespace"
-	ObservabilityOtelCollectorPort      = "observability.otel.collector.port"
-	ObservabilityOtelCollectorProtocol  = "observability.otel.collector.protocol"
-	ObservabilityOtelCollectorService   = "observability.otel.collector.service"
-	ObservabilityOtelCollectorTLS       = "observability.otel.collector.tls"
-	ObservabilityOtelEnabled            = "observability.otel.enabled"
-	ProxyAddress                        = "proxy.address"
-	ProxyExclude                        = "proxy.exclude"
-	RateLimitBurst                      = "ratelimit.burst"
-	RateLimitQPS                        = "ratelimit.qps"
-	SecurelogsConfigMapReloadImage      = "securelogs.configmap-reload-image"
-	SecurelogsFluentdImage              = "securelogs.fluentd-image"
-	SynchronizerRolloutCheckInterval    = "synchronizer.rollout-check-interval"
-	SynchronizerRolloutTimeout          = "synchronizer.rollout-timeout"
-	SynchronizerSynchronizationTimeout  = "synchronizer.synchronization-timeout"
-	VaultAddress                        = "vault.address"
-	VaultAuthPath                       = "vault.auth-path"
-	VaultInitContainerImage             = "vault.init-container-image"
-	VaultKvPath                         = "vault.kv-path"
-	WonderwallImage                     = "wonderwall.image"
+	AivenProject                                  = "aiven-project"
+	AivenRange                                    = "aiven-range"
+	ApiServerIp                                   = "api-server-ip"
+	Bind                                          = "bind"
+	HealthProbeBindAddress                        = "health-probe-bind-address"
+	ClusterName                                   = "cluster-name"
+	DryRun                                        = "dry-run"
+	NaisNamespace                                 = "nais-namespace"
+	FeaturesAccessPolicyNotAllowedCIDRs           = "features.access-policy-not-allowed-cidrs"
+	FeaturesAzurerator                            = "features.azurerator"
+	FeaturesGCP                                   = "features.gcp"
+	FeaturesIDPorten                              = "features.idporten"
+	FeaturesJwker                                 = "features.jwker"
+	FeaturesCNRM                                  = "features.cnrm"
+	FeaturesKafkarator                            = "features.kafkarator"
+	FeaturesLinkerd                               = "features.linkerd"
+	FeaturesMaskinporten                          = "features.maskinporten"
+	FeaturesNetworkPolicy                         = "features.network-policy"
+	FeaturesPrometheusOperator                    = "features.prometheus-operator"
+	FeaturesVault                                 = "features.vault"
+	FeaturesWebhook                               = "features.webhook"
+	FeaturesWonderwall                            = "features.wonderwall"
+	FeaturesLegacyGCP                             = "features.legacy-gcp"
+	FQDNPolicyEnabled                             = "fqdn-policy.enabled"
+	GoogleCloudSQLProxyContainerImage             = "google-cloud-sql-proxy-container-image"
+	GoogleProjectId                               = "google-project-id"
+	InformerFullSynchronizationInterval           = "informer.full-sync-interval"
+	KafkaBrokers                                  = "kafka.brokers"
+	KafkaEnabled                                  = "kafka.enabled"
+	KafkaLogVerbosity                             = "kafka.log-verbosity"
+	KafkaTLSCAPath                                = "kafka.tls.ca-path"
+	KafkaTLSCertificatePath                       = "kafka.tls.certificate-path"
+	KafkaTLSEnabled                               = "kafka.tls.enabled"
+	KafkaTLSInsecure                              = "kafka.tls.insecure"
+	KafkaTLSPrivateKeyPath                        = "kafka.tls.private-key-path"
+	KafkaTopic                                    = "kafka.topic"
+	KubeConfig                                    = "kubeconfig"
+	LeaderElectionImage                           = "leader-election.image"
+	MaxConcurrentReconciles                       = "max-concurrent-reconciles"
+	ObservabilityLoggingDestinations              = "observability.logging.destinations"
+	ObservabilityOtelCollectorLabels              = "observability.otel.collector.labels"
+	ObservabilityOtelCollectorNamespace           = "observability.otel.collector.namespace"
+	ObservabilityOtelCollectorPort                = "observability.otel.collector.port"
+	ObservabilityOtelCollectorProtocol            = "observability.otel.collector.protocol"
+	ObservabilityOtelCollectorService             = "observability.otel.collector.service"
+	ObservabilityOtelCollectorTLS                 = "observability.otel.collector.tls"
+	ObservabilityOtelEnabled                      = "observability.otel.enabled"
+	ObservabilityOtelAutoInstrumentationAppConfig = "observability.otel.auto-instrumentation.app-config"
+	ObservabilityOtelAutoInstrumentationEnabled   = "observability.otel.auto-instrumentation.enabled"
+	ProxyAddress                                  = "proxy.address"
+	ProxyExclude                                  = "proxy.exclude"
+	RateLimitBurst                                = "ratelimit.burst"
+	RateLimitQPS                                  = "ratelimit.qps"
+	SecurelogsConfigMapReloadImage                = "securelogs.configmap-reload-image"
+	SecurelogsFluentdImage                        = "securelogs.fluentd-image"
+	SynchronizerRolloutCheckInterval              = "synchronizer.rollout-check-interval"
+	SynchronizerRolloutTimeout                    = "synchronizer.rollout-timeout"
+	SynchronizerSynchronizationTimeout            = "synchronizer.synchronization-timeout"
+	VaultAddress                                  = "vault.address"
+	VaultAuthPath                                 = "vault.auth-path"
+	VaultInitContainerImage                       = "vault.init-container-image"
+	VaultKvPath                                   = "vault.kv-path"
+	WonderwallImage                               = "wonderwall.image"
 )
 
 func bindNAIS() {
@@ -303,6 +311,8 @@ func init() {
 	flag.String(ObservabilityOtelCollectorNamespace, "nais-system", "namespace of the OpenTelemetry collector")
 	flag.String(ObservabilityOtelCollectorService, "opentelmetry-collector", "service name of the OpenTelemetry collector")
 	flag.String(ObservabilityOtelCollectorProtocol, "grpc", "protocol used by the OpenTelemetry collector")
+	flag.Bool(ObservabilityOtelAutoInstrumentationEnabled, false, "enable OpenTelemetry auto-instrumentation")
+	flag.String(ObservabilityOtelAutoInstrumentationAppConfig, "nais-system/apps", "path to OpenTelemetry auto-instrumentation config")
 	flag.Int(ObservabilityOtelCollectorPort, 4317, "port used by the OpenTelemetry collector")
 	flag.Bool(ObservabilityOtelCollectorTLS, false, "use TLS for the OpenTelemetry collector")
 	flag.StringArray(ObservabilityOtelCollectorLabels, []string{}, "list of labels to be used by the OpenTelemetry collector")
diff --git a/pkg/resourcecreator/observability/observability.go b/pkg/resourcecreator/observability/observability.go
@@ -37,6 +37,9 @@ const otelExporterInsecure = "OTEL_EXPORTER_OTLP_INSECURE"
 const logLabelDefault = "logs.nais.io/flow-default"
 const logLabelPrefix = "logs.nais.io/flow-"
 
+const autoInstrumentationInjectAnnotationPrefix = "instrumentation.opentelemetry.io/inject-"
+const autoInstrumentationContainerNamesAnnotation = "instrumentation.opentelemetry.io/container-names"
+
 func otelEndpointFromConfig(collector config.OtelCollector) string {
 	schema := "http"
 	if collector.Tls {
@@ -73,6 +76,16 @@ func otelEnvVars(source Source, otel config.Otel) []corev1.EnvVar {
 	}
 }
 
+func otelAutoInstrumentAnnotations(source Source, otel config.Otel) map[string]string {
+	runtime := source.GetObservability().AutoInstrumentation.Runtime
+	autoInstrumentationInjectAnnotation := autoInstrumentationInjectAnnotationPrefix + runtime
+
+	return map[string]string{
+		autoInstrumentationInjectAnnotation:         otel.AutoInstrumentation.AppConfig,
+		autoInstrumentationContainerNamesAnnotation: source.GetName(),
+	}
+}
+
 func labelsFromCollectorConfig(collector config.OtelCollector) map[string]string {
 	labels := collector.Labels
 	labelMap := make(map[string]string, len(labels))
@@ -147,16 +160,26 @@ func Create(source Source, ast *resource.Ast, config Config) error {
 		return nil
 	}
 
-	if obs.Tracing != nil && obs.Tracing.Enabled {
+	if obs.AutoInstrumentation != nil && obs.AutoInstrumentation.Enabled && obs.Tracing != nil && obs.Tracing.Enabled {
+		return fmt.Errorf("auto-instrumentation and tracing cannot be enabled at the same time")
+	}
+
+	if (obs.Tracing != nil && obs.Tracing.Enabled) || (obs.AutoInstrumentation != nil && obs.AutoInstrumentation.Enabled) {
 		if !cfg.Otel.Enabled {
-			return fmt.Errorf("tracing is not supported in this cluster configuration")
+			return fmt.Errorf("tracing and auto-instrumentation are not supported in this cluster")
 		}
 
 		netpol, err := tracingNetpol(source, cfg.Otel)
 		if err != nil {
 			return err
 		}
 
+		if obs.AutoInstrumentation != nil && obs.AutoInstrumentation.Enabled {
+			for k, v := range otelAutoInstrumentAnnotations(source, cfg.Otel) {
+				ast.Annotations[k] = v
+			}
+		}
+
 		ast.Env = append(ast.Env, otelEnvVars(source, cfg.Otel)...)
 		ast.AppendOperation(resource.OperationCreateOrUpdate, netpol)
 	}
diff --git a/pkg/resourcecreator/pod/pod.go b/pkg/resourcecreator/pod/pod.go
@@ -373,7 +373,7 @@ func CreateAppObjectMeta(app Source, ast *resource.Ast, cfg Config) metav1.Objec
 		port = strconv.Itoa(app.GetPort())
 	}
 
-	objectMeta.Annotations = map[string]string{}
+	//objectMeta.Annotations = map[string]string{}
 
 	objectMeta.Annotations["kubectl.kubernetes.io/default-container"] = app.GetName()
 
diff --git a/pkg/resourcecreator/testdata/observability_autoinstrument_enabled.yaml b/pkg/resourcecreator/testdata/observability_autoinstrument_enabled.yaml
@@ -0,0 +1,92 @@
+testconfig:
+  description: enable opentelemetry auto-instrumentation
+
+config:
+  observability:
+    otel:
+      enabled: true
+      auto-instrumentation:
+        enabled: true
+        app-config: "system-namespace/app-config"
+      collector:
+        labels:
+          - key1=value1
+          - key2=value2
+          - key3=value3
+        namespace: "system-namespace"
+        port: 4317
+        service: "my-collector"
+        tls: false
+        protocol: "grpc"
+
+input:
+  kind: Application
+  apiVersion: nais.io/v1alpha1
+  metadata:
+    name: myapplication
+    namespace: mynamespace
+    uid: "123456"
+    labels:
+      team: myteam
+  spec:
+    image: navikt/myapplication:1.2.3
+    observability:
+      autoInstrumentation:
+        enabled: true
+        runtime: "java"
+
+tests:
+  - apiVersion: apps/v1
+    kind: Deployment
+    name: myapplication
+    operation: CreateOrUpdate
+    match:
+      - type: subset
+        name: "otel annotation config set in pod"
+        resource:
+          spec:
+            template:
+              metadata:
+                annotations:
+                  "instrumentation.opentelemetry.io/inject-java": "system-namespace/app-config"
+                  "instrumentation.opentelemetry.io/container-names": "myapplication"
+      - type: subset
+        name: "otel environment variables set in pod"
+        resource:
+          spec:
+            template:
+              spec:
+                containers:
+                  - image: navikt/myapplication:1.2.3
+                    name: myapplication
+                    env:
+                      - name: OTEL_SERVICE_NAME
+                        value: myapplication
+                      - name: OTEL_RESOURCE_ATTRIBUTES
+                        value: service.name=myapplication,service.namespace=mynamespace
+                      - name: OTEL_EXPORTER_OTLP_ENDPOINT
+                        value: http://my-collector.system-namespace:4317
+                      - name: OTEL_EXPORTER_OTLP_PROTOCOL
+                        value: grpc
+                      - name: OTEL_EXPORTER_OTLP_INSECURE
+                        value: "true"
+
+  - apiVersion: networking.k8s.io/v1
+    kind: NetworkPolicy
+    name: myapplication-tracing-438f0c6d
+    operation: CreateOrUpdate
+    match:
+      - type: subset
+        name: "network policy allows traffic to data collector"
+        resource:
+          spec:
+            egress:
+              - to:
+                  - namespaceSelector:
+                      matchLabels:
+                        kubernetes.io/metadata.name: system-namespace
+                    podSelector:
+                      matchLabels:
+                        key1: value1
+                        key2: value2
+                        key3: value3
diff --git a/pkg/resourcecreator/testdata/observability_tracing_enabled_error.yaml b/pkg/resourcecreator/testdata/observability_tracing_enabled_error.yaml
@@ -21,4 +21,4 @@ input:
       tracing:
         enabled: true
 
-error: "tracing is not supported in this cluster configuration"
+error: "tracing and auto-instrumentation are not supported in this cluster"

Original file line number	Diff line number	Diff line change
`@@ -373,7 +373,7 @@ func CreateAppObjectMeta(app Source, ast *resource.Ast, cfg Config) metav1.Objec`
`373`	`373`	`port = strconv.Itoa(app.GetPort())`
`374`	`374`	`}`
`375`	`375`
`376`		`- objectMeta.Annotations = map[string]string{}`
	`376`	`+ //objectMeta.Annotations = map[string]string{}`
`377`	`377`
`378`	`378`	`objectMeta.Annotations["kubectl.kubernetes.io/default-container"] = app.GetName()`
`379`	`379`