feat: Upgrade OTel Collector to version 0.120.0 (contrib 0.120.1)

.env

@@ -16,6 +16,6 @@ ENV_GORELEASER_VERSION=v1.23.0
 ## Default Docker Images
 DEFAULT_FLUENTBIT_EXPORTER_IMAGE="europe-docker.pkg.dev/kyma-project/prod/directory-size-exporter:v20250217-6a3cdc4a"
 DEFAULT_FLUENTBIT_IMAGE="europe-docker.pkg.dev/kyma-project/prod/external/fluent/fluent-bit:3.2.7"
-DEFAULT_OTEL_COLLECTOR_IMAGE="europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.118.0-main"
+DEFAULT_OTEL_COLLECTOR_IMAGE="europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.120.0-main"
 DEFAULT_SELFMONITOR_IMAGE="europe-docker.pkg.dev/kyma-project/prod/tpi/telemetry-self-monitor:3.2.0-825b449"
-DEFAULT_TEST_TELEMETRYGEN_IMAGE="ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.118.0"
+DEFAULT_TEST_TELEMETRYGEN_IMAGE="ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.120.0"

docs/user/resources/01-telemetry.md

@@ -66,6 +66,24 @@ For further examples, see the [samples](https://github.com/kyma-project/telemetr
 
 For details, see the [Telemetry specification file](https://github.com/kyma-project/telemetry-manager/blob/main/apis/operator/v1alpha1/telemetry_types.go).
 
+### Annotations
+
+Backward compatibility for internal metrics in OpenTelemetry:
+The recent version of OpenTelemetry Collector 0.120.0 introduced a new suffix for internal metrics exposed through the Prometheus endpoint and breaks the stability of internal metrics.
+To maintain backward compatibility, the Telemetry Manager introduces new annotations `telemetry.kyma-project.io/internal-metrics-compatibility-mode` to control the internal metrics suffix.
+To enable the backward compatibility mode, set the annotation `telemetry.kyma-project.io/internal-metrics-compatibility-mode: true` in the Telemetry CR.
+```yaml
+apiVersion: operator.kyma-project.io/v1alpha1
+kind: Telemetry
+metadata:
+  name: default
+  namespace: kyma-system
+  annotations:
+    telemetry.kyma-project.io/internal-metrics-compatibility-mode: "true"
+```
+
+> [! WARNING]
+> The backward compatibility mode is disabled by default. This annotation will be removed in the future.
 <!-- The table below was generated automatically -->
 <!-- Some special tags (html comments) are at the end of lines due to markdown requirements. -->
 <!-- The content between "TABLE-START" and "TABLE-END" will be replaced -->

hack/load-tests/run-load-test.sh

@@ -297,8 +297,8 @@ EOF
 
 function get_result_and_cleanup_trace() {
   RESULT_TYPE="span"
-  QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_spans{service="telemetry-trace-gateway-metrics"}[20m])))'
-  QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_spans{exporter=~"otlp/load-test.*"}[20m])))'
+  QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_spans_total{service="telemetry-trace-gateway-metrics"}[20m])))'
+  QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_spans_total{exporter=~"otlp/load-test.*"}[20m])))'
   QUERY_QUEUE='query=avg(sum(otelcol_exporter_queue_size{service="telemetry-trace-gateway-metrics"}))'
   QUERY_MEMORY='query=round(sum(avg_over_time(container_memory_working_set_bytes{namespace="kyma-system", container="collector"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="kyma-system", workload="telemetry-trace-gateway"}[20m])) by (pod) / 1024 / 1024)'
   QUERY_CPU='query=round(sum(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="kyma-system"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="kyma-system", workload="telemetry-trace-gateway"}[20m])) by (pod), 0.1)'
@@ -327,8 +327,8 @@ function get_result_and_cleanup_trace() {
 
 function get_result_and_cleanup_metric() {
     RESULT_TYPE="metric"
-    QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_metric_points{service="telemetry-metric-gateway-metrics"}[20m])))'
-    QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_metric_points{exporter=~"otlp/load-test.*"}[20m])))'
+    QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_metric_points_total{service="telemetry-metric-gateway-metrics"}[20m])))'
+    QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_metric_points_total{exporter=~"otlp/load-test.*"}[20m])))'
     QUERY_QUEUE='query=avg(sum(otelcol_exporter_queue_size{service="telemetry-metric-gateway-metrics"}))'
     QUERY_MEMORY='query=round(sum(avg_over_time(container_memory_working_set_bytes{namespace="kyma-system", container="collector"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="kyma-system", workload="telemetry-metric-gateway"}[20m])) by (pod) / 1024 / 1024)'
     QUERY_CPU='query=round(sum(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="kyma-system"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="kyma-system", workload="telemetry-metric-gateway"}[20m])) by (pod), 0.1)'
@@ -358,8 +358,8 @@ function get_result_and_cleanup_metric() {
 
 function get_result_and_cleanup_metricagent() {
     RESULT_TYPE="metric"
-    QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_metric_points{service="telemetry-metric-agent-metrics"}[20m])))'
-    QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_metric_points{service=~"telemetry-metric-agent-metrics"}[20m])))'
+    QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_metric_points_total{service="telemetry-metric-agent-metrics"}[20m])))'
+    QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_metric_points_total{service=~"telemetry-metric-agent-metrics"}[20m])))'
     QUERY_QUEUE='query=avg(sum(otelcol_exporter_queue_size{service="telemetry-metric-agent-metrics"}))'
     QUERY_MEMORY='query=round(sum(avg_over_time(container_memory_working_set_bytes{namespace="kyma-system", container="collector"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="kyma-system", workload="telemetry-metric-agent"}[20m])) by (pod) / 1024 / 1024)'
     QUERY_CPU='query=round(sum(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="kyma-system"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="kyma-system", workload="telemetry-metric-agent"}[20m])) by (pod), 0.1)'
@@ -385,8 +385,8 @@ function get_result_and_cleanup_metricagent() {
 
 function get_result_and_cleanup_log_otel() {
   RESULT_TYPE="log"
-  QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_log_records{service=~"log-gateway-metrics"}[20m])))'
-  QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_log_records{service=~"log-gateway-metrics"}[20m])))'
+  QUERY_RECEIVED='query=round(sum(rate(otelcol_receiver_accepted_log_records_total{service=~"log-gateway-metrics"}[20m])))'
+  QUERY_EXPORTED='query=round(sum(rate(otelcol_exporter_sent_log_records_total{service=~"log-gateway-metrics"}[20m])))'
   QUERY_QUEUE='query=avg(sum(otelcol_exporter_queue_size{service=~"log-gateway-metrics"}))'
   QUERY_MEMORY='query=round(sum(avg_over_time(container_memory_working_set_bytes{namespace="log-load-test", container="collector"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="log-load-test", workload="log-gateway"}[20m])) by (pod) / 1024 / 1024)'
   QUERY_CPU='query=round(sum(avg_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="log-load-test"}[20m]) * on(namespace,pod) group_left(workload) avg_over_time(namespace_workload_pod:kube_pod_owner:relabel{namespace="log-load-test", workload="log-gateway"}[20m])) by (pod), 0.1)'

internal/images/images.go

@@ -6,6 +6,6 @@ package images
 const (
 	DefaultFluentBitExporterImage = "europe-docker.pkg.dev/kyma-project/prod/directory-size-exporter:v20250217-6a3cdc4a"
 	DefaultFluentBitImage         = "europe-docker.pkg.dev/kyma-project/prod/external/fluent/fluent-bit:3.2.7"
-	DefaultOTelCollectorImage     = "europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.118.0-main"
+	DefaultOTelCollectorImage     = "europe-docker.pkg.dev/kyma-project/prod/kyma-otel-collector:0.120.0-main"
 	DefaultSelfMonitorImage       = "europe-docker.pkg.dev/kyma-project/prod/tpi/telemetry-self-monitor:3.2.0-825b449"
 )

internal/otelcollector/config/config.go

@@ -56,25 +56,31 @@ type MetricExporter struct {
 }
 
 type PrometheusMetricExporter struct {
-	Host string `yaml:"host"`
-	Port int32  `yaml:"port"`
+	Host              string `yaml:"host"`
+	Port              int32  `yaml:"port"`
+	WithoutScopeInfo  bool   `yaml:"without_scope_info,omitempty"`
+	WithoutTypeSuffix bool   `yaml:"without_type_suffix,omitempty"`
+	WithoutUnits      bool   `yaml:"without_units,omitempty"`
 }
 
 type Logs struct {
 	Level    string `yaml:"level"`
 	Encoding string `yaml:"encoding"`
 }
 
-func DefaultService(pipelines Pipelines) Service {
+func DefaultService(pipelines Pipelines, compatibilityMode bool) Service {
 	telemetry := Telemetry{
 		Metrics: Metrics{
 			Readers: []MetricReader{
 				{
 					Pull: PullMetricReader{
 						Exporter: MetricExporter{
 							Prometheus: PrometheusMetricExporter{
-								Host: fmt.Sprintf("${%s}", EnvVarCurrentPodIP),
-								Port: ports.Metrics,
+								Host:              fmt.Sprintf("${%s}", EnvVarCurrentPodIP),
+								Port:              ports.Metrics,
+								WithoutScopeInfo:  compatibilityMode,
+								WithoutTypeSuffix: compatibilityMode,
+								WithoutUnits:      compatibilityMode,
 							},
 						},
 					},

internal/otelcollector/config/gatewayprocs/resolve_service_name_proc.go

@@ -27,35 +27,34 @@ func ResolveServiceNameStatements() []config.TransformProcessorStatements {
 
 	return []config.TransformProcessorStatements{
 		{
-			Context:    "resource",
 			Statements: statements,
 		},
 	}
 }
 
 // serviceNameNotDefinedBasicCondition specifies the cases for which the service.name attribute is not defined
 // without considering the "unknown_service" and the "unknown_service:<process.executable.name>" cases
-const serviceNameNotDefinedBasicCondition = "attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\""
+const serviceNameNotDefinedBasicCondition = "resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\""
 
 func inferServiceNameFromAttr(attrKey string) string {
 	// serviceNameNotDefinedCondition builds up on the serviceNameNotDefinedBasicCondition
 	// to consider the "unknown_service" and the "unknown_service:<process.executable.name>" cases
 	serviceNameNotDefinedCondition := fmt.Sprintf(
 		"%s or %s",
 		serviceNameNotDefinedBasicCondition,
-		ottlexpr.IsMatch("attributes[\"service.name\"]", "^unknown_service(:.+)?$"),
+		ottlexpr.IsMatch("resource.attributes[\"service.name\"]", "^unknown_service(:.+)?$"),
 	)
 
 	return fmt.Sprintf(
-		"set(attributes[\"service.name\"], attributes[\"%s\"]) where %s",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"%s\"]) where %s",
 		attrKey,
 		serviceNameNotDefinedCondition,
 	)
 }
 
 func setDefaultServiceName() string {
 	return fmt.Sprintf(
-		"set(attributes[\"service.name\"], \"unknown_service\") where %s",
+		"set(resource.attributes[\"service.name\"], \"unknown_service\") where %s",
 		serviceNameNotDefinedBasicCondition,
 	)
 }

internal/otelcollector/config/gatewayprocs/resolve_service_name_proc_test.go

@@ -10,19 +10,18 @@ func TestResolveServiceNameStatements(t *testing.T) {
 	require := require.New(t)
 
 	expectedStatements := []string{
-		"set(attributes[\"service.name\"], attributes[\"kyma.kubernetes_io_app_name\"]) where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\" or IsMatch(attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
-		"set(attributes[\"service.name\"], attributes[\"kyma.app_name\"]) where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\" or IsMatch(attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
-		"set(attributes[\"service.name\"], attributes[\"k8s.deployment.name\"]) where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\" or IsMatch(attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
-		"set(attributes[\"service.name\"], attributes[\"k8s.daemonset.name\"]) where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\" or IsMatch(attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
-		"set(attributes[\"service.name\"], attributes[\"k8s.statefulset.name\"]) where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\" or IsMatch(attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
-		"set(attributes[\"service.name\"], attributes[\"k8s.job.name\"]) where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\" or IsMatch(attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
-		"set(attributes[\"service.name\"], attributes[\"k8s.pod.name\"]) where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\" or IsMatch(attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
-		"set(attributes[\"service.name\"], \"unknown_service\") where attributes[\"service.name\"] == nil or attributes[\"service.name\"] == \"\"",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"kyma.kubernetes_io_app_name\"]) where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\" or IsMatch(resource.attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"kyma.app_name\"]) where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\" or IsMatch(resource.attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"k8s.deployment.name\"]) where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\" or IsMatch(resource.attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"k8s.daemonset.name\"]) where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\" or IsMatch(resource.attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"k8s.statefulset.name\"]) where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\" or IsMatch(resource.attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"k8s.job.name\"]) where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\" or IsMatch(resource.attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
+		"set(resource.attributes[\"service.name\"], resource.attributes[\"k8s.pod.name\"]) where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\" or IsMatch(resource.attributes[\"service.name\"], \"^unknown_service(:.+)?$\")",
+		"set(resource.attributes[\"service.name\"], \"unknown_service\") where resource.attributes[\"service.name\"] == nil or resource.attributes[\"service.name\"] == \"\"",
 	}
 
 	statements := ResolveServiceNameStatements()
 
 	require.Len(statements, 1)
-	require.Equal("resource", statements[0].Context)
 	require.Equal(expectedStatements, statements[0].Statements)
 }

internal/otelcollector/config/log/agent/config_builder.go

@@ -18,12 +18,13 @@ type Builder struct {
 }
 
 type BuildOptions struct {
-	InstrumentationScopeVersion string
-	AgentNamespace              string
+	InstrumentationScopeVersion     string
+	AgentNamespace                  string
+	InternalMetricCompatibilityMode bool
 }
 
 func (b *Builder) Build(logPipelines []telemetryv1alpha1.LogPipeline, opts BuildOptions) *Config {
-	logService := config.DefaultService(makePipelinesConfig())
+	logService := config.DefaultService(makePipelinesConfig(), opts.InternalMetricCompatibilityMode)
 	// Overwrite the extension from default service name
 	logService.Extensions = []string{"health_check", "pprof", "file_storage"}
 

internal/otelcollector/config/log/agent/config_builder_test.go

@@ -90,17 +90,42 @@ func TestBuildAgentConfig(t *testing.T) {
 		})
 	})
 	t.Run("marshaling", func(t *testing.T) {
-		goldenFileName := "config.yaml"
-
-		collectorConfig := sut.Build([]telemetryv1alpha1.LogPipeline{
-			testutils.NewLogPipelineBuilder().WithApplicationInput(true).WithKeepOriginalBody(true).Build()}, BuildOptions{InstrumentationScopeVersion: "main", AgentNamespace: "kyma-system"})
-		configYAML, err := yaml.Marshal(collectorConfig)
-		require.NoError(t, err, "failed to marshal config")
-
-		goldenFilePath := filepath.Join("testdata", goldenFileName)
-		goldenFile, err := os.ReadFile(goldenFilePath)
-		require.NoError(t, err, "failed to load golden file")
-
-		require.Equal(t, string(goldenFile), string(configYAML))
+		tests := []struct {
+			name              string
+			goldenFileName    string
+			withOTLPInput     bool
+			compatibilityMode bool
+		}{
+			{
+				name:              "Compatibility mode disabled",
+				goldenFileName:    "config.yaml",
+				compatibilityMode: false,
+			},
+			{
+				name:              "Compatibility mode enabled",
+				goldenFileName:    "config_compatibility_enabled.yaml",
+				compatibilityMode: true,
+			},
+		}
+		for _, tt := range tests {
+			t.Run(tt.name, func(t *testing.T) {
+				goldenFileName := tt.goldenFileName
+
+				collectorConfig := sut.Build([]telemetryv1alpha1.LogPipeline{
+					testutils.NewLogPipelineBuilder().WithApplicationInput(true).WithKeepOriginalBody(true).Build()}, BuildOptions{
+					InstrumentationScopeVersion:     "main",
+					AgentNamespace:                  "kyma-system",
+					InternalMetricCompatibilityMode: tt.compatibilityMode,
+				})
+				configYAML, err := yaml.Marshal(collectorConfig)
+				require.NoError(t, err, "failed to marshal config")
+
+				goldenFilePath := filepath.Join("testdata", goldenFileName)
+				goldenFile, err := os.ReadFile(goldenFilePath)
+				require.NoError(t, err, "failed to load golden file")
+
+				require.Equal(t, string(goldenFile), string(configYAML))
+			})
+		}
 	})
 }

internal/otelcollector/config/log/agent/processors.go

@@ -35,10 +35,9 @@ func makeInstrumentationScopeRuntime(instrumentationScopeVersion string) *log.Tr
 		ErrorMode: "ignore",
 		LogStatements: []config.TransformProcessorStatements{
 			{
-				Context: "scope",
 				Statements: []string{
-					fmt.Sprintf("set(version, %q)", instrumentationScopeVersion),
-					fmt.Sprintf("set(name, %q)", InstrumentationScopeRuntime),
+					fmt.Sprintf("set(scope.version, %q)", instrumentationScopeVersion),
+					fmt.Sprintf("set(scope.name, %q)", InstrumentationScopeRuntime),
 				},
 			},
 		},

internal/otelcollector/config/log/agent/processors_test.go

@@ -8,10 +8,9 @@ import (
 
 func TestProcessorConfig(t *testing.T) {
 	processorsConfig := makeProcessorsConfig("v1.0.0")
-	require.Equal(t, "scope", processorsConfig.SetInstrumentationScopeRuntime.LogStatements[0].Context)
 	require.Len(t, processorsConfig.SetInstrumentationScopeRuntime.LogStatements[0].Statements, 2)
-	require.Equal(t, "set(version, \"v1.0.0\")", processorsConfig.SetInstrumentationScopeRuntime.LogStatements[0].Statements[0])
-	require.Equal(t, "set(name, \"io.kyma-project.telemetry/runtime\")", processorsConfig.SetInstrumentationScopeRuntime.LogStatements[0].Statements[1])
+	require.Equal(t, "set(scope.version, \"v1.0.0\")", processorsConfig.SetInstrumentationScopeRuntime.LogStatements[0].Statements[0])
+	require.Equal(t, "set(scope.name, \"io.kyma-project.telemetry/runtime\")", processorsConfig.SetInstrumentationScopeRuntime.LogStatements[0].Statements[1])
 
 	require.Equal(t, "5s", processorsConfig.BaseProcessors.MemoryLimiter.CheckInterval)
 	require.Equal(t, 80, processorsConfig.BaseProcessors.MemoryLimiter.LimitPercentage)

internal/otelcollector/config/log/agent/testdata/config.yaml

@@ -86,10 +86,9 @@ processors:
     transform/set-instrumentation-scope-runtime:
         error_mode: ignore
         log_statements:
-            - context: scope
-              statements:
-                - set(version, "main")
-                - set(name, "io.kyma-project.telemetry/runtime")
+            - statements:
+                - set(scope.version, "main")
+                - set(scope.name, "io.kyma-project.telemetry/runtime")
 exporters:
     otlp:
         endpoint: logs.telemetry-system.svc.cluster.local:4317