From c41281d3ee8e67ffa2a2ab780f282d2254d27328 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 4 Apr 2024 00:41:55 -0600 Subject: [PATCH 01/20] Add crd metrics usage information Signed-off-by: Ruben Vargas --- .chloggen/usage_metrics.yaml | 22 +++++ apis/v1beta1/collector_webhook.go | 51 +++++++++- apis/v1beta1/metrics.go | 149 ++++++++++++++++++++++++++++++ config/manager/kustomization.yaml | 6 ++ go.mod | 1 + go.sum | 2 + main.go | 10 ++ pkg/constants/env.go | 2 + 8 files changed, 239 insertions(+), 4 deletions(-) create mode 100755 .chloggen/usage_metrics.yaml create mode 100644 apis/v1beta1/metrics.go diff --git a/.chloggen/usage_metrics.yaml b/.chloggen/usage_metrics.yaml new file mode 100755 index 0000000000..ad28214065 --- /dev/null +++ b/.chloggen/usage_metrics.yaml @@ -0,0 +1,22 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action) +component: collector + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add usage metrics for the collector + +# One or more tracking issues related to the change +issues: [2829] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + This change will add metrics to the OpenTelemetry operator about how the collector is used in the cluster, + it will add the following metrics to the opentelemetry-operator metrics endpoint + opentelemetry_collector_receivers{collector_name="collector_name", namespace="ns", type="otlp"} 1 + opentelemetry_collector_exporters{collector_name="collector_name", namespace="ns", type="otlp"} 1 + opentelemetry_collector_processors{collector_name="collector_name", namespace="ns", type="otlp"} 1 + opentelemetry_collector_processors{collector_name="collector_name", namespace="ns", type="otlp"} 0 diff --git a/apis/v1beta1/collector_webhook.go b/apis/v1beta1/collector_webhook.go index 2c4bc80d85..bd2476af66 100644 --- a/apis/v1beta1/collector_webhook.go +++ b/apis/v1beta1/collector_webhook.go @@ -166,15 +166,48 @@ func (c CollectorWebhook) ValidateCreate(ctx context.Context, obj runtime.Object if !ok { return nil, fmt.Errorf("expected an OpenTelemetryCollector, received %T", obj) } - return c.validate(ctx, otelcol) + + warnings, err := c.validate(ctx, otelcol) + if err != nil { + return warnings, err + } + + err = IncCounters(ctx, otelcol) + if err != nil { + return warnings, err + } + + return warnings, nil } -func (c CollectorWebhook) ValidateUpdate(ctx context.Context, _, newObj runtime.Object) (admission.Warnings, error) { +func (c CollectorWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { otelcol, ok := newObj.(*OpenTelemetryCollector) if !ok { return nil, fmt.Errorf("expected an OpenTelemetryCollector, received %T", newObj) } - return c.validate(ctx, otelcol) + + otelcolOld, ok := oldObj.(*OpenTelemetryCollector) + if !ok { + return nil, fmt.Errorf("expected an OpenTelemetryCollector, received %T", oldObj) + } + + warnings, err := c.validate(ctx, otelcol) + if err != nil { + return warnings, err + } + + // Decrease all metrics related to old CR + err = DecCounters(ctx, otelcolOld) + if err != nil { + return warnings, err + } + + // Increase all metrics related to new CR + err = IncCounters(ctx, otelcolOld) + if err != nil { + return warnings, err + } + return warnings, nil } func (c CollectorWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { @@ -182,7 +215,17 @@ func (c CollectorWebhook) ValidateDelete(ctx context.Context, obj runtime.Object if !ok || otelcol == nil { return nil, fmt.Errorf("expected an OpenTelemetryCollector, received %T", obj) } - return c.validate(ctx, otelcol) + + warnings, err := c.validate(ctx, otelcol) + if err != nil { + return warnings, err + } + err = DecCounters(ctx, otelcol) + if err != nil { + return warnings, err + } + + return warnings, nil } func (c CollectorWebhook) validate(ctx context.Context, r *OpenTelemetryCollector) (admission.Warnings, error) { diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go new file mode 100644 index 0000000000..64aaf40709 --- /dev/null +++ b/apis/v1beta1/metrics.go @@ -0,0 +1,149 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1beta1 + +import ( + "context" + "strings" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + meterName = "crd-metrics" +) + +// Metric labels + +const ( + prefix = "opentelemetry_collector_" + receivers = prefix + "receivers" + exporters = prefix + "exporters" + processors = prefix + "processors" + extensions = prefix + "extensions" + mode = prefix + "info" +) + +type components struct { + receivers []string + processors []string + exporters []string + extensions []string +} + +// BootstrapMetrics configures the OpenTelemetry meter provider with the Prometheus exporter. +func BootstrapMetrics() error { + + exporter, err := prometheus.New(prometheus.WithRegisterer(metrics.Registry)) + if err != nil { + return err + } + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)) + otel.SetMeterProvider(provider) + return err +} + +func extractElements(elements map[string]interface{}) []string { + if elements == nil { + return []string{} + } + + itemsMap := map[string]struct{}{} + var items []string + for key := range elements { + itemName := strings.SplitN(key, "/", 2)[0] + itemsMap[itemName] = struct{}{} + } + for key := range itemsMap { + items = append(items, key) + } + return items +} + +func getComponentsFromConfigV1Beta1(yamlContent Config) *components { + + info := &components{ + receivers: extractElements(yamlContent.Receivers.Object), + exporters: extractElements(yamlContent.Exporters.Object), + } + + if yamlContent.Processors != nil { + info.processors = extractElements(yamlContent.Processors.Object) + } + + if yamlContent.Extensions != nil { + info.extensions = extractElements(yamlContent.Extensions.Object) + } + return info +} + +func IncCounters(ctx context.Context, collector *OpenTelemetryCollector) error { + return updateCounter(ctx, collector, true) +} + +func DecCounters(ctx context.Context, collector *OpenTelemetryCollector) error { + return updateCounter(ctx, collector, false) +} + +func updateCounter(ctx context.Context, collector *OpenTelemetryCollector, up bool) error { + meter := otel.Meter(meterName) + receiversCounter, err := meter.Int64UpDownCounter(receivers) + if err != nil { + return err + } + + exporterCounter, err := meter.Int64UpDownCounter(exporters) + if err != nil { + return err + } + + processorCounter, err := meter.Int64UpDownCounter(processors) + if err != nil { + return err + } + + extensionsCounter, err := meter.Int64UpDownCounter(extensions) + if err != nil { + return err + } + + components := getComponentsFromConfigV1Beta1(collector.Spec.Config) + moveCounter(ctx, collector, components.receivers, receiversCounter, up) + moveCounter(ctx, collector, components.exporters, exporterCounter, up) + moveCounter(ctx, collector, components.processors, processorCounter, up) + moveCounter(ctx, collector, components.extensions, extensionsCounter, up) + + return nil +} + +func moveCounter( + ctx context.Context, collector *OpenTelemetryCollector, types []string, upDown metric.Int64UpDownCounter, up bool) { + for _, exporter := range types { + inc := 1 + if !up { + inc = -1 + } + upDown.Add(ctx, int64(inc), metric.WithAttributes( + attribute.Key("collector_name").String(collector.Name), + attribute.Key("namespace").String(collector.Namespace), + attribute.Key("type").String(exporter), + )) + } +} diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84cb..963273dd37 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/rvargasp/opentelemetry-operator + newTag: 1714976402.0.0 diff --git a/go.mod b/go.mod index c004b55bc5..2be03fe53f 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( go.opentelemetry.io/collector/featuregate v1.5.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 + go.opentelemetry.io/otel/exporters/prometheus v0.48.0 go.opentelemetry.io/otel/metric v1.27.0 go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/sdk/metric v1.27.0 diff --git a/go.sum b/go.sum index 6d5a46661c..4e292ac41c 100644 --- a/go.sum +++ b/go.sum @@ -649,6 +649,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 h1:dT33yIHtmsqpixFsSQP go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0/go.mod h1:h95q0LBGh7hlAC08X2DhSeyIG02YQ0UyioTCVAqRPmc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0 h1:Mbi5PKN7u322woPa85d7ebZ+SOvEoPvoiBu+ryHWgfA= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0/go.mod h1:e7ciERRhZaOZXVjx5MiL8TK5+Xv7G5Gv5PA2ZDEJdL8= +go.opentelemetry.io/otel/exporters/prometheus v0.48.0 h1:sBQe3VNGUjY9IKWQC6z2lNqa5iGbDSxhs60ABwK4y0s= +go.opentelemetry.io/otel/exporters/prometheus v0.48.0/go.mod h1:DtrbMzoZWwQHyrQmCfLam5DZbnmorsGbOtTbYHycU5o= go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= diff --git a/main.go b/main.go index 4ee518ce35..9f1b9f6db0 100644 --- a/main.go +++ b/main.go @@ -119,6 +119,7 @@ func main() { enableNginxInstrumentation bool enableNodeJSInstrumentation bool enableJavaInstrumentation bool + enableCRMetrics bool collectorImage string targetAllocatorImage string operatorOpAMPBridgeImage string @@ -154,6 +155,8 @@ func main() { pflag.BoolVar(&enableNginxInstrumentation, constants.FlagNginx, false, "Controls whether the operator supports nginx auto-instrumentation") pflag.BoolVar(&enableNodeJSInstrumentation, constants.FlagNodeJS, true, "Controls whether the operator supports nodejs auto-instrumentation") pflag.BoolVar(&enableJavaInstrumentation, constants.FlagJava, true, "Controls whether the operator supports java auto-instrumentation") + pflag.BoolVar(&enableCRMetrics, constants.FlagCRMetrics, false, "Controls whether the CR metrics is enabled") + stringFlagOrEnv(&collectorImage, "collector-image", "RELATED_IMAGE_COLLECTOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:%s", v.OpenTelemetryCollector), "The default OpenTelemetry collector image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&targetAllocatorImage, "target-allocator-image", "RELATED_IMAGE_TARGET_ALLOCATOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:%s", v.TargetAllocator), "The default OpenTelemetry target allocator image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&operatorOpAMPBridgeImage, "operator-opamp-bridge-image", "RELATED_IMAGE_OPERATOR_OPAMP_BRIDGE", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/operator-opamp-bridge:%s", v.OperatorOpAMPBridge), "The default OpenTelemetry Operator OpAMP Bridge image. This image is used when no image is specified in the CustomResource.") @@ -334,6 +337,13 @@ func main() { } } } + + if enableCRMetrics { + if metricsErr := otelv1beta1.BootstrapMetrics(); metricsErr != nil { + setupLog.Error(metricsErr, "Error bootstrapping CRD metrics") + } + } + if cfg.LabelsFilter() != nil { for _, basePattern := range cfg.LabelsFilter() { _, compileErr := regexp.Compile(basePattern) diff --git a/pkg/constants/env.go b/pkg/constants/env.go index 8ebd1bb5d9..a8933eab4d 100644 --- a/pkg/constants/env.go +++ b/pkg/constants/env.go @@ -37,6 +37,8 @@ const ( EnvNodeName = "OTEL_RESOURCE_ATTRIBUTES_NODE_NAME" EnvNodeIP = "OTEL_NODE_IP" + FlagCRMetrics = "enable-cr-metrics" + FlagApacheHttpd = "enable-apache-httpd-instrumentation" FlagDotNet = "enable-dotnet-instrumentation" FlagGo = "enable-go-instrumentation" From 531c5e9d56fede53ba63a59100be636c351087c8 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Tue, 14 May 2024 22:05:15 -0600 Subject: [PATCH 02/20] Add mode metric Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index 64aaf40709..e6e5b72b74 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -95,14 +95,37 @@ func getComponentsFromConfigV1Beta1(yamlContent Config) *components { } func IncCounters(ctx context.Context, collector *OpenTelemetryCollector) error { - return updateCounter(ctx, collector, true) + if err := updateComponentCounters(ctx, collector, true); err != nil { + return err + } + return updateGeneralCRMetricsComponents(ctx, collector, true) } func DecCounters(ctx context.Context, collector *OpenTelemetryCollector) error { - return updateCounter(ctx, collector, false) + if err := updateComponentCounters(ctx, collector, false); err != nil { + return err + } + return updateGeneralCRMetricsComponents(ctx, collector, false) } -func updateCounter(ctx context.Context, collector *OpenTelemetryCollector, up bool) error { +func updateGeneralCRMetricsComponents(ctx context.Context, collector *OpenTelemetryCollector, up bool) error { + meter := otel.Meter(meterName) + modeCounter, err := meter.Int64UpDownCounter(mode) + if err != nil { + return err + } + inc := 1 + if !up { + inc = -1 + } + modeCounter.Add(ctx, int64(inc), metric.WithAttributes( + attribute.Key("collector_name").String(collector.Name), + attribute.Key("namespace").String(collector.Namespace), + attribute.Key("type").String(string(collector.Spec.Mode)), + )) + return nil +} +func updateComponentCounters(ctx context.Context, collector *OpenTelemetryCollector, up bool) error { meter := otel.Meter(meterName) receiversCounter, err := meter.Int64UpDownCounter(receivers) if err != nil { From b31a3110d95d8eeb36ad9d710c30ec171ca7dfbe Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 15 May 2024 19:39:12 -0600 Subject: [PATCH 03/20] Refactor CR metrics Signed-off-by: Ruben Vargas --- apis/v1beta1/collector_webhook.go | 30 ++-- apis/v1beta1/metrics.go | 137 ++++++++++-------- config/manager/kustomization.yaml | 6 - controllers/suite_test.go | 2 +- .../podmutation/webhookhandler_suite_test.go | 2 +- main.go | 22 ++- pkg/collector/upgrade/suite_test.go | 2 +- 7 files changed, 104 insertions(+), 97 deletions(-) diff --git a/apis/v1beta1/collector_webhook.go b/apis/v1beta1/collector_webhook.go index bd2476af66..2632899094 100644 --- a/apis/v1beta1/collector_webhook.go +++ b/apis/v1beta1/collector_webhook.go @@ -76,6 +76,7 @@ type CollectorWebhook struct { cfg config.Config scheme *runtime.Scheme reviewer *rbac.Reviewer + metrics *Metrics } func (c CollectorWebhook) Default(_ context.Context, obj runtime.Object) error { @@ -171,10 +172,8 @@ func (c CollectorWebhook) ValidateCreate(ctx context.Context, obj runtime.Object if err != nil { return warnings, err } - - err = IncCounters(ctx, otelcol) - if err != nil { - return warnings, err + if c.metrics != nil { + c.metrics.incCounters(ctx, otelcol) } return warnings, nil @@ -196,17 +195,13 @@ func (c CollectorWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj run return warnings, err } - // Decrease all metrics related to old CR - err = DecCounters(ctx, otelcolOld) - if err != nil { - return warnings, err + if c.metrics != nil { + // Decrease all metrics related to old CR + c.metrics.decCounters(ctx, otelcolOld) + // Increase all metrics related to new CR + c.metrics.incCounters(ctx, otelcol) } - // Increase all metrics related to new CR - err = IncCounters(ctx, otelcolOld) - if err != nil { - return warnings, err - } return warnings, nil } @@ -220,9 +215,9 @@ func (c CollectorWebhook) ValidateDelete(ctx context.Context, obj runtime.Object if err != nil { return warnings, err } - err = DecCounters(ctx, otelcol) - if err != nil { - return warnings, err + + if c.metrics != nil { + c.metrics.decCounters(ctx, otelcol) } return warnings, nil @@ -462,12 +457,13 @@ func checkAutoscalerSpec(autoscaler *AutoscalerSpec) error { return nil } -func SetupCollectorWebhook(mgr ctrl.Manager, cfg config.Config, reviewer *rbac.Reviewer) error { +func SetupCollectorWebhook(mgr ctrl.Manager, cfg config.Config, reviewer *rbac.Reviewer, metrics *Metrics) error { cvw := &CollectorWebhook{ reviewer: reviewer, logger: mgr.GetLogger().WithValues("handler", "CollectorWebhook", "version", "v1beta1"), scheme: mgr.GetScheme(), cfg: cfg, + metrics: metrics, } return ctrl.NewWebhookManagedBy(mgr). For(&OpenTelemetryCollector{}). diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index e6e5b72b74..d648e6d8e6 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -48,9 +48,16 @@ type components struct { extensions []string } +type Metrics struct { + modeCounter metric.Int64UpDownCounter + receiversCounter metric.Int64UpDownCounter + exporterCounter metric.Int64UpDownCounter + processorCounter metric.Int64UpDownCounter + extensionsCounter metric.Int64UpDownCounter +} + // BootstrapMetrics configures the OpenTelemetry meter provider with the Prometheus exporter. func BootstrapMetrics() error { - exporter, err := prometheus.New(prometheus.WithRegisterer(metrics.Registry)) if err != nil { return err @@ -60,6 +67,72 @@ func BootstrapMetrics() error { return err } +func NewMetrics() (*Metrics, error) { + meter := otel.Meter(meterName) + modeCounter, err := meter.Int64UpDownCounter(mode) + if err != nil { + return nil, err + } + receiversCounter, err := meter.Int64UpDownCounter(receivers) + if err != nil { + return nil, err + } + + exporterCounter, err := meter.Int64UpDownCounter(exporters) + if err != nil { + return nil, err + } + + processorCounter, err := meter.Int64UpDownCounter(processors) + if err != nil { + return nil, err + } + + extensionsCounter, err := meter.Int64UpDownCounter(extensions) + if err != nil { + return nil, err + } + + return &Metrics{ + modeCounter: modeCounter, + receiversCounter: receiversCounter, + exporterCounter: exporterCounter, + processorCounter: processorCounter, + extensionsCounter: extensionsCounter, + }, nil + +} + +func (m *Metrics) incCounters(ctx context.Context, collector *OpenTelemetryCollector) { + m.updateComponentCounters(ctx, collector, true) + m.updateGeneralCRMetricsComponents(ctx, collector, true) +} + +func (m *Metrics) decCounters(ctx context.Context, collector *OpenTelemetryCollector) { + m.updateComponentCounters(ctx, collector, false) + m.updateGeneralCRMetricsComponents(ctx, collector, false) +} + +func (m *Metrics) updateGeneralCRMetricsComponents(ctx context.Context, collector *OpenTelemetryCollector, up bool) { + + inc := 1 + if !up { + inc = -1 + } + m.modeCounter.Add(ctx, int64(inc), metric.WithAttributes( + attribute.Key("collector_name").String(collector.Name), + attribute.Key("namespace").String(collector.Namespace), + attribute.Key("type").String(string(collector.Spec.Mode)), + )) +} +func (m *Metrics) updateComponentCounters(ctx context.Context, collector *OpenTelemetryCollector, up bool) { + components := getComponentsFromConfigV1Beta1(collector.Spec.Config) + moveCounter(ctx, collector, components.receivers, m.receiversCounter, up) + moveCounter(ctx, collector, components.exporters, m.exporterCounter, up) + moveCounter(ctx, collector, components.processors, m.processorCounter, up) + moveCounter(ctx, collector, components.extensions, m.extensionsCounter, up) +} + func extractElements(elements map[string]interface{}) []string { if elements == nil { return []string{} @@ -94,68 +167,6 @@ func getComponentsFromConfigV1Beta1(yamlContent Config) *components { return info } -func IncCounters(ctx context.Context, collector *OpenTelemetryCollector) error { - if err := updateComponentCounters(ctx, collector, true); err != nil { - return err - } - return updateGeneralCRMetricsComponents(ctx, collector, true) -} - -func DecCounters(ctx context.Context, collector *OpenTelemetryCollector) error { - if err := updateComponentCounters(ctx, collector, false); err != nil { - return err - } - return updateGeneralCRMetricsComponents(ctx, collector, false) -} - -func updateGeneralCRMetricsComponents(ctx context.Context, collector *OpenTelemetryCollector, up bool) error { - meter := otel.Meter(meterName) - modeCounter, err := meter.Int64UpDownCounter(mode) - if err != nil { - return err - } - inc := 1 - if !up { - inc = -1 - } - modeCounter.Add(ctx, int64(inc), metric.WithAttributes( - attribute.Key("collector_name").String(collector.Name), - attribute.Key("namespace").String(collector.Namespace), - attribute.Key("type").String(string(collector.Spec.Mode)), - )) - return nil -} -func updateComponentCounters(ctx context.Context, collector *OpenTelemetryCollector, up bool) error { - meter := otel.Meter(meterName) - receiversCounter, err := meter.Int64UpDownCounter(receivers) - if err != nil { - return err - } - - exporterCounter, err := meter.Int64UpDownCounter(exporters) - if err != nil { - return err - } - - processorCounter, err := meter.Int64UpDownCounter(processors) - if err != nil { - return err - } - - extensionsCounter, err := meter.Int64UpDownCounter(extensions) - if err != nil { - return err - } - - components := getComponentsFromConfigV1Beta1(collector.Spec.Config) - moveCounter(ctx, collector, components.receivers, receiversCounter, up) - moveCounter(ctx, collector, components.exporters, exporterCounter, up) - moveCounter(ctx, collector, components.processors, processorCounter, up) - moveCounter(ctx, collector, components.extensions, extensionsCounter, up) - - return nil -} - func moveCounter( ctx context.Context, collector *OpenTelemetryCollector, types []string, upDown metric.Int64UpDownCounter, up bool) { for _, exporter := range types { diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 963273dd37..5c5f0b84cb 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,8 +1,2 @@ resources: - manager.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: quay.io/rvargasp/opentelemetry-operator - newTag: 1714976402.0.0 diff --git a/controllers/suite_test.go b/controllers/suite_test.go index b17379dd8b..0b8ee89adf 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -177,7 +177,7 @@ func TestMain(m *testing.M) { } reviewer := rbac.NewReviewer(clientset) - if err = v1beta1.SetupCollectorWebhook(mgr, config.New(), reviewer); err != nil { + if err = v1beta1.SetupCollectorWebhook(mgr, config.New(), reviewer, nil); err != nil { fmt.Printf("failed to SetupWebhookWithManager: %v", err) os.Exit(1) } diff --git a/internal/webhook/podmutation/webhookhandler_suite_test.go b/internal/webhook/podmutation/webhookhandler_suite_test.go index 464649f489..1336cab0e8 100644 --- a/internal/webhook/podmutation/webhookhandler_suite_test.go +++ b/internal/webhook/podmutation/webhookhandler_suite_test.go @@ -105,7 +105,7 @@ func TestMain(m *testing.M) { } reviewer := rbac.NewReviewer(clientset) - if err = v1beta1.SetupCollectorWebhook(mgr, config.New(), reviewer); err != nil { + if err = v1beta1.SetupCollectorWebhook(mgr, config.New(), reviewer, nil); err != nil { fmt.Printf("failed to SetupWebhookWithManager: %v", err) os.Exit(1) } diff --git a/main.go b/main.go index 9f1b9f6db0..677e031726 100644 --- a/main.go +++ b/main.go @@ -155,7 +155,7 @@ func main() { pflag.BoolVar(&enableNginxInstrumentation, constants.FlagNginx, false, "Controls whether the operator supports nginx auto-instrumentation") pflag.BoolVar(&enableNodeJSInstrumentation, constants.FlagNodeJS, true, "Controls whether the operator supports nodejs auto-instrumentation") pflag.BoolVar(&enableJavaInstrumentation, constants.FlagJava, true, "Controls whether the operator supports java auto-instrumentation") - pflag.BoolVar(&enableCRMetrics, constants.FlagCRMetrics, false, "Controls whether the CR metrics is enabled") + pflag.BoolVar(&enableCRMetrics, constants.FlagCRMetrics, true, "Controls whether the CR metrics is enabled") stringFlagOrEnv(&collectorImage, "collector-image", "RELATED_IMAGE_COLLECTOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:%s", v.OpenTelemetryCollector), "The default OpenTelemetry collector image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&targetAllocatorImage, "target-allocator-image", "RELATED_IMAGE_TARGET_ALLOCATOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:%s", v.TargetAllocator), "The default OpenTelemetry target allocator image. This image is used when no image is specified in the CustomResource.") @@ -338,12 +338,6 @@ func main() { } } - if enableCRMetrics { - if metricsErr := otelv1beta1.BootstrapMetrics(); metricsErr != nil { - setupLog.Error(metricsErr, "Error bootstrapping CRD metrics") - } - } - if cfg.LabelsFilter() != nil { for _, basePattern := range cfg.LabelsFilter() { _, compileErr := regexp.Compile(basePattern) @@ -382,7 +376,19 @@ func main() { } if os.Getenv("ENABLE_WEBHOOKS") != "false" { - if err = otelv1beta1.SetupCollectorWebhook(mgr, cfg, reviewer); err != nil { + var crdMetrics *otelv1beta1.Metrics + + if enableCRMetrics { + if metricsErr := otelv1beta1.BootstrapMetrics(); metricsErr != nil { + setupLog.Error(metricsErr, "Error bootstrapping CRD metrics") + } + crdMetrics, err = otelv1beta1.NewMetrics() + if err != nil { + setupLog.Error(err, "Error bootstrapping CRD metrics") + } + } + + if err = otelv1beta1.SetupCollectorWebhook(mgr, cfg, reviewer, crdMetrics); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "OpenTelemetryCollector") os.Exit(1) } diff --git a/pkg/collector/upgrade/suite_test.go b/pkg/collector/upgrade/suite_test.go index e6e505b760..fdafdca245 100644 --- a/pkg/collector/upgrade/suite_test.go +++ b/pkg/collector/upgrade/suite_test.go @@ -105,7 +105,7 @@ func TestMain(m *testing.M) { } reviewer := rbac.NewReviewer(clientset) - if err = v1beta1.SetupCollectorWebhook(mgr, config.New(), reviewer); err != nil { + if err = v1beta1.SetupCollectorWebhook(mgr, config.New(), reviewer, nil); err != nil { fmt.Printf("failed to SetupWebhookWithManager: %v", err) os.Exit(1) } From 99aa71b896af963027815fe17cd2bd3dcc057a5a Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 15 May 2024 20:02:55 -0600 Subject: [PATCH 04/20] Add annotation to avoid generate Metrics Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index d648e6d8e6..93f44dd704 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -48,6 +48,8 @@ type components struct { extensions []string } +// Metrics hold all gauges for the different metrics related to the CRs +// +kubebuilder:object:generate=false type Metrics struct { modeCounter metric.Int64UpDownCounter receiversCounter metric.Int64UpDownCounter From 02fcf951e7208cfafae4e9934aeb48e5130c31a5 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 01:45:46 -0600 Subject: [PATCH 05/20] Add unit tests Signed-off-by: Ruben Vargas --- apis/v1beta1/collector_webhook.go | 9 +- apis/v1beta1/metrics.go | 29 +- apis/v1beta1/metrics_test.go | 840 ++++++++++++++++++++++++++++++ config/manager/kustomization.yaml | 6 + main.go | 4 + 5 files changed, 880 insertions(+), 8 deletions(-) create mode 100644 apis/v1beta1/metrics_test.go diff --git a/apis/v1beta1/collector_webhook.go b/apis/v1beta1/collector_webhook.go index 2632899094..ffe4eff5e5 100644 --- a/apis/v1beta1/collector_webhook.go +++ b/apis/v1beta1/collector_webhook.go @@ -173,7 +173,7 @@ func (c CollectorWebhook) ValidateCreate(ctx context.Context, obj runtime.Object return warnings, err } if c.metrics != nil { - c.metrics.incCounters(ctx, otelcol) + c.metrics.create(ctx, otelcol) } return warnings, nil @@ -196,10 +196,7 @@ func (c CollectorWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj run } if c.metrics != nil { - // Decrease all metrics related to old CR - c.metrics.decCounters(ctx, otelcolOld) - // Increase all metrics related to new CR - c.metrics.incCounters(ctx, otelcol) + c.metrics.update(ctx, otelcolOld, otelcol) } return warnings, nil @@ -217,7 +214,7 @@ func (c CollectorWebhook) ValidateDelete(ctx context.Context, obj runtime.Object } if c.metrics != nil { - c.metrics.decCounters(ctx, otelcol) + c.metrics.delete(ctx, otelcol) } return warnings, nil diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index 93f44dd704..e27683dfe2 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -16,6 +16,7 @@ package v1beta1 import ( "context" + "fmt" "strings" "go.opentelemetry.io/otel" @@ -23,6 +24,7 @@ import ( "go.opentelemetry.io/otel/exporters/prometheus" "go.opentelemetry.io/otel/metric" sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -105,16 +107,39 @@ func NewMetrics() (*Metrics, error) { } -func (m *Metrics) incCounters(ctx context.Context, collector *OpenTelemetryCollector) { +// Init metrics from the first time the operator starts. +func (m *Metrics) Init(ctx context.Context, cl client.Client) error { + opts := []client.ListOption{ + client.MatchingLabels(map[string]string{ + "app.kubernetes.io/managed-by": "opentelemetry-operator", + }), + } + list := &OpenTelemetryCollectorList{} + if err := cl.List(ctx, list, opts...); err != nil { + return fmt.Errorf("failed to list: %w", err) + } + + for i := range list.Items { + m.create(ctx, &list.Items[i]) + } + return nil +} + +func (m *Metrics) create(ctx context.Context, collector *OpenTelemetryCollector) { m.updateComponentCounters(ctx, collector, true) m.updateGeneralCRMetricsComponents(ctx, collector, true) } -func (m *Metrics) decCounters(ctx context.Context, collector *OpenTelemetryCollector) { +func (m *Metrics) delete(ctx context.Context, collector *OpenTelemetryCollector) { m.updateComponentCounters(ctx, collector, false) m.updateGeneralCRMetricsComponents(ctx, collector, false) } +func (m *Metrics) update(ctx context.Context, oldCollector *OpenTelemetryCollector, newCollector *OpenTelemetryCollector) { + m.delete(ctx, oldCollector) + m.create(ctx, newCollector) +} + func (m *Metrics) updateGeneralCRMetricsComponents(ctx context.Context, collector *OpenTelemetryCollector, up bool) { inc := 1 diff --git a/apis/v1beta1/metrics_test.go b/apis/v1beta1/metrics_test.go new file mode 100644 index 0000000000..ffced15fcc --- /dev/null +++ b/apis/v1beta1/metrics_test.go @@ -0,0 +1,840 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1beta1 + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/instrumentation" + "go.opentelemetry.io/otel/sdk/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var wantInstrumentationScope = instrumentation.Scope{ + Name: "crd-metrics", +} + +func TestOTELCollectorCRDMetrics(t *testing.T) { + + otelcollector1 := &OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "collector1", + Namespace: "test1", + }, + Spec: OpenTelemetryCollectorSpec{ + Mode: ModeDeployment, + Config: Config{ + Processors: &AnyConfig{ + Object: map[string]interface{}{ + "batch": nil, + "foo": nil, + }, + }, + Extensions: &AnyConfig{ + Object: map[string]interface{}{ + "extfoo": nil, + }, + }, + }, + }, + } + + otelcollector2 := &OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "collector2", + Namespace: "test2", + }, + Spec: OpenTelemetryCollectorSpec{ + Mode: ModeSidecar, + Config: Config{ + Processors: &AnyConfig{ + Object: map[string]interface{}{ + "x": nil, + "y": nil, + }, + }, + Extensions: &AnyConfig{ + Object: map[string]interface{}{ + "z/r": nil, + }, + }, + Exporters: AnyConfig{ + Object: map[string]interface{}{ + "w": nil, + }, + }, + }, + }, + } + + updatedCollector1 := &OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "collector1", + Namespace: "test1", + }, + Spec: OpenTelemetryCollectorSpec{ + Mode: ModeSidecar, + Config: Config{ + Processors: &AnyConfig{ + Object: map[string]interface{}{ + "foo": nil, + "y": nil, + }, + }, + Extensions: &AnyConfig{ + Object: map[string]interface{}{ + "z/r": nil, + }, + }, + Exporters: AnyConfig{ + Object: map[string]interface{}{ + "w": nil, + }, + }, + }, + }, + } + crdMetrics, err := NewMetrics() + assert.NoError(t, err) + + var tests = []struct { + name string + testFunction func(t *testing.T, m *Metrics, collectors []*OpenTelemetryCollector, reader metric.Reader) + }{ + { + name: "create", + testFunction: checkCreate, + }, + { + name: "update", + testFunction: checkUpdate, + }, + { + name: "delete", + testFunction: checkDelete, + }, + } + + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.testFunction(t, crdMetrics, []*OpenTelemetryCollector{otelcollector1, otelcollector2, updatedCollector1}, reader) + }) + } + +} + +func TestOTELCollectorInitMetrics(t *testing.T) { + otelcollector1 := OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "collector1", + Namespace: "test1", + Labels: map[string]string{"app.kubernetes.io/managed-by": "opentelemetry-operator"}, + }, + Spec: OpenTelemetryCollectorSpec{ + Mode: ModeDeployment, + Config: Config{ + Processors: &AnyConfig{ + Object: map[string]interface{}{ + "batch": nil, + "foo": nil, + }, + }, + Extensions: &AnyConfig{ + Object: map[string]interface{}{ + "extfoo": nil, + }, + }, + }, + }, + } + + otelcollector2 := OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "collector2", + Namespace: "test2", + Labels: map[string]string{"app.kubernetes.io/managed-by": "opentelemetry-operator"}, + }, + Spec: OpenTelemetryCollectorSpec{ + Mode: ModeSidecar, + Config: Config{ + Processors: &AnyConfig{ + Object: map[string]interface{}{ + "x": nil, + "y": nil, + }, + }, + Extensions: &AnyConfig{ + Object: map[string]interface{}{ + "z/r": nil, + }, + }, + Exporters: AnyConfig{ + Object: map[string]interface{}{ + "w": nil, + }, + }, + }, + }, + } + + schemeBuilder := runtime.NewSchemeBuilder(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &OpenTelemetryCollector{}, &OpenTelemetryCollectorList{}) + metav1.AddToGroupVersion(s, GroupVersion) + return nil + }) + scheme := runtime.NewScheme() + err := schemeBuilder.AddToScheme(scheme) + list := &OpenTelemetryCollectorList{ + Items: []OpenTelemetryCollector{otelcollector1, otelcollector2}, + } + require.NoError(t, err, "Should be able to add custom types") + cl := fake.NewClientBuilder().WithLists(list).WithScheme(scheme).Build() + crdMetrics, err := NewMetrics() + assert.NoError(t, err) + + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + err = crdMetrics.Init(context.Background(), cl) + assert.NoError(t, err) + + rm := metricdata.ResourceMetrics{} + err = reader.Collect(context.Background(), &rm) + assert.NoError(t, err) + require.Len(t, rm.ScopeMetrics, 1) + + want := metricdata.ScopeMetrics{ + Scope: wantInstrumentationScope, + Metrics: []metricdata.Metrics{ + { + Name: "opentelemetry_collector_info", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("deployment"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String(string(ModeSidecar)), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_processors", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("batch"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("foo"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("x"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("y"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_extensions", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("extfoo"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("z"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_exporters", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("w"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + }, + } + + metricdatatest.AssertEqual(t, want, rm.ScopeMetrics[0], metricdatatest.IgnoreTimestamp()) + +} + +func checkCreate(t *testing.T, m *Metrics, collectors []*OpenTelemetryCollector, reader metric.Reader) { + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + + m.create(context.Background(), collectors[0]) + rm := metricdata.ResourceMetrics{} + err := reader.Collect(context.Background(), &rm) + assert.NoError(t, err) + + want := metricdata.ScopeMetrics{ + Scope: wantInstrumentationScope, + Metrics: []metricdata.Metrics{ + { + Name: "opentelemetry_collector_info", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("deployment"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_processors", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("batch"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("foo"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_extensions", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("extfoo"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + }, + } + require.Len(t, rm.ScopeMetrics, 1) + metricdatatest.AssertEqual(t, want, rm.ScopeMetrics[0], metricdatatest.IgnoreTimestamp()) + + m.create(context.Background(), collectors[1]) + + rm = metricdata.ResourceMetrics{} + err = reader.Collect(context.Background(), &rm) + assert.NoError(t, err) + require.Len(t, rm.ScopeMetrics, 1) + + want = metricdata.ScopeMetrics{ + Scope: wantInstrumentationScope, + Metrics: []metricdata.Metrics{ + { + Name: "opentelemetry_collector_info", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("deployment"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String(string(ModeSidecar)), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_processors", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("batch"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("foo"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("x"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("y"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_extensions", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("extfoo"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("z"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_exporters", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("w"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + }, + } + + metricdatatest.AssertEqual(t, want, rm.ScopeMetrics[0], metricdatatest.IgnoreTimestamp()) +} + +func checkUpdate(t *testing.T, m *Metrics, collectors []*OpenTelemetryCollector, reader metric.Reader) { + + m.update(context.Background(), collectors[0], collectors[2]) + + rm := metricdata.ResourceMetrics{} + err := reader.Collect(context.Background(), &rm) + assert.NoError(t, err) + require.Len(t, rm.ScopeMetrics, 1) + + want := metricdata.ScopeMetrics{ + Scope: wantInstrumentationScope, + Metrics: []metricdata.Metrics{ + { + Name: "opentelemetry_collector_info", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String(string(ModeDeployment)), + ), + Value: 0, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String(string(ModeSidecar)), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String(string(ModeSidecar)), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_processors", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("batch"), + ), + Value: 0, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("foo"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("y"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("x"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("y"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_extensions", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("extfoo"), + ), + Value: 0, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("z"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("z"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_exporters", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("w"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("w"), + ), + Value: 1, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + }, + } + metricdatatest.AssertEqual(t, want, rm.ScopeMetrics[0], metricdatatest.IgnoreTimestamp()) +} + +func checkDelete(t *testing.T, m *Metrics, collectors []*OpenTelemetryCollector, reader metric.Reader) { + m.delete(context.Background(), collectors[1]) + rm := metricdata.ResourceMetrics{} + err := reader.Collect(context.Background(), &rm) + assert.NoError(t, err) + require.Len(t, rm.ScopeMetrics, 1) + want := metricdata.ScopeMetrics{ + Scope: wantInstrumentationScope, + Metrics: []metricdata.Metrics{ + { + Name: "opentelemetry_collector_info", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String(string(ModeDeployment)), + ), + Value: 0, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String(string(ModeSidecar)), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String(string(ModeSidecar)), + ), + Value: 0, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_processors", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("batch"), + ), + Value: 0, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("foo"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("y"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("x"), + ), + Value: 0, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("y"), + ), + Value: 0, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_extensions", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("extfoo"), + ), + Value: 0, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("z"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("z"), + ), + Value: 0, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + { + Name: "opentelemetry_collector_exporters", + Data: metricdata.Sum[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector1"), + attribute.Key("namespace").String("test1"), + attribute.Key("type").String("w"), + ), + Value: 1, + }, + { + Attributes: attribute.NewSet( + attribute.Key("collector_name").String("collector2"), + attribute.Key("namespace").String("test2"), + attribute.Key("type").String("w"), + ), + Value: 0, + }, + }, + Temporality: metricdata.CumulativeTemporality, + }, + }, + }, + } + metricdatatest.AssertEqual(t, want, rm.ScopeMetrics[0], metricdatatest.IgnoreTimestamp()) +} diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84cb..5859f55bb3 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/rvargasp/opentelemetry-operator + newTag: 1715831609.0.0 diff --git a/main.go b/main.go index 677e031726..678b11986f 100644 --- a/main.go +++ b/main.go @@ -386,6 +386,10 @@ func main() { if err != nil { setupLog.Error(err, "Error bootstrapping CRD metrics") } + err = crdMetrics.Init(ctx, mgr.GetClient()) + if err != nil { + setupLog.Error(err, "Error intiializing CRD metrics") + } } if err = otelv1beta1.SetupCollectorWebhook(mgr, cfg, reviewer, crdMetrics); err != nil { From dc1e2cbf265ef5c5a989dce37f2c5d52f558fc16 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 01:47:05 -0600 Subject: [PATCH 06/20] remove space Signed-off-by: Ruben Vargas --- pkg/constants/env.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/constants/env.go b/pkg/constants/env.go index a8933eab4d..ed6d5e0f2e 100644 --- a/pkg/constants/env.go +++ b/pkg/constants/env.go @@ -37,8 +37,7 @@ const ( EnvNodeName = "OTEL_RESOURCE_ATTRIBUTES_NODE_NAME" EnvNodeIP = "OTEL_NODE_IP" - FlagCRMetrics = "enable-cr-metrics" - + FlagCRMetrics = "enable-cr-metrics" FlagApacheHttpd = "enable-apache-httpd-instrumentation" FlagDotNet = "enable-dotnet-instrumentation" FlagGo = "enable-go-instrumentation" From 7130931b8681cbc3ee6e72c86c012431e4f088b9 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 02:50:39 -0600 Subject: [PATCH 07/20] remove global provider Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics.go | 13 +++++-------- apis/v1beta1/metrics_test.go | 14 +++++--------- main.go | 6 ++++-- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index e27683dfe2..3ad2e55444 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -19,7 +19,6 @@ import ( "fmt" "strings" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/prometheus" "go.opentelemetry.io/otel/metric" @@ -61,18 +60,16 @@ type Metrics struct { } // BootstrapMetrics configures the OpenTelemetry meter provider with the Prometheus exporter. -func BootstrapMetrics() error { +func BootstrapMetrics() (metric.MeterProvider, error) { exporter, err := prometheus.New(prometheus.WithRegisterer(metrics.Registry)) if err != nil { - return err + return nil, err } - provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)) - otel.SetMeterProvider(provider) - return err + return sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)), err } -func NewMetrics() (*Metrics, error) { - meter := otel.Meter(meterName) +func NewMetrics(prv metric.MeterProvider) (*Metrics, error) { + meter := prv.Meter(meterName) modeCounter, err := meter.Int64UpDownCounter(mode) if err != nil { return nil, err diff --git a/apis/v1beta1/metrics_test.go b/apis/v1beta1/metrics_test.go index ffced15fcc..6dd1dff9fc 100644 --- a/apis/v1beta1/metrics_test.go +++ b/apis/v1beta1/metrics_test.go @@ -116,8 +116,6 @@ func TestOTELCollectorCRDMetrics(t *testing.T) { }, }, } - crdMetrics, err := NewMetrics() - assert.NoError(t, err) var tests = []struct { name string @@ -139,14 +137,14 @@ func TestOTELCollectorCRDMetrics(t *testing.T) { reader := sdkmetric.NewManualReader() provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) - otel.SetMeterProvider(provider) + crdMetrics, err := NewMetrics(provider) + assert.NoError(t, err) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.testFunction(t, crdMetrics, []*OpenTelemetryCollector{otelcollector1, otelcollector2, updatedCollector1}, reader) }) } - } func TestOTELCollectorInitMetrics(t *testing.T) { @@ -215,12 +213,11 @@ func TestOTELCollectorInitMetrics(t *testing.T) { } require.NoError(t, err, "Should be able to add custom types") cl := fake.NewClientBuilder().WithLists(list).WithScheme(scheme).Build() - crdMetrics, err := NewMetrics() - assert.NoError(t, err) - reader := sdkmetric.NewManualReader() provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) - otel.SetMeterProvider(provider) + crdMetrics, err := NewMetrics(provider) + assert.NoError(t, err) + err = crdMetrics.Init(context.Background(), cl) assert.NoError(t, err) @@ -340,7 +337,6 @@ func TestOTELCollectorInitMetrics(t *testing.T) { } metricdatatest.AssertEqual(t, want, rm.ScopeMetrics[0], metricdatatest.IgnoreTimestamp()) - } func checkCreate(t *testing.T, m *Metrics, collectors []*OpenTelemetryCollector, reader metric.Reader) { diff --git a/main.go b/main.go index 678b11986f..8de9151d4c 100644 --- a/main.go +++ b/main.go @@ -379,10 +379,12 @@ func main() { var crdMetrics *otelv1beta1.Metrics if enableCRMetrics { - if metricsErr := otelv1beta1.BootstrapMetrics(); metricsErr != nil { + meterProvider, metricsErr := otelv1beta1.BootstrapMetrics() + if metricsErr != nil { setupLog.Error(metricsErr, "Error bootstrapping CRD metrics") } - crdMetrics, err = otelv1beta1.NewMetrics() + + crdMetrics, err = otelv1beta1.NewMetrics(meterProvider) if err != nil { setupLog.Error(err, "Error bootstrapping CRD metrics") } From edf5e1c1e28298525ff1ff72d5ddf26c026c65e2 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 08:02:28 -0600 Subject: [PATCH 08/20] Update main.go Co-authored-by: Israel Blancas --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 8de9151d4c..9030ea299e 100644 --- a/main.go +++ b/main.go @@ -155,7 +155,7 @@ func main() { pflag.BoolVar(&enableNginxInstrumentation, constants.FlagNginx, false, "Controls whether the operator supports nginx auto-instrumentation") pflag.BoolVar(&enableNodeJSInstrumentation, constants.FlagNodeJS, true, "Controls whether the operator supports nodejs auto-instrumentation") pflag.BoolVar(&enableJavaInstrumentation, constants.FlagJava, true, "Controls whether the operator supports java auto-instrumentation") - pflag.BoolVar(&enableCRMetrics, constants.FlagCRMetrics, true, "Controls whether the CR metrics is enabled") + pflag.BoolVar(&enableCRMetrics, constants.FlagCRMetrics, true, "Controls whether exposing the CR metrics is enabled") stringFlagOrEnv(&collectorImage, "collector-image", "RELATED_IMAGE_COLLECTOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:%s", v.OpenTelemetryCollector), "The default OpenTelemetry collector image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&targetAllocatorImage, "target-allocator-image", "RELATED_IMAGE_TARGET_ALLOCATOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:%s", v.TargetAllocator), "The default OpenTelemetry target allocator image. This image is used when no image is specified in the CustomResource.") From abeabb921618e950f620f1a3caab23f489b05959 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 08:04:04 -0600 Subject: [PATCH 09/20] revert kusttomization.yaml Signed-off-by: Ruben Vargas --- config/manager/kustomization.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5859f55bb3..5c5f0b84cb 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,8 +1,2 @@ resources: - manager.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: quay.io/rvargasp/opentelemetry-operator - newTag: 1715831609.0.0 From fc43894accc55602bf7e134ee7d20ca390a43b44 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 17:17:05 -0600 Subject: [PATCH 10/20] rename some constants Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index 3ad2e55444..49ae22d709 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -34,12 +34,12 @@ const ( // Metric labels const ( - prefix = "opentelemetry_collector_" - receivers = prefix + "receivers" - exporters = prefix + "exporters" - processors = prefix + "processors" - extensions = prefix + "extensions" - mode = prefix + "info" + prefix = "opentelemetry_collector_" + receiversMetricName = prefix + "receivers" + exportersMetricName = prefix + "exporters" + processorsMetricName = prefix + "processors" + extensionsMetricName = prefix + "extensions" + modeMetricName = prefix + "info" ) type components struct { @@ -70,26 +70,26 @@ func BootstrapMetrics() (metric.MeterProvider, error) { func NewMetrics(prv metric.MeterProvider) (*Metrics, error) { meter := prv.Meter(meterName) - modeCounter, err := meter.Int64UpDownCounter(mode) + modeCounter, err := meter.Int64UpDownCounter(modeMetricName) if err != nil { return nil, err } - receiversCounter, err := meter.Int64UpDownCounter(receivers) + receiversCounter, err := meter.Int64UpDownCounter(receiversMetricName) if err != nil { return nil, err } - exporterCounter, err := meter.Int64UpDownCounter(exporters) + exporterCounter, err := meter.Int64UpDownCounter(exportersMetricName) if err != nil { return nil, err } - processorCounter, err := meter.Int64UpDownCounter(processors) + processorCounter, err := meter.Int64UpDownCounter(processorsMetricName) if err != nil { return nil, err } - extensionsCounter, err := meter.Int64UpDownCounter(extensions) + extensionsCounter, err := meter.Int64UpDownCounter(extensionsMetricName) if err != nil { return nil, err } @@ -158,6 +158,8 @@ func (m *Metrics) updateComponentCounters(ctx context.Context, collector *OpenTe } func extractElements(elements map[string]interface{}) []string { + // TODO: we should get rid of this method and centralize the parse logic + // see https://github.com/open-telemetry/opentelemetry-operator/issues/2603 if elements == nil { return []string{} } From da34c785933303795e7d0b01807a6fd6693150f0 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 17:23:35 -0600 Subject: [PATCH 11/20] Add connectors metrics Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index 49ae22d709..fb13bc099c 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -39,14 +39,17 @@ const ( exportersMetricName = prefix + "exporters" processorsMetricName = prefix + "processors" extensionsMetricName = prefix + "extensions" + connectorsMetricName = prefix + "connectors" modeMetricName = prefix + "info" ) +// TODO: Refactor this logic, centralize it. type components struct { receivers []string processors []string exporters []string extensions []string + connectors []string } // Metrics hold all gauges for the different metrics related to the CRs @@ -57,6 +60,7 @@ type Metrics struct { exporterCounter metric.Int64UpDownCounter processorCounter metric.Int64UpDownCounter extensionsCounter metric.Int64UpDownCounter + connectorsCounter metric.Int64UpDownCounter } // BootstrapMetrics configures the OpenTelemetry meter provider with the Prometheus exporter. @@ -94,12 +98,18 @@ func NewMetrics(prv metric.MeterProvider) (*Metrics, error) { return nil, err } + connectorsCounter, err := meter.Int64UpDownCounter(connectorsMetricName) + if err != nil { + return nil, err + } + return &Metrics{ modeCounter: modeCounter, receiversCounter: receiversCounter, exporterCounter: exporterCounter, processorCounter: processorCounter, extensionsCounter: extensionsCounter, + connectorsCounter: connectorsCounter, }, nil } @@ -155,6 +165,8 @@ func (m *Metrics) updateComponentCounters(ctx context.Context, collector *OpenTe moveCounter(ctx, collector, components.exporters, m.exporterCounter, up) moveCounter(ctx, collector, components.processors, m.processorCounter, up) moveCounter(ctx, collector, components.extensions, m.extensionsCounter, up) + moveCounter(ctx, collector, components.connectors, m.connectorsCounter, up) + } func extractElements(elements map[string]interface{}) []string { @@ -190,6 +202,11 @@ func getComponentsFromConfigV1Beta1(yamlContent Config) *components { if yamlContent.Extensions != nil { info.extensions = extractElements(yamlContent.Extensions.Object) } + + if yamlContent.Connectors != nil { + info.connectors = extractElements(yamlContent.Connectors.Object) + } + return info } From 5ebe2adc209a29b41021e4bd77da80d913885c02 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Thu, 23 May 2024 17:26:23 -0600 Subject: [PATCH 12/20] Update chlog Signed-off-by: Ruben Vargas --- .chloggen/usage_metrics.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/usage_metrics.yaml b/.chloggen/usage_metrics.yaml index ad28214065..16b1c78fcc 100755 --- a/.chloggen/usage_metrics.yaml +++ b/.chloggen/usage_metrics.yaml @@ -19,4 +19,4 @@ subtext: | opentelemetry_collector_receivers{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_exporters{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_processors{collector_name="collector_name", namespace="ns", type="otlp"} 1 - opentelemetry_collector_processors{collector_name="collector_name", namespace="ns", type="otlp"} 0 + opentelemetry_collector_connectors{collector_name="collector_name", namespace="ns", type="myconnector"} 0 From e9c0cb2a060b50908983d90b02be9f24a61864f6 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 06:43:13 -0600 Subject: [PATCH 13/20] merge new with init, rename some functions, improve changelog entry Signed-off-by: Ruben Vargas --- .chloggen/usage_metrics.yaml | 2 ++ apis/v1beta1/metrics.go | 17 +++++++++++------ main.go | 7 ++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.chloggen/usage_metrics.yaml b/.chloggen/usage_metrics.yaml index 16b1c78fcc..68e67c2415 100755 --- a/.chloggen/usage_metrics.yaml +++ b/.chloggen/usage_metrics.yaml @@ -16,7 +16,9 @@ issues: [2829] subtext: | This change will add metrics to the OpenTelemetry operator about how the collector is used in the cluster, it will add the following metrics to the opentelemetry-operator metrics endpoint + ``` opentelemetry_collector_receivers{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_exporters{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_processors{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_connectors{collector_name="collector_name", namespace="ns", type="myconnector"} 0 + ``` \ No newline at end of file diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index fb13bc099c..e5e42e5b80 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -72,7 +72,7 @@ func BootstrapMetrics() (metric.MeterProvider, error) { return sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)), err } -func NewMetrics(prv metric.MeterProvider) (*Metrics, error) { +func NewMetrics(prv metric.MeterProvider, ctx context.Context, cl client.Client) (*Metrics, error) { meter := prv.Meter(meterName) modeCounter, err := meter.Int64UpDownCounter(modeMetricName) if err != nil { @@ -103,19 +103,24 @@ func NewMetrics(prv metric.MeterProvider) (*Metrics, error) { return nil, err } - return &Metrics{ + m := &Metrics{ modeCounter: modeCounter, receiversCounter: receiversCounter, exporterCounter: exporterCounter, processorCounter: processorCounter, extensionsCounter: extensionsCounter, connectorsCounter: connectorsCounter, - }, nil + } + err = m.init(ctx, cl) + if err != nil { + return nil, err + } + return m, nil } // Init metrics from the first time the operator starts. -func (m *Metrics) Init(ctx context.Context, cl client.Client) error { +func (m *Metrics) init(ctx context.Context, cl client.Client) error { opts := []client.ListOption{ client.MatchingLabels(map[string]string{ "app.kubernetes.io/managed-by": "opentelemetry-operator", @@ -160,7 +165,7 @@ func (m *Metrics) updateGeneralCRMetricsComponents(ctx context.Context, collecto )) } func (m *Metrics) updateComponentCounters(ctx context.Context, collector *OpenTelemetryCollector, up bool) { - components := getComponentsFromConfigV1Beta1(collector.Spec.Config) + components := getComponentsFromConfig(collector.Spec.Config) moveCounter(ctx, collector, components.receivers, m.receiversCounter, up) moveCounter(ctx, collector, components.exporters, m.exporterCounter, up) moveCounter(ctx, collector, components.processors, m.processorCounter, up) @@ -188,7 +193,7 @@ func extractElements(elements map[string]interface{}) []string { return items } -func getComponentsFromConfigV1Beta1(yamlContent Config) *components { +func getComponentsFromConfig(yamlContent Config) *components { info := &components{ receivers: extractElements(yamlContent.Receivers.Object), diff --git a/main.go b/main.go index 9030ea299e..a701d6c963 100644 --- a/main.go +++ b/main.go @@ -384,14 +384,11 @@ func main() { setupLog.Error(metricsErr, "Error bootstrapping CRD metrics") } - crdMetrics, err = otelv1beta1.NewMetrics(meterProvider) + crdMetrics, err = otelv1beta1.NewMetrics(meterProvider, ctx, mgr.GetClient()) if err != nil { setupLog.Error(err, "Error bootstrapping CRD metrics") } - err = crdMetrics.Init(ctx, mgr.GetClient()) - if err != nil { - setupLog.Error(err, "Error intiializing CRD metrics") - } + } if err = otelv1beta1.SetupCollectorWebhook(mgr, cfg, reviewer, crdMetrics); err != nil { From 40fb661d294b52ba33a3ccfa4bcd8e1c59ea1035 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 06:45:13 -0600 Subject: [PATCH 14/20] improve todo comment Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index e5e42e5b80..78595cb189 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -43,7 +43,7 @@ const ( modeMetricName = prefix + "info" ) -// TODO: Refactor this logic, centralize it. +// TODO: Refactor this logic, centralize it. See: https://github.com/open-telemetry/opentelemetry-operator/issues/2603 type components struct { receivers []string processors []string From cb0eb80fd9d15ec11266d1b7af4ff2ba1c76e379 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 07:21:32 -0600 Subject: [PATCH 15/20] fix tests Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/apis/v1beta1/metrics_test.go b/apis/v1beta1/metrics_test.go index 6dd1dff9fc..aa38db0497 100644 --- a/apis/v1beta1/metrics_test.go +++ b/apis/v1beta1/metrics_test.go @@ -134,10 +134,18 @@ func TestOTELCollectorCRDMetrics(t *testing.T) { testFunction: checkDelete, }, } - + schemeBuilder := runtime.NewSchemeBuilder(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, &OpenTelemetryCollector{}, &OpenTelemetryCollectorList{}) + metav1.AddToGroupVersion(s, GroupVersion) + return nil + }) + scheme := runtime.NewScheme() + err := schemeBuilder.AddToScheme(scheme) + require.NoError(t, err) reader := sdkmetric.NewManualReader() provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) - crdMetrics, err := NewMetrics(provider) + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + crdMetrics, err := NewMetrics(provider, context.Background(), cl) assert.NoError(t, err) for _, tt := range tests { @@ -208,6 +216,7 @@ func TestOTELCollectorInitMetrics(t *testing.T) { }) scheme := runtime.NewScheme() err := schemeBuilder.AddToScheme(scheme) + require.NoError(t, err) list := &OpenTelemetryCollectorList{ Items: []OpenTelemetryCollector{otelcollector1, otelcollector2}, } @@ -215,10 +224,9 @@ func TestOTELCollectorInitMetrics(t *testing.T) { cl := fake.NewClientBuilder().WithLists(list).WithScheme(scheme).Build() reader := sdkmetric.NewManualReader() provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) - crdMetrics, err := NewMetrics(provider) + NewMetrics(provider, context.Background(), cl) assert.NoError(t, err) - err = crdMetrics.Init(context.Background(), cl) assert.NoError(t, err) rm := metricdata.ResourceMetrics{} From ba3a479c186b67991a1d2a15ad7973ccb86e8656 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 07:22:50 -0600 Subject: [PATCH 16/20] set flag to default false Signed-off-by: Ruben Vargas --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index a701d6c963..b8fba2a0ca 100644 --- a/main.go +++ b/main.go @@ -155,7 +155,7 @@ func main() { pflag.BoolVar(&enableNginxInstrumentation, constants.FlagNginx, false, "Controls whether the operator supports nginx auto-instrumentation") pflag.BoolVar(&enableNodeJSInstrumentation, constants.FlagNodeJS, true, "Controls whether the operator supports nodejs auto-instrumentation") pflag.BoolVar(&enableJavaInstrumentation, constants.FlagJava, true, "Controls whether the operator supports java auto-instrumentation") - pflag.BoolVar(&enableCRMetrics, constants.FlagCRMetrics, true, "Controls whether exposing the CR metrics is enabled") + pflag.BoolVar(&enableCRMetrics, constants.FlagCRMetrics, false, "Controls whether exposing the CR metrics is enabled") stringFlagOrEnv(&collectorImage, "collector-image", "RELATED_IMAGE_COLLECTOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:%s", v.OpenTelemetryCollector), "The default OpenTelemetry collector image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&targetAllocatorImage, "target-allocator-image", "RELATED_IMAGE_TARGET_ALLOCATOR", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:%s", v.TargetAllocator), "The default OpenTelemetry target allocator image. This image is used when no image is specified in the CustomResource.") From 9c05ef2a298ce70aaaad20c0a400f39f41deba88 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 07:33:26 -0600 Subject: [PATCH 17/20] fix lint issues Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apis/v1beta1/metrics_test.go b/apis/v1beta1/metrics_test.go index aa38db0497..71df095c53 100644 --- a/apis/v1beta1/metrics_test.go +++ b/apis/v1beta1/metrics_test.go @@ -224,9 +224,7 @@ func TestOTELCollectorInitMetrics(t *testing.T) { cl := fake.NewClientBuilder().WithLists(list).WithScheme(scheme).Build() reader := sdkmetric.NewManualReader() provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) - NewMetrics(provider, context.Background(), cl) - assert.NoError(t, err) - + _, err = NewMetrics(provider, context.Background(), cl) assert.NoError(t, err) rm := metricdata.ResourceMetrics{} From b891c8fcf500af509251460811ec37f4cafca8b8 Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 08:14:37 -0600 Subject: [PATCH 18/20] breaking line Signed-off-by: Ruben Vargas --- .chloggen/usage_metrics.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/usage_metrics.yaml b/.chloggen/usage_metrics.yaml index 68e67c2415..145e8b8945 100755 --- a/.chloggen/usage_metrics.yaml +++ b/.chloggen/usage_metrics.yaml @@ -21,4 +21,4 @@ subtext: | opentelemetry_collector_exporters{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_processors{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_connectors{collector_name="collector_name", namespace="ns", type="myconnector"} 0 - ``` \ No newline at end of file + ``` From e9454224ba0dfa90e57e467c2e9de19e90c4131e Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 23:31:35 -0600 Subject: [PATCH 19/20] Use api reader to avoid cache issues Signed-off-by: Ruben Vargas --- apis/v1beta1/metrics.go | 4 ++-- main.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apis/v1beta1/metrics.go b/apis/v1beta1/metrics.go index 78595cb189..395306059d 100644 --- a/apis/v1beta1/metrics.go +++ b/apis/v1beta1/metrics.go @@ -72,7 +72,7 @@ func BootstrapMetrics() (metric.MeterProvider, error) { return sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)), err } -func NewMetrics(prv metric.MeterProvider, ctx context.Context, cl client.Client) (*Metrics, error) { +func NewMetrics(prv metric.MeterProvider, ctx context.Context, cl client.Reader) (*Metrics, error) { meter := prv.Meter(meterName) modeCounter, err := meter.Int64UpDownCounter(modeMetricName) if err != nil { @@ -120,7 +120,7 @@ func NewMetrics(prv metric.MeterProvider, ctx context.Context, cl client.Client) } // Init metrics from the first time the operator starts. -func (m *Metrics) init(ctx context.Context, cl client.Client) error { +func (m *Metrics) init(ctx context.Context, cl client.Reader) error { opts := []client.ListOption{ client.MatchingLabels(map[string]string{ "app.kubernetes.io/managed-by": "opentelemetry-operator", diff --git a/main.go b/main.go index b8fba2a0ca..798bbeada8 100644 --- a/main.go +++ b/main.go @@ -384,9 +384,9 @@ func main() { setupLog.Error(metricsErr, "Error bootstrapping CRD metrics") } - crdMetrics, err = otelv1beta1.NewMetrics(meterProvider, ctx, mgr.GetClient()) + crdMetrics, err = otelv1beta1.NewMetrics(meterProvider, ctx, mgr.GetAPIReader()) if err != nil { - setupLog.Error(err, "Error bootstrapping CRD metrics") + setupLog.Error(err, "Error init CRD metrics") } } From 0d76e7ccbe7f4ca8baeaa28ccdd1ed4caefcf96b Mon Sep 17 00:00:00 2001 From: Ruben Vargas Date: Wed, 29 May 2024 23:32:37 -0600 Subject: [PATCH 20/20] Add info metric to changelog entry Signed-off-by: Ruben Vargas --- .chloggen/usage_metrics.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.chloggen/usage_metrics.yaml b/.chloggen/usage_metrics.yaml index 145e8b8945..c4051bde5a 100755 --- a/.chloggen/usage_metrics.yaml +++ b/.chloggen/usage_metrics.yaml @@ -21,4 +21,5 @@ subtext: | opentelemetry_collector_exporters{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_processors{collector_name="collector_name", namespace="ns", type="otlp"} 1 opentelemetry_collector_connectors{collector_name="collector_name", namespace="ns", type="myconnector"} 0 + opentelemetry_collector_info{collector_name="simplest",namespace="default", type="deployment"} 1 ```