Fix: opentelemetry_allocator_targets_remaining should be a gauge instead of counter (#3590)

Co-authored-by: Israel Blancas <[email protected]>

Author: chenlujjj

.chloggen/fix-allocator-metric.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
+component: target allocator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "change metric `opentelemetry_allocator_targets_remaining` type from counter to gauge"
+
+# One or more tracking issues related to the change
+issues: [3578]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

cmd/otel-allocator/allocation/strategy.go

@@ -47,7 +47,7 @@ var (
 		Name: "opentelemetry_allocator_time_to_allocate",
 		Help: "The time it takes to allocate",
 	}, []string{"method", "strategy"})
-	targetsRemaining = promauto.NewCounter(prometheus.CounterOpts{
+	TargetsRemaining = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "opentelemetry_allocator_targets_remaining",
 		Help: "Number of targets kept after filtering.",
 	})
@@ -80,7 +80,7 @@ func WithFallbackStrategy(fallbackStrategy string) AllocationOption {
 }
 
 func RecordTargetsKept(targets map[string]*target.Item) {
-	targetsRemaining.Add(float64(len(targets)))
+	TargetsRemaining.Set(float64(len(targets)))
 }
 
 func New(name string, log logr.Logger, opts ...AllocationOption) (Allocator, error) {