Scrape config and probe support in target allocator

cmd/otel-allocator/watcher/promOperator.go

@@ -18,12 +18,9 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
-	"os"
 	"time"
 
 	"github.com/blang/semver/v4"
-	"github.com/go-kit/log"
-	"github.com/go-kit/log/level"
 	"github.com/go-logr/logr"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	promv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1"
@@ -53,6 +50,7 @@ const (
 )
 
 func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocatorconfig.Config) (*PrometheusCRWatcher, error) {
+	slogger := slog.New(logr.ToSlogHandler(logger))
 	var resourceSelector *prometheus.ResourceSelector
 	mClient, err := monitoringclient.NewForConfig(cfg.ClusterConfig)
 	if err != nil {
@@ -92,9 +90,7 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 		},
 	}
 
-	promOperatorLogger := level.NewFilter(log.NewLogfmtLogger(os.Stderr), level.AllowWarn())
-	promOperatorSlogLogger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
-	generator, err := prometheus.NewConfigGenerator(promOperatorLogger, prom, true)
+	generator, err := prometheus.NewConfigGenerator(slogger, prom, prometheus.WithEndpointSliceSupport())
 
 	if err != nil {
 		return nil, err
@@ -112,21 +108,21 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 			logger.Error(err, "Retrying namespace informer creation in promOperator CRD watcher")
 			return true
 		}, func() error {
-			nsMonInf, err = getNamespaceInformer(ctx, map[string]struct{}{v1.NamespaceAll: {}}, promOperatorLogger, clientset, operatorMetrics)
+			nsMonInf, err = getNamespaceInformer(ctx, map[string]struct{}{v1.NamespaceAll: {}}, slogger, clientset, operatorMetrics)
 			return err
 		})
 	if getNamespaceInformerErr != nil {
 		logger.Error(getNamespaceInformerErr, "Failed to create namespace informer in promOperator CRD watcher")
 		return nil, getNamespaceInformerErr
 	}
 
-	resourceSelector, err = prometheus.NewResourceSelector(promOperatorSlogLogger, prom, store, nsMonInf, operatorMetrics, eventRecorder)
+	resourceSelector, err = prometheus.NewResourceSelector(slogger, prom, store, nsMonInf, operatorMetrics, eventRecorder)
 	if err != nil {
 		logger.Error(err, "Failed to create resource selector in promOperator CRD watcher")
 	}
 
 	return &PrometheusCRWatcher{
-		logger:                          logger,
+		logger:                          slogger,
 		kubeMonitoringClient:            mClient,
 		k8sClient:                       clientset,
 		informers:                       monitoringInformers,
@@ -145,7 +141,7 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 }
 
 type PrometheusCRWatcher struct {
-	logger                          logr.Logger
+	logger                          *slog.Logger
 	kubeMonitoringClient            monitoringclient.Interface
 	k8sClient                       kubernetes.Interface
 	informers                       map[string]*informers.ForResource
@@ -162,7 +158,7 @@ type PrometheusCRWatcher struct {
 	store                           *assets.StoreBuilder
 }
 
-func getNamespaceInformer(ctx context.Context, allowList map[string]struct{}, promOperatorLogger log.Logger, clientset kubernetes.Interface, operatorMetrics *operator.Metrics) (cache.SharedIndexInformer, error) {
+func getNamespaceInformer(ctx context.Context, allowList map[string]struct{}, promOperatorLogger *slog.Logger, clientset kubernetes.Interface, operatorMetrics *operator.Metrics) (cache.SharedIndexInformer, error) {
 	kubernetesVersion, err := clientset.Discovery().ServerVersion()
 	if err != nil {
 		return nil, err
@@ -252,7 +248,7 @@ func (w *PrometheusCRWatcher) Watch(upstreamEvents chan Event, upstreamErrors ch
 				} {
 					sync, err := k8sutil.LabelSelectionHasChanged(old.Labels, cur.Labels, selector)
 					if err != nil {
-						w.logger.Error(err, "Failed to check label selection between namespaces while handling namespace updates", "selector", name)
+						w.logger.Error(err.Error(), "Failed to check label selection between namespaces while handling namespace updates", "selector", name)
 						return
 					}
 
@@ -273,17 +269,20 @@ func (w *PrometheusCRWatcher) Watch(upstreamEvents chan Event, upstreamErrors ch
 	for name, resource := range w.informers {
 		resource.Start(w.stopChannel)
 
-		if ok := cache.WaitForNamedCacheSync(name, w.stopChannel, resource.HasSynced); !ok {
-			success = false
+		if ok := w.WaitForNamedCacheSync(name, resource.HasSynced); !ok {
+			w.logger.Info("skipping informer", "informer", name)
+			continue
 		}
 
 		// only send an event notification if there isn't one already
 		resource.AddEventHandler(cache.ResourceEventHandlerFuncs{
 			// these functions only write to the notification channel if it's empty to avoid blocking
 			// if scrape config updates are being rate-limited
 			AddFunc: func(obj interface{}) {
+				w.logger.Info("added", "obj", obj)
 				select {
 				case notifyEvents <- struct{}{}:
+					w.logger.Info("added")
 				default:
 				}
 			},
@@ -378,7 +377,7 @@ func (w *PrometheusCRWatcher) LoadConfig(ctx context.Context) (*promconfig.Confi
 			"",
 			nil,
 			nil,
-			monitoringv1.TSDBSpec{},
+			&monitoringv1.TSDBSpec{},
 			nil,
 			nil,
 			serviceMonitorInstances,
@@ -415,3 +414,41 @@ func (w *PrometheusCRWatcher) LoadConfig(ctx context.Context) (*promconfig.Confi
 		return promCfg, nil
 	}
 }
+
+// WaitForNamedCacheSync adds a timeout to the informer's wait for the cache to be ready.
+// If the PrometheusCRWatcher is unable to load an informer within 15 seconds, the method is
+// cancelled and returns false. A successful informer load will return true. This method also
+// will be cancelled if the target allocator's stopChannel is called before it returns.
+//
+// This method is inspired by the upstream prometheus-operator implementation, with a shorter timeout
+// and support for the PrometheusCRWatcher's stopChannel.
+// https://github.com/prometheus-operator/prometheus-operator/blob/293c16c854ce69d1da9fdc8f0705de2d67bfdbfa/pkg/operator/operator.go#L433
+func (w *PrometheusCRWatcher) WaitForNamedCacheSync(controllerName string, inf cache.InformerSynced) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+	t := time.NewTicker(time.Second * 5)
+	defer t.Stop()
+
+	go func() {
+		for {
+			select {
+			case <-t.C:
+				w.logger.Debug("cache sync not yet completed")
+			case <-ctx.Done():
+				return
+			case <-w.stopChannel:
+				w.logger.Warn("stop received, shutting down cache syncing")
+				cancel()
+				return
+			}
+		}
+	}()
+
+	ok := cache.WaitForNamedCacheSync(controllerName, ctx.Done(), inf)
+	if !ok {
+		w.logger.Error("failed to sync cache")
+	} else {
+		w.logger.Debug("successfully synced cache")
+	}
+
+	return ok
+}

go.mod

@@ -1,6 +1,8 @@
 module github.com/open-telemetry/opentelemetry-operator
 
-go 1.22.0
+go 1.23
+
+toolchain go1.23.2
 
 retract v1.51.0
 
@@ -22,9 +24,9 @@ require (
 	github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e
 	github.com/operator-framework/api v0.27.0
 	github.com/operator-framework/operator-lib v0.15.0
-	github.com/prometheus-operator/prometheus-operator v0.76.0
-	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2
-	github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2
+	github.com/prometheus-operator/prometheus-operator v0.77.2
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.2
+	github.com/prometheus-operator/prometheus-operator/pkg/client v0.77.2
 	github.com/prometheus/client_golang v1.20.5
 	github.com/prometheus/common v0.60.0
 	github.com/prometheus/prometheus v0.54.1
@@ -205,16 +207,16 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/arch v0.8.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
-	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
-	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/term v0.25.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/tools v0.25.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/api v0.198.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect

go.sum

@@ -539,10 +539,16 @@ github.com/prometheus-community/prom-label-proxy v0.11.0 h1:IO02WiiFMfcIqvjhwMbC
 github.com/prometheus-community/prom-label-proxy v0.11.0/go.mod h1:lfvrG70XqsxWDrSh1843QXBG0fSg8EbIXmAo8xGsvw8=
 github.com/prometheus-operator/prometheus-operator v0.76.0 h1:EjGJiQVF3BUy/iygeRlN6iMBIAySMGZobEm7+7A95pI=
 github.com/prometheus-operator/prometheus-operator v0.76.0/go.mod h1:y4PxsSBsOBwK1vXIw9U8DGLi8EptquItyP2IpqUtTGs=
+github.com/prometheus-operator/prometheus-operator v0.77.2 h1:HRewl8m49URAOfRTZql2YjjNGZ8+25srcGji5wx/tic=
+github.com/prometheus-operator/prometheus-operator v0.77.2/go.mod h1:y2XLK9xQ4ulaAtWmvKDM+hQNl7w2Wv/iE+tuF5Xnb8A=
 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2 h1:BpGDC87A2SaxbKgONsFLEX3kRcRJee2aLQbjXsuz0hA=
 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2/go.mod h1:Rd8YnCqz+2FYsiGmE2DMlaLjQRB4v2jFNnzCt9YY4IM=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.2 h1:F/MALZ518KfI1zEg+Kg8/uTzoXKDyqw+LNC/5irJlJE=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.2/go.mod h1:D0KY8md81DQKdaR/cXwnhoWB3MYYyc/UjvqE8GFkIvA=
 github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2 h1:yncs8NglhE3hB+viNsabCAF9TBBDOBljHUyxHC5fSGY=
 github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2/go.mod h1:AfbzyEUFxJmSoTiMcgNHHjDKcorBVd9TIwx0viURgEw=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.77.2 h1:rEBVlnqBSkNc6ufXnDu/GtSyS2jL1l6imtufzJUfOXI=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.77.2/go.mod h1:Q+R7L3DIlJ2a8R+W+7wjk0B25Ci/VTy7V07Zvq327x0=
 github.com/prometheus/alertmanager v0.27.0 h1:V6nTa2J5V4s8TG4C4HtrBP/WNSebCCTYGGv4qecA/+I=
 github.com/prometheus/alertmanager v0.27.0/go.mod h1:8Ia/R3urPmbzJ8OsdvmZvIprDwvwmYCmUbwBL+jlPOE=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -706,6 +712,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI=
 golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -730,6 +738,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
 golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -931,6 +941,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
 golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
+golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE=
+golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=