Skip to content

Commit 1fe348b

Browse files
swiatekmswiatekm
committed
Add a benchmark for the whole targets pipeline
1 parent b563a6e commit 1fe348b

File tree

2 files changed

+150
-13
lines changed

2 files changed

+150
-13
lines changed

‎cmd/otel-allocator/benchmark_test.go

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
// Copyright The OpenTelemetry Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package main
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"os"
21+
"testing"
22+
23+
gokitlog "github.com/go-kit/log"
24+
"github.com/prometheus/client_golang/prometheus"
25+
"github.com/prometheus/common/model"
26+
"github.com/prometheus/prometheus/discovery"
27+
"github.com/prometheus/prometheus/discovery/targetgroup"
28+
"github.com/stretchr/testify/require"
29+
ctrl "sigs.k8s.io/controller-runtime"
30+
31+
"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/allocation"
32+
"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/prehook"
33+
"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/server"
34+
"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/target"
35+
)
36+
37+
// BenchmarkProcessTargets benchmarks the whole target allocation pipeline. It starts with data the prometheus
38+
// discovery manager would normally output, and pushes it all the way into the allocator. It notably doe *not* check
39+
// the HTTP server afterward. Test data is chosen to be reasonably representative of what the Prometheus service discovery
40+
// outputs in the real world.
41+
func BenchmarkProcessTargets(b *testing.B) {
42+
numTargets := 10000
43+
targetsPerGroup := 5
44+
numGroups := numTargets / targetsPerGroup
45+
groupsPerJob := 20
46+
numJobs := numGroups / groupsPerJob
47+
jobNamePrefix := "test-"
48+
groupLabels := model.LabelSet{
49+
"__meta_kubernetes_pod_controller_name": "example",
50+
"__meta_kubernetes_pod_ip": "10.244.0.251",
51+
"__meta_kubernetes_pod_uid": "676ebee7-14f8-481e-a937-d2affaec4105",
52+
"__meta_kubernetes_endpointslice_port_protocol": "TCP",
53+
"__meta_kubernetes_endpointslice_endpoint_conditions_ready": "true",
54+
"__meta_kubernetes_service_annotation_kubectl_kubernetes_io_last_applied_configuration": "{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{},\"labels\":{\"app\":\"example\"},\"name\":\"example-svc\",\"namespace\":\"example\"},\"spec\":{\"clusterIP\":\"None\",\"ports\":[{\"name\":\"http-example\",\"port\":9006,\"targetPort\":9006}],\"selector\":{\"app\":\"example\"},\"type\":\"ClusterIP\"}}\n",
55+
"__meta_kubernetes_endpointslice_labelpresent_app": "true",
56+
"__meta_kubernetes_endpointslice_name": "example-svc-qgwxf",
57+
"__address__": "10.244.0.251:9006",
58+
"__meta_kubernetes_endpointslice_endpoint_conditions_terminating": "false",
59+
"__meta_kubernetes_pod_labelpresent_pod_template_hash": "true",
60+
"__meta_kubernetes_endpointslice_label_kubernetes_io_service_name": "example-svc",
61+
"__meta_kubernetes_endpointslice_labelpresent_service_kubernetes_io_headless": "true",
62+
"__meta_kubernetes_pod_label_pod_template_hash": "6b549885f8",
63+
"__meta_kubernetes_endpointslice_address_target_name": "example-6b549885f8-7tbcw",
64+
"__meta_kubernetes_pod_labelpresent_app": "true",
65+
"somelabel": "somevalue",
66+
}
67+
exampleTarget := model.LabelSet{
68+
"__meta_kubernetes_endpointslice_port": "9006",
69+
"__meta_kubernetes_service_label_app": "example",
70+
"__meta_kubernetes_endpointslice_port_name": "http-example",
71+
"__meta_kubernetes_pod_ready": "true",
72+
"__meta_kubernetes_endpointslice_address_type": "IPv4",
73+
"__meta_kubernetes_endpointslice_label_endpointslice_kubernetes_io_managed_by": "endpointslice-controller.k8s.io",
74+
"__meta_kubernetes_endpointslice_labelpresent_endpointslice_kubernetes_io_managed_by": "true",
75+
"__meta_kubernetes_endpointslice_label_app": "example",
76+
"__meta_kubernetes_endpointslice_endpoint_conditions_serving": "true",
77+
"__meta_kubernetes_pod_phase": "Running",
78+
"__meta_kubernetes_pod_controller_kind": "ReplicaSet",
79+
"__meta_kubernetes_service_annotationpresent_kubectl_kubernetes_io_last_applied_configuration": "true",
80+
"__meta_kubernetes_service_labelpresent_app": "true",
81+
"__meta_kubernetes_endpointslice_labelpresent_kubernetes_io_service_name": "true",
82+
"__meta_kubernetes_endpointslice_annotation_endpoints_kubernetes_io_last_change_trigger_time": "2023-09-27T16:01:29Z",
83+
"__meta_kubernetes_pod_name": "example-6b549885f8-7tbcw",
84+
"__meta_kubernetes_service_name": "example-svc",
85+
"__meta_kubernetes_namespace": "example",
86+
"__meta_kubernetes_endpointslice_annotationpresent_endpoints_kubernetes_io_last_change_trigger_time": "true",
87+
"__meta_kubernetes_pod_node_name": "kind-control-plane",
88+
"__meta_kubernetes_endpointslice_address_target_kind": "Pod",
89+
"__meta_kubernetes_pod_host_ip": "172.18.0.2",
90+
"__meta_kubernetes_endpointslice_label_service_kubernetes_io_headless": "",
91+
"__meta_kubernetes_pod_label_app": "example",
92+
}
93+
94+
ctx := context.Background()
95+
logger := ctrl.Log.WithName("test")
96+
allocatorPrehook := prehook.New("relabel-config", logger)
97+
allocator, err := allocation.New("consistent-hashing", logger, allocation.WithFilter(allocatorPrehook))
98+
srv := server.NewServer(logger, allocator, "0.0.0.0:8080")
99+
if err != nil {
100+
setupLog.Error(err, "Unable to initialize allocation strategy")
101+
os.Exit(1)
102+
}
103+
registry := prometheus.NewRegistry()
104+
sdMetrics, err := discovery.CreateAndRegisterSDMetrics(registry)
105+
require.NoError(b, err)
106+
discoveryManager := discovery.NewManager(ctx, gokitlog.NewNopLogger(), registry, sdMetrics)
107+
targetDiscoverer := target.NewDiscoverer(logger, discoveryManager, allocatorPrehook, srv)
108+
109+
targets := []model.LabelSet{}
110+
for i := 0; i < numTargets; i++ {
111+
targets = append(targets, exampleTarget.Clone())
112+
}
113+
groups := make([]*targetgroup.Group, numGroups)
114+
for i := 0; i < numGroups; i++ {
115+
groupTargets := targets[(i * targetsPerGroup):(i*targetsPerGroup + targetsPerGroup)]
116+
groups[i] = &targetgroup.Group{
117+
Labels: groupLabels,
118+
Targets: groupTargets,
119+
}
120+
}
121+
tsets := make(map[string][]*targetgroup.Group, numJobs)
122+
for i := 0; i < numJobs; i++ {
123+
jobGroups := groups[(i * groupsPerJob):(i*groupsPerJob + groupsPerJob)]
124+
jobName := fmt.Sprintf("%s%d", jobNamePrefix, i)
125+
tsets[jobName] = jobGroups
126+
}
127+
128+
b.ResetTimer()
129+
for i := 0; i < b.N; i++ {
130+
targetDiscoverer.ProcessTargets(tsets, allocator.SetTargets)
131+
}
132+
}

‎cmd/otel-allocator/target/discovery.go

+18-13
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/prometheus/common/model"
2525
promconfig "github.com/prometheus/prometheus/config"
2626
"github.com/prometheus/prometheus/discovery"
27+
"github.com/prometheus/prometheus/discovery/targetgroup"
2728
"github.com/prometheus/prometheus/model/relabel"
2829
"gopkg.in/yaml.v3"
2930

@@ -110,22 +111,26 @@ func (m *Discoverer) Watch(fn func(targets map[string]*Item)) error {
110111
m.log.Info("Service Discovery watch event stopped: discovery manager closed")
111112
return nil
112113
case tsets := <-m.manager.SyncCh():
113-
targets := map[string]*Item{}
114-
115-
for jobName, tgs := range tsets {
116-
var count float64 = 0
117-
for _, tg := range tgs {
118-
for _, t := range tg.Targets {
119-
count++
120-
item := NewItem(jobName, string(t[model.AddressLabel]), t.Merge(tg.Labels), "")
121-
targets[item.Hash()] = item
122-
}
123-
}
124-
targetsDiscovered.WithLabelValues(jobName).Set(count)
114+
m.ProcessTargets(tsets, fn)
115+
}
116+
}
117+
}
118+
119+
func (m *Discoverer) ProcessTargets(tsets map[string][]*targetgroup.Group, fn func(targets map[string]*Item)) {
120+
targets := map[string]*Item{}
121+
122+
for jobName, tgs := range tsets {
123+
var count float64 = 0
124+
for _, tg := range tgs {
125+
for _, t := range tg.Targets {
126+
count++
127+
item := NewItem(jobName, string(t[model.AddressLabel]), t.Merge(tg.Labels), "")
128+
targets[item.Hash()] = item
125129
}
126-
fn(targets)
127130
}
131+
targetsDiscovered.WithLabelValues(jobName).Set(count)
128132
}
133+
fn(targets)
129134
}
130135

131136
func (m *Discoverer) Close() {

0 commit comments

Comments
 (0)