Skip to content

Commit faa3aa8

Browse files
neilcookneilcook
authored
Merge pull request #110 from PowerDNS/metrics
Additional metrics and some tests for metrics
2 parents 57dad9f + 5df19dd commit faa3aa8

File tree

5 files changed

+107
-1
lines changed

5 files changed

+107
-1
lines changed

‎backends/s3/metrics.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,27 @@ var (
2626
},
2727
[]string{"method"},
2828
)
29+
metricCallErrorsType = prometheus.NewCounterVec(
30+
prometheus.CounterOpts{
31+
Name: "storage_s3_call_error_by_type_total",
32+
Help: "S3 API call errors by method and error type",
33+
},
34+
[]string{"method", "error"},
35+
)
36+
metricCallHistogram = prometheus.NewHistogramVec(
37+
prometheus.HistogramOpts{
38+
Name: "storage_s3_call_duration_seconds",
39+
Help: "S3 API call duration by method",
40+
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60},
41+
},
42+
[]string{"method"},
43+
)
2944
)
3045

3146
func init() {
3247
prometheus.MustRegister(metricLastCallTimestamp)
3348
prometheus.MustRegister(metricCalls)
3449
prometheus.MustRegister(metricCallErrors)
50+
prometheus.MustRegister(metricCallErrorsType)
51+
prometheus.MustRegister(metricCallHistogram)
3552
}

‎backends/s3/s3.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,9 +236,16 @@ func (b *Backend) List(ctx context.Context, prefix string) (blobList simpleblob.
236236
return blobs.WithPrefix(prefix), nil
237237
}
238238

239+
func recordMinioDurationMetric(method string, start time.Time) {
240+
elapsed := time.Since(start)
241+
metricCallHistogram.WithLabelValues(method).Observe(elapsed.Seconds())
242+
}
243+
239244
func (b *Backend) doList(ctx context.Context, prefix string) (simpleblob.BlobList, error) {
240245
var blobs simpleblob.BlobList
241246

247+
defer recordMinioDurationMetric("list", time.Now())
248+
242249
// Runes to strip from blob names for GlobalPrefix
243250
// This is fine, because we can trust the API to only return with the prefix.
244251
// TODO: trust but verify
@@ -252,6 +259,7 @@ func (b *Backend) doList(ctx context.Context, prefix string) (simpleblob.BlobLis
252259
// Handle error returned by MinIO client
253260
if err := convertMinioError(obj.Err, true); err != nil {
254261
metricCallErrors.WithLabelValues("list").Inc()
262+
metricCallErrorsType.WithLabelValues("list", errorToMetricsLabel(err)).Inc()
255263
return nil, err
256264
}
257265

@@ -303,9 +311,12 @@ func (b *Backend) doLoadReader(ctx context.Context, name string) (io.ReadCloser,
303311
metricCalls.WithLabelValues("load").Inc()
304312
metricLastCallTimestamp.WithLabelValues("load").SetToCurrentTime()
305313

314+
defer recordMinioDurationMetric("load", time.Now())
315+
306316
obj, err := b.client.GetObject(ctx, b.opt.Bucket, name, minio.GetObjectOptions{})
307317
if err = convertMinioError(err, false); err != nil {
308318
metricCallErrors.WithLabelValues("load").Inc()
319+
metricCallErrorsType.WithLabelValues("load", errorToMetricsLabel(err)).Inc()
309320
return nil, err
310321
}
311322
if obj == nil {
@@ -314,6 +325,7 @@ func (b *Backend) doLoadReader(ctx context.Context, name string) (io.ReadCloser,
314325
info, err := obj.Stat()
315326
if err = convertMinioError(err, false); err != nil {
316327
metricCallErrors.WithLabelValues("load").Inc()
328+
metricCallErrorsType.WithLabelValues("load", errorToMetricsLabel(err)).Inc()
317329
return nil, err
318330
}
319331
if info.Key == "" {
@@ -347,6 +359,7 @@ func (b *Backend) doStore(ctx context.Context, name string, data []byte) (minio.
347359
func (b *Backend) doStoreReader(ctx context.Context, name string, r io.Reader, size int64) (minio.UploadInfo, error) {
348360
metricCalls.WithLabelValues("store").Inc()
349361
metricLastCallTimestamp.WithLabelValues("store").SetToCurrentTime()
362+
defer recordMinioDurationMetric("store", time.Now())
350363

351364
putObjectOptions := minio.PutObjectOptions{
352365
NumThreads: b.opt.NumMinioThreads,
@@ -358,6 +371,7 @@ func (b *Backend) doStoreReader(ctx context.Context, name string, r io.Reader, s
358371
err = convertMinioError(err, false)
359372
if err != nil {
360373
metricCallErrors.WithLabelValues("store").Inc()
374+
metricCallErrorsType.WithLabelValues("store", errorToMetricsLabel(err)).Inc()
361375
}
362376
return info, err
363377
}
@@ -377,10 +391,12 @@ func (b *Backend) Delete(ctx context.Context, name string) error {
377391
func (b *Backend) doDelete(ctx context.Context, name string) error {
378392
metricCalls.WithLabelValues("delete").Inc()
379393
metricLastCallTimestamp.WithLabelValues("delete").SetToCurrentTime()
394+
defer recordMinioDurationMetric("delete", time.Now())
380395

381396
err := b.client.RemoveObject(ctx, b.opt.Bucket, name, minio.RemoveObjectOptions{})
382397
if err = convertMinioError(err, false); err != nil {
383398
metricCallErrors.WithLabelValues("delete").Inc()
399+
metricCallErrorsType.WithLabelValues("delete", errorToMetricsLabel(err)).Inc()
384400
}
385401
return err
386402
}
@@ -555,6 +571,40 @@ func convertMinioError(err error, isList bool) error {
555571
return err
556572
}
557573

574+
// errorToMetricsLabel converts an error into a prometheus label.
575+
// If error is a NotExist error, "NotFound" is returned.
576+
// If error is a timeout, "Timeout" is returned.
577+
// If error is a DNS error, the DNS error is returned.
578+
// If error is a URL error, the URL error is returned.
579+
// If error is a MinIO error, the MinIO error code is returned.
580+
// Otherwise "Unknown" is returned.
581+
func errorToMetricsLabel(err error) string {
582+
if err == nil {
583+
return "ok"
584+
}
585+
if errors.Is(err, os.ErrNotExist) {
586+
return "NotFound"
587+
}
588+
var netError *net.OpError
589+
if errors.Is(err, context.DeadlineExceeded) ||
590+
(errors.As(err, &netError) && netError.Timeout()) {
591+
return "Timeout"
592+
}
593+
var dnsErr *net.DNSError
594+
if errors.As(err, &dnsErr) {
595+
return "DNSError"
596+
}
597+
var urlErr *url.Error
598+
if errors.As(err, &urlErr) {
599+
return "URLError"
600+
}
601+
errRes := minio.ToErrorResponse(err)
602+
if errRes.Code != "" {
603+
return errRes.Code
604+
}
605+
return "Unknown"
606+
}
607+
558608
func getOpt[T comparable](optVal, defaultVal T) T {
559609
var zero T
560610
if optVal == zero {

‎backends/s3/s3_test.go‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ package s3
22

33
import (
44
"context"
5+
"strings"
56
"testing"
67
"time"
78

89
"github.com/minio/minio-go/v7"
10+
"github.com/prometheus/client_golang/prometheus/testutil"
911
"github.com/stretchr/testify/assert"
1012
"github.com/stretchr/testify/require"
1113
"github.com/testcontainers/testcontainers-go"
@@ -64,15 +66,49 @@ func getBackend(ctx context.Context, t *testing.T) (b *Backend) {
6466
return b
6567
}
6668

69+
func getBadBackend(ctx context.Context, url string, t *testing.T) (b *Backend) {
70+
b, err := New(ctx, Options{
71+
EndpointURL: url,
72+
AccessKey: "foo",
73+
SecretKey: "bar",
74+
Bucket: "test-bucket",
75+
CreateBucket: false,
76+
DialTimeout: 1 * time.Second,
77+
})
78+
require.NoError(t, err)
79+
return b
80+
}
81+
6782
func TestBackend(t *testing.T) {
6883
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
6984
defer cancel()
70-
7185
b := getBackend(ctx, t)
7286
tester.DoBackendTests(t, b)
7387
assert.Len(t, b.lastMarker, 0)
7488
}
7589

90+
func TestMetrics(t *testing.T) {
91+
bTimeout := getBadBackend(context.Background(), "http://1.2.3.4:1234", t)
92+
93+
_, err := bTimeout.List(context.Background(), "")
94+
assert.Error(t, err)
95+
96+
expectedMetric := "# HELP storage_s3_call_error_by_type_total S3 API call errors by method and error type\n# TYPE storage_s3_call_error_by_type_total counter\nstorage_s3_call_error_by_type_total{error=\"Timeout\",method=\"list\"} 1\nstorage_s3_call_error_by_type_total{error=\"NotFound\",method=\"load\"} 3\n"
97+
98+
err = testutil.CollectAndCompare(metricCallErrorsType, strings.NewReader(expectedMetric), "storage_s3_call_error_by_type_total")
99+
assert.NoError(t, err)
100+
101+
bBadHost := getBadBackend(context.Background(), "http://nosuchhost:1234", t)
102+
103+
_, err = bBadHost.List(context.Background(), "")
104+
assert.Error(t, err)
105+
106+
expectedMetric += "storage_s3_call_error_by_type_total{error=\"DNSError\",method=\"list\"} 1\n"
107+
108+
err = testutil.CollectAndCompare(metricCallErrorsType, strings.NewReader(expectedMetric), "storage_s3_call_error_by_type_total")
109+
assert.NoError(t, err)
110+
}
111+
76112
func TestBackend_marker(t *testing.T) {
77113
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
78114
defer cancel()

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ require (
4444
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
4545
github.com/klauspost/compress v1.18.0 // indirect
4646
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
47+
github.com/kylelemons/godebug v1.1.0 // indirect
4748
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
4849
github.com/magiconair/properties v1.8.10 // indirect
4950
github.com/minio/crc64nvme v1.0.2 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
7474
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
7575
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
7676
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
77+
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
78+
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
7779
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
7880
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
7981
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=

0 commit comments

Comments
 (0)