From 3c775254896f0ee7b18d6b4da1ad1cc4ae35fc53 Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Fri, 13 Dec 2024 12:37:09 -0800 Subject: [PATCH] impr: BKTCLT-34 Go client: add Prometheus metrics Add metrics when requests are processed, in order for them to be exposed by the API user and scraped by Prometheus. --- go/bucketclient.go | 1 + go/bucketclientmetrics.go | 87 ++++++++++++++++++++++++++++++++++ go/bucketclientmetrics_test.go | 75 +++++++++++++++++++++++++++++ go/bucketclientrequest.go | 58 +++++++++++++++++++---- go/constants.go | 6 +++ go/go.mod | 10 ++++ go/go.sum | 36 ++++++++++++-- 7 files changed, 259 insertions(+), 14 deletions(-) create mode 100644 go/bucketclientmetrics.go create mode 100644 go/bucketclientmetrics_test.go diff --git a/go/bucketclient.go b/go/bucketclient.go index 5465aff..a168c83 100644 --- a/go/bucketclient.go +++ b/go/bucketclient.go @@ -7,6 +7,7 @@ import ( type BucketClient struct { Endpoint string HTTPClient *http.Client + Metrics *BucketClientMetrics } // New creates a new BucketClient instance, with the provided endpoint (e.g. "localhost:9000") diff --git a/go/bucketclientmetrics.go b/go/bucketclientmetrics.go new file mode 100644 index 0000000..cc9d106 --- /dev/null +++ b/go/bucketclientmetrics.go @@ -0,0 +1,87 @@ +package bucketclient + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +type BucketClientMetrics struct { + RequestsTotal *prometheus.CounterVec + RequestDurationSeconds *prometheus.SummaryVec + RequestBytesSentTotal *prometheus.CounterVec + ResponseBytesReceivedTotal *prometheus.CounterVec +} + +var metricsLabels = []string{ + "endpoint", + "method", + "action", + "code", +} + +var globalMetrics *BucketClientMetrics +var globalMetricsLock sync.Mutex + +// EnableMetrics enables Prometheus metrics gathering for the provided client and registers +// them in the provided registerer. +// +// Metrics implemented: +// - `s3_metadata_bucketclient_requests_total`: +// Number of requests processed (counter) +// - `s3_metadata_bucketclient_request_duration_seconds`: +// Time elapsed processing requests to bucketd, in seconds (summary) +// - `s3_metadata_bucketclient_request_bytes_sent_total`: +// Number of request body bytes sent to bucketd (counter) +// - `s3_metadata_bucketclient_response_bytes_received_total`: +// Number of response body bytes received from bucketd (counter) +// +// Metrics have the following labels attached: +// - `endpoint`: +// bucketd endpoint such as `http://localhost:9000` +// - `method`: +// HTTP method +// - `action`: +// name of the API action, such as `CreateBucket`. Admin actions are prefixed with `Admin`. +// - `code`: +// HTTP status code returned, or "0" for generic network or protocol errors +func (client *BucketClient) EnableMetrics(registerer prometheus.Registerer) { + globalMetricsLock.Lock() + defer globalMetricsLock.Unlock() + + if globalMetrics == nil { + globalMetrics = &BucketClientMetrics{ + RequestsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: "requests_total", + Help: "Number of requests processed", + }, metricsLabels), + + RequestDurationSeconds: prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: MetricsNamespace, + Name: "request_duration_seconds", + Help: "Time elapsed processing requests to bucketd, in seconds", + Objectives: MetricsSummaryDefaultObjectives, + }, metricsLabels), + + RequestBytesSentTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: "request_bytes_sent_total", + Help: "Number of request body bytes sent to bucketd", + }, metricsLabels), + + ResponseBytesReceivedTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: "response_bytes_received_total", + Help: "Number of response body bytes received from bucketd", + }, metricsLabels), + } + registerer.MustRegister( + globalMetrics.RequestsTotal, + globalMetrics.RequestDurationSeconds, + globalMetrics.RequestBytesSentTotal, + globalMetrics.ResponseBytesReceivedTotal, + ) + } + client.Metrics = globalMetrics +} diff --git a/go/bucketclientmetrics_test.go b/go/bucketclientmetrics_test.go new file mode 100644 index 0000000..dabb8c8 --- /dev/null +++ b/go/bucketclientmetrics_test.go @@ -0,0 +1,75 @@ +package bucketclient_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "io" + "net/http" + "strings" + + "github.com/jarcoal/httpmock" + "github.com/prometheus/client_golang/prometheus" + promTestutil "github.com/prometheus/client_golang/prometheus/testutil" + + "github.com/scality/bucketclient/go" +) + +var _ = Describe("BucketClientMetrics", func() { + Describe("RegisterMetrics()", func() { + It("enables metrics collection for all requests of all clients with metrics enabled", func(ctx SpecContext) { + client1 := bucketclient.New("http://localhost:9000") + Expect(client1).ToNot(BeNil()) + client2 := bucketclient.New("http://localhost:9001") + Expect(client2).ToNot(BeNil()) + + registry := prometheus.NewPedanticRegistry() + client1.EnableMetrics(registry) + client2.EnableMetrics(registry) + + mockAttributes := `{"foo":"bar"}` + httpmock.RegisterResponder( + "GET", "http://localhost:9000/default/attributes/my-bucket", + httpmock.NewStringResponder(200, mockAttributes), + ) + expectedBatch := `{"batch":[{"key":"foo","value":"{}"}]}` + batchErrorResponse := "OOPS!" + httpmock.RegisterResponder( + "POST", "http://localhost:9001/default/batch/my-bucket", + func(req *http.Request) (*http.Response, error) { + defer req.Body.Close() + Expect(io.ReadAll(req.Body)).To(Equal([]byte(expectedBatch))) + return httpmock.NewStringResponse(500, batchErrorResponse), nil + }, + ) + + _, err := client1.GetBucketAttributes(ctx, "my-bucket") + Expect(err).ToNot(HaveOccurred()) + + err = client2.PostBatch(ctx, "my-bucket", []bucketclient.PostBatchEntry{ + {Key: "foo", Value: "{}"}, + }) + Expect(err).To(HaveOccurred()) + + Expect(promTestutil.GatherAndCompare(registry, strings.NewReader(` +# HELP s3_metadata_bucketclient_requests_total Number of requests processed +# TYPE s3_metadata_bucketclient_requests_total counter +s3_metadata_bucketclient_requests_total{action="GetBucketAttributes",code="200",endpoint="http://localhost:9000",method="GET"} 1 +s3_metadata_bucketclient_requests_total{action="PostBatch",code="500",endpoint="http://localhost:9001",method="POST"} 1 + +# HELP s3_metadata_bucketclient_request_bytes_sent_total Number of request body bytes sent to bucketd +# TYPE s3_metadata_bucketclient_request_bytes_sent_total counter +s3_metadata_bucketclient_request_bytes_sent_total{action="PostBatch",code="500",endpoint="http://localhost:9001",method="POST"} 38 + +# HELP s3_metadata_bucketclient_response_bytes_received_total Number of response body bytes received from bucketd +# TYPE s3_metadata_bucketclient_response_bytes_received_total counter +s3_metadata_bucketclient_response_bytes_received_total{action="GetBucketAttributes",code="200",endpoint="http://localhost:9000",method="GET"} 13 +s3_metadata_bucketclient_response_bytes_received_total{action="PostBatch",code="500",endpoint="http://localhost:9001",method="POST"} 5 +`), + "s3_metadata_bucketclient_requests_total", + "s3_metadata_bucketclient_request_bytes_sent_total", + "s3_metadata_bucketclient_response_bytes_received_total", + )).To(Succeed()) + }) + }) +}) diff --git a/go/bucketclientrequest.go b/go/bucketclientrequest.go index b6770e8..8bc9e83 100644 --- a/go/bucketclientrequest.go +++ b/go/bucketclientrequest.go @@ -6,7 +6,11 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" ) type requestOptionSet struct { @@ -52,11 +56,43 @@ func parseRequestOptions(opts ...RequestOption) (requestOptionSet, error) { return parsedOpts, nil } +// updateMetrics is a local helper to update Prometheus metrics in a generic way before +// returning a response or an error to the API caller. +func (client *BucketClient) updateMetrics(apiMethod, httpMethod string, + startTime time.Time, requestBody []byte, + httpCode int, responseBody []byte) { + if client.Metrics == nil { + return + } + labels := prometheus.Labels{ + "endpoint": client.Endpoint, + "method": httpMethod, + "action": apiMethod, + "code": strconv.Itoa(httpCode), + } + elapsedSeconds := time.Since(startTime).Seconds() + + client.Metrics.RequestsTotal.With(labels).Inc() + client.Metrics.RequestDurationSeconds.With(labels).Observe(elapsedSeconds) + + // only update this metric when the request body exists, to avoid unneeded metrics + if requestBody != nil { + client.Metrics.RequestBytesSentTotal.With(labels).Add(float64(len(requestBody))) + } + + var responseBodyLength int + if responseBody != nil { + responseBodyLength = len(responseBody) + } + client.Metrics.ResponseBytesReceivedTotal.With(labels).Add(float64(responseBodyLength)) +} + func (client *BucketClient) Request(ctx context.Context, apiMethod string, httpMethod string, resource string, opts ...RequestOption) ([]byte, error) { var response *http.Response var err error + startTime := time.Now() options, err := parseRequestOptions(opts...) if err == nil { url := fmt.Sprintf("%s%s", client.Endpoint, resource) @@ -83,13 +119,24 @@ func (client *BucketClient) Request(ctx context.Context, } } if err != nil { + client.updateMetrics(apiMethod, httpMethod, startTime, options.requestBody, 0, nil) return nil, &BucketClientError{ apiMethod, httpMethod, client.Endpoint, resource, 0, "", err, } } - if response.Body != nil { - defer response.Body.Close() + defer response.Body.Close() + responseBody, err := io.ReadAll(response.Body) + if err != nil { + // We have a HTTP status code but we couldn't read the whole response body, + // so use "0" as status code for a generic transport error + client.updateMetrics(apiMethod, httpMethod, startTime, options.requestBody, 0, nil) + return nil, &BucketClientError{ + apiMethod, httpMethod, client.Endpoint, resource, 0, "", + fmt.Errorf("error reading response body: %w", err), + } } + client.updateMetrics(apiMethod, httpMethod, startTime, options.requestBody, + response.StatusCode, responseBody) if response.StatusCode/100 != 2 { splitStatus := strings.Split(response.Status, " ") @@ -102,12 +149,5 @@ func (client *BucketClient) Request(ctx context.Context, response.StatusCode, errorType, nil, } } - responseBody, err := io.ReadAll(response.Body) - if err != nil { - return nil, &BucketClientError{ - apiMethod, httpMethod, client.Endpoint, resource, 0, "", - fmt.Errorf("error reading response body: %w", err), - } - } return responseBody, nil } diff --git a/go/constants.go b/go/constants.go index d7a1959..700bf85 100644 --- a/go/constants.go +++ b/go/constants.go @@ -14,3 +14,9 @@ const ( DBMethodBatch DBMethodType = 8 DBMethodNoop DBMethodType = 9 ) + +const ( + MetricsNamespace = "s3_metadata_bucketclient" +) + +var MetricsSummaryDefaultObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001, 1.0: 0.0001} diff --git a/go/go.mod b/go/go.mod index 6670b09..b0f2fe0 100644 --- a/go/go.mod +++ b/go/go.mod @@ -6,16 +6,26 @@ require ( github.com/jarcoal/httpmock v1.3.1 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 + github.com/prometheus/client_golang v1.20.5 ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/sys v0.24.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.24.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go/go.sum b/go/go.sum index f3af108..d0dc4c7 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,3 +1,8 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -10,16 +15,36 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSF github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= @@ -28,9 +53,10 @@ golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=