From 8ec51bbede59e1fa6ac24de3607f6e7bb3e3a654 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Thu, 17 Sep 2020 14:01:20 -0400 Subject: metrics: Initial integration of Prometheus metrics (#3709) Signed-off-by: Dave Henderson --- modules/caddyhttp/app.go | 2 + modules/caddyhttp/metrics.go | 195 ++++++++++++++++++++++++++++++++++++++ modules/caddyhttp/metrics_test.go | 66 +++++++++++++ modules/caddyhttp/routes.go | 12 ++- modules/metrics/adminmetrics.go | 55 +++++++++++ modules/metrics/metrics.go | 109 +++++++++++++++++++++ modules/standard/imports.go | 1 + 7 files changed, 437 insertions(+), 3 deletions(-) create mode 100644 modules/caddyhttp/metrics.go create mode 100644 modules/caddyhttp/metrics_test.go create mode 100644 modules/metrics/adminmetrics.go create mode 100644 modules/metrics/metrics.go (limited to 'modules') diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go index 375ca4d..c8bbc95 100644 --- a/modules/caddyhttp/app.go +++ b/modules/caddyhttp/app.go @@ -226,6 +226,8 @@ func (app *App) Provision(ctx caddy.Context) error { // route handler so that important security checks are done, etc. primaryRoute := emptyHandler if srv.Routes != nil { + // inject the server name for observability purposes + ctx.Context = contextWithServerName(ctx.Context, srvName) err := srv.Routes.ProvisionHandlers(ctx) if err != nil { return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err) diff --git a/modules/caddyhttp/metrics.go b/modules/caddyhttp/metrics.go new file mode 100644 index 0000000..dbf1033 --- /dev/null +++ b/modules/caddyhttp/metrics.go @@ -0,0 +1,195 @@ +package caddyhttp + +import ( + "context" + "net/http" + "strconv" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var httpMetrics = struct { + init sync.Once + requestInFlight *prometheus.GaugeVec + requestCount *prometheus.CounterVec + requestErrors *prometheus.CounterVec + requestDuration *prometheus.HistogramVec + requestSize *prometheus.HistogramVec + responseSize *prometheus.HistogramVec + responseDuration *prometheus.HistogramVec +}{ + init: sync.Once{}, +} + +func initHTTPMetrics() { + const ns, sub = "caddy", "http" + + basicLabels := []string{"server", "handler"} + httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ns, + Subsystem: sub, + Name: "requests_in_flight", + Help: "Number of requests currently handled by this server.", + }, basicLabels) + httpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: ns, + Subsystem: sub, + Name: "request_errors_total", + Help: "Number of requests resulting in middleware errors.", + }, basicLabels) + httpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: ns, + Subsystem: sub, + Name: "requests_total", + Help: "Counter of HTTP(S) requests made.", + }, basicLabels) + + // TODO: allow these to be customized in the config + durationBuckets := prometheus.DefBuckets + sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8) + + httpLabels := []string{"server", "handler", "code", "method"} + httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: ns, + Subsystem: sub, + Name: "request_duration_seconds", + Help: "Histogram of round-trip request durations.", + Buckets: durationBuckets, + }, httpLabels) + httpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: ns, + Subsystem: sub, + Name: "request_size_bytes", + Help: "Total size of the request. Includes body", + Buckets: sizeBuckets, + }, httpLabels) + httpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: ns, + Subsystem: sub, + Name: "response_size_bytes", + Help: "Size of the returned response.", + Buckets: sizeBuckets, + }, httpLabels) + httpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: ns, + Subsystem: sub, + Name: "response_duration_seconds", + Help: "Histogram of times to first byte in response bodies.", + Buckets: durationBuckets, + }, httpLabels) +} + +type ctxKeyServerName struct{} + +// serverNameFromContext extracts the current server name from the context. +// Returns "UNKNOWN" if none is available (should probably never happen?) +func serverNameFromContext(ctx context.Context) string { + srvName, ok := ctx.Value(ctxKeyServerName{}).(string) + if !ok { + return "UNKNOWN" + } + return srvName +} + +func contextWithServerName(ctx context.Context, serverName string) context.Context { + return context.WithValue(ctx, ctxKeyServerName{}, serverName) +} + +type metricsInstrumentedHandler struct { + labels prometheus.Labels + statusLabels prometheus.Labels + mh MiddlewareHandler +} + +func newMetricsInstrumentedHandler(server, handler string, mh MiddlewareHandler) *metricsInstrumentedHandler { + httpMetrics.init.Do(func() { + initHTTPMetrics() + }) + + labels := prometheus.Labels{"server": server, "handler": handler} + statusLabels := prometheus.Labels{"server": server, "handler": handler, "code": "", "method": ""} + return &metricsInstrumentedHandler{labels, statusLabels, mh} +} + +func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error { + inFlight := httpMetrics.requestInFlight.With(h.labels) + inFlight.Inc() + defer inFlight.Dec() + + statusLabels := prometheus.Labels{"method": r.Method} + for k, v := range h.labels { + statusLabels[k] = v + } + + start := time.Now() + + // This is a _bit_ of a hack - it depends on the ShouldBufferFunc always + // being called when the headers are written. + // Effectively the same behaviour as promhttp.InstrumentHandlerTimeToWriteHeader. + writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool { + statusLabels["code"] = sanitizeCode(status) + ttfb := time.Since(start).Seconds() + observeWithExemplar(statusLabels, httpMetrics.responseDuration, ttfb) + return false + }) + wrec := NewResponseRecorder(w, nil, writeHeaderRecorder) + err := h.mh.ServeHTTP(wrec, r, next) + dur := time.Since(start).Seconds() + httpMetrics.requestCount.With(h.labels).Inc() + if err != nil { + httpMetrics.requestErrors.With(h.labels).Inc() + return err + } + + observeWithExemplar(statusLabels, httpMetrics.requestDuration, dur) + observeWithExemplar(statusLabels, httpMetrics.requestSize, float64(computeApproximateRequestSize(r))) + httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size())) + + return nil +} + +func observeWithExemplar(l prometheus.Labels, o *prometheus.HistogramVec, value float64) { + obs := o.With(l) + if oe, ok := obs.(prometheus.ExemplarObserver); ok { + oe.ObserveWithExemplar(value, l) + return + } + // _should_ be a noop, but here just in case... + obs.Observe(value) +} + +func sanitizeCode(code int) string { + if code == 0 { + return "200" + } + return strconv.Itoa(code) + +} + +// taken from https://github.com/prometheus/client_golang/blob/6007b2b5cae01203111de55f753e76d8dac1f529/prometheus/promhttp/instrument_server.go#L298 +func computeApproximateRequestSize(r *http.Request) int { + s := 0 + if r.URL != nil { + s += len(r.URL.String()) + } + + s += len(r.Method) + s += len(r.Proto) + for name, values := range r.Header { + s += len(name) + for _, value := range values { + s += len(value) + } + } + s += len(r.Host) + + // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. + + if r.ContentLength != -1 { + s += int(r.ContentLength) + } + return s +} diff --git a/modules/caddyhttp/metrics_test.go b/modules/caddyhttp/metrics_test.go new file mode 100644 index 0000000..5c3bc1d --- /dev/null +++ b/modules/caddyhttp/metrics_test.go @@ -0,0 +1,66 @@ +package caddyhttp + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestServerNameFromContext(t *testing.T) { + ctx := context.Background() + expected := "UNKNOWN" + if actual := serverNameFromContext(ctx); actual != expected { + t.Errorf("Not equal: expected %q, but got %q", expected, actual) + } + + in := "foo" + ctx = contextWithServerName(ctx, in) + if actual := serverNameFromContext(ctx); actual != in { + t.Errorf("Not equal: expected %q, but got %q", in, actual) + } +} + +func TestMetricsInstrumentedHandler(t *testing.T) { + handlerErr := errors.New("oh noes") + response := []byte("hello world!") + h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { + if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 { + t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual) + } + if handlerErr == nil { + w.Write(response) + } + return handlerErr + }) + + mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error { + return h.ServeHTTP(w, r) + }) + + ih := newMetricsInstrumentedHandler("foo", "bar", mh) + + r := httptest.NewRequest("GET", "/", nil) + w := httptest.NewRecorder() + + if actual := ih.ServeHTTP(w, r, h); actual != handlerErr { + t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual) + } + if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 { + t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual) + } + + handlerErr = nil + if err := ih.ServeHTTP(w, r, h); err != nil { + t.Errorf("Received unexpected error: %w", err) + } +} + +type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error + +func (f middlewareHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, h Handler) error { + return f(w, r, h) +} diff --git a/modules/caddyhttp/routes.go b/modules/caddyhttp/routes.go index 10e0c9d..be23d39 100644 --- a/modules/caddyhttp/routes.go +++ b/modules/caddyhttp/routes.go @@ -157,7 +157,7 @@ func (routes RouteList) ProvisionHandlers(ctx caddy.Context) error { // pre-compile the middleware handler chain for _, midhandler := range routes[i].Handlers { - routes[i].middleware = append(routes[i].middleware, wrapMiddleware(midhandler)) + routes[i].middleware = append(routes[i].middleware, wrapMiddleware(ctx, midhandler)) } } return nil @@ -242,7 +242,13 @@ func wrapRoute(route Route) Middleware { // we need to pull this particular MiddlewareHandler // pointer into its own stack frame to preserve it so it // won't be overwritten in future loop iterations. -func wrapMiddleware(mh MiddlewareHandler) Middleware { +func wrapMiddleware(ctx caddy.Context, mh MiddlewareHandler) Middleware { + // first, wrap the middleware with metrics instrumentation + metricsHandler := newMetricsInstrumentedHandler( + serverNameFromContext(ctx.Context), + caddy.GetModuleName(mh), + mh, + ) return func(next Handler) Handler { // copy the next handler (it's an interface, so it's // just a very lightweight copy of a pointer); this @@ -253,7 +259,7 @@ func wrapMiddleware(mh MiddlewareHandler) Middleware { return HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { // TODO: This is where request tracing could be implemented // TODO: see what the std lib gives us in terms of stack tracing too - return mh.ServeHTTP(w, r, nextCopy) + return metricsHandler.ServeHTTP(w, r, nextCopy) }) } } diff --git a/modules/metrics/adminmetrics.go b/modules/metrics/adminmetrics.go new file mode 100644 index 0000000..f6a55e8 --- /dev/null +++ b/modules/metrics/adminmetrics.go @@ -0,0 +1,55 @@ +// Copyright 2020 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "net/http" + + "github.com/caddyserver/caddy/v2" +) + +func init() { + caddy.RegisterModule(AdminMetrics{}) +} + +// AdminMetrics is a module that serves a metrics endpoint so that any gathered +// metrics can be exposed for scraping. This module is not configurable, and +// is permanently mounted to the admin API endpoint at "/metrics". +// See the Metrics module for a configurable endpoint that is usable if the +// Admin API is disabled. +type AdminMetrics struct{} + +// CaddyModule returns the Caddy module information. +func (AdminMetrics) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "admin.api.metrics", + New: func() caddy.Module { return new(AdminMetrics) }, + } +} + +// Routes returns a route for the /metrics endpoint. +func (m *AdminMetrics) Routes() []caddy.AdminRoute { + metricsHandler := createMetricsHandler(nil) + h := caddy.AdminHandlerFunc(func(w http.ResponseWriter, r *http.Request) error { + metricsHandler.ServeHTTP(w, r) + return nil + }) + return []caddy.AdminRoute{{Pattern: "/metrics", Handler: h}} +} + +// Interface guards +var ( + _ caddy.AdminRouter = (*AdminMetrics)(nil) +) diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go new file mode 100644 index 0000000..eab6342 --- /dev/null +++ b/modules/metrics/metrics.go @@ -0,0 +1,109 @@ +// Copyright 2020 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "net/http" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" + "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + + "go.uber.org/zap" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func init() { + caddy.RegisterModule(Metrics{}) + httpcaddyfile.RegisterHandlerDirective("metrics", parseCaddyfile) +} + +// Metrics is a module that serves a /metrics endpoint so that any gathered +// metrics can be exposed for scraping. This module is configurable by end-users +// unlike AdminMetrics. +type Metrics struct { + metricsHandler http.Handler +} + +// CaddyModule returns the Caddy module information. +func (Metrics) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.handlers.metrics", + New: func() caddy.Module { return new(Metrics) }, + } +} + +type zapLogger struct { + zl *zap.Logger +} + +func (l *zapLogger) Println(v ...interface{}) { + l.zl.Sugar().Error(v...) +} + +// Provision sets up m. +func (m *Metrics) Provision(ctx caddy.Context) error { + log := ctx.Logger(m) + m.metricsHandler = createMetricsHandler(&zapLogger{log}) + return nil +} + +func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { + var m Metrics + err := m.UnmarshalCaddyfile(h.Dispenser) + return m, err +} + +// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax: +// +// metrics +// +func (m *Metrics) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + for d.Next() { + args := d.RemainingArgs() + if len(args) > 0 { + return d.ArgErr() + } + } + return nil +} + +func (m Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + m.metricsHandler.ServeHTTP(w, r) + return next.ServeHTTP(w, r) +} + +// Interface guards +var ( + _ caddy.Provisioner = (*Metrics)(nil) + _ caddyhttp.MiddlewareHandler = (*Metrics)(nil) + _ caddyfile.Unmarshaler = (*Metrics)(nil) +) + +func createMetricsHandler(logger promhttp.Logger) http.Handler { + return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, + promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + // will only log errors if logger is non-nil + ErrorLog: logger, + + // Allow OpenMetrics format to be negotiated - largely compatible, + // except quantile/le label values always have a decimal. + EnableOpenMetrics: true, + }), + ) +} diff --git a/modules/standard/imports.go b/modules/standard/imports.go index b8ea7a8..bc2d955 100644 --- a/modules/standard/imports.go +++ b/modules/standard/imports.go @@ -11,4 +11,5 @@ import ( _ "github.com/caddyserver/caddy/v2/modules/caddytls/standardstek" _ "github.com/caddyserver/caddy/v2/modules/filestorage" _ "github.com/caddyserver/caddy/v2/modules/logging" + _ "github.com/caddyserver/caddy/v2/modules/metrics" ) -- cgit v1.2.3