summaryrefslogtreecommitdiff
path: root/modules
diff options
context:
space:
mode:
authorDave Henderson <dhenderson@gmail.com>2020-09-17 14:01:20 -0400
committerGitHub <noreply@github.com>2020-09-17 12:01:20 -0600
commit8ec51bbede59e1fa6ac24de3607f6e7bb3e3a654 (patch)
treef598667a352f4606979965e200eedecc9aa4e4e8 /modules
parentbc453fa6ae36287c90d2bf6941cb686490090df2 (diff)
metrics: Initial integration of Prometheus metrics (#3709)
Signed-off-by: Dave Henderson <dhenderson@gmail.com>
Diffstat (limited to 'modules')
-rw-r--r--modules/caddyhttp/app.go2
-rw-r--r--modules/caddyhttp/metrics.go195
-rw-r--r--modules/caddyhttp/metrics_test.go66
-rw-r--r--modules/caddyhttp/routes.go12
-rw-r--r--modules/metrics/adminmetrics.go55
-rw-r--r--modules/metrics/metrics.go109
-rw-r--r--modules/standard/imports.go1
7 files changed, 437 insertions, 3 deletions
diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go
index 375ca4d..c8bbc95 100644
--- a/modules/caddyhttp/app.go
+++ b/modules/caddyhttp/app.go
@@ -226,6 +226,8 @@ func (app *App) Provision(ctx caddy.Context) error {
// route handler so that important security checks are done, etc.
primaryRoute := emptyHandler
if srv.Routes != nil {
+ // inject the server name for observability purposes
+ ctx.Context = contextWithServerName(ctx.Context, srvName)
err := srv.Routes.ProvisionHandlers(ctx)
if err != nil {
return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
diff --git a/modules/caddyhttp/metrics.go b/modules/caddyhttp/metrics.go
new file mode 100644
index 0000000..dbf1033
--- /dev/null
+++ b/modules/caddyhttp/metrics.go
@@ -0,0 +1,195 @@
+package caddyhttp
+
+import (
+ "context"
+ "net/http"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var httpMetrics = struct {
+ init sync.Once
+ requestInFlight *prometheus.GaugeVec
+ requestCount *prometheus.CounterVec
+ requestErrors *prometheus.CounterVec
+ requestDuration *prometheus.HistogramVec
+ requestSize *prometheus.HistogramVec
+ responseSize *prometheus.HistogramVec
+ responseDuration *prometheus.HistogramVec
+}{
+ init: sync.Once{},
+}
+
+func initHTTPMetrics() {
+ const ns, sub = "caddy", "http"
+
+ basicLabels := []string{"server", "handler"}
+ httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{
+ Namespace: ns,
+ Subsystem: sub,
+ Name: "requests_in_flight",
+ Help: "Number of requests currently handled by this server.",
+ }, basicLabels)
+ httpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+ Namespace: ns,
+ Subsystem: sub,
+ Name: "request_errors_total",
+ Help: "Number of requests resulting in middleware errors.",
+ }, basicLabels)
+ httpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{
+ Namespace: ns,
+ Subsystem: sub,
+ Name: "requests_total",
+ Help: "Counter of HTTP(S) requests made.",
+ }, basicLabels)
+
+ // TODO: allow these to be customized in the config
+ durationBuckets := prometheus.DefBuckets
+ sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8)
+
+ httpLabels := []string{"server", "handler", "code", "method"}
+ httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+ Namespace: ns,
+ Subsystem: sub,
+ Name: "request_duration_seconds",
+ Help: "Histogram of round-trip request durations.",
+ Buckets: durationBuckets,
+ }, httpLabels)
+ httpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+ Namespace: ns,
+ Subsystem: sub,
+ Name: "request_size_bytes",
+ Help: "Total size of the request. Includes body",
+ Buckets: sizeBuckets,
+ }, httpLabels)
+ httpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+ Namespace: ns,
+ Subsystem: sub,
+ Name: "response_size_bytes",
+ Help: "Size of the returned response.",
+ Buckets: sizeBuckets,
+ }, httpLabels)
+ httpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+ Namespace: ns,
+ Subsystem: sub,
+ Name: "response_duration_seconds",
+ Help: "Histogram of times to first byte in response bodies.",
+ Buckets: durationBuckets,
+ }, httpLabels)
+}
+
+type ctxKeyServerName struct{}
+
+// serverNameFromContext extracts the current server name from the context.
+// Returns "UNKNOWN" if none is available (should probably never happen?)
+func serverNameFromContext(ctx context.Context) string {
+ srvName, ok := ctx.Value(ctxKeyServerName{}).(string)
+ if !ok {
+ return "UNKNOWN"
+ }
+ return srvName
+}
+
+func contextWithServerName(ctx context.Context, serverName string) context.Context {
+ return context.WithValue(ctx, ctxKeyServerName{}, serverName)
+}
+
+type metricsInstrumentedHandler struct {
+ labels prometheus.Labels
+ statusLabels prometheus.Labels
+ mh MiddlewareHandler
+}
+
+func newMetricsInstrumentedHandler(server, handler string, mh MiddlewareHandler) *metricsInstrumentedHandler {
+ httpMetrics.init.Do(func() {
+ initHTTPMetrics()
+ })
+
+ labels := prometheus.Labels{"server": server, "handler": handler}
+ statusLabels := prometheus.Labels{"server": server, "handler": handler, "code": "", "method": ""}
+ return &metricsInstrumentedHandler{labels, statusLabels, mh}
+}
+
+func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {
+ inFlight := httpMetrics.requestInFlight.With(h.labels)
+ inFlight.Inc()
+ defer inFlight.Dec()
+
+ statusLabels := prometheus.Labels{"method": r.Method}
+ for k, v := range h.labels {
+ statusLabels[k] = v
+ }
+
+ start := time.Now()
+
+ // This is a _bit_ of a hack - it depends on the ShouldBufferFunc always
+ // being called when the headers are written.
+ // Effectively the same behaviour as promhttp.InstrumentHandlerTimeToWriteHeader.
+ writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool {
+ statusLabels["code"] = sanitizeCode(status)
+ ttfb := time.Since(start).Seconds()
+ observeWithExemplar(statusLabels, httpMetrics.responseDuration, ttfb)
+ return false
+ })
+ wrec := NewResponseRecorder(w, nil, writeHeaderRecorder)
+ err := h.mh.ServeHTTP(wrec, r, next)
+ dur := time.Since(start).Seconds()
+ httpMetrics.requestCount.With(h.labels).Inc()
+ if err != nil {
+ httpMetrics.requestErrors.With(h.labels).Inc()
+ return err
+ }
+
+ observeWithExemplar(statusLabels, httpMetrics.requestDuration, dur)
+ observeWithExemplar(statusLabels, httpMetrics.requestSize, float64(computeApproximateRequestSize(r)))
+ httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
+
+ return nil
+}
+
+func observeWithExemplar(l prometheus.Labels, o *prometheus.HistogramVec, value float64) {
+ obs := o.With(l)
+ if oe, ok := obs.(prometheus.ExemplarObserver); ok {
+ oe.ObserveWithExemplar(value, l)
+ return
+ }
+ // _should_ be a noop, but here just in case...
+ obs.Observe(value)
+}
+
+func sanitizeCode(code int) string {
+ if code == 0 {
+ return "200"
+ }
+ return strconv.Itoa(code)
+
+}
+
+// taken from https://github.com/prometheus/client_golang/blob/6007b2b5cae01203111de55f753e76d8dac1f529/prometheus/promhttp/instrument_server.go#L298
+func computeApproximateRequestSize(r *http.Request) int {
+ s := 0
+ if r.URL != nil {
+ s += len(r.URL.String())
+ }
+
+ s += len(r.Method)
+ s += len(r.Proto)
+ for name, values := range r.Header {
+ s += len(name)
+ for _, value := range values {
+ s += len(value)
+ }
+ }
+ s += len(r.Host)
+
+ // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL.
+
+ if r.ContentLength != -1 {
+ s += int(r.ContentLength)
+ }
+ return s
+}
diff --git a/modules/caddyhttp/metrics_test.go b/modules/caddyhttp/metrics_test.go
new file mode 100644
index 0000000..5c3bc1d
--- /dev/null
+++ b/modules/caddyhttp/metrics_test.go
@@ -0,0 +1,66 @@
+package caddyhttp
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestServerNameFromContext(t *testing.T) {
+ ctx := context.Background()
+ expected := "UNKNOWN"
+ if actual := serverNameFromContext(ctx); actual != expected {
+ t.Errorf("Not equal: expected %q, but got %q", expected, actual)
+ }
+
+ in := "foo"
+ ctx = contextWithServerName(ctx, in)
+ if actual := serverNameFromContext(ctx); actual != in {
+ t.Errorf("Not equal: expected %q, but got %q", in, actual)
+ }
+}
+
+func TestMetricsInstrumentedHandler(t *testing.T) {
+ handlerErr := errors.New("oh noes")
+ response := []byte("hello world!")
+ h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+ if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 {
+ t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
+ }
+ if handlerErr == nil {
+ w.Write(response)
+ }
+ return handlerErr
+ })
+
+ mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
+ return h.ServeHTTP(w, r)
+ })
+
+ ih := newMetricsInstrumentedHandler("foo", "bar", mh)
+
+ r := httptest.NewRequest("GET", "/", nil)
+ w := httptest.NewRecorder()
+
+ if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
+ t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
+ }
+ if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 {
+ t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
+ }
+
+ handlerErr = nil
+ if err := ih.ServeHTTP(w, r, h); err != nil {
+ t.Errorf("Received unexpected error: %w", err)
+ }
+}
+
+type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error
+
+func (f middlewareHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, h Handler) error {
+ return f(w, r, h)
+}
diff --git a/modules/caddyhttp/routes.go b/modules/caddyhttp/routes.go
index 10e0c9d..be23d39 100644
--- a/modules/caddyhttp/routes.go
+++ b/modules/caddyhttp/routes.go
@@ -157,7 +157,7 @@ func (routes RouteList) ProvisionHandlers(ctx caddy.Context) error {
// pre-compile the middleware handler chain
for _, midhandler := range routes[i].Handlers {
- routes[i].middleware = append(routes[i].middleware, wrapMiddleware(midhandler))
+ routes[i].middleware = append(routes[i].middleware, wrapMiddleware(ctx, midhandler))
}
}
return nil
@@ -242,7 +242,13 @@ func wrapRoute(route Route) Middleware {
// we need to pull this particular MiddlewareHandler
// pointer into its own stack frame to preserve it so it
// won't be overwritten in future loop iterations.
-func wrapMiddleware(mh MiddlewareHandler) Middleware {
+func wrapMiddleware(ctx caddy.Context, mh MiddlewareHandler) Middleware {
+ // first, wrap the middleware with metrics instrumentation
+ metricsHandler := newMetricsInstrumentedHandler(
+ serverNameFromContext(ctx.Context),
+ caddy.GetModuleName(mh),
+ mh,
+ )
return func(next Handler) Handler {
// copy the next handler (it's an interface, so it's
// just a very lightweight copy of a pointer); this
@@ -253,7 +259,7 @@ func wrapMiddleware(mh MiddlewareHandler) Middleware {
return HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// TODO: This is where request tracing could be implemented
// TODO: see what the std lib gives us in terms of stack tracing too
- return mh.ServeHTTP(w, r, nextCopy)
+ return metricsHandler.ServeHTTP(w, r, nextCopy)
})
}
}
diff --git a/modules/metrics/adminmetrics.go b/modules/metrics/adminmetrics.go
new file mode 100644
index 0000000..f6a55e8
--- /dev/null
+++ b/modules/metrics/adminmetrics.go
@@ -0,0 +1,55 @@
+// Copyright 2020 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+ "net/http"
+
+ "github.com/caddyserver/caddy/v2"
+)
+
+func init() {
+ caddy.RegisterModule(AdminMetrics{})
+}
+
+// AdminMetrics is a module that serves a metrics endpoint so that any gathered
+// metrics can be exposed for scraping. This module is not configurable, and
+// is permanently mounted to the admin API endpoint at "/metrics".
+// See the Metrics module for a configurable endpoint that is usable if the
+// Admin API is disabled.
+type AdminMetrics struct{}
+
+// CaddyModule returns the Caddy module information.
+func (AdminMetrics) CaddyModule() caddy.ModuleInfo {
+ return caddy.ModuleInfo{
+ ID: "admin.api.metrics",
+ New: func() caddy.Module { return new(AdminMetrics) },
+ }
+}
+
+// Routes returns a route for the /metrics endpoint.
+func (m *AdminMetrics) Routes() []caddy.AdminRoute {
+ metricsHandler := createMetricsHandler(nil)
+ h := caddy.AdminHandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+ metricsHandler.ServeHTTP(w, r)
+ return nil
+ })
+ return []caddy.AdminRoute{{Pattern: "/metrics", Handler: h}}
+}
+
+// Interface guards
+var (
+ _ caddy.AdminRouter = (*AdminMetrics)(nil)
+)
diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go
new file mode 100644
index 0000000..eab6342
--- /dev/null
+++ b/modules/metrics/metrics.go
@@ -0,0 +1,109 @@
+// Copyright 2020 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+ "net/http"
+
+ "github.com/caddyserver/caddy/v2"
+ "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+ "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
+ "github.com/caddyserver/caddy/v2/modules/caddyhttp"
+
+ "go.uber.org/zap"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func init() {
+ caddy.RegisterModule(Metrics{})
+ httpcaddyfile.RegisterHandlerDirective("metrics", parseCaddyfile)
+}
+
+// Metrics is a module that serves a /metrics endpoint so that any gathered
+// metrics can be exposed for scraping. This module is configurable by end-users
+// unlike AdminMetrics.
+type Metrics struct {
+ metricsHandler http.Handler
+}
+
+// CaddyModule returns the Caddy module information.
+func (Metrics) CaddyModule() caddy.ModuleInfo {
+ return caddy.ModuleInfo{
+ ID: "http.handlers.metrics",
+ New: func() caddy.Module { return new(Metrics) },
+ }
+}
+
+type zapLogger struct {
+ zl *zap.Logger
+}
+
+func (l *zapLogger) Println(v ...interface{}) {
+ l.zl.Sugar().Error(v...)
+}
+
+// Provision sets up m.
+func (m *Metrics) Provision(ctx caddy.Context) error {
+ log := ctx.Logger(m)
+ m.metricsHandler = createMetricsHandler(&zapLogger{log})
+ return nil
+}
+
+func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
+ var m Metrics
+ err := m.UnmarshalCaddyfile(h.Dispenser)
+ return m, err
+}
+
+// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
+//
+// metrics <matcher>
+//
+func (m *Metrics) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+ for d.Next() {
+ args := d.RemainingArgs()
+ if len(args) > 0 {
+ return d.ArgErr()
+ }
+ }
+ return nil
+}
+
+func (m Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+ m.metricsHandler.ServeHTTP(w, r)
+ return next.ServeHTTP(w, r)
+}
+
+// Interface guards
+var (
+ _ caddy.Provisioner = (*Metrics)(nil)
+ _ caddyhttp.MiddlewareHandler = (*Metrics)(nil)
+ _ caddyfile.Unmarshaler = (*Metrics)(nil)
+)
+
+func createMetricsHandler(logger promhttp.Logger) http.Handler {
+ return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer,
+ promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
+ // will only log errors if logger is non-nil
+ ErrorLog: logger,
+
+ // Allow OpenMetrics format to be negotiated - largely compatible,
+ // except quantile/le label values always have a decimal.
+ EnableOpenMetrics: true,
+ }),
+ )
+}
diff --git a/modules/standard/imports.go b/modules/standard/imports.go
index b8ea7a8..bc2d955 100644
--- a/modules/standard/imports.go
+++ b/modules/standard/imports.go
@@ -11,4 +11,5 @@ import (
_ "github.com/caddyserver/caddy/v2/modules/caddytls/standardstek"
_ "github.com/caddyserver/caddy/v2/modules/filestorage"
_ "github.com/caddyserver/caddy/v2/modules/logging"
+ _ "github.com/caddyserver/caddy/v2/modules/metrics"
)