From 0c57facc67910a6b0994763ac8e6e55e693f20e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Szab=C3=B3?= Date: Sat, 27 Aug 2022 20:30:23 +0200 Subject: reverseproxy: Add upstreams healthy metrics (#4935) --- modules/caddyhttp/reverseproxy/metrics.go | 81 ++++++++++++++++++++++++++ modules/caddyhttp/reverseproxy/reverseproxy.go | 3 + 2 files changed, 84 insertions(+) create mode 100644 modules/caddyhttp/reverseproxy/metrics.go (limited to 'modules/caddyhttp') diff --git a/modules/caddyhttp/reverseproxy/metrics.go b/modules/caddyhttp/reverseproxy/metrics.go new file mode 100644 index 0000000..4272bc4 --- /dev/null +++ b/modules/caddyhttp/reverseproxy/metrics.go @@ -0,0 +1,81 @@ +package reverseproxy + +import ( + "runtime/debug" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.uber.org/zap" +) + +var reverseProxyMetrics = struct { + init sync.Once + upstreamsHealthy *prometheus.GaugeVec + logger *zap.Logger +}{} + +func initReverseProxyMetrics(handler *Handler) { + const ns, sub = "caddy", "reverse_proxy" + + upstreamsLabels := []string{"upstream"} + reverseProxyMetrics.upstreamsHealthy = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ns, + Subsystem: sub, + Name: "upstreams_healthy", + Help: "Health status of reverse proxy upstreams.", + }, upstreamsLabels) + + reverseProxyMetrics.logger = handler.logger.Named("reverse_proxy.metrics") +} + +type metricsUpstreamsHealthyUpdater struct { + handler *Handler +} + +func newMetricsUpstreamsHealthyUpdater(handler *Handler) *metricsUpstreamsHealthyUpdater { + reverseProxyMetrics.init.Do(func() { + initReverseProxyMetrics(handler) + }) + + return &metricsUpstreamsHealthyUpdater{handler} +} + +func (m *metricsUpstreamsHealthyUpdater) Init() { + go func() { + defer func() { + if err := recover(); err != nil { + reverseProxyMetrics.logger.Error("upstreams healthy metrics updater panicked", + zap.Any("error", err), + zap.ByteString("stack", debug.Stack())) + } + }() + + m.update() + + ticker := time.NewTicker(10 * time.Second) + for { + select { + case <-ticker.C: + m.update() + case <-m.handler.ctx.Done(): + ticker.Stop() + return + } + } + }() +} + +func (m *metricsUpstreamsHealthyUpdater) update() { + for _, upstream := range m.handler.Upstreams { + labels := prometheus.Labels{"upstream": upstream.Dial} + + gaugeValue := 0.0 + if upstream.Healthy() { + gaugeValue = 1.0 + } + + reverseProxyMetrics.upstreamsHealthy.With(labels).Set(gaugeValue) + } +} diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go index 0890306..fd3eb2c 100644 --- a/modules/caddyhttp/reverseproxy/reverseproxy.go +++ b/modules/caddyhttp/reverseproxy/reverseproxy.go @@ -392,6 +392,9 @@ func (h *Handler) Provision(ctx caddy.Context) error { } } + upstreamHealthyUpdater := newMetricsUpstreamsHealthyUpdater(h) + upstreamHealthyUpdater.Init() + return nil } -- cgit v1.2.3