From 0c57facc67910a6b0994763ac8e6e55e693f20e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Szab=C3=B3?= <david.szabo97@gmail.com>
Date: Sat, 27 Aug 2022 20:30:23 +0200
Subject: reverseproxy: Add upstreams healthy metrics (#4935)

---
 modules/caddyhttp/reverseproxy/metrics.go      | 81 ++++++++++++++++++++++++++
 modules/caddyhttp/reverseproxy/reverseproxy.go |  3 +
 2 files changed, 84 insertions(+)
 create mode 100644 modules/caddyhttp/reverseproxy/metrics.go

(limited to 'modules/caddyhttp')

diff --git a/modules/caddyhttp/reverseproxy/metrics.go b/modules/caddyhttp/reverseproxy/metrics.go
new file mode 100644
index 0000000..4272bc4
--- /dev/null
+++ b/modules/caddyhttp/reverseproxy/metrics.go
@@ -0,0 +1,81 @@
+package reverseproxy
+
+import (
+	"runtime/debug"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"go.uber.org/zap"
+)
+
+var reverseProxyMetrics = struct {
+	init             sync.Once
+	upstreamsHealthy *prometheus.GaugeVec
+	logger           *zap.Logger
+}{}
+
+func initReverseProxyMetrics(handler *Handler) {
+	const ns, sub = "caddy", "reverse_proxy"
+
+	upstreamsLabels := []string{"upstream"}
+	reverseProxyMetrics.upstreamsHealthy = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: ns,
+		Subsystem: sub,
+		Name:      "upstreams_healthy",
+		Help:      "Health status of reverse proxy upstreams.",
+	}, upstreamsLabels)
+
+	reverseProxyMetrics.logger = handler.logger.Named("reverse_proxy.metrics")
+}
+
+type metricsUpstreamsHealthyUpdater struct {
+	handler *Handler
+}
+
+func newMetricsUpstreamsHealthyUpdater(handler *Handler) *metricsUpstreamsHealthyUpdater {
+	reverseProxyMetrics.init.Do(func() {
+		initReverseProxyMetrics(handler)
+	})
+
+	return &metricsUpstreamsHealthyUpdater{handler}
+}
+
+func (m *metricsUpstreamsHealthyUpdater) Init() {
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				reverseProxyMetrics.logger.Error("upstreams healthy metrics updater panicked",
+					zap.Any("error", err),
+					zap.ByteString("stack", debug.Stack()))
+			}
+		}()
+
+		m.update()
+
+		ticker := time.NewTicker(10 * time.Second)
+		for {
+			select {
+			case <-ticker.C:
+				m.update()
+			case <-m.handler.ctx.Done():
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+}
+
+func (m *metricsUpstreamsHealthyUpdater) update() {
+	for _, upstream := range m.handler.Upstreams {
+		labels := prometheus.Labels{"upstream": upstream.Dial}
+
+		gaugeValue := 0.0
+		if upstream.Healthy() {
+			gaugeValue = 1.0
+		}
+
+		reverseProxyMetrics.upstreamsHealthy.With(labels).Set(gaugeValue)
+	}
+}
diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go
index 0890306..fd3eb2c 100644
--- a/modules/caddyhttp/reverseproxy/reverseproxy.go
+++ b/modules/caddyhttp/reverseproxy/reverseproxy.go
@@ -392,6 +392,9 @@ func (h *Handler) Provision(ctx caddy.Context) error {
 		}
 	}
 
+	upstreamHealthyUpdater := newMetricsUpstreamsHealthyUpdater(h)
+	upstreamHealthyUpdater.Init()
+
 	return nil
 }
 
-- 
cgit v1.2.3