proxyprotocol: Add PROXY protocol support to `reverse_proxy`, add HTTP listener wrapper (#5424)

Co-authored-by: WeidiDeng <weidi_deng@icloud.com>
Co-authored-by: Francis Lavoie <lavofr@gmail.com>

4 files changed, 98 insertions, 1 deletions

diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go
index cf84ba7..fab3099 100644
--- a/modules/caddyhttp/reverseproxy/caddyfile.go
+++ b/modules/caddyhttp/reverseproxy/caddyfile.go
@@ -918,6 +918,17 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				}
 				h.MaxResponseHeaderSize = int64(size)
 
+			case "proxy_protocol":
+				if !d.NextArg() {
+					return d.ArgErr()
+				}
+				switch proxyProtocol := d.Val(); proxyProtocol {
+				case "v1", "v2":
+					h.ProxyProtocol = proxyProtocol
+				default:
+					return d.Errf("invalid proxy protocol version '%s'", proxyProtocol)
+				}
+
 			case "dial_timeout":
 				if !d.NextArg() {
 					return d.ArgErr()
diff --git a/modules/caddyhttp/reverseproxy/hosts.go b/modules/caddyhttp/reverseproxy/hosts.go
index a973ecb..b97c8b4 100644
--- a/modules/caddyhttp/reverseproxy/hosts.go
+++ b/modules/caddyhttp/reverseproxy/hosts.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/netip"
 	"strconv"
 	"sync/atomic"
 
@@ -259,3 +260,13 @@ var hosts = caddy.NewUsagePool()
 // dialInfoVarKey is the key used for the variable that holds
 // the dial info for the upstream connection.
 const dialInfoVarKey = "reverse_proxy.dial_info"
+
+// proxyProtocolInfoVarKey is the key used for the variable that holds
+// the proxy protocol info for the upstream connection.
+const proxyProtocolInfoVarKey = "reverse_proxy.proxy_protocol_info"
+
+// ProxyProtocolInfo contains information needed to write proxy protocol to a
+// connection to an upstream host.
+type ProxyProtocolInfo struct {
+	AddrPort netip.AddrPort
+}
diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go
index 71d06d5..1135862 100644
--- a/modules/caddyhttp/reverseproxy/httptransport.go
+++ b/modules/caddyhttp/reverseproxy/httptransport.go
@@ -29,7 +29,9 @@ import (
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"github.com/mastercactapus/proxyprotocol"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
 )
@@ -64,6 +66,10 @@ type HTTPTransport struct {
 	// Maximum number of connections per host. Default: 0 (no limit)
 	MaxConnsPerHost int `json:"max_conns_per_host,omitempty"`
 
+	// If non-empty, which PROXY protocol version to send when
+	// connecting to an upstream. Default: off.
+	ProxyProtocol string `json:"proxy_protocol,omitempty"`
+
 	// How long to wait before timing out trying to connect to
 	// an upstream. Default: `3s`.
 	DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`
@@ -195,6 +201,57 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e
 			return nil, DialError{err}
 		}
 
+		if h.ProxyProtocol != "" {
+			proxyProtocolInfo, ok := caddyhttp.GetVar(ctx, proxyProtocolInfoVarKey).(ProxyProtocolInfo)
+			if !ok {
+				return nil, fmt.Errorf("failed to get proxy protocol info from context")
+			}
+
+			// The src and dst have to be of the some address family. As we don't know the original
+			// dst address (it's kind of impossible to know) and this address is generelly of very
+			// little interest, we just set it to all zeros.
+			var destIP net.IP
+			switch {
+			case proxyProtocolInfo.AddrPort.Addr().Is4():
+				destIP = net.IPv4zero
+			case proxyProtocolInfo.AddrPort.Addr().Is6():
+				destIP = net.IPv6zero
+			default:
+				return nil, fmt.Errorf("unexpected remote addr type in proxy protocol info")
+			}
+
+			// TODO: We should probably migrate away from net.IP to use netip.Addr,
+			// but due to the upstream dependency, we can't do that yet.
+			switch h.ProxyProtocol {
+			case "v1":
+				header := proxyprotocol.HeaderV1{
+					SrcIP:    net.IP(proxyProtocolInfo.AddrPort.Addr().AsSlice()),
+					SrcPort:  int(proxyProtocolInfo.AddrPort.Port()),
+					DestIP:   destIP,
+					DestPort: 0,
+				}
+				caddyCtx.Logger().Debug("sending proxy protocol header v1", zap.Any("header", header))
+				_, err = header.WriteTo(conn)
+			case "v2":
+				header := proxyprotocol.HeaderV2{
+					Command: proxyprotocol.CmdProxy,
+					Src:     &net.TCPAddr{IP: net.IP(proxyProtocolInfo.AddrPort.Addr().AsSlice()), Port: int(proxyProtocolInfo.AddrPort.Port())},
+					Dest:    &net.TCPAddr{IP: destIP, Port: 0},
+				}
+				caddyCtx.Logger().Debug("sending proxy protocol header v2", zap.Any("header", header))
+				_, err = header.WriteTo(conn)
+			default:
+				return nil, fmt.Errorf("unexpected proxy protocol version")
+			}
+
+			if err != nil {
+				// identify this error as one that occurred during
+				// dialing, which can be important when trying to
+				// decide whether to retry a request
+				return nil, DialError{err}
+			}
+		}
+
 		// if read/write timeouts are configured and this is a TCP connection,
 		// enforce the timeouts by wrapping the connection with our own type
 		if tcpConn, ok := conn.(*net.TCPConn); ok && (h.ReadTimeout > 0 || h.WriteTimeout > 0) {
@@ -239,6 +296,14 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e
 		rt.IdleConnTimeout = time.Duration(h.KeepAlive.IdleConnTimeout)
 	}
 
+	// The proxy protocol header can only be sent once right after opening the connection.
+	// So single connection must not be used for multiple requests, which can potentially
+	// come from different clients.
+	if !rt.DisableKeepAlives && h.ProxyProtocol != "" {
+		caddyCtx.Logger().Warn("disabling keepalives, they are incompatible with using PROXY protocol")
+		rt.DisableKeepAlives = true
+	}
+
 	if h.Compression != nil {
 		rt.DisableCompression = !*h.Compression
 	}
diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go
index 8fd24fe..ff22d49 100644
--- a/modules/caddyhttp/reverseproxy/reverseproxy.go
+++ b/modules/caddyhttp/reverseproxy/reverseproxy.go
@@ -688,8 +688,18 @@ func (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http.
 		req.Header.Set("Upgrade", reqUpType)
 	}
 
+	// Set up the PROXY protocol info
+	address := caddyhttp.GetVar(req.Context(), caddyhttp.ClientIPVarKey).(string)
+	addrPort, err := netip.ParseAddrPort(address)
+	if err != nil {
+		// OK; probably didn't have a port
+		addrPort, _ = netip.ParseAddrPort(address + ":0")
+	}
+	proxyProtocolInfo := ProxyProtocolInfo{AddrPort: addrPort}
+	caddyhttp.SetVar(req.Context(), proxyProtocolInfoVarKey, proxyProtocolInfo)
+
 	// Add the supported X-Forwarded-* headers
-	err := h.addForwardedHeaders(req)
+	err = h.addForwardedHeaders(req)
 	if err != nil {
 		return nil, err
 	}