From b6fe5d4b41d07e70a502ed58d40e8b0e75067db5 Mon Sep 17 00:00:00 2001 From: Corin Langosch Date: Fri, 31 Mar 2023 23:44:53 +0200 Subject: proxyprotocol: Add PROXY protocol support to `reverse_proxy`, add HTTP listener wrapper (#5424) Co-authored-by: WeidiDeng Co-authored-by: Francis Lavoie --- modules/caddyhttp/proxyprotocol/listenerwrapper.go | 68 ++++++++++++++++++++ modules/caddyhttp/proxyprotocol/module.go | 75 ++++++++++++++++++++++ modules/caddyhttp/reverseproxy/caddyfile.go | 11 ++++ modules/caddyhttp/reverseproxy/hosts.go | 11 ++++ modules/caddyhttp/reverseproxy/httptransport.go | 65 +++++++++++++++++++ modules/caddyhttp/reverseproxy/reverseproxy.go | 12 +++- modules/caddyhttp/standard/imports.go | 1 + 7 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 modules/caddyhttp/proxyprotocol/listenerwrapper.go create mode 100644 modules/caddyhttp/proxyprotocol/module.go (limited to 'modules') diff --git a/modules/caddyhttp/proxyprotocol/listenerwrapper.go b/modules/caddyhttp/proxyprotocol/listenerwrapper.go new file mode 100644 index 0000000..fd90499 --- /dev/null +++ b/modules/caddyhttp/proxyprotocol/listenerwrapper.go @@ -0,0 +1,68 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxyprotocol + +import ( + "fmt" + "net" + "time" + + "github.com/caddyserver/caddy/v2" + "github.com/mastercactapus/proxyprotocol" +) + +// ListenerWrapper provides PROXY protocol support to Caddy by implementing +// the caddy.ListenerWrapper interface. It must be loaded before the `tls` listener. +// +// Credit goes to https://github.com/mastercactapus/caddy2-proxyprotocol for having +// initially implemented this as a plugin. +type ListenerWrapper struct { + // Timeout specifies an optional maximum time for + // the PROXY header to be received. + // If zero, timeout is disabled. Default is 5s. + Timeout caddy.Duration `json:"timeout,omitempty"` + + // Allow is an optional list of CIDR ranges to + // allow/require PROXY headers from. + Allow []string `json:"allow,omitempty"` + + rules []proxyprotocol.Rule +} + +// Provision sets up the listener wrapper. +func (pp *ListenerWrapper) Provision(ctx caddy.Context) error { + rules := make([]proxyprotocol.Rule, 0, len(pp.Allow)) + for _, s := range pp.Allow { + _, n, err := net.ParseCIDR(s) + if err != nil { + return fmt.Errorf("invalid subnet '%s': %w", s, err) + } + rules = append(rules, proxyprotocol.Rule{ + Timeout: time.Duration(pp.Timeout), + Subnet: n, + }) + } + + pp.rules = rules + + return nil +} + +// WrapListener adds PROXY protocol support to the listener. +func (pp *ListenerWrapper) WrapListener(l net.Listener) net.Listener { + pl := proxyprotocol.NewListener(l, time.Duration(pp.Timeout)) + pl.SetFilter(pp.rules) + return pl +} diff --git a/modules/caddyhttp/proxyprotocol/module.go b/modules/caddyhttp/proxyprotocol/module.go new file mode 100644 index 0000000..78f89c6 --- /dev/null +++ b/modules/caddyhttp/proxyprotocol/module.go @@ -0,0 +1,75 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxyprotocol + +import ( + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" +) + +func init() { + caddy.RegisterModule(ListenerWrapper{}) +} + +func (ListenerWrapper) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.listeners.proxy_protocol", + New: func() caddy.Module { return new(ListenerWrapper) }, + } +} + +// UnmarshalCaddyfile sets up the listener Listenerwrapper from Caddyfile tokens. Syntax: +// +// proxy_protocol { +// timeout +// allow +// } +func (w *ListenerWrapper) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + for d.Next() { + // No same-line options are supported + if d.NextArg() { + return d.ArgErr() + } + + for d.NextBlock(0) { + switch d.Val() { + case "timeout": + if !d.NextArg() { + return d.ArgErr() + } + dur, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("parsing proxy_protocol timeout duration: %v", err) + } + w.Timeout = caddy.Duration(dur) + + case "allow": + w.Allow = append(w.Allow, d.RemainingArgs()...) + + default: + return d.ArgErr() + } + } + } + return nil +} + +// Interface guards +var ( + _ caddy.Provisioner = (*ListenerWrapper)(nil) + _ caddy.Module = (*ListenerWrapper)(nil) + _ caddy.ListenerWrapper = (*ListenerWrapper)(nil) + _ caddyfile.Unmarshaler = (*ListenerWrapper)(nil) +) diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index cf84ba7..fab3099 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -918,6 +918,17 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } h.MaxResponseHeaderSize = int64(size) + case "proxy_protocol": + if !d.NextArg() { + return d.ArgErr() + } + switch proxyProtocol := d.Val(); proxyProtocol { + case "v1", "v2": + h.ProxyProtocol = proxyProtocol + default: + return d.Errf("invalid proxy protocol version '%s'", proxyProtocol) + } + case "dial_timeout": if !d.NextArg() { return d.ArgErr() diff --git a/modules/caddyhttp/reverseproxy/hosts.go b/modules/caddyhttp/reverseproxy/hosts.go index a973ecb..b97c8b4 100644 --- a/modules/caddyhttp/reverseproxy/hosts.go +++ b/modules/caddyhttp/reverseproxy/hosts.go @@ -19,6 +19,7 @@ import ( "fmt" "net" "net/http" + "net/netip" "strconv" "sync/atomic" @@ -259,3 +260,13 @@ var hosts = caddy.NewUsagePool() // dialInfoVarKey is the key used for the variable that holds // the dial info for the upstream connection. const dialInfoVarKey = "reverse_proxy.dial_info" + +// proxyProtocolInfoVarKey is the key used for the variable that holds +// the proxy protocol info for the upstream connection. +const proxyProtocolInfoVarKey = "reverse_proxy.proxy_protocol_info" + +// ProxyProtocolInfo contains information needed to write proxy protocol to a +// connection to an upstream host. +type ProxyProtocolInfo struct { + AddrPort netip.AddrPort +} diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index 71d06d5..1135862 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -29,7 +29,9 @@ import ( "time" "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/mastercactapus/proxyprotocol" "go.uber.org/zap" "golang.org/x/net/http2" ) @@ -64,6 +66,10 @@ type HTTPTransport struct { // Maximum number of connections per host. Default: 0 (no limit) MaxConnsPerHost int `json:"max_conns_per_host,omitempty"` + // If non-empty, which PROXY protocol version to send when + // connecting to an upstream. Default: off. + ProxyProtocol string `json:"proxy_protocol,omitempty"` + // How long to wait before timing out trying to connect to // an upstream. Default: `3s`. DialTimeout caddy.Duration `json:"dial_timeout,omitempty"` @@ -195,6 +201,57 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e return nil, DialError{err} } + if h.ProxyProtocol != "" { + proxyProtocolInfo, ok := caddyhttp.GetVar(ctx, proxyProtocolInfoVarKey).(ProxyProtocolInfo) + if !ok { + return nil, fmt.Errorf("failed to get proxy protocol info from context") + } + + // The src and dst have to be of the some address family. As we don't know the original + // dst address (it's kind of impossible to know) and this address is generelly of very + // little interest, we just set it to all zeros. + var destIP net.IP + switch { + case proxyProtocolInfo.AddrPort.Addr().Is4(): + destIP = net.IPv4zero + case proxyProtocolInfo.AddrPort.Addr().Is6(): + destIP = net.IPv6zero + default: + return nil, fmt.Errorf("unexpected remote addr type in proxy protocol info") + } + + // TODO: We should probably migrate away from net.IP to use netip.Addr, + // but due to the upstream dependency, we can't do that yet. + switch h.ProxyProtocol { + case "v1": + header := proxyprotocol.HeaderV1{ + SrcIP: net.IP(proxyProtocolInfo.AddrPort.Addr().AsSlice()), + SrcPort: int(proxyProtocolInfo.AddrPort.Port()), + DestIP: destIP, + DestPort: 0, + } + caddyCtx.Logger().Debug("sending proxy protocol header v1", zap.Any("header", header)) + _, err = header.WriteTo(conn) + case "v2": + header := proxyprotocol.HeaderV2{ + Command: proxyprotocol.CmdProxy, + Src: &net.TCPAddr{IP: net.IP(proxyProtocolInfo.AddrPort.Addr().AsSlice()), Port: int(proxyProtocolInfo.AddrPort.Port())}, + Dest: &net.TCPAddr{IP: destIP, Port: 0}, + } + caddyCtx.Logger().Debug("sending proxy protocol header v2", zap.Any("header", header)) + _, err = header.WriteTo(conn) + default: + return nil, fmt.Errorf("unexpected proxy protocol version") + } + + if err != nil { + // identify this error as one that occurred during + // dialing, which can be important when trying to + // decide whether to retry a request + return nil, DialError{err} + } + } + // if read/write timeouts are configured and this is a TCP connection, // enforce the timeouts by wrapping the connection with our own type if tcpConn, ok := conn.(*net.TCPConn); ok && (h.ReadTimeout > 0 || h.WriteTimeout > 0) { @@ -239,6 +296,14 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e rt.IdleConnTimeout = time.Duration(h.KeepAlive.IdleConnTimeout) } + // The proxy protocol header can only be sent once right after opening the connection. + // So single connection must not be used for multiple requests, which can potentially + // come from different clients. + if !rt.DisableKeepAlives && h.ProxyProtocol != "" { + caddyCtx.Logger().Warn("disabling keepalives, they are incompatible with using PROXY protocol") + rt.DisableKeepAlives = true + } + if h.Compression != nil { rt.DisableCompression = !*h.Compression } diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go index 8fd24fe..ff22d49 100644 --- a/modules/caddyhttp/reverseproxy/reverseproxy.go +++ b/modules/caddyhttp/reverseproxy/reverseproxy.go @@ -688,8 +688,18 @@ func (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http. req.Header.Set("Upgrade", reqUpType) } + // Set up the PROXY protocol info + address := caddyhttp.GetVar(req.Context(), caddyhttp.ClientIPVarKey).(string) + addrPort, err := netip.ParseAddrPort(address) + if err != nil { + // OK; probably didn't have a port + addrPort, _ = netip.ParseAddrPort(address + ":0") + } + proxyProtocolInfo := ProxyProtocolInfo{AddrPort: addrPort} + caddyhttp.SetVar(req.Context(), proxyProtocolInfoVarKey, proxyProtocolInfo) + // Add the supported X-Forwarded-* headers - err := h.addForwardedHeaders(req) + err = h.addForwardedHeaders(req) if err != nil { return nil, err } diff --git a/modules/caddyhttp/standard/imports.go b/modules/caddyhttp/standard/imports.go index 435569d..d7bb280 100644 --- a/modules/caddyhttp/standard/imports.go +++ b/modules/caddyhttp/standard/imports.go @@ -11,6 +11,7 @@ import ( _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/map" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/proxyprotocol" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/push" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/requestbody" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" -- cgit v1.2.3