caddyhttp: Enable HTTP/3 by default (#4707)

3 files changed, 136 insertions, 46 deletions

diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go
index c28e09d..a3d0f7e 100644
--- a/modules/caddyhttp/app.go
+++ b/modules/caddyhttp/app.go
@@ -25,7 +25,6 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"github.com/lucas-clemente/quic-go/http3"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
@@ -185,6 +184,17 @@ func (app *App) Provision(ctx caddy.Context) error {
 			srv.accessLogger = app.logger.Named("log.access")
 		}
 
+		// the Go standard library does not let us serve only HTTP/2 using
+		// http.Server; we would probably need to write our own server
+		if !srv.protocol("h1") && (srv.protocol("h2") || srv.protocol("h2c")) {
+			return fmt.Errorf("server %s: cannot enable HTTP/2 or H2C without enabling HTTP/1.1; add h1 to protocols or remove h2/h2c", srvName)
+		}
+
+		// if no protocols configured explicitly, enable all except h2c
+		if len(srv.Protocols) == 0 {
+			srv.Protocols = []string{"h1", "h2", "h3"}
+		}
+
 		// if not explicitly configured by the user, disallow TLS
 		// client auth bypass (domain fronting) which could
 		// otherwise be exploited by sending an unprotected SNI
@@ -196,8 +206,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		// based on hostname
 		if srv.StrictSNIHost == nil && srv.hasTLSClientAuth() {
 			app.logger.Warn("enabling strict SNI-Host enforcement because TLS client auth is configured",
-				zap.String("server_id", srvName),
-			)
+				zap.String("server_id", srvName))
 			trueBool := true
 			srv.StrictSNIHost = &trueBool
 		}
@@ -206,8 +215,7 @@ func (app *App) Provision(ctx caddy.Context) error {
 		for i := range srv.Listen {
 			lnOut, err := repl.ReplaceOrErr(srv.Listen[i], true, true)
 			if err != nil {
-				return fmt.Errorf("server %s, listener %d: %v",
-					srvName, i, err)
+				return fmt.Errorf("server %s, listener %d: %v", srvName, i, err)
 			}
 			srv.Listen[i] = lnOut
 		}
@@ -324,10 +332,34 @@ func (app *App) Start() error {
 			Handler:           srv,
 			ErrorLog:          serverLogger,
 		}
+
+		// disable HTTP/2, which we enabled by default during provisioning
+		if !srv.protocol("h2") {
+			srv.server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+			for _, cp := range srv.TLSConnPolicies {
+				// the TLSConfig was already provisioned, so... manually remove it
+				for i, np := range cp.TLSConfig.NextProtos {
+					if np == "h2" {
+						cp.TLSConfig.NextProtos = append(cp.TLSConfig.NextProtos[:i], cp.TLSConfig.NextProtos[i+1:]...)
+						break
+					}
+				}
+				// remove it from the parent connection policy too, just to keep things tidy
+				for i, alpn := range cp.ALPN {
+					if alpn == "h2" {
+						cp.ALPN = append(cp.ALPN[:i], cp.ALPN[i+1:]...)
+						break
+					}
+				}
+			}
+		}
+
+		// this TLS config is used by the std lib to choose the actual TLS config for connections
+		// by looking through the connection policies to find the first one that matches
 		tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
 
-		// enable h2c if configured
-		if srv.AllowH2C {
+		// enable H2C if configured
+		if srv.protocol("h2c") {
 			h2server := &http2.Server{
 				IdleTimeout: time.Duration(srv.IdleTimeout),
 			}
@@ -362,27 +394,15 @@ func (app *App) Start() error {
 				// enable TLS if there is a policy and if this is not the HTTP port
 				useTLS := len(srv.TLSConnPolicies) > 0 && int(listenAddr.StartPort+portOffset) != app.httpPort()
 				if useTLS {
-					// create TLS listener
+					// create TLS listener - this enables and terminates TLS
 					ln = tls.NewListener(ln, tlsCfg)
 
-					// TODO: HTTP/3 support is experimental for now
-					if srv.ExperimentalHTTP3 {
-						if srv.h3server == nil {
-							srv.h3server = &http3.Server{
-								Handler:        srv,
-								TLSConfig:      tlsCfg,
-								MaxHeaderBytes: srv.MaxHeaderBytes,
-							}
+					// enable HTTP/3 if configured
+					if srv.protocol("h3") {
+						app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport))
+						if err := srv.serveHTTP3(hostport, tlsCfg); err != nil {
+							return err
 						}
-
-						app.logger.Info("enabling experimental HTTP/3 listener", zap.String("addr", hostport))
-						h3ln, err := caddy.ListenQUIC(hostport, tlsCfg)
-						if err != nil {
-							return fmt.Errorf("getting HTTP/3 QUIC listener: %v", err)
-						}
-
-						//nolint:errcheck
-						go srv.h3server.ServeListener(h3ln)
 					}
 				}
 
@@ -402,16 +422,22 @@ func (app *App) Start() error {
 
 				app.logger.Debug("starting server loop",
 					zap.String("address", ln.Addr().String()),
-					zap.Bool("http3", srv.ExperimentalHTTP3),
 					zap.Bool("tls", useTLS),
-				)
+					zap.Bool("http3", srv.h3server != nil))
 
 				srv.listeners = append(srv.listeners, ln)
 
-				//nolint:errcheck
-				go srv.server.Serve(ln)
+				// enable HTTP/1 if configured
+				if srv.protocol("h1") {
+					//nolint:errcheck
+					go srv.server.Serve(ln)
+				}
 			}
 		}
+
+		srv.logger.Info("server running",
+			zap.String("name", srvName),
+			zap.Strings("protocols", srv.Protocols))
 	}
 
 	// finish automatic HTTPS by finally beginning
diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go
index 62c0fd9..f8cd612 100644
--- a/modules/caddyhttp/server.go
+++ b/modules/caddyhttp/server.go
@@ -16,6 +16,7 @@ package caddyhttp
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -25,6 +26,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
@@ -37,6 +39,8 @@ import (
 
 // Server describes an HTTP server.
 type Server struct {
+	activeRequests int64 // accessed atomically
+
 	// Socket addresses to which to bind listeners. Accepts
 	// [network addresses](/docs/conventions#network-addresses)
 	// that may include port ranges. Listener addresses must
@@ -112,21 +116,35 @@ type Server struct {
 	// to a non-null, empty struct.
 	Logs *ServerLogConfig `json:"logs,omitempty"`
 
-	// Enable experimental HTTP/3 support. Note that HTTP/3 is not a
-	// finished standard and has extremely limited client support.
-	// This field is not subject to compatibility promises.
-	ExperimentalHTTP3 bool `json:"experimental_http3,omitempty"`
-
-	// Enables H2C ("Cleartext HTTP/2" or "H2 over TCP") support,
-	// which will serve HTTP/2 over plaintext TCP connections if
-	// the client supports it. Because this is not implemented by the
-	// Go standard library, using H2C is incompatible with most
-	// of the other options for this server. Do not enable this
+	// Protocols specifies which HTTP protocols to enable.
+	// Supported values are:
+	//
+	// - `h1` (HTTP/1.1)
+	// - `h2` (HTTP/2)
+	// - `h2c` (cleartext HTTP/2)
+	// - `h3` (HTTP/3)
+	//
+	// If enabling `h2` or `h2c`, `h1` must also be enabled;
+	// this is due to current limitations in the Go standard
+	// library.
+	//
+	// HTTP/2 operates only over TLS (HTTPS). HTTP/3 opens
+	// a UDP socket to serve QUIC connections.
+	//
+	// H2C operates over plain TCP if the client supports it;
+	// however, because this is not implemented by the Go
+	// standard library, other server options are not compatible
+	// and will not be applied to H2C requests. Do not enable this
 	// only to achieve maximum client compatibility. In practice,
 	// very few clients implement H2C, and even fewer require it.
-	// This setting applies only to unencrypted HTTP listeners.
-	// ⚠️ Experimental feature; subject to change or removal.
-	AllowH2C bool `json:"allow_h2c,omitempty"`
+	// Enabling H2C can be useful for serving/proxying gRPC
+	// if encryption is not possible or desired.
+	//
+	// We recommend for most users to simply let Caddy use the
+	// default settings.
+	//
+	// Default: `[h1 h2 h3]`
+	Protocols []string `json:"protocols,omitempty"`
 
 	name string
 
@@ -152,7 +170,12 @@ type Server struct {
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Server", "Caddy")
 
+	// advertise HTTP/3, if enabled
 	if s.h3server != nil {
+		// keep track of active requests for QUIC transport purposes (See AcceptToken callback in quic.Config)
+		atomic.AddInt64(&s.activeRequests, 1)
+		defer atomic.AddInt64(&s.activeRequests, -1)
+
 		err := s.h3server.SetQuicHeaders(w.Header())
 		if err != nil {
 			s.logger.Error("setting HTTP/3 Alt-Svc header", zap.Error(err))
@@ -445,6 +468,30 @@ func (s *Server) findLastRouteWithHostMatcher() int {
 	return lastIndex
 }
 
+// serveHTTP3 creates a QUIC listener, configures an HTTP/3 server if
+// not already done, and then uses that server to serve HTTP/3 over
+// the listener, with Server s as the handler.
+func (s *Server) serveHTTP3(hostport string, tlsCfg *tls.Config) error {
+	h3ln, err := caddy.ListenQUIC(hostport, tlsCfg, &s.activeRequests)
+	if err != nil {
+		return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err)
+	}
+
+	// create HTTP/3 server if not done already
+	if s.h3server == nil {
+		s.h3server = &http3.Server{
+			Handler:        s,
+			TLSConfig:      tlsCfg,
+			MaxHeaderBytes: s.MaxHeaderBytes,
+		}
+	}
+
+	//nolint:errcheck
+	go s.h3server.ServeListener(h3ln)
+
+	return nil
+}
+
 // HTTPErrorConfig determines how to handle errors
 // from the HTTP handlers.
 type HTTPErrorConfig struct {
@@ -509,6 +556,16 @@ func (s *Server) shouldLogRequest(r *http.Request) bool {
 	return true
 }
 
+// protocol returns true if the protocol proto is configured/enabled.
+func (s *Server) protocol(proto string) bool {
+	for _, p := range s.Protocols {
+		if p == proto {
+			return true
+		}
+	}
+	return false
+}
+
 // ServerLogConfig describes a server's logging configuration. If
 // enabled without customization, all requests to this server are
 // logged to the default logger; logger destinations may be
diff --git a/modules/caddytls/connpolicy.go b/modules/caddytls/connpolicy.go
index 285e9f6..f7b9c46 100644
--- a/modules/caddytls/connpolicy.go
+++ b/modules/caddytls/connpolicy.go
@@ -112,7 +112,7 @@ func (cp ConnectionPolicies) TLSConfig(_ caddy.Context) *tls.Config {
 						continue policyLoop
 					}
 				}
-				return pol.stdTLSConfig, nil
+				return pol.TLSConfig, nil
 			}
 
 			return nil, fmt.Errorf("no server TLS configuration available for ClientHello: %+v", hello)
@@ -156,8 +156,15 @@ type ConnectionPolicy struct {
 	// is no policy configured for the empty SNI value.
 	DefaultSNI string `json:"default_sni,omitempty"`
 
-	matchers     []ConnectionMatcher
-	stdTLSConfig *tls.Config
+	// TLSConfig is the fully-formed, standard lib TLS config
+	// used to serve TLS connections. Provision all
+	// ConnectionPolicies to populate this. It is exported only
+	// so it can be minimally adjusted after provisioning
+	// if necessary (like to adjust NextProtos to disable HTTP/2),
+	// and may be unexported in the future.
+	TLSConfig *tls.Config `json:"-"`
+
+	matchers []ConnectionMatcher
 }
 
 func (p *ConnectionPolicy) buildStandardTLSConfig(ctx caddy.Context) error {
@@ -275,7 +282,7 @@ func (p *ConnectionPolicy) buildStandardTLSConfig(ctx caddy.Context) error {
 
 	setDefaultTLSParams(cfg)
 
-	p.stdTLSConfig = cfg
+	p.TLSConfig = cfg
 
 	return nil
 }