From e3e8aabbcf65d37516bb97f9dc0f77df52f8cf55 Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Wed, 28 Sep 2022 13:35:51 -0600 Subject: core: Refactor and improve listener logic (#5089) * core: Refactor, improve listener logic Deprecate: - caddy.Listen - caddy.ListenTimeout - caddy.ListenPacket Prefer caddy.NetworkAddress.Listen() instead. Change: - caddy.ListenQUIC (hopefully to remove later) - caddy.ListenerFunc signature (add context and ListenConfig) - Don't emit Alt-Svc header advertising h3 over HTTP/3 - Use quic.ListenEarly instead of quic.ListenEarlyAddr; this gives us more flexibility (e.g. possibility of HTTP/3 over UDS) but also introduces a new issue: https://github.com/lucas-clemente/quic-go/issues/3560#issuecomment-1258959608 - Unlink unix socket before and after use * Appease the linter * Keep ListenAll --- modules/caddyhttp/app.go | 48 +++++++++++++++++++++++++++++++++++-------- modules/caddyhttp/matchers.go | 4 +++- modules/caddyhttp/server.go | 40 ++++++++++++++++++++++++++++-------- 3 files changed, 74 insertions(+), 18 deletions(-) (limited to 'modules') diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go index c9a5543..33d96d8 100644 --- a/modules/caddyhttp/app.go +++ b/modules/caddyhttp/app.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "fmt" + "net" "net/http" "strconv" "sync" @@ -387,10 +388,11 @@ func (app *App) Start() error { for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ { // create the listener for this socket hostport := listenAddr.JoinHostPort(portOffset) - ln, err := caddy.ListenTimeout(listenAddr.Network, hostport, time.Duration(srv.KeepAliveInterval)) + lnAny, err := listenAddr.Listen(app.ctx, portOffset, net.ListenConfig{KeepAlive: time.Duration(srv.KeepAliveInterval)}) if err != nil { - return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err) + return fmt.Errorf("listening on %s: %v", listenAddr.At(portOffset), err) } + ln := lnAny.(net.Listener) // wrap listener before TLS (up to the TLS placeholder wrapper) var lnWrapperIdx int @@ -409,10 +411,27 @@ func (app *App) Start() error { ln = tls.NewListener(ln, tlsCfg) // enable HTTP/3 if configured - if srv.protocol("h3") && !listenAddr.IsUnixNetwork() { - app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport)) - if err := srv.serveHTTP3(hostport, tlsCfg); err != nil { - return err + if srv.protocol("h3") { + // Can't serve HTTP/3 on the same socket as HTTP/1 and 2 because it uses + // a different transport mechanism... which is fine, but the OS doesn't + // differentiate between a SOCK_STREAM file and a SOCK_DGRAM file; they + // are still one file on the system. So even though "unixpacket" and + // "unixgram" are different network types just as "tcp" and "udp" are, + // the OS will not let us use the same file as both STREAM and DGRAM. + if len(srv.Protocols) > 1 && listenAddr.IsUnixNetwork() { + app.logger.Warn("HTTP/3 disabled because Unix can't multiplex STREAM and DGRAM on same socket", + zap.String("file", hostport)) + for i := range srv.Protocols { + if srv.Protocols[i] == "h3" { + srv.Protocols = append(srv.Protocols[:i], srv.Protocols[i+1:]...) + break + } + } + } else { + app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport)) + if err := srv.serveHTTP3(listenAddr.At(portOffset), tlsCfg); err != nil { + return err + } } } } @@ -424,11 +443,10 @@ func (app *App) Start() error { // if binding to port 0, the OS chooses a port for us; // but the user won't know the port unless we print it - if listenAddr.StartPort == 0 && listenAddr.EndPort == 0 { + if !listenAddr.IsUnixNetwork() && listenAddr.StartPort == 0 && listenAddr.EndPort == 0 { app.logger.Info("port 0 listener", zap.String("input_address", lnAddr), - zap.String("actual_address", ln.Addr().String()), - ) + zap.String("actual_address", ln.Addr().String())) } app.logger.Debug("starting server loop", @@ -533,6 +551,18 @@ func (app *App) Stop() error { if server.h3server == nil { return } + + // TODO: we have to manually close our listeners because quic-go won't + // close listeners it didn't create along with the server itself... + // see https://github.com/lucas-clemente/quic-go/issues/3560 + for _, el := range server.h3listeners { + if err := el.Close(); err != nil { + app.logger.Error("HTTP/3 listener close", + zap.Error(err), + zap.String("address", el.LocalAddr().String())) + } + } + // TODO: CloseGracefully, once implemented upstream (see https://github.com/lucas-clemente/quic-go/issues/2103) if err := server.h3server.Close(); err != nil { app.logger.Error("HTTP/3 server shutdown", diff --git a/modules/caddyhttp/matchers.go b/modules/caddyhttp/matchers.go index bca94be..e39ba3f 100644 --- a/modules/caddyhttp/matchers.go +++ b/modules/caddyhttp/matchers.go @@ -156,7 +156,9 @@ type ( MatchHeaderRE map[string]*MatchRegexp // MatchProtocol matches requests by protocol. Recognized values are - // "http", "https", and "grpc". + // "http", "https", and "grpc" for broad protocol matches, or specific + // HTTP versions can be specified like so: "http/1", "http/1.1", + // "http/2", "http/3", or minimum versions: "http/2+", etc. MatchProtocol string // MatchRemoteIP matches requests by client IP (or CIDR range). diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index 5daa5b2..4d47d26 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -170,9 +170,10 @@ type Server struct { errorLogger *zap.Logger ctx caddy.Context - server *http.Server - h3server *http3.Server - addresses []caddy.NetworkAddress + server *http.Server + h3server *http3.Server + h3listeners []net.PacketConn // TODO: we have to hold these because quic-go won't close listeners it didn't create + addresses []caddy.NetworkAddress shutdownAt time.Time shutdownAtMu *sync.RWMutex @@ -193,9 +194,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { atomic.AddInt64(&s.activeRequests, 1) defer atomic.AddInt64(&s.activeRequests, -1) - err := s.h3server.SetQuicHeaders(w.Header()) - if err != nil { - s.logger.Error("setting HTTP/3 Alt-Svc header", zap.Error(err)) + if r.ProtoMajor < 3 { + err := s.h3server.SetQuicHeaders(w.Header()) + if err != nil { + s.logger.Error("setting HTTP/3 Alt-Svc header", zap.Error(err)) + } } } @@ -493,8 +496,27 @@ func (s *Server) findLastRouteWithHostMatcher() int { // serveHTTP3 creates a QUIC listener, configures an HTTP/3 server if // not already done, and then uses that server to serve HTTP/3 over // the listener, with Server s as the handler. -func (s *Server) serveHTTP3(hostport string, tlsCfg *tls.Config) error { - h3ln, err := caddy.ListenQUIC(hostport, tlsCfg, &s.activeRequests) +func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error { + switch addr.Network { + case "unix": + addr.Network = "unixgram" + case "tcp": + addr.Network = "udp" + case "tcp4": + addr.Network = "udp4" + case "tcp6": + addr.Network = "udp6" + default: + return fmt.Errorf("unsure what network to use for HTTP/3 given network type: %s", addr.Network) + } + + lnAny, err := addr.Listen(s.ctx, 0, net.ListenConfig{}) + if err != nil { + return err + } + ln := lnAny.(net.PacketConn) + + h3ln, err := caddy.ListenQUIC(ln, tlsCfg, &s.activeRequests) if err != nil { return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err) } @@ -512,6 +534,8 @@ func (s *Server) serveHTTP3(hostport string, tlsCfg *tls.Config) error { } } + s.h3listeners = append(s.h3listeners, lnAny.(net.PacketConn)) + //nolint:errcheck go s.h3server.ServeListener(h3ln) -- cgit v1.2.3