From e3e8aabbcf65d37516bb97f9dc0f77df52f8cf55 Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Wed, 28 Sep 2022 13:35:51 -0600 Subject: core: Refactor and improve listener logic (#5089) * core: Refactor, improve listener logic Deprecate: - caddy.Listen - caddy.ListenTimeout - caddy.ListenPacket Prefer caddy.NetworkAddress.Listen() instead. Change: - caddy.ListenQUIC (hopefully to remove later) - caddy.ListenerFunc signature (add context and ListenConfig) - Don't emit Alt-Svc header advertising h3 over HTTP/3 - Use quic.ListenEarly instead of quic.ListenEarlyAddr; this gives us more flexibility (e.g. possibility of HTTP/3 over UDS) but also introduces a new issue: https://github.com/lucas-clemente/quic-go/issues/3560#issuecomment-1258959608 - Unlink unix socket before and after use * Appease the linter * Keep ListenAll --- listen_unix.go | 103 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 50 deletions(-) (limited to 'listen_unix.go') diff --git a/listen_unix.go b/listen_unix.go index f7b6279..dc955d8 100644 --- a/listen_unix.go +++ b/listen_unix.go @@ -24,78 +24,88 @@ import ( "errors" "io/fs" "net" - "sync" + "sync/atomic" "syscall" - "time" "go.uber.org/zap" "golang.org/x/sys/unix" ) -// ListenTimeout is the same as Listen, but with a configurable keep-alive timeout duration. -func ListenTimeout(network, addr string, keepalivePeriod time.Duration) (net.Listener, error) { - // check to see if plugin provides listener - if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil { - return ln, err +// reuseUnixSocket copies and reuses the unix domain socket (UDS) if we already +// have it open; if not, unlink it so we can have it. No-op if not a unix network. +func reuseUnixSocket(network, addr string) (any, error) { + if !isUnixNetwork(network) { + return nil, nil } socketKey := listenerKey(network, addr) - if isUnixNetwork(network) { - unixSocketsMu.Lock() - defer unixSocketsMu.Unlock() - socket, exists := unixSockets[socketKey] - if exists { - // make copy of file descriptor - socketFile, err := socket.File() // dup() deep down + socket, exists := unixSockets[socketKey] + if exists { + // make copy of file descriptor + socketFile, err := socket.File() // does dup() deep down + if err != nil { + return nil, err + } + + // use copied fd to make new Listener or PacketConn, then replace + // it in the map so that future copies always come from the most + // recent fd (as the previous ones will be closed, and we'd get + // "use of closed network connection" errors) -- note that we + // preserve the *pointer* to the counter (not just the value) so + // that all socket wrappers will refer to the same value + switch unixSocket := socket.(type) { + case *unixListener: + ln, err := net.FileListener(socketFile) if err != nil { return nil, err } + atomic.AddInt32(unixSocket.count, 1) + unixSockets[socketKey] = &unixListener{ln.(*net.UnixListener), socketKey, unixSocket.count} - // use copy to make new listener - ln, err := net.FileListener(socketFile) + case *unixConn: + pc, err := net.FilePacketConn(socketFile) if err != nil { return nil, err } - - // the old socket fd will likely be closed soon, so replace it in the map - unixSockets[socketKey] = ln.(*net.UnixListener) - - return ln.(*net.UnixListener), nil + atomic.AddInt32(unixSocket.count, 1) + unixSockets[socketKey] = &unixConn{pc.(*net.UnixConn), addr, socketKey, unixSocket.count} } - // from what I can tell after some quick research, it's quite common for programs to - // leave their socket file behind after they close, so the typical pattern is to - // unlink it before you bind to it -- this is often crucial if the last program using - // it was killed forcefully without a chance to clean up the socket, but there is a - // race, as the comment in net.UnixListener.close() explains... oh well? - if err := syscall.Unlink(addr); err != nil && !errors.Is(err, fs.ErrNotExist) { - return nil, err - } + return unixSockets[socketKey], nil } - config := &net.ListenConfig{Control: reusePort, KeepAlive: keepalivePeriod} - - ln, err := config.Listen(context.Background(), network, addr) - if err != nil { + // from what I can tell after some quick research, it's quite common for programs to + // leave their socket file behind after they close, so the typical pattern is to + // unlink it before you bind to it -- this is often crucial if the last program using + // it was killed forcefully without a chance to clean up the socket, but there is a + // race, as the comment in net.UnixListener.close() explains... oh well, I guess? + if err := syscall.Unlink(addr); err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } - if uln, ok := ln.(*net.UnixListener); ok { - // TODO: ideally, we should unlink the socket once we know we're done using it - // (i.e. either on exit or a new config that doesn't use this socket; in UsagePool - // terms, when the reference count reaches 0), but given that we unlink existing - // socket before we create the new one anyway (see above), we don't necessarily - // need to clean up after ourselves; still, doing so would probably be more tidy - uln.SetUnlinkOnClose(false) - unixSockets[socketKey] = uln - } + return nil, nil +} - return ln, nil +func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, config net.ListenConfig) (net.Listener, error) { + // wrap any Control function set by the user so we can also add our reusePort control without clobbering theirs + oldControl := config.Control + config.Control = func(network, address string, c syscall.RawConn) error { + if oldControl != nil { + if err := oldControl(network, address, c); err != nil { + return err + } + } + return reusePort(network, address, c) + } + return config.Listen(ctx, network, address) } // reusePort sets SO_REUSEPORT. Ineffective for unix sockets. func reusePort(network, address string, conn syscall.RawConn) error { + if isUnixNetwork(network) { + return nil + } return conn.Control(func(descriptor uintptr) { if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil { Log().Error("setting SO_REUSEPORT", @@ -106,10 +116,3 @@ func reusePort(network, address string, conn syscall.RawConn) error { } }) } - -// unixSockets keeps track of the currently-active unix sockets -// so we can transfer their FDs gracefully during reloads. -var ( - unixSockets = make(map[string]*net.UnixListener) - unixSocketsMu sync.Mutex -) -- cgit v1.2.3