From 99d47050e97a8ccac2aad3bda46be46d4fec85ed Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Thu, 2 Mar 2023 21:00:18 -0700 Subject: core: Eliminate unnecessary shutdown delay on Unix (#5413) * core: Eliminate unnecessary shutdown delay on Unix Fix #5393, alternate to #5405 * Comments, cleanup, adjust logs * Fix build constraint --- listen_unix.go | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) (limited to 'listen_unix.go') diff --git a/listen_unix.go b/listen_unix.go index 7ea6745..50d9a66 100644 --- a/listen_unix.go +++ b/listen_unix.go @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// TODO: Go 1.19 introduced the "unix" build tag. We have to support Go 1.18 until Go 1.20 is released. -// When Go 1.19 is our minimum, remove this build tag, since "_unix" in the filename will do this. -// (see also change needed in listen.go) -//go:build aix || android || darwin || dragonfly || freebsd || hurd || illumos || ios || linux || netbsd || openbsd || solaris +// Even though the filename ends in _unix.go, we still have to specify the +// build constraint here, because the filename convention only works for +// literal GOOS values, and "unix" is a shortcut unique to build tags. +//go:build unix package caddy @@ -98,7 +98,19 @@ func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, } return reusePort(network, address, c) } - return config.Listen(ctx, network, address) + + // even though SO_REUSEPORT lets us bind the socket multiple times, + // we still put it in the listenerPool so we can count how many + // configs are using this socket; necessary to ensure we can know + // whether to enforce shutdown delays, for example (see #5393). + ln, err := config.Listen(ctx, network, address) + if err == nil { + listenerPool.LoadOrStore(lnKey, nil) + } + + // lightly wrap the listener so that when it is closed, + // we can decrement the usage pool counter + return deleteListener{ln, lnKey}, err } // reusePort sets SO_REUSEPORT. Ineffective for unix sockets. @@ -116,3 +128,17 @@ func reusePort(network, address string, conn syscall.RawConn) error { } }) } + +// deleteListener is a type that simply deletes itself +// from the listenerPool when it closes. It is used +// solely for the purpose of reference counting (i.e. +// counting how many configs are using a given socket). +type deleteListener struct { + net.Listener + lnKey string +} + +func (dl deleteListener) Close() error { + _, _ = listenerPool.Delete(dl.lnKey) + return dl.Listener.Close() +} -- cgit v1.2.3 From 942fbb37ec46f1907e57a04c44ca790bd47ed52c Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 23 May 2023 17:56:00 +0100 Subject: core: Use SO_REUSEPORT_LB on FreeBSD (#5554) to balance load between threads. --- listen_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'listen_unix.go') diff --git a/listen_unix.go b/listen_unix.go index 50d9a66..7011a1e 100644 --- a/listen_unix.go +++ b/listen_unix.go @@ -119,7 +119,7 @@ func reusePort(network, address string, conn syscall.RawConn) error { return nil } return conn.Control(func(descriptor uintptr) { - if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil { + if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unixSOREUSEPORT, 1); err != nil { Log().Error("setting SO_REUSEPORT", zap.String("network", network), zap.String("address", address), -- cgit v1.2.3 From 9563666bfb93f1708b044c7b1e5d0aa91afd029a Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 21 Jun 2023 17:47:23 -0600 Subject: Fix compile on Windows, hopefully --- listen_unix.go | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'listen_unix.go') diff --git a/listen_unix.go b/listen_unix.go index 7011a1e..e31ecac 100644 --- a/listen_unix.go +++ b/listen_unix.go @@ -108,6 +108,15 @@ func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, listenerPool.LoadOrStore(lnKey, nil) } + // if new listener is a unix socket, make sure we can reuse it later + // (we do our own "unlink on close" -- not required, but more tidy) + one := int32(1) + if unix, ok := ln.(*net.UnixListener); ok { + unix.SetUnlinkOnClose(false) + ln = &unixListener{unix, lnKey, &one} + unixSockets[lnKey] = ln.(*unixListener) + } + // lightly wrap the listener so that when it is closed, // we can decrement the usage pool counter return deleteListener{ln, lnKey}, err -- cgit v1.2.3 From 2b2addebb8cc5dfc7d4d1d34c82f3c46159fea12 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 21 Jun 2023 17:59:54 -0600 Subject: Appease linter --- listen_unix.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'listen_unix.go') diff --git a/listen_unix.go b/listen_unix.go index e31ecac..8870da5 100644 --- a/listen_unix.go +++ b/listen_unix.go @@ -138,6 +138,26 @@ func reusePort(network, address string, conn syscall.RawConn) error { }) } +type unixListener struct { + *net.UnixListener + mapKey string + count *int32 // accessed atomically +} + +func (uln *unixListener) Close() error { + newCount := atomic.AddInt32(uln.count, -1) + if newCount == 0 { + defer func() { + addr := uln.Addr().String() + unixSocketsMu.Lock() + delete(unixSockets, uln.mapKey) + unixSocketsMu.Unlock() + _ = syscall.Unlink(addr) + }() + } + return uln.UnixListener.Close() +} + // deleteListener is a type that simply deletes itself // from the listenerPool when it closes. It is used // solely for the purpose of reference counting (i.e. -- cgit v1.2.3