From d3c3fa10bd72029720969cff0c29e2b79a1b2cdf Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Fri, 2 Sep 2022 16:59:11 -0600 Subject: core: Refactor listeners; use SO_REUSEPORT on Unix (#4705) * core: Refactor listeners; use SO_REUSEPORT on Unix Just an experiment for now * Fix lint by logging error * TCP Keepalive configuration (#4865) * initial attempt at TCP Keepalive configuration * core: implement tcp-keepalive for linux * move canSetKeepAlive interface * Godoc for keepalive server parameter * handle return values * log keepalive errors * Clean up after bad merge * Merge in pluggable network types From 1edc1a45e3aee1f7d86b68c3ddaf2fd16ba8ab73 * Slight refactor, fix from recent merge conflict Co-authored-by: Karmanyaah Malhotra --- listeners.go | 148 +++++------------------------------------------------------ 1 file changed, 11 insertions(+), 137 deletions(-) (limited to 'listeners.go') diff --git a/listeners.go b/listeners.go index a95c98c..5914443 100644 --- a/listeners.go +++ b/listeners.go @@ -24,10 +24,8 @@ import ( "os" "strconv" "strings" - "sync" "sync/atomic" "syscall" - "time" "github.com/lucas-clemente/quic-go" "github.com/lucas-clemente/quic-go/http3" @@ -43,6 +41,14 @@ import ( // the socket have been finished. Always be sure to close listeners // when you are done with them, just like normal listeners. func Listen(network, addr string) (net.Listener, error) { + // a 0 timeout means Go uses its default + return ListenTimeout(network, addr, 0) +} + +// getListenerFromPlugin returns a listener on the given network and address +// if a plugin has registered the network name. It may return (nil, nil) if +// no plugin can provide a listener. +func getListenerFromPlugin(network, addr string) (net.Listener, error) { network = strings.TrimSpace(strings.ToLower(network)) // get listener from plugin if network type is registered @@ -51,24 +57,7 @@ func Listen(network, addr string) (net.Listener, error) { return getListener(network, addr) } - lnKey := listenerKey(network, addr) - - sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) { - ln, err := net.Listen(network, addr) - if err != nil { - // https://github.com/caddyserver/caddy/pull/4534 - if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) { - return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err) - } - return nil, err - } - return &sharedListener{Listener: ln, key: lnKey}, nil - }) - if err != nil { - return nil, err - } - - return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener)}, nil + return nil, nil } // ListenPacket returns a net.PacketConn suitable for use in a Caddy module. @@ -124,80 +113,14 @@ func ListenQUIC(addr string, tlsConf *tls.Config, activeRequests *int64) (quic.E }, err } -func listenerKey(network, addr string) string { - return network + "/" + addr -} - // ListenerUsage returns the current usage count of the given listener address. func ListenerUsage(network, addr string) int { count, _ := listenerPool.References(listenerKey(network, addr)) return count } -// fakeCloseListener is a private wrapper over a listener that -// is shared. The state of fakeCloseListener is not shared. -// This allows one user of a socket to "close" the listener -// while in reality the socket stays open for other users of -// the listener. In this way, servers become hot-swappable -// while the listener remains running. Listeners should be -// re-wrapped in a new fakeCloseListener each time the listener -// is reused. This type is atomic and values must not be copied. -type fakeCloseListener struct { - closed int32 // accessed atomically; belongs to this struct only - *sharedListener // embedded, so we also become a net.Listener -} - -func (fcl *fakeCloseListener) Accept() (net.Conn, error) { - // if the listener is already "closed", return error - if atomic.LoadInt32(&fcl.closed) == 1 { - return nil, fakeClosedErr(fcl) - } - - // call underlying accept - conn, err := fcl.sharedListener.Accept() - if err == nil { - return conn, nil - } - - // since Accept() returned an error, it may be because our reference to - // the listener (this fakeCloseListener) may have been closed, i.e. the - // server is shutting down; in that case, we need to clear the deadline - // that we set when Close() was called, and return a non-temporary and - // non-timeout error value to the caller, masking the "true" error, so - // that server loops / goroutines won't retry, linger, and leak - if atomic.LoadInt32(&fcl.closed) == 1 { - // we dereference the sharedListener explicitly even though it's embedded - // so that it's clear in the code that side-effects are shared with other - // users of this listener, not just our own reference to it; we also don't - // do anything with the error because all we could do is log it, but we - // expliclty assign it to nothing so we don't forget it's there if needed - _ = fcl.sharedListener.clearDeadline() - - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - return nil, fakeClosedErr(fcl) - } - } - - return nil, err -} - -// Close stops accepting new connections without closing the -// underlying listener. The underlying listener is only closed -// if the caller is the last known user of the socket. -func (fcl *fakeCloseListener) Close() error { - if atomic.CompareAndSwapInt32(&fcl.closed, 0, 1) { - // There are two ways I know of to get an Accept() - // function to return to the server loop that called - // it: close the listener, or set a deadline in the - // past. Obviously, we can't close the socket yet - // since others may be using it (hence this whole - // file). But we can set the deadline in the past, - // and this is kind of cheating, but it works, and - // it apparently even works on Windows. - _ = fcl.sharedListener.setDeadline() - _, _ = listenerPool.Delete(fcl.sharedListener.key) - } - return nil +func listenerKey(network, addr string) string { + return network + "/" + addr } type fakeCloseQuicListener struct { @@ -283,55 +206,6 @@ func (fcpc fakeClosePacketConn) SyscallConn() (syscall.RawConn, error) { return nil, fmt.Errorf("SyscallConn() not implemented for %T", fcpc.PacketConn) } -// sharedListener is a wrapper over an underlying listener. The listener -// and the other fields on the struct are shared state that is synchronized, -// so sharedListener structs must never be copied (always use a pointer). -type sharedListener struct { - net.Listener - key string // uniquely identifies this listener - deadline bool // whether a deadline is currently set - deadlineMu sync.Mutex -} - -func (sl *sharedListener) clearDeadline() error { - var err error - sl.deadlineMu.Lock() - if sl.deadline { - switch ln := sl.Listener.(type) { - case *net.TCPListener: - err = ln.SetDeadline(time.Time{}) - case *net.UnixListener: - err = ln.SetDeadline(time.Time{}) - } - sl.deadline = false - } - sl.deadlineMu.Unlock() - return err -} - -func (sl *sharedListener) setDeadline() error { - timeInPast := time.Now().Add(-1 * time.Minute) - var err error - sl.deadlineMu.Lock() - if !sl.deadline { - switch ln := sl.Listener.(type) { - case *net.TCPListener: - err = ln.SetDeadline(timeInPast) - case *net.UnixListener: - err = ln.SetDeadline(timeInPast) - } - sl.deadline = true - } - sl.deadlineMu.Unlock() - return err -} - -// Destruct is called by the UsagePool when the listener is -// finally not being used anymore. It closes the socket. -func (sl *sharedListener) Destruct() error { - return sl.Listener.Close() -} - // sharedQuicListener is like sharedListener, but for quic.EarlyListeners. type sharedQuicListener struct { quic.EarlyListener -- cgit v1.2.3