all: Recover from panics in goroutines

6 files changed, 52 insertions, 0 deletions

diff --git a/modules/caddyhttp/reverseproxy/healthchecks.go b/modules/caddyhttp/reverseproxy/healthchecks.go
index 02d8438..96c9be2 100644
--- a/modules/caddyhttp/reverseproxy/healthchecks.go
+++ b/modules/caddyhttp/reverseproxy/healthchecks.go
@@ -19,10 +19,12 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net"
 	"net/http"
 	"net/url"
 	"regexp"
+	"runtime/debug"
 	"strconv"
 	"time"
 
@@ -124,6 +126,11 @@ type CircuitBreaker interface {
 // regular basis and blocks until
 // h.HealthChecks.Active.stopChan is closed.
 func (h *Handler) activeHealthChecker() {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Printf("[PANIC] active health checks: %v\n%s", err, debug.Stack())
+		}
+	}()
 	ticker := time.NewTicker(time.Duration(h.HealthChecks.Active.Interval))
 	h.doActiveHealthCheckForAllHosts()
 	for {
@@ -143,6 +150,11 @@ func (h *Handler) activeHealthChecker() {
 func (h *Handler) doActiveHealthCheckForAllHosts() {
 	for _, upstream := range h.Upstreams {
 		go func(upstream *Upstream) {
+			defer func() {
+				if err := recover(); err != nil {
+					log.Printf("[PANIC] active health check: %v\n%s", err, debug.Stack())
+				}
+			}()
 			networkAddr := upstream.Dial
 			addr, err := caddy.ParseNetworkAddress(networkAddr)
 			if err != nil {
@@ -335,6 +347,11 @@ func (h *Handler) countFailure(upstream *Upstream) {
 
 	// forget it later
 	go func(host Host, failDuration time.Duration) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Printf("[PANIC] health check failure forgetter: %v\n%s", err, debug.Stack())
+			}
+		}()
 		time.Sleep(failDuration)
 		err := host.CountFail(-1)
 		if err != nil {
diff --git a/modules/caddypki/maintain.go b/modules/caddypki/maintain.go
index 2fce0d9..c0b277d 100644
--- a/modules/caddypki/maintain.go
+++ b/modules/caddypki/maintain.go
@@ -17,12 +17,20 @@ package caddypki
 import (
 	"crypto/x509"
 	"fmt"
+	"log"
+	"runtime/debug"
 	"time"
 
 	"go.uber.org/zap"
 )
 
 func (p *PKI) maintenance() {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Printf("[PANIC] PKI maintenance: %v\n%s", err, debug.Stack())
+		}
+	}()
+
 	ticker := time.NewTicker(10 * time.Minute) // TODO: make configurable
 	defer ticker.Stop()
 
diff --git a/modules/caddytls/distributedstek/distributedstek.go b/modules/caddytls/distributedstek/distributedstek.go
index 6fc48a2..807f2bb 100644
--- a/modules/caddytls/distributedstek/distributedstek.go
+++ b/modules/caddytls/distributedstek/distributedstek.go
@@ -28,6 +28,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"runtime/debug"
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
@@ -193,6 +194,11 @@ func (s *Provider) rotateKeys(oldSTEK distributedSTEK) (distributedSTEK, error)
 // rotate rotates keys on a regular basis, sending each updated set of
 // keys down keysChan, until doneChan is closed.
 func (s *Provider) rotate(doneChan <-chan struct{}, keysChan chan<- [][32]byte) {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Printf("[PANIC] distributed STEK rotation: %v\n%s", err, debug.Stack())
+		}
+	}()
 	for {
 		select {
 		case <-s.timer.C:
diff --git a/modules/caddytls/sessiontickets.go b/modules/caddytls/sessiontickets.go
index 258c135..bfc5628 100644
--- a/modules/caddytls/sessiontickets.go
+++ b/modules/caddytls/sessiontickets.go
@@ -20,6 +20,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -118,6 +120,12 @@ func (s *SessionTicketService) start() error {
 // the keys whenever new ones are sent. It reads
 // from keysChan until s.stop() is called.
 func (s *SessionTicketService) stayUpdated() {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Printf("[PANIC] session ticket service: %v\n%s", err, debug.Stack())
+		}
+	}()
+
 	// this call is essential when Initialize()
 	// returns without error, because the stop
 	// channel is the only way the key source
diff --git a/modules/caddytls/standardstek/stek.go b/modules/caddytls/standardstek/stek.go
index eb609ca..61cab0e 100644
--- a/modules/caddytls/standardstek/stek.go
+++ b/modules/caddytls/standardstek/stek.go
@@ -16,6 +16,7 @@ package standardstek
 
 import (
 	"log"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -81,6 +82,11 @@ func (s *standardSTEKProvider) Next(doneChan <-chan struct{}) <-chan [][32]byte
 // rotate rotates keys on a regular basis, sending each updated set of
 // keys down keysChan, until doneChan is closed.
 func (s *standardSTEKProvider) rotate(doneChan <-chan struct{}, keysChan chan<- [][32]byte) {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Printf("[PANIC] standard STEK rotation: %v\n%s", err, debug.Stack())
+		}
+	}()
 	for {
 		select {
 		case now := <-s.timer.C:
diff --git a/modules/caddytls/tls.go b/modules/caddytls/tls.go
index 0e92f05..7f2d23e 100644
--- a/modules/caddytls/tls.go
+++ b/modules/caddytls/tls.go
@@ -19,9 +19,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
 	"path/filepath"
+	"runtime/debug"
 	"strings"
 	"sync"
 	"time"
@@ -367,6 +369,11 @@ func (t *TLS) keepStorageClean() {
 	t.storageCleanTicker = time.NewTicker(storageCleanInterval)
 	t.storageCleanStop = make(chan struct{})
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Printf("[PANIC] storage cleaner: %v\n%s", err, debug.Stack())
+			}
+		}()
 		for {
 			select {
 			case <-t.storageCleanStop: