From aef560c7fc52092a412d9e97112b8cb879c5eda5 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 12 May 2020 11:36:20 -0600 Subject: all: Recover from panics in goroutines --- modules/caddyhttp/reverseproxy/healthchecks.go | 17 +++++++++++++++++ modules/caddypki/maintain.go | 8 ++++++++ modules/caddytls/distributedstek/distributedstek.go | 6 ++++++ modules/caddytls/sessiontickets.go | 8 ++++++++ modules/caddytls/standardstek/stek.go | 6 ++++++ modules/caddytls/tls.go | 7 +++++++ 6 files changed, 52 insertions(+) (limited to 'modules') diff --git a/modules/caddyhttp/reverseproxy/healthchecks.go b/modules/caddyhttp/reverseproxy/healthchecks.go index 02d8438..96c9be2 100644 --- a/modules/caddyhttp/reverseproxy/healthchecks.go +++ b/modules/caddyhttp/reverseproxy/healthchecks.go @@ -19,10 +19,12 @@ import ( "fmt" "io" "io/ioutil" + "log" "net" "net/http" "net/url" "regexp" + "runtime/debug" "strconv" "time" @@ -124,6 +126,11 @@ type CircuitBreaker interface { // regular basis and blocks until // h.HealthChecks.Active.stopChan is closed. func (h *Handler) activeHealthChecker() { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] active health checks: %v\n%s", err, debug.Stack()) + } + }() ticker := time.NewTicker(time.Duration(h.HealthChecks.Active.Interval)) h.doActiveHealthCheckForAllHosts() for { @@ -143,6 +150,11 @@ func (h *Handler) activeHealthChecker() { func (h *Handler) doActiveHealthCheckForAllHosts() { for _, upstream := range h.Upstreams { go func(upstream *Upstream) { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] active health check: %v\n%s", err, debug.Stack()) + } + }() networkAddr := upstream.Dial addr, err := caddy.ParseNetworkAddress(networkAddr) if err != nil { @@ -335,6 +347,11 @@ func (h *Handler) countFailure(upstream *Upstream) { // forget it later go func(host Host, failDuration time.Duration) { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] health check failure forgetter: %v\n%s", err, debug.Stack()) + } + }() time.Sleep(failDuration) err := host.CountFail(-1) if err != nil { diff --git a/modules/caddypki/maintain.go b/modules/caddypki/maintain.go index 2fce0d9..c0b277d 100644 --- a/modules/caddypki/maintain.go +++ b/modules/caddypki/maintain.go @@ -17,12 +17,20 @@ package caddypki import ( "crypto/x509" "fmt" + "log" + "runtime/debug" "time" "go.uber.org/zap" ) func (p *PKI) maintenance() { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] PKI maintenance: %v\n%s", err, debug.Stack()) + } + }() + ticker := time.NewTicker(10 * time.Minute) // TODO: make configurable defer ticker.Stop() diff --git a/modules/caddytls/distributedstek/distributedstek.go b/modules/caddytls/distributedstek/distributedstek.go index 6fc48a2..807f2bb 100644 --- a/modules/caddytls/distributedstek/distributedstek.go +++ b/modules/caddytls/distributedstek/distributedstek.go @@ -28,6 +28,7 @@ import ( "encoding/json" "fmt" "log" + "runtime/debug" "time" "github.com/caddyserver/caddy/v2" @@ -193,6 +194,11 @@ func (s *Provider) rotateKeys(oldSTEK distributedSTEK) (distributedSTEK, error) // rotate rotates keys on a regular basis, sending each updated set of // keys down keysChan, until doneChan is closed. func (s *Provider) rotate(doneChan <-chan struct{}, keysChan chan<- [][32]byte) { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] distributed STEK rotation: %v\n%s", err, debug.Stack()) + } + }() for { select { case <-s.timer.C: diff --git a/modules/caddytls/sessiontickets.go b/modules/caddytls/sessiontickets.go index 258c135..bfc5628 100644 --- a/modules/caddytls/sessiontickets.go +++ b/modules/caddytls/sessiontickets.go @@ -20,6 +20,8 @@ import ( "encoding/json" "fmt" "io" + "log" + "runtime/debug" "sync" "time" @@ -118,6 +120,12 @@ func (s *SessionTicketService) start() error { // the keys whenever new ones are sent. It reads // from keysChan until s.stop() is called. func (s *SessionTicketService) stayUpdated() { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] session ticket service: %v\n%s", err, debug.Stack()) + } + }() + // this call is essential when Initialize() // returns without error, because the stop // channel is the only way the key source diff --git a/modules/caddytls/standardstek/stek.go b/modules/caddytls/standardstek/stek.go index eb609ca..61cab0e 100644 --- a/modules/caddytls/standardstek/stek.go +++ b/modules/caddytls/standardstek/stek.go @@ -16,6 +16,7 @@ package standardstek import ( "log" + "runtime/debug" "sync" "time" @@ -81,6 +82,11 @@ func (s *standardSTEKProvider) Next(doneChan <-chan struct{}) <-chan [][32]byte // rotate rotates keys on a regular basis, sending each updated set of // keys down keysChan, until doneChan is closed. func (s *standardSTEKProvider) rotate(doneChan <-chan struct{}, keysChan chan<- [][32]byte) { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] standard STEK rotation: %v\n%s", err, debug.Stack()) + } + }() for { select { case now := <-s.timer.C: diff --git a/modules/caddytls/tls.go b/modules/caddytls/tls.go index 0e92f05..7f2d23e 100644 --- a/modules/caddytls/tls.go +++ b/modules/caddytls/tls.go @@ -19,9 +19,11 @@ import ( "encoding/json" "fmt" "io/ioutil" + "log" "net/http" "os" "path/filepath" + "runtime/debug" "strings" "sync" "time" @@ -367,6 +369,11 @@ func (t *TLS) keepStorageClean() { t.storageCleanTicker = time.NewTicker(storageCleanInterval) t.storageCleanStop = make(chan struct{}) go func() { + defer func() { + if err := recover(); err != nil { + log.Printf("[PANIC] storage cleaner: %v\n%s", err, debug.Stack()) + } + }() for { select { case <-t.storageCleanStop: -- cgit v1.2.3