From f3e8b9d95fefc81c347ec38d913c9ccd9edb626a Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 4 May 2023 16:29:03 -0600 Subject: logging: Soft start for net writer (close #5520) If enabled and there is an error when opening the net writer, ignore the error and report it along with subsequent logs to stderr. --- modules/logging/netwriter.go | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) (limited to 'modules/logging') diff --git a/modules/logging/netwriter.go b/modules/logging/netwriter.go index 5a6cf39..954a09b 100644 --- a/modules/logging/netwriter.go +++ b/modules/logging/netwriter.go @@ -40,6 +40,11 @@ type NetWriter struct { // The timeout to wait while connecting to the socket. DialTimeout caddy.Duration `json:"dial_timeout,omitempty"` + // If enabled, allow connections errors when first opening the + // writer. The error and subsequent log entries will be reported + // to stderr instead until a connection can be re-established. + SoftStart bool `json:"soft_start,omitempty"` + addr caddy.NetworkAddress } @@ -92,7 +97,9 @@ func (nw NetWriter) OpenWriter() (io.WriteCloser, error) { } conn, err := reconn.dial() if err != nil { - return nil, err + // don't block config load if remote is down or some other external problem; + // we can dump logs to stderr for now (see issue #5520) + fmt.Fprintf(os.Stderr, "[ERROR] net log writer failed to connect: %v (will retry connection and print errors here in the meantime)\n", err) } reconn.connMu.Lock() reconn.Conn = conn @@ -104,6 +111,7 @@ func (nw NetWriter) OpenWriter() (io.WriteCloser, error) { // // net
{ // dial_timeout +// soft_start // } func (nw *NetWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { for d.Next() { @@ -128,6 +136,12 @@ func (nw *NetWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return d.ArgErr() } nw.DialTimeout = caddy.Duration(timeout) + + case "soft_start": + if d.NextArg() { + return d.ArgErr() + } + nw.SoftStart = true } } } @@ -151,8 +165,10 @@ func (reconn *redialerConn) Write(b []byte) (n int, err error) { reconn.connMu.RLock() conn := reconn.Conn reconn.connMu.RUnlock() - if n, err = conn.Write(b); err == nil { - return + if conn != nil { + if n, err = conn.Write(b); err == nil { + return + } } // problem with the connection - lock it and try to fix it @@ -161,8 +177,10 @@ func (reconn *redialerConn) Write(b []byte) (n int, err error) { // if multiple concurrent writes failed on the same broken conn, then // one of them might have already re-dialed by now; try writing again - if n, err = reconn.Conn.Write(b); err == nil { - return + if reconn.Conn != nil { + if n, err = reconn.Conn.Write(b); err == nil { + return + } } // there's still a problem, so try to re-attempt dialing the socket @@ -178,7 +196,9 @@ func (reconn *redialerConn) Write(b []byte) (n int, err error) { return } if n, err = conn2.Write(b); err == nil { - reconn.Conn.Close() + if reconn.Conn != nil { + reconn.Conn.Close() + } reconn.Conn = conn2 } } else { -- cgit v1.2.3 From cdce452edc5e9cf9127789a868d01864d3276af5 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 4 May 2023 16:30:34 -0600 Subject: logging: Actually honor the SoftStart parameter --- modules/logging/netwriter.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'modules/logging') diff --git a/modules/logging/netwriter.go b/modules/logging/netwriter.go index 954a09b..1939cb7 100644 --- a/modules/logging/netwriter.go +++ b/modules/logging/netwriter.go @@ -97,6 +97,9 @@ func (nw NetWriter) OpenWriter() (io.WriteCloser, error) { } conn, err := reconn.dial() if err != nil { + if !nw.SoftStart { + return nil, err + } // don't block config load if remote is down or some other external problem; // we can dump logs to stderr for now (see issue #5520) fmt.Fprintf(os.Stderr, "[ERROR] net log writer failed to connect: %v (will retry connection and print errors here in the meantime)\n", err) -- cgit v1.2.3 From b32f265ecad60404c3818cc9d42e367a8e4eb7d4 Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Tue, 8 Aug 2023 03:40:31 +0800 Subject: ci: Use gofumpt to format code (#5707) --- modules/logging/filewriter.go | 2 +- modules/logging/filters.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'modules/logging') diff --git a/modules/logging/filewriter.go b/modules/logging/filewriter.go index f902a6d..afc4afc 100644 --- a/modules/logging/filewriter.go +++ b/modules/logging/filewriter.go @@ -126,7 +126,7 @@ func (fw FileWriter) OpenWriter() (io.WriteCloser, error) { } // otherwise just open a regular file - return os.OpenFile(fw.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) + return os.OpenFile(fw.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) } // UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax: diff --git a/modules/logging/filters.go b/modules/logging/filters.go index 8cc84c7..af9be74 100644 --- a/modules/logging/filters.go +++ b/modules/logging/filters.go @@ -81,8 +81,7 @@ func hash(s string) string { // of the SHA-256 hash of the content. Operates // on string fields, or on arrays of strings // where each string is hashed. -type HashFilter struct { -} +type HashFilter struct{} // CaddyModule returns the Caddy module information. func (HashFilter) CaddyModule() caddy.ModuleInfo { -- cgit v1.2.3 From d6f86cccf5fa5b4eb30141da390cf2439746c5da Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Mon, 14 Aug 2023 23:41:15 +0800 Subject: ci: use gci linter (#5708) * use gofmput to format code * use gci to format imports * reconfigure gci * linter autofixes * rearrange imports a little * export GOOS=windows golangci-lint run ./... --fix --- modules/logging/encoders.go | 5 +++-- modules/logging/filewriter.go | 5 +++-- modules/logging/filterencoder.go | 7 ++++--- modules/logging/filters.go | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) (limited to 'modules/logging') diff --git a/modules/logging/encoders.go b/modules/logging/encoders.go index 1cfab8e..a4409e7 100644 --- a/modules/logging/encoders.go +++ b/modules/logging/encoders.go @@ -17,11 +17,12 @@ package logging import ( "time" - "github.com/caddyserver/caddy/v2" - "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "go.uber.org/zap" "go.uber.org/zap/buffer" "go.uber.org/zap/zapcore" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" ) func init() { diff --git a/modules/logging/filewriter.go b/modules/logging/filewriter.go index afc4afc..11c051d 100644 --- a/modules/logging/filewriter.go +++ b/modules/logging/filewriter.go @@ -22,10 +22,11 @@ import ( "path/filepath" "strconv" - "github.com/caddyserver/caddy/v2" - "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/dustin/go-humanize" "gopkg.in/natefinch/lumberjack.v2" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" ) func init() { diff --git a/modules/logging/filterencoder.go b/modules/logging/filterencoder.go index 1a6842e..4d51e64 100644 --- a/modules/logging/filterencoder.go +++ b/modules/logging/filterencoder.go @@ -19,12 +19,13 @@ import ( "fmt" "time" - "github.com/caddyserver/caddy/v2" - "github.com/caddyserver/caddy/v2/caddyconfig" - "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "go.uber.org/zap" "go.uber.org/zap/buffer" "go.uber.org/zap/zapcore" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" ) func init() { diff --git a/modules/logging/filters.go b/modules/logging/filters.go index af9be74..ed4c7af 100644 --- a/modules/logging/filters.go +++ b/modules/logging/filters.go @@ -25,10 +25,11 @@ import ( "strconv" "strings" + "go.uber.org/zap/zapcore" + "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" - "go.uber.org/zap/zapcore" ) func init() { -- cgit v1.2.3 From c46ec3b500774e554ef0612530acc941b8b92bd4 Mon Sep 17 00:00:00 2001 From: Francis Lavoie Date: Tue, 29 Aug 2023 13:41:39 -0400 Subject: logging: Clone array on log filters, prevent side-effects (#5786) Fixes https://caddy.community/t/is-caddy-mutating-header-content-from-logging-settings/20947 --- modules/logging/filters.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'modules/logging') diff --git a/modules/logging/filters.go b/modules/logging/filters.go index ed4c7af..69ba9d1 100644 --- a/modules/logging/filters.go +++ b/modules/logging/filters.go @@ -100,12 +100,15 @@ func (f *HashFilter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // Filter filters the input field with the replacement value. func (f *HashFilter) Filter(in zapcore.Field) zapcore.Field { if array, ok := in.Interface.(caddyhttp.LoggableStringArray); ok { + newArray := make(caddyhttp.LoggableStringArray, len(array)) for i, s := range array { - array[i] = hash(s) + newArray[i] = hash(s) } + in.Interface = newArray } else { in.String = hash(in.String) } + return in } @@ -219,9 +222,11 @@ func (m *IPMaskFilter) Provision(ctx caddy.Context) error { // Filter filters the input field. func (m IPMaskFilter) Filter(in zapcore.Field) zapcore.Field { if array, ok := in.Interface.(caddyhttp.LoggableStringArray); ok { + newArray := make(caddyhttp.LoggableStringArray, len(array)) for i, s := range array { - array[i] = m.mask(s) + newArray[i] = m.mask(s) } + in.Interface = newArray } else { in.String = m.mask(in.String) } @@ -580,9 +585,11 @@ func (m *RegexpFilter) Provision(ctx caddy.Context) error { // Filter filters the input field with the replacement value if it matches the regexp. func (f *RegexpFilter) Filter(in zapcore.Field) zapcore.Field { if array, ok := in.Interface.(caddyhttp.LoggableStringArray); ok { + newArray := make(caddyhttp.LoggableStringArray, len(array)) for i, s := range array { - array[i] = f.regexp.ReplaceAllString(s, f.Value) + newArray[i] = f.regexp.ReplaceAllString(s, f.Value) } + in.Interface = newArray } else { in.String = f.regexp.ReplaceAllString(in.String, f.Value) } -- cgit v1.2.3 From 1b73e3862d312ac2057265bf2a5fd95760dbe9da Mon Sep 17 00:00:00 2001 From: Paul Jeannot Date: Wed, 30 Aug 2023 00:59:43 +0200 Subject: logging: query filter for array of strings (#5779) Co-authored-by: Matt Holt Co-authored-by: Francis Lavoie --- modules/logging/filters.go | 22 +++++++++++++++++----- modules/logging/filters_test.go | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) (limited to 'modules/logging') diff --git a/modules/logging/filters.go b/modules/logging/filters.go index 69ba9d1..233d5d7 100644 --- a/modules/logging/filters.go +++ b/modules/logging/filters.go @@ -373,9 +373,23 @@ func (m *QueryFilter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // Filter filters the input field. func (m QueryFilter) Filter(in zapcore.Field) zapcore.Field { - u, err := url.Parse(in.String) + if array, ok := in.Interface.(caddyhttp.LoggableStringArray); ok { + newArray := make(caddyhttp.LoggableStringArray, len(array)) + for i, s := range array { + newArray[i] = m.processQueryString(s) + } + in.Interface = newArray + } else { + in.String = m.processQueryString(in.String) + } + + return in +} + +func (m QueryFilter) processQueryString(s string) string { + u, err := url.Parse(s) if err != nil { - return in + return s } q := u.Query() @@ -397,9 +411,7 @@ func (m QueryFilter) Filter(in zapcore.Field) zapcore.Field { } u.RawQuery = q.Encode() - in.String = u.String() - - return in + return u.String() } type cookieFilterAction struct { diff --git a/modules/logging/filters_test.go b/modules/logging/filters_test.go index e9c3e77..8f7ba0d 100644 --- a/modules/logging/filters_test.go +++ b/modules/logging/filters_test.go @@ -83,7 +83,7 @@ func TestIPMaskMultiValue(t *testing.T) { } } -func TestQueryFilter(t *testing.T) { +func TestQueryFilterSingleValue(t *testing.T) { f := QueryFilter{[]queryFilterAction{ {replaceAction, "foo", "REDACTED"}, {replaceAction, "notexist", "REDACTED"}, @@ -102,6 +102,40 @@ func TestQueryFilter(t *testing.T) { } } +func TestQueryFilterMultiValue(t *testing.T) { + f := QueryFilter{ + Actions: []queryFilterAction{ + {Type: replaceAction, Parameter: "foo", Value: "REDACTED"}, + {Type: replaceAction, Parameter: "notexist", Value: "REDACTED"}, + {Type: deleteAction, Parameter: "bar"}, + {Type: deleteAction, Parameter: "notexist"}, + {Type: hashAction, Parameter: "hash"}, + }, + } + + if f.Validate() != nil { + t.Fatalf("the filter must be valid") + } + + out := f.Filter(zapcore.Field{Interface: caddyhttp.LoggableStringArray{ + "/path1?foo=a&foo=b&bar=c&bar=d&baz=e&hash=hashed", + "/path2?foo=c&foo=d&bar=e&bar=f&baz=g&hash=hashed", + }}) + arr, ok := out.Interface.(caddyhttp.LoggableStringArray) + if !ok { + t.Fatalf("field is wrong type: %T", out.Interface) + } + + expected1 := "/path1?baz=e&foo=REDACTED&foo=REDACTED&hash=e3b0c442" + expected2 := "/path2?baz=g&foo=REDACTED&foo=REDACTED&hash=e3b0c442" + if arr[0] != expected1 { + t.Fatalf("query parameters in entry 0 have not been filtered correctly: got %s, expected %s", arr[0], expected1) + } + if arr[1] != expected2 { + t.Fatalf("query parameters in entry 1 have not been filtered correctly: got %s, expected %s", arr[1], expected2) + } +} + func TestValidateQueryFilter(t *testing.T) { f := QueryFilter{[]queryFilterAction{ {}, -- cgit v1.2.3