v2: Logging! (#2831)

* logging: Initial implementation

* logging: More encoder formats, better defaults

* logging: Fix repetition bug with FilterEncoder; add more presets

* logging: DiscardWriter; delete or no-op logs that discard their output

* logging: Add http.handlers.log module; enhance Replacer methods

The Replacer interface has new methods to customize how to handle empty
or unrecognized placeholders. Closes #2815.

* logging: Overhaul HTTP logging, fix bugs, improve filtering, etc.

* logging: General cleanup, begin transitioning to using new loggers

* Fixes after merge conflict

1 files changed, 321 insertions, 0 deletions

diff --git a/modules/logging/filterencoder.go b/modules/logging/filterencoder.go
new file mode 100644
index 0000000..eff0279
--- /dev/null
+++ b/modules/logging/filterencoder.go
@@ -0,0 +1,321 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logging
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
+	"go.uber.org/zap/buffer"
+	"go.uber.org/zap/zapcore"
+)
+
+func init() {
+	caddy.RegisterModule(FilterEncoder{})
+}
+
+// FilterEncoder wraps an underlying encoder. It does
+// not do any encoding itself, but it can manipulate
+// (filter) fields before they are actually encoded.
+// A wrapped encoder is required.
+type FilterEncoder struct {
+	WrappedRaw json.RawMessage            `json:"wrap,omitempty"`
+	FieldsRaw  map[string]json.RawMessage `json:"fields,omitempty"`
+
+	wrapped zapcore.Encoder
+	Fields  map[string]LogFieldFilter `json:"-"`
+
+	// used to keep keys unique across nested objects
+	keyPrefix string
+}
+
+// CaddyModule returns the Caddy module information.
+func (FilterEncoder) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		Name: "caddy.logging.encoders.filter",
+		New:  func() caddy.Module { return new(FilterEncoder) },
+	}
+}
+
+// Provision sets up the encoder.
+func (fe *FilterEncoder) Provision(ctx caddy.Context) error {
+	if fe.WrappedRaw == nil {
+		return fmt.Errorf("missing \"wrap\" (must specify an underlying encoder)")
+	}
+
+	// set up wrapped encoder (required)
+	val, err := ctx.LoadModuleInline("format", "caddy.logging.encoders", fe.WrappedRaw)
+	if err != nil {
+		return fmt.Errorf("loading fallback encoder module: %v", err)
+	}
+	fe.WrappedRaw = nil // allow GC to deallocate
+	fe.wrapped = val.(zapcore.Encoder)
+
+	// set up each field filter
+	if fe.Fields == nil {
+		fe.Fields = make(map[string]LogFieldFilter)
+	}
+	for field, filterRaw := range fe.FieldsRaw {
+		if filterRaw == nil {
+			continue
+		}
+		val, err := ctx.LoadModuleInline("filter", "caddy.logging.encoders.filter", filterRaw)
+		if err != nil {
+			return fmt.Errorf("loading log filter module: %v", err)
+		}
+		fe.Fields[field] = val.(LogFieldFilter)
+	}
+	fe.FieldsRaw = nil // allow GC to deallocate
+
+	return nil
+}
+
+// AddArray is part of the zapcore.ObjectEncoder interface.
+// Array elements do not get filtered.
+func (fe FilterEncoder) AddArray(key string, marshaler zapcore.ArrayMarshaler) error {
+	if filter, ok := fe.Fields[fe.keyPrefix+key]; ok {
+		filter.Filter(zap.Array(key, marshaler)).AddTo(fe.wrapped)
+		return nil
+	}
+	return fe.wrapped.AddArray(key, marshaler)
+}
+
+// AddObject is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddObject(key string, marshaler zapcore.ObjectMarshaler) error {
+	fe.keyPrefix += key + ">"
+	return fe.wrapped.AddObject(key, logObjectMarshalerWrapper{
+		enc:   fe,
+		marsh: marshaler,
+	})
+}
+
+// AddBinary is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddBinary(key string, value []byte) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddBinary(key, value)
+	}
+}
+
+// AddByteString is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddByteString(key string, value []byte) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddByteString(key, value)
+	}
+}
+
+// AddBool is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddBool(key string, value bool) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddBool(key, value)
+	}
+}
+
+// AddComplex128 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddComplex128(key string, value complex128) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddComplex128(key, value)
+	}
+}
+
+// AddComplex64 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddComplex64(key string, value complex64) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddComplex64(key, value)
+	}
+}
+
+// AddDuration is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddDuration(key string, value time.Duration) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddDuration(key, value)
+	}
+}
+
+// AddFloat64 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddFloat64(key string, value float64) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddFloat64(key, value)
+	}
+}
+
+// AddFloat32 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddFloat32(key string, value float32) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddFloat32(key, value)
+	}
+}
+
+// AddInt is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddInt(key string, value int) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddInt(key, value)
+	}
+}
+
+// AddInt64 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddInt64(key string, value int64) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddInt64(key, value)
+	}
+}
+
+// AddInt32 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddInt32(key string, value int32) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddInt32(key, value)
+	}
+}
+
+// AddInt16 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddInt16(key string, value int16) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddInt16(key, value)
+	}
+}
+
+// AddInt8 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddInt8(key string, value int8) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddInt8(key, value)
+	}
+}
+
+// AddString is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddString(key, value string) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddString(key, value)
+	}
+}
+
+// AddTime is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddTime(key string, value time.Time) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddTime(key, value)
+	}
+}
+
+// AddUint is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddUint(key string, value uint) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddUint(key, value)
+	}
+}
+
+// AddUint64 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddUint64(key string, value uint64) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddUint64(key, value)
+	}
+}
+
+// AddUint32 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddUint32(key string, value uint32) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddUint32(key, value)
+	}
+}
+
+// AddUint16 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddUint16(key string, value uint16) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddUint16(key, value)
+	}
+}
+
+// AddUint8 is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddUint8(key string, value uint8) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddUint8(key, value)
+	}
+}
+
+// AddUintptr is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddUintptr(key string, value uintptr) {
+	if !fe.filtered(key, value) {
+		fe.wrapped.AddUintptr(key, value)
+	}
+}
+
+// AddReflected is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) AddReflected(key string, value interface{}) error {
+	if !fe.filtered(key, value) {
+		return fe.wrapped.AddReflected(key, value)
+	}
+	return nil
+}
+
+// OpenNamespace is part of the zapcore.ObjectEncoder interface.
+func (fe FilterEncoder) OpenNamespace(key string) {
+	fe.wrapped.OpenNamespace(key)
+}
+
+// Clone is part of the zapcore.ObjectEncoder interface.
+// We don't use it as of Oct 2019 (v2 beta 7), I'm not
+// really sure what it'd be useful for in our case.
+func (fe FilterEncoder) Clone() zapcore.Encoder {
+	return FilterEncoder{
+		Fields:    fe.Fields,
+		wrapped:   fe.wrapped.Clone(),
+		keyPrefix: fe.keyPrefix,
+	}
+}
+
+// EncodeEntry partially implements the zapcore.Encoder interface.
+func (fe FilterEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) {
+	// without this clone and storing it to fe.wrapped, fields
+	// from subsequent log entries get appended to previous
+	// ones, and I'm not 100% sure why; see end of
+	// https://github.com/uber-go/zap/issues/750
+	fe.wrapped = fe.wrapped.Clone()
+	for _, field := range fields {
+		field.AddTo(fe)
+	}
+	return fe.wrapped.EncodeEntry(ent, nil)
+}
+
+// filtered returns true if the field was filtered.
+// If true is returned, the field was filtered and
+// added to the underlying encoder (so do not do
+// that again). If false was returned, the field has
+// not yet been added to the underlying encoder.
+func (fe FilterEncoder) filtered(key string, value interface{}) bool {
+	filter, ok := fe.Fields[fe.keyPrefix+key]
+	if !ok {
+		return false
+	}
+	filter.Filter(zap.Any(key, value)).AddTo(fe.wrapped)
+	return true
+}
+
+// logObjectMarshalerWrapper allows us to recursively
+// filter fields of objects as they get encoded.
+type logObjectMarshalerWrapper struct {
+	enc   FilterEncoder
+	marsh zapcore.ObjectMarshaler
+}
+
+// MarshalLogObject implements the zapcore.ObjectMarshaler interface.
+func (mom logObjectMarshalerWrapper) MarshalLogObject(_ zapcore.ObjectEncoder) error {
+	return mom.marsh.MarshalLogObject(mom.enc)
+}
+
+// Interface guards
+var (
+	_ zapcore.Encoder         = (*FilterEncoder)(nil)
+	_ zapcore.ObjectMarshaler = (*logObjectMarshalerWrapper)(nil)
+)