Refactor for CertMagic v0.10; prepare for PKI app

This is a breaking change primarily in two areas:
 - Storage paths for certificates have changed
 - Slight changes to JSON config parameters

Huge improvements in this commit, to be detailed more in
the release notes.

The upcoming PKI app will be powered by Smallstep libraries.

5 files changed, 254 insertions, 33 deletions

diff --git a/caddyconfig/httpcaddyfile/addresses.go b/caddyconfig/httpcaddyfile/addresses.go
index 19ac197..3aedb60 100644
--- a/caddyconfig/httpcaddyfile/addresses.go
+++ b/caddyconfig/httpcaddyfile/addresses.go
@@ -23,7 +23,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/certmagic"
 )
 
 // mapAddressToServerBlocks returns a map of listener address to list of server
diff --git a/caddyconfig/httpcaddyfile/builtins.go b/caddyconfig/httpcaddyfile/builtins.go
index 8cfca18..a085fcb 100644
--- a/caddyconfig/httpcaddyfile/builtins.go
+++ b/caddyconfig/httpcaddyfile/builtins.go
@@ -24,8 +24,10 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
+	"go.uber.org/zap/zapcore"
 )
 
 func init() {
@@ -37,6 +39,7 @@ func init() {
 	RegisterHandlerDirective("route", parseRoute)
 	RegisterHandlerDirective("handle", parseSegmentAsSubroute)
 	RegisterDirective("handle_errors", parseHandleErrors)
+	RegisterDirective("log", parseLog)
 }
 
 // parseBind parses the bind directive. Syntax:
@@ -108,7 +111,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	var cp *caddytls.ConnectionPolicy
 	var fileLoader caddytls.FileLoader
 	var folderLoader caddytls.FolderLoader
-	var mgr caddytls.ACMEManagerMaker
+	var mgr caddytls.ACMEIssuer
 
 	// fill in global defaults, if configured
 	if email := h.Option("email"); email != nil {
@@ -307,9 +310,9 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	}
 
 	// automation policy
-	if !reflect.DeepEqual(mgr, caddytls.ACMEManagerMaker{}) {
+	if !reflect.DeepEqual(mgr, caddytls.ACMEIssuer{}) {
 		configVals = append(configVals, ConfigValue{
-			Class: "tls.automation_manager",
+			Class: "tls.cert_issuer",
 			Value: mgr,
 		})
 	}
@@ -426,9 +429,116 @@ func parseHandleErrors(h Helper) ([]ConfigValue, error) {
 	}, nil
 }
 
+// parseLog parses the log directive. Syntax:
+//
+//     log {
+//         output <writer_module> ...
+//         format <encoder_module> ...
+//         level  <level>
+//     }
+//
+func parseLog(h Helper) ([]ConfigValue, error) {
+	var configValues []ConfigValue
+	for h.Next() {
+		cl := new(caddy.CustomLog)
+
+		for h.NextBlock(0) {
+			switch h.Val() {
+			case "output":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				moduleName := h.Val()
+
+				// can't use the usual caddyfile.Unmarshaler flow with the
+				// standard writers because they are in the caddy package
+				// (because they are the default) and implementing that
+				// interface there would unfortunately create circular import
+				var wo caddy.WriterOpener
+				switch moduleName {
+				case "stdout":
+					wo = caddy.StdoutWriter{}
+				case "stderr":
+					wo = caddy.StderrWriter{}
+				case "discard":
+					wo = caddy.DiscardWriter{}
+				default:
+					mod, err := caddy.GetModule("caddy.logging.writers." + moduleName)
+					if err != nil {
+						return nil, h.Errf("getting log writer module named '%s': %v", moduleName, err)
+					}
+					unm, ok := mod.New().(caddyfile.Unmarshaler)
+					if !ok {
+						return nil, h.Errf("log writer module '%s' is not a Caddyfile unmarshaler", mod)
+					}
+					err = unm.UnmarshalCaddyfile(h.NewFromNextSegment())
+					if err != nil {
+						return nil, err
+					}
+					wo, ok = unm.(caddy.WriterOpener)
+					if !ok {
+						return nil, h.Errf("module %s is not a WriterOpener", mod)
+					}
+				}
+				cl.WriterRaw = caddyconfig.JSONModuleObject(wo, "output", moduleName, h.warnings)
+
+			case "format":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				moduleName := h.Val()
+				mod, err := caddy.GetModule("caddy.logging.encoders." + moduleName)
+				if err != nil {
+					return nil, h.Errf("getting log encoder module named '%s': %v", moduleName, err)
+				}
+				unm, ok := mod.New().(caddyfile.Unmarshaler)
+				if !ok {
+					return nil, h.Errf("log encoder module '%s' is not a Caddyfile unmarshaler", mod)
+				}
+				err = unm.UnmarshalCaddyfile(h.NewFromNextSegment())
+				if err != nil {
+					return nil, err
+				}
+				enc, ok := unm.(zapcore.Encoder)
+				if !ok {
+					return nil, h.Errf("module %s is not a zapcore.Encoder", mod)
+				}
+				cl.EncoderRaw = caddyconfig.JSONModuleObject(enc, "format", moduleName, h.warnings)
+
+			case "level":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				cl.Level = h.Val()
+				if h.NextArg() {
+					return nil, h.ArgErr()
+				}
+
+			default:
+				return nil, h.Errf("unrecognized subdirective: %s", h.Val())
+			}
+		}
+
+		var val namedCustomLog
+		if !reflect.DeepEqual(cl, new(caddy.CustomLog)) {
+			cl.Include = []string{"http.log.access"}
+			val.name = fmt.Sprintf("log%d", logCounter)
+			val.log = cl
+			logCounter++
+		}
+		configValues = append(configValues, ConfigValue{
+			Class: "custom_log",
+			Value: val,
+		})
+	}
+	return configValues, nil
+}
+
 // tlsCertTags maps certificate filenames to their tag.
 // This is used to remember which tag is used for each
 // certificate files, since we need to avoid loading
 // the same certificate files more than once, overwriting
 // previous tags
 var tlsCertTags = make(map[string]string)
+
+var logCounter int
diff --git a/caddyconfig/httpcaddyfile/directives.go b/caddyconfig/httpcaddyfile/directives.go
index 3c03d30..f82e2a8 100644
--- a/caddyconfig/httpcaddyfile/directives.go
+++ b/caddyconfig/httpcaddyfile/directives.go
@@ -17,6 +17,7 @@ package httpcaddyfile
 import (
 	"encoding/json"
 	"sort"
+	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
@@ -298,17 +299,43 @@ func sortRoutes(routes []ConfigValue) {
 // and returned.
 func parseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 	var allResults []ConfigValue
+
 	for h.Next() {
+		// slice the linear list of tokens into top-level segments
+		var segments []caddyfile.Segment
 		for nesting := h.Nesting(); h.NextBlock(nesting); {
-			dir := h.Val()
+			segments = append(segments, h.NextSegment())
+		}
 
+		// copy existing matcher definitions so we can augment
+		// new ones that are defined only in this scope
+		matcherDefs := make(map[string]caddy.ModuleMap, len(h.matcherDefs))
+		for key, val := range h.matcherDefs {
+			matcherDefs[key] = val
+		}
+
+		// find and extract any embedded matcher definitions in this scope
+		for i, seg := range segments {
+			if strings.HasPrefix(seg.Directive(), matcherPrefix) {
+				err := parseMatcherDefinitions(caddyfile.NewDispenser(seg), matcherDefs)
+				if err != nil {
+					return nil, err
+				}
+				segments = append(segments[:i], segments[i+1:]...)
+			}
+		}
+
+		// with matchers ready to go, evaluate each directive's segment
+		for _, seg := range segments {
+			dir := seg.Directive()
 			dirFunc, ok := registeredDirectives[dir]
 			if !ok {
 				return nil, h.Errf("unrecognized directive: %s", dir)
 			}
 
 			subHelper := h
-			subHelper.Dispenser = h.NewFromNextSegment()
+			subHelper.Dispenser = caddyfile.NewDispenser(seg)
+			subHelper.matcherDefs = matcherDefs
 
 			results, err := dirFunc(subHelper)
 			if err != nil {
@@ -319,9 +346,9 @@ func parseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) {
 				allResults = append(allResults, result)
 			}
 		}
-		return buildSubroute(allResults, h.groupCounter) // TODO:  should we move this outside the loop?
 	}
-	return nil, nil
+
+	return buildSubroute(allResults, h.groupCounter)
 }
 
 // serverBlock pairs a Caddyfile server block
diff --git a/caddyconfig/httpcaddyfile/httptype.go b/caddyconfig/httpcaddyfile/httptype.go
index 03234b3..aaec2e9 100644
--- a/caddyconfig/httpcaddyfile/httptype.go
+++ b/caddyconfig/httpcaddyfile/httptype.go
@@ -26,7 +26,7 @@ import (
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
-	"github.com/mholt/certmagic"
+	"github.com/caddyserver/certmagic"
 )
 
 func init() {
@@ -88,6 +88,13 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 			"{remote}", "{http.request.remote}",
 			"{scheme}", "{http.request.scheme}",
 			"{uri}", "{http.request.uri}",
+
+			"{tls_cipher}", "{http.request.tls.cipher_suite}",
+			"{tls_version}", "{http.request.tls.version}",
+			"{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}",
+			"{tls_client_issuer}", "{http.request.tls.client.issuer}",
+			"{tls_client_serial}", "{http.request.tls.client.serial}",
+			"{tls_client_subject}", "{http.request.tls.client.subject}",
 		)
 		for _, segment := range sb.block.Segments {
 			for i := 0; i < len(segment); i++ {
@@ -173,9 +180,9 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 	for _, p := range pairings {
 		for i, sblock := range p.serverBlocks {
 			// tls automation policies
-			if mmVals, ok := sblock.pile["tls.automation_manager"]; ok {
+			if mmVals, ok := sblock.pile["tls.cert_issuer"]; ok {
 				for _, mmVal := range mmVals {
-					mm := mmVal.Value.(caddytls.ManagerMaker)
+					mm := mmVal.Value.(certmagic.Issuer)
 					sblockHosts, err := st.autoHTTPSHosts(sblock)
 					if err != nil {
 						return nil, warnings, err
@@ -185,8 +192,8 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 							tlsApp.Automation = new(caddytls.AutomationConfig)
 						}
 						tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, &caddytls.AutomationPolicy{
-							Hosts:         sblockHosts,
-							ManagementRaw: caddyconfig.JSONModuleObject(mm, "module", mm.(caddy.Module).CaddyModule().ID.Name(), &warnings),
+							Hosts:     sblockHosts,
+							IssuerRaw: caddyconfig.JSONModuleObject(mm, "module", mm.(caddy.Module).CaddyModule().ID.Name(), &warnings),
 						})
 					} else {
 						warnings = append(warnings, caddyconfig.Warning{
@@ -245,7 +252,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 		if !hasEmail {
 			email = ""
 		}
-		mgr := caddytls.ACMEManagerMaker{
+		mgr := caddytls.ACMEIssuer{
 			CA:    acmeCA.(string),
 			Email: email.(string),
 		}
@@ -260,7 +267,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 			}
 		}
 		tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, &caddytls.AutomationPolicy{
-			ManagementRaw: caddyconfig.JSONModuleObject(mgr, "module", "acme", &warnings),
+			IssuerRaw: caddyconfig.JSONModuleObject(mgr, "module", "acme", &warnings),
 		})
 	}
 	if tlsApp.Automation != nil {
@@ -275,6 +282,35 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 		}
 	}
 
+	// extract any custom logs, and enforce configured levels
+	var customLogs []namedCustomLog
+	var hasDefaultLog bool
+	for _, sb := range serverBlocks {
+		for _, clVal := range sb.pile["custom_log"] {
+			ncl := clVal.Value.(namedCustomLog)
+			if ncl.name == "" {
+				continue
+			}
+			if ncl.name == "default" {
+				hasDefaultLog = true
+			}
+			if _, ok := options["debug"]; ok && ncl.log.Level == "" {
+				ncl.log.Level = "DEBUG"
+			}
+			customLogs = append(customLogs, ncl)
+		}
+	}
+	if !hasDefaultLog {
+		// if the default log was not customized, ensure we
+		// configure it with any applicable options
+		if _, ok := options["debug"]; ok {
+			customLogs = append(customLogs, namedCustomLog{
+				name: "default",
+				log:  &caddy.CustomLog{Level: "DEBUG"},
+			})
+		}
+	}
+
 	// annnd the top-level config, then we're done!
 	cfg := &caddy.Config{AppsRaw: make(caddy.ModuleMap)}
 	if !reflect.DeepEqual(httpApp, caddyhttp.App{}) {
@@ -292,6 +328,18 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 	if adminConfig, ok := options["admin"].(string); ok && adminConfig != "" {
 		cfg.Admin = &caddy.AdminConfig{Listen: adminConfig}
 	}
+	if len(customLogs) > 0 {
+		if cfg.Logging == nil {
+			cfg.Logging = &caddy.Logging{
+				Logs: make(map[string]*caddy.CustomLog),
+			}
+		}
+		for _, ncl := range customLogs {
+			if ncl.name != "" {
+				cfg.Logging.Logs[ncl.name] = ncl.log
+			}
+		}
+	}
 
 	return cfg, warnings, nil
 }
@@ -316,6 +364,8 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 			val, err = parseOptHTTPPort(disp)
 		case "https_port":
 			val, err = parseOptHTTPSPort(disp)
+		case "default_sni":
+			val, err = parseOptSingleString(disp)
 		case "order":
 			val, err = parseOptOrder(disp)
 		case "experimental_http3":
@@ -323,11 +373,13 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options
 		case "storage":
 			val, err = parseOptStorage(disp)
 		case "acme_ca", "acme_dns", "acme_ca_root":
-			val, err = parseOptACME(disp)
+			val, err = parseOptSingleString(disp)
 		case "email":
-			val, err = parseOptEmail(disp)
+			val, err = parseOptSingleString(disp)
 		case "admin":
 			val, err = parseOptAdmin(disp)
+		case "debug":
+			options["debug"] = true
 		default:
 			return nil, fmt.Errorf("unrecognized parameter name: %s", dir)
 		}
@@ -426,6 +478,7 @@ func (st *ServerType) serversFromPairings(
 			}
 
 			// tls: connection policies and toggle auto HTTPS
+			defaultSNI := tryString(options["default_sni"], warnings)
 			autoHTTPSQualifiedHosts, err := st.autoHTTPSHosts(sblock)
 			if err != nil {
 				return nil, err
@@ -438,6 +491,7 @@ func (st *ServerType) serversFromPairings(
 				srv.AutoHTTPS.Skip = append(srv.AutoHTTPS.Skip, autoHTTPSQualifiedHosts...)
 			} else if cpVals, ok := sblock.pile["tls.connection_policy"]; ok {
 				// tls connection policies
+
 				for _, cpVal := range cpVals {
 					cp := cpVal.Value.(*caddytls.ConnectionPolicy)
 
@@ -446,6 +500,13 @@ func (st *ServerType) serversFromPairings(
 					if err != nil {
 						return nil, err
 					}
+					for _, h := range hosts {
+						if h == defaultSNI {
+							hosts = append(hosts, "")
+							cp.DefaultSNI = defaultSNI
+							break
+						}
+					}
 
 					// TODO: are matchers needed if every hostname of the resulting config is matched?
 					if len(hosts) > 0 {
@@ -459,6 +520,11 @@ func (st *ServerType) serversFromPairings(
 					srv.TLSConnPolicies = append(srv.TLSConnPolicies, cp)
 				}
 				// TODO: consolidate equal conn policies
+			} else if defaultSNI != "" {
+				hasCatchAllTLSConnPolicy = true
+				srv.TLSConnPolicies = append(srv.TLSConnPolicies, &caddytls.ConnectionPolicy{
+					DefaultSNI: defaultSNI,
+				})
 			}
 
 			// exclude any hosts that were defined explicitly with
@@ -499,6 +565,25 @@ func (st *ServerType) serversFromPairings(
 					srv.Errors.Routes = appendSubrouteToRouteList(srv.Errors.Routes, sr, matcherSetsEnc, p, warnings)
 				}
 			}
+
+			// add log associations
+			for _, cval := range sblock.pile["custom_log"] {
+				ncl := cval.Value.(namedCustomLog)
+				if srv.Logs == nil {
+					srv.Logs = &caddyhttp.ServerLogConfig{
+						LoggerNames: make(map[string]string),
+					}
+				}
+				hosts, err := st.hostsFromServerBlockKeys(sblock.block)
+				if err != nil {
+					return nil, err
+				}
+				for _, h := range hosts {
+					if ncl.name != "" {
+						srv.Logs.LoggerNames[h] = ncl.name
+					}
+				}
+			}
 		}
 
 		// a catch-all TLS conn policy is necessary to ensure TLS can
@@ -690,7 +775,7 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls
 			// otherwise the one without any hosts (a catch-all) would be
 			// eaten up by the one with hosts; and if both have hosts, we
 			// need to combine their lists
-			if reflect.DeepEqual(aps[i].ManagementRaw, aps[j].ManagementRaw) &&
+			if reflect.DeepEqual(aps[i].IssuerRaw, aps[j].IssuerRaw) &&
 				aps[i].ManageSync == aps[j].ManageSync {
 				if len(aps[i].Hosts) == 0 && len(aps[j].Hosts) > 0 {
 					aps = append(aps[:j], aps[j+1:]...)
@@ -882,6 +967,14 @@ func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int {
 	return intVal
 }
 
+func tryString(val interface{}, warnings *[]caddyconfig.Warning) string {
+	stringVal, ok := val.(string)
+	if val != nil && !ok && warnings != nil {
+		*warnings = append(*warnings, caddyconfig.Warning{Message: "not a string type"})
+	}
+	return stringVal
+}
+
 // sliceContains returns true if needle is in haystack.
 func sliceContains(haystack []string, needle string) bool {
 	for _, s := range haystack {
@@ -933,6 +1026,11 @@ type matcherSetAndTokens struct {
 	tokens     []caddyfile.Token
 }
 
+type namedCustomLog struct {
+	name string
+	log  *caddy.CustomLog
+}
+
 // sbAddrAssocation is a mapping from a list of
 // addresses to a list of server blocks that are
 // served on those addresses.
diff --git a/caddyconfig/httpcaddyfile/options.go b/caddyconfig/httpcaddyfile/options.go
index fdecfa4..f8c221c 100644
--- a/caddyconfig/httpcaddyfile/options.go
+++ b/caddyconfig/httpcaddyfile/options.go
@@ -162,19 +162,7 @@ func parseOptStorage(d *caddyfile.Dispenser) (caddy.StorageConverter, error) {
 	return storage, nil
 }
 
-func parseOptACME(d *caddyfile.Dispenser) (string, error) {
-	d.Next() // consume parameter name
-	if !d.Next() {
-		return "", d.ArgErr()
-	}
-	val := d.Val()
-	if d.Next() {
-		return "", d.ArgErr()
-	}
-	return val, nil
-}
-
-func parseOptEmail(d *caddyfile.Dispenser) (string, error) {
+func parseOptSingleString(d *caddyfile.Dispenser) (string, error) {
 	d.Next() // consume parameter name
 	if !d.Next() {
 		return "", d.ArgErr()
@@ -190,11 +178,9 @@ func parseOptAdmin(d *caddyfile.Dispenser) (string, error) {
 	if d.Next() {
 		var listenAddress string
 		d.AllArgs(&listenAddress)
-
 		if listenAddress == "" {
 			listenAddress = caddy.DefaultAdminListen
 		}
-
 		return listenAddress, nil
 	}
 	return "", nil