From b8cba62643abf849411856bd92c42b59b98779f4 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 6 Mar 2020 23:15:25 -0700 Subject: Refactor for CertMagic v0.10; prepare for PKI app This is a breaking change primarily in two areas: - Storage paths for certificates have changed - Slight changes to JSON config parameters Huge improvements in this commit, to be detailed more in the release notes. The upcoming PKI app will be powered by Smallstep libraries. --- caddyconfig/httpcaddyfile/addresses.go | 2 +- caddyconfig/httpcaddyfile/builtins.go | 116 ++++++++++++++++++++++++++++++- caddyconfig/httpcaddyfile/directives.go | 35 ++++++++-- caddyconfig/httpcaddyfile/httptype.go | 118 +++++++++++++++++++++++++++++--- caddyconfig/httpcaddyfile/options.go | 16 +---- 5 files changed, 254 insertions(+), 33 deletions(-) (limited to 'caddyconfig') diff --git a/caddyconfig/httpcaddyfile/addresses.go b/caddyconfig/httpcaddyfile/addresses.go index 19ac197..3aedb60 100644 --- a/caddyconfig/httpcaddyfile/addresses.go +++ b/caddyconfig/httpcaddyfile/addresses.go @@ -23,7 +23,7 @@ import ( "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" - "github.com/mholt/certmagic" + "github.com/caddyserver/certmagic" ) // mapAddressToServerBlocks returns a map of listener address to list of server diff --git a/caddyconfig/httpcaddyfile/builtins.go b/caddyconfig/httpcaddyfile/builtins.go index 8cfca18..a085fcb 100644 --- a/caddyconfig/httpcaddyfile/builtins.go +++ b/caddyconfig/httpcaddyfile/builtins.go @@ -24,8 +24,10 @@ import ( "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/caddyserver/caddy/v2/modules/caddytls" + "go.uber.org/zap/zapcore" ) func init() { @@ -37,6 +39,7 @@ func init() { RegisterHandlerDirective("route", parseRoute) RegisterHandlerDirective("handle", parseSegmentAsSubroute) RegisterDirective("handle_errors", parseHandleErrors) + RegisterDirective("log", parseLog) } // parseBind parses the bind directive. Syntax: @@ -108,7 +111,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) { var cp *caddytls.ConnectionPolicy var fileLoader caddytls.FileLoader var folderLoader caddytls.FolderLoader - var mgr caddytls.ACMEManagerMaker + var mgr caddytls.ACMEIssuer // fill in global defaults, if configured if email := h.Option("email"); email != nil { @@ -307,9 +310,9 @@ func parseTLS(h Helper) ([]ConfigValue, error) { } // automation policy - if !reflect.DeepEqual(mgr, caddytls.ACMEManagerMaker{}) { + if !reflect.DeepEqual(mgr, caddytls.ACMEIssuer{}) { configVals = append(configVals, ConfigValue{ - Class: "tls.automation_manager", + Class: "tls.cert_issuer", Value: mgr, }) } @@ -426,9 +429,116 @@ func parseHandleErrors(h Helper) ([]ConfigValue, error) { }, nil } +// parseLog parses the log directive. Syntax: +// +// log { +// output ... +// format ... +// level +// } +// +func parseLog(h Helper) ([]ConfigValue, error) { + var configValues []ConfigValue + for h.Next() { + cl := new(caddy.CustomLog) + + for h.NextBlock(0) { + switch h.Val() { + case "output": + if !h.NextArg() { + return nil, h.ArgErr() + } + moduleName := h.Val() + + // can't use the usual caddyfile.Unmarshaler flow with the + // standard writers because they are in the caddy package + // (because they are the default) and implementing that + // interface there would unfortunately create circular import + var wo caddy.WriterOpener + switch moduleName { + case "stdout": + wo = caddy.StdoutWriter{} + case "stderr": + wo = caddy.StderrWriter{} + case "discard": + wo = caddy.DiscardWriter{} + default: + mod, err := caddy.GetModule("caddy.logging.writers." + moduleName) + if err != nil { + return nil, h.Errf("getting log writer module named '%s': %v", moduleName, err) + } + unm, ok := mod.New().(caddyfile.Unmarshaler) + if !ok { + return nil, h.Errf("log writer module '%s' is not a Caddyfile unmarshaler", mod) + } + err = unm.UnmarshalCaddyfile(h.NewFromNextSegment()) + if err != nil { + return nil, err + } + wo, ok = unm.(caddy.WriterOpener) + if !ok { + return nil, h.Errf("module %s is not a WriterOpener", mod) + } + } + cl.WriterRaw = caddyconfig.JSONModuleObject(wo, "output", moduleName, h.warnings) + + case "format": + if !h.NextArg() { + return nil, h.ArgErr() + } + moduleName := h.Val() + mod, err := caddy.GetModule("caddy.logging.encoders." + moduleName) + if err != nil { + return nil, h.Errf("getting log encoder module named '%s': %v", moduleName, err) + } + unm, ok := mod.New().(caddyfile.Unmarshaler) + if !ok { + return nil, h.Errf("log encoder module '%s' is not a Caddyfile unmarshaler", mod) + } + err = unm.UnmarshalCaddyfile(h.NewFromNextSegment()) + if err != nil { + return nil, err + } + enc, ok := unm.(zapcore.Encoder) + if !ok { + return nil, h.Errf("module %s is not a zapcore.Encoder", mod) + } + cl.EncoderRaw = caddyconfig.JSONModuleObject(enc, "format", moduleName, h.warnings) + + case "level": + if !h.NextArg() { + return nil, h.ArgErr() + } + cl.Level = h.Val() + if h.NextArg() { + return nil, h.ArgErr() + } + + default: + return nil, h.Errf("unrecognized subdirective: %s", h.Val()) + } + } + + var val namedCustomLog + if !reflect.DeepEqual(cl, new(caddy.CustomLog)) { + cl.Include = []string{"http.log.access"} + val.name = fmt.Sprintf("log%d", logCounter) + val.log = cl + logCounter++ + } + configValues = append(configValues, ConfigValue{ + Class: "custom_log", + Value: val, + }) + } + return configValues, nil +} + // tlsCertTags maps certificate filenames to their tag. // This is used to remember which tag is used for each // certificate files, since we need to avoid loading // the same certificate files more than once, overwriting // previous tags var tlsCertTags = make(map[string]string) + +var logCounter int diff --git a/caddyconfig/httpcaddyfile/directives.go b/caddyconfig/httpcaddyfile/directives.go index 3c03d30..f82e2a8 100644 --- a/caddyconfig/httpcaddyfile/directives.go +++ b/caddyconfig/httpcaddyfile/directives.go @@ -17,6 +17,7 @@ package httpcaddyfile import ( "encoding/json" "sort" + "strings" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig" @@ -298,17 +299,43 @@ func sortRoutes(routes []ConfigValue) { // and returned. func parseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) { var allResults []ConfigValue + for h.Next() { + // slice the linear list of tokens into top-level segments + var segments []caddyfile.Segment for nesting := h.Nesting(); h.NextBlock(nesting); { - dir := h.Val() + segments = append(segments, h.NextSegment()) + } + // copy existing matcher definitions so we can augment + // new ones that are defined only in this scope + matcherDefs := make(map[string]caddy.ModuleMap, len(h.matcherDefs)) + for key, val := range h.matcherDefs { + matcherDefs[key] = val + } + + // find and extract any embedded matcher definitions in this scope + for i, seg := range segments { + if strings.HasPrefix(seg.Directive(), matcherPrefix) { + err := parseMatcherDefinitions(caddyfile.NewDispenser(seg), matcherDefs) + if err != nil { + return nil, err + } + segments = append(segments[:i], segments[i+1:]...) + } + } + + // with matchers ready to go, evaluate each directive's segment + for _, seg := range segments { + dir := seg.Directive() dirFunc, ok := registeredDirectives[dir] if !ok { return nil, h.Errf("unrecognized directive: %s", dir) } subHelper := h - subHelper.Dispenser = h.NewFromNextSegment() + subHelper.Dispenser = caddyfile.NewDispenser(seg) + subHelper.matcherDefs = matcherDefs results, err := dirFunc(subHelper) if err != nil { @@ -319,9 +346,9 @@ func parseSegmentAsSubroute(h Helper) (caddyhttp.MiddlewareHandler, error) { allResults = append(allResults, result) } } - return buildSubroute(allResults, h.groupCounter) // TODO: should we move this outside the loop? } - return nil, nil + + return buildSubroute(allResults, h.groupCounter) } // serverBlock pairs a Caddyfile server block diff --git a/caddyconfig/httpcaddyfile/httptype.go b/caddyconfig/httpcaddyfile/httptype.go index 03234b3..aaec2e9 100644 --- a/caddyconfig/httpcaddyfile/httptype.go +++ b/caddyconfig/httpcaddyfile/httptype.go @@ -26,7 +26,7 @@ import ( "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/caddyserver/caddy/v2/modules/caddytls" - "github.com/mholt/certmagic" + "github.com/caddyserver/certmagic" ) func init() { @@ -88,6 +88,13 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, "{remote}", "{http.request.remote}", "{scheme}", "{http.request.scheme}", "{uri}", "{http.request.uri}", + + "{tls_cipher}", "{http.request.tls.cipher_suite}", + "{tls_version}", "{http.request.tls.version}", + "{tls_client_fingerprint}", "{http.request.tls.client.fingerprint}", + "{tls_client_issuer}", "{http.request.tls.client.issuer}", + "{tls_client_serial}", "{http.request.tls.client.serial}", + "{tls_client_subject}", "{http.request.tls.client.subject}", ) for _, segment := range sb.block.Segments { for i := 0; i < len(segment); i++ { @@ -173,9 +180,9 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, for _, p := range pairings { for i, sblock := range p.serverBlocks { // tls automation policies - if mmVals, ok := sblock.pile["tls.automation_manager"]; ok { + if mmVals, ok := sblock.pile["tls.cert_issuer"]; ok { for _, mmVal := range mmVals { - mm := mmVal.Value.(caddytls.ManagerMaker) + mm := mmVal.Value.(certmagic.Issuer) sblockHosts, err := st.autoHTTPSHosts(sblock) if err != nil { return nil, warnings, err @@ -185,8 +192,8 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, tlsApp.Automation = new(caddytls.AutomationConfig) } tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, &caddytls.AutomationPolicy{ - Hosts: sblockHosts, - ManagementRaw: caddyconfig.JSONModuleObject(mm, "module", mm.(caddy.Module).CaddyModule().ID.Name(), &warnings), + Hosts: sblockHosts, + IssuerRaw: caddyconfig.JSONModuleObject(mm, "module", mm.(caddy.Module).CaddyModule().ID.Name(), &warnings), }) } else { warnings = append(warnings, caddyconfig.Warning{ @@ -245,7 +252,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, if !hasEmail { email = "" } - mgr := caddytls.ACMEManagerMaker{ + mgr := caddytls.ACMEIssuer{ CA: acmeCA.(string), Email: email.(string), } @@ -260,7 +267,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, } } tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, &caddytls.AutomationPolicy{ - ManagementRaw: caddyconfig.JSONModuleObject(mgr, "module", "acme", &warnings), + IssuerRaw: caddyconfig.JSONModuleObject(mgr, "module", "acme", &warnings), }) } if tlsApp.Automation != nil { @@ -275,6 +282,35 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, } } + // extract any custom logs, and enforce configured levels + var customLogs []namedCustomLog + var hasDefaultLog bool + for _, sb := range serverBlocks { + for _, clVal := range sb.pile["custom_log"] { + ncl := clVal.Value.(namedCustomLog) + if ncl.name == "" { + continue + } + if ncl.name == "default" { + hasDefaultLog = true + } + if _, ok := options["debug"]; ok && ncl.log.Level == "" { + ncl.log.Level = "DEBUG" + } + customLogs = append(customLogs, ncl) + } + } + if !hasDefaultLog { + // if the default log was not customized, ensure we + // configure it with any applicable options + if _, ok := options["debug"]; ok { + customLogs = append(customLogs, namedCustomLog{ + name: "default", + log: &caddy.CustomLog{Level: "DEBUG"}, + }) + } + } + // annnd the top-level config, then we're done! cfg := &caddy.Config{AppsRaw: make(caddy.ModuleMap)} if !reflect.DeepEqual(httpApp, caddyhttp.App{}) { @@ -292,6 +328,18 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, if adminConfig, ok := options["admin"].(string); ok && adminConfig != "" { cfg.Admin = &caddy.AdminConfig{Listen: adminConfig} } + if len(customLogs) > 0 { + if cfg.Logging == nil { + cfg.Logging = &caddy.Logging{ + Logs: make(map[string]*caddy.CustomLog), + } + } + for _, ncl := range customLogs { + if ncl.name != "" { + cfg.Logging.Logs[ncl.name] = ncl.log + } + } + } return cfg, warnings, nil } @@ -316,6 +364,8 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options val, err = parseOptHTTPPort(disp) case "https_port": val, err = parseOptHTTPSPort(disp) + case "default_sni": + val, err = parseOptSingleString(disp) case "order": val, err = parseOptOrder(disp) case "experimental_http3": @@ -323,11 +373,13 @@ func (ServerType) evaluateGlobalOptionsBlock(serverBlocks []serverBlock, options case "storage": val, err = parseOptStorage(disp) case "acme_ca", "acme_dns", "acme_ca_root": - val, err = parseOptACME(disp) + val, err = parseOptSingleString(disp) case "email": - val, err = parseOptEmail(disp) + val, err = parseOptSingleString(disp) case "admin": val, err = parseOptAdmin(disp) + case "debug": + options["debug"] = true default: return nil, fmt.Errorf("unrecognized parameter name: %s", dir) } @@ -426,6 +478,7 @@ func (st *ServerType) serversFromPairings( } // tls: connection policies and toggle auto HTTPS + defaultSNI := tryString(options["default_sni"], warnings) autoHTTPSQualifiedHosts, err := st.autoHTTPSHosts(sblock) if err != nil { return nil, err @@ -438,6 +491,7 @@ func (st *ServerType) serversFromPairings( srv.AutoHTTPS.Skip = append(srv.AutoHTTPS.Skip, autoHTTPSQualifiedHosts...) } else if cpVals, ok := sblock.pile["tls.connection_policy"]; ok { // tls connection policies + for _, cpVal := range cpVals { cp := cpVal.Value.(*caddytls.ConnectionPolicy) @@ -446,6 +500,13 @@ func (st *ServerType) serversFromPairings( if err != nil { return nil, err } + for _, h := range hosts { + if h == defaultSNI { + hosts = append(hosts, "") + cp.DefaultSNI = defaultSNI + break + } + } // TODO: are matchers needed if every hostname of the resulting config is matched? if len(hosts) > 0 { @@ -459,6 +520,11 @@ func (st *ServerType) serversFromPairings( srv.TLSConnPolicies = append(srv.TLSConnPolicies, cp) } // TODO: consolidate equal conn policies + } else if defaultSNI != "" { + hasCatchAllTLSConnPolicy = true + srv.TLSConnPolicies = append(srv.TLSConnPolicies, &caddytls.ConnectionPolicy{ + DefaultSNI: defaultSNI, + }) } // exclude any hosts that were defined explicitly with @@ -499,6 +565,25 @@ func (st *ServerType) serversFromPairings( srv.Errors.Routes = appendSubrouteToRouteList(srv.Errors.Routes, sr, matcherSetsEnc, p, warnings) } } + + // add log associations + for _, cval := range sblock.pile["custom_log"] { + ncl := cval.Value.(namedCustomLog) + if srv.Logs == nil { + srv.Logs = &caddyhttp.ServerLogConfig{ + LoggerNames: make(map[string]string), + } + } + hosts, err := st.hostsFromServerBlockKeys(sblock.block) + if err != nil { + return nil, err + } + for _, h := range hosts { + if ncl.name != "" { + srv.Logs.LoggerNames[h] = ncl.name + } + } + } } // a catch-all TLS conn policy is necessary to ensure TLS can @@ -690,7 +775,7 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls // otherwise the one without any hosts (a catch-all) would be // eaten up by the one with hosts; and if both have hosts, we // need to combine their lists - if reflect.DeepEqual(aps[i].ManagementRaw, aps[j].ManagementRaw) && + if reflect.DeepEqual(aps[i].IssuerRaw, aps[j].IssuerRaw) && aps[i].ManageSync == aps[j].ManageSync { if len(aps[i].Hosts) == 0 && len(aps[j].Hosts) > 0 { aps = append(aps[:j], aps[j+1:]...) @@ -882,6 +967,14 @@ func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int { return intVal } +func tryString(val interface{}, warnings *[]caddyconfig.Warning) string { + stringVal, ok := val.(string) + if val != nil && !ok && warnings != nil { + *warnings = append(*warnings, caddyconfig.Warning{Message: "not a string type"}) + } + return stringVal +} + // sliceContains returns true if needle is in haystack. func sliceContains(haystack []string, needle string) bool { for _, s := range haystack { @@ -933,6 +1026,11 @@ type matcherSetAndTokens struct { tokens []caddyfile.Token } +type namedCustomLog struct { + name string + log *caddy.CustomLog +} + // sbAddrAssocation is a mapping from a list of // addresses to a list of server blocks that are // served on those addresses. diff --git a/caddyconfig/httpcaddyfile/options.go b/caddyconfig/httpcaddyfile/options.go index fdecfa4..f8c221c 100644 --- a/caddyconfig/httpcaddyfile/options.go +++ b/caddyconfig/httpcaddyfile/options.go @@ -162,19 +162,7 @@ func parseOptStorage(d *caddyfile.Dispenser) (caddy.StorageConverter, error) { return storage, nil } -func parseOptACME(d *caddyfile.Dispenser) (string, error) { - d.Next() // consume parameter name - if !d.Next() { - return "", d.ArgErr() - } - val := d.Val() - if d.Next() { - return "", d.ArgErr() - } - return val, nil -} - -func parseOptEmail(d *caddyfile.Dispenser) (string, error) { +func parseOptSingleString(d *caddyfile.Dispenser) (string, error) { d.Next() // consume parameter name if !d.Next() { return "", d.ArgErr() @@ -190,11 +178,9 @@ func parseOptAdmin(d *caddyfile.Dispenser) (string, error) { if d.Next() { var listenAddress string d.AllArgs(&listenAddress) - if listenAddress == "" { listenAddress = caddy.DefaultAdminListen } - return listenAddress, nil } return "", nil -- cgit v1.2.3