diff options
-rwxr-xr-x | caddyconfig/caddyfile/dispenser.go | 20 | ||||
-rw-r--r-- | caddyconfig/httpcaddyfile/builtins.go | 34 | ||||
-rw-r--r-- | caddyconfig/httpcaddyfile/directives.go | 124 | ||||
-rw-r--r-- | caddyconfig/httpcaddyfile/httptype.go | 285 | ||||
-rw-r--r-- | caddyconfig/httpcaddyfile/httptype_test.go | 33 | ||||
-rw-r--r-- | caddyconfig/httpcaddyfile/options.go | 87 | ||||
-rw-r--r-- | cmd/commandfuncs.go | 12 | ||||
-rw-r--r-- | modules/caddyhttp/autohttps.go | 347 | ||||
-rw-r--r-- | modules/caddyhttp/caddyhttp.go | 275 | ||||
-rw-r--r-- | modules/caddyhttp/fileserver/caddyfile.go | 5 | ||||
-rw-r--r-- | modules/caddyhttp/matchers.go | 6 | ||||
-rw-r--r-- | modules/caddyhttp/matchers_test.go | 2 | ||||
-rw-r--r-- | modules/caddyhttp/reverseproxy/circuitbreaker.go | 37 | ||||
-rw-r--r-- | modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go | 31 | ||||
-rw-r--r-- | modules/caddyhttp/reverseproxy/selectionpolicies.go | 3 | ||||
-rw-r--r-- | modules/caddyhttp/rewrite/rewrite.go | 28 | ||||
-rw-r--r-- | modules/caddyhttp/rewrite/rewrite_test.go | 10 | ||||
-rw-r--r-- | modules/caddyhttp/routes.go | 30 | ||||
-rw-r--r-- | modules/caddyhttp/server.go | 53 | ||||
-rw-r--r-- | modules/caddyhttp/staticresp.go | 25 |
20 files changed, 958 insertions, 489 deletions
diff --git a/caddyconfig/caddyfile/dispenser.go b/caddyconfig/caddyfile/dispenser.go index 5b90b73..4ed9325 100755 --- a/caddyconfig/caddyfile/dispenser.go +++ b/caddyconfig/caddyfile/dispenser.go @@ -152,8 +152,10 @@ func (d *Dispenser) NextBlock(initialNestingLevel int) bool { if !d.Next() { return false // should be EOF error } - if d.Val() == "}" { + if d.Val() == "}" && !d.nextOnSameLine() { d.nesting-- + } else if d.Val() == "{" && !d.nextOnSameLine() { + d.nesting++ } return d.nesting > initialNestingLevel } @@ -262,9 +264,9 @@ func (d *Dispenser) NewFromNextTokens() *Dispenser { if !openedBlock { // because NextBlock() consumes the initial open // curly brace, we rewind here to append it, since - // our case is special in that we want to include - // all the tokens including surrounding curly braces - // for a new dispenser to have + // our case is special in that we want the new + // dispenser to have all the tokens including + // surrounding curly braces d.Prev() tkns = append(tkns, d.Token()) d.Next() @@ -273,12 +275,12 @@ func (d *Dispenser) NewFromNextTokens() *Dispenser { tkns = append(tkns, d.Token()) } if openedBlock { - // include closing brace accordingly + // include closing brace tkns = append(tkns, d.Token()) - // since NewFromNextTokens is intended to consume the entire - // directive, we must call Next() here and consume the closing - // curly brace - d.Next() + + // do not consume the closing curly brace; the + // next iteration of the enclosing loop will + // call Next() and consume it } return NewDispenser(tkns) } diff --git a/caddyconfig/httpcaddyfile/builtins.go b/caddyconfig/httpcaddyfile/builtins.go index ebb03cc..daba03b 100644 --- a/caddyconfig/httpcaddyfile/builtins.go +++ b/caddyconfig/httpcaddyfile/builtins.go @@ -34,6 +34,7 @@ func init() { RegisterHandlerDirective("redir", parseRedir) RegisterHandlerDirective("respond", parseRespond) RegisterHandlerDirective("route", parseRoute) + RegisterHandlerDirective("handle", parseHandle) } func parseBind(h Helper) ([]ConfigValue, error) { @@ -76,7 +77,7 @@ func parseRoot(h Helper) ([]ConfigValue, error) { route.MatcherSetsRaw = []caddy.ModuleMap{matcherSet} } - return h.NewVarsRoute(route), nil + return []ConfigValue{{Class: "route", Value: route}}, nil } func parseTLS(h Helper) ([]ConfigValue, error) { @@ -330,3 +331,34 @@ func parseRoute(h Helper) (caddyhttp.MiddlewareHandler, error) { return sr, nil } + +func parseHandle(h Helper) (caddyhttp.MiddlewareHandler, error) { + var allResults []ConfigValue + + for h.Next() { + for nesting := h.Nesting(); h.NextBlock(nesting); { + dir := h.Val() + + dirFunc, ok := registeredDirectives[dir] + if !ok { + return nil, h.Errf("unrecognized directive: %s", dir) + } + + subHelper := h + subHelper.Dispenser = h.NewFromNextTokens() + + results, err := dirFunc(subHelper) + if err != nil { + return nil, h.Errf("parsing caddyfile tokens for '%s': %v", dir, err) + } + for _, result := range results { + result.directive = dir + allResults = append(allResults, result) + } + } + + return buildSubroute(allResults, h.groupCounter) + } + + return nil, nil +} diff --git a/caddyconfig/httpcaddyfile/directives.go b/caddyconfig/httpcaddyfile/directives.go index 19ecb26..7acdb8c 100644 --- a/caddyconfig/httpcaddyfile/directives.go +++ b/caddyconfig/httpcaddyfile/directives.go @@ -16,6 +16,7 @@ package httpcaddyfile import ( "encoding/json" + "sort" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig" @@ -23,27 +24,45 @@ import ( "github.com/caddyserver/caddy/v2/modules/caddyhttp" ) -// defaultDirectiveOrder specifies the order +// directiveOrder specifies the order // to apply directives in HTTP routes. -var defaultDirectiveOrder = []string{ +var directiveOrder = []string{ + "root", + + "redir", "rewrite", + "strip_prefix", "strip_suffix", "uri_replace", "try_files", + "basicauth", "headers", "request_header", "encode", "templates", + + "handle", "route", - "redir", + "respond", "reverse_proxy", "php_fastcgi", "file_server", } +// directiveIsOrdered returns true if dir is +// a known, ordered (sorted) directive. +func directiveIsOrdered(dir string) bool { + for _, d := range directiveOrder { + if d == dir { + return true + } + } + return false +} + // RegisterDirective registers a unique directive dir with an // associated unmarshaling (setup) function. When directive dir // is encountered in a Caddyfile, setupFunc will be called to @@ -92,10 +111,11 @@ func RegisterHandlerDirective(dir string, setupFunc UnmarshalHandlerFunc) { // Caddyfile tokens. type Helper struct { *caddyfile.Dispenser - options map[string]interface{} - warnings *[]caddyconfig.Warning - matcherDefs map[string]caddy.ModuleMap - parentBlock caddyfile.ServerBlock + options map[string]interface{} + warnings *[]caddyconfig.Warning + matcherDefs map[string]caddy.ModuleMap + parentBlock caddyfile.ServerBlock + groupCounter counter } // Option gets the option keyed by name. @@ -127,8 +147,8 @@ func (h Helper) JSON(val interface{}) json.RawMessage { return caddyconfig.JSON(val, h.warnings) } -// MatcherToken assumes the current token is (possibly) a matcher, and -// if so, returns the matcher set along with a true value. If the current +// MatcherToken assumes the next argument token is (possibly) a matcher, +// and if so, returns the matcher set along with a true value. If the next // token is not a matcher, nil and false is returned. Note that a true // value may be returned with a nil matcher set if it is a catch-all. func (h Helper) MatcherToken() (caddy.ModuleMap, bool, error) { @@ -165,18 +185,39 @@ func (h Helper) NewRoute(matcherSet caddy.ModuleMap, } } +// GroupRoutes adds the routes (caddyhttp.Route type) in vals to the +// same group, if there is more than one route in vals. +func (h Helper) GroupRoutes(vals []ConfigValue) { + // ensure there's at least two routes; group of one is pointless + var count int + for _, v := range vals { + if _, ok := v.Value.(caddyhttp.Route); ok { + count++ + if count > 1 { + break + } + } + } + if count < 2 { + return + } + + // now that we know the group will have some effect, do it + groupName := h.groupCounter.nextGroup() + for i := range vals { + if route, ok := vals[i].Value.(caddyhttp.Route); ok { + route.Group = groupName + vals[i].Value = route + } + } +} + // NewBindAddresses returns config values relevant to adding // listener bind addresses to the config. func (h Helper) NewBindAddresses(addrs []string) []ConfigValue { return []ConfigValue{{Class: "bind", Value: addrs}} } -// NewVarsRoute returns config values relevant to adding a -// "vars" wrapper route to the config. -func (h Helper) NewVarsRoute(route caddyhttp.Route) []ConfigValue { - return []ConfigValue{{Class: "var", Value: route}} -} - // ConfigValue represents a value to be added to the final // configuration, or a value to be consulted when building // the final configuration. @@ -197,6 +238,59 @@ type ConfigValue struct { directive string } +func sortRoutes(routes []ConfigValue) { + dirPositions := make(map[string]int) + for i, dir := range directiveOrder { + dirPositions[dir] = i + } + + // while we are sorting, we will need to decode a route's path matcher + // in order to sub-sort by path length; we can amortize this operation + // for efficiency by storing the decoded matchers in a slice + decodedMatchers := make([]caddyhttp.MatchPath, len(routes)) + + sort.SliceStable(routes, func(i, j int) bool { + iDir, jDir := routes[i].directive, routes[j].directive + if iDir == jDir { + // directives are the same; sub-sort by path matcher length + // if there's only one matcher set and one path (common case) + iRoute, ok := routes[i].Value.(caddyhttp.Route) + if !ok { + return false + } + jRoute, ok := routes[j].Value.(caddyhttp.Route) + if !ok { + return false + } + + if len(iRoute.MatcherSetsRaw) == 1 && len(jRoute.MatcherSetsRaw) == 1 { + // use already-decoded matcher, or decode if it's the first time seeing it + iPM, jPM := decodedMatchers[i], decodedMatchers[j] + if iPM == nil { + var pathMatcher caddyhttp.MatchPath + _ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &pathMatcher) + decodedMatchers[i] = pathMatcher + iPM = pathMatcher + } + if jPM == nil { + var pathMatcher caddyhttp.MatchPath + _ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &pathMatcher) + decodedMatchers[j] = pathMatcher + jPM = pathMatcher + } + + // if there is only one path in the matcher, sort by + // longer path (more specific) first + if len(iPM) == 1 && len(jPM) == 1 { + return len(iPM[0]) > len(jPM[0]) + } + } + } + + return dirPositions[iDir] < dirPositions[jDir] + }) +} + // serverBlock pairs a Caddyfile server block // with a "pile" of config values, keyed by class // name. diff --git a/caddyconfig/httpcaddyfile/httptype.go b/caddyconfig/httpcaddyfile/httptype.go index bec5045..a57b6e9 100644 --- a/caddyconfig/httpcaddyfile/httptype.go +++ b/caddyconfig/httpcaddyfile/httptype.go @@ -41,6 +41,7 @@ type ServerType struct { func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, options map[string]interface{}) (*caddy.Config, []caddyconfig.Warning, error) { var warnings []caddyconfig.Warning + gc := counter{new(int)} var serverBlocks []serverBlock for _, sblock := range originalServerBlocks { @@ -64,8 +65,8 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, val, err = parseOptHTTPPort(disp) case "https_port": val, err = parseOptHTTPSPort(disp) - case "handler_order": - val, err = parseOptHandlerOrder(disp) + case "order": + val, err = parseOptOrder(disp) case "experimental_http3": val, err = parseOptExperimentalHTTP3(disp) case "storage": @@ -140,11 +141,12 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, } results, err := dirFunc(Helper{ - Dispenser: caddyfile.NewDispenser(segment), - options: options, - warnings: &warnings, - matcherDefs: matcherDefs, - parentBlock: sb.block, + Dispenser: caddyfile.NewDispenser(segment), + options: options, + warnings: &warnings, + matcherDefs: matcherDefs, + parentBlock: sb.block, + groupCounter: gc, }) if err != nil { return nil, warnings, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err) @@ -167,7 +169,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, // each pairing of listener addresses to list of server // blocks is basically a server definition - servers, err := st.serversFromPairings(pairings, options, &warnings) + servers, err := st.serversFromPairings(pairings, options, &warnings, gc) if err != nil { return nil, warnings, err } @@ -304,6 +306,7 @@ func (st *ServerType) serversFromPairings( pairings []sbAddrAssociation, options map[string]interface{}, warnings *[]caddyconfig.Warning, + groupCounter counter, ) (map[string]*caddyhttp.Server, error) { servers := make(map[string]*caddyhttp.Server) @@ -312,16 +315,48 @@ func (st *ServerType) serversFromPairings( Listen: p.addresses, } + // sort server blocks by their keys; this is important because + // only the first matching site should be evaluated, and we should + // attempt to match most specific site first (host and path), in + // case their matchers overlap; we do this somewhat naively by + // descending sort by length of host then path + sort.SliceStable(p.serverBlocks, func(i, j int) bool { + // TODO: we could pre-process the specificities for efficiency, + // but I don't expect many blocks will have SO many keys... + var iLongestPath, jLongestPath string + var iLongestHost, jLongestHost string + for _, key := range p.serverBlocks[i].block.Keys { + addr, _ := ParseAddress(key) + if specificity(addr.Host) > specificity(iLongestHost) { + iLongestHost = addr.Host + } + if specificity(addr.Path) > specificity(iLongestPath) { + iLongestPath = addr.Path + } + } + for _, key := range p.serverBlocks[j].block.Keys { + addr, _ := ParseAddress(key) + if specificity(addr.Host) > specificity(jLongestHost) { + jLongestHost = addr.Host + } + if specificity(addr.Path) > specificity(jLongestPath) { + jLongestPath = addr.Path + } + } + if specificity(iLongestHost) == specificity(jLongestHost) { + return len(iLongestPath) > len(jLongestPath) + } + return specificity(iLongestHost) > specificity(jLongestHost) + }) + + // create a subroute for each site in the server block for _, sblock := range p.serverBlocks { matcherSetsEnc, err := st.compileEncodedMatcherSets(sblock.block) if err != nil { return nil, fmt.Errorf("server block %v: compiling matcher sets: %v", sblock.block.Keys, err) } - siteSubroute := new(caddyhttp.Subroute) - // tls: connection policies and toggle auto HTTPS - autoHTTPSQualifiedHosts, err := st.autoHTTPSHosts(sblock) if err != nil { return nil, err @@ -354,105 +389,126 @@ func (st *ServerType) serversFromPairings( // TODO: consolidate equal conn policies } - // vars: make sure these are linked in first so future - // routes can use the variables they define - for _, cfgVal := range sblock.pile["var"] { - siteSubroute.Routes = append(siteSubroute.Routes, cfgVal.Value.(caddyhttp.Route)) - } - - // set up each handler directive - the order of the handlers - // as they are added to the routes depends on user preference + // set up each handler directive, making sure to honor directive order dirRoutes := sblock.pile["route"] - handlerOrder, ok := options["handler_order"].([]string) - if !ok { - handlerOrder = defaultDirectiveOrder - } - if len(handlerOrder) == 1 && handlerOrder[0] == "appearance" { - handlerOrder = nil + siteSubroute, err := buildSubroute(dirRoutes, groupCounter) + if err != nil { + return nil, err } - if handlerOrder != nil { - dirPositions := make(map[string]int) - for i, dir := range handlerOrder { - dirPositions[dir] = i - } - sort.SliceStable(dirRoutes, func(i, j int) bool { - iDir, jDir := dirRoutes[i].directive, dirRoutes[j].directive - if iDir == jDir { - // TODO: we really need to refactor this into a separate function or method... - // sub-sort by path matcher length, if there's only one - iRoute := dirRoutes[i].Value.(caddyhttp.Route) - jRoute := dirRoutes[j].Value.(caddyhttp.Route) - if len(iRoute.MatcherSetsRaw) == 1 && len(jRoute.MatcherSetsRaw) == 1 { - // for slightly better efficiency, only decode the path matchers once, - // then just store them arbitrarily in the decoded MatcherSets field, - // ours should be the only thing in there - var iPM, jPM caddyhttp.MatchPath - if len(iRoute.MatcherSets) == 1 { - iPM = iRoute.MatcherSets[0][0].(caddyhttp.MatchPath) - } - if len(jRoute.MatcherSets) == 1 { - jPM = jRoute.MatcherSets[0][0].(caddyhttp.MatchPath) - } - // if it's our first time seeing this route's path matcher, decode it - if iPM == nil { - var pathMatcher caddyhttp.MatchPath - _ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &pathMatcher) - iRoute.MatcherSets = caddyhttp.MatcherSets{{pathMatcher}} - iPM = pathMatcher - } - if jPM == nil { - var pathMatcher caddyhttp.MatchPath - _ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &pathMatcher) - jRoute.MatcherSets = caddyhttp.MatcherSets{{pathMatcher}} - jPM = pathMatcher - } - // finally, if there is only one path in the - // matcher, sort by longer path first - if len(iPM) == 1 && len(jPM) == 1 { - return len(iPM[0]) > len(jPM[0]) - } - } - } - return dirPositions[iDir] < dirPositions[jDir] + + if len(matcherSetsEnc) == 0 && len(p.serverBlocks) == 1 { + // no need to wrap the handlers in a subroute if this is + // the only server block and there is no matcher for it + srv.Routes = append(srv.Routes, siteSubroute.Routes...) + } else { + srv.Routes = append(srv.Routes, caddyhttp.Route{ + MatcherSetsRaw: matcherSetsEnc, + HandlersRaw: []json.RawMessage{ + caddyconfig.JSONModuleObject(siteSubroute, "handler", "subroute", warnings), + }, + Terminal: true, // only first matching site block should be evaluated }) } + } - // add all the routes piled in from directives - for _, r := range dirRoutes { - // as a special case, group rewrite directives so that they are mutually exclusive; - // this means that only the first matching rewrite will be evaluated, and that's - // probably a good thing, since there should never be a need to do more than one - // rewrite (I think?), and cascading rewrites smell bad... imagine these rewrites: - // rewrite /docs/json/* /docs/json/index.html - // rewrite /docs/* /docs/index.html - // (We use this on the Caddy website, or at least we did once.) The first rewrite's - // result is also matched by the second rewrite, making the first rewrite pointless. - // See issue #2959. - if r.directive == "rewrite" { - route := r.Value.(caddyhttp.Route) - route.Group = "rewriting" - r.Value = route - } + srv.Routes = consolidateRoutes(srv.Routes) + + servers[fmt.Sprintf("srv%d", i)] = srv + } + + return servers, nil +} + +func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subroute, error) { + for _, val := range routes { + if !directiveIsOrdered(val.directive) { + return nil, fmt.Errorf("directive '%s' is not ordered, so it cannot be used here", val.directive) + } + } + + sortRoutes(routes) + + subroute := new(caddyhttp.Subroute) - siteSubroute.Routes = append(siteSubroute.Routes, r.Value.(caddyhttp.Route)) + // get a group name for rewrite directives, if needed + var rewriteGroupName string + var rewriteCount int + for _, r := range routes { + if r.directive == "rewrite" { + rewriteCount++ + if rewriteCount > 1 { + break } + } + } + if rewriteCount > 1 { + rewriteGroupName = groupCounter.nextGroup() + } - siteSubroute.Routes = consolidateRoutes(siteSubroute.Routes) + // get a group name for handle blocks, if needed + var handleGroupName string + var handleCount int + for _, r := range routes { + if r.directive == "handle" { + handleCount++ + if handleCount > 1 { + break + } + } + } + if handleCount > 1 { + handleGroupName = groupCounter.nextGroup() + } - srv.Routes = append(srv.Routes, caddyhttp.Route{ - MatcherSetsRaw: matcherSetsEnc, - HandlersRaw: []json.RawMessage{ - caddyconfig.JSONModuleObject(siteSubroute, "handler", "subroute", warnings), - }, - }) + // add all the routes piled in from directives + for _, r := range routes { + // as a special case, group rewrite directives so that they are mutually exclusive; + // this means that only the first matching rewrite will be evaluated, and that's + // probably a good thing, since there should never be a need to do more than one + // rewrite (I think?), and cascading rewrites smell bad... imagine these rewrites: + // rewrite /docs/json/* /docs/json/index.html + // rewrite /docs/* /docs/index.html + // (We use this on the Caddy website, or at least we did once.) The first rewrite's + // result is also matched by the second rewrite, making the first rewrite pointless. + // See issue #2959. + if r.directive == "rewrite" { + route := r.Value.(caddyhttp.Route) + route.Group = rewriteGroupName + r.Value = route } - srv.Routes = consolidateRoutes(srv.Routes) + // handle blocks are also mutually exclusive by definition + if r.directive == "handle" { + route := r.Value.(caddyhttp.Route) + route.Group = handleGroupName + r.Value = route + } - servers[fmt.Sprintf("srv%d", i)] = srv + switch route := r.Value.(type) { + case caddyhttp.Subroute: + // if a route-class config value is actually a Subroute handler + // with nothing but a list of routes, then it is the intention + // of the directive to keep these handlers together and in this + // same order, but not necessarily in a subroute (if it wanted + // to keep them in a subroute, the directive would have returned + // a route with a Subroute as its handler); this is useful to + // keep multiple handlers/routes together and in the same order + // so that the sorting procedure we did above doesn't reorder them + if route.Errors != nil { + // if error handlers are also set, this is confusing; it's + // probably supposed to be wrapped in a Route and encoded + // as a regular handler route... programmer error. + panic("found subroute with more than just routes; perhaps it should have been wrapped in a route?") + } + subroute.Routes = append(subroute.Routes, route.Routes...) + case caddyhttp.Route: + subroute.Routes = append(subroute.Routes, route) + } } - return servers, nil + subroute.Routes = consolidateRoutes(subroute.Routes) + + return subroute, nil } func (st ServerType) autoHTTPSHosts(sb serverBlock) ([]string, error) { @@ -530,7 +586,6 @@ func matcherSetFromMatcherToken( } return m, true, nil } - return nil, false, nil } @@ -668,6 +723,42 @@ func tryInt(val interface{}, warnings *[]caddyconfig.Warning) int { return intVal } +// specifity returns len(s) minus any wildcards (*) and +// placeholders ({...}). Basically, it's a length count +// that penalizes the use of wildcards and placeholders. +// This is useful for comparing hostnames and paths. +// However, wildcards in paths are not a sure answer to +// the question of specificity. For exmaple, +// '*.example.com' is clearly less specific than +// 'a.example.com', but is '/a' more or less specific +// than '/a*'? +func specificity(s string) int { + l := len(s) - strings.Count(s, "*") + for len(s) > 0 { + start := strings.Index(s, "{") + if start < 0 { + return l + } + end := strings.Index(s[start:], "}") + start + 1 + if end <= start { + return l + } + l -= end - start + s = s[end:] + } + return l +} + +type counter struct { + n *int +} + +func (c counter) nextGroup() string { + name := fmt.Sprintf("group%d", *c.n) + *c.n++ + return name +} + type matcherSetAndTokens struct { matcherSet caddy.ModuleMap tokens []caddyfile.Token diff --git a/caddyconfig/httpcaddyfile/httptype_test.go b/caddyconfig/httpcaddyfile/httptype_test.go new file mode 100644 index 0000000..ae4f042 --- /dev/null +++ b/caddyconfig/httpcaddyfile/httptype_test.go @@ -0,0 +1,33 @@ +package httpcaddyfile + +import "testing" + +func TestSpecificity(t *testing.T) { + for i, tc := range []struct { + input string + expect int + }{ + {"", 0}, + {"*", 0}, + {"*.*", 1}, + {"{placeholder}", 0}, + {"/{placeholder}", 1}, + {"foo", 3}, + {"example.com", 11}, + {"a.example.com", 13}, + {"*.example.com", 12}, + {"/foo", 4}, + {"/foo*", 4}, + {"{placeholder}.example.com", 12}, + {"{placeholder.example.com", 24}, + {"}.", 2}, + {"}{", 2}, + {"{}", 0}, + {"{{{}}", 1}, + } { + actual := specificity(tc.input) + if actual != tc.expect { + t.Errorf("Test %d (%s): Expected %d but got %d", i, tc.input, tc.expect, actual) + } + } +} diff --git a/caddyconfig/httpcaddyfile/options.go b/caddyconfig/httpcaddyfile/options.go index e87b30f..e81528e 100644 --- a/caddyconfig/httpcaddyfile/options.go +++ b/caddyconfig/httpcaddyfile/options.go @@ -58,27 +58,80 @@ func parseOptExperimentalHTTP3(d *caddyfile.Dispenser) (bool, error) { return true, nil } -func parseOptHandlerOrder(d *caddyfile.Dispenser) ([]string, error) { - if !d.Next() { - return nil, d.ArgErr() - } - order := d.RemainingArgs() - if len(order) == 1 && order[0] == "appearance" { - return []string{"appearance"}, nil - } - if len(order) > 0 && d.NextBlock(0) { - return nil, d.Err("cannot open block if there are arguments") - } - for d.NextBlock(0) { - order = append(order, d.Val()) +func parseOptOrder(d *caddyfile.Dispenser) ([]string, error) { + newOrder := directiveOrder + + for d.Next() { + // get directive name + if !d.Next() { + return nil, d.ArgErr() + } + dirName := d.Val() + if _, ok := registeredDirectives[dirName]; !ok { + return nil, fmt.Errorf("%s is not a registered directive", dirName) + } + + // get positional token + if !d.Next() { + return nil, d.ArgErr() + } + pos := d.Val() + + // if directive exists, first remove it + for i, d := range newOrder { + if d == dirName { + newOrder = append(newOrder[:i], newOrder[i+1:]...) + break + } + } + + // act on the positional + switch pos { + case "first": + newOrder = append([]string{dirName}, newOrder...) + if d.NextArg() { + return nil, d.ArgErr() + } + directiveOrder = newOrder + return newOrder, nil + case "last": + newOrder = append(newOrder, dirName) + if d.NextArg() { + return nil, d.ArgErr() + } + directiveOrder = newOrder + return newOrder, nil + case "before": + case "after": + default: + return nil, fmt.Errorf("unknown positional '%s'", pos) + } + + // get name of other directive + if !d.NextArg() { + return nil, d.ArgErr() + } + otherDir := d.Val() if d.NextArg() { return nil, d.ArgErr() } + + // insert directive into proper position + for i, d := range newOrder { + if d == otherDir { + if pos == "before" { + newOrder = append(newOrder[:i], append([]string{dirName}, newOrder[i:]...)...) + } else if pos == "after" { + newOrder = append(newOrder[:i+1], append([]string{dirName}, newOrder[i+1:]...)...) + } + break + } + } } - if len(order) == 0 { - return nil, d.ArgErr() - } - return order, nil + + directiveOrder = newOrder + + return newOrder, nil } func parseOptStorage(d *caddyfile.Dispenser) (caddy.StorageConverter, error) { diff --git a/cmd/commandfuncs.go b/cmd/commandfuncs.go index 0fd7786..83fc52d 100644 --- a/cmd/commandfuncs.go +++ b/cmd/commandfuncs.go @@ -312,12 +312,18 @@ func cmdReload(fl Flags) (int, error) { func cmdVersion(_ Flags) (int, error) { goModule := caddy.GoModule() + fmt.Print(goModule.Version) if goModule.Sum != "" { // a build with a known version will also have a checksum - fmt.Printf("%s %s\n", goModule.Version, goModule.Sum) - } else { - fmt.Println(goModule.Version) + fmt.Printf(" %s", goModule.Sum) } + if goModule.Replace != nil { + fmt.Printf(" => %s", goModule.Replace.Path) + if goModule.Replace.Version != "" { + fmt.Printf(" %s", goModule.Replace.Version) + } + } + fmt.Println() return caddy.ExitCodeSuccess, nil } diff --git a/modules/caddyhttp/autohttps.go b/modules/caddyhttp/autohttps.go new file mode 100644 index 0000000..69e3318 --- /dev/null +++ b/modules/caddyhttp/autohttps.go @@ -0,0 +1,347 @@ +package caddyhttp + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/mholt/certmagic" + "go.uber.org/zap" +) + +// AutoHTTPSConfig is used to disable automatic HTTPS +// or certain aspects of it for a specific server. +// HTTPS is enabled automatically and by default when +// qualifying hostnames are available from the config. +type AutoHTTPSConfig struct { + // If true, automatic HTTPS will be entirely disabled. + Disabled bool `json:"disable,omitempty"` + + // If true, only automatic HTTP->HTTPS redirects will + // be disabled. + DisableRedir bool `json:"disable_redirects,omitempty"` + + // Hosts/domain names listed here will not be included + // in automatic HTTPS (they will not have certificates + // loaded nor redirects applied). + Skip []string `json:"skip,omitempty"` + + // Hosts/domain names listed here will still be enabled + // for automatic HTTPS (unless in the Skip list), except + // that certificates will not be provisioned and managed + // for these names. + SkipCerts []string `json:"skip_certificates,omitempty"` + + // By default, automatic HTTPS will obtain and renew + // certificates for qualifying hostnames. However, if + // a certificate with a matching SAN is already loaded + // into the cache, certificate management will not be + // enabled. To force automated certificate management + // regardless of loaded certificates, set this to true. + IgnoreLoadedCerts bool `json:"ignore_loaded_certificates,omitempty"` + + domainSet map[string]struct{} +} + +// Skipped returns true if name is in skipSlice, which +// should be one of the Skip* fields on ahc. +func (ahc AutoHTTPSConfig) Skipped(name string, skipSlice []string) bool { + for _, n := range skipSlice { + if name == n { + return true + } + } + return false +} + +// automaticHTTPSPhase1 provisions all route matchers, determines +// which domain names found in the routes qualify for automatic +// HTTPS, and sets up HTTP->HTTPS redirects. This phase must occur +// at the beginning of provisioning, because it may add routes and +// even servers to the app, which still need to be set up with the +// rest of them. +func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) error { + // this map will store associations of HTTP listener + // addresses to the routes that do HTTP->HTTPS redirects + lnAddrRedirRoutes := make(map[string]Route) + + for srvName, srv := range app.Servers { + // as a prerequisite, provision route matchers; this is + // required for all routes on all servers, and must be + // done before we attempt to do phase 1 of auto HTTPS, + // since we have to access the decoded host matchers the + // handlers will be provisioned later + if srv.Routes != nil { + err := srv.Routes.ProvisionMatchers(ctx) + if err != nil { + return fmt.Errorf("server %s: setting up route matchers: %v", srvName, err) + } + } + + // prepare for automatic HTTPS + if srv.AutoHTTPS == nil { + srv.AutoHTTPS = new(AutoHTTPSConfig) + } + if srv.AutoHTTPS.Disabled { + continue + } + + // skip if all listeners use the HTTP port + if !srv.listenersUseAnyPortOtherThan(app.httpPort()) { + app.logger.Info("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server", + zap.String("server_name", srvName), + zap.Int("http_port", app.httpPort()), + ) + srv.AutoHTTPS.Disabled = true + continue + } + + defaultConnPolicies := caddytls.ConnectionPolicies{ + &caddytls.ConnectionPolicy{ALPN: defaultALPN}, + } + + // if all listeners are on the HTTPS port, make sure + // there is at least one TLS connection policy; it + // should be obvious that they want to use TLS without + // needing to specify one empty policy to enable it + if srv.TLSConnPolicies == nil && + !srv.listenersUseAnyPortOtherThan(app.httpsPort()) { + app.logger.Info("server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS", + zap.String("server_name", srvName), + zap.Int("https_port", app.httpsPort()), + ) + srv.TLSConnPolicies = defaultConnPolicies + } + + // find all qualifying domain names in this server + srv.AutoHTTPS.domainSet = make(map[string]struct{}) + for routeIdx, route := range srv.Routes { + for matcherSetIdx, matcherSet := range route.MatcherSets { + for matcherIdx, m := range matcherSet { + if hm, ok := m.(*MatchHost); ok { + for hostMatcherIdx, d := range *hm { + var err error + d, err = repl.ReplaceOrErr(d, true, false) + if err != nil { + return fmt.Errorf("%s: route %d, matcher set %d, matcher %d, host matcher %d: %v", + srvName, routeIdx, matcherSetIdx, matcherIdx, hostMatcherIdx, err) + } + if certmagic.HostQualifies(d) && + !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) { + srv.AutoHTTPS.domainSet[d] = struct{}{} + } + } + } + } + } + } + + // nothing more to do here if there are no + // domains that qualify for automatic HTTPS + if len(srv.AutoHTTPS.domainSet) == 0 { + continue + } + + // tell the server to use TLS if it is not already doing so + if srv.TLSConnPolicies == nil { + srv.TLSConnPolicies = defaultConnPolicies + } + + // nothing left to do if auto redirects are disabled + if srv.AutoHTTPS.DisableRedir { + continue + } + + app.logger.Info("enabling automatic HTTP->HTTPS redirects", + zap.String("server_name", srvName), + ) + + // create HTTP->HTTPS redirects + for _, addr := range srv.Listen { + netw, host, port, err := caddy.SplitNetworkAddress(addr) + if err != nil { + return fmt.Errorf("%s: invalid listener address: %v", srvName, addr) + } + + if parts := strings.SplitN(port, "-", 2); len(parts) == 2 { + port = parts[0] + } + redirTo := "https://{http.request.host}" + + if port != strconv.Itoa(app.httpsPort()) { + redirTo += ":" + port + } + redirTo += "{http.request.uri}" + + // build the plaintext HTTP variant of this address + httpRedirLnAddr := caddy.JoinNetworkAddress(netw, host, strconv.Itoa(app.httpPort())) + + // build the matcher set for this redirect route + // (note that we happen to bypass Provision and + // Validate steps for these matcher modules) + matcherSet := MatcherSet{MatchProtocol("http")} + if len(srv.AutoHTTPS.Skip) > 0 { + matcherSet = append(matcherSet, MatchNegate{ + Matchers: MatcherSet{MatchHost(srv.AutoHTTPS.Skip)}, + }) + } + + // create the route that does the redirect and associate + // it with the listener address it will be served from + // (note that we happen to bypass any Provision or Validate + // steps on the handler modules created here) + lnAddrRedirRoutes[httpRedirLnAddr] = Route{ + MatcherSets: []MatcherSet{matcherSet}, + Handlers: []MiddlewareHandler{ + StaticResponse{ + StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)), + Headers: http.Header{ + "Location": []string{redirTo}, + "Connection": []string{"close"}, + }, + Close: true, + }, + }, + } + } + } + + // if there are HTTP->HTTPS redirects to add, do so now + if len(lnAddrRedirRoutes) == 0 { + return nil + } + + var redirServerAddrs []string + var redirRoutes RouteList + + // for each redirect listener, see if there's already a + // server configured to listen on that exact address; if so, + // simply add the redirect route to the end of its route + // list; otherwise, we'll create a new server for all the + // listener addresses that are unused and serve the + // remaining redirects from it +redirRoutesLoop: + for addr, redirRoute := range lnAddrRedirRoutes { + for srvName, srv := range app.Servers { + if srv.hasListenerAddress(addr) { + // user has configured a server for the same address + // that the redirect runs from; simply append our + // redirect route to the existing routes, with a + // caveat that their config might override ours + app.logger.Warn("server is listening on same interface as redirects, so automatic HTTP->HTTPS redirects might be overridden by your own configuration", + zap.String("server_name", srvName), + zap.String("interface", addr), + ) + srv.Routes = append(srv.Routes, redirRoute) + continue redirRoutesLoop + } + } + // no server with this listener address exists; + // save this address and route for custom server + redirServerAddrs = append(redirServerAddrs, addr) + redirRoutes = append(redirRoutes, redirRoute) + } + + // if there are routes remaining which do not belong + // in any existing server, make our own to serve the + // rest of the redirects + if len(redirServerAddrs) > 0 { + app.Servers["remaining_auto_https_redirects"] = &Server{ + Listen: redirServerAddrs, + Routes: redirRoutes, + } + } + + return nil +} + +// automaticHTTPSPhase2 attaches a TLS app pointer to each +// server and begins certificate management for all names +// in the qualifying domain set for each server. This phase +// must occur after provisioning, and at the beginning of +// the app start, before starting each of the servers. +func (app *App) automaticHTTPSPhase2() error { + tlsAppIface, err := app.ctx.App("tls") + if err != nil { + return fmt.Errorf("getting tls app: %v", err) + } + tlsApp := tlsAppIface.(*caddytls.TLS) + + // set the tlsApp pointer before starting any + // challenges, since it is required to solve + // the ACME HTTP challenge + for _, srv := range app.Servers { + srv.tlsApp = tlsApp + } + + // begin managing certificates for enabled servers + for srvName, srv := range app.Servers { + if srv.AutoHTTPS == nil || + srv.AutoHTTPS.Disabled || + len(srv.AutoHTTPS.domainSet) == 0 { + continue + } + + // marshal the domains into a slice + var domains, domainsForCerts []string + for d := range srv.AutoHTTPS.domainSet { + domains = append(domains, d) + if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) { + // if a certificate for this name is already loaded, + // don't obtain another one for it, unless we are + // supposed to ignore loaded certificates + if !srv.AutoHTTPS.IgnoreLoadedCerts && + len(tlsApp.AllMatchingCertificates(d)) > 0 { + app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded", + zap.String("domain", d), + zap.String("server_name", srvName), + ) + continue + } + domainsForCerts = append(domainsForCerts, d) + } + } + + // ensure that these certificates are managed properly; + // for example, it's implied that the HTTPPort should also + // be the port the HTTP challenge is solved on, and so + // for HTTPS port and TLS-ALPN challenge also - we need + // to tell the TLS app to manage these certs by honoring + // those port configurations + acmeManager := &caddytls.ACMEManagerMaker{ + Challenges: &caddytls.ChallengesConfig{ + HTTP: &caddytls.HTTPChallengeConfig{ + AlternatePort: app.HTTPPort, // we specifically want the user-configured port, if any + }, + TLSALPN: &caddytls.TLSALPNChallengeConfig{ + AlternatePort: app.HTTPSPort, // we specifically want the user-configured port, if any + }, + }, + } + if tlsApp.Automation == nil { + tlsApp.Automation = new(caddytls.AutomationConfig) + } + tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, + caddytls.AutomationPolicy{ + Hosts: domainsForCerts, + Management: acmeManager, + }) + + // manage their certificates + app.logger.Info("enabling automatic TLS certificate management", + zap.Strings("domains", domainsForCerts), + ) + err := tlsApp.Manage(domainsForCerts) + if err != nil { + return fmt.Errorf("%s: managing certificate for %s: %s", srvName, domains, err) + } + + // no longer needed; allow GC to deallocate + srv.AutoHTTPS.domainSet = nil + } + + return nil +} diff --git a/modules/caddyhttp/caddyhttp.go b/modules/caddyhttp/caddyhttp.go index 73c4863..37f9670 100644 --- a/modules/caddyhttp/caddyhttp.go +++ b/modules/caddyhttp/caddyhttp.go @@ -25,13 +25,10 @@ import ( "net" "net/http" "strconv" - "strings" "time" "github.com/caddyserver/caddy/v2" - "github.com/caddyserver/caddy/v2/modules/caddytls" "github.com/lucas-clemente/quic-go/http3" - "github.com/mholt/certmagic" "go.uber.org/zap" ) @@ -52,7 +49,7 @@ func init() { // only on the HTTPS port but which do not have any TLS connection policies // defined by adding a good, default TLS connection policy. // -// In HTTP routes, additional placeholders are available: +// In HTTP routes, additional placeholders are available (replace any `*`): // // Placeholder | Description // ------------|--------------- @@ -127,6 +124,14 @@ func (app *App) Provision(ctx caddy.Context) error { repl := caddy.NewReplacer() + // this provisions the matchers for each route, + // and prepares auto HTTP->HTTP redirects, and + // is required before we provision each server + err := app.automaticHTTPSPhase1(ctx, repl) + if err != nil { + return err + } + for srvName, srv := range app.Servers { srv.logger = app.logger.Named("log") srv.errorLogger = app.logger.Named("log.error") @@ -136,11 +141,6 @@ func (app *App) Provision(ctx caddy.Context) error { srv.accessLogger = app.logger.Named("log.access") } - if srv.AutoHTTPS == nil { - // avoid nil pointer dereferences - srv.AutoHTTPS = new(AutoHTTPSConfig) - } - // if not explicitly configured by the user, disallow TLS // client auth bypass (domain fronting) which could // otherwise be exploited by sending an unprotected SNI @@ -151,6 +151,9 @@ func (app *App) Provision(ctx caddy.Context) error { // domain fronting is desired and access is not restricted // based on hostname if srv.StrictSNIHost == nil && srv.hasTLSClientAuth() { + app.logger.Info("enabling strict SNI-Host matching because TLS client auth is configured", + zap.String("server_name", srvName), + ) trueBool := true srv.StrictSNIHost = &trueBool } @@ -164,18 +167,19 @@ func (app *App) Provision(ctx caddy.Context) error { srv.Listen[i] = lnOut } + // pre-compile the primary handler chain, and be sure to wrap it in our + // route handler so that important security checks are done, etc. primaryRoute := emptyHandler if srv.Routes != nil { - err := srv.Routes.Provision(ctx) + err := srv.Routes.ProvisionHandlers(ctx) if err != nil { - return fmt.Errorf("server %s: setting up server routes: %v", srvName, err) + return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err) } - // pre-compile the handler chain, and be sure to wrap it in our - // route handler so that important security checks are done, etc. primaryRoute = srv.Routes.Compile(emptyHandler) } srv.primaryHandlerChain = srv.wrapPrimaryRoute(primaryRoute) + // pre-compile the error handler chain if srv.Errors != nil { err := srv.Errors.Routes.Provision(ctx) if err != nil { @@ -213,9 +217,12 @@ func (app *App) Validate() error { return nil } -// Start runs the app. It sets up automatic HTTPS if enabled. +// Start runs the app. It finishes automatic HTTPS if enabled, +// including management of certificates. func (app *App) Start() error { - err := app.automaticHTTPS() + // finish setting up automatic HTTPS and manage certs; + // this must happen before each server is started + err := app.automaticHTTPSPhase2() if err != nil { return fmt.Errorf("enabling automatic HTTPS: %v", err) } @@ -235,8 +242,8 @@ func (app *App) Start() error { if err != nil { return fmt.Errorf("%s: parsing listen address '%s': %v", srvName, lnAddr, err) } - for i := uint(0); i < listenAddr.PortRangeSize(); i++ { - hostport := listenAddr.JoinHostPort(i) + for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ { + hostport := listenAddr.JoinHostPort(portOffset) ln, err := caddy.Listen(listenAddr.Network, hostport) if err != nil { return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err) @@ -249,8 +256,10 @@ func (app *App) Start() error { } } - // enable TLS - if len(srv.TLSConnPolicies) > 0 && int(i) != app.httpPort() { + // enable TLS if there is a policy and if this is not the HTTP port + if len(srv.TLSConnPolicies) > 0 && + int(listenAddr.StartPort+portOffset) != app.httpPort() { + // create TLS listener tlsCfg, err := srv.TLSConnPolicies.TLSConfig(app.ctx) if err != nil { return fmt.Errorf("%s/%s: making TLS configuration: %v", listenAddr.Network, hostport, err) @@ -330,230 +339,6 @@ func (app *App) Stop() error { return nil } -func (app *App) automaticHTTPS() error { - tlsAppIface, err := app.ctx.App("tls") - if err != nil { - return fmt.Errorf("getting tls app: %v", err) - } - tlsApp := tlsAppIface.(*caddytls.TLS) - - // this map will store associations of HTTP listener - // addresses to the routes that do HTTP->HTTPS redirects - lnAddrRedirRoutes := make(map[string]Route) - - repl := caddy.NewReplacer() - - for srvName, srv := range app.Servers { - srv.tlsApp = tlsApp - - if srv.AutoHTTPS.Disabled { - continue - } - - // skip if all listeners use the HTTP port - if !srv.listenersUseAnyPortOtherThan(app.httpPort()) { - app.logger.Info("server is only listening on the HTTP port, so no automatic HTTPS will be applied to this server", - zap.String("server_name", srvName), - zap.Int("http_port", app.httpPort()), - ) - continue - } - - // if all listeners are on the HTTPS port, make sure - // there is at least one TLS connection policy; it - // should be obvious that they want to use TLS without - // needing to specify one empty policy to enable it - if !srv.listenersUseAnyPortOtherThan(app.httpsPort()) && len(srv.TLSConnPolicies) == 0 { - app.logger.Info("server is only listening on the HTTPS port but has no TLS connection policies; adding one to enable TLS", - zap.String("server_name", srvName), - zap.Int("https_port", app.httpsPort()), - ) - srv.TLSConnPolicies = append(srv.TLSConnPolicies, new(caddytls.ConnectionPolicy)) - } - - // find all qualifying domain names, de-duplicated - domainSet := make(map[string]struct{}) - for routeIdx, route := range srv.Routes { - for matcherSetIdx, matcherSet := range route.MatcherSets { - for matcherIdx, m := range matcherSet { - if hm, ok := m.(*MatchHost); ok { - for hostMatcherIdx, d := range *hm { - d, err = repl.ReplaceOrErr(d, true, false) - if err != nil { - return fmt.Errorf("%s: route %d, matcher set %d, matcher %d, host matcher %d: %v", - srvName, routeIdx, matcherSetIdx, matcherIdx, hostMatcherIdx, err) - } - if certmagic.HostQualifies(d) && - !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) { - domainSet[d] = struct{}{} - } - } - } - } - } - } - - if len(domainSet) > 0 { - // marshal the domains into a slice - var domains, domainsForCerts []string - for d := range domainSet { - domains = append(domains, d) - if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) { - // if a certificate for this name is already loaded, - // don't obtain another one for it, unless we are - // supposed to ignore loaded certificates - if !srv.AutoHTTPS.IgnoreLoadedCerts && - len(tlsApp.AllMatchingCertificates(d)) > 0 { - app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded", - zap.String("domain", d), - zap.String("server_name", srvName), - ) - continue - } - domainsForCerts = append(domainsForCerts, d) - } - } - - // ensure that these certificates are managed properly; - // for example, it's implied that the HTTPPort should also - // be the port the HTTP challenge is solved on, and so - // for HTTPS port and TLS-ALPN challenge also - we need - // to tell the TLS app to manage these certs by honoring - // those port configurations - acmeManager := &caddytls.ACMEManagerMaker{ - Challenges: &caddytls.ChallengesConfig{ - HTTP: &caddytls.HTTPChallengeConfig{ - AlternatePort: app.HTTPPort, // we specifically want the user-configured port, if any - }, - TLSALPN: &caddytls.TLSALPNChallengeConfig{ - AlternatePort: app.HTTPSPort, // we specifically want the user-configured port, if any - }, - }, - } - if tlsApp.Automation == nil { - tlsApp.Automation = new(caddytls.AutomationConfig) - } - tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, - caddytls.AutomationPolicy{ - Hosts: domainsForCerts, - Management: acmeManager, - }) - - // manage their certificates - app.logger.Info("enabling automatic TLS certificate management", - zap.Strings("domains", domainsForCerts), - ) - err := tlsApp.Manage(domainsForCerts) - if err != nil { - return fmt.Errorf("%s: managing certificate for %s: %s", srvName, domains, err) - } - - // tell the server to use TLS if it is not already doing so - if srv.TLSConnPolicies == nil { - srv.TLSConnPolicies = caddytls.ConnectionPolicies{ - &caddytls.ConnectionPolicy{ALPN: defaultALPN}, - } - } - - if srv.AutoHTTPS.DisableRedir { - continue - } - - app.logger.Info("enabling automatic HTTP->HTTPS redirects", - zap.Strings("domains", domains), - ) - - // create HTTP->HTTPS redirects - for _, addr := range srv.Listen { - netw, host, port, err := caddy.SplitNetworkAddress(addr) - if err != nil { - return fmt.Errorf("%s: invalid listener address: %v", srvName, addr) - } - - if parts := strings.SplitN(port, "-", 2); len(parts) == 2 { - port = parts[0] - } - redirTo := "https://{http.request.host}" - - if port != strconv.Itoa(app.httpsPort()) { - redirTo += ":" + port - } - redirTo += "{http.request.uri}" - - // build the plaintext HTTP variant of this address - httpRedirLnAddr := caddy.JoinNetworkAddress(netw, host, strconv.Itoa(app.httpPort())) - - // create the route that does the redirect and associate - // it with the listener address it will be served from - lnAddrRedirRoutes[httpRedirLnAddr] = Route{ - MatcherSets: []MatcherSet{{MatchProtocol("http")}}, - Handlers: []MiddlewareHandler{ - StaticResponse{ - StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)), - Headers: http.Header{ - "Location": []string{redirTo}, - "Connection": []string{"close"}, - }, - Close: true, - }, - }, - } - - } - } - } - - // if there are HTTP->HTTPS redirects to add, do so now - if len(lnAddrRedirRoutes) > 0 { - var redirServerAddrs []string - var redirRoutes RouteList - - // for each redirect listener, see if there's already a - // server configured to listen on that exact address; if so, - // simply add the redirect route to the end of its route - // list; otherwise, we'll create a new server for all the - // listener addresses that are unused and serve the - // remaining redirects from it - redirRoutesLoop: - for addr, redirRoute := range lnAddrRedirRoutes { - for srvName, srv := range app.Servers { - if srv.hasListenerAddress(addr) { - // user has configured a server for the same address - // that the redirect runs from; simply append our - // redirect route to the existing routes, with a - // caveat that their config might override ours - app.logger.Warn("server is listening on same interface as redirects, so automatic HTTP->HTTPS redirects might be overridden by your own configuration", - zap.String("server_name", srvName), - zap.String("interface", addr), - ) - srv.Routes = append(srv.Routes, redirRoute) - continue redirRoutesLoop - } - } - // no server with this listener address exists; - // save this address and route for custom server - redirServerAddrs = append(redirServerAddrs, addr) - redirRoutes = append(redirRoutes, redirRoute) - } - - // if there are routes remaining which do not belong - // in any existing server, make our own to serve the - // rest of the redirects - if len(redirServerAddrs) > 0 { - app.Servers["remaining_auto_https_redirects"] = &Server{ - Listen: redirServerAddrs, - Routes: redirRoutes, - tlsApp: tlsApp, // required to solve HTTP challenge - logger: app.logger.Named("log"), - errorLogger: app.logger.Named("log.error"), - primaryHandlerChain: redirRoutes.Compile(emptyHandler), - } - } - } - - return nil -} - func (app *App) httpPort() int { if app.HTTPPort == 0 { return DefaultHTTPPort @@ -709,7 +494,9 @@ func StatusCodeMatches(actual, configured int) bool { if actual == configured { return true } - if configured < 100 && actual >= configured*100 && actual < (configured+1)*100 { + if configured < 100 && + actual >= configured*100 && + actual < (configured+1)*100 { return true } return false diff --git a/modules/caddyhttp/fileserver/caddyfile.go b/modules/caddyhttp/fileserver/caddyfile.go index d26b435..67ae4f4 100644 --- a/modules/caddyhttp/fileserver/caddyfile.go +++ b/modules/caddyhttp/fileserver/caddyfile.go @@ -163,5 +163,10 @@ func parseTryFiles(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) result = append(result, makeRoute(try, "")...) } + // ensure that multiple routes (possible if rewrite targets + // have query strings, for example) are grouped together + // so only the first matching rewrite is performed (#2891) + h.GroupRoutes(result) + return result, nil } diff --git a/modules/caddyhttp/matchers.go b/modules/caddyhttp/matchers.go index f63e48e..49fe859 100644 --- a/modules/caddyhttp/matchers.go +++ b/modules/caddyhttp/matchers.go @@ -35,7 +35,7 @@ import ( type ( // MatchHost matches requests by the Host value (case-insensitive). // - // When used in an HTTP route, + // When used in a top-level HTTP route, // [qualifying domain names](/docs/automatic-https#hostname-requirements) // may trigger [automatic HTTPS](/docs/automatic-https), which automatically // provisions and renews certificates for you. Before doing this, you @@ -55,8 +55,8 @@ type ( // - In the middle, for a globular match (`/accounts/*/info`) // // This matcher is fast, so it does not support regular expressions or - // capture groups. For slower but more capable matching, use the path_regexp - // matcher. + // capture groups. For slower but more powerful matching, use the + // path_regexp matcher. MatchPath []string // MatchPathRE matches requests by a regular expression on the URI's path. diff --git a/modules/caddyhttp/matchers_test.go b/modules/caddyhttp/matchers_test.go index 8e06546..06f137e 100644 --- a/modules/caddyhttp/matchers_test.go +++ b/modules/caddyhttp/matchers_test.go @@ -199,7 +199,7 @@ func TestPathMatcher(t *testing.T) { }, { match: MatchPath{"*.ext"}, - input: "/foo.ext", + input: "/foo/bar.ext", expect: true, }, { diff --git a/modules/caddyhttp/reverseproxy/circuitbreaker.go b/modules/caddyhttp/reverseproxy/circuitbreaker.go index 474f1c6..00b38a8 100644 --- a/modules/caddyhttp/reverseproxy/circuitbreaker.go +++ b/modules/caddyhttp/reverseproxy/circuitbreaker.go @@ -31,7 +31,7 @@ func init() { // for requests within this process over a sliding time window. type localCircuitBreaker struct { tripped int32 - cbType int32 + cbFactor int32 threshold float64 metrics *memmetrics.RTMetrics tripTime time.Duration @@ -48,7 +48,7 @@ func (localCircuitBreaker) CaddyModule() caddy.ModuleInfo { // Provision sets up a configured circuit breaker. func (c *localCircuitBreaker) Provision(ctx caddy.Context) error { - t, ok := typeCB[c.Type] + f, ok := typeCB[c.Factor] if !ok { return fmt.Errorf("type is not defined") } @@ -67,7 +67,7 @@ func (c *localCircuitBreaker) Provision(ctx caddy.Context) error { return fmt.Errorf("cannot create new metrics: %v", err.Error()) } - c.cbType = t + c.cbFactor = f c.tripTime = tw c.threshold = c.Threshold c.metrics = mt @@ -92,13 +92,13 @@ func (c *localCircuitBreaker) RecordMetric(statusCode int, latency time.Duration func (c *localCircuitBreaker) checkAndSet() { var isTripped bool - switch c.cbType { - case typeErrorRatio: + switch c.cbFactor { + case factorErrorRatio: // check if amount of network errors exceed threshold over sliding window, threshold for comparison should be < 1.0 i.e. .5 = 50th percentile if c.metrics.NetworkErrorRatio() > c.threshold { isTripped = true } - case typeLatency: + case factorLatency: // check if threshold in milliseconds is reached and trip hist, err := c.metrics.LatencyHistogram() if err != nil { @@ -109,7 +109,7 @@ func (c *localCircuitBreaker) checkAndSet() { if l.Nanoseconds()/int64(time.Millisecond) > int64(c.threshold) { isTripped = true } - case typeStatusCodeRatio: + case factorStatusCodeRatio: // check ratio of error status codes of sliding window, threshold for comparison should be < 1.0 i.e. .5 = 50th percentile if c.metrics.ResponseCodeRatio(500, 600, 0, 600) > c.threshold { isTripped = true @@ -130,23 +130,28 @@ func (c *localCircuitBreaker) checkAndSet() { // Config represents the configuration of a circuit breaker. type Config struct { + // The threshold over sliding window that would trip the circuit breaker Threshold float64 `json:"threshold"` - Type string `json:"type"` - TripTime string `json:"trip_time"` + // Possible values: latency, error_ratio, and status_ratio. It + // defaults to latency. + Factor string `json:"factor"` + // How long to wait after the circuit is tripped before allowing operations to resume. + // The default is 5s. + TripTime string `json:"trip_time"` } const ( - typeLatency = iota + 1 - typeErrorRatio - typeStatusCodeRatio + factorLatency = iota + 1 + factorErrorRatio + factorStatusCodeRatio defaultTripTime = "5s" ) var ( - // typeCB handles converting a Config Type value to the internal circuit breaker types. + // typeCB handles converting a Config Factor value to the internal circuit breaker types. typeCB = map[string]int32{ - "latency": typeLatency, - "error_ratio": typeErrorRatio, - "status_ratio": typeStatusCodeRatio, + "latency": factorLatency, + "error_ratio": factorErrorRatio, + "status_ratio": factorStatusCodeRatio, } ) diff --git a/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go b/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go index dee6eb5..8c9fd38 100644 --- a/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go @@ -81,7 +81,7 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // // php_fastcgi localhost:7777 // -// is equivalent to: +// is equivalent to a route consisting of: // // @canonicalPath { // file { @@ -104,8 +104,8 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // } // } // -// Thus, this directive produces multiple routes, each with a different -// matcher because multiple consecutive routes are necessary to support +// Thus, this directive produces multiple handlers, each with a different +// matcher because multiple consecutive hgandlers are necessary to support // the common PHP use case. If this "common" config is not compatible // with a user's PHP requirements, they can use a manual approach based // on the example above to configure it precisely as they need. @@ -114,7 +114,7 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // // php_fastcgi /subpath localhost:7777 // -// then the resulting routes are wrapped in a subroute that uses the +// then the resulting handlers are wrapped in a subroute that uses the // user's matcher as a prerequisite to enter the subroute. In other // words, the directive's matcher is necessary, but not sufficient. func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) { @@ -198,12 +198,13 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error HandlersRaw: []json.RawMessage{caddyconfig.JSONModuleObject(rpHandler, "handler", "reverse_proxy", nil)}, } + subroute := caddyhttp.Subroute{ + Routes: caddyhttp.RouteList{redirRoute, rewriteRoute, rpRoute}, + } + // the user's matcher is a prerequisite for ours, so // wrap ours in a subroute and return that if hasUserMatcher { - subroute := caddyhttp.Subroute{ - Routes: caddyhttp.RouteList{redirRoute, rewriteRoute, rpRoute}, - } return []httpcaddyfile.ConfigValue{ { Class: "route", @@ -215,20 +216,14 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error }, nil } - // if the user did not specify a matcher, then - // we can just use our own matchers + // otherwise, return the literal subroute instead of + // individual routes, to ensure they stay together and + // are treated as a single unit, without necessarily + // creating an actual subroute in the output return []httpcaddyfile.ConfigValue{ { Class: "route", - Value: redirRoute, - }, - { - Class: "route", - Value: rewriteRoute, - }, - { - Class: "route", - Value: rpRoute, + Value: subroute, }, }, nil } diff --git a/modules/caddyhttp/reverseproxy/selectionpolicies.go b/modules/caddyhttp/reverseproxy/selectionpolicies.go index 937ae37..330dbdc 100644 --- a/modules/caddyhttp/reverseproxy/selectionpolicies.go +++ b/modules/caddyhttp/reverseproxy/selectionpolicies.go @@ -78,6 +78,8 @@ func (r RandomSelection) Select(pool UpstreamPool, request *http.Request) *Upstr // two or more available hosts at random, then // chooses the one with the least load. type RandomChoiceSelection struct { + // The size of the sub-pool created from the larger upstream pool. The default value + // is 2 and the maximum at selection time is the size of the upstream pool. Choose int `json:"choose,omitempty"` } @@ -283,6 +285,7 @@ func (URIHashSelection) Select(pool UpstreamPool, req *http.Request) *Upstream { // HeaderHashSelection is a policy that selects // a host based on a given request header. type HeaderHashSelection struct { + // The HTTP header field whose value is to be hashed and used for upstream selection. Field string `json:"field,omitempty"` } diff --git a/modules/caddyhttp/rewrite/rewrite.go b/modules/caddyhttp/rewrite/rewrite.go index abcaa84..ad05486 100644 --- a/modules/caddyhttp/rewrite/rewrite.go +++ b/modules/caddyhttp/rewrite/rewrite.go @@ -50,7 +50,7 @@ type Rewrite struct { // You can also use placeholders. For example, to preserve the existing // query string, you might use: "?{http.request.uri.query}&a=b". Any // key-value pairs you add to the query string will not overwrite - // existing values. + // existing values (individual pairs are append-only). // // To clear the query string, explicitly set an empty one: "?" URI string `json:"uri,omitempty"` @@ -112,7 +112,7 @@ func (rewr Rewrite) rewrite(r *http.Request, repl *caddy.Replacer, logger *zap.L r.Method = strings.ToUpper(repl.ReplaceAll(rewr.Method, "")) } - // uri (path, query string, and fragment just because) + // uri (path, query string, and fragment... because why not) if uri := rewr.URI; uri != "" { // find the bounds of each part of the URI that exist pathStart, qsStart, fragStart := -1, -1, -1 @@ -134,14 +134,30 @@ func (rewr Rewrite) rewrite(r *http.Request, repl *caddy.Replacer, logger *zap.L qsEnd = len(uri) } + // build components which are specified, and store them + // in a temporary variable so that they all read the + // same version of the URI + var newPath, newQuery, newFrag string if pathStart >= 0 { - r.URL.Path = repl.ReplaceAll(uri[pathStart:pathEnd], "") + newPath = repl.ReplaceAll(uri[pathStart:pathEnd], "") } if qsStart >= 0 { - r.URL.RawQuery = buildQueryString(uri[qsStart:qsEnd], repl) + newQuery = buildQueryString(uri[qsStart:qsEnd], repl) } if fragStart >= 0 { - r.URL.Fragment = repl.ReplaceAll(uri[fragStart:], "") + newFrag = repl.ReplaceAll(uri[fragStart:], "") + } + + // update the URI with the new components + // only after building them + if pathStart >= 0 { + r.URL.Path = newPath + } + if qsStart >= 0 { + r.URL.RawQuery = newQuery + } + if fragStart >= 0 { + r.URL.Fragment = newFrag } } @@ -206,7 +222,7 @@ func buildQueryString(qs string, repl *caddy.Replacer) string { // if previous iteration wrote a value, // that means we are writing a key if wroteVal { - if sb.Len() > 0 { + if sb.Len() > 0 && len(comp) > 0 { sb.WriteRune('&') } } else { diff --git a/modules/caddyhttp/rewrite/rewrite_test.go b/modules/caddyhttp/rewrite/rewrite_test.go index 3dbc2d6..34a0cdb 100644 --- a/modules/caddyhttp/rewrite/rewrite_test.go +++ b/modules/caddyhttp/rewrite/rewrite_test.go @@ -64,6 +64,16 @@ func TestRewrite(t *testing.T) { expect: newRequest(t, "GET", "/foo/bar"), }, { + rule: Rewrite{URI: "/index.php?p={http.request.uri.path}"}, + input: newRequest(t, "GET", "/foo/bar"), + expect: newRequest(t, "GET", "/index.php?p=%2Ffoo%2Fbar"), + }, + { + rule: Rewrite{URI: "?a=b&{http.request.uri.query}"}, + input: newRequest(t, "GET", "/"), + expect: newRequest(t, "GET", "/?a=b"), + }, + { rule: Rewrite{URI: "/?c=d"}, input: newRequest(t, "GET", "/"), expect: newRequest(t, "GET", "/?c=d"), diff --git a/modules/caddyhttp/routes.go b/modules/caddyhttp/routes.go index 431d1a5..1224e32 100644 --- a/modules/caddyhttp/routes.go +++ b/modules/caddyhttp/routes.go @@ -113,23 +113,43 @@ func (r Route) Empty() bool { // create a middleware chain. type RouteList []Route -// Provision sets up all the routes by loading the modules. +// Provision sets up both the matchers and handlers in the route. func (routes RouteList) Provision(ctx caddy.Context) error { + err := routes.ProvisionMatchers(ctx) + if err != nil { + return err + } + return routes.ProvisionHandlers(ctx) +} + +// ProvisionMatchers sets up all the matchers by loading the +// matcher modules. Only call this method directly if you need +// to set up matchers and handlers separately without having +// to provision a second time; otherwise use Provision instead. +func (routes RouteList) ProvisionMatchers(ctx caddy.Context) error { for i := range routes { // matchers matchersIface, err := ctx.LoadModule(&routes[i], "MatcherSetsRaw") if err != nil { - return fmt.Errorf("loading matchers in route %d: %v", i, err) + return fmt.Errorf("route %d: loading matcher modules: %v", i, err) } err = routes[i].MatcherSets.FromInterface(matchersIface) if err != nil { return fmt.Errorf("route %d: %v", i, err) } + } + return nil +} - // handlers +// ProvisionHandlers sets up all the handlers by loading the +// handler modules. Only call this method directly if you need +// to set up matchers and handlers separately without having +// to provision a second time; otherwise use Provision instead. +func (routes RouteList) ProvisionHandlers(ctx caddy.Context) error { + for i := range routes { handlersIface, err := ctx.LoadModule(&routes[i], "HandlersRaw") if err != nil { - return fmt.Errorf("loading handler modules in route %d: %v", i, err) + return fmt.Errorf("route %d: loading handler modules: %v", i, err) } for _, handler := range handlersIface.([]interface{}) { routes[i].Handlers = append(routes[i].Handlers, handler.(MiddlewareHandler)) @@ -140,7 +160,6 @@ func (routes RouteList) Provision(ctx caddy.Context) error { routes[i].middleware = append(routes[i].middleware, wrapMiddleware(midhandler)) } } - return nil } @@ -233,7 +252,6 @@ func wrapMiddleware(mh MiddlewareHandler) Middleware { return HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { // TODO: This is where request tracing could be implemented - // TODO: Trace a diff of the request, would be cool too... see what changed since the last middleware (host, headers, URI...) // TODO: see what the std lib gives us in terms of stack tracing too return mh.ServeHTTP(w, r, nextCopy) }) diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index ce61b13..1c896a4 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -173,7 +173,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } log("handled request", - zap.String("common_log", repl.ReplaceAll(CommonLogFormat, "-")), + zap.String("common_log", repl.ReplaceAll(commonLogFormat, "-")), zap.Duration("latency", latency), zap.Int("size", wrec.Size()), zap.Int("status", wrec.Status()), @@ -317,49 +317,6 @@ func (s *Server) hasTLSClientAuth() bool { return false } -// AutoHTTPSConfig is used to disable automatic HTTPS -// or certain aspects of it for a specific server. -// HTTPS is enabled automatically and by default when -// qualifying hostnames are available from the config. -type AutoHTTPSConfig struct { - // If true, automatic HTTPS will be entirely disabled. - Disabled bool `json:"disable,omitempty"` - - // If true, only automatic HTTP->HTTPS redirects will - // be disabled. - DisableRedir bool `json:"disable_redirects,omitempty"` - - // Hosts/domain names listed here will not be included - // in automatic HTTPS (they will not have certificates - // loaded nor redirects applied). - Skip []string `json:"skip,omitempty"` - - // Hosts/domain names listed here will still be enabled - // for automatic HTTPS (unless in the Skip list), except - // that certificates will not be provisioned and managed - // for these names. - SkipCerts []string `json:"skip_certificates,omitempty"` - - // By default, automatic HTTPS will obtain and renew - // certificates for qualifying hostnames. However, if - // a certificate with a matching SAN is already loaded - // into the cache, certificate management will not be - // enabled. To force automated certificate management - // regardless of loaded certificates, set this to true. - IgnoreLoadedCerts bool `json:"ignore_loaded_certificates,omitempty"` -} - -// Skipped returns true if name is in skipSlice, which -// should be one of the Skip* fields on ahc. -func (ahc AutoHTTPSConfig) Skipped(name string, skipSlice []string) bool { - for _, n := range skipSlice { - if name == n { - return true - } - } - return false -} - // HTTPErrorConfig determines how to handle errors // from the HTTP handlers. type HTTPErrorConfig struct { @@ -466,11 +423,11 @@ func cloneURL(from, to *url.URL) { } const ( - // CommonLogFormat is the common log format. https://en.wikipedia.org/wiki/Common_Log_Format - CommonLogFormat = `{http.request.remote.host} ` + CommonLogEmptyValue + ` {http.authentication.user.id} [{time.now.common_log}] "{http.request.orig_method} {http.request.orig_uri} {http.request.proto}" {http.response.status} {http.response.size}` + // commonLogFormat is the common log format. https://en.wikipedia.org/wiki/Common_Log_Format + commonLogFormat = `{http.request.remote.host} ` + commonLogEmptyValue + ` {http.authentication.user.id} [{time.now.common_log}] "{http.request.orig_method} {http.request.orig_uri} {http.request.proto}" {http.response.status} {http.response.size}` - // CommonLogEmptyValue is the common empty log value. - CommonLogEmptyValue = "-" + // commonLogEmptyValue is the common empty log value. + commonLogEmptyValue = "-" ) // Context keys for HTTP request context values. diff --git a/modules/caddyhttp/staticresp.go b/modules/caddyhttp/staticresp.go index 0ca2f43..777ecb2 100644 --- a/modules/caddyhttp/staticresp.go +++ b/modules/caddyhttp/staticresp.go @@ -54,24 +54,39 @@ func (StaticResponse) CaddyModule() caddy.ModuleInfo { // UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax: // -// respond [<matcher>] <status> { +// respond [<matcher>] [<status>|[<body> [<status>]] { // body <text> // close // } // +// If there is just one argument (other than the matcher), it is considered +// to be a status code if it's a valid positive integer of 3 digits. func (s *StaticResponse) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { for d.Next() { - var statusCodeStr string - if d.Args(&statusCodeStr) { - s.StatusCode = WeakString(statusCodeStr) + args := d.RemainingArgs() + switch len(args) { + case 1: + if len(args[0]) == 3 { + if num, err := strconv.Atoi(args[0]); err == nil && num > 0 { + s.StatusCode = WeakString(args[0]) + break + } + } + s.Body = args[0] + case 2: + s.Body = args[0] + s.StatusCode = WeakString(args[1]) + default: + return d.ArgErr() } + for d.NextBlock(0) { switch d.Val() { case "body": if s.Body != "" { return d.Err("body already specified") } - if !d.Args(&s.Body) { + if !d.AllArgs(&s.Body) { return d.ArgErr() } case "close": |