diff options
Diffstat (limited to 'modules')
-rw-r--r-- | modules/caddyhttp/autohttps.go | 347 | ||||
-rw-r--r-- | modules/caddyhttp/caddyhttp.go | 275 | ||||
-rw-r--r-- | modules/caddyhttp/fileserver/caddyfile.go | 5 | ||||
-rw-r--r-- | modules/caddyhttp/matchers.go | 6 | ||||
-rw-r--r-- | modules/caddyhttp/matchers_test.go | 2 | ||||
-rw-r--r-- | modules/caddyhttp/reverseproxy/circuitbreaker.go | 37 | ||||
-rw-r--r-- | modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go | 31 | ||||
-rw-r--r-- | modules/caddyhttp/reverseproxy/selectionpolicies.go | 3 | ||||
-rw-r--r-- | modules/caddyhttp/rewrite/rewrite.go | 28 | ||||
-rw-r--r-- | modules/caddyhttp/rewrite/rewrite_test.go | 10 | ||||
-rw-r--r-- | modules/caddyhttp/routes.go | 30 | ||||
-rw-r--r-- | modules/caddyhttp/server.go | 53 | ||||
-rw-r--r-- | modules/caddyhttp/staticresp.go | 25 |
13 files changed, 505 insertions, 347 deletions
diff --git a/modules/caddyhttp/autohttps.go b/modules/caddyhttp/autohttps.go new file mode 100644 index 0000000..69e3318 --- /dev/null +++ b/modules/caddyhttp/autohttps.go @@ -0,0 +1,347 @@ +package caddyhttp + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/mholt/certmagic" + "go.uber.org/zap" +) + +// AutoHTTPSConfig is used to disable automatic HTTPS +// or certain aspects of it for a specific server. +// HTTPS is enabled automatically and by default when +// qualifying hostnames are available from the config. +type AutoHTTPSConfig struct { + // If true, automatic HTTPS will be entirely disabled. + Disabled bool `json:"disable,omitempty"` + + // If true, only automatic HTTP->HTTPS redirects will + // be disabled. + DisableRedir bool `json:"disable_redirects,omitempty"` + + // Hosts/domain names listed here will not be included + // in automatic HTTPS (they will not have certificates + // loaded nor redirects applied). + Skip []string `json:"skip,omitempty"` + + // Hosts/domain names listed here will still be enabled + // for automatic HTTPS (unless in the Skip list), except + // that certificates will not be provisioned and managed + // for these names. + SkipCerts []string `json:"skip_certificates,omitempty"` + + // By default, automatic HTTPS will obtain and renew + // certificates for qualifying hostnames. However, if + // a certificate with a matching SAN is already loaded + // into the cache, certificate management will not be + // enabled. To force automated certificate management + // regardless of loaded certificates, set this to true. + IgnoreLoadedCerts bool `json:"ignore_loaded_certificates,omitempty"` + + domainSet map[string]struct{} +} + +// Skipped returns true if name is in skipSlice, which +// should be one of the Skip* fields on ahc. +func (ahc AutoHTTPSConfig) Skipped(name string, skipSlice []string) bool { + for _, n := range skipSlice { + if name == n { + return true + } + } + return false +} + +// automaticHTTPSPhase1 provisions all route matchers, determines +// which domain names found in the routes qualify for automatic +// HTTPS, and sets up HTTP->HTTPS redirects. This phase must occur +// at the beginning of provisioning, because it may add routes and +// even servers to the app, which still need to be set up with the +// rest of them. +func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) error { + // this map will store associations of HTTP listener + // addresses to the routes that do HTTP->HTTPS redirects + lnAddrRedirRoutes := make(map[string]Route) + + for srvName, srv := range app.Servers { + // as a prerequisite, provision route matchers; this is + // required for all routes on all servers, and must be + // done before we attempt to do phase 1 of auto HTTPS, + // since we have to access the decoded host matchers the + // handlers will be provisioned later + if srv.Routes != nil { + err := srv.Routes.ProvisionMatchers(ctx) + if err != nil { + return fmt.Errorf("server %s: setting up route matchers: %v", srvName, err) + } + } + + // prepare for automatic HTTPS + if srv.AutoHTTPS == nil { + srv.AutoHTTPS = new(AutoHTTPSConfig) + } + if srv.AutoHTTPS.Disabled { + continue + } + + // skip if all listeners use the HTTP port + if !srv.listenersUseAnyPortOtherThan(app.httpPort()) { + app.logger.Info("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server", + zap.String("server_name", srvName), + zap.Int("http_port", app.httpPort()), + ) + srv.AutoHTTPS.Disabled = true + continue + } + + defaultConnPolicies := caddytls.ConnectionPolicies{ + &caddytls.ConnectionPolicy{ALPN: defaultALPN}, + } + + // if all listeners are on the HTTPS port, make sure + // there is at least one TLS connection policy; it + // should be obvious that they want to use TLS without + // needing to specify one empty policy to enable it + if srv.TLSConnPolicies == nil && + !srv.listenersUseAnyPortOtherThan(app.httpsPort()) { + app.logger.Info("server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS", + zap.String("server_name", srvName), + zap.Int("https_port", app.httpsPort()), + ) + srv.TLSConnPolicies = defaultConnPolicies + } + + // find all qualifying domain names in this server + srv.AutoHTTPS.domainSet = make(map[string]struct{}) + for routeIdx, route := range srv.Routes { + for matcherSetIdx, matcherSet := range route.MatcherSets { + for matcherIdx, m := range matcherSet { + if hm, ok := m.(*MatchHost); ok { + for hostMatcherIdx, d := range *hm { + var err error + d, err = repl.ReplaceOrErr(d, true, false) + if err != nil { + return fmt.Errorf("%s: route %d, matcher set %d, matcher %d, host matcher %d: %v", + srvName, routeIdx, matcherSetIdx, matcherIdx, hostMatcherIdx, err) + } + if certmagic.HostQualifies(d) && + !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) { + srv.AutoHTTPS.domainSet[d] = struct{}{} + } + } + } + } + } + } + + // nothing more to do here if there are no + // domains that qualify for automatic HTTPS + if len(srv.AutoHTTPS.domainSet) == 0 { + continue + } + + // tell the server to use TLS if it is not already doing so + if srv.TLSConnPolicies == nil { + srv.TLSConnPolicies = defaultConnPolicies + } + + // nothing left to do if auto redirects are disabled + if srv.AutoHTTPS.DisableRedir { + continue + } + + app.logger.Info("enabling automatic HTTP->HTTPS redirects", + zap.String("server_name", srvName), + ) + + // create HTTP->HTTPS redirects + for _, addr := range srv.Listen { + netw, host, port, err := caddy.SplitNetworkAddress(addr) + if err != nil { + return fmt.Errorf("%s: invalid listener address: %v", srvName, addr) + } + + if parts := strings.SplitN(port, "-", 2); len(parts) == 2 { + port = parts[0] + } + redirTo := "https://{http.request.host}" + + if port != strconv.Itoa(app.httpsPort()) { + redirTo += ":" + port + } + redirTo += "{http.request.uri}" + + // build the plaintext HTTP variant of this address + httpRedirLnAddr := caddy.JoinNetworkAddress(netw, host, strconv.Itoa(app.httpPort())) + + // build the matcher set for this redirect route + // (note that we happen to bypass Provision and + // Validate steps for these matcher modules) + matcherSet := MatcherSet{MatchProtocol("http")} + if len(srv.AutoHTTPS.Skip) > 0 { + matcherSet = append(matcherSet, MatchNegate{ + Matchers: MatcherSet{MatchHost(srv.AutoHTTPS.Skip)}, + }) + } + + // create the route that does the redirect and associate + // it with the listener address it will be served from + // (note that we happen to bypass any Provision or Validate + // steps on the handler modules created here) + lnAddrRedirRoutes[httpRedirLnAddr] = Route{ + MatcherSets: []MatcherSet{matcherSet}, + Handlers: []MiddlewareHandler{ + StaticResponse{ + StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)), + Headers: http.Header{ + "Location": []string{redirTo}, + "Connection": []string{"close"}, + }, + Close: true, + }, + }, + } + } + } + + // if there are HTTP->HTTPS redirects to add, do so now + if len(lnAddrRedirRoutes) == 0 { + return nil + } + + var redirServerAddrs []string + var redirRoutes RouteList + + // for each redirect listener, see if there's already a + // server configured to listen on that exact address; if so, + // simply add the redirect route to the end of its route + // list; otherwise, we'll create a new server for all the + // listener addresses that are unused and serve the + // remaining redirects from it +redirRoutesLoop: + for addr, redirRoute := range lnAddrRedirRoutes { + for srvName, srv := range app.Servers { + if srv.hasListenerAddress(addr) { + // user has configured a server for the same address + // that the redirect runs from; simply append our + // redirect route to the existing routes, with a + // caveat that their config might override ours + app.logger.Warn("server is listening on same interface as redirects, so automatic HTTP->HTTPS redirects might be overridden by your own configuration", + zap.String("server_name", srvName), + zap.String("interface", addr), + ) + srv.Routes = append(srv.Routes, redirRoute) + continue redirRoutesLoop + } + } + // no server with this listener address exists; + // save this address and route for custom server + redirServerAddrs = append(redirServerAddrs, addr) + redirRoutes = append(redirRoutes, redirRoute) + } + + // if there are routes remaining which do not belong + // in any existing server, make our own to serve the + // rest of the redirects + if len(redirServerAddrs) > 0 { + app.Servers["remaining_auto_https_redirects"] = &Server{ + Listen: redirServerAddrs, + Routes: redirRoutes, + } + } + + return nil +} + +// automaticHTTPSPhase2 attaches a TLS app pointer to each +// server and begins certificate management for all names +// in the qualifying domain set for each server. This phase +// must occur after provisioning, and at the beginning of +// the app start, before starting each of the servers. +func (app *App) automaticHTTPSPhase2() error { + tlsAppIface, err := app.ctx.App("tls") + if err != nil { + return fmt.Errorf("getting tls app: %v", err) + } + tlsApp := tlsAppIface.(*caddytls.TLS) + + // set the tlsApp pointer before starting any + // challenges, since it is required to solve + // the ACME HTTP challenge + for _, srv := range app.Servers { + srv.tlsApp = tlsApp + } + + // begin managing certificates for enabled servers + for srvName, srv := range app.Servers { + if srv.AutoHTTPS == nil || + srv.AutoHTTPS.Disabled || + len(srv.AutoHTTPS.domainSet) == 0 { + continue + } + + // marshal the domains into a slice + var domains, domainsForCerts []string + for d := range srv.AutoHTTPS.domainSet { + domains = append(domains, d) + if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) { + // if a certificate for this name is already loaded, + // don't obtain another one for it, unless we are + // supposed to ignore loaded certificates + if !srv.AutoHTTPS.IgnoreLoadedCerts && + len(tlsApp.AllMatchingCertificates(d)) > 0 { + app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded", + zap.String("domain", d), + zap.String("server_name", srvName), + ) + continue + } + domainsForCerts = append(domainsForCerts, d) + } + } + + // ensure that these certificates are managed properly; + // for example, it's implied that the HTTPPort should also + // be the port the HTTP challenge is solved on, and so + // for HTTPS port and TLS-ALPN challenge also - we need + // to tell the TLS app to manage these certs by honoring + // those port configurations + acmeManager := &caddytls.ACMEManagerMaker{ + Challenges: &caddytls.ChallengesConfig{ + HTTP: &caddytls.HTTPChallengeConfig{ + AlternatePort: app.HTTPPort, // we specifically want the user-configured port, if any + }, + TLSALPN: &caddytls.TLSALPNChallengeConfig{ + AlternatePort: app.HTTPSPort, // we specifically want the user-configured port, if any + }, + }, + } + if tlsApp.Automation == nil { + tlsApp.Automation = new(caddytls.AutomationConfig) + } + tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, + caddytls.AutomationPolicy{ + Hosts: domainsForCerts, + Management: acmeManager, + }) + + // manage their certificates + app.logger.Info("enabling automatic TLS certificate management", + zap.Strings("domains", domainsForCerts), + ) + err := tlsApp.Manage(domainsForCerts) + if err != nil { + return fmt.Errorf("%s: managing certificate for %s: %s", srvName, domains, err) + } + + // no longer needed; allow GC to deallocate + srv.AutoHTTPS.domainSet = nil + } + + return nil +} diff --git a/modules/caddyhttp/caddyhttp.go b/modules/caddyhttp/caddyhttp.go index 73c4863..37f9670 100644 --- a/modules/caddyhttp/caddyhttp.go +++ b/modules/caddyhttp/caddyhttp.go @@ -25,13 +25,10 @@ import ( "net" "net/http" "strconv" - "strings" "time" "github.com/caddyserver/caddy/v2" - "github.com/caddyserver/caddy/v2/modules/caddytls" "github.com/lucas-clemente/quic-go/http3" - "github.com/mholt/certmagic" "go.uber.org/zap" ) @@ -52,7 +49,7 @@ func init() { // only on the HTTPS port but which do not have any TLS connection policies // defined by adding a good, default TLS connection policy. // -// In HTTP routes, additional placeholders are available: +// In HTTP routes, additional placeholders are available (replace any `*`): // // Placeholder | Description // ------------|--------------- @@ -127,6 +124,14 @@ func (app *App) Provision(ctx caddy.Context) error { repl := caddy.NewReplacer() + // this provisions the matchers for each route, + // and prepares auto HTTP->HTTP redirects, and + // is required before we provision each server + err := app.automaticHTTPSPhase1(ctx, repl) + if err != nil { + return err + } + for srvName, srv := range app.Servers { srv.logger = app.logger.Named("log") srv.errorLogger = app.logger.Named("log.error") @@ -136,11 +141,6 @@ func (app *App) Provision(ctx caddy.Context) error { srv.accessLogger = app.logger.Named("log.access") } - if srv.AutoHTTPS == nil { - // avoid nil pointer dereferences - srv.AutoHTTPS = new(AutoHTTPSConfig) - } - // if not explicitly configured by the user, disallow TLS // client auth bypass (domain fronting) which could // otherwise be exploited by sending an unprotected SNI @@ -151,6 +151,9 @@ func (app *App) Provision(ctx caddy.Context) error { // domain fronting is desired and access is not restricted // based on hostname if srv.StrictSNIHost == nil && srv.hasTLSClientAuth() { + app.logger.Info("enabling strict SNI-Host matching because TLS client auth is configured", + zap.String("server_name", srvName), + ) trueBool := true srv.StrictSNIHost = &trueBool } @@ -164,18 +167,19 @@ func (app *App) Provision(ctx caddy.Context) error { srv.Listen[i] = lnOut } + // pre-compile the primary handler chain, and be sure to wrap it in our + // route handler so that important security checks are done, etc. primaryRoute := emptyHandler if srv.Routes != nil { - err := srv.Routes.Provision(ctx) + err := srv.Routes.ProvisionHandlers(ctx) if err != nil { - return fmt.Errorf("server %s: setting up server routes: %v", srvName, err) + return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err) } - // pre-compile the handler chain, and be sure to wrap it in our - // route handler so that important security checks are done, etc. primaryRoute = srv.Routes.Compile(emptyHandler) } srv.primaryHandlerChain = srv.wrapPrimaryRoute(primaryRoute) + // pre-compile the error handler chain if srv.Errors != nil { err := srv.Errors.Routes.Provision(ctx) if err != nil { @@ -213,9 +217,12 @@ func (app *App) Validate() error { return nil } -// Start runs the app. It sets up automatic HTTPS if enabled. +// Start runs the app. It finishes automatic HTTPS if enabled, +// including management of certificates. func (app *App) Start() error { - err := app.automaticHTTPS() + // finish setting up automatic HTTPS and manage certs; + // this must happen before each server is started + err := app.automaticHTTPSPhase2() if err != nil { return fmt.Errorf("enabling automatic HTTPS: %v", err) } @@ -235,8 +242,8 @@ func (app *App) Start() error { if err != nil { return fmt.Errorf("%s: parsing listen address '%s': %v", srvName, lnAddr, err) } - for i := uint(0); i < listenAddr.PortRangeSize(); i++ { - hostport := listenAddr.JoinHostPort(i) + for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ { + hostport := listenAddr.JoinHostPort(portOffset) ln, err := caddy.Listen(listenAddr.Network, hostport) if err != nil { return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err) @@ -249,8 +256,10 @@ func (app *App) Start() error { } } - // enable TLS - if len(srv.TLSConnPolicies) > 0 && int(i) != app.httpPort() { + // enable TLS if there is a policy and if this is not the HTTP port + if len(srv.TLSConnPolicies) > 0 && + int(listenAddr.StartPort+portOffset) != app.httpPort() { + // create TLS listener tlsCfg, err := srv.TLSConnPolicies.TLSConfig(app.ctx) if err != nil { return fmt.Errorf("%s/%s: making TLS configuration: %v", listenAddr.Network, hostport, err) @@ -330,230 +339,6 @@ func (app *App) Stop() error { return nil } -func (app *App) automaticHTTPS() error { - tlsAppIface, err := app.ctx.App("tls") - if err != nil { - return fmt.Errorf("getting tls app: %v", err) - } - tlsApp := tlsAppIface.(*caddytls.TLS) - - // this map will store associations of HTTP listener - // addresses to the routes that do HTTP->HTTPS redirects - lnAddrRedirRoutes := make(map[string]Route) - - repl := caddy.NewReplacer() - - for srvName, srv := range app.Servers { - srv.tlsApp = tlsApp - - if srv.AutoHTTPS.Disabled { - continue - } - - // skip if all listeners use the HTTP port - if !srv.listenersUseAnyPortOtherThan(app.httpPort()) { - app.logger.Info("server is only listening on the HTTP port, so no automatic HTTPS will be applied to this server", - zap.String("server_name", srvName), - zap.Int("http_port", app.httpPort()), - ) - continue - } - - // if all listeners are on the HTTPS port, make sure - // there is at least one TLS connection policy; it - // should be obvious that they want to use TLS without - // needing to specify one empty policy to enable it - if !srv.listenersUseAnyPortOtherThan(app.httpsPort()) && len(srv.TLSConnPolicies) == 0 { - app.logger.Info("server is only listening on the HTTPS port but has no TLS connection policies; adding one to enable TLS", - zap.String("server_name", srvName), - zap.Int("https_port", app.httpsPort()), - ) - srv.TLSConnPolicies = append(srv.TLSConnPolicies, new(caddytls.ConnectionPolicy)) - } - - // find all qualifying domain names, de-duplicated - domainSet := make(map[string]struct{}) - for routeIdx, route := range srv.Routes { - for matcherSetIdx, matcherSet := range route.MatcherSets { - for matcherIdx, m := range matcherSet { - if hm, ok := m.(*MatchHost); ok { - for hostMatcherIdx, d := range *hm { - d, err = repl.ReplaceOrErr(d, true, false) - if err != nil { - return fmt.Errorf("%s: route %d, matcher set %d, matcher %d, host matcher %d: %v", - srvName, routeIdx, matcherSetIdx, matcherIdx, hostMatcherIdx, err) - } - if certmagic.HostQualifies(d) && - !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) { - domainSet[d] = struct{}{} - } - } - } - } - } - } - - if len(domainSet) > 0 { - // marshal the domains into a slice - var domains, domainsForCerts []string - for d := range domainSet { - domains = append(domains, d) - if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) { - // if a certificate for this name is already loaded, - // don't obtain another one for it, unless we are - // supposed to ignore loaded certificates - if !srv.AutoHTTPS.IgnoreLoadedCerts && - len(tlsApp.AllMatchingCertificates(d)) > 0 { - app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded", - zap.String("domain", d), - zap.String("server_name", srvName), - ) - continue - } - domainsForCerts = append(domainsForCerts, d) - } - } - - // ensure that these certificates are managed properly; - // for example, it's implied that the HTTPPort should also - // be the port the HTTP challenge is solved on, and so - // for HTTPS port and TLS-ALPN challenge also - we need - // to tell the TLS app to manage these certs by honoring - // those port configurations - acmeManager := &caddytls.ACMEManagerMaker{ - Challenges: &caddytls.ChallengesConfig{ - HTTP: &caddytls.HTTPChallengeConfig{ - AlternatePort: app.HTTPPort, // we specifically want the user-configured port, if any - }, - TLSALPN: &caddytls.TLSALPNChallengeConfig{ - AlternatePort: app.HTTPSPort, // we specifically want the user-configured port, if any - }, - }, - } - if tlsApp.Automation == nil { - tlsApp.Automation = new(caddytls.AutomationConfig) - } - tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, - caddytls.AutomationPolicy{ - Hosts: domainsForCerts, - Management: acmeManager, - }) - - // manage their certificates - app.logger.Info("enabling automatic TLS certificate management", - zap.Strings("domains", domainsForCerts), - ) - err := tlsApp.Manage(domainsForCerts) - if err != nil { - return fmt.Errorf("%s: managing certificate for %s: %s", srvName, domains, err) - } - - // tell the server to use TLS if it is not already doing so - if srv.TLSConnPolicies == nil { - srv.TLSConnPolicies = caddytls.ConnectionPolicies{ - &caddytls.ConnectionPolicy{ALPN: defaultALPN}, - } - } - - if srv.AutoHTTPS.DisableRedir { - continue - } - - app.logger.Info("enabling automatic HTTP->HTTPS redirects", - zap.Strings("domains", domains), - ) - - // create HTTP->HTTPS redirects - for _, addr := range srv.Listen { - netw, host, port, err := caddy.SplitNetworkAddress(addr) - if err != nil { - return fmt.Errorf("%s: invalid listener address: %v", srvName, addr) - } - - if parts := strings.SplitN(port, "-", 2); len(parts) == 2 { - port = parts[0] - } - redirTo := "https://{http.request.host}" - - if port != strconv.Itoa(app.httpsPort()) { - redirTo += ":" + port - } - redirTo += "{http.request.uri}" - - // build the plaintext HTTP variant of this address - httpRedirLnAddr := caddy.JoinNetworkAddress(netw, host, strconv.Itoa(app.httpPort())) - - // create the route that does the redirect and associate - // it with the listener address it will be served from - lnAddrRedirRoutes[httpRedirLnAddr] = Route{ - MatcherSets: []MatcherSet{{MatchProtocol("http")}}, - Handlers: []MiddlewareHandler{ - StaticResponse{ - StatusCode: WeakString(strconv.Itoa(http.StatusPermanentRedirect)), - Headers: http.Header{ - "Location": []string{redirTo}, - "Connection": []string{"close"}, - }, - Close: true, - }, - }, - } - - } - } - } - - // if there are HTTP->HTTPS redirects to add, do so now - if len(lnAddrRedirRoutes) > 0 { - var redirServerAddrs []string - var redirRoutes RouteList - - // for each redirect listener, see if there's already a - // server configured to listen on that exact address; if so, - // simply add the redirect route to the end of its route - // list; otherwise, we'll create a new server for all the - // listener addresses that are unused and serve the - // remaining redirects from it - redirRoutesLoop: - for addr, redirRoute := range lnAddrRedirRoutes { - for srvName, srv := range app.Servers { - if srv.hasListenerAddress(addr) { - // user has configured a server for the same address - // that the redirect runs from; simply append our - // redirect route to the existing routes, with a - // caveat that their config might override ours - app.logger.Warn("server is listening on same interface as redirects, so automatic HTTP->HTTPS redirects might be overridden by your own configuration", - zap.String("server_name", srvName), - zap.String("interface", addr), - ) - srv.Routes = append(srv.Routes, redirRoute) - continue redirRoutesLoop - } - } - // no server with this listener address exists; - // save this address and route for custom server - redirServerAddrs = append(redirServerAddrs, addr) - redirRoutes = append(redirRoutes, redirRoute) - } - - // if there are routes remaining which do not belong - // in any existing server, make our own to serve the - // rest of the redirects - if len(redirServerAddrs) > 0 { - app.Servers["remaining_auto_https_redirects"] = &Server{ - Listen: redirServerAddrs, - Routes: redirRoutes, - tlsApp: tlsApp, // required to solve HTTP challenge - logger: app.logger.Named("log"), - errorLogger: app.logger.Named("log.error"), - primaryHandlerChain: redirRoutes.Compile(emptyHandler), - } - } - } - - return nil -} - func (app *App) httpPort() int { if app.HTTPPort == 0 { return DefaultHTTPPort @@ -709,7 +494,9 @@ func StatusCodeMatches(actual, configured int) bool { if actual == configured { return true } - if configured < 100 && actual >= configured*100 && actual < (configured+1)*100 { + if configured < 100 && + actual >= configured*100 && + actual < (configured+1)*100 { return true } return false diff --git a/modules/caddyhttp/fileserver/caddyfile.go b/modules/caddyhttp/fileserver/caddyfile.go index d26b435..67ae4f4 100644 --- a/modules/caddyhttp/fileserver/caddyfile.go +++ b/modules/caddyhttp/fileserver/caddyfile.go @@ -163,5 +163,10 @@ func parseTryFiles(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) result = append(result, makeRoute(try, "")...) } + // ensure that multiple routes (possible if rewrite targets + // have query strings, for example) are grouped together + // so only the first matching rewrite is performed (#2891) + h.GroupRoutes(result) + return result, nil } diff --git a/modules/caddyhttp/matchers.go b/modules/caddyhttp/matchers.go index f63e48e..49fe859 100644 --- a/modules/caddyhttp/matchers.go +++ b/modules/caddyhttp/matchers.go @@ -35,7 +35,7 @@ import ( type ( // MatchHost matches requests by the Host value (case-insensitive). // - // When used in an HTTP route, + // When used in a top-level HTTP route, // [qualifying domain names](/docs/automatic-https#hostname-requirements) // may trigger [automatic HTTPS](/docs/automatic-https), which automatically // provisions and renews certificates for you. Before doing this, you @@ -55,8 +55,8 @@ type ( // - In the middle, for a globular match (`/accounts/*/info`) // // This matcher is fast, so it does not support regular expressions or - // capture groups. For slower but more capable matching, use the path_regexp - // matcher. + // capture groups. For slower but more powerful matching, use the + // path_regexp matcher. MatchPath []string // MatchPathRE matches requests by a regular expression on the URI's path. diff --git a/modules/caddyhttp/matchers_test.go b/modules/caddyhttp/matchers_test.go index 8e06546..06f137e 100644 --- a/modules/caddyhttp/matchers_test.go +++ b/modules/caddyhttp/matchers_test.go @@ -199,7 +199,7 @@ func TestPathMatcher(t *testing.T) { }, { match: MatchPath{"*.ext"}, - input: "/foo.ext", + input: "/foo/bar.ext", expect: true, }, { diff --git a/modules/caddyhttp/reverseproxy/circuitbreaker.go b/modules/caddyhttp/reverseproxy/circuitbreaker.go index 474f1c6..00b38a8 100644 --- a/modules/caddyhttp/reverseproxy/circuitbreaker.go +++ b/modules/caddyhttp/reverseproxy/circuitbreaker.go @@ -31,7 +31,7 @@ func init() { // for requests within this process over a sliding time window. type localCircuitBreaker struct { tripped int32 - cbType int32 + cbFactor int32 threshold float64 metrics *memmetrics.RTMetrics tripTime time.Duration @@ -48,7 +48,7 @@ func (localCircuitBreaker) CaddyModule() caddy.ModuleInfo { // Provision sets up a configured circuit breaker. func (c *localCircuitBreaker) Provision(ctx caddy.Context) error { - t, ok := typeCB[c.Type] + f, ok := typeCB[c.Factor] if !ok { return fmt.Errorf("type is not defined") } @@ -67,7 +67,7 @@ func (c *localCircuitBreaker) Provision(ctx caddy.Context) error { return fmt.Errorf("cannot create new metrics: %v", err.Error()) } - c.cbType = t + c.cbFactor = f c.tripTime = tw c.threshold = c.Threshold c.metrics = mt @@ -92,13 +92,13 @@ func (c *localCircuitBreaker) RecordMetric(statusCode int, latency time.Duration func (c *localCircuitBreaker) checkAndSet() { var isTripped bool - switch c.cbType { - case typeErrorRatio: + switch c.cbFactor { + case factorErrorRatio: // check if amount of network errors exceed threshold over sliding window, threshold for comparison should be < 1.0 i.e. .5 = 50th percentile if c.metrics.NetworkErrorRatio() > c.threshold { isTripped = true } - case typeLatency: + case factorLatency: // check if threshold in milliseconds is reached and trip hist, err := c.metrics.LatencyHistogram() if err != nil { @@ -109,7 +109,7 @@ func (c *localCircuitBreaker) checkAndSet() { if l.Nanoseconds()/int64(time.Millisecond) > int64(c.threshold) { isTripped = true } - case typeStatusCodeRatio: + case factorStatusCodeRatio: // check ratio of error status codes of sliding window, threshold for comparison should be < 1.0 i.e. .5 = 50th percentile if c.metrics.ResponseCodeRatio(500, 600, 0, 600) > c.threshold { isTripped = true @@ -130,23 +130,28 @@ func (c *localCircuitBreaker) checkAndSet() { // Config represents the configuration of a circuit breaker. type Config struct { + // The threshold over sliding window that would trip the circuit breaker Threshold float64 `json:"threshold"` - Type string `json:"type"` - TripTime string `json:"trip_time"` + // Possible values: latency, error_ratio, and status_ratio. It + // defaults to latency. + Factor string `json:"factor"` + // How long to wait after the circuit is tripped before allowing operations to resume. + // The default is 5s. + TripTime string `json:"trip_time"` } const ( - typeLatency = iota + 1 - typeErrorRatio - typeStatusCodeRatio + factorLatency = iota + 1 + factorErrorRatio + factorStatusCodeRatio defaultTripTime = "5s" ) var ( - // typeCB handles converting a Config Type value to the internal circuit breaker types. + // typeCB handles converting a Config Factor value to the internal circuit breaker types. typeCB = map[string]int32{ - "latency": typeLatency, - "error_ratio": typeErrorRatio, - "status_ratio": typeStatusCodeRatio, + "latency": factorLatency, + "error_ratio": factorErrorRatio, + "status_ratio": factorStatusCodeRatio, } ) diff --git a/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go b/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go index dee6eb5..8c9fd38 100644 --- a/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go @@ -81,7 +81,7 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // // php_fastcgi localhost:7777 // -// is equivalent to: +// is equivalent to a route consisting of: // // @canonicalPath { // file { @@ -104,8 +104,8 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // } // } // -// Thus, this directive produces multiple routes, each with a different -// matcher because multiple consecutive routes are necessary to support +// Thus, this directive produces multiple handlers, each with a different +// matcher because multiple consecutive hgandlers are necessary to support // the common PHP use case. If this "common" config is not compatible // with a user's PHP requirements, they can use a manual approach based // on the example above to configure it precisely as they need. @@ -114,7 +114,7 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // // php_fastcgi /subpath localhost:7777 // -// then the resulting routes are wrapped in a subroute that uses the +// then the resulting handlers are wrapped in a subroute that uses the // user's matcher as a prerequisite to enter the subroute. In other // words, the directive's matcher is necessary, but not sufficient. func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) { @@ -198,12 +198,13 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error HandlersRaw: []json.RawMessage{caddyconfig.JSONModuleObject(rpHandler, "handler", "reverse_proxy", nil)}, } + subroute := caddyhttp.Subroute{ + Routes: caddyhttp.RouteList{redirRoute, rewriteRoute, rpRoute}, + } + // the user's matcher is a prerequisite for ours, so // wrap ours in a subroute and return that if hasUserMatcher { - subroute := caddyhttp.Subroute{ - Routes: caddyhttp.RouteList{redirRoute, rewriteRoute, rpRoute}, - } return []httpcaddyfile.ConfigValue{ { Class: "route", @@ -215,20 +216,14 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error }, nil } - // if the user did not specify a matcher, then - // we can just use our own matchers + // otherwise, return the literal subroute instead of + // individual routes, to ensure they stay together and + // are treated as a single unit, without necessarily + // creating an actual subroute in the output return []httpcaddyfile.ConfigValue{ { Class: "route", - Value: redirRoute, - }, - { - Class: "route", - Value: rewriteRoute, - }, - { - Class: "route", - Value: rpRoute, + Value: subroute, }, }, nil } diff --git a/modules/caddyhttp/reverseproxy/selectionpolicies.go b/modules/caddyhttp/reverseproxy/selectionpolicies.go index 937ae37..330dbdc 100644 --- a/modules/caddyhttp/reverseproxy/selectionpolicies.go +++ b/modules/caddyhttp/reverseproxy/selectionpolicies.go @@ -78,6 +78,8 @@ func (r RandomSelection) Select(pool UpstreamPool, request *http.Request) *Upstr // two or more available hosts at random, then // chooses the one with the least load. type RandomChoiceSelection struct { + // The size of the sub-pool created from the larger upstream pool. The default value + // is 2 and the maximum at selection time is the size of the upstream pool. Choose int `json:"choose,omitempty"` } @@ -283,6 +285,7 @@ func (URIHashSelection) Select(pool UpstreamPool, req *http.Request) *Upstream { // HeaderHashSelection is a policy that selects // a host based on a given request header. type HeaderHashSelection struct { + // The HTTP header field whose value is to be hashed and used for upstream selection. Field string `json:"field,omitempty"` } diff --git a/modules/caddyhttp/rewrite/rewrite.go b/modules/caddyhttp/rewrite/rewrite.go index abcaa84..ad05486 100644 --- a/modules/caddyhttp/rewrite/rewrite.go +++ b/modules/caddyhttp/rewrite/rewrite.go @@ -50,7 +50,7 @@ type Rewrite struct { // You can also use placeholders. For example, to preserve the existing // query string, you might use: "?{http.request.uri.query}&a=b". Any // key-value pairs you add to the query string will not overwrite - // existing values. + // existing values (individual pairs are append-only). // // To clear the query string, explicitly set an empty one: "?" URI string `json:"uri,omitempty"` @@ -112,7 +112,7 @@ func (rewr Rewrite) rewrite(r *http.Request, repl *caddy.Replacer, logger *zap.L r.Method = strings.ToUpper(repl.ReplaceAll(rewr.Method, "")) } - // uri (path, query string, and fragment just because) + // uri (path, query string, and fragment... because why not) if uri := rewr.URI; uri != "" { // find the bounds of each part of the URI that exist pathStart, qsStart, fragStart := -1, -1, -1 @@ -134,14 +134,30 @@ func (rewr Rewrite) rewrite(r *http.Request, repl *caddy.Replacer, logger *zap.L qsEnd = len(uri) } + // build components which are specified, and store them + // in a temporary variable so that they all read the + // same version of the URI + var newPath, newQuery, newFrag string if pathStart >= 0 { - r.URL.Path = repl.ReplaceAll(uri[pathStart:pathEnd], "") + newPath = repl.ReplaceAll(uri[pathStart:pathEnd], "") } if qsStart >= 0 { - r.URL.RawQuery = buildQueryString(uri[qsStart:qsEnd], repl) + newQuery = buildQueryString(uri[qsStart:qsEnd], repl) } if fragStart >= 0 { - r.URL.Fragment = repl.ReplaceAll(uri[fragStart:], "") + newFrag = repl.ReplaceAll(uri[fragStart:], "") + } + + // update the URI with the new components + // only after building them + if pathStart >= 0 { + r.URL.Path = newPath + } + if qsStart >= 0 { + r.URL.RawQuery = newQuery + } + if fragStart >= 0 { + r.URL.Fragment = newFrag } } @@ -206,7 +222,7 @@ func buildQueryString(qs string, repl *caddy.Replacer) string { // if previous iteration wrote a value, // that means we are writing a key if wroteVal { - if sb.Len() > 0 { + if sb.Len() > 0 && len(comp) > 0 { sb.WriteRune('&') } } else { diff --git a/modules/caddyhttp/rewrite/rewrite_test.go b/modules/caddyhttp/rewrite/rewrite_test.go index 3dbc2d6..34a0cdb 100644 --- a/modules/caddyhttp/rewrite/rewrite_test.go +++ b/modules/caddyhttp/rewrite/rewrite_test.go @@ -64,6 +64,16 @@ func TestRewrite(t *testing.T) { expect: newRequest(t, "GET", "/foo/bar"), }, { + rule: Rewrite{URI: "/index.php?p={http.request.uri.path}"}, + input: newRequest(t, "GET", "/foo/bar"), + expect: newRequest(t, "GET", "/index.php?p=%2Ffoo%2Fbar"), + }, + { + rule: Rewrite{URI: "?a=b&{http.request.uri.query}"}, + input: newRequest(t, "GET", "/"), + expect: newRequest(t, "GET", "/?a=b"), + }, + { rule: Rewrite{URI: "/?c=d"}, input: newRequest(t, "GET", "/"), expect: newRequest(t, "GET", "/?c=d"), diff --git a/modules/caddyhttp/routes.go b/modules/caddyhttp/routes.go index 431d1a5..1224e32 100644 --- a/modules/caddyhttp/routes.go +++ b/modules/caddyhttp/routes.go @@ -113,23 +113,43 @@ func (r Route) Empty() bool { // create a middleware chain. type RouteList []Route -// Provision sets up all the routes by loading the modules. +// Provision sets up both the matchers and handlers in the route. func (routes RouteList) Provision(ctx caddy.Context) error { + err := routes.ProvisionMatchers(ctx) + if err != nil { + return err + } + return routes.ProvisionHandlers(ctx) +} + +// ProvisionMatchers sets up all the matchers by loading the +// matcher modules. Only call this method directly if you need +// to set up matchers and handlers separately without having +// to provision a second time; otherwise use Provision instead. +func (routes RouteList) ProvisionMatchers(ctx caddy.Context) error { for i := range routes { // matchers matchersIface, err := ctx.LoadModule(&routes[i], "MatcherSetsRaw") if err != nil { - return fmt.Errorf("loading matchers in route %d: %v", i, err) + return fmt.Errorf("route %d: loading matcher modules: %v", i, err) } err = routes[i].MatcherSets.FromInterface(matchersIface) if err != nil { return fmt.Errorf("route %d: %v", i, err) } + } + return nil +} - // handlers +// ProvisionHandlers sets up all the handlers by loading the +// handler modules. Only call this method directly if you need +// to set up matchers and handlers separately without having +// to provision a second time; otherwise use Provision instead. +func (routes RouteList) ProvisionHandlers(ctx caddy.Context) error { + for i := range routes { handlersIface, err := ctx.LoadModule(&routes[i], "HandlersRaw") if err != nil { - return fmt.Errorf("loading handler modules in route %d: %v", i, err) + return fmt.Errorf("route %d: loading handler modules: %v", i, err) } for _, handler := range handlersIface.([]interface{}) { routes[i].Handlers = append(routes[i].Handlers, handler.(MiddlewareHandler)) @@ -140,7 +160,6 @@ func (routes RouteList) Provision(ctx caddy.Context) error { routes[i].middleware = append(routes[i].middleware, wrapMiddleware(midhandler)) } } - return nil } @@ -233,7 +252,6 @@ func wrapMiddleware(mh MiddlewareHandler) Middleware { return HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { // TODO: This is where request tracing could be implemented - // TODO: Trace a diff of the request, would be cool too... see what changed since the last middleware (host, headers, URI...) // TODO: see what the std lib gives us in terms of stack tracing too return mh.ServeHTTP(w, r, nextCopy) }) diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index ce61b13..1c896a4 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -173,7 +173,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } log("handled request", - zap.String("common_log", repl.ReplaceAll(CommonLogFormat, "-")), + zap.String("common_log", repl.ReplaceAll(commonLogFormat, "-")), zap.Duration("latency", latency), zap.Int("size", wrec.Size()), zap.Int("status", wrec.Status()), @@ -317,49 +317,6 @@ func (s *Server) hasTLSClientAuth() bool { return false } -// AutoHTTPSConfig is used to disable automatic HTTPS -// or certain aspects of it for a specific server. -// HTTPS is enabled automatically and by default when -// qualifying hostnames are available from the config. -type AutoHTTPSConfig struct { - // If true, automatic HTTPS will be entirely disabled. - Disabled bool `json:"disable,omitempty"` - - // If true, only automatic HTTP->HTTPS redirects will - // be disabled. - DisableRedir bool `json:"disable_redirects,omitempty"` - - // Hosts/domain names listed here will not be included - // in automatic HTTPS (they will not have certificates - // loaded nor redirects applied). - Skip []string `json:"skip,omitempty"` - - // Hosts/domain names listed here will still be enabled - // for automatic HTTPS (unless in the Skip list), except - // that certificates will not be provisioned and managed - // for these names. - SkipCerts []string `json:"skip_certificates,omitempty"` - - // By default, automatic HTTPS will obtain and renew - // certificates for qualifying hostnames. However, if - // a certificate with a matching SAN is already loaded - // into the cache, certificate management will not be - // enabled. To force automated certificate management - // regardless of loaded certificates, set this to true. - IgnoreLoadedCerts bool `json:"ignore_loaded_certificates,omitempty"` -} - -// Skipped returns true if name is in skipSlice, which -// should be one of the Skip* fields on ahc. -func (ahc AutoHTTPSConfig) Skipped(name string, skipSlice []string) bool { - for _, n := range skipSlice { - if name == n { - return true - } - } - return false -} - // HTTPErrorConfig determines how to handle errors // from the HTTP handlers. type HTTPErrorConfig struct { @@ -466,11 +423,11 @@ func cloneURL(from, to *url.URL) { } const ( - // CommonLogFormat is the common log format. https://en.wikipedia.org/wiki/Common_Log_Format - CommonLogFormat = `{http.request.remote.host} ` + CommonLogEmptyValue + ` {http.authentication.user.id} [{time.now.common_log}] "{http.request.orig_method} {http.request.orig_uri} {http.request.proto}" {http.response.status} {http.response.size}` + // commonLogFormat is the common log format. https://en.wikipedia.org/wiki/Common_Log_Format + commonLogFormat = `{http.request.remote.host} ` + commonLogEmptyValue + ` {http.authentication.user.id} [{time.now.common_log}] "{http.request.orig_method} {http.request.orig_uri} {http.request.proto}" {http.response.status} {http.response.size}` - // CommonLogEmptyValue is the common empty log value. - CommonLogEmptyValue = "-" + // commonLogEmptyValue is the common empty log value. + commonLogEmptyValue = "-" ) // Context keys for HTTP request context values. diff --git a/modules/caddyhttp/staticresp.go b/modules/caddyhttp/staticresp.go index 0ca2f43..777ecb2 100644 --- a/modules/caddyhttp/staticresp.go +++ b/modules/caddyhttp/staticresp.go @@ -54,24 +54,39 @@ func (StaticResponse) CaddyModule() caddy.ModuleInfo { // UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax: // -// respond [<matcher>] <status> { +// respond [<matcher>] [<status>|[<body> [<status>]] { // body <text> // close // } // +// If there is just one argument (other than the matcher), it is considered +// to be a status code if it's a valid positive integer of 3 digits. func (s *StaticResponse) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { for d.Next() { - var statusCodeStr string - if d.Args(&statusCodeStr) { - s.StatusCode = WeakString(statusCodeStr) + args := d.RemainingArgs() + switch len(args) { + case 1: + if len(args[0]) == 3 { + if num, err := strconv.Atoi(args[0]); err == nil && num > 0 { + s.StatusCode = WeakString(args[0]) + break + } + } + s.Body = args[0] + case 2: + s.Body = args[0] + s.StatusCode = WeakString(args[1]) + default: + return d.ArgErr() } + for d.NextBlock(0) { switch d.Val() { case "body": if s.Body != "" { return d.Err("body already specified") } - if !d.Args(&s.Body) { + if !d.AllArgs(&s.Body) { return d.ArgErr() } case "close": |