v2: Implement 'pki' app powered by Smallstep for localhost certificates (#3125)

* pki: Initial commit of PKI app (WIP) (see #2502 and #3021)

* pki: Ability to use root/intermediates, and sign with root

* pki: Fix benign misnamings left over from copy+paste

* pki: Only install root if not already trusted

* Make HTTPS port the default; all names use auto-HTTPS; bug fixes

* Fix build - what happened to our CI tests??

* Fix go.mod

4 files changed, 115 insertions, 70 deletions

diff --git a/modules/caddyhttp/autohttps.go b/modules/caddyhttp/autohttps.go
index 7dab359..6a23ca0 100644
--- a/modules/caddyhttp/autohttps.go
+++ b/modules/caddyhttp/autohttps.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddytls"
@@ -130,8 +131,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 								return fmt.Errorf("%s: route %d, matcher set %d, matcher %d, host matcher %d: %v",
 									srvName, routeIdx, matcherSetIdx, matcherIdx, hostMatcherIdx, err)
 							}
-							if certmagic.HostQualifies(d) &&
-								!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) {
+							if !srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.Skip) {
 								serverDomainSet[d] = struct{}{}
 							}
 						}
@@ -161,6 +161,15 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 					)
 					continue
 				}
+
+				// most clients don't accept wildcards like *.tld... we
+				// can handle that, but as a courtesy, warn the user
+				if strings.Contains(d, "*") &&
+					strings.Count(strings.Trim(d, "."), ".") == 1 {
+					app.logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
+						zap.String("domain", d))
+				}
+
 				uniqueDomainsForCerts[d] = struct{}{}
 			}
 		}
@@ -202,12 +211,18 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 	// we now have a list of all the unique names for which we need certs;
 	// turn the set into a slice so that phase 2 can use it
 	app.allCertDomains = make([]string, 0, len(uniqueDomainsForCerts))
+	var internal, external []string
 	for d := range uniqueDomainsForCerts {
+		if certmagic.SubjectQualifiesForPublicCert(d) {
+			external = append(external, d)
+		} else {
+			internal = append(internal, d)
+		}
 		app.allCertDomains = append(app.allCertDomains, d)
 	}
 
 	// ensure there is an automation policy to handle these certs
-	err := app.createAutomationPolicy(ctx)
+	err := app.createAutomationPolicies(ctx, external, internal)
 	if err != nil {
 		return err
 	}
@@ -354,23 +369,29 @@ redirServersLoop:
 	return nil
 }
 
-// createAutomationPolicy ensures that certificates for this app are
-// managed properly; for example, it's implied that the HTTPPort
-// should also be the port the HTTP challenge is solved on; the same
-// for HTTPS port and TLS-ALPN challenge also. We need to tell the
-// TLS app to manage these certs by honoring those port configurations,
-// so we either find an existing matching automation policy with an
-// ACME issuer, or make a new one and append it.
-func (app *App) createAutomationPolicy(ctx caddy.Context) error {
+// createAutomationPolicy ensures that automated certificates for this
+// app are managed properly. This adds up to two automation policies:
+// one for the public names, and one for the internal names. If a catch-all
+// automation policy exists, it will be shallow-copied and used as the
+// base for the new ones (this is important for preserving behavior the
+// user intends to be "defaults").
+func (app *App) createAutomationPolicies(ctx caddy.Context, publicNames, internalNames []string) error {
+	// nothing to do if no names to manage certs for
+	if len(publicNames) == 0 && len(internalNames) == 0 {
+		return nil
+	}
+
+	// start by finding a base policy that the user may have defined
+	// which should, in theory, apply to any policies derived from it;
+	// typically this would be a "catch-all" policy with no host filter
 	var matchingPolicy *caddytls.AutomationPolicy
-	var acmeIssuer *caddytls.ACMEIssuer
 	if app.tlsApp.Automation != nil {
-		// maybe we can find an exisitng one that matches; this is
-		// useful if the user made a single automation policy to
-		// set the CA endpoint to a test/staging endpoint (very
-		// common), but forgot to customize the ports here, while
-		// setting them in the HTTP app instead (I did this too
-		// many times)
+		// if an existing policy matches (specifically, a catch-all policy),
+		// we should inherit from it, because that is what the user expects;
+		// this is very common for user setting a default issuer, with a
+		// custom CA endpoint, for example - whichever one we choose must
+		// have a host list that is a superset of the policy we make...
+		// the policy with no host filter is guaranteed to qualify
 		for _, ap := range app.tlsApp.Automation.Policies {
 			if len(ap.Hosts) == 0 {
 				matchingPolicy = ap
@@ -378,51 +399,78 @@ func (app *App) createAutomationPolicy(ctx caddy.Context) error {
 			}
 		}
 	}
-	if matchingPolicy != nil {
-		// if it has an ACME issuer, maybe we can just use that
-		acmeIssuer, _ = matchingPolicy.Issuer.(*caddytls.ACMEIssuer)
-	}
-	if acmeIssuer == nil {
-		acmeIssuer = new(caddytls.ACMEIssuer)
-	}
-	if acmeIssuer.Challenges == nil {
-		acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
-	}
-	if acmeIssuer.Challenges.HTTP == nil {
-		acmeIssuer.Challenges.HTTP = new(caddytls.HTTPChallengeConfig)
-	}
-	if acmeIssuer.Challenges.HTTP.AlternatePort == 0 {
-		// don't overwrite existing explicit config
-		acmeIssuer.Challenges.HTTP.AlternatePort = app.HTTPPort
-	}
-	if acmeIssuer.Challenges.TLSALPN == nil {
-		acmeIssuer.Challenges.TLSALPN = new(caddytls.TLSALPNChallengeConfig)
+	if matchingPolicy == nil {
+		matchingPolicy = new(caddytls.AutomationPolicy)
 	}
-	if acmeIssuer.Challenges.TLSALPN.AlternatePort == 0 {
-		// don't overwrite existing explicit config
-		acmeIssuer.Challenges.TLSALPN.AlternatePort = app.HTTPSPort
+
+	// addPolicy adds an automation policy that uses issuer for hosts.
+	addPolicy := func(issuer certmagic.Issuer, hosts []string) error {
+		// shallow-copy the matching policy; we want to inherit
+		// from it, not replace it... this takes two lines to
+		// overrule compiler optimizations
+		policyCopy := *matchingPolicy
+		newPolicy := &policyCopy
+
+		// very important to provision it, since we are
+		// bypassing the JSON-unmarshaling step
+		if prov, ok := issuer.(caddy.Provisioner); ok {
+			err := prov.Provision(ctx)
+			if err != nil {
+				return err
+			}
+		}
+		newPolicy.Issuer = issuer
+		newPolicy.Hosts = hosts
+
+		return app.tlsApp.AddAutomationPolicy(newPolicy)
 	}
 
-	if matchingPolicy == nil {
-		// if there was no matching policy, we'll have to append our own
-		err := app.tlsApp.AddAutomationPolicy(&caddytls.AutomationPolicy{
-			Hosts:  app.allCertDomains,
-			Issuer: acmeIssuer,
-		})
-		if err != nil {
+	if len(publicNames) > 0 {
+		var acmeIssuer *caddytls.ACMEIssuer
+		// if it has an ACME issuer, maybe we can just use that
+		// TODO: we might need a deep copy here, like a Clone() method on ACMEIssuer...
+		acmeIssuer, _ = matchingPolicy.Issuer.(*caddytls.ACMEIssuer)
+		if acmeIssuer == nil {
+			acmeIssuer = new(caddytls.ACMEIssuer)
+		}
+		if app.HTTPPort > 0 || app.HTTPSPort > 0 {
+			if acmeIssuer.Challenges == nil {
+				acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
+			}
+		}
+		if app.HTTPPort > 0 {
+			if acmeIssuer.Challenges.HTTP == nil {
+				acmeIssuer.Challenges.HTTP = new(caddytls.HTTPChallengeConfig)
+			}
+			// don't overwrite existing explicit config
+			if acmeIssuer.Challenges.HTTP.AlternatePort == 0 {
+				acmeIssuer.Challenges.HTTP.AlternatePort = app.HTTPPort
+			}
+		}
+		if app.HTTPSPort > 0 {
+			if acmeIssuer.Challenges.TLSALPN == nil {
+				acmeIssuer.Challenges.TLSALPN = new(caddytls.TLSALPNChallengeConfig)
+			}
+			// don't overwrite existing explicit config
+			if acmeIssuer.Challenges.TLSALPN.AlternatePort == 0 {
+				acmeIssuer.Challenges.TLSALPN.AlternatePort = app.HTTPSPort
+			}
+		}
+		if err := addPolicy(acmeIssuer, publicNames); err != nil {
 			return err
 		}
-	} else {
-		// if there was an existing matching policy, we need to reprovision
-		// its issuer (because we just changed its port settings and it has
-		// to re-build its stored certmagic config template with the new
-		// values), then re-assign the Issuer pointer on the policy struct
-		// because our type assertion changed the address
-		err := acmeIssuer.Provision(ctx)
-		if err != nil {
+	}
+
+	if len(internalNames) > 0 {
+		internalIssuer := new(caddytls.InternalIssuer)
+		if err := addPolicy(internalIssuer, internalNames); err != nil {
 			return err
 		}
-		matchingPolicy.Issuer = acmeIssuer
+	}
+
+	err := app.tlsApp.Validate()
+	if err != nil {
+		return err
 	}
 
 	return nil
diff --git a/modules/caddyhttp/caddyhttp.go b/modules/caddyhttp/caddyhttp.go
index 6ad70f5..06719b5 100644
--- a/modules/caddyhttp/caddyhttp.go
+++ b/modules/caddyhttp/caddyhttp.go
@@ -286,8 +286,8 @@ func (app *App) Start() error {
 				}
 
 				// enable TLS if there is a policy and if this is not the HTTP port
-				if len(srv.TLSConnPolicies) > 0 &&
-					int(listenAddr.StartPort+portOffset) != app.httpPort() {
+				useTLS := len(srv.TLSConnPolicies) > 0 && int(listenAddr.StartPort+portOffset) != app.httpPort()
+				if useTLS {
 					// create TLS listener
 					tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
 					ln = tls.NewListener(ln, tlsCfg)
@@ -317,6 +317,12 @@ func (app *App) Start() error {
 					/////////
 				}
 
+				app.logger.Debug("starting server loop",
+					zap.String("address", lnAddr),
+					zap.Bool("http3", srv.ExperimentalHTTP3),
+					zap.Bool("tls", useTLS),
+				)
+
 				go s.Serve(ln)
 				app.servers = append(app.servers, s)
 			}
diff --git a/modules/caddyhttp/fileserver/command.go b/modules/caddyhttp/fileserver/command.go
index fa6560b..18e9be3 100644
--- a/modules/caddyhttp/fileserver/command.go
+++ b/modules/caddyhttp/fileserver/command.go
@@ -23,7 +23,6 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	caddycmd "github.com/caddyserver/caddy/v2/cmd"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/certmagic"
@@ -90,11 +89,7 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 		Routes:            caddyhttp.RouteList{route},
 	}
 	if listen == "" {
-		if certmagic.HostQualifies(domain) {
-			listen = ":" + strconv.Itoa(certmagic.HTTPSPort)
-		} else {
-			listen = ":" + httpcaddyfile.DefaultPort
-		}
+		listen = ":" + strconv.Itoa(certmagic.HTTPSPort)
 	}
 	server.Listen = []string{listen}
 
diff --git a/modules/caddyhttp/reverseproxy/command.go b/modules/caddyhttp/reverseproxy/command.go
index 6f70d14..6110ca8 100644
--- a/modules/caddyhttp/reverseproxy/command.go
+++ b/modules/caddyhttp/reverseproxy/command.go
@@ -25,11 +25,9 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	caddycmd "github.com/caddyserver/caddy/v2/cmd"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
-	"github.com/caddyserver/certmagic"
 )
 
 func init() {
@@ -67,7 +65,7 @@ func cmdReverseProxy(fs caddycmd.Flags) (int, error) {
 	changeHost := fs.Bool("change-host-header")
 
 	if from == "" {
-		from = "localhost:" + httpcaddyfile.DefaultPort
+		from = "localhost:443"
 	}
 
 	// URLs need a scheme in order to parse successfully
@@ -129,11 +127,9 @@ func cmdReverseProxy(fs caddycmd.Flags) (int, error) {
 		}
 	}
 
-	listen := ":80"
+	listen := ":443"
 	if urlPort := fromURL.Port(); urlPort != "" {
 		listen = ":" + urlPort
-	} else if certmagic.HostQualifies(urlHost) {
-		listen = ":443"
 	}
 
 	server := &caddyhttp.Server{