v2: Implement 'pki' app powered by Smallstep for localhost certificates (#3125)

* pki: Initial commit of PKI app (WIP) (see #2502 and #3021)

* pki: Ability to use root/intermediates, and sign with root

* pki: Fix benign misnamings left over from copy+paste

* pki: Only install root if not already trusted

* Make HTTPS port the default; all names use auto-HTTPS; bug fixes

* Fix build - what happened to our CI tests??

* Fix go.mod

4 files changed, 69 insertions, 85 deletions

diff --git a/caddyconfig/httpcaddyfile/addresses.go b/caddyconfig/httpcaddyfile/addresses.go
index 64c5d4f..2d17833 100644
--- a/caddyconfig/httpcaddyfile/addresses.go
+++ b/caddyconfig/httpcaddyfile/addresses.go
@@ -172,20 +172,14 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 		httpsPort = strconv.Itoa(hsport.(int))
 	}
 
-	lnPort := DefaultPort
+	// default port is the HTTPS port
+	lnPort := httpsPort
 	if addr.Port != "" {
 		// port explicitly defined
 		lnPort = addr.Port
-	} else if addr.Scheme != "" {
+	} else if addr.Scheme == "http" {
 		// port inferred from scheme
-		if addr.Scheme == "http" {
-			lnPort = httpPort
-		} else if addr.Scheme == "https" {
-			lnPort = httpsPort
-		}
-	} else if certmagic.HostQualifies(addr.Host) {
-		// automatic HTTPS
-		lnPort = httpsPort
+		lnPort = httpPort
 	}
 
 	// error if scheme and port combination violate convention
@@ -213,7 +207,6 @@ func (st *ServerType) listenerAddrsForServerBlockKey(sblock serverBlock, key str
 	for lnStr := range listeners {
 		listenersList = append(listenersList, lnStr)
 	}
-	// sort.Strings(listenersList) // TODO: is sorting necessary?
 
 	return listenersList, nil
 }
@@ -317,9 +310,6 @@ func (a Address) String() string {
 // Normalize returns a normalized version of a.
 func (a Address) Normalize() Address {
 	path := a.Path
-	if !caseSensitivePath {
-		path = strings.ToLower(path)
-	}
 
 	// ensure host is normalized if it's an IP address
 	host := a.Host
@@ -357,10 +347,3 @@ func (a Address) Key() string {
 	}
 	return res
 }
-
-const (
-	// DefaultPort is the default port to use.
-	DefaultPort = "2015"
-
-	caseSensitivePath = false // TODO: Used?
-)
diff --git a/caddyconfig/httpcaddyfile/addresses_test.go b/caddyconfig/httpcaddyfile/addresses_test.go
index e22535c..8de1f09 100644
--- a/caddyconfig/httpcaddyfile/addresses_test.go
+++ b/caddyconfig/httpcaddyfile/addresses_test.go
@@ -1,7 +1,6 @@
 package httpcaddyfile
 
 import (
-	"strings"
 	"testing"
 )
 
@@ -156,15 +155,8 @@ func TestKeyNormalization(t *testing.T) {
 			t.Errorf("Test %d: Parsing address '%s': %v", i, tc.input, err)
 			continue
 		}
-		expect := tc.expect
-		if !caseSensitivePath {
-			// every other part of the address should be lowercased when normalized,
-			// so simply lower-case the whole thing to do case-insensitive comparison
-			// of the path as well
-			expect = strings.ToLower(expect)
-		}
-		if actual := addr.Normalize().Key(); actual != expect {
-			t.Errorf("Test %d: Normalized key for address '%s' was '%s' but expected '%s'", i, tc.input, actual, expect)
+		if actual := addr.Normalize().Key(); actual != tc.expect {
+			t.Errorf("Test %d: Normalized key for address '%s' was '%s' but expected '%s'", i, tc.input, actual, tc.expect)
 		}
 
 	}
diff --git a/caddyconfig/httpcaddyfile/builtins.go b/caddyconfig/httpcaddyfile/builtins.go
index 3b5a4f5..91c1c0a 100644
--- a/caddyconfig/httpcaddyfile/builtins.go
+++ b/caddyconfig/httpcaddyfile/builtins.go
@@ -95,7 +95,7 @@ func parseRoot(h Helper) ([]ConfigValue, error) {
 
 // parseTLS parses the tls directive. Syntax:
 //
-//     tls [<email>]|[<cert_file> <key_file>] {
+//     tls [<email>|internal]|[<cert_file> <key_file>] {
 //         protocols <min> [<max>]
 //         ciphers   <cipher_suites...>
 //         curves    <curves...>
@@ -106,23 +106,11 @@ func parseRoot(h Helper) ([]ConfigValue, error) {
 //     }
 //
 func parseTLS(h Helper) ([]ConfigValue, error) {
-	var configVals []ConfigValue
-
 	var cp *caddytls.ConnectionPolicy
 	var fileLoader caddytls.FileLoader
 	var folderLoader caddytls.FolderLoader
-	var mgr caddytls.ACMEIssuer
-
-	// fill in global defaults, if configured
-	if email := h.Option("email"); email != nil {
-		mgr.Email = email.(string)
-	}
-	if acmeCA := h.Option("acme_ca"); acmeCA != nil {
-		mgr.CA = acmeCA.(string)
-	}
-	if caPemFile := h.Option("acme_ca_root"); caPemFile != nil {
-		mgr.TrustedRootsPEMFiles = append(mgr.TrustedRootsPEMFiles, caPemFile.(string))
-	}
+	var acmeIssuer *caddytls.ACMEIssuer
+	var internalIssuer *caddytls.InternalIssuer
 
 	for h.Next() {
 		// file certificate loader
@@ -130,10 +118,17 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 		switch len(firstLine) {
 		case 0:
 		case 1:
-			if !strings.Contains(firstLine[0], "@") {
-				return nil, h.Err("single argument must be an email address")
+			if firstLine[0] == "internal" {
+				internalIssuer = new(caddytls.InternalIssuer)
+			} else if !strings.Contains(firstLine[0], "@") {
+				return nil, h.Err("single argument must either be 'internal' or an email address")
+			} else {
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				acmeIssuer.Email = firstLine[0]
 			}
-			mgr.Email = firstLine[0]
+
 		case 2:
 			certFilename := firstLine[0]
 			keyFilename := firstLine[1]
@@ -143,7 +138,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			// https://github.com/caddyserver/caddy/issues/2588 ... but we
 			// must be careful about how we do this; being careless will
 			// lead to failed handshakes
-
+			//
 			// we need to remember which cert files we've seen, since we
 			// must load each cert only once; otherwise, they each get a
 			// different tag... since a cert loaded twice has the same
@@ -152,7 +147,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			// policy that is looking for any tag but the last one to be
 			// loaded won't find it, and TLS handshakes will fail (see end)
 			// of issue #3004)
-
+			//
 			// tlsCertTags maps certificate filenames to their tag.
 			// This is used to remember which tag is used for each
 			// certificate files, since we need to avoid loading
@@ -256,29 +251,38 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				if len(arg) != 1 {
 					return nil, h.ArgErr()
 				}
-				mgr.CA = arg[0]
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				acmeIssuer.CA = arg[0]
 
 			// DNS provider for ACME DNS challenge
 			case "dns":
 				if !h.Next() {
 					return nil, h.ArgErr()
 				}
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
 				provName := h.Val()
-				if mgr.Challenges == nil {
-					mgr.Challenges = new(caddytls.ChallengesConfig)
+				if acmeIssuer.Challenges == nil {
+					acmeIssuer.Challenges = new(caddytls.ChallengesConfig)
 				}
 				dnsProvModule, err := caddy.GetModule("tls.dns." + provName)
 				if err != nil {
 					return nil, h.Errf("getting DNS provider module named '%s': %v", provName, err)
 				}
-				mgr.Challenges.DNSRaw = caddyconfig.JSONModuleObject(dnsProvModule.New(), "provider", provName, h.warnings)
+				acmeIssuer.Challenges.DNSRaw = caddyconfig.JSONModuleObject(dnsProvModule.New(), "provider", provName, h.warnings)
 
 			case "ca_root":
 				arg := h.RemainingArgs()
 				if len(arg) != 1 {
 					return nil, h.ArgErr()
 				}
-				mgr.TrustedRootsPEMFiles = append(mgr.TrustedRootsPEMFiles, arg[0])
+				if acmeIssuer == nil {
+					acmeIssuer = new(caddytls.ACMEIssuer)
+				}
+				acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, arg[0])
 
 			default:
 				return nil, h.Errf("unknown subdirective: %s", h.Val())
@@ -291,6 +295,9 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 		}
 	}
 
+	// begin building the final config values
+	var configVals []ConfigValue
+
 	// certificate loaders
 	if len(fileLoader) > 0 {
 		configVals = append(configVals, ConfigValue{
@@ -322,10 +329,30 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	}
 
 	// automation policy
-	if !reflect.DeepEqual(mgr, caddytls.ACMEIssuer{}) {
+	if acmeIssuer != nil && internalIssuer != nil {
+		// the logic to support this would be complex
+		return nil, h.Err("cannot use both ACME and internal issuers in same server block")
+	}
+	if acmeIssuer != nil {
+		// fill in global defaults, if configured
+		if email := h.Option("email"); email != nil && acmeIssuer.Email == "" {
+			acmeIssuer.Email = email.(string)
+		}
+		if acmeCA := h.Option("acme_ca"); acmeCA != nil && acmeIssuer.CA == "" {
+			acmeIssuer.CA = acmeCA.(string)
+		}
+		if caPemFile := h.Option("acme_ca_root"); caPemFile != nil {
+			acmeIssuer.TrustedRootsPEMFiles = append(acmeIssuer.TrustedRootsPEMFiles, caPemFile.(string))
+		}
+
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_issuer",
+			Value: acmeIssuer,
+		})
+	} else if internalIssuer != nil {
 		configVals = append(configVals, ConfigValue{
 			Class: "tls.cert_issuer",
-			Value: mgr,
+			Value: internalIssuer,
 		})
 	}
 
diff --git a/caddyconfig/httpcaddyfile/httptype.go b/caddyconfig/httpcaddyfile/httptype.go
index d880d97..96f2bb0 100644
--- a/caddyconfig/httpcaddyfile/httptype.go
+++ b/caddyconfig/httpcaddyfile/httptype.go
@@ -185,10 +185,10 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 	for _, p := range pairings {
 		for i, sblock := range p.serverBlocks {
 			// tls automation policies
-			if mmVals, ok := sblock.pile["tls.cert_issuer"]; ok {
-				for _, mmVal := range mmVals {
-					mm := mmVal.Value.(certmagic.Issuer)
-					sblockHosts, err := st.autoHTTPSHosts(sblock)
+			if issuerVals, ok := sblock.pile["tls.cert_issuer"]; ok {
+				for _, issuerVal := range issuerVals {
+					issuer := issuerVal.Value.(certmagic.Issuer)
+					sblockHosts, err := st.hostsFromServerBlockKeys(sblock.block)
 					if err != nil {
 						return nil, warnings, err
 					}
@@ -198,7 +198,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock,
 						}
 						tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, &caddytls.AutomationPolicy{
 							Hosts:     sblockHosts,
-							IssuerRaw: caddyconfig.JSONModuleObject(mm, "module", mm.(caddy.Module).CaddyModule().ID.Name(), &warnings),
+							IssuerRaw: caddyconfig.JSONModuleObject(issuer, "module", issuer.(caddy.Module).CaddyModule().ID.Name(), &warnings),
 						})
 					} else {
 						warnings = append(warnings, caddyconfig.Warning{
@@ -500,16 +500,13 @@ func (st *ServerType) serversFromPairings(
 
 			// tls: connection policies and toggle auto HTTPS
 			defaultSNI := tryString(options["default_sni"], warnings)
-			autoHTTPSQualifiedHosts, err := st.autoHTTPSHosts(sblock)
-			if err != nil {
-				return nil, err
-			}
-			if _, ok := sblock.pile["tls.off"]; ok && len(autoHTTPSQualifiedHosts) > 0 {
+			if _, ok := sblock.pile["tls.off"]; ok {
+				// TODO: right now, no directives yield any tls.off value...
 				// tls off: disable TLS (and automatic HTTPS) for server block's names
 				if srv.AutoHTTPS == nil {
 					srv.AutoHTTPS = new(caddyhttp.AutoHTTPSConfig)
 				}
-				srv.AutoHTTPS.Skip = append(srv.AutoHTTPS.Skip, autoHTTPSQualifiedHosts...)
+				srv.AutoHTTPS.Disabled = true
 			} else if cpVals, ok := sblock.pile["tls.connection_policy"]; ok {
 				// tls connection policies
 
@@ -741,25 +738,10 @@ func buildSubroute(routes []ConfigValue, groupCounter counter) (*caddyhttp.Subro
 	return subroute, nil
 }
 
-func (st ServerType) autoHTTPSHosts(sb serverBlock) ([]string, error) {
-	// get the hosts for this server block...
-	hosts, err := st.hostsFromServerBlockKeys(sb.block)
-	if err != nil {
-		return nil, err
-	}
-	// ...and of those, which ones qualify for auto HTTPS
-	var autoHTTPSQualifiedHosts []string
-	for _, h := range hosts {
-		if certmagic.HostQualifies(h) {
-			autoHTTPSQualifiedHosts = append(autoHTTPSQualifiedHosts, h)
-		}
-	}
-	return autoHTTPSQualifiedHosts, nil
-}
-
 // consolidateRoutes combines routes with the same properties
 // (same matchers, same Terminal and Group settings) for a
 // cleaner overall output.
+// FIXME: See caddyserver/caddy#3108
 func consolidateRoutes(routes caddyhttp.RouteList) caddyhttp.RouteList {
 	for i := 0; i < len(routes)-1; i++ {
 		if reflect.DeepEqual(routes[i].MatcherSetsRaw, routes[i+1].MatcherSetsRaw) &&