pki: Embedded ACME server (#3198)

* pki: Initial commit of embedded ACME server (#3021)

* reverseproxy: Support auto-managed TLS client certificates (#3021)

* A little cleanup after today's review session

7 files changed, 277 insertions, 45 deletions

diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go
index f0fbbd6..327cc89 100644
--- a/modules/caddyhttp/reverseproxy/httptransport.go
+++ b/modules/caddyhttp/reverseproxy/httptransport.go
@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddytls"
 	"golang.org/x/net/http2"
 )
 
@@ -140,9 +141,8 @@ func (h *HTTPTransport) Provision(ctx caddy.Context) error {
 	return nil
 }
 
-// NewTransport builds a standard-lib-compatible
-// http.Transport value from h.
-func (h *HTTPTransport) NewTransport(_ caddy.Context) (*http.Transport, error) {
+// NewTransport builds a standard-lib-compatible http.Transport value from h.
+func (h *HTTPTransport) NewTransport(ctx caddy.Context) (*http.Transport, error) {
 	dialer := &net.Dialer{
 		Timeout:       time.Duration(h.DialTimeout),
 		FallbackDelay: time.Duration(h.FallbackDelay),
@@ -175,9 +175,8 @@ func (h *HTTPTransport) NewTransport(_ caddy.Context) (*http.Transport, error) {
 
 	if h.TLS != nil {
 		rt.TLSHandshakeTimeout = time.Duration(h.TLS.HandshakeTimeout)
-
 		var err error
-		rt.TLSClientConfig, err = h.TLS.MakeTLSClientConfig()
+		rt.TLSClientConfig, err = h.TLS.MakeTLSClientConfig(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("making TLS client config: %v", err)
 		}
@@ -267,6 +266,10 @@ type TLSConfig struct {
 	// PEM-encoded key to use with the client certificate.
 	ClientCertificateKeyFile string `json:"client_certificate_key_file,omitempty"`
 
+	// If specified, Caddy will use and automate a client certificate
+	// with this subject name.
+	ClientCertificateAutomate string `json:"client_certificate_automate,omitempty"`
+
 	// If true, TLS verification of server certificates will be disabled.
 	// This is insecure and may be removed in the future. Do not use this
 	// option except in testing or local development environments.
@@ -281,7 +284,7 @@ type TLSConfig struct {
 
 // MakeTLSClientConfig returns a tls.Config usable by a client to a backend.
 // If there is no custom TLS configuration, a nil config may be returned.
-func (t TLSConfig) MakeTLSClientConfig() (*tls.Config, error) {
+func (t TLSConfig) MakeTLSClientConfig(ctx caddy.Context) (*tls.Config, error) {
 	cfg := new(tls.Config)
 
 	// client auth
@@ -298,6 +301,28 @@ func (t TLSConfig) MakeTLSClientConfig() (*tls.Config, error) {
 		}
 		cfg.Certificates = []tls.Certificate{cert}
 	}
+	if t.ClientCertificateAutomate != "" {
+		tlsAppIface, err := ctx.App("tls")
+		if err != nil {
+			return nil, fmt.Errorf("getting tls app: %v", err)
+		}
+		tlsApp := tlsAppIface.(*caddytls.TLS)
+		err = tlsApp.Manage([]string{t.ClientCertificateAutomate})
+		if err != nil {
+			return nil, fmt.Errorf("managing client certificate: %v", err)
+		}
+		cfg.GetClientCertificate = func(cri *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			certs := tlsApp.AllMatchingCertificates(t.ClientCertificateAutomate)
+			var err error
+			for _, cert := range certs {
+				err = cri.SupportsCertificate(&cert.Certificate)
+				if err == nil {
+					return &cert.Certificate, nil
+				}
+			}
+			return nil, err
+		}
+	}
 
 	// trusted root CAs
 	if len(t.RootCAPool) > 0 || len(t.RootCAPEMFiles) > 0 {
diff --git a/modules/caddypki/acmeserver/acmeserver.go b/modules/caddypki/acmeserver/acmeserver.go
new file mode 100644
index 0000000..8dc0f01
--- /dev/null
+++ b/modules/caddypki/acmeserver/acmeserver.go
@@ -0,0 +1,165 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package acmeserver
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddypki"
+	"github.com/go-chi/chi"
+	"github.com/smallstep/certificates/acme"
+	acmeAPI "github.com/smallstep/certificates/acme/api"
+	"github.com/smallstep/certificates/authority"
+	"github.com/smallstep/certificates/authority/provisioner"
+	"github.com/smallstep/certificates/db"
+	"github.com/smallstep/nosql"
+)
+
+func init() {
+	caddy.RegisterModule(Handler{})
+}
+
+// Handler is an ACME server handler.
+type Handler struct {
+	// The ID of the CA to use for signing. This refers to
+	// the ID given to the CA in the `pki` app. If omitted,
+	// the default ID is "local".
+	CA string `json:"ca,omitempty"`
+
+	// The hostname or IP address by which ACME clients
+	// will access the server. This is used to populate
+	// the ACME directory endpoint. Default: localhost.
+	// TODO: this is probably not needed - check with smallstep
+	Host string `json:"host,omitempty"`
+
+	// The path prefix under which to serve all ACME
+	// endpoints. All other requests will not be served
+	// by this handler and will be passed through to
+	// the next one. Default: "/acme/"
+	PathPrefix string `json:"path_prefix,omitempty"`
+
+	acmeEndpoints http.Handler
+}
+
+// CaddyModule returns the Caddy module information.
+func (Handler) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.handlers.acme_server",
+		New: func() caddy.Module { return new(Handler) },
+	}
+}
+
+// Provision sets up the ACME server handler.
+func (ash *Handler) Provision(ctx caddy.Context) error {
+	// set some defaults
+	if ash.CA == "" {
+		ash.CA = caddypki.DefaultCAID
+	}
+	if ash.Host == "" {
+		ash.Host = defaultHost
+	}
+	if ash.PathPrefix == "" {
+		ash.PathPrefix = defaultPathPrefix
+	}
+
+	// get a reference to the configured CA
+	appModule, err := ctx.App("pki")
+	if err != nil {
+		return err
+	}
+	pkiApp := appModule.(*caddypki.PKI)
+	ca, ok := pkiApp.CAs[ash.CA]
+	if !ok {
+		return fmt.Errorf("no certificate authority configured with id: %s", ash.CA)
+	}
+
+	dbFolder := filepath.Join(caddy.AppDataDir(), "acme_server", "db")
+
+	// TODO: See https://github.com/smallstep/nosql/issues/7
+	err = os.MkdirAll(dbFolder, 0755)
+	if err != nil {
+		return fmt.Errorf("making folder for ACME server database: %v", err)
+	}
+
+	authorityConfig := caddypki.AuthorityConfig{
+		AuthConfig: &authority.AuthConfig{
+			Provisioners: provisioner.List{
+				&provisioner.ACME{
+					Name: ash.CA,
+					Type: provisioner.TypeACME.String(),
+					Claims: &provisioner.Claims{
+						MinTLSDur:     &provisioner.Duration{Duration: 5 * time.Minute},
+						MaxTLSDur:     &provisioner.Duration{Duration: 24 * time.Hour * 365},
+						DefaultTLSDur: &provisioner.Duration{Duration: 12 * time.Hour},
+					},
+				},
+			},
+		},
+		DB: &db.Config{
+			Type:       "badger",
+			DataSource: dbFolder,
+		},
+	}
+
+	auth, err := ca.NewAuthority(authorityConfig)
+	if err != nil {
+		return err
+	}
+
+	acmeAuth, err := acme.NewAuthority(
+		auth.GetDatabase().(nosql.DB),     // stores all the server state
+		ash.Host,                          // used for directory links; TODO: not needed
+		strings.Trim(ash.PathPrefix, "/"), // used for directory links
+		auth)                              // configures the signing authority
+	if err != nil {
+		return err
+	}
+
+	// create the router for the ACME endpoints
+	acmeRouterHandler := acmeAPI.New(acmeAuth)
+	r := chi.NewRouter()
+	r.Route(ash.PathPrefix, func(r chi.Router) {
+		acmeRouterHandler.Route(r)
+	})
+	ash.acmeEndpoints = r
+
+	return nil
+}
+
+func (ash Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+	if strings.HasPrefix(r.URL.Path, ash.PathPrefix) {
+		ash.acmeEndpoints.ServeHTTP(w, r)
+		return nil
+	}
+	return next.ServeHTTP(w, r)
+}
+
+const (
+	defaultHost       = "localhost"
+	defaultPathPrefix = "/acme/"
+)
+
+// Interface guards
+var (
+	_ caddyhttp.MiddlewareHandler = (*Handler)(nil)
+	_ caddy.Provisioner           = (*Handler)(nil)
+)
diff --git a/modules/caddypki/ca.go b/modules/caddypki/ca.go
index 21a8bd5..610e7f6 100644
--- a/modules/caddypki/ca.go
+++ b/modules/caddypki/ca.go
@@ -15,6 +15,7 @@
 package caddypki
 
 import (
+	"crypto"
 	"crypto/x509"
 	"encoding/json"
 	"fmt"
@@ -24,6 +25,8 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/certmagic"
+	"github.com/smallstep/certificates/authority"
+	"github.com/smallstep/certificates/db"
 	"github.com/smallstep/truststore"
 	"go.uber.org/zap"
 )
@@ -171,6 +174,52 @@ func (ca CA) IntermediateKey() interface{} {
 	return ca.interKey
 }
 
+// NewAuthority returns a new Smallstep-powered signing authority for this CA.
+func (ca CA) NewAuthority(authorityConfig AuthorityConfig) (*authority.Authority, error) {
+	cfg := &authority.Config{
+		// TODO: eliminate these placeholders / needless values
+		// see https://github.com/smallstep/certificates/issues/218
+		Address:          "placeholder_Address:1",
+		Root:             []string{"placeholder_Root"},
+		IntermediateCert: "placeholder_IntermediateCert",
+		IntermediateKey:  "placeholder_IntermediateKey",
+		DNSNames:         []string{"placeholder_DNSNames"},
+
+		AuthorityConfig: authorityConfig.AuthConfig,
+		DB:              authorityConfig.DB,
+	}
+	// TODO: this also seems unnecessary, see above issue
+	if cfg.AuthorityConfig == nil {
+		cfg.AuthorityConfig = new(authority.AuthConfig)
+	}
+
+	// get the root certificate and the issuer cert+key
+	rootCert := ca.RootCertificate()
+	var issuerCert *x509.Certificate
+	var issuerKey interface{}
+	if authorityConfig.SignWithRoot {
+		issuerCert = rootCert
+		var err error
+		issuerKey, err = ca.RootKey()
+		if err != nil {
+			return nil, fmt.Errorf("loading signing key: %v", err)
+		}
+	} else {
+		issuerCert = ca.IntermediateCertificate()
+		issuerKey = ca.IntermediateKey()
+	}
+
+	auth, err := authority.New(cfg,
+		authority.WithX509Signer(issuerCert, issuerKey.(crypto.Signer)),
+		authority.WithX509RootCerts(rootCert),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("initializing certificate authority: %v", err)
+	}
+
+	return auth, nil
+}
+
 func (ca CA) loadOrGenRoot() (rootCert *x509.Certificate, rootKey interface{}, err error) {
 	rootCertPEM, err := ca.storage.Load(ca.storageKeyRootCert())
 	if err != nil {
@@ -345,8 +394,20 @@ func (ca CA) installRoot() error {
 	)
 }
 
+// AuthorityConfig is used to help a CA configure
+// the underlying signing authority.
+type AuthorityConfig struct {
+	SignWithRoot bool
+
+	// TODO: should we just embed the underlying authority.Config struct type?
+	DB         *db.Config
+	AuthConfig *authority.AuthConfig
+}
+
 const (
-	defaultCAID                   = "local"
+	// DefaultCAID is the default CA ID.
+	DefaultCAID = "local"
+
 	defaultCAName                 = "Caddy Local Authority"
 	defaultRootCommonName         = "{pki.ca.name} - {time.now.year} ECC Root"
 	defaultIntermediateCommonName = "{pki.ca.name} - ECC Intermediate"
diff --git a/modules/caddypki/command.go b/modules/caddypki/command.go
index 9117f3f..34daefa 100644
--- a/modules/caddypki/command.go
+++ b/modules/caddypki/command.go
@@ -88,7 +88,7 @@ func cmdTrust(fs caddycmd.Flags) (int, error) {
 	ca := CA{
 		storage: caddy.DefaultStorage,
 	}
-	err := ca.Provision(ctx, defaultCAID, caddy.Log())
+	err := ca.Provision(ctx, DefaultCAID, caddy.Log())
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
@@ -109,7 +109,7 @@ func cmdUntrust(fs caddycmd.Flags) (int, error) {
 		return caddy.ExitCodeFailedStartup, fmt.Errorf("conflicting command line arguments")
 	}
 	if ca == "" && cert == "" {
-		ca = defaultCAID
+		ca = DefaultCAID
 	}
 	if ca != "" {
 		cert = filepath.Join(caddy.AppDataDir(), "pki", "authorities", ca, "root.crt")
diff --git a/modules/caddypki/pki.go b/modules/caddypki/pki.go
index f9aa372..7737079 100644
--- a/modules/caddypki/pki.go
+++ b/modules/caddypki/pki.go
@@ -52,7 +52,7 @@ func (p *PKI) Provision(ctx caddy.Context) error {
 	// if this app is initialized at all, ensure there's
 	// at least a default CA that can be used
 	if len(p.CAs) == 0 {
-		p.CAs = map[string]*CA{defaultCAID: new(CA)}
+		p.CAs = map[string]*CA{DefaultCAID: new(CA)}
 	}
 
 	for caID, ca := range p.CAs {
diff --git a/modules/caddytls/internalissuer.go b/modules/caddytls/internalissuer.go
index 53a1d00..ca43bf8 100644
--- a/modules/caddytls/internalissuer.go
+++ b/modules/caddytls/internalissuer.go
@@ -17,7 +17,6 @@ package caddytls
 import (
 	"bytes"
 	"context"
-	"crypto"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -26,7 +25,6 @@ import (
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddypki"
 	"github.com/caddyserver/certmagic"
-	"github.com/smallstep/certificates/authority"
 	"github.com/smallstep/certificates/authority/provisioner"
 	"github.com/smallstep/cli/crypto/x509util"
 )
@@ -73,7 +71,7 @@ func (li *InternalIssuer) Provision(ctx caddy.Context) error {
 	}
 	pkiApp := appModule.(*caddypki.PKI)
 	if li.CA == "" {
-		li.CA = defaultInternalCAName
+		li.CA = caddypki.DefaultCAID
 	}
 	ca, ok := pkiApp.CAs[li.CA]
 	if !ok {
@@ -98,40 +96,20 @@ func (li InternalIssuer) IssuerKey() string {
 // Issue issues a certificate to satisfy the CSR.
 func (li InternalIssuer) Issue(ctx context.Context, csr *x509.CertificateRequest) (*certmagic.IssuedCertificate, error) {
 	// prepare the signing authority
-	// TODO: eliminate placeholders / needless values
-	cfg := &authority.Config{
-		Address:          "placeholder_Address:1",
-		Root:             []string{"placeholder_Root"},
-		IntermediateCert: "placeholder_IntermediateCert",
-		IntermediateKey:  "placeholder_IntermediateKey",
-		DNSNames:         []string{"placeholder_DNSNames"},
-		AuthorityConfig: &authority.AuthConfig{
-			Provisioners: provisioner.List{},
-		},
+	authCfg := caddypki.AuthorityConfig{
+		SignWithRoot: li.SignWithRoot,
+	}
+	auth, err := li.ca.NewAuthority(authCfg)
+	if err != nil {
+		return nil, err
 	}
 
-	// get the root certificate and the issuer cert+key
-	rootCert := li.ca.RootCertificate()
+	// get the cert (public key) that will be used for signing
 	var issuerCert *x509.Certificate
-	var issuerKey interface{}
 	if li.SignWithRoot {
-		issuerCert = rootCert
-		var err error
-		issuerKey, err = li.ca.RootKey()
-		if err != nil {
-			return nil, fmt.Errorf("loading signing key: %v", err)
-		}
+		issuerCert = li.ca.RootCertificate()
 	} else {
 		issuerCert = li.ca.IntermediateCertificate()
-		issuerKey = li.ca.IntermediateKey()
-	}
-
-	auth, err := authority.New(cfg,
-		authority.WithX509Signer(issuerCert, issuerKey.(crypto.Signer)),
-		authority.WithX509RootCerts(rootCert),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("initializing certificate authority: %v", err)
 	}
 
 	// ensure issued certificate does not expire later than its issuer
@@ -161,13 +139,16 @@ func (li InternalIssuer) Issue(ctx context.Context, csr *x509.CertificateRequest
 	}, nil
 }
 
-// TODO: borrowing from https://github.com/smallstep/certificates/blob/806abb6232a5691198b891d76b9898ea7f269da0/authority/provisioner/sign_options.go#L191-L211
-// as per https://github.com/smallstep/certificates/issues/198.
 // profileDefaultDuration is a wrapper against x509util.WithOption to conform
 // the SignOption interface.
+//
+// This type is borrowed from the smallstep libraries:
+// https://github.com/smallstep/certificates/blob/806abb6232a5691198b891d76b9898ea7f269da0/authority/provisioner/sign_options.go#L191-L211
+// as per https://github.com/smallstep/certificates/issues/198.
+//
+// TODO: In the future, this approach to custom cert lifetimes may not be necessary
 type profileDefaultDuration time.Duration
 
-// TODO: is there a better way to set cert lifetimes than copying from the smallstep libs?
 func (d profileDefaultDuration) Option(so provisioner.Options) x509util.WithOption {
 	var backdate time.Duration
 	notBefore := so.NotBefore.Time()
@@ -188,7 +169,6 @@ func (d profileDefaultDuration) Option(so provisioner.Options) x509util.WithOpti
 }
 
 const (
-	defaultInternalCAName       = "local"
 	defaultInternalCertLifetime = 12 * time.Hour
 )
 
diff --git a/modules/standard/import.go b/modules/standard/import.go
index dddf712..b8ea7a8 100644
--- a/modules/standard/import.go
+++ b/modules/standard/import.go
@@ -5,6 +5,7 @@ import (
 	_ "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/standard"
 	_ "github.com/caddyserver/caddy/v2/modules/caddypki"
+	_ "github.com/caddyserver/caddy/v2/modules/caddypki/acmeserver"
 	_ "github.com/caddyserver/caddy/v2/modules/caddytls"
 	_ "github.com/caddyserver/caddy/v2/modules/caddytls/distributedstek"
 	_ "github.com/caddyserver/caddy/v2/modules/caddytls/standardstek"