Refactor for CertMagic v0.10; prepare for PKI app

This is a breaking change primarily in two areas:
 - Storage paths for certificates have changed
 - Slight changes to JSON config parameters

Huge improvements in this commit, to be detailed more in
the release notes.

The upcoming PKI app will be powered by Smallstep libraries.

1 files changed, 189 insertions, 59 deletions

diff --git a/modules/caddytls/tls.go b/modules/caddytls/tls.go
index 6be480a..a490ffe 100644
--- a/modules/caddytls/tls.go
+++ b/modules/caddytls/tls.go
@@ -23,8 +23,8 @@ import (
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/certmagic"
 	"github.com/go-acme/lego/v3/challenge"
-	"github.com/mholt/certmagic"
 	"go.uber.org/zap"
 )
 
@@ -71,13 +71,15 @@ func (TLS) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up the configuration for the TLS app.
 func (t *TLS) Provision(ctx caddy.Context) error {
+	// TODO: Move assets to the new folder structure!!
+
 	t.ctx = ctx
 	t.logger = ctx.Logger(t)
 
 	// set up a new certificate cache; this (re)loads all certificates
 	cacheOpts := certmagic.CacheOptions{
-		GetConfigForCert: func(cert certmagic.Certificate) (certmagic.Config, error) {
-			return t.getConfigForName(cert.Names[0])
+		GetConfigForCert: func(cert certmagic.Certificate) (*certmagic.Config, error) {
+			return t.getConfigForName(cert.Names[0]), nil
 		},
 	}
 	if t.Automation != nil {
@@ -87,20 +89,25 @@ func (t *TLS) Provision(ctx caddy.Context) error {
 	t.certCache = certmagic.NewCache(cacheOpts)
 
 	// automation/management policies
-	if t.Automation != nil {
-		for i, ap := range t.Automation.Policies {
-			val, err := ctx.LoadModule(ap, "ManagementRaw")
-			if err != nil {
-				return fmt.Errorf("loading TLS automation management module: %s", err)
-			}
-			t.Automation.Policies[i].Management = val.(ManagerMaker)
+	if t.Automation == nil {
+		t.Automation = new(AutomationConfig)
+	}
+	t.Automation.defaultAutomationPolicy = new(AutomationPolicy)
+	err := t.Automation.defaultAutomationPolicy.provision(t)
+	if err != nil {
+		return fmt.Errorf("provisioning default automation policy: %v", err)
+	}
+	for i, ap := range t.Automation.Policies {
+		err := ap.provision(t)
+		if err != nil {
+			return fmt.Errorf("provisioning automation policy %d: %v", i, err)
 		}
 	}
 
 	// certificate loaders
 	val, err := ctx.LoadModule(t, "CertificatesRaw")
 	if err != nil {
-		return fmt.Errorf("loading TLS automation management module: %s", err)
+		return fmt.Errorf("loading certificate loader modules: %s", err)
 	}
 	for modName, modIface := range val.(map[string]interface{}) {
 		if modName == "automate" {
@@ -216,12 +223,11 @@ func (t *TLS) Manage(names []string) error {
 	// certmagic.Config for each (potentially large) group of names
 	// and call ManageSync/ManageAsync just once for the whole batch
 	for ap, names := range policyToNames {
-		magic := certmagic.New(t.certCache, ap.makeCertMagicConfig(t.ctx))
 		var err error
 		if ap.ManageSync {
-			err = magic.ManageSync(names)
+			err = ap.magic.ManageSync(names)
 		} else {
-			err = magic.ManageAsync(t.ctx.Context, names)
+			err = ap.magic.ManageAsync(t.ctx.Context, names)
 		}
 		if err != nil {
 			return fmt.Errorf("automate: manage %v: %v", names, err)
@@ -232,36 +238,54 @@ func (t *TLS) Manage(names []string) error {
 }
 
 // HandleHTTPChallenge ensures that the HTTP challenge is handled for the
-// certificate named by r.Host, if it is an HTTP challenge request.
+// certificate named by r.Host, if it is an HTTP challenge request. It
+// requires that the automation policy for r.Host has an issue of type
+// *certmagic.ACMEManager.
 func (t *TLS) HandleHTTPChallenge(w http.ResponseWriter, r *http.Request) bool {
 	if !certmagic.LooksLikeHTTPChallenge(r) {
 		return false
 	}
 	ap := t.getAutomationPolicyForName(r.Host)
-	magic := certmagic.New(t.certCache, ap.makeCertMagicConfig(t.ctx))
-	return magic.HandleHTTPChallenge(w, r)
+	if ap.magic.Issuer == nil {
+		return false
+	}
+	if am, ok := ap.magic.Issuer.(*certmagic.ACMEManager); ok {
+		return am.HandleHTTPChallenge(w, r)
+	}
+	return false
+}
+
+// AddAutomationPolicy provisions and adds ap to the list of the app's
+// automation policies.
+func (t *TLS) AddAutomationPolicy(ap *AutomationPolicy) error {
+	if t.Automation == nil {
+		t.Automation = new(AutomationConfig)
+	}
+	err := ap.provision(t)
+	if err != nil {
+		return err
+	}
+	t.Automation.Policies = append(t.Automation.Policies, ap)
+	return nil
 }
 
-func (t *TLS) getConfigForName(name string) (certmagic.Config, error) {
+func (t *TLS) getConfigForName(name string) *certmagic.Config {
 	ap := t.getAutomationPolicyForName(name)
-	return ap.makeCertMagicConfig(t.ctx), nil
+	return ap.magic
 }
 
 func (t *TLS) getAutomationPolicyForName(name string) *AutomationPolicy {
-	if t.Automation != nil {
-		for _, ap := range t.Automation.Policies {
-			if len(ap.Hosts) == 0 {
-				// no host filter is an automatic match
+	for _, ap := range t.Automation.Policies {
+		if len(ap.Hosts) == 0 {
+			return ap // no host filter is an automatic match
+		}
+		for _, h := range ap.Hosts {
+			if h == name {
 				return ap
 			}
-			for _, h := range ap.Hosts {
-				if h == name {
-					return ap
-				}
-			}
 		}
 	}
-	return defaultAutomationPolicy
+	return t.Automation.defaultAutomationPolicy
 }
 
 // AllMatchingCertificates returns the list of all certificates in
@@ -309,10 +333,8 @@ func (t *TLS) cleanStorageUnits() {
 	// then clean each storage defined in ACME automation policies
 	if t.Automation != nil {
 		for _, ap := range t.Automation.Policies {
-			if acmeMgmt, ok := ap.Management.(ACMEManagerMaker); ok {
-				if acmeMgmt.storage != nil {
-					certmagic.CleanStorage(acmeMgmt.storage, options)
-				}
+			if ap.storage != nil {
+				certmagic.CleanStorage(ap.storage, options)
 			}
 		}
 	}
@@ -355,23 +377,56 @@ type AutomationConfig struct {
 	OCSPCheckInterval caddy.Duration `json:"ocsp_interval,omitempty"`
 
 	// Every so often, Caddy will scan all loaded, managed
-	// certificates for expiration. Certificates which are
-	// about 2/3 into their valid lifetime are due for
-	// renewal. This setting changes how frequently the scan
-	// is performed. If your certificate lifetimes are very
-	// short (less than ~1 week), you should customize this.
+	// certificates for expiration. This setting changes how
+	// frequently the scan for expiring certificates is
+	// performed. If your certificate lifetimes are very
+	// short (less than ~24 hours), you should set this to
+	// a low value.
 	RenewCheckInterval caddy.Duration `json:"renew_interval,omitempty"`
+
+	defaultAutomationPolicy *AutomationPolicy
 }
 
 // AutomationPolicy designates the policy for automating the
 // management (obtaining, renewal, and revocation) of managed
 // TLS certificates.
+//
+// An AutomationPolicy value is not valid until it has been
+// provisioned; use the `AddAutomationPolicy()` method on the
+// TLS app to properly provision a new policy.
 type AutomationPolicy struct {
 	// Which hostnames this policy applies to.
 	Hosts []string `json:"hosts,omitempty"`
 
-	// How to manage certificates.
-	ManagementRaw json.RawMessage `json:"management,omitempty" caddy:"namespace=tls.management inline_key=module"`
+	// The module that will issue certificates. Default: acme
+	IssuerRaw json.RawMessage `json:"issuer,omitempty" caddy:"namespace=tls.issuance inline_key=module"`
+
+	// If true, certificates will be requested with MustStaple. Not all
+	// CAs support this, and there are potentially serious consequences
+	// of enabling this feature without proper threat modeling.
+	MustStaple bool `json:"must_staple,omitempty"`
+
+	// How long before a certificate's expiration to try renewing it,
+	// as a function of its total lifetime. As a general and conservative
+	// rule, it is a good idea to renew a certificate when it has about
+	// 1/3 of its total lifetime remaining. This utilizes the majority
+	// of the certificate's lifetime while still saving time to
+	// troubleshoot problems. However, for extremely short-lived certs,
+	// you may wish to increase the ratio to ~1/2.
+	RenewalWindowRatio float64 `json:"renewal_window_ratio,omitempty"`
+
+	// The type of key to generate for certificates.
+	// Supported values: `ed25519`, `p256`, `p384`, `rsa2048`, `rsa4096`.
+	KeyType string `json:"key_type,omitempty"`
+
+	// Optionally configure a separate storage module associated with this
+	// manager, instead of using Caddy's global/default-configured storage.
+	StorageRaw json.RawMessage `json:"storage,omitempty" caddy:"namespace=caddy.storage inline_key=module"`
+
+	// If true, certificates will be managed "on demand", that is, during
+	// TLS handshakes or when needed, as opposed to at startup or config
+	// load.
+	OnDemand bool `json:"on_demand,omitempty"`
 
 	// If true, certificate management will be conducted
 	// in the foreground; this will block config reloads
@@ -381,23 +436,96 @@ type AutomationPolicy struct {
 	// of your control. Default: false
 	ManageSync bool `json:"manage_sync,omitempty"`
 
-	Management ManagerMaker `json:"-"`
+	Issuer certmagic.Issuer `json:"-"`
+
+	magic   *certmagic.Config
+	storage certmagic.Storage
 }
 
-// makeCertMagicConfig converts ap into a CertMagic config. Passing onDemand
-// is necessary because the automation policy does not have convenient access
-// to the TLS app's global on-demand policies;
-func (ap AutomationPolicy) makeCertMagicConfig(ctx caddy.Context) certmagic.Config {
-	// default manager (ACME) is a special case because of how CertMagic is designed
-	// TODO: refactor certmagic so that ACME manager is not a special case by extracting
-	// its config fields out of the certmagic.Config struct, or something...
-	if acmeMgmt, ok := ap.Management.(*ACMEManagerMaker); ok {
-		return acmeMgmt.makeCertMagicConfig(ctx)
+// provision converts ap into a CertMagic config.
+func (ap *AutomationPolicy) provision(tlsApp *TLS) error {
+	// policy-specific storage implementation
+	if ap.StorageRaw != nil {
+		val, err := tlsApp.ctx.LoadModule(ap, "StorageRaw")
+		if err != nil {
+			return fmt.Errorf("loading TLS storage module: %v", err)
+		}
+		cmStorage, err := val.(caddy.StorageConverter).CertMagicStorage()
+		if err != nil {
+			return fmt.Errorf("creating TLS storage configuration: %v", err)
+		}
+		ap.storage = cmStorage
+	}
+
+	var ond *certmagic.OnDemandConfig
+	if ap.OnDemand {
+		var onDemand *OnDemandConfig
+		if tlsApp.Automation != nil {
+			onDemand = tlsApp.Automation.OnDemand
+		}
+
+		ond = &certmagic.OnDemandConfig{
+			DecisionFunc: func(name string) error {
+				if onDemand != nil {
+					if onDemand.Ask != "" {
+						err := onDemandAskRequest(onDemand.Ask, name)
+						if err != nil {
+							return err
+						}
+					}
+					// check the rate limiter last because
+					// doing so makes a reservation
+					if !onDemandRateLimiter.Allow() {
+						return fmt.Errorf("on-demand rate limit exceeded")
+					}
+				}
+				return nil
+			},
+		}
+	}
+
+	keySource := certmagic.StandardKeyGenerator{
+		KeyType: supportedCertKeyTypes[ap.KeyType],
+	}
+
+	storage := ap.storage
+	if storage == nil {
+		storage = tlsApp.ctx.Storage()
 	}
 
-	return certmagic.Config{
-		NewManager: ap.Management.NewManager,
+	template := certmagic.Config{
+		MustStaple:         ap.MustStaple,
+		RenewalWindowRatio: ap.RenewalWindowRatio,
+		KeySource:          keySource,
+		OnDemand:           ond,
+		Storage:            storage,
 	}
+	cfg := certmagic.New(tlsApp.certCache, template)
+	ap.magic = cfg
+
+	if ap.IssuerRaw != nil {
+		val, err := tlsApp.ctx.LoadModule(ap, "IssuerRaw")
+		if err != nil {
+			return fmt.Errorf("loading TLS automation management module: %s", err)
+		}
+		ap.Issuer = val.(certmagic.Issuer)
+	}
+
+	// sometimes issuers may need the parent certmagic.Config in
+	// order to function properly (for example, ACMEIssuer needs
+	// access to the correct storage and cache so it can solve
+	// ACME challenges -- it's an annoying, inelegant circular
+	// dependency that I don't know how to resolve nicely!)
+	if configger, ok := ap.Issuer.(ConfigSetter); ok {
+		configger.SetConfig(cfg)
+	}
+
+	cfg.Issuer = ap.Issuer
+	if rev, ok := ap.Issuer.(certmagic.Revoker); ok {
+		cfg.Revoker = rev
+	}
+
+	return nil
 }
 
 // ChallengesConfig configures the ACME challenges.
@@ -482,11 +610,6 @@ type RateLimit struct {
 	Burst int `json:"burst,omitempty"`
 }
 
-// ManagerMaker makes a certificate manager.
-type ManagerMaker interface {
-	NewManager(interactive bool) (certmagic.Manager, error)
-}
-
 // AutomateLoader is a no-op certificate loader module
 // that is treated as a special case: it uses this app's
 // automation features to load certificates for the
@@ -502,6 +625,15 @@ func (AutomateLoader) CaddyModule() caddy.ModuleInfo {
 	}
 }
 
+// ConfigSetter is implemented by certmagic.Issuers that
+// need access to a parent certmagic.Config as part of
+// their provisioning phase. For example, the ACMEIssuer
+// requires a config so it can access storage and the
+// cache to solve ACME challenges.
+type ConfigSetter interface {
+	SetConfig(cfg *certmagic.Config)
+}
+
 // These perpetual values are used for on-demand TLS.
 var (
 	onDemandRateLimiter = certmagic.NewRateLimiter(0, 0)
@@ -521,8 +653,6 @@ var (
 	storageCleanMu sync.Mutex
 )
 
-var defaultAutomationPolicy = &AutomationPolicy{Management: new(ACMEManagerMaker)}
-
 // Interface guards
 var (
 	_ caddy.App          = (*TLS)(nil)