caddytls: Support external certificate Managers (like Tailscale) (#4541)

Huge thank-you to Tailscale (https://tailscale.com) for making this change possible!
This is a great feature for Caddy and Tailscale is a great fit for a standard implementation.

* caddytls: GetCertificate modules; Tailscale

* Caddyfile support for get_certificate

Also fix AP provisioning in case of empty subject list (persist loaded
module on struct, much like Issuers, to surive reprovisioning).

And implement start of HTTP cert getter, still WIP.

* Update modules/caddytls/automation.go

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

* Use tsclient package, check status for name

* Implement HTTP cert getter

And use reuse CertMagic's PEM functions for private keys.

* Remove cache option from Tailscale getter

Tailscale does its own caching and we don't need the added complexity...
for now, at least.

* Several updates

- Option to disable cert automation in auto HTTPS
- Support multiple cert managers
- Remove cache feature from cert manager modules
- Minor improvements to auto HTTPS logging

* Run go mod tidy

* Try to get certificates from Tailscale implicitly

Only for domains ending in .ts.net.

I think this is really cool!

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

3 files changed, 259 insertions, 28 deletions

diff --git a/modules/caddytls/automation.go b/modules/caddytls/automation.go
index 2a701bf..95b1772 100644
--- a/modules/caddytls/automation.go
+++ b/modules/caddytls/automation.go
@@ -89,6 +89,14 @@ type AutomationPolicy struct {
 	// zerossl.
 	IssuersRaw []json.RawMessage `json:"issuers,omitempty" caddy:"namespace=tls.issuance inline_key=module"`
 
+	// Modules that can get a custom certificate to use for any
+	// given TLS handshake at handshake-time. Custom certificates
+	// can be useful if another entity is managing certificates
+	// and Caddy need only get it and serve it.
+	//
+	// TODO: This is an EXPERIMENTAL feature. It is subject to change or removal.
+	ManagersRaw []json.RawMessage `json:"get_certificate,omitempty" caddy:"namespace=tls.get_certificate inline_key=via"`
+
 	// If true, certificates will be requested with MustStaple. Not all
 	// CAs support this, and there are potentially serious consequences
 	// of enabling this feature without proper threat modeling.
@@ -113,7 +121,7 @@ type AutomationPolicy struct {
 
 	// If true, certificates will be managed "on demand"; that is, during
 	// TLS handshakes or when needed, as opposed to at startup or config
-	// load.
+	// load. This enables On-Demand TLS for this policy.
 	OnDemand bool `json:"on_demand,omitempty"`
 
 	// Disables OCSP stapling. Disabling OCSP stapling puts clients at
@@ -129,10 +137,12 @@ type AutomationPolicy struct {
 	// EXPERIMENTAL. Subject to change.
 	OCSPOverrides map[string]string `json:"ocsp_overrides,omitempty"`
 
-	// Issuers stores the decoded issuer parameters. This is only
-	// used to populate an underlying certmagic.Config's Issuers
-	// field; it is not referenced thereafter.
-	Issuers []certmagic.Issuer `json:"-"`
+	// Issuers and Managers store the decoded issuer and manager modules;
+	// they are only used to populate an underlying certmagic.Config's
+	// fields during provisioning so that the modules can survive a
+	// re-provisioning.
+	Issuers  []certmagic.Issuer             `json:"-"`
+	Managers []certmagic.CertificateManager `json:"-"`
 
 	magic   *certmagic.Config
 	storage certmagic.Storage
@@ -153,6 +163,7 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 		ap.storage = cmStorage
 	}
 
+	// on-demand TLS
 	var ond *certmagic.OnDemandConfig
 	if ap.OnDemand {
 		ond = &certmagic.OnDemandConfig{
@@ -176,6 +187,22 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 		}
 	}
 
+	// we don't store loaded modules directly in the certmagic config since
+	// policy provisioning may happen more than once (during auto-HTTPS) and
+	// loading a module clears its config bytes; thus, load the module and
+	// store them on the policy before putting it on the config
+
+	// load and provision any cert manager modules
+	if ap.ManagersRaw != nil {
+		vals, err := tlsApp.ctx.LoadModule(ap, "ManagersRaw")
+		if err != nil {
+			return fmt.Errorf("loading external certificate manager modules: %v", err)
+		}
+		for _, getCertVal := range vals.([]interface{}) {
+			ap.Managers = append(ap.Managers, getCertVal.(certmagic.CertificateManager))
+		}
+	}
+
 	// load and provision any explicitly-configured issuer modules
 	if ap.IssuersRaw != nil {
 		val, err := tlsApp.ctx.LoadModule(ap, "IssuersRaw")
@@ -225,9 +252,10 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 			DisableStapling:    ap.DisableOCSPStapling,
 			ResponderOverrides: ap.OCSPOverrides,
 		},
-		Storage: storage,
-		Issuers: issuers,
-		Logger:  tlsApp.logger,
+		Storage:  storage,
+		Issuers:  issuers,
+		Managers: ap.Managers,
+		Logger:   tlsApp.logger,
 	}
 	ap.magic = certmagic.New(tlsApp.certCache, template)
 
diff --git a/modules/caddytls/certmanagers.go b/modules/caddytls/certmanagers.go
new file mode 100644
index 0000000..653e9f5
--- /dev/null
+++ b/modules/caddytls/certmanagers.go
@@ -0,0 +1,208 @@
+package caddytls
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/certmagic"
+	"github.com/tailscale/tscert"
+	"go.uber.org/zap"
+)
+
+func init() {
+	caddy.RegisterModule(Tailscale{})
+	caddy.RegisterModule(HTTPCertGetter{})
+}
+
+// Tailscale is a module that can get certificates from the local Tailscale process.
+type Tailscale struct {
+	// If true, this module will operate in "best-effort" mode and
+	// ignore "soft" errors; i.e. try Tailscale, and if it doesn't connect
+	// or return a certificate, oh well. Failure to connect to Tailscale
+	// results in a no-op instead of an error. Intended for the use case
+	// where this module is added implicitly for convenience, even if
+	// Tailscale isn't necessarily running.
+	Optional bool `json:"optional,omitempty"`
+
+	logger *zap.Logger
+}
+
+// CaddyModule returns the Caddy module information.
+func (Tailscale) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "tls.get_certificate.tailscale",
+		New: func() caddy.Module { return new(Tailscale) },
+	}
+}
+
+func (ts *Tailscale) Provision(ctx caddy.Context) error {
+	ts.logger = ctx.Logger(ts)
+	return nil
+}
+
+func (ts Tailscale) GetCertificate(ctx context.Context, hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	canGetCert, err := ts.canHazCertificate(ctx, hello)
+	if err == nil && !canGetCert {
+		return nil, nil // pass-thru: Tailscale can't offer a cert for this name
+	}
+	if err != nil {
+		ts.logger.Warn("could not get status; will try to get certificate anyway", zap.Error(err))
+	}
+	return tscert.GetCertificate(hello)
+}
+
+// canHazCertificate returns true if Tailscale reports it can get a certificate for the given ClientHello.
+func (ts Tailscale) canHazCertificate(ctx context.Context, hello *tls.ClientHelloInfo) (bool, error) {
+	if ts.Optional && !strings.HasSuffix(strings.ToLower(hello.ServerName), tailscaleDomainAliasEnding) {
+		return false, nil
+	}
+	status, err := tscert.GetStatus(ctx)
+	if err != nil {
+		if ts.Optional {
+			return false, nil // ignore error if we don't expect/require it to work anyway
+		}
+		return false, err
+	}
+	for _, domain := range status.CertDomains {
+		if certmagic.MatchWildcard(hello.ServerName, domain) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// UnmarshalCaddyfile deserializes Caddyfile tokens into ts.
+//
+//     ... tailscale
+//
+func (Tailscale) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+	}
+	return nil
+}
+
+// tailscaleDomainAliasEnding is the ending for all Tailscale custom domains.
+const tailscaleDomainAliasEnding = ".ts.net"
+
+// HTTPCertGetter can get a certificate via HTTP(S) request.
+type HTTPCertGetter struct {
+	// The URL from which to download the certificate. Required.
+	//
+	// The URL will be augmented with query string parameters taken
+	// from the TLS handshake:
+	//
+	// - server_name: The SNI value
+	// - signature_schemes: Comma-separated list of hex IDs of signatures
+	// - cipher_suites: Comma-separated list of hex IDs of cipher suites
+	//
+	// To be valid, the response must be HTTP 200 with a PEM body
+	// consisting of blocks for the certificate chain and the private
+	// key.
+	URL string `json:"url,omitempty"`
+
+	ctx context.Context
+}
+
+// CaddyModule returns the Caddy module information.
+func (hcg HTTPCertGetter) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "tls.get_certificate.http",
+		New: func() caddy.Module { return new(HTTPCertGetter) },
+	}
+}
+
+func (hcg *HTTPCertGetter) Provision(ctx caddy.Context) error {
+	hcg.ctx = ctx
+	if hcg.URL == "" {
+		return fmt.Errorf("URL is required")
+	}
+	return nil
+}
+
+func (hcg HTTPCertGetter) GetCertificate(ctx context.Context, hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	sigs := make([]string, len(hello.SignatureSchemes))
+	for i, sig := range hello.SignatureSchemes {
+		sigs[i] = fmt.Sprintf("%x", uint16(sig)) // you won't believe what %x uses if the val is a Stringer
+	}
+	suites := make([]string, len(hello.CipherSuites))
+	for i, cs := range hello.CipherSuites {
+		suites[i] = fmt.Sprintf("%x", cs)
+	}
+
+	parsed, err := url.Parse(hcg.URL)
+	if err != nil {
+		return nil, err
+	}
+	qs := parsed.Query()
+	qs.Set("server_name", hello.ServerName)
+	qs.Set("signature_schemes", strings.Join(sigs, ","))
+	qs.Set("cipher_suites", strings.Join(suites, ","))
+	parsed.RawQuery = qs.Encode()
+
+	req, err := http.NewRequestWithContext(hcg.ctx, http.MethodGet, parsed.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("got HTTP %d", resp.StatusCode)
+	}
+
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error reading response body: %v", err)
+	}
+
+	cert, err := tlsCertFromCertAndKeyPEMBundle(bodyBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cert, nil
+}
+
+// UnmarshalCaddyfile deserializes Caddyfile tokens into ts.
+//
+//     ... http <url>
+//
+func (hcg *HTTPCertGetter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	for d.Next() {
+		if !d.NextArg() {
+			return d.ArgErr()
+		}
+		hcg.URL = d.Val()
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+		for nesting := d.Nesting(); d.NextBlock(nesting); {
+			return d.Err("block not allowed here")
+		}
+	}
+	return nil
+}
+
+// Interface guards
+var (
+	_ certmagic.CertificateManager = (*Tailscale)(nil)
+	_ caddy.Provisioner            = (*Tailscale)(nil)
+	_ caddyfile.Unmarshaler        = (*Tailscale)(nil)
+
+	_ certmagic.CertificateManager = (*HTTPCertGetter)(nil)
+	_ caddy.Provisioner            = (*HTTPCertGetter)(nil)
+	_ caddyfile.Unmarshaler        = (*HTTPCertGetter)(nil)
+)
diff --git a/modules/caddytls/folderloader.go b/modules/caddytls/folderloader.go
index 0ff0629..33b31a5 100644
--- a/modules/caddytls/folderloader.go
+++ b/modules/caddytls/folderloader.go
@@ -61,10 +61,14 @@ func (fl FolderLoader) LoadCertificates() ([]Certificate, error) {
 				return nil
 			}
 
-			cert, err := x509CertFromCertAndKeyPEMFile(fpath)
+			bundle, err := os.ReadFile(fpath)
 			if err != nil {
 				return err
 			}
+			cert, err := tlsCertFromCertAndKeyPEMBundle(bundle)
+			if err != nil {
+				return fmt.Errorf("%s: %w", fpath, err)
+			}
 
 			certs = append(certs, Certificate{Certificate: cert})
 
@@ -77,12 +81,7 @@ func (fl FolderLoader) LoadCertificates() ([]Certificate, error) {
 	return certs, nil
 }
 
-func x509CertFromCertAndKeyPEMFile(fpath string) (tls.Certificate, error) {
-	bundle, err := os.ReadFile(fpath)
-	if err != nil {
-		return tls.Certificate{}, err
-	}
-
+func tlsCertFromCertAndKeyPEMBundle(bundle []byte) (tls.Certificate, error) {
 	certBuilder, keyBuilder := new(bytes.Buffer), new(bytes.Buffer)
 	var foundKey bool // use only the first key in the file
 
@@ -96,8 +95,7 @@ func x509CertFromCertAndKeyPEMFile(fpath string) (tls.Certificate, error) {
 
 		if derBlock.Type == "CERTIFICATE" {
 			// Re-encode certificate as PEM, appending to certificate chain
-			err = pem.Encode(certBuilder, derBlock)
-			if err != nil {
+			if err := pem.Encode(certBuilder, derBlock); err != nil {
 				return tls.Certificate{}, err
 			}
 		} else if derBlock.Type == "EC PARAMETERS" {
@@ -105,18 +103,16 @@ func x509CertFromCertAndKeyPEMFile(fpath string) (tls.Certificate, error) {
 			// parameters and key (parameter block should come first)
 			if !foundKey {
 				// Encode parameters
-				err = pem.Encode(keyBuilder, derBlock)
-				if err != nil {
+				if err := pem.Encode(keyBuilder, derBlock); err != nil {
 					return tls.Certificate{}, err
 				}
 
 				// Key must immediately follow
 				derBlock, bundle = pem.Decode(bundle)
 				if derBlock == nil || derBlock.Type != "EC PRIVATE KEY" {
-					return tls.Certificate{}, fmt.Errorf("%s: expected elliptic private key to immediately follow EC parameters", fpath)
+					return tls.Certificate{}, fmt.Errorf("expected elliptic private key to immediately follow EC parameters")
 				}
-				err = pem.Encode(keyBuilder, derBlock)
-				if err != nil {
+				if err := pem.Encode(keyBuilder, derBlock); err != nil {
 					return tls.Certificate{}, err
 				}
 				foundKey = true
@@ -124,28 +120,27 @@ func x509CertFromCertAndKeyPEMFile(fpath string) (tls.Certificate, error) {
 		} else if derBlock.Type == "PRIVATE KEY" || strings.HasSuffix(derBlock.Type, " PRIVATE KEY") {
 			// RSA key
 			if !foundKey {
-				err = pem.Encode(keyBuilder, derBlock)
-				if err != nil {
+				if err := pem.Encode(keyBuilder, derBlock); err != nil {
 					return tls.Certificate{}, err
 				}
 				foundKey = true
 			}
 		} else {
-			return tls.Certificate{}, fmt.Errorf("%s: unrecognized PEM block type: %s", fpath, derBlock.Type)
+			return tls.Certificate{}, fmt.Errorf("unrecognized PEM block type: %s", derBlock.Type)
 		}
 	}
 
 	certPEMBytes, keyPEMBytes := certBuilder.Bytes(), keyBuilder.Bytes()
 	if len(certPEMBytes) == 0 {
-		return tls.Certificate{}, fmt.Errorf("%s: failed to parse PEM data", fpath)
+		return tls.Certificate{}, fmt.Errorf("failed to parse PEM data")
 	}
 	if len(keyPEMBytes) == 0 {
-		return tls.Certificate{}, fmt.Errorf("%s: no private key block found", fpath)
+		return tls.Certificate{}, fmt.Errorf("no private key block found")
 	}
 
 	cert, err := tls.X509KeyPair(certPEMBytes, keyPEMBytes)
 	if err != nil {
-		return tls.Certificate{}, fmt.Errorf("%s: making X509 key pair: %v", fpath, err)
+		return tls.Certificate{}, fmt.Errorf("making X509 key pair: %v", err)
 	}
 
 	return cert, nil