caddytls: Support external certificate Managers (like Tailscale) (#4541)

Huge thank-you to Tailscale (https://tailscale.com) for making this change possible!
This is a great feature for Caddy and Tailscale is a great fit for a standard implementation.

* caddytls: GetCertificate modules; Tailscale

* Caddyfile support for get_certificate

Also fix AP provisioning in case of empty subject list (persist loaded
module on struct, much like Issuers, to surive reprovisioning).

And implement start of HTTP cert getter, still WIP.

* Update modules/caddytls/automation.go

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

* Use tsclient package, check status for name

* Implement HTTP cert getter

And use reuse CertMagic's PEM functions for private keys.

* Remove cache option from Tailscale getter

Tailscale does its own caching and we don't need the added complexity...
for now, at least.

* Several updates

- Option to disable cert automation in auto HTTPS
- Support multiple cert managers
- Remove cache feature from cert manager modules
- Minor improvements to auto HTTPS logging

* Run go mod tidy

* Try to get certificates from Tailscale implicitly

Only for domains ending in .ts.net.

I think this is really cool!

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

3 files changed, 13 insertions, 78 deletions

diff --git a/modules/caddypki/ca.go b/modules/caddypki/ca.go
index e3102fb..7fefee6 100644
--- a/modules/caddypki/ca.go
+++ b/modules/caddypki/ca.go
@@ -239,7 +239,7 @@ func (ca CA) loadOrGenRoot() (rootCert *x509.Certificate, rootKey interface{}, e
 		if err != nil {
 			return nil, nil, fmt.Errorf("loading root key: %v", err)
 		}
-		rootKey, err = pemDecodePrivateKey(rootKeyPEM)
+		rootKey, err = certmagic.PEMDecodePrivateKey(rootKeyPEM)
 		if err != nil {
 			return nil, nil, fmt.Errorf("decoding root key: %v", err)
 		}
@@ -263,7 +263,7 @@ func (ca CA) genRoot() (rootCert *x509.Certificate, rootKey interface{}, err err
 	if err != nil {
 		return nil, nil, fmt.Errorf("saving root certificate: %v", err)
 	}
-	rootKeyPEM, err := pemEncodePrivateKey(rootKey)
+	rootKeyPEM, err := certmagic.PEMEncodePrivateKey(rootKey)
 	if err != nil {
 		return nil, nil, fmt.Errorf("encoding root key: %v", err)
 	}
@@ -275,7 +275,7 @@ func (ca CA) genRoot() (rootCert *x509.Certificate, rootKey interface{}, err err
 	return rootCert, rootKey, nil
 }
 
-func (ca CA) loadOrGenIntermediate(rootCert *x509.Certificate, rootKey interface{}) (interCert *x509.Certificate, interKey interface{}, err error) {
+func (ca CA) loadOrGenIntermediate(rootCert *x509.Certificate, rootKey crypto.PrivateKey) (interCert *x509.Certificate, interKey crypto.PrivateKey, err error) {
 	interCertPEM, err := ca.storage.Load(ca.storageKeyIntermediateCert())
 	if err != nil {
 		if _, ok := err.(certmagic.ErrNotExist); !ok {
@@ -301,7 +301,7 @@ func (ca CA) loadOrGenIntermediate(rootCert *x509.Certificate, rootKey interface
 		if err != nil {
 			return nil, nil, fmt.Errorf("loading intermediate key: %v", err)
 		}
-		interKey, err = pemDecodePrivateKey(interKeyPEM)
+		interKey, err = certmagic.PEMDecodePrivateKey(interKeyPEM)
 		if err != nil {
 			return nil, nil, fmt.Errorf("decoding intermediate key: %v", err)
 		}
@@ -310,7 +310,7 @@ func (ca CA) loadOrGenIntermediate(rootCert *x509.Certificate, rootKey interface
 	return interCert, interKey, nil
 }
 
-func (ca CA) genIntermediate(rootCert *x509.Certificate, rootKey interface{}) (interCert *x509.Certificate, interKey interface{}, err error) {
+func (ca CA) genIntermediate(rootCert *x509.Certificate, rootKey crypto.PrivateKey) (interCert *x509.Certificate, interKey crypto.PrivateKey, err error) {
 	repl := ca.newReplacer()
 
 	interCert, interKey, err = generateIntermediate(repl.ReplaceAll(ca.IntermediateCommonName, ""), rootCert, rootKey)
@@ -325,7 +325,7 @@ func (ca CA) genIntermediate(rootCert *x509.Certificate, rootKey interface{}) (i
 	if err != nil {
 		return nil, nil, fmt.Errorf("saving intermediate certificate: %v", err)
 	}
-	interKeyPEM, err := pemEncodePrivateKey(interKey)
+	interKeyPEM, err := certmagic.PEMEncodePrivateKey(interKey)
 	if err != nil {
 		return nil, nil, fmt.Errorf("encoding intermediate key: %v", err)
 	}
diff --git a/modules/caddypki/certificates.go b/modules/caddypki/certificates.go
index a55c165..bd260da 100644
--- a/modules/caddypki/certificates.go
+++ b/modules/caddypki/certificates.go
@@ -15,6 +15,7 @@
 package caddypki
 
 import (
+	"crypto"
 	"crypto/x509"
 	"time"
 
@@ -30,7 +31,7 @@ func generateRoot(commonName string) (rootCrt *x509.Certificate, privateKey inte
 	return newCert(rootProfile)
 }
 
-func generateIntermediate(commonName string, rootCrt *x509.Certificate, rootKey interface{}) (cert *x509.Certificate, privateKey interface{}, err error) {
+func generateIntermediate(commonName string, rootCrt *x509.Certificate, rootKey crypto.PrivateKey) (cert *x509.Certificate, privateKey crypto.PrivateKey, err error) {
 	interProfile, err := x509util.NewIntermediateProfile(commonName, rootCrt, rootKey)
 	if err != nil {
 		return
@@ -39,7 +40,7 @@ func generateIntermediate(commonName string, rootCrt *x509.Certificate, rootKey
 	return newCert(interProfile)
 }
 
-func newCert(profile x509util.Profile) (cert *x509.Certificate, privateKey interface{}, err error) {
+func newCert(profile x509util.Profile) (cert *x509.Certificate, privateKey crypto.PrivateKey, err error) {
 	certBytes, err := profile.CreateCertificate()
 	if err != nil {
 		return
diff --git a/modules/caddypki/crypto.go b/modules/caddypki/crypto.go
index dbc6f38..386ce62 100644
--- a/modules/caddypki/crypto.go
+++ b/modules/caddypki/crypto.go
@@ -17,14 +17,12 @@ package caddypki
 import (
 	"bytes"
 	"crypto"
-	"crypto/ecdsa"
-	"crypto/ed25519"
-	"crypto/rsa"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
 	"os"
-	"strings"
+
+	"github.com/caddyserver/certmagic"
 )
 
 func pemDecodeSingleCert(pemDER []byte) (*x509.Certificate, error) {
@@ -45,70 +43,6 @@ func pemEncodeCert(der []byte) ([]byte, error) {
 	return pemEncode("CERTIFICATE", der)
 }
 
-// pemEncodePrivateKey marshals a EC or RSA private key into a PEM-encoded array of bytes.
-// TODO: this is the same thing as in certmagic. Should we reuse that code somehow? It's unexported.
-func pemEncodePrivateKey(key crypto.PrivateKey) ([]byte, error) {
-	var pemType string
-	var keyBytes []byte
-	switch key := key.(type) {
-	case *ecdsa.PrivateKey:
-		var err error
-		pemType = "EC"
-		keyBytes, err = x509.MarshalECPrivateKey(key)
-		if err != nil {
-			return nil, err
-		}
-	case *rsa.PrivateKey:
-		pemType = "RSA"
-		keyBytes = x509.MarshalPKCS1PrivateKey(key)
-	case *ed25519.PrivateKey:
-		var err error
-		pemType = "ED25519"
-		keyBytes, err = x509.MarshalPKCS8PrivateKey(key)
-		if err != nil {
-			return nil, err
-		}
-	default:
-		return nil, fmt.Errorf("unsupported key type: %T", key)
-	}
-	return pemEncode(pemType+" PRIVATE KEY", keyBytes)
-}
-
-// pemDecodePrivateKey loads a PEM-encoded ECC/RSA private key from an array of bytes.
-// Borrowed from Go standard library, to handle various private key and PEM block types.
-// https://github.com/golang/go/blob/693748e9fa385f1e2c3b91ca9acbb6c0ad2d133d/src/crypto/tls/tls.go#L291-L308
-// https://github.com/golang/go/blob/693748e9fa385f1e2c3b91ca9acbb6c0ad2d133d/src/crypto/tls/tls.go#L238)
-// TODO: this is the same thing as in certmagic. Should we reuse that code somehow? It's unexported.
-func pemDecodePrivateKey(keyPEMBytes []byte) (crypto.PrivateKey, error) {
-	keyBlockDER, _ := pem.Decode(keyPEMBytes)
-	if keyBlockDER == nil {
-		return nil, fmt.Errorf("no PEM data found")
-	}
-
-	if keyBlockDER.Type != "PRIVATE KEY" && !strings.HasSuffix(keyBlockDER.Type, " PRIVATE KEY") {
-		return nil, fmt.Errorf("unknown PEM header %q", keyBlockDER.Type)
-	}
-
-	if key, err := x509.ParsePKCS1PrivateKey(keyBlockDER.Bytes); err == nil {
-		return key, nil
-	}
-
-	if key, err := x509.ParsePKCS8PrivateKey(keyBlockDER.Bytes); err == nil {
-		switch key := key.(type) {
-		case *rsa.PrivateKey, *ecdsa.PrivateKey, ed25519.PrivateKey:
-			return key, nil
-		default:
-			return nil, fmt.Errorf("found unknown private key type in PKCS#8 wrapping: %T", key)
-		}
-	}
-
-	if key, err := x509.ParseECPrivateKey(keyBlockDER.Bytes); err == nil {
-		return key, nil
-	}
-
-	return nil, fmt.Errorf("unknown private key type")
-}
-
 func pemEncode(blockType string, b []byte) ([]byte, error) {
 	var buf bytes.Buffer
 	err := pem.Encode(&buf, &pem.Block{Type: blockType, Bytes: b})
@@ -137,7 +71,7 @@ type KeyPair struct {
 }
 
 // Load loads the certificate and key.
-func (kp KeyPair) Load() (*x509.Certificate, interface{}, error) {
+func (kp KeyPair) Load() (*x509.Certificate, crypto.Signer, error) {
 	switch kp.Format {
 	case "", "pem_file":
 		certData, err := os.ReadFile(kp.Certificate)
@@ -153,7 +87,7 @@ func (kp KeyPair) Load() (*x509.Certificate, interface{}, error) {
 		if err != nil {
 			return nil, nil, err
 		}
-		key, err := pemDecodePrivateKey(keyData)
+		key, err := certmagic.PEMDecodePrivateKey(keyData)
 		if err != nil {
 			return nil, nil, err
 		}