pki: Implement API endpoints for certs and `caddy trust` (#4443)

* admin: Implement /pki/certificates/<id> API

* pki: Lower "skip_install_trust" log level to INFO

See https://github.com/caddyserver/caddy/issues/4058#issuecomment-976132935

It's not necessary to warn about this, because this was an option explicitly configured by the user. Still useful to log, but we don't need to be so loud about it.

* cmd: Export functions needed for PKI app, return API response to caller

* pki: Rewrite `caddy trust` command to use new admin endpoint instead

* pki: Rewrite `caddy untrust` command to support using admin endpoint

* Refactor cmd and pki packages for determining admin API endpoint

3 files changed, 347 insertions, 50 deletions

diff --git a/modules/caddypki/adminpki.go b/modules/caddypki/adminpki.go
new file mode 100644
index 0000000..5933bcd
--- /dev/null
+++ b/modules/caddypki/adminpki.go
@@ -0,0 +1,194 @@
+// Copyright 2020 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddypki
+
+import (
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
+)
+
+func init() {
+	caddy.RegisterModule(adminPKI{})
+}
+
+// adminPKI is a module that serves a PKI endpoint to retrieve
+// information about the CAs being managed by Caddy.
+type adminPKI struct {
+	ctx    caddy.Context
+	log    *zap.Logger
+	pkiApp *PKI
+}
+
+// CaddyModule returns the Caddy module information.
+func (adminPKI) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "admin.api.pki",
+		New: func() caddy.Module { return new(adminPKI) },
+	}
+}
+
+// Provision sets up the adminPKI module.
+func (a *adminPKI) Provision(ctx caddy.Context) error {
+	a.ctx = ctx
+	a.log = ctx.Logger(a)
+
+	// First check if the PKI app was configured, because
+	// a.ctx.App() has the side effect of instantiating
+	// and provisioning an app even if it wasn't configured.
+	pkiAppConfigured := a.ctx.AppIsConfigured("pki")
+	if !pkiAppConfigured {
+		return nil
+	}
+
+	// Load the PKI app, so we can query it for information.
+	appModule, err := a.ctx.App("pki")
+	if err != nil {
+		return err
+	}
+	a.pkiApp = appModule.(*PKI)
+
+	return nil
+}
+
+// Routes returns the admin routes for the PKI app.
+func (a *adminPKI) Routes() []caddy.AdminRoute {
+	return []caddy.AdminRoute{
+		{
+			Pattern: adminPKICertificatesEndpoint,
+			Handler: caddy.AdminHandlerFunc(a.handleCertificates),
+		},
+	}
+}
+
+// handleCertificates returns certificate information about a particular
+// CA, by its ID. If the CA ID is the default, then the CA will be
+// provisioned if it has not already been. Other CA IDs will return an
+// error if they have not been previously provisioned.
+func (a *adminPKI) handleCertificates(w http.ResponseWriter, r *http.Request) error {
+	if r.Method != http.MethodGet {
+		return caddy.APIError{
+			HTTPStatus: http.StatusMethodNotAllowed,
+			Err:        fmt.Errorf("method not allowed"),
+		}
+	}
+
+	// Prep for a JSON response
+	w.Header().Set("Content-Type", "application/json")
+	enc := json.NewEncoder(w)
+
+	idPath := r.URL.Path
+
+	// Grab the CA ID from the request path, it should be the 4th segment
+	parts := strings.Split(idPath, "/")
+	if len(parts) < 4 || parts[3] == "" {
+		return caddy.APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("request path is missing the CA ID"),
+		}
+	}
+	if parts[0] != "" || parts[1] != "pki" || parts[2] != "certificates" {
+		return caddy.APIError{
+			HTTPStatus: http.StatusBadRequest,
+			Err:        fmt.Errorf("malformed object path"),
+		}
+	}
+	id := parts[3]
+
+	// Find the CA by ID, if PKI is configured
+	var ca *CA
+	ok := false
+	if a.pkiApp != nil {
+		ca, ok = a.pkiApp.CAs[id]
+	}
+
+	// If we didn't find the CA, and PKI is not configured
+	// then we'll either error out if the CA ID is not the
+	// default. If the CA ID is the default, then we'll
+	// provision it, because the user probably aims to
+	// change their config to enable PKI immediately after
+	// if they actually requested the local CA ID.
+	if !ok {
+		if id != DefaultCAID {
+			return caddy.APIError{
+				HTTPStatus: http.StatusNotFound,
+				Err:        fmt.Errorf("no certificate authority configured with id: %s", id),
+			}
+		}
+
+		// Provision the default CA, which generates and stores a root
+		// certificate in storage, if one doesn't already exist.
+		ca = new(CA)
+		err := ca.Provision(a.ctx, id, a.log)
+		if err != nil {
+			return caddy.APIError{
+				HTTPStatus: http.StatusInternalServerError,
+				Err:        fmt.Errorf("failed to provision CA %s, %w", id, err),
+			}
+		}
+	}
+
+	// Convert the root certificate to PEM
+	rootPem := string(pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: ca.RootCertificate().Raw,
+	}))
+
+	// Convert the intermediate certificate to PEM
+	interPem := string(pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: ca.IntermediateCertificate().Raw,
+	}))
+
+	// Build the response
+	response := CAInfo{
+		ID:           ca.ID,
+		Name:         ca.Name,
+		Root:         rootPem,
+		Intermediate: interPem,
+	}
+
+	// Encode and write the JSON response
+	err := enc.Encode(response)
+	if err != nil {
+		return caddy.APIError{
+			HTTPStatus: http.StatusInternalServerError,
+			Err:        err,
+		}
+	}
+
+	return nil
+}
+
+// CAInfo is the response from the certificates API endpoint
+type CAInfo struct {
+	ID           string `json:"id"`
+	Name         string `json:"name"`
+	Root         string `json:"root"`
+	Intermediate string `json:"intermediate"`
+}
+
+const adminPKICertificatesEndpoint = "/pki/certificates/"
+
+// Interface guards
+var (
+	_ caddy.AdminRouter = (*adminPKI)(nil)
+	_ caddy.Provisioner = (*adminPKI)(nil)
+)
diff --git a/modules/caddypki/command.go b/modules/caddypki/command.go
index 34daefa..fa37ab0 100644
--- a/modules/caddypki/command.go
+++ b/modules/caddypki/command.go
@@ -15,11 +15,13 @@
 package caddypki
 
 import (
-	"context"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
-	"path/filepath"
 
 	"github.com/caddyserver/caddy/v2"
 	caddycmd "github.com/caddyserver/caddy/v2/cmd"
@@ -30,69 +32,110 @@ func init() {
 	caddycmd.RegisterCommand(caddycmd.Command{
 		Name:  "trust",
 		Func:  cmdTrust,
+		Usage: "[--ca <id>] [--address <listen>] [--config <path> [--adapter <name>]]",
 		Short: "Installs a CA certificate into local trust stores",
 		Long: `
-Adds a root certificate into the local trust stores. Intended for
-development environments only.
-
-Since Caddy will install its root certificates into the local trust
-stores automatically when they are first generated, this command is
-only necessary if you need to pre-install the certificates before
-using them; for example, if you have elevated privileges at one
-point but not later, you will want to use this command so that a
-password prompt is not required later.
-
-This command installs the root certificate only for Caddy's
-default CA.`,
+Adds a root certificate into the local trust stores.
+
+Caddy will attempt to install its root certificates into the local
+trust stores automatically when they are first generated, but it
+might fail if Caddy doesn't have the appropriate permissions to
+write to the trust store. This command is necessary to pre-install
+the certificates before using them, if the server process runs as an
+unprivileged user (such as via systemd).
+
+By default, this command installs the root certificate for Caddy's
+default CA (i.e. 'local'). You may specify the ID of another CA
+with the --ca flag.
+
+Also, this command will attempt to connect to the Caddy's admin API
+running at '` + caddy.DefaultAdminListen + `' to fetch the root certificate. You may
+explicitly specify the --address, or use the --config flag to load
+the admin address from your config, if not using the default.`,
+		Flags: func() *flag.FlagSet {
+			fs := flag.NewFlagSet("trust", flag.ExitOnError)
+			fs.String("ca", "", "The ID of the CA to trust (defaults to 'local')")
+			fs.String("address", "", "Address of the administration API listener (if --config is not used)")
+			fs.String("config", "", "Configuration file (if --address is not used)")
+			fs.String("adapter", "", "Name of config adapter to apply (if --config is used)")
+			return fs
+		}(),
 	})
 
 	caddycmd.RegisterCommand(caddycmd.Command{
 		Name:  "untrust",
 		Func:  cmdUntrust,
-		Usage: "[--ca <id> | --cert <path>]",
+		Usage: "[--cert <path>] | [[--ca <id>] [--address <listen>] [--config <path> [--adapter <name>]]]",
 		Short: "Untrusts a locally-trusted CA certificate",
 		Long: `
-Untrusts a root certificate from the local trust store(s). Intended
-for development environments only.
+Untrusts a root certificate from the local trust store(s).
 
 This command uninstalls trust; it does not necessarily delete the
 root certificate from trust stores entirely. Thus, repeatedly
 trusting and untrusting new certificates can fill up trust databases.
 
-This command does not delete or modify certificate files.
+This command does not delete or modify certificate files from Caddy's
+configured storage.
 
-Specify which certificate to untrust either by the ID of its CA with
-the --ca flag, or the direct path to the certificate file with the
---cert flag. If the --ca flag is used, only the default storage paths
-are assumed (i.e. using --ca flag with custom storage backends or file
-paths will not work).
+This command can be used in one of two ways. Either by specifying
+which certificate to untrust by a direct path to the certificate
+file with the --cert flag, or by fetching the root certificate for
+the CA from the admin API (default behaviour).
 
-If no flags are specified, --ca=local is assumed.`,
+If the admin API is used, then the CA defaults to 'local'. You may
+specify the ID of another CA with the --ca flag. By default, this
+will attempt to connect to the Caddy's admin API running at
+'` + caddy.DefaultAdminListen + `' to fetch the root certificate.
+You may explicitly specify the --address, or use the --config flag
+to load the admin address from your config, if not using the default.`,
 		Flags: func() *flag.FlagSet {
 			fs := flag.NewFlagSet("untrust", flag.ExitOnError)
-			fs.String("ca", "", "The ID of the CA to untrust")
 			fs.String("cert", "", "The path to the CA certificate to untrust")
+			fs.String("ca", "", "The ID of the CA to untrust (defaults to 'local')")
+			fs.String("address", "", "Address of the administration API listener (if --config is not used)")
+			fs.String("config", "", "Configuration file (if --address is not used)")
+			fs.String("adapter", "", "Name of config adapter to apply (if --config is used)")
 			return fs
 		}(),
 	})
 }
 
-func cmdTrust(fs caddycmd.Flags) (int, error) {
-	// we have to create a sort of dummy context so that
-	// the CA can provision itself...
-	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
-	defer cancel()
+func cmdTrust(fl caddycmd.Flags) (int, error) {
+	caID := fl.String("ca")
+	addrFlag := fl.String("address")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
 
-	// provision the CA, which generates and stores a root
-	// certificate if one doesn't already exist in storage
-	ca := CA{
-		storage: caddy.DefaultStorage,
+	// Prepare the URI to the admin endpoint
+	if caID == "" {
+		caID = DefaultCAID
 	}
-	err := ca.Provision(ctx, DefaultCAID, caddy.Log())
+
+	// Determine where we're sending the request to get the CA info
+	adminAddr, err := caddycmd.DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
+	}
+
+	// Fetch the root cert from the admin API
+	rootCert, err := rootCertFromAdmin(adminAddr, caID)
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
 
+	// Set up the CA struct; we only need to fill in the root
+	// because we're only using it to make use of the installRoot()
+	// function. Also needs a logger for warnings, and a "cert path"
+	// for the root cert; since we're loading from the API and we
+	// don't know the actual storage path via this flow, we'll just
+	// pass through the admin API address instead.
+	ca := CA{
+		log:          caddy.Log(),
+		root:         rootCert,
+		rootCertPath: adminAddr + adminPKICertificatesEndpoint + caID,
+	}
+
+	// Install the cert!
 	err = ca.installRoot()
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
@@ -101,33 +144,93 @@ func cmdTrust(fs caddycmd.Flags) (int, error) {
 	return caddy.ExitCodeSuccess, nil
 }
 
-func cmdUntrust(fs caddycmd.Flags) (int, error) {
-	ca := fs.String("ca")
-	cert := fs.String("cert")
+func cmdUntrust(fl caddycmd.Flags) (int, error) {
+	certFile := fl.String("cert")
+	caID := fl.String("ca")
+	addrFlag := fl.String("address")
+	configFlag := fl.String("config")
+	configAdapterFlag := fl.String("adapter")
 
-	if ca != "" && cert != "" {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("conflicting command line arguments")
+	if certFile != "" && (caID != "" || addrFlag != "" || configFlag != "") {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("conflicting command line arguments, cannot use --cert with other flags")
 	}
-	if ca == "" && cert == "" {
-		ca = DefaultCAID
+
+	// If a file was specified, try to uninstall the cert matching that file
+	if certFile != "" {
+		// Sanity check, make sure cert file exists first
+		_, err := os.Stat(certFile)
+		if err != nil {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("accessing certificate file: %v", err)
+		}
+
+		// Uninstall the file!
+		err = truststore.UninstallFile(certFile,
+			truststore.WithDebug(),
+			truststore.WithFirefox(),
+			truststore.WithJava())
+		if err != nil {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("failed to uninstall certificate file: %v", err)
+		}
+
+		return caddy.ExitCodeSuccess, nil
 	}
-	if ca != "" {
-		cert = filepath.Join(caddy.AppDataDir(), "pki", "authorities", ca, "root.crt")
+
+	// Prepare the URI to the admin endpoint
+	if caID == "" {
+		caID = DefaultCAID
 	}
 
-	// sanity check, make sure cert file exists first
-	_, err := os.Stat(cert)
+	// Determine where we're sending the request to get the CA info
+	adminAddr, err := caddycmd.DetermineAdminAPIAddress(addrFlag, configFlag, configAdapterFlag)
 	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("accessing certificate file: %v", err)
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("couldn't determine admin API address: %v", err)
 	}
 
-	err = truststore.UninstallFile(cert,
+	// Fetch the root cert from the admin API
+	rootCert, err := rootCertFromAdmin(adminAddr, caID)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	// Uninstall the cert!
+	err = truststore.Uninstall(rootCert,
 		truststore.WithDebug(),
 		truststore.WithFirefox(),
 		truststore.WithJava())
 	if err != nil {
-		return caddy.ExitCodeFailedStartup, err
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("failed to uninstall certificate file: %v", err)
 	}
 
 	return caddy.ExitCodeSuccess, nil
 }
+
+// rootCertFromAdmin makes the API request to fetch the
+func rootCertFromAdmin(adminAddr string, caID string) (*x509.Certificate, error) {
+	uri := adminPKICertificatesEndpoint + caID
+
+	// Make the request to fetch the CA info
+	resp, err := caddycmd.AdminAPIRequest(adminAddr, http.MethodGet, uri, make(http.Header), nil)
+	if err != nil {
+		return nil, fmt.Errorf("requesting CA info: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Decode the resposne
+	caInfo := new(CAInfo)
+	err = json.NewDecoder(resp.Body).Decode(caInfo)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode JSON response: %v", err)
+	}
+
+	// Decode the root
+	rootBlock, _ := pem.Decode([]byte(caInfo.Root))
+	if rootBlock == nil {
+		return nil, fmt.Errorf("failed to decode root certificate: %v", err)
+	}
+	rootCert, err := x509.ParseCertificate(rootBlock.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse root certificate: %v", err)
+	}
+
+	return rootCert, nil
+}
diff --git a/modules/caddypki/pki.go b/modules/caddypki/pki.go
index c19bd0f..4fd0bb5 100644
--- a/modules/caddypki/pki.go
+++ b/modules/caddypki/pki.go
@@ -91,7 +91,7 @@ func (p *PKI) Start() error {
 	// install roots to trust store, if not disabled
 	for _, ca := range p.CAs {
 		if ca.InstallTrust != nil && !*ca.InstallTrust {
-			ca.log.Warn("root certificate trust store installation disabled; unconfigured clients may show warnings",
+			ca.log.Info("root certificate trust store installation disabled; unconfigured clients may show warnings",
 				zap.String("path", ca.rootCertPath))
 			continue
 		}