cmd: New `add-package` and `remove-package` commands (#4226)

* adding package command

* add-package command name

* refactoring duplicate code

* fixed by review

* fixed by review

* remove-package command

* commands in different files, common utils

* fix add, remove, upgrade packages in 1 file

* copyright and downloadPath moved

* refactor

* downloadPath do no export

* adding/removing multiple packages

* addPackages/removePackages, comments, command-desc

* add-package, process case len(args) == 0

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

3 files changed, 330 insertions, 197 deletions

diff --git a/cmd/commandfuncs.go b/cmd/commandfuncs.go
index 58b9720..c70b57a 100644
--- a/cmd/commandfuncs.go
+++ b/cmd/commandfuncs.go
@@ -25,10 +25,8 @@ import (
 	"log"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
-	"reflect"
 	"runtime"
 	"runtime/debug"
 	"sort"
@@ -570,151 +568,6 @@ func cmdFmt(fl Flags) (int, error) {
 	return caddy.ExitCodeSuccess, nil
 }
 
-func cmdUpgrade(_ Flags) (int, error) {
-	l := caddy.Log()
-
-	thisExecPath, err := os.Executable()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err)
-	}
-	thisExecStat, err := os.Stat(thisExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err)
-	}
-	l.Info("this executable will be replaced", zap.String("path", thisExecPath))
-
-	// get the list of nonstandard plugins
-	_, nonstandard, _, err := getModules()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
-	}
-	pluginPkgs := make(map[string]struct{})
-	for _, mod := range nonstandard {
-		if mod.goModule.Replace != nil {
-			return caddy.ExitCodeFailedStartup, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s",
-				mod.goModule.Path, mod.goModule.Replace.Path)
-		}
-		l.Info("found non-standard module",
-			zap.String("id", mod.caddyModuleID),
-			zap.String("package", mod.goModule.Path))
-		pluginPkgs[mod.goModule.Path] = struct{}{}
-	}
-
-	// build the request URL to download this custom build
-	qs := url.Values{
-		"os":   {runtime.GOOS},
-		"arch": {runtime.GOARCH},
-	}
-	for pkg := range pluginPkgs {
-		qs.Add("p", pkg)
-	}
-	urlStr := fmt.Sprintf("https://caddyserver.com/api/download?%s", qs.Encode())
-
-	// initiate the build
-	l.Info("requesting build",
-		zap.String("os", qs.Get("os")),
-		zap.String("arch", qs.Get("arch")),
-		zap.Strings("packages", qs["p"]))
-	resp, err := http.Get(urlStr)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("secure request failed: %v", err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		var details struct {
-			StatusCode int `json:"status_code"`
-			Error      struct {
-				Message string `json:"message"`
-				ID      string `json:"id"`
-			} `json:"error"`
-		}
-		err2 := json.NewDecoder(resp.Body).Decode(&details)
-		if err2 != nil {
-			return caddy.ExitCodeFailedStartup, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2)
-		}
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID)
-	}
-
-	// back up the current binary, in case something goes wrong we can replace it
-	backupExecPath := thisExecPath + ".tmp"
-	l.Info("build acquired; backing up current executable",
-		zap.String("current_path", thisExecPath),
-		zap.String("backup_path", backupExecPath))
-	err = os.Rename(thisExecPath, backupExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err)
-	}
-	defer func() {
-		if err != nil {
-			err2 := os.Rename(backupExecPath, thisExecPath)
-			if err2 != nil {
-				l.Error("restoring original executable failed; will need to be restored manually",
-					zap.String("backup_path", backupExecPath),
-					zap.String("original_path", thisExecPath),
-					zap.Error(err2))
-			}
-		}
-	}()
-
-	// download the file; do this in a closure to close reliably before we execute it
-	writeFile := func() error {
-		destFile, err := os.OpenFile(thisExecPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, thisExecStat.Mode())
-		if err != nil {
-			return fmt.Errorf("unable to open destination file: %v", err)
-		}
-		defer destFile.Close()
-
-		l.Info("downloading binary", zap.String("source", urlStr), zap.String("destination", thisExecPath))
-
-		_, err = io.Copy(destFile, resp.Body)
-		if err != nil {
-			return fmt.Errorf("unable to download file: %v", err)
-		}
-
-		err = destFile.Sync()
-		if err != nil {
-			return fmt.Errorf("syncing downloaded file to device: %v", err)
-		}
-
-		return nil
-	}
-	err = writeFile()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, err
-	}
-
-	l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath))
-
-	// use the new binary to print out version and module info
-	fmt.Print("\nModule versions:\n\n")
-	cmd := exec.Command(thisExecPath, "list-modules", "--versions")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	fmt.Println("\nVersion:")
-	cmd = exec.Command(thisExecPath, "version")
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
-	}
-	fmt.Println()
-
-	// clean up the backup file
-	err = os.Remove(backupExecPath)
-	if err != nil {
-		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
-	}
-
-	l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath))
-
-	return caddy.ExitCodeSuccess, nil
-}
-
 func cmdHelp(fl Flags) (int, error) {
 	const fullDocs = `Full documentation is available at:
 https://caddyserver.com/docs/command-line`
@@ -779,56 +632,6 @@ commands:
 	return caddy.ExitCodeSuccess, nil
 }
 
-func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
-	bi, ok := debug.ReadBuildInfo()
-	if !ok {
-		err = fmt.Errorf("no build info")
-		return
-	}
-
-	for _, modID := range caddy.Modules() {
-		modInfo, err := caddy.GetModule(modID)
-		if err != nil {
-			// that's weird, shouldn't happen
-			unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err})
-			continue
-		}
-
-		// to get the Caddy plugin's version info, we need to know
-		// the package that the Caddy module's value comes from; we
-		// can use reflection but we need a non-pointer value (I'm
-		// not sure why), and since New() should return a pointer
-		// value, we need to dereference it first
-		iface := interface{}(modInfo.New())
-		if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr {
-			iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface()
-		}
-		modPkgPath := reflect.TypeOf(iface).PkgPath()
-
-		// now we find the Go module that the Caddy module's package
-		// belongs to; we assume the Caddy module package path will
-		// be prefixed by its Go module path, and we will choose the
-		// longest matching prefix in case there are nested modules
-		var matched *debug.Module
-		for _, dep := range bi.Deps {
-			if strings.HasPrefix(modPkgPath, dep.Path) {
-				if matched == nil || len(dep.Path) > len(matched.Path) {
-					matched = dep
-				}
-			}
-		}
-
-		caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched}
-
-		if strings.HasPrefix(modPkgPath, caddy.ImportPath) {
-			standard = append(standard, caddyModGoMod)
-		} else {
-			nonstandard = append(nonstandard, caddyModGoMod)
-		}
-	}
-	return
-}
-
 // apiRequest makes an API request to the endpoint adminAddr with the
 // given HTTP method and request URI. If body is non-nil, it will be
 // assumed to be Content-Type application/json.
diff --git a/cmd/commands.go b/cmd/commands.go
index 36e9c04..89c4fe4 100644
--- a/cmd/commands.go
+++ b/cmd/commands.go
@@ -291,6 +291,30 @@ Downloads an updated Caddy binary with the same modules/plugins at the
 latest versions. EXPERIMENTAL: May be changed or removed.`,
 	})
 
+	RegisterCommand(Command{
+		Name:  "add-package",
+		Func:  cmdAddPackage,
+		Usage: "<packages...>",
+		Short: "Adds Caddy packages (EXPERIMENTAL)",
+		Long: `
+Downloads an updated Caddy binary with the specified packages (module/plugin)
+added. Retains existing packages. Returns an error if the any of packages are 
+already included. EXPERIMENTAL: May be changed or removed.
+`,
+	})
+
+	RegisterCommand(Command{
+		Name:  "remove-package",
+		Func:  cmdRemovePackage,
+		Usage: "<packages...>",
+		Short: "Removes Caddy packages (EXPERIMENTAL)",
+		Long: `
+Downloads an updated Caddy binaries without the specified packages (module/plugin). 
+Returns an error if any of the packages are not included. 
+EXPERIMENTAL: May be changed or removed.
+`,
+	})
+
 }
 
 // RegisterCommand registers the command cmd.
diff --git a/cmd/packagesfuncs.go b/cmd/packagesfuncs.go
new file mode 100644
index 0000000..6aaf52b
--- /dev/null
+++ b/cmd/packagesfuncs.go
@@ -0,0 +1,306 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package caddycmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"reflect"
+	"runtime"
+	"runtime/debug"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"go.uber.org/zap"
+)
+
+func cmdUpgrade(_ Flags) (int, error) {
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	return upgradeBuild(pluginPkgs)
+}
+
+func cmdAddPackage(fl Flags) (int, error) {
+	if len(fl.Args()) == 0 {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified")
+	}
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	for _, arg := range fl.Args() {
+		if _, ok := pluginPkgs[arg]; ok {
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("package is already added")
+		}
+		pluginPkgs[arg] = struct{}{}
+	}
+
+	return upgradeBuild(pluginPkgs)
+}
+
+func cmdRemovePackage(fl Flags) (int, error) {
+	if len(fl.Args()) == 0 {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified")
+	}
+	_, nonstandard, _, err := getModules()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err)
+	}
+	pluginPkgs, err := getPluginPackages(nonstandard)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	for _, arg := range fl.Args() {
+		if _, ok := pluginPkgs[arg]; !ok {
+			// package does not exist
+			return caddy.ExitCodeFailedStartup, fmt.Errorf("package is not added")
+		}
+		delete(pluginPkgs, arg)
+	}
+
+	return upgradeBuild(pluginPkgs)
+}
+
+func upgradeBuild(pluginPkgs map[string]struct{}) (int, error) {
+	l := caddy.Log()
+
+	thisExecPath, err := os.Executable()
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err)
+	}
+	thisExecStat, err := os.Stat(thisExecPath)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err)
+	}
+	l.Info("this executable will be replaced", zap.String("path", thisExecPath))
+
+	// build the request URL to download this custom build
+	qs := url.Values{
+		"os":   {runtime.GOOS},
+		"arch": {runtime.GOARCH},
+	}
+	for pkg := range pluginPkgs {
+		qs.Add("p", pkg)
+	}
+
+	// initiate the build
+	resp, err := downloadBuild(qs)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// back up the current binary, in case something goes wrong we can replace it
+	backupExecPath := thisExecPath + ".tmp"
+	l.Info("build acquired; backing up current executable",
+		zap.String("current_path", thisExecPath),
+		zap.String("backup_path", backupExecPath))
+	err = os.Rename(thisExecPath, backupExecPath)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			err2 := os.Rename(backupExecPath, thisExecPath)
+			if err2 != nil {
+				l.Error("restoring original executable failed; will need to be restored manually",
+					zap.String("backup_path", backupExecPath),
+					zap.String("original_path", thisExecPath),
+					zap.Error(err2))
+			}
+		}
+	}()
+
+	// download the file; do this in a closure to close reliably before we execute it
+	err = writeCaddyBinary(thisExecPath, &resp.Body, thisExecStat)
+	if err != nil {
+		return caddy.ExitCodeFailedStartup, err
+	}
+
+	l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath))
+
+	// use the new binary to print out version and module info
+	fmt.Print("\nModule versions:\n\n")
+	if err = listModules(thisExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	fmt.Println("\nVersion:")
+	if err = showVersion(thisExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	fmt.Println()
+
+	// clean up the backup file
+	if err = os.Remove(backupExecPath); err != nil {
+		return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err)
+	}
+	l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath))
+
+	return caddy.ExitCodeSuccess, nil
+}
+
+func getModules() (standard, nonstandard, unknown []moduleInfo, err error) {
+	bi, ok := debug.ReadBuildInfo()
+	if !ok {
+		err = fmt.Errorf("no build info")
+		return
+	}
+
+	for _, modID := range caddy.Modules() {
+		modInfo, err := caddy.GetModule(modID)
+		if err != nil {
+			// that's weird, shouldn't happen
+			unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err})
+			continue
+		}
+
+		// to get the Caddy plugin's version info, we need to know
+		// the package that the Caddy module's value comes from; we
+		// can use reflection but we need a non-pointer value (I'm
+		// not sure why), and since New() should return a pointer
+		// value, we need to dereference it first
+		iface := interface{}(modInfo.New())
+		if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr {
+			iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface()
+		}
+		modPkgPath := reflect.TypeOf(iface).PkgPath()
+
+		// now we find the Go module that the Caddy module's package
+		// belongs to; we assume the Caddy module package path will
+		// be prefixed by its Go module path, and we will choose the
+		// longest matching prefix in case there are nested modules
+		var matched *debug.Module
+		for _, dep := range bi.Deps {
+			if strings.HasPrefix(modPkgPath, dep.Path) {
+				if matched == nil || len(dep.Path) > len(matched.Path) {
+					matched = dep
+				}
+			}
+		}
+
+		caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched}
+
+		if strings.HasPrefix(modPkgPath, caddy.ImportPath) {
+			standard = append(standard, caddyModGoMod)
+		} else {
+			nonstandard = append(nonstandard, caddyModGoMod)
+		}
+	}
+	return
+}
+
+func listModules(path string) error {
+	cmd := exec.Command(path, "list-modules", "--versions")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	return nil
+}
+
+func showVersion(path string) error {
+	cmd := exec.Command(path, "version")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("download succeeded, but unable to execute: %v", err)
+	}
+	return nil
+}
+
+func downloadBuild(qs url.Values) (*http.Response, error) {
+	l := caddy.Log()
+	l.Info("requesting build",
+		zap.String("os", qs.Get("os")),
+		zap.String("arch", qs.Get("arch")),
+		zap.Strings("packages", qs["p"]))
+	resp, err := http.Get(fmt.Sprintf("%s?%s", downloadPath, qs.Encode()))
+	if err != nil {
+		return nil, fmt.Errorf("secure request failed: %v", err)
+	}
+	if resp.StatusCode >= 400 {
+		var details struct {
+			StatusCode int `json:"status_code"`
+			Error      struct {
+				Message string `json:"message"`
+				ID      string `json:"id"`
+			} `json:"error"`
+		}
+		err2 := json.NewDecoder(resp.Body).Decode(&details)
+		if err2 != nil {
+			return nil, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2)
+		}
+		return nil, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID)
+	}
+	return resp, nil
+}
+
+func getPluginPackages(modules []moduleInfo) (map[string]struct{}, error) {
+	pluginPkgs := make(map[string]struct{})
+	for _, mod := range modules {
+		if mod.goModule.Replace != nil {
+			return nil, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s",
+				mod.goModule.Path, mod.goModule.Replace.Path)
+		}
+		pluginPkgs[mod.goModule.Path] = struct{}{}
+	}
+	return pluginPkgs, nil
+}
+
+func writeCaddyBinary(path string, body *io.ReadCloser, fileInfo os.FileInfo) error {
+	l := caddy.Log()
+	destFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("unable to open destination file: %v", err)
+	}
+	defer destFile.Close()
+
+	l.Info("downloading binary", zap.String("destination", path))
+
+	_, err = io.Copy(destFile, *body)
+	if err != nil {
+		return fmt.Errorf("unable to download file: %v", err)
+	}
+
+	err = destFile.Sync()
+	if err != nil {
+		return fmt.Errorf("syncing downloaded file to device: %v", err)
+	}
+
+	return nil
+}
+
+const downloadPath = "https://caddyserver.com/api/download"