From 68c5c71659109b10226f10873f7dc67102b9dc14 Mon Sep 17 00:00:00 2001 From: Oleg Date: Thu, 12 Aug 2021 02:31:41 +0300 Subject: [PATCH] cmd: New `add-package` and `remove-package` commands (#4226) * adding package command * add-package command name * refactoring duplicate code * fixed by review * fixed by review * remove-package command * commands in different files, common utils * fix add, remove, upgrade packages in 1 file * copyright and downloadPath moved * refactor * downloadPath do no export * adding/removing multiple packages * addPackages/removePackages, comments, command-desc * add-package, process case len(args) == 0 Co-authored-by: Francis Lavoie --- cmd/commandfuncs.go | 197 ---------------------------- cmd/commands.go | 24 ++++ cmd/packagesfuncs.go | 306 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 330 insertions(+), 197 deletions(-) create mode 100644 cmd/packagesfuncs.go diff --git a/cmd/commandfuncs.go b/cmd/commandfuncs.go index 58b97204..c70b57af 100644 --- a/cmd/commandfuncs.go +++ b/cmd/commandfuncs.go @@ -25,10 +25,8 @@ import ( "log" "net" "net/http" - "net/url" "os" "os/exec" - "reflect" "runtime" "runtime/debug" "sort" @@ -570,151 +568,6 @@ func cmdFmt(fl Flags) (int, error) { return caddy.ExitCodeSuccess, nil } -func cmdUpgrade(_ Flags) (int, error) { - l := caddy.Log() - - thisExecPath, err := os.Executable() - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err) - } - thisExecStat, err := os.Stat(thisExecPath) - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err) - } - l.Info("this executable will be replaced", zap.String("path", thisExecPath)) - - // get the list of nonstandard plugins - _, nonstandard, _, err := getModules() - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err) - } - pluginPkgs := make(map[string]struct{}) - for _, mod := range nonstandard { - if mod.goModule.Replace != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s", - mod.goModule.Path, mod.goModule.Replace.Path) - } - l.Info("found non-standard module", - zap.String("id", mod.caddyModuleID), - zap.String("package", mod.goModule.Path)) - pluginPkgs[mod.goModule.Path] = struct{}{} - } - - // build the request URL to download this custom build - qs := url.Values{ - "os": {runtime.GOOS}, - "arch": {runtime.GOARCH}, - } - for pkg := range pluginPkgs { - qs.Add("p", pkg) - } - urlStr := fmt.Sprintf("https://caddyserver.com/api/download?%s", qs.Encode()) - - // initiate the build - l.Info("requesting build", - zap.String("os", qs.Get("os")), - zap.String("arch", qs.Get("arch")), - zap.Strings("packages", qs["p"])) - resp, err := http.Get(urlStr) - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("secure request failed: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - var details struct { - StatusCode int `json:"status_code"` - Error struct { - Message string `json:"message"` - ID string `json:"id"` - } `json:"error"` - } - err2 := json.NewDecoder(resp.Body).Decode(&details) - if err2 != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2) - } - return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID) - } - - // back up the current binary, in case something goes wrong we can replace it - backupExecPath := thisExecPath + ".tmp" - l.Info("build acquired; backing up current executable", - zap.String("current_path", thisExecPath), - zap.String("backup_path", backupExecPath)) - err = os.Rename(thisExecPath, backupExecPath) - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err) - } - defer func() { - if err != nil { - err2 := os.Rename(backupExecPath, thisExecPath) - if err2 != nil { - l.Error("restoring original executable failed; will need to be restored manually", - zap.String("backup_path", backupExecPath), - zap.String("original_path", thisExecPath), - zap.Error(err2)) - } - } - }() - - // download the file; do this in a closure to close reliably before we execute it - writeFile := func() error { - destFile, err := os.OpenFile(thisExecPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, thisExecStat.Mode()) - if err != nil { - return fmt.Errorf("unable to open destination file: %v", err) - } - defer destFile.Close() - - l.Info("downloading binary", zap.String("source", urlStr), zap.String("destination", thisExecPath)) - - _, err = io.Copy(destFile, resp.Body) - if err != nil { - return fmt.Errorf("unable to download file: %v", err) - } - - err = destFile.Sync() - if err != nil { - return fmt.Errorf("syncing downloaded file to device: %v", err) - } - - return nil - } - err = writeFile() - if err != nil { - return caddy.ExitCodeFailedStartup, err - } - - l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath)) - - // use the new binary to print out version and module info - fmt.Print("\nModule versions:\n\n") - cmd := exec.Command(thisExecPath, "list-modules", "--versions") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err) - } - fmt.Println("\nVersion:") - cmd = exec.Command(thisExecPath, "version") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err) - } - fmt.Println() - - // clean up the backup file - err = os.Remove(backupExecPath) - if err != nil { - return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err) - } - - l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath)) - - return caddy.ExitCodeSuccess, nil -} - func cmdHelp(fl Flags) (int, error) { const fullDocs = `Full documentation is available at: https://caddyserver.com/docs/command-line` @@ -779,56 +632,6 @@ commands: return caddy.ExitCodeSuccess, nil } -func getModules() (standard, nonstandard, unknown []moduleInfo, err error) { - bi, ok := debug.ReadBuildInfo() - if !ok { - err = fmt.Errorf("no build info") - return - } - - for _, modID := range caddy.Modules() { - modInfo, err := caddy.GetModule(modID) - if err != nil { - // that's weird, shouldn't happen - unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err}) - continue - } - - // to get the Caddy plugin's version info, we need to know - // the package that the Caddy module's value comes from; we - // can use reflection but we need a non-pointer value (I'm - // not sure why), and since New() should return a pointer - // value, we need to dereference it first - iface := interface{}(modInfo.New()) - if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr { - iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface() - } - modPkgPath := reflect.TypeOf(iface).PkgPath() - - // now we find the Go module that the Caddy module's package - // belongs to; we assume the Caddy module package path will - // be prefixed by its Go module path, and we will choose the - // longest matching prefix in case there are nested modules - var matched *debug.Module - for _, dep := range bi.Deps { - if strings.HasPrefix(modPkgPath, dep.Path) { - if matched == nil || len(dep.Path) > len(matched.Path) { - matched = dep - } - } - } - - caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched} - - if strings.HasPrefix(modPkgPath, caddy.ImportPath) { - standard = append(standard, caddyModGoMod) - } else { - nonstandard = append(nonstandard, caddyModGoMod) - } - } - return -} - // apiRequest makes an API request to the endpoint adminAddr with the // given HTTP method and request URI. If body is non-nil, it will be // assumed to be Content-Type application/json. diff --git a/cmd/commands.go b/cmd/commands.go index 36e9c04f..89c4fe43 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -291,6 +291,30 @@ Downloads an updated Caddy binary with the same modules/plugins at the latest versions. EXPERIMENTAL: May be changed or removed.`, }) + RegisterCommand(Command{ + Name: "add-package", + Func: cmdAddPackage, + Usage: "", + Short: "Adds Caddy packages (EXPERIMENTAL)", + Long: ` +Downloads an updated Caddy binary with the specified packages (module/plugin) +added. Retains existing packages. Returns an error if the any of packages are +already included. EXPERIMENTAL: May be changed or removed. +`, + }) + + RegisterCommand(Command{ + Name: "remove-package", + Func: cmdRemovePackage, + Usage: "", + Short: "Removes Caddy packages (EXPERIMENTAL)", + Long: ` +Downloads an updated Caddy binaries without the specified packages (module/plugin). +Returns an error if any of the packages are not included. +EXPERIMENTAL: May be changed or removed. +`, + }) + } // RegisterCommand registers the command cmd. diff --git a/cmd/packagesfuncs.go b/cmd/packagesfuncs.go new file mode 100644 index 00000000..6aaf52bf --- /dev/null +++ b/cmd/packagesfuncs.go @@ -0,0 +1,306 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package caddycmd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "reflect" + "runtime" + "runtime/debug" + "strings" + + "github.com/caddyserver/caddy/v2" + "go.uber.org/zap" +) + +func cmdUpgrade(_ Flags) (int, error) { + _, nonstandard, _, err := getModules() + if err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err) + } + pluginPkgs, err := getPluginPackages(nonstandard) + if err != nil { + return caddy.ExitCodeFailedStartup, err + } + + return upgradeBuild(pluginPkgs) +} + +func cmdAddPackage(fl Flags) (int, error) { + if len(fl.Args()) == 0 { + return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified") + } + _, nonstandard, _, err := getModules() + if err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err) + } + pluginPkgs, err := getPluginPackages(nonstandard) + if err != nil { + return caddy.ExitCodeFailedStartup, err + } + + for _, arg := range fl.Args() { + if _, ok := pluginPkgs[arg]; ok { + return caddy.ExitCodeFailedStartup, fmt.Errorf("package is already added") + } + pluginPkgs[arg] = struct{}{} + } + + return upgradeBuild(pluginPkgs) +} + +func cmdRemovePackage(fl Flags) (int, error) { + if len(fl.Args()) == 0 { + return caddy.ExitCodeFailedStartup, fmt.Errorf("at least one package name must be specified") + } + _, nonstandard, _, err := getModules() + if err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("unable to enumerate installed plugins: %v", err) + } + pluginPkgs, err := getPluginPackages(nonstandard) + if err != nil { + return caddy.ExitCodeFailedStartup, err + } + + for _, arg := range fl.Args() { + if _, ok := pluginPkgs[arg]; !ok { + // package does not exist + return caddy.ExitCodeFailedStartup, fmt.Errorf("package is not added") + } + delete(pluginPkgs, arg) + } + + return upgradeBuild(pluginPkgs) +} + +func upgradeBuild(pluginPkgs map[string]struct{}) (int, error) { + l := caddy.Log() + + thisExecPath, err := os.Executable() + if err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("determining current executable path: %v", err) + } + thisExecStat, err := os.Stat(thisExecPath) + if err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("retrieving current executable permission bits: %v", err) + } + l.Info("this executable will be replaced", zap.String("path", thisExecPath)) + + // build the request URL to download this custom build + qs := url.Values{ + "os": {runtime.GOOS}, + "arch": {runtime.GOARCH}, + } + for pkg := range pluginPkgs { + qs.Add("p", pkg) + } + + // initiate the build + resp, err := downloadBuild(qs) + if err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("download failed: %v", err) + } + defer resp.Body.Close() + + // back up the current binary, in case something goes wrong we can replace it + backupExecPath := thisExecPath + ".tmp" + l.Info("build acquired; backing up current executable", + zap.String("current_path", thisExecPath), + zap.String("backup_path", backupExecPath)) + err = os.Rename(thisExecPath, backupExecPath) + if err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("backing up current binary: %v", err) + } + defer func() { + if err != nil { + err2 := os.Rename(backupExecPath, thisExecPath) + if err2 != nil { + l.Error("restoring original executable failed; will need to be restored manually", + zap.String("backup_path", backupExecPath), + zap.String("original_path", thisExecPath), + zap.Error(err2)) + } + } + }() + + // download the file; do this in a closure to close reliably before we execute it + err = writeCaddyBinary(thisExecPath, &resp.Body, thisExecStat) + if err != nil { + return caddy.ExitCodeFailedStartup, err + } + + l.Info("download successful; displaying new binary details", zap.String("location", thisExecPath)) + + // use the new binary to print out version and module info + fmt.Print("\nModule versions:\n\n") + if err = listModules(thisExecPath); err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err) + } + fmt.Println("\nVersion:") + if err = showVersion(thisExecPath); err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to execute: %v", err) + } + fmt.Println() + + // clean up the backup file + if err = os.Remove(backupExecPath); err != nil { + return caddy.ExitCodeFailedStartup, fmt.Errorf("download succeeded, but unable to clean up backup binary: %v", err) + } + l.Info("upgrade successful; please restart any running Caddy instances", zap.String("executable", thisExecPath)) + + return caddy.ExitCodeSuccess, nil +} + +func getModules() (standard, nonstandard, unknown []moduleInfo, err error) { + bi, ok := debug.ReadBuildInfo() + if !ok { + err = fmt.Errorf("no build info") + return + } + + for _, modID := range caddy.Modules() { + modInfo, err := caddy.GetModule(modID) + if err != nil { + // that's weird, shouldn't happen + unknown = append(unknown, moduleInfo{caddyModuleID: modID, err: err}) + continue + } + + // to get the Caddy plugin's version info, we need to know + // the package that the Caddy module's value comes from; we + // can use reflection but we need a non-pointer value (I'm + // not sure why), and since New() should return a pointer + // value, we need to dereference it first + iface := interface{}(modInfo.New()) + if rv := reflect.ValueOf(iface); rv.Kind() == reflect.Ptr { + iface = reflect.New(reflect.TypeOf(iface).Elem()).Elem().Interface() + } + modPkgPath := reflect.TypeOf(iface).PkgPath() + + // now we find the Go module that the Caddy module's package + // belongs to; we assume the Caddy module package path will + // be prefixed by its Go module path, and we will choose the + // longest matching prefix in case there are nested modules + var matched *debug.Module + for _, dep := range bi.Deps { + if strings.HasPrefix(modPkgPath, dep.Path) { + if matched == nil || len(dep.Path) > len(matched.Path) { + matched = dep + } + } + } + + caddyModGoMod := moduleInfo{caddyModuleID: modID, goModule: matched} + + if strings.HasPrefix(modPkgPath, caddy.ImportPath) { + standard = append(standard, caddyModGoMod) + } else { + nonstandard = append(nonstandard, caddyModGoMod) + } + } + return +} + +func listModules(path string) error { + cmd := exec.Command(path, "list-modules", "--versions") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("download succeeded, but unable to execute: %v", err) + } + return nil +} + +func showVersion(path string) error { + cmd := exec.Command(path, "version") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("download succeeded, but unable to execute: %v", err) + } + return nil +} + +func downloadBuild(qs url.Values) (*http.Response, error) { + l := caddy.Log() + l.Info("requesting build", + zap.String("os", qs.Get("os")), + zap.String("arch", qs.Get("arch")), + zap.Strings("packages", qs["p"])) + resp, err := http.Get(fmt.Sprintf("%s?%s", downloadPath, qs.Encode())) + if err != nil { + return nil, fmt.Errorf("secure request failed: %v", err) + } + if resp.StatusCode >= 400 { + var details struct { + StatusCode int `json:"status_code"` + Error struct { + Message string `json:"message"` + ID string `json:"id"` + } `json:"error"` + } + err2 := json.NewDecoder(resp.Body).Decode(&details) + if err2 != nil { + return nil, fmt.Errorf("download and error decoding failed: HTTP %d: %v", resp.StatusCode, err2) + } + return nil, fmt.Errorf("download failed: HTTP %d: %s (id=%s)", resp.StatusCode, details.Error.Message, details.Error.ID) + } + return resp, nil +} + +func getPluginPackages(modules []moduleInfo) (map[string]struct{}, error) { + pluginPkgs := make(map[string]struct{}) + for _, mod := range modules { + if mod.goModule.Replace != nil { + return nil, fmt.Errorf("cannot auto-upgrade when Go module has been replaced: %s => %s", + mod.goModule.Path, mod.goModule.Replace.Path) + } + pluginPkgs[mod.goModule.Path] = struct{}{} + } + return pluginPkgs, nil +} + +func writeCaddyBinary(path string, body *io.ReadCloser, fileInfo os.FileInfo) error { + l := caddy.Log() + destFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode()) + if err != nil { + return fmt.Errorf("unable to open destination file: %v", err) + } + defer destFile.Close() + + l.Info("downloading binary", zap.String("destination", path)) + + _, err = io.Copy(destFile, *body) + if err != nil { + return fmt.Errorf("unable to download file: %v", err) + } + + err = destFile.Sync() + if err != nil { + return fmt.Errorf("syncing downloaded file to device: %v", err) + } + + return nil +} + +const downloadPath = "https://caddyserver.com/api/download"