Change storage paths to follow OS conventions; migrate folder (#2955)

caddy.go

@@ -348,7 +348,7 @@ func run(newCfg *Config, start bool) error {
 		}
 
 		if newCfg.storage == nil {
-			newCfg.storage = &certmagic.FileStorage{Path: dataDir()}
+			newCfg.storage = &certmagic.FileStorage{Path: AppDataDir()}
 		}
 		certmagic.Default.Storage = newCfg.storage
 

cmd/commandfuncs.go

@@ -154,6 +154,8 @@ func cmdRun(fl Flags) (int, error) {
 	if err != nil {
 		return caddy.ExitCodeFailedStartup, err
 	}
+	// TODO: This is TEMPORARY, until the RCs
+	moveStorage()
 
 	// set a fitting User-Agent for ACME requests
 	goModule := caddy.GoModule()
@@ -190,6 +192,25 @@ func cmdRun(fl Flags) (int, error) {
 		}
 	}
 
+	// warn if the environment does not provide enough information about the disk
+	hasXDG := os.Getenv("XDG_DATA_HOME") != "" &&
+		os.Getenv("XDG_CONFIG_HOME") != "" &&
+		os.Getenv("XDG_CACHE_HOME") != ""
+	switch runtime.GOOS {
+	case "windows":
+		if os.Getenv("HOME") == "" && os.Getenv("USERPROFILE") == "" && !hasXDG {
+			caddy.Log().Warn("neither HOME nor USERPROFILE environment variables are set - please fix; some assets might be stored in ./caddy")
+		}
+	case "plan9":
+		if os.Getenv("home") == "" && !hasXDG {
+			caddy.Log().Warn("$home environment variable is empty - please fix; some assets might be stored in ./caddy")
+		}
+	default:
+		if os.Getenv("HOME") == "" && !hasXDG {
+			caddy.Log().Warn("$HOME environment variable is empty - please fix; some assets might be stored in ./caddy")
+		}
+	}
+
 	select {}
 }
 

cmd/main.go

@@ -22,12 +22,15 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
+	"go.uber.org/zap"
 )
 
 // Main implements the main function of the caddy command.
@@ -226,3 +229,71 @@ func printEnvironment() {
 		fmt.Println(v)
 	}
 }
+
+// moveStorage moves the old default dataDir to the new default dataDir.
+// TODO: This is TEMPORARY until the release candidates.
+func moveStorage() {
+	// get the home directory (the old way)
+	oldHome := os.Getenv("HOME")
+	if oldHome == "" && runtime.GOOS == "windows" {
+		drive := os.Getenv("HOMEDRIVE")
+		path := os.Getenv("HOMEPATH")
+		oldHome = drive + path
+		if drive == "" || path == "" {
+			oldHome = os.Getenv("USERPROFILE")
+		}
+	}
+	if oldHome == "" {
+		oldHome = "."
+	}
+	oldDataDir := filepath.Join(oldHome, ".local", "share", "caddy")
+
+	// nothing to do if old data dir doesn't exist
+	_, err := os.Stat(oldDataDir)
+	if os.IsNotExist(err) {
+		return
+	}
+
+	// nothing to do if the new data dir is the same as the old one
+	newDataDir := caddy.AppDataDir()
+	if oldDataDir == newDataDir {
+		return
+	}
+
+	logger := caddy.Log().Named("automigrate").With(
+		zap.String("old_dir", oldDataDir),
+		zap.String("new_dir", newDataDir))
+
+	logger.Info("beginning one-time data directory migration",
+		zap.String("details", "https://github.com/caddyserver/caddy/issues/2955"))
+
+	// if new data directory exists, avoid auto-migration as a conservative safety measure
+	_, err = os.Stat(newDataDir)
+	if !os.IsNotExist(err) {
+		logger.Error("new data directory already exists; skipping auto-migration as conservative safety measure",
+			zap.Error(err),
+			zap.String("instructions", "https://github.com/caddyserver/caddy/issues/2955#issuecomment-570000333"))
+		return
+	}
+
+	// construct the new data directory's parent folder
+	err = os.MkdirAll(filepath.Dir(newDataDir), 0700)
+	if err != nil {
+		logger.Error("unable to make new datadirectory - follow link for instructions",
+			zap.String("instructions", "https://github.com/caddyserver/caddy/issues/2955#issuecomment-570000333"),
+			zap.Error(err))
+		return
+	}
+
+	// folder structure is same, so just try to rename (move) it;
+	// this fails if the new path is on a separate device
+	err = os.Rename(oldDataDir, newDataDir)
+	if err != nil {
+		logger.Error("new data directory already exists; skipping auto-migration as conservative safety measure - follow link for instructions",
+			zap.String("instructions", "https://github.com/caddyserver/caddy/issues/2955#issuecomment-570000333"),
+			zap.Error(err))
+	}
+
+	logger.Info("successfully completed one-time migration of data directory",
+		zap.String("details", "https://github.com/caddyserver/caddy/issues/2955"))
+}

go.sum

@@ -63,6 +63,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/cloudflare-go v0.10.2 h1:VBodKICVPnwmDxstcW3biKcDSpFIfS/RELUXsZSBYK4=
 github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=

storage.go

@@ -20,6 +20,7 @@ import (
 	"runtime"
 
 	"github.com/mholt/certmagic"
+	"go.uber.org/zap"
 )
 
 // StorageConverter is a type that can convert itself
@@ -31,10 +32,28 @@ type StorageConverter interface {
 	CertMagicStorage() (certmagic.Storage, error)
 }
 
-// homeDir returns the best guess of the current user's home
+// HomeDir returns the best guess of the current user's home
 // directory from environment variables. If unknown, "." (the
-// current directory) is returned instead.
+// current directory) is returned instead, except GOOS=android,
-func homeDir() string {
+// which returns "/sdcard".
+func HomeDir() string {
+	home := homeDirUnsafe()
+	if home == "" && runtime.GOOS == "android" {
+		home = "/sdcard"
+	}
+	if home == "" {
+		home = "."
+	}
+	return home
+}
+
+// homeDirUnsafe is a low-level function that returns
+// the user's home directory from environment
+// variables. Careful: if it cannot be determined, an
+// empty string is returned. If not accounting for
+// that case, use HomeDir() instead; otherwise you
+// may end up using the root of the file system.
+func homeDirUnsafe() string {
 	home := os.Getenv("HOME")
 	if home == "" && runtime.GOOS == "windows" {
 		drive := os.Getenv("HOMEDRIVE")
@@ -44,21 +63,96 @@ func homeDir() string {
 			home = os.Getenv("USERPROFILE")
 		}
 	}
-	if home == "" {
+	if home == "" && runtime.GOOS == "plan9" {
-		home = "."
+		home = os.Getenv("home")
 	}
 	return home
 }
 
-// dataDir returns a directory path that is suitable for storage.
+// AppConfigDir returns the directory where to store user's config.
-// https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables
+//
-func dataDir() string {
+// If XDG_CONFIG_HOME is set, it returns: $XDG_CONFIG_HOME/caddy.
-	baseDir := filepath.Join(homeDir(), ".local", "share")
+// Otherwise, os.UserConfigDir() is used; if successful, it appends
-	if xdgData := os.Getenv("XDG_DATA_HOME"); xdgData != "" {
+// "Caddy" (Windows & Mac) or "caddy" (every other OS) to the path.
-		baseDir = xdgData
+// If it returns an error, the fallback path "./caddy" is returned.
+//
+// The config directory is not guaranteed to be different from
+// AppDataDir().
+//
+// Unlike os.UserConfigDir(), this function prefers the
+// XDG_CONFIG_HOME env var on all platforms, not just Unix.
+//
+// Ref: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
+func AppConfigDir() string {
+	basedir := os.Getenv("XDG_CONFIG_HOME")
+	if basedir == "" {
+		var err error
+		basedir, err = os.UserConfigDir()
+		if err != nil {
+			Log().Warn("unable to determine directory for user configuration; falling back to current directory", zap.Error(err))
+			return "./caddy"
 		}
-	return filepath.Join(baseDir, "caddy")
+	}
+	subdir := "caddy"
+	switch runtime.GOOS {
+	case "windows", "darwin":
+		subdir = "Caddy"
+	}
+	return filepath.Join(basedir, subdir)
 }
 
-// TODO: Consider using Go 1.13's os.UserConfigDir() (https://golang.org/pkg/os/#UserConfigDir)
+// AppDataDir returns a directory path that is suitable for storing
-// if we are going to store the last-loaded config anywhere
+// application data on disk. It uses the environment for finding the
+// best place to store data, and appends a "caddy" or "Caddy" (depending
+// on OS and environment) subdirectory.
+//
+// For a base directory path:
+// If XDG_DATA_HOME is set, it returns: $XDG_DATA_HOME/caddy; otherwise,
+// on Windows it returns: %AppData%/Caddy,
+// on Mac: $HOME/Library/Application Support/Caddy,
+// on Plan9: $home/lib/caddy,
+// on Android: $HOME/caddy,
+// and on everything else: $HOME/.local/share/caddy.
+//
+// If a data directory cannot be determined, it returns "./caddy"
+// (this is not ideal, and the environment should be fixed).
+//
+// The data directory is not guaranteed to be different from AppConfigDir().
+//
+// Ref: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
+func AppDataDir() string {
+	if basedir := os.Getenv("XDG_DATA_HOME"); basedir != "" {
+		return filepath.Join(basedir, "caddy")
+	}
+	switch runtime.GOOS {
+	case "windows":
+		appData := os.Getenv("AppData")
+		if appData != "" {
+			return filepath.Join(appData, "Caddy")
+		}
+	case "darwin":
+		home := homeDirUnsafe()
+		if home != "" {
+			return filepath.Join(home, "Library", "Application Support", "Caddy")
+		}
+	case "plan9":
+		home := homeDirUnsafe()
+		if home != "" {
+			return filepath.Join(home, "lib", "caddy")
+		}
+	case "android":
+		home := homeDirUnsafe()
+		if home != "" {
+			return filepath.Join(home, "caddy")
+		}
+	default:
+		home := homeDirUnsafe()
+		if home != "" {
+			return filepath.Join(home, ".local", "share", "caddy")
+		}
+	}
+	return "./caddy"
+}
+
+// ConfigAutosavePath is the default path to which the last config will be persisted.
+var ConfigAutosavePath = filepath.Join(AppConfigDir(), "autosave.json")