Change "mox backup $destdir" from storing only data files to $destdir to storing those under $destdir/data and now also copying config files to $destdir/config. (#150)

Upgrade note: Admins may want to check their backup scripts.

Based on feedback in issue #150.

.gitignore

@@ -5,7 +5,7 @@
 /local/
 /testdata/check/
 /testdata/*/data/
-/testdata/ctl/dkim/
+/testdata/ctl/config/dkim/
 /testdata/empty/
 /testdata/exportmaildir/
 /testdata/exportmbox/

README.md

@@ -347,15 +347,18 @@ in place and restart. If manual actions are required, the release notes mention
 them. Check the release notes of all version between your current installation
 and the release you're upgrading to.
 
-Before upgrading, make a backup of the data directory with `mox backup
+Before upgrading, make a backup of the config & data directory with `mox backup
-<destdir>`. This writes consistent snapshots of the database files, and
+<destdir>`. This copies all files from the config directory to
-duplicates message files from the outgoing queue and accounts.  Using the new
+`<destdir>/config`, and creates `<destdir>/data` with a consistent snapshots of
-mox binary, run `mox verifydata <backupdir>` (do NOT use the "live" data
+the database files, and message files from the outgoing queue and accounts.
-directory!) for a dry run. If this fails, an upgrade will probably fail too.
+Using the new mox binary, run `mox verifydata <destdir>/data` (do NOT use the
-Important: verifydata with the new mox binary can modify the database files (due
+"live" data directory!) for a dry run. If this fails, an upgrade will probably
-to automatic schema upgrades). So make a fresh backup again before the actual
+fail too.
-upgrade. See the help output of the "backup" and "verifydata" commands for more
+
-details.
+Important: verifydata with the new mox binary can modify the database files
+(due to automatic schema upgrades). So make a fresh backup again before the
+actual upgrade. See the help output of the "backup" and "verifydata" commands
+for more details.
 
 During backup, message files are hardlinked if possible, and copied otherwise.
 Using a destination directory like `data/tmp/backup` increases the odds

backup.go

@@ -40,7 +40,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
 	// "src" or "dst" are incomplete paths relative to the source or destination data
 	// directories.
 
-	dstDataDir := ctl.xread()
+	dstDir := ctl.xread()
 	verbose := ctl.xread() == "verbose"
 
 	// Set when an error is encountered. At the end, we warn if set.
@@ -93,8 +93,94 @@ func backupctl(ctx context.Context, ctl *ctl) {
 		}
 	}
 
+	dstConfigDir := filepath.Join(dstDir, "config")
+	dstDataDir := filepath.Join(dstDir, "data")
+
+	// Warn if directories already exist, will likely cause failures when trying to
+	// write files that already exist.
+	if _, err := os.Stat(dstConfigDir); err == nil {
+		xwarnx("destination config directory already exists", nil, slog.String("configdir", dstConfigDir))
+	}
 	if _, err := os.Stat(dstDataDir); err == nil {
-		xwarnx("destination data directory already exists", nil, slog.String("dir", dstDataDir))
+		xwarnx("destination data directory already exists", nil, slog.String("datadir", dstDataDir))
+	}
+
+	os.MkdirAll(dstDir, 0770)
+	os.MkdirAll(dstConfigDir, 0770)
+	os.MkdirAll(dstDataDir, 0770)
+
+	// Copy all files in the config dir.
+	srcConfigDir := filepath.Clean(mox.ConfigDirPath("."))
+	err := filepath.WalkDir(srcConfigDir, func(srcPath string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if srcConfigDir == srcPath {
+			return nil
+		}
+
+		// Trim directory and separator.
+		relPath := srcPath[len(srcConfigDir)+1:]
+
+		destPath := filepath.Join(dstConfigDir, relPath)
+
+		if d.IsDir() {
+			if info, err := os.Stat(srcPath); err != nil {
+				return fmt.Errorf("stat config dir %s: %v", srcPath, err)
+			} else if err := os.Mkdir(destPath, info.Mode()&0777); err != nil {
+				return fmt.Errorf("mkdir %s: %v", destPath, err)
+			}
+			return nil
+		}
+		if d.Type()&fs.ModeSymlink != 0 {
+			linkDest, err := os.Readlink(srcPath)
+			if err != nil {
+				return fmt.Errorf("reading symlink %s: %v", srcPath, err)
+			}
+			if err := os.Symlink(linkDest, destPath); err != nil {
+				return fmt.Errorf("creating symlink %s: %v", destPath, err)
+			}
+			return nil
+		}
+		if !d.Type().IsRegular() {
+			xwarnx("skipping non-regular/dir/symlink file in config dir", nil, slog.String("path", srcPath))
+			return nil
+		}
+
+		sf, err := os.Open(srcPath)
+		if err != nil {
+			return fmt.Errorf("open config file %s: %v", srcPath, err)
+		}
+		info, err := sf.Stat()
+		if err != nil {
+			return fmt.Errorf("stat config file %s: %v", srcPath, err)
+		}
+		df, err := os.OpenFile(destPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0777&info.Mode())
+		if err != nil {
+			return fmt.Errorf("create destination config file %s: %v", destPath, err)
+		}
+		defer func() {
+			if df != nil {
+				err := df.Close()
+				ctl.log.Check(err, "closing file")
+			}
+		}()
+		defer func() {
+			err := sf.Close()
+			ctl.log.Check(err, "closing file")
+		}()
+		if _, err := io.Copy(df, sf); err != nil {
+			return fmt.Errorf("copying config file %s to %s: %v", srcPath, destPath, err)
+		}
+		if err := df.Close(); err != nil {
+			return fmt.Errorf("closing destination config file %s: %v", srcPath, err)
+		}
+		df = nil
+		return nil
+	})
+	if err != nil {
+		xerrx("storing config directory", err)
 	}
 
 	srcDataDir := filepath.Clean(mox.DataDirPath("."))
@@ -280,8 +366,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
 
 	ctl.log.Print("making backup", slog.String("destdir", dstDataDir))
 
-	err := os.MkdirAll(dstDataDir, 0770)
+	if err := os.MkdirAll(dstDataDir, 0770); err != nil {
-	if err != nil {
 		xerrx("creating destination data directory", err)
 	}
 

ctl_test.go

@@ -43,8 +43,8 @@ func tcheck(t *testing.T, err error, errmsg string) {
 // unhandled errors would cause a panic.
 func TestCtl(t *testing.T) {
 	os.RemoveAll("testdata/ctl/data")
-	mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/mox.conf")
+	mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/config/mox.conf")
-	mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/domains.conf")
+	mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/config/domains.conf")
 	if errs := mox.LoadConfig(ctxbg, pkglog, true, false); len(errs) > 0 {
 		t.Fatalf("loading mox config: %v", errs)
 	}
@@ -485,16 +485,16 @@ func TestCtl(t *testing.T) {
 	tcheck(t, err, "tlsrptdb init")
 	defer tlsrptdb.Close()
 	testctl(func(ctl *ctl) {
-		os.RemoveAll("testdata/ctl/data/tmp/backup-data")
+		os.RemoveAll("testdata/ctl/data/tmp/backup")
 		err := os.WriteFile("testdata/ctl/data/receivedid.key", make([]byte, 16), 0600)
 		tcheck(t, err, "writing receivedid.key")
-		ctlcmdBackup(ctl, filepath.FromSlash("testdata/ctl/data/tmp/backup-data"), false)
+		ctlcmdBackup(ctl, filepath.FromSlash("testdata/ctl/data/tmp/backup"), false)
 	})
 
 	// Verify the backup.
 	xcmd := cmd{
 		flag:     flag.NewFlagSet("", flag.ExitOnError),
-		flagArgs: []string{filepath.FromSlash("testdata/ctl/data/tmp/backup-data")},
+		flagArgs: []string{filepath.FromSlash("testdata/ctl/data/tmp/backup/data")},
 	}
 	cmdVerifydata(&xcmd)
 }

doc.go

@@ -55,7 +55,7 @@ any parameters. Followed by the help and usage information for each command.
 	mox export mbox [-single] dst-dir account-path [mailbox]
 	mox localserve
 	mox help [command ...]
-	mox backup dest-dir
+	mox backup destdir
 	mox verifydata data-dir
 	mox licenses
 	mox config test
@@ -819,13 +819,14 @@ If a single command matches, its usage and full help text is printed.
 
 # mox backup
 
-Creates a backup of the data directory.
+Creates a backup of the config and data directory.
 
-Backup creates consistent snapshots of the databases and message files and
+Backup copies the config directory to <destdir>/config, and creates
-copies other files in the data directory. Empty directories are not copied.
+<destdir>/data with a consistent snapshot of the databases and message files
-These files can then be stored elsewhere for long-term storage, or used to fall
+and copies other files from the data directory. Empty directories are not
-back to should an upgrade fail. Simply copying files in the data directory
+copied. The backup can then be stored elsewhere for long-term storage, or used
-while mox is running can result in unusable database files.
+to fall back to should an upgrade fail. Simply copying files in the data
+directory while mox is running can result in unusable database files.
 
 Message files never change (they are read-only, though can be removed) and are
 hard-linked so they don't consume additional space. If hardlinking fails, for
@@ -847,18 +848,19 @@ not print any output, but may print warnings. Use the -verbose flag for
 details, including timing.
 
 To restore a backup, first shut down mox, move away the old data directory and
-move an earlier backed up directory in its place, run "mox verifydata",
+move an earlier backed up directory in its place, run "mox verifydata
-possibly with the "-fix" option, and restart mox. After the restore, you may
+<datadir>", possibly with the "-fix" option, and restart mox. After the
-also want to run "mox bumpuidvalidity" for each account for which messages in a
+restore, you may also want to run "mox bumpuidvalidity" for each account for
-mailbox changed, to force IMAP clients to synchronize mailbox state.
+which messages in a mailbox changed, to force IMAP clients to synchronize
+mailbox state.
 
 Before upgrading, to check if the upgrade will likely succeed, first make a
-backup, then use the new mox binary to run "mox verifydata" on the backup. This
+backup, then use the new mox binary to run "mox verifydata <backupdir>/data".
-can change the backup files (e.g. upgrade database files, move away
+This can change the backup files (e.g. upgrade database files, move away
 unrecognized message files), so you should make a new backup before actually
 upgrading.
 
-	usage: mox backup dest-dir
+	usage: mox backup destdir
 	  -verbose
 	    	print progress
 

gentestdata.go

@@ -30,7 +30,7 @@ import (
 
 func cmdGentestdata(c *cmd) {
 	c.unlisted = true
-	c.params = "dest-dir"
+	c.params = "destdir"
 	c.help = `Generate a data directory populated, for testing upgrades.`
 	args := c.Parse()
 	if len(args) != 1 {

main.go

@@ -1561,14 +1561,15 @@ new mail deliveries.
 }
 
 func cmdBackup(c *cmd) {
-	c.params = "dest-dir"
+	c.params = "destdir"
-	c.help = `Creates a backup of the data directory.
+	c.help = `Creates a backup of the config and data directory.
 
-Backup creates consistent snapshots of the databases and message files and
+Backup copies the config directory to <destdir>/config, and creates
-copies other files in the data directory. Empty directories are not copied.
+<destdir>/data with a consistent snapshot of the databases and message files
-These files can then be stored elsewhere for long-term storage, or used to fall
+and copies other files from the data directory. Empty directories are not
-back to should an upgrade fail. Simply copying files in the data directory
+copied. The backup can then be stored elsewhere for long-term storage, or used
-while mox is running can result in unusable database files.
+to fall back to should an upgrade fail. Simply copying files in the data
+directory while mox is running can result in unusable database files.
 
 Message files never change (they are read-only, though can be removed) and are
 hard-linked so they don't consume additional space. If hardlinking fails, for
@@ -1590,14 +1591,15 @@ not print any output, but may print warnings. Use the -verbose flag for
 details, including timing.
 
 To restore a backup, first shut down mox, move away the old data directory and
-move an earlier backed up directory in its place, run "mox verifydata",
+move an earlier backed up directory in its place, run "mox verifydata
-possibly with the "-fix" option, and restart mox. After the restore, you may
+<datadir>", possibly with the "-fix" option, and restart mox. After the
-also want to run "mox bumpuidvalidity" for each account for which messages in a
+restore, you may also want to run "mox bumpuidvalidity" for each account for
-mailbox changed, to force IMAP clients to synchronize mailbox state.
+which messages in a mailbox changed, to force IMAP clients to synchronize
+mailbox state.
 
 Before upgrading, to check if the upgrade will likely succeed, first make a
-backup, then use the new mox binary to run "mox verifydata" on the backup. This
+backup, then use the new mox binary to run "mox verifydata <backupdir>/data".
-can change the backup files (e.g. upgrade database files, move away
+This can change the backup files (e.g. upgrade database files, move away
 unrecognized message files), so you should make a new backup before actually
 upgrading.
 `

testdata/ctl/config/mox.conf

@@ -1,4 +1,4 @@
-DataDir: data
+DataDir: ../data
 User: 1000
 LogLevel: trace
 Hostname: mox.example