From 76e96ee673f83be3566fd8d2a252a32eaf920c0d Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Fri, 24 Jan 2025 11:35:28 +0100 Subject: [PATCH] Change "mox backup $destdir" from storing only data files to $destdir to storing those under $destdir/data and now also copying config files to $destdir/config. (#150) Upgrade note: Admins may want to check their backup scripts. Based on feedback in issue #150. --- .gitignore | 2 +- README.md | 21 +++--- backup.go | 93 ++++++++++++++++++++++++-- ctl_test.go | 10 +-- doc.go | 30 +++++---- gentestdata.go | 2 +- main.go | 28 ++++---- testdata/ctl/{ => config}/domains.conf | 0 testdata/ctl/{ => config}/mox.conf | 2 +- 9 files changed, 140 insertions(+), 48 deletions(-) rename testdata/ctl/{ => config}/domains.conf (100%) rename testdata/ctl/{ => config}/mox.conf (87%) diff --git a/.gitignore b/.gitignore index 88d33e8..e76c9c5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ /local/ /testdata/check/ /testdata/*/data/ -/testdata/ctl/dkim/ +/testdata/ctl/config/dkim/ /testdata/empty/ /testdata/exportmaildir/ /testdata/exportmbox/ diff --git a/README.md b/README.md index e060ec8..eff3b9b 100644 --- a/README.md +++ b/README.md @@ -347,15 +347,18 @@ in place and restart. If manual actions are required, the release notes mention them. Check the release notes of all version between your current installation and the release you're upgrading to. -Before upgrading, make a backup of the data directory with `mox backup -`. This writes consistent snapshots of the database files, and -duplicates message files from the outgoing queue and accounts. Using the new -mox binary, run `mox verifydata ` (do NOT use the "live" data -directory!) for a dry run. If this fails, an upgrade will probably fail too. -Important: verifydata with the new mox binary can modify the database files (due -to automatic schema upgrades). So make a fresh backup again before the actual -upgrade. See the help output of the "backup" and "verifydata" commands for more -details. +Before upgrading, make a backup of the config & data directory with `mox backup +`. This copies all files from the config directory to +`/config`, and creates `/data` with a consistent snapshots of +the database files, and message files from the outgoing queue and accounts. +Using the new mox binary, run `mox verifydata /data` (do NOT use the +"live" data directory!) for a dry run. If this fails, an upgrade will probably +fail too. + +Important: verifydata with the new mox binary can modify the database files +(due to automatic schema upgrades). So make a fresh backup again before the +actual upgrade. See the help output of the "backup" and "verifydata" commands +for more details. During backup, message files are hardlinked if possible, and copied otherwise. Using a destination directory like `data/tmp/backup` increases the odds diff --git a/backup.go b/backup.go index b8e172a..e3787af 100644 --- a/backup.go +++ b/backup.go @@ -40,7 +40,7 @@ func backupctl(ctx context.Context, ctl *ctl) { // "src" or "dst" are incomplete paths relative to the source or destination data // directories. - dstDataDir := ctl.xread() + dstDir := ctl.xread() verbose := ctl.xread() == "verbose" // Set when an error is encountered. At the end, we warn if set. @@ -93,8 +93,94 @@ func backupctl(ctx context.Context, ctl *ctl) { } } + dstConfigDir := filepath.Join(dstDir, "config") + dstDataDir := filepath.Join(dstDir, "data") + + // Warn if directories already exist, will likely cause failures when trying to + // write files that already exist. + if _, err := os.Stat(dstConfigDir); err == nil { + xwarnx("destination config directory already exists", nil, slog.String("configdir", dstConfigDir)) + } if _, err := os.Stat(dstDataDir); err == nil { - xwarnx("destination data directory already exists", nil, slog.String("dir", dstDataDir)) + xwarnx("destination data directory already exists", nil, slog.String("datadir", dstDataDir)) + } + + os.MkdirAll(dstDir, 0770) + os.MkdirAll(dstConfigDir, 0770) + os.MkdirAll(dstDataDir, 0770) + + // Copy all files in the config dir. + srcConfigDir := filepath.Clean(mox.ConfigDirPath(".")) + err := filepath.WalkDir(srcConfigDir, func(srcPath string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if srcConfigDir == srcPath { + return nil + } + + // Trim directory and separator. + relPath := srcPath[len(srcConfigDir)+1:] + + destPath := filepath.Join(dstConfigDir, relPath) + + if d.IsDir() { + if info, err := os.Stat(srcPath); err != nil { + return fmt.Errorf("stat config dir %s: %v", srcPath, err) + } else if err := os.Mkdir(destPath, info.Mode()&0777); err != nil { + return fmt.Errorf("mkdir %s: %v", destPath, err) + } + return nil + } + if d.Type()&fs.ModeSymlink != 0 { + linkDest, err := os.Readlink(srcPath) + if err != nil { + return fmt.Errorf("reading symlink %s: %v", srcPath, err) + } + if err := os.Symlink(linkDest, destPath); err != nil { + return fmt.Errorf("creating symlink %s: %v", destPath, err) + } + return nil + } + if !d.Type().IsRegular() { + xwarnx("skipping non-regular/dir/symlink file in config dir", nil, slog.String("path", srcPath)) + return nil + } + + sf, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("open config file %s: %v", srcPath, err) + } + info, err := sf.Stat() + if err != nil { + return fmt.Errorf("stat config file %s: %v", srcPath, err) + } + df, err := os.OpenFile(destPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0777&info.Mode()) + if err != nil { + return fmt.Errorf("create destination config file %s: %v", destPath, err) + } + defer func() { + if df != nil { + err := df.Close() + ctl.log.Check(err, "closing file") + } + }() + defer func() { + err := sf.Close() + ctl.log.Check(err, "closing file") + }() + if _, err := io.Copy(df, sf); err != nil { + return fmt.Errorf("copying config file %s to %s: %v", srcPath, destPath, err) + } + if err := df.Close(); err != nil { + return fmt.Errorf("closing destination config file %s: %v", srcPath, err) + } + df = nil + return nil + }) + if err != nil { + xerrx("storing config directory", err) } srcDataDir := filepath.Clean(mox.DataDirPath(".")) @@ -280,8 +366,7 @@ func backupctl(ctx context.Context, ctl *ctl) { ctl.log.Print("making backup", slog.String("destdir", dstDataDir)) - err := os.MkdirAll(dstDataDir, 0770) - if err != nil { + if err := os.MkdirAll(dstDataDir, 0770); err != nil { xerrx("creating destination data directory", err) } diff --git a/ctl_test.go b/ctl_test.go index 1a7c846..afb17a3 100644 --- a/ctl_test.go +++ b/ctl_test.go @@ -43,8 +43,8 @@ func tcheck(t *testing.T, err error, errmsg string) { // unhandled errors would cause a panic. func TestCtl(t *testing.T) { os.RemoveAll("testdata/ctl/data") - mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/mox.conf") - mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/domains.conf") + mox.ConfigStaticPath = filepath.FromSlash("testdata/ctl/config/mox.conf") + mox.ConfigDynamicPath = filepath.FromSlash("testdata/ctl/config/domains.conf") if errs := mox.LoadConfig(ctxbg, pkglog, true, false); len(errs) > 0 { t.Fatalf("loading mox config: %v", errs) } @@ -485,16 +485,16 @@ func TestCtl(t *testing.T) { tcheck(t, err, "tlsrptdb init") defer tlsrptdb.Close() testctl(func(ctl *ctl) { - os.RemoveAll("testdata/ctl/data/tmp/backup-data") + os.RemoveAll("testdata/ctl/data/tmp/backup") err := os.WriteFile("testdata/ctl/data/receivedid.key", make([]byte, 16), 0600) tcheck(t, err, "writing receivedid.key") - ctlcmdBackup(ctl, filepath.FromSlash("testdata/ctl/data/tmp/backup-data"), false) + ctlcmdBackup(ctl, filepath.FromSlash("testdata/ctl/data/tmp/backup"), false) }) // Verify the backup. xcmd := cmd{ flag: flag.NewFlagSet("", flag.ExitOnError), - flagArgs: []string{filepath.FromSlash("testdata/ctl/data/tmp/backup-data")}, + flagArgs: []string{filepath.FromSlash("testdata/ctl/data/tmp/backup/data")}, } cmdVerifydata(&xcmd) } diff --git a/doc.go b/doc.go index 0761186..69b1156 100644 --- a/doc.go +++ b/doc.go @@ -55,7 +55,7 @@ any parameters. Followed by the help and usage information for each command. mox export mbox [-single] dst-dir account-path [mailbox] mox localserve mox help [command ...] - mox backup dest-dir + mox backup destdir mox verifydata data-dir mox licenses mox config test @@ -819,13 +819,14 @@ If a single command matches, its usage and full help text is printed. # mox backup -Creates a backup of the data directory. +Creates a backup of the config and data directory. -Backup creates consistent snapshots of the databases and message files and -copies other files in the data directory. Empty directories are not copied. -These files can then be stored elsewhere for long-term storage, or used to fall -back to should an upgrade fail. Simply copying files in the data directory -while mox is running can result in unusable database files. +Backup copies the config directory to /config, and creates +/data with a consistent snapshot of the databases and message files +and copies other files from the data directory. Empty directories are not +copied. The backup can then be stored elsewhere for long-term storage, or used +to fall back to should an upgrade fail. Simply copying files in the data +directory while mox is running can result in unusable database files. Message files never change (they are read-only, though can be removed) and are hard-linked so they don't consume additional space. If hardlinking fails, for @@ -847,18 +848,19 @@ not print any output, but may print warnings. Use the -verbose flag for details, including timing. To restore a backup, first shut down mox, move away the old data directory and -move an earlier backed up directory in its place, run "mox verifydata", -possibly with the "-fix" option, and restart mox. After the restore, you may -also want to run "mox bumpuidvalidity" for each account for which messages in a -mailbox changed, to force IMAP clients to synchronize mailbox state. +move an earlier backed up directory in its place, run "mox verifydata +", possibly with the "-fix" option, and restart mox. After the +restore, you may also want to run "mox bumpuidvalidity" for each account for +which messages in a mailbox changed, to force IMAP clients to synchronize +mailbox state. Before upgrading, to check if the upgrade will likely succeed, first make a -backup, then use the new mox binary to run "mox verifydata" on the backup. This -can change the backup files (e.g. upgrade database files, move away +backup, then use the new mox binary to run "mox verifydata /data". +This can change the backup files (e.g. upgrade database files, move away unrecognized message files), so you should make a new backup before actually upgrading. - usage: mox backup dest-dir + usage: mox backup destdir -verbose print progress diff --git a/gentestdata.go b/gentestdata.go index 47ad4c7..16d65f7 100644 --- a/gentestdata.go +++ b/gentestdata.go @@ -30,7 +30,7 @@ import ( func cmdGentestdata(c *cmd) { c.unlisted = true - c.params = "dest-dir" + c.params = "destdir" c.help = `Generate a data directory populated, for testing upgrades.` args := c.Parse() if len(args) != 1 { diff --git a/main.go b/main.go index 2a62f4f..54c65bc 100644 --- a/main.go +++ b/main.go @@ -1561,14 +1561,15 @@ new mail deliveries. } func cmdBackup(c *cmd) { - c.params = "dest-dir" - c.help = `Creates a backup of the data directory. + c.params = "destdir" + c.help = `Creates a backup of the config and data directory. -Backup creates consistent snapshots of the databases and message files and -copies other files in the data directory. Empty directories are not copied. -These files can then be stored elsewhere for long-term storage, or used to fall -back to should an upgrade fail. Simply copying files in the data directory -while mox is running can result in unusable database files. +Backup copies the config directory to /config, and creates +/data with a consistent snapshot of the databases and message files +and copies other files from the data directory. Empty directories are not +copied. The backup can then be stored elsewhere for long-term storage, or used +to fall back to should an upgrade fail. Simply copying files in the data +directory while mox is running can result in unusable database files. Message files never change (they are read-only, though can be removed) and are hard-linked so they don't consume additional space. If hardlinking fails, for @@ -1590,14 +1591,15 @@ not print any output, but may print warnings. Use the -verbose flag for details, including timing. To restore a backup, first shut down mox, move away the old data directory and -move an earlier backed up directory in its place, run "mox verifydata", -possibly with the "-fix" option, and restart mox. After the restore, you may -also want to run "mox bumpuidvalidity" for each account for which messages in a -mailbox changed, to force IMAP clients to synchronize mailbox state. +move an earlier backed up directory in its place, run "mox verifydata +", possibly with the "-fix" option, and restart mox. After the +restore, you may also want to run "mox bumpuidvalidity" for each account for +which messages in a mailbox changed, to force IMAP clients to synchronize +mailbox state. Before upgrading, to check if the upgrade will likely succeed, first make a -backup, then use the new mox binary to run "mox verifydata" on the backup. This -can change the backup files (e.g. upgrade database files, move away +backup, then use the new mox binary to run "mox verifydata /data". +This can change the backup files (e.g. upgrade database files, move away unrecognized message files), so you should make a new backup before actually upgrading. ` diff --git a/testdata/ctl/domains.conf b/testdata/ctl/config/domains.conf similarity index 100% rename from testdata/ctl/domains.conf rename to testdata/ctl/config/domains.conf diff --git a/testdata/ctl/mox.conf b/testdata/ctl/config/mox.conf similarity index 87% rename from testdata/ctl/mox.conf rename to testdata/ctl/config/mox.conf index e1286db..b9062bf 100644 --- a/testdata/ctl/mox.conf +++ b/testdata/ctl/config/mox.conf @@ -1,4 +1,4 @@ -DataDir: data +DataDir: ../data User: 1000 LogLevel: trace Hostname: mox.example