add a "backup" subcommand to make consistent backups, and a "verifydata" subcommand to verify a backup before restoring, and add tests for future upgrades

the backup command will make consistent snapshots of all the database files. i
had been copying the db files before, and it usually works. but if the file is
modified during the backup, it is inconsistent and is likely to generate errors
when reading (can be at any moment in the future, when reading some db page).
"mox backup" opens the database file and writes out a copy in a transaction.
it also duplicates the message files.

before doing a restore, you could run "mox verifydata" on the to-be-restored
"data" directory. it check the database files, and compares the message files
with the database.

the new "gentestdata" subcommand generates a basic "data" directory, with a
queue and a few accounts. we will use it in the future along with "verifydata"
to test upgrades from old version to the latest version. both when going to the
next version, and when skipping several versions. the script test-upgrades.sh
executes these tests and doesn't do anything at the moment, because no releases
have this subcommand yet.

inspired by a failed upgrade attempt of a pre-release version.

Makefile

@@ -18,6 +18,9 @@ test-race:
 	CGO_ENABLED=1 go test -race -shuffle=on -covermode atomic -coverprofile cover.out ./...
 	go tool cover -html=cover.out -o cover.html
 
+test-upgrade:
+	./test-upgrade.sh
+
 check:
 	staticcheck ./...
 	staticcheck -tags integration

README.md

@@ -282,3 +282,34 @@ Mox also has an "admin" web interface where the mox instance administrator can
 make changes, e.g. add/remove/modify domains/accounts/addresses.
 
 Mox does not have a webmail yet, so there are no screenshots of actual email.
+
+## How do I upgrade my mox installation?
+
+We try to make upgrades effortless and you can typically just put a new binary
+in place and restart. If manual actions are required, the release notes mention
+them. Check the release notes of all version between your current installation
+and the release you're upgrading to.
+
+Before upgrading, make a backup of the data directory with `mox backup
+<destdir>`. This writes consistent snapshots of the database files, and
+duplicates message files from the queue and accounts.  Using the new mox
+binary, run `mox verifydata <backupdir>` (do NOT use the "live" data
+directory!) for a dry run. If this fails, an upgrade will probably fail too.
+Important: verifydata with the new mox binary can modify the database files
+(due to automatic schema upgrades). So make a fresh backup again before the
+actual upgrade. See the help output of the "backup" and "verifydata" commands
+for more details.
+
+During backup, message files are hardlinked if possible. Using a destination
+directory like `data/tmp/backup` increases the odds hardlinking succeeds: the
+default systemd service file specifically mounts the data directory, causing
+attempts to outside it to fail with an error about cross-device linking.
+
+If an upgrade fails and you have to restore (parts) of the data directory, you
+should run `mox verifydata <datadir>` (with the original binary) on the
+restored directory before starting mox again. If problematic files are found,
+for example queue or account message files that are not in the database, run
+`mox verifydata -fix <datadir>` to move away those files. After a restore, you may
+also want to run `mox bumpuidvalidity <account>` for each account for which
+messages in a mailbox changed, to force IMAP clients to synchronize mailbox
+state.

backup.go

@@ -0,0 +1,556 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/dmarcdb"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/mtastsdb"
+	"github.com/mjl-/mox/queue"
+	"github.com/mjl-/mox/store"
+	"github.com/mjl-/mox/tlsrptdb"
+)
+
+func backupctl(ctx context.Context, ctl *ctl) {
+	/* protocol:
+	> "backup"
+	> destdir
+	> "verbose" or ""
+	< stream
+	< "ok" or error
+	*/
+
+	// Convention in this function: variables containing "src" or "dst" are file system
+	// paths that can be passed to os.Open and such. Variables with dirs/paths without
+	// "src" or "dst" are incomplete paths relative to the source or destination data
+	// directories.
+
+	dstDataDir := ctl.xread()
+	verbose := ctl.xread() == "verbose"
+
+	// Set when an error is encountered. At the end, we warn if set.
+	var incomplete bool
+
+	// We'll be writing output, and logging both to mox and the ctl stream.
+	writer := ctl.writer()
+
+	// Format easily readable output for the user.
+	formatLog := func(prefix, text string, err error, fields ...mlog.Pair) []byte {
+		var b bytes.Buffer
+		fmt.Fprint(&b, prefix)
+		fmt.Fprint(&b, text)
+		if err != nil {
+			fmt.Fprint(&b, ": "+err.Error())
+		}
+		for _, f := range fields {
+			fmt.Fprintf(&b, "; %s=%v", f.Key, f.Value)
+		}
+		fmt.Fprint(&b, "\n")
+		return b.Bytes()
+	}
+
+	// Log an error to both the mox service as the user running "mox backup".
+	xlogx := func(prefix, text string, err error, fields ...mlog.Pair) {
+		ctl.log.Errorx(text, err, fields...)
+
+		_, werr := writer.Write(formatLog(prefix, text, err, fields...))
+		ctl.xcheck(werr, "write to ctl")
+	}
+
+	// Log an error but don't mark backup as failed.
+	xwarnx := func(text string, err error, fields ...mlog.Pair) {
+		xlogx("warning: ", text, err, fields...)
+	}
+
+	// Log an error that causes the backup to be marked as failed. We typically
+	// continue processing though.
+	xerrx := func(text string, err error, fields ...mlog.Pair) {
+		incomplete = true
+		xlogx("error: ", text, err, fields...)
+	}
+
+	// If verbose is enabled, log to the cli command. Always log as info level.
+	xvlog := func(text string, fields ...mlog.Pair) {
+		ctl.log.Info(text, fields...)
+		if verbose {
+			_, werr := writer.Write(formatLog("", text, nil, fields...))
+			ctl.xcheck(werr, "write to ctl")
+		}
+	}
+
+	if _, err := os.Stat(dstDataDir); err == nil {
+		xwarnx("destination data directory already exists", nil, mlog.Field("dir", dstDataDir))
+	}
+
+	srcDataDir := filepath.Clean(mox.DataDirPath("."))
+
+	// When creating a file in the destination, we first ensure its directory exists.
+	// We track which directories we created, to prevent needless syscalls.
+	createdDirs := map[string]struct{}{}
+	ensureDestDir := func(dstpath string) {
+		dstdir := filepath.Dir(dstpath)
+		if _, ok := createdDirs[dstdir]; !ok {
+			err := os.MkdirAll(dstdir, 0770)
+			if err != nil {
+				xerrx("creating directory", err)
+			}
+			createdDirs[dstdir] = struct{}{}
+		}
+	}
+
+	// Backup a single file by copying (never hardlinking, the file may change).
+	backupFile := func(path string) {
+		tmFile := time.Now()
+		srcpath := filepath.Join(srcDataDir, path)
+		dstpath := filepath.Join(dstDataDir, path)
+
+		sf, err := os.Open(srcpath)
+		if err != nil {
+			xerrx("open source file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
+			return
+		}
+		defer sf.Close()
+
+		ensureDestDir(dstpath)
+		df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
+		if err != nil {
+			xerrx("creating destination file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
+			return
+		}
+		defer func() {
+			if df != nil {
+				df.Close()
+			}
+		}()
+		if _, err := io.Copy(df, sf); err != nil {
+			xerrx("copying file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
+			return
+		}
+		err = df.Close()
+		df = nil
+		if err != nil {
+			xerrx("closing destination file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
+			return
+		}
+		xvlog("backed up file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmFile)))
+	}
+
+	// Back up the files in a directory (by copying).
+	backupDir := func(dir string) {
+		tmDir := time.Now()
+		srcdir := filepath.Join(srcDataDir, dir)
+		dstdir := filepath.Join(dstDataDir, dir)
+		err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error {
+			if err != nil {
+				xerrx("walking file (not backed up)", err, mlog.Field("srcpath", srcpath))
+				return nil
+			}
+			if d.IsDir() {
+				return nil
+			}
+			backupFile(srcpath[len(srcDataDir)+1:])
+			return nil
+		})
+		if err != nil {
+			xerrx("copying directory (not backed up properly)", err, mlog.Field("srcdir", srcdir), mlog.Field("dstdir", dstdir), mlog.Field("duration", time.Since(tmDir)))
+			return
+		}
+		xvlog("backed up directory", mlog.Field("dir", dir), mlog.Field("duration", time.Since(tmDir)))
+	}
+
+	// Backup a database by copying it in a readonly transaction.
+	// Always logs on error, so caller doesn't have to, but also returns the error so
+	// callers can see result.
+	backupDB := func(db *bstore.DB, path string) (rerr error) {
+		defer func() {
+			if rerr != nil {
+				xerrx("backing up database", rerr, mlog.Field("path", path))
+			}
+		}()
+
+		tmDB := time.Now()
+
+		dstpath := filepath.Join(dstDataDir, path)
+		ensureDestDir(dstpath)
+		df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
+		if err != nil {
+			return fmt.Errorf("creating destination file: %v", err)
+		}
+		defer func() {
+			if df != nil {
+				df.Close()
+			}
+		}()
+		err = db.Read(ctx, func(tx *bstore.Tx) error {
+			// Using regular WriteTo seems fine, and fast. It just copies pages.
+			//
+			// bolt.Compact is slower, it writes all key/value pairs, building up new data
+			// structures. My compacted test database was ~60% of original size. Lz4 on the
+			// uncompacted database got it to 14%. Lz4 on the compacted database got it to 13%.
+			// Backups are likely archived somewhere with compression, so we don't compact.
+			//
+			// Tests with WriteTo and os.O_DIRECT were slower than without O_DIRECT, but
+			// probably because everything fit in the page cache. It may be better to use
+			// O_DIRECT when copying many large or inactive databases.
+			_, err := tx.WriteTo(df)
+			return err
+		})
+		if err != nil {
+			return fmt.Errorf("copying database: %v", err)
+		}
+		err = df.Close()
+		df = nil
+		if err != nil {
+			return fmt.Errorf("closing destination database after copy: %v", err)
+		}
+		xvlog("backed up database file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmDB)))
+		return nil
+	}
+
+	// Try to create a hardlink. Fall back to copying the file (e.g. when on different file system).
+	warnedHardlink := false // We warn once about failing to hardlink.
+	linkOrCopy := func(srcpath, dstpath string) (bool, error) {
+		ensureDestDir(dstpath)
+
+		if err := os.Link(srcpath, dstpath); err == nil {
+			return true, nil
+		} else if os.IsNotExist(err) {
+			// No point in trying with regular copy, we would warn twice.
+			return false, err
+		} else if !warnedHardlink {
+			xwarnx("creating hardlink to message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
+			warnedHardlink = true
+		}
+
+		// Fall back to copying.
+		sf, err := os.Open(srcpath)
+		if err != nil {
+			return false, fmt.Errorf("open source path %s: %v", srcpath, err)
+		}
+		defer sf.Close()
+
+		df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
+		if err != nil {
+			return false, fmt.Errorf("open destination path %s: %v", dstpath, err)
+		}
+		defer func() {
+			if df != nil {
+				df.Close()
+			}
+		}()
+		if _, err := io.Copy(df, sf); err != nil {
+			return false, fmt.Errorf("coping: %v", err)
+		}
+		err = df.Close()
+		df = nil
+		if err != nil {
+			return false, fmt.Errorf("close: %v", err)
+		}
+		return false, nil
+	}
+
+	// Start making the backup.
+	tmStart := time.Now()
+
+	ctl.log.Print("making backup", mlog.Field("destdir", dstDataDir))
+
+	err := os.MkdirAll(dstDataDir, 0770)
+	if err != nil {
+		xerrx("creating destination data directory", err)
+	}
+
+	if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil {
+		xerrx("writing moxversion", err)
+	}
+	backupDB(dmarcdb.DB, "dmarcrpt.db")
+	backupDB(mtastsdb.DB, "mtasts.db")
+	backupDB(tlsrptdb.DB, "tlsrpt.db")
+	backupFile("receivedid.key")
+
+	// Acme directory is optional.
+	srcAcmeDir := filepath.Join(srcDataDir, "acme")
+	if _, err := os.Stat(srcAcmeDir); err == nil {
+		backupDir("acme")
+	} else if err != nil && !os.IsNotExist(err) {
+		xerrx("copying acme/", err)
+	}
+
+	// Copy the queue database and all message files.
+	backupQueue := func(path string) {
+		tmQueue := time.Now()
+
+		if err := backupDB(queue.DB, path); err != nil {
+			xerrx("queue not backed up", err, mlog.Field("path", path), mlog.Field("duration", time.Since(tmQueue)))
+			return
+		}
+
+		dstdbpath := filepath.Join(dstDataDir, path)
+		db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, queue.DBTypes...)
+		if err != nil {
+			xerrx("open copied queue database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmQueue)))
+			return
+		}
+
+		defer func() {
+			if db != nil {
+				err := db.Close()
+				ctl.log.Check(err, "closing new queue db")
+			}
+		}()
+
+		// Link/copy known message files. Warn if files are missing or unexpected
+		// (though a message file could have been removed just now due to delivery, or a
+		// new message may have been queued).
+		tmMsgs := time.Now()
+		seen := map[string]struct{}{}
+		var nlinked, ncopied int
+		err = bstore.QueryDB[queue.Msg](ctx, db).ForEach(func(m queue.Msg) error {
+			mp := store.MessagePath(m.ID)
+			seen[mp] = struct{}{}
+			srcpath := filepath.Join(srcDataDir, "queue", mp)
+			dstpath := filepath.Join(dstDataDir, "queue", mp)
+			if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
+				xerrx("linking/copying queue message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
+			} else if linked {
+				nlinked++
+			} else {
+				ncopied++
+			}
+			return nil
+		})
+		if err != nil {
+			xerrx("processing queue messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs)))
+		} else {
+			xvlog("queue message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs)))
+		}
+
+		// Read through all files in queue directory and warn about anything we haven't handled yet.
+		tmWalk := time.Now()
+		srcqdir := filepath.Join(srcDataDir, "queue")
+		err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error {
+			if err != nil {
+				xerrx("walking files in queue", err, mlog.Field("srcpath", srcqpath))
+				return nil
+			}
+			if d.IsDir() {
+				return nil
+			}
+			p := srcqpath[len(srcqdir)+1:]
+			if _, ok := seen[p]; ok {
+				return nil
+			}
+			if p == "index.db" {
+				return nil
+			}
+			qp := filepath.Join("queue", p)
+			xwarnx("backing up unrecognized file in queue directory", nil, mlog.Field("path", qp))
+			backupFile(qp)
+			return nil
+		})
+		if err != nil {
+			xerrx("walking queue directory (not backed up properly)", err, mlog.Field("dir", "queue"), mlog.Field("duration", time.Since(tmWalk)))
+		} else {
+			xvlog("walked queue directory", mlog.Field("duration", time.Since(tmWalk)))
+		}
+
+		xvlog("queue backed finished", mlog.Field("duration", time.Since(tmQueue)))
+	}
+	backupQueue("queue/index.db")
+
+	backupAccount := func(acc *store.Account) {
+		defer acc.Close()
+
+		tmAccount := time.Now()
+
+		// Copy database file.
+		dbpath := filepath.Join("accounts", acc.Name, "index.db")
+		err := backupDB(acc.DB, dbpath)
+		if err != nil {
+			xerrx("copying account database", err, mlog.Field("path", dbpath), mlog.Field("duration", time.Since(tmAccount)))
+		}
+
+		// todo: should document/check not taking a rlock on account.
+
+		// Copy junkfilter files, if configured.
+		if jf, _, err := acc.OpenJunkFilter(ctx, ctl.log); err != nil {
+			if !errors.Is(err, store.ErrNoJunkFilter) {
+				xerrx("opening junk filter for account (not backed up)", err)
+			}
+		} else {
+			db := jf.DB()
+			jfpath := filepath.Join("accounts", acc.Name, "junkfilter.db")
+			backupDB(db, jfpath)
+			bloompath := filepath.Join("accounts", acc.Name, "junkfilter.bloom")
+			backupFile(bloompath)
+			db = nil
+			err := jf.Close()
+			ctl.log.Check(err, "closing junkfilter")
+		}
+
+		dstdbpath := filepath.Join(dstDataDir, dbpath)
+		db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, store.DBTypes...)
+		if err != nil {
+			xerrx("open copied account database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmAccount)))
+			return
+		}
+
+		defer func() {
+			if db != nil {
+				err := db.Close()
+				ctl.log.Check(err, "close account database")
+			}
+		}()
+
+		// Link/copy known message files. Warn if files are missing or unexpected (though a
+		// message file could have been added just now due to delivery, or a message have
+		// been removed).
+		tmMsgs := time.Now()
+		seen := map[string]struct{}{}
+		var nlinked, ncopied int
+		err = bstore.QueryDB[store.Message](ctx, db).ForEach(func(m store.Message) error {
+			mp := store.MessagePath(m.ID)
+			seen[mp] = struct{}{}
+			amp := filepath.Join("accounts", acc.Name, "msg", mp)
+			srcpath := filepath.Join(srcDataDir, amp)
+			dstpath := filepath.Join(dstDataDir, amp)
+			if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
+				xerrx("linking/copying account message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
+			} else if linked {
+				nlinked++
+			} else {
+				ncopied++
+			}
+			return nil
+		})
+		if err != nil {
+			xerrx("processing account messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs)))
+		} else {
+			xvlog("account message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs)))
+		}
+
+		// Read through all files in account directory and warn about anything we haven't handled yet.
+		tmWalk := time.Now()
+		srcadir := filepath.Join(srcDataDir, "accounts", acc.Name)
+		err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error {
+			if err != nil {
+				xerrx("walking files in account", err, mlog.Field("srcpath", srcapath))
+				return nil
+			}
+			if d.IsDir() {
+				return nil
+			}
+			p := srcapath[len(srcadir)+1:]
+			l := strings.Split(p, string(filepath.Separator))
+			if l[0] == "msg" {
+				mp := filepath.Join(l[1:]...)
+				if _, ok := seen[mp]; ok {
+					return nil
+				}
+			}
+			switch p {
+			case "index.db", "junkfilter.db", "junkfilter.bloom":
+				return nil
+			}
+			ap := filepath.Join("accounts", acc.Name, p)
+			if strings.HasPrefix(p, "msg/") {
+				xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, mlog.Field("path", ap))
+			} else {
+				xwarnx("backing up unrecognized file in account directory", nil, mlog.Field("path", ap))
+			}
+			backupFile(ap)
+			return nil
+		})
+		if err != nil {
+			xerrx("walking account directory (not backed up properly)", err, mlog.Field("srcdir", srcadir), mlog.Field("duration", time.Since(tmWalk)))
+		} else {
+			xvlog("walked account directory", mlog.Field("duration", time.Since(tmWalk)))
+		}
+
+		xvlog("account backup finished", mlog.Field("dir", filepath.Join("accounts", acc.Name)), mlog.Field("duration", time.Since(tmAccount)))
+	}
+
+	// For each configured account, open it, make a copy of the database and
+	// hardlink/copy the messages. We track the accounts we handled, and skip the
+	// account directories when handling "all other files" below.
+	accounts := map[string]struct{}{}
+	for _, accName := range mox.Conf.Accounts() {
+		acc, err := store.OpenAccount(accName)
+		if err != nil {
+			xerrx("opening account for copying (will try to copy as regular files later)", err, mlog.Field("account", accName))
+			continue
+		}
+		accounts[accName] = struct{}{}
+		backupAccount(acc)
+	}
+
+	// Copy all other files, that aren't part of the known files, databases, queue or accounts.
+	tmWalk := time.Now()
+	err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error {
+		if err != nil {
+			xerrx("walking path", err, mlog.Field("path", srcpath))
+			return nil
+		}
+
+		if srcpath == srcDataDir {
+			return nil
+		}
+		p := srcpath[len(srcDataDir)+1:]
+		if p == "queue" || p == "acme" || p == "tmp" {
+			if p == "tmp" {
+				xwarnx("skipping entire tmp directory", nil, mlog.Field("path", p))
+			}
+			return fs.SkipDir
+		}
+		l := strings.Split(p, string(filepath.Separator))
+		if len(l) >= 2 && l[0] == "accounts" {
+			name := l[1]
+			if _, ok := accounts[name]; ok {
+				return fs.SkipDir
+			}
+		}
+
+		// Only files are explicitly backed up.
+		if d.IsDir() {
+			return nil
+		}
+
+		switch p {
+		case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
+			// Already handled.
+			return nil
+		case "lastknownversion": // Optional file, not yet handled.
+		default:
+			xwarnx("backing up unrecognized file", nil, mlog.Field("path", p))
+		}
+		backupFile(p)
+		return nil
+	})
+	if err != nil {
+		xerrx("walking other files (not backed up properly)", err, mlog.Field("duration", time.Since(tmWalk)))
+	} else {
+		xvlog("walking other files finished", mlog.Field("duration", time.Since(tmWalk)))
+	}
+
+	xvlog("backup finished", mlog.Field("duration", time.Since(tmStart)))
+
+	writer.xclose()
+
+	if incomplete {
+		ctl.xwrite("errors were encountered during backup")
+	} else {
+		ctl.xwriteok()
+	}
+}

ctl.go

@@ -30,6 +30,7 @@ import (
 // ctl represents a connection to the ctl unix domain socket of a running mox instance.
 // ctl provides functions to read/write commands/responses/data streams.
 type ctl struct {
+	cmd  string // Set for server-side of commands.
 	conn net.Conn
 	r    *bufio.Reader // Set for first reader.
 	x    any           // If set, errors are handled by calling panic(x) instead of log.Fatal.
@@ -57,7 +58,7 @@ func (c *ctl) xerror(msg string) {
 	if c.x == nil {
 		log.Fatalln(msg)
 	}
-	c.log.Debugx("ctl error", fmt.Errorf("%s", msg))
+	c.log.Debugx("ctl error", fmt.Errorf("%s", msg), mlog.Field("cmd", c.cmd))
 	c.xwrite(msg)
 	panic(c.x)
 }
@@ -72,7 +73,7 @@ func (c *ctl) xcheck(err error, msg string) {
 	if c.x == nil {
 		log.Fatalf("%s: %s", msg, err)
 	}
-	c.log.Debugx(msg, err)
+	c.log.Debugx(msg, err, mlog.Field("cmd", c.cmd))
 	fmt.Fprintf(c.conn, "%s: %s\n", msg, err)
 	panic(c.x)
 }
@@ -124,7 +125,7 @@ func (c *ctl) xstreamfrom(src io.Reader) {
 // When done writing, caller must call xclose to signal the end of the stream.
 // Behaviour of "x" is copied from ctl.
 func (c *ctl) writer() *ctlwriter {
-	return &ctlwriter{conn: c.conn, x: c.x, log: c.log}
+	return &ctlwriter{cmd: c.cmd, conn: c.conn, x: c.x, log: c.log}
 }
 
 // Reader returns an io.Reader for a data stream from ctl.
@@ -133,7 +134,7 @@ func (c *ctl) reader() *ctlreader {
 	if c.r == nil {
 		c.r = bufio.NewReader(c.conn)
 	}
-	return &ctlreader{conn: c.conn, r: c.r, x: c.x, log: c.log}
+	return &ctlreader{cmd: c.cmd, conn: c.conn, r: c.r, x: c.x, log: c.log}
 }
 
 /*
@@ -154,6 +155,7 @@ Followed by a end of stream indicated by zero data bytes message:
 */
 
 type ctlwriter struct {
+	cmd  string   // Set for server-side of commands.
 	conn net.Conn // Ctl socket from which messages are read.
 	buf  []byte   // Scratch buffer, for reading response.
 	x    any      // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal.
@@ -181,7 +183,7 @@ func (s *ctlwriter) xerror(msg string) {
 	if s.x == nil {
 		log.Fatalln(msg)
 	} else {
-		s.log.Debugx("error", fmt.Errorf("%s", msg))
+		s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd))
 		panic(s.x)
 	}
 }
@@ -193,7 +195,7 @@ func (s *ctlwriter) xcheck(err error, msg string) {
 	if s.x == nil {
 		log.Fatalf("%s: %s", msg, err)
 	} else {
-		s.log.Debugx(msg, err)
+		s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd))
 		panic(s.x)
 	}
 }
@@ -204,6 +206,7 @@ func (s *ctlwriter) xclose() {
 }
 
 type ctlreader struct {
+	cmd      string        // Set for server-side of command.
 	conn     net.Conn      // For writing "ok" after reading.
 	r        *bufio.Reader // Buffered ctl socket.
 	err      error         // If set, returned for each read. can also be io.EOF.
@@ -246,7 +249,7 @@ func (s *ctlreader) xerror(msg string) {
 	if s.x == nil {
 		log.Fatalln(msg)
 	} else {
-		s.log.Debugx("error", fmt.Errorf("%s", msg))
+		s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd))
 		panic(s.x)
 	}
 }
@@ -258,7 +261,7 @@ func (s *ctlreader) xcheck(err error, msg string) {
 	if s.x == nil {
 		log.Fatalf("%s: %s", msg, err)
 	} else {
-		s.log.Debugx(msg, err)
+		s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd))
 		panic(s.x)
 	}
 }
@@ -267,33 +270,30 @@ func (s *ctlreader) xcheck(err error, msg string) {
 func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()) {
 	log.Debug("ctl connection")
 
-	var cmd string
-
 	var stop = struct{}{} // Sentinel value for panic and recover.
+	ctl := &ctl{conn: conn, x: stop, log: log}
 	defer func() {
 		x := recover()
 		if x == nil || x == stop {
 			return
 		}
-		log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", cmd))
+		log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", ctl.cmd))
 		debug.PrintStack()
 		metrics.PanicInc("ctl")
 	}()
 
 	defer conn.Close()
 
-	ctl := &ctl{conn: conn, x: stop, log: log}
 	ctl.xwrite("ctlv0")
-
 	for {
-		servectlcmd(ctx, log, ctl, &cmd, shutdown)
+		servectlcmd(ctx, log, ctl, shutdown)
 	}
 }
 
-func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shutdown func()) {
+func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, shutdown func()) {
 	cmd := ctl.xread()
+	ctl.cmd = cmd
 	log.Info("ctl command", mlog.Field("cmd", cmd))
-	*xcmd = cmd
 	switch cmd {
 	case "stop":
 		shutdown()
@@ -641,6 +641,9 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
 
 		ctl.xwriteok()
 
+	case "backup":
+		backupctl(ctx, ctl)
+
 	default:
 		log.Info("unrecognized command", mlog.Field("cmd", cmd))
 		ctl.xwrite("unrecognized command")

develop.txt

@@ -167,6 +167,7 @@ testing purposes.
 - Write release notes, use instructions from updating.txt.
 - Build and run tests with previous major Go release.
 - Run all (integration) tests, including with race detector.
+- Test upgrades.
 - Run fuzzing tests for a while.
 - Deploy to test environment. Test the update instructions.
 - Generate a config with quickstart, check if it results in a working setup.

dmarcdb/db.go

@@ -28,7 +28,8 @@ import (
 var xlog = mlog.New("dmarcdb")
 
 var (
-	dmarcDB *bstore.DB
+	DBTypes = []any{DomainFeedback{}} // Types stored in DB.
+	DB      *bstore.DB                // Exported for backups.
 	mutex   sync.Mutex
 )
 
@@ -70,16 +71,16 @@ type DomainFeedback struct {
 func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
 	mutex.Lock()
 	defer mutex.Unlock()
-	if dmarcDB == nil {
+	if DB == nil {
 		p := mox.DataDirPath("dmarcrpt.db")
 		os.MkdirAll(filepath.Dir(p), 0770)
-		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DomainFeedback{})
+		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
 		if err != nil {
 			return nil, err
 		}
-		dmarcDB = db
+		DB = db
 	}
-	return dmarcDB, nil
+	return DB, nil
 }
 
 // Init opens the database.

doc.go

@@ -29,6 +29,8 @@ low-maintenance self-hosted email.
 	mox export mbox dst-dir account-path [mailbox]
 	mox localserve
 	mox help [command ...]
+	mox backup dest-dir
+	mox verifydata data-dir
 	mox config test
 	mox config dnscheck domain
 	mox config dnsrecords domain
@@ -331,6 +333,69 @@ If a single command matches, its usage and full help text is printed.
 
 	usage: mox help [command ...]
 
+# mox backup
+
+Creates a backup of the data directory.
+
+Backup creates consistent snapshots of the databases and message files and
+copies other files in the data directory. Empty directories are not copied.
+These files can then be stored elsewhere for long-term storage, or used to fall
+back to should an upgrade fail. Simply copying files in the data directory
+while mox is running can result in unusable database files.
+
+Message files never change (they are read-only, though can be removed) and are
+hardlinked so they don't consume additional space. If hardlinking fails, for
+example when the backup destination directory is on a different file system, a
+regular copy is made. Using a destination directory like "data/tmp/backup"
+increases the odds hardlinking succeeds: the default systemd service file
+specifically mounts the data directory, causing attempts to hardlink outside it
+to fail with an error about cross-device linking.
+
+All files in the data directory that aren't recognized (i.e. other than known
+database files, message files, an acme directory, etc), are stored, but with a
+warning.
+
+A clean successful backup does not print any output by default. Use the
+-verbose flag for details, including timing.
+
+To restore a backup, first shut down mox, move away the old data directory and
+move an earlier backed up directory in its place, run "mox verifydata",
+possibly with the "-fix" option, and restart mox. After the restore, you may
+also want to run "mox bumpuidvalidity" for each account for which messages in a
+mailbox changed, to force IMAP clients to synchronize mailbox state.
+
+Before upgrading, to check if the upgrade will likely succeed, first make a
+backup, then use the new mox binary to run "mox verifydata" on the backup. This
+can change the backup files (e.g. upgrade database files, move away
+unrecognized message files), so you should make a new backup before actually
+upgrading.
+
+	usage: mox backup dest-dir
+	  -verbose
+	    	print progress
+
+# mox verifydata
+
+Verify the contents of a data directory, typically of a backup.
+
+Verifydata checks all database files to see if they are valid BoltDB/bstore
+databases. It checks that all messages in the database have a corresponding
+on-disk message file and there are no unrecognized files. If option -fix is
+specified, unrecognized message files are moved away. This may be needed after
+a restore, because messages enqueued or delivered in the future may get those
+message sequence numbers assigned and writing the message file would fail.
+
+Because verifydata opens the database files, schema upgrades may automatically
+be applied. This can happen if you use a new mox release. It is useful to run
+"mox verifydata" with a new binary before attempting an upgrade, but only on a
+copy of the database files, as made with "mox backup". Before upgrading, make a
+new backup again since "mox verifydata" may have upgraded the database files,
+possibly making them potentially no longer readable by the previous version.
+
+	usage: mox verifydata data-dir
+	  -fix
+	    	fix fixable problems, such as moving away message files not referenced by their database
+
 # mox config test
 
 Parses and validates the configuration files.

export.go

@@ -57,7 +57,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) {
 	}
 
 	dbpath := filepath.Join(accountDir, "index.db")
-	db, err := bstore.Open(context.Background(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{})
+	db, err := bstore.Open(context.Background(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.DBTypes...)
 	xcheckf(err, "open database %q", dbpath)
 	defer func() {
 		if err := db.Close(); err != nil {

gentestdata.go

@@ -0,0 +1,361 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/mjl-/bstore"
+	"github.com/mjl-/sconf"
+
+	"github.com/mjl-/mox/config"
+	"github.com/mjl-/mox/dmarcdb"
+	"github.com/mjl-/mox/dmarcrpt"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/mtasts"
+	"github.com/mjl-/mox/mtastsdb"
+	"github.com/mjl-/mox/queue"
+	"github.com/mjl-/mox/smtp"
+	"github.com/mjl-/mox/store"
+	"github.com/mjl-/mox/tlsrpt"
+	"github.com/mjl-/mox/tlsrptdb"
+)
+
+func cmdGentestdata(c *cmd) {
+	c.unlisted = true
+	c.params = "dest-dir"
+	c.help = `Generate a data directory populated, for testing upgrades.`
+	args := c.Parse()
+	if len(args) != 1 {
+		c.Usage()
+	}
+
+	destDataDir, err := filepath.Abs(args[0])
+	xcheckf(err, "making destination directory an absolute path")
+
+	if _, err := os.Stat(destDataDir); err == nil {
+		log.Fatalf("destination directory already exists, refusing to generate test data")
+	}
+	err = os.MkdirAll(destDataDir, 0770)
+	xcheckf(err, "creating destination data directory")
+	err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)
+	xcheckf(err, "creating tmp directory")
+
+	tempfile := func() *os.File {
+		f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")
+		xcheckf(err, "creating temp file")
+		return f
+	}
+
+	ctxbg := context.Background()
+	mox.Shutdown = ctxbg
+	mox.Context = ctxbg
+	mox.Conf.Log[""] = mlog.LevelInfo
+	mlog.SetConfig(mox.Conf.Log)
+
+	const domainsConf = `
+Domains:
+	mox.example: nil
+	☺.example: nil
+Accounts:
+	test0:
+		Domain: mox.example
+		Destinations:
+			test0@mox.example: nil
+	test1:
+		Domain: mox.example
+		Destinations:
+			test1@mox.example: nil
+	test2:
+		Domain: ☺.example
+		Destinations:
+			☹@☺.example: nil
+		JunkFilter:
+			Threshold: 0.95
+			Params:
+				Twograms: true
+				MaxPower: 0.1
+				TopWords: 10
+				IgnoreWords: 0.1
+`
+
+	mox.ConfigStaticPath = "/tmp/mox-bogus/mox.conf"
+	mox.ConfigDynamicPath = "/tmp/mox-bogus/domains.conf"
+	mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.
+	mox.Conf.Static = config.Static{
+		DataDir: destDataDir,
+	}
+	err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)
+	xcheckf(err, "parsing domains config")
+
+	const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
+<feedback>
+  <report_metadata>
+    <org_name>google.com</org_name>
+    <email>noreply-dmarc-support@google.com</email>
+    <extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
+    <report_id>10051505501689795560</report_id>
+    <date_range>
+      <begin>1596412800</begin>
+      <end>1596499199</end>
+    </date_range>
+  </report_metadata>
+  <policy_published>
+    <domain>mox.example</domain>
+    <adkim>r</adkim>
+    <aspf>r</aspf>
+    <p>reject</p>
+    <sp>reject</sp>
+    <pct>100</pct>
+  </policy_published>
+  <record>
+    <row>
+      <source_ip>127.0.0.1</source_ip>
+      <count>1</count>
+      <policy_evaluated>
+        <disposition>none</disposition>
+        <dkim>pass</dkim>
+        <spf>pass</spf>
+      </policy_evaluated>
+    </row>
+    <identifiers>
+      <header_from>example.org</header_from>
+    </identifiers>
+    <auth_results>
+      <dkim>
+        <domain>example.org</domain>
+        <result>pass</result>
+        <selector>example</selector>
+      </dkim>
+      <spf>
+        <domain>example.org</domain>
+        <result>pass</result>
+      </spf>
+    </auth_results>
+  </record>
+</feedback>
+`
+
+	const tlsReport = `{
+     "organization-name": "Company-X",
+     "date-range": {
+       "start-datetime": "2016-04-01T00:00:00Z",
+       "end-datetime": "2016-04-01T23:59:59Z"
+     },
+     "contact-info": "sts-reporting@company-x.example",
+     "report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
+     "policies": [{
+       "policy": {
+         "policy-type": "sts",
+         "policy-string": ["version: STSv1","mode: testing",
+               "mx: *.mail.company-y.example","max_age: 86400"],
+         "policy-domain": "mox.example",
+         "mx-host": ["*.mail.company-y.example"]
+       },
+       "summary": {
+         "total-successful-session-count": 5326,
+         "total-failure-session-count": 303
+       },
+       "failure-details": [{
+         "result-type": "certificate-expired",
+         "sending-mta-ip": "2001:db8:abcd:0012::1",
+         "receiving-mx-hostname": "mx1.mail.company-y.example",
+         "failed-session-count": 100
+       }, {
+         "result-type": "starttls-not-supported",
+         "sending-mta-ip": "2001:db8:abcd:0013::1",
+         "receiving-mx-hostname": "mx2.mail.company-y.example",
+         "receiving-ip": "203.0.113.56",
+         "failed-session-count": 200,
+         "additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
+       }, {
+         "result-type": "validation-failure",
+         "sending-mta-ip": "198.51.100.62",
+         "receiving-ip": "203.0.113.58",
+         "receiving-mx-hostname": "mx-backup.mail.company-y.example",
+         "failed-session-count": 3,
+         "failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
+       }]
+     }]
+   }`
+
+	err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
+	xcheckf(err, "writing moxversion")
+
+	// Populate dmarc.db.
+	err = dmarcdb.Init()
+	xcheckf(err, "dmarcdb init")
+	report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
+	xcheckf(err, "parsing dmarc report")
+	err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
+	xcheckf(err, "adding dmarc report")
+
+	// Populate mtasts.db.
+	err = mtastsdb.Init(false)
+	xcheckf(err, "mtastsdb init")
+	mtastsPolicy := mtasts.Policy{
+		Version: "STSv1",
+		Mode:    mtasts.ModeTesting,
+		MX: []mtasts.STSMX{
+			{Domain: dns.Domain{ASCII: "mx1.example.com"}},
+			{Domain: dns.Domain{ASCII: "mx2.example.com"}},
+			{Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},
+		},
+		MaxAgeSeconds: 1296000,
+	}
+	err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy)
+	xcheckf(err, "adding mtastsdb report")
+
+	// Populate tlsrpt.db.
+	err = tlsrptdb.Init()
+	xcheckf(err, "tlsrptdb init")
+	tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport))
+	xcheckf(err, "parsing tls report")
+	err = tlsrptdb.AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", tlsr)
+	xcheckf(err, "adding tls report")
+
+	// Populate queue, with a message.
+	err = queue.Init()
+	xcheckf(err, "queue init")
+	mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}
+	rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
+	prefix := []byte{}
+	mf := tempfile()
+	xcheckf(err, "temp file for queue message")
+	defer mf.Close()
+	const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
+	_, err = fmt.Fprint(mf, qmsg)
+	xcheckf(err, "writing message")
+	err = queue.Add(ctxbg, mlog.New("gentestdata"), "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), prefix, mf, nil, true)
+	xcheckf(err, "enqueue message")
+
+	// Create three accounts.
+	// First account without messages.
+	accTest0, err := store.OpenAccount("test0")
+	xcheckf(err, "open account test0")
+	err = accTest0.Close()
+	xcheckf(err, "close account")
+
+	// Second account with one message.
+	accTest1, err := store.OpenAccount("test1")
+	xcheckf(err, "open account test1")
+	err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
+		inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
+		xcheckf(err, "looking up inbox")
+		const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
+		m := store.Message{
+			MailboxID:          inbox.ID,
+			MailboxOrigID:      inbox.ID,
+			MailboxDestinedID:  inbox.ID,
+			RemoteIP:           "1.2.3.4",
+			RemoteIPMasked1:    "1.2.3.4",
+			RemoteIPMasked2:    "1.2.3.0",
+			RemoteIPMasked3:    "1.2.0.0",
+			EHLODomain:         "other.example",
+			MailFrom:           "other@remote.example",
+			MailFromLocalpart:  smtp.Localpart("other"),
+			MailFromDomain:     "remote.example",
+			RcptToLocalpart:    "test1",
+			RcptToDomain:       "mox.example",
+			MsgFromLocalpart:   "other",
+			MsgFromDomain:      "remote.example",
+			MsgFromOrgDomain:   "remote.example",
+			EHLOValidated:      true,
+			MailFromValidated:  true,
+			MsgFromValidated:   true,
+			EHLOValidation:     store.ValidationStrict,
+			MailFromValidation: store.ValidationPass,
+			MsgFromValidation:  store.ValidationStrict,
+			DKIMDomains:        []string{"other.example"},
+			Size:               int64(len(msg)),
+		}
+		mf := tempfile()
+		xcheckf(err, "creating temp file for delivery")
+		_, err = fmt.Fprint(mf, msg)
+		xcheckf(err, "writing deliver message to file")
+		err = accTest1.DeliverMessage(mlog.New("gentestdata"), tx, &m, mf, true, false, false, true)
+		xcheckf(err, "add message to account test1")
+		err = mf.Close()
+		xcheckf(err, "closing file")
+		return nil
+	})
+	xcheckf(err, "write transaction with new message")
+	err = accTest1.Close()
+	xcheckf(err, "close account")
+
+	// Third account with two messages and junkfilter.
+	accTest2, err := store.OpenAccount("test2")
+	xcheckf(err, "open account test2")
+	err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
+		inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
+		xcheckf(err, "looking up inbox")
+		const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
+		m0 := store.Message{
+			MailboxID:          inbox.ID,
+			MailboxOrigID:      inbox.ID,
+			MailboxDestinedID:  inbox.ID,
+			RemoteIP:           "::1",
+			RemoteIPMasked1:    "::",
+			RemoteIPMasked2:    "::",
+			RemoteIPMasked3:    "::",
+			EHLODomain:         "other.example",
+			MailFrom:           "other@remote.example",
+			MailFromLocalpart:  smtp.Localpart("other"),
+			MailFromDomain:     "remote.example",
+			RcptToLocalpart:    "☹",
+			RcptToDomain:       "☺.example",
+			MsgFromLocalpart:   "other",
+			MsgFromDomain:      "remote.example",
+			MsgFromOrgDomain:   "remote.example",
+			EHLOValidated:      true,
+			MailFromValidated:  true,
+			MsgFromValidated:   true,
+			EHLOValidation:     store.ValidationStrict,
+			MailFromValidation: store.ValidationPass,
+			MsgFromValidation:  store.ValidationStrict,
+			DKIMDomains:        []string{"other.example"},
+			Size:               int64(len(msg0)),
+		}
+		mf0 := tempfile()
+		xcheckf(err, "creating temp file for delivery")
+		_, err = fmt.Fprint(mf0, msg0)
+		xcheckf(err, "writing deliver message to file")
+		err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m0, mf0, true, false, false, false)
+		xcheckf(err, "add message to account test2")
+		err = mf0.Close()
+		xcheckf(err, "closing file")
+
+		sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()
+		xcheckf(err, "looking up inbox")
+		const prefix1 = "Extra: test\r\n"
+		const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
+		m1 := store.Message{
+			MailboxID:         sent.ID,
+			MailboxOrigID:     sent.ID,
+			MailboxDestinedID: sent.ID,
+			Flags:             store.Flags{Seen: true, Junk: true},
+			Size:              int64(len(prefix1) + len(msg1)),
+			MsgPrefix:         []byte(prefix),
+		}
+		mf1 := tempfile()
+		xcheckf(err, "creating temp file for delivery")
+		_, err = fmt.Fprint(mf1, msg1)
+		xcheckf(err, "writing deliver message to file")
+		err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m1, mf1, true, true, false, false)
+		xcheckf(err, "add message to account test2")
+		err = mf1.Close()
+		xcheckf(err, "closing file")
+
+		return nil
+	})
+	xcheckf(err, "write transaction with new message")
+	err = accTest2.Close()
+	xcheckf(err, "close account")
+}

integration_test.go

@@ -82,7 +82,7 @@ func TestDeliver(t *testing.T) {
 	latestMsgID := func(username string) int64 {
 		// We open the account index database created by mox for the test user. And we keep looking for the email we sent.
 		dbpath := fmt.Sprintf("testdata/integration/data/accounts/%s/index.db", username)
-		db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{}, store.Password{})
+		db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.DBTypes...)
 		if err != nil && errors.Is(err, bolt.ErrTimeout) {
 			log.Printf("db open timeout (normal delay for new sender with account and db file kept open)")
 			return 0

junk/filter.go

@@ -56,6 +56,8 @@ type Params struct {
 	RareWords   int     `sconf:"optional" sconf-doc:"Occurrences in word database until a word is considered rare and its influence in calculating probability reduced. E.g. 1 or 2."`
 }
 
+var DBTypes = []any{wordscore{}} // Stored in DB.
+
 type Filter struct {
 	Params
 
@@ -228,7 +230,7 @@ func newDB(ctx context.Context, log *mlog.Log, path string) (db *bstore.DB, rerr
 		}
 	}()
 
-	db, err := bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
+	db, err := bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
 	if err != nil {
 		return nil, fmt.Errorf("open new database: %w", err)
 	}
@@ -239,7 +241,7 @@ func openDB(ctx context.Context, path string) (*bstore.DB, error) {
 	if _, err := os.Stat(path); err != nil {
 		return nil, fmt.Errorf("stat db file: %w", err)
 	}
-	return bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
+	return bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
 }
 
 // Save stores modifications, e.g. from training, to the database and bloom
@@ -744,3 +746,8 @@ func (f *Filter) fileSize(p string) int {
 	}
 	return int(fi.Size())
 }
+
+// DB returns the database, for backups.
+func (f *Filter) DB() *bstore.DB {
+	return f.db
+}

main.go

@@ -88,6 +88,8 @@ var commands = []struct {
 	{"export mbox", cmdExportMbox},
 	{"localserve", cmdLocalserve},
 	{"help", cmdHelp},
+	{"backup", cmdBackup},
+	{"verifydata", cmdVerifydata},
 
 	{"config test", cmdConfigTest},
 	{"config dnscheck", cmdConfigDNSCheck},
@@ -146,6 +148,7 @@ var commands = []struct {
 	{"updates pubkey", cmdUpdatesPubkey},
 	{"updates serve", cmdUpdatesServe},
 	{"updates verify", cmdUpdatesVerify},
+	{"gentestdata", cmdGentestdata},
 }
 
 var cmds []cmd
@@ -961,6 +964,67 @@ new mail deliveries.
 	fmt.Println("mox stopped")
 }
 
+func cmdBackup(c *cmd) {
+	c.params = "dest-dir"
+	c.help = `Creates a backup of the data directory.
+
+Backup creates consistent snapshots of the databases and message files and
+copies other files in the data directory. Empty directories are not copied.
+These files can then be stored elsewhere for long-term storage, or used to fall
+back to should an upgrade fail. Simply copying files in the data directory
+while mox is running can result in unusable database files.
+
+Message files never change (they are read-only, though can be removed) and are
+hardlinked so they don't consume additional space. If hardlinking fails, for
+example when the backup destination directory is on a different file system, a
+regular copy is made. Using a destination directory like "data/tmp/backup"
+increases the odds hardlinking succeeds: the default systemd service file
+specifically mounts the data directory, causing attempts to hardlink outside it
+to fail with an error about cross-device linking.
+
+All files in the data directory that aren't recognized (i.e. other than known
+database files, message files, an acme directory, etc), are stored, but with a
+warning.
+
+A clean successful backup does not print any output by default. Use the
+-verbose flag for details, including timing.
+
+To restore a backup, first shut down mox, move away the old data directory and
+move an earlier backed up directory in its place, run "mox verifydata",
+possibly with the "-fix" option, and restart mox. After the restore, you may
+also want to run "mox bumpuidvalidity" for each account for which messages in a
+mailbox changed, to force IMAP clients to synchronize mailbox state.
+
+Before upgrading, to check if the upgrade will likely succeed, first make a
+backup, then use the new mox binary to run "mox verifydata" on the backup. This
+can change the backup files (e.g. upgrade database files, move away
+unrecognized message files), so you should make a new backup before actually
+upgrading.
+`
+
+	var verbose bool
+	c.flag.BoolVar(&verbose, "verbose", false, "print progress")
+	args := c.Parse()
+	if len(args) != 1 {
+		c.Usage()
+	}
+	mustLoadConfig()
+
+	dstDataDir, err := filepath.Abs(args[0])
+	xcheckf(err, "making path absolute")
+
+	ctl := xctl()
+	ctl.xwrite("backup")
+	ctl.xwrite(dstDataDir)
+	if verbose {
+		ctl.xwrite("verbose")
+	} else {
+		ctl.xwrite("")
+	}
+	ctl.xstreamto(os.Stdout)
+	ctl.xreadok()
+}
+
 func cmdSetadminpassword(c *cmd) {
 	c.help = `Set a new admin password, for the web interface.
 

mlog/log.go

@@ -96,8 +96,8 @@ func SetConfig(c map[string]Level) {
 
 // Pair is a field/value pair, for use in logged lines.
 type Pair struct {
-	key   string
+	Key   string
-	value any
+	Value any
 }
 
 // Field is a shorthand for making a Pair.
@@ -339,7 +339,7 @@ func (l *Log) plog(level Level, err error, text string, fields ...Pair) {
 		}
 		for i := 0; i < len(fields); i++ {
 			kv := fields[i]
-			fmt.Fprintf(b, " %s=%s", kv.key, logfmtValue(stringValue(kv.key == "cid", false, kv.value)))
+			fmt.Fprintf(b, " %s=%s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
 		}
 		b.WriteString("\n")
 	} else {
@@ -354,7 +354,7 @@ func (l *Log) plog(level Level, err error, text string, fields ...Pair) {
 					fmt.Fprint(b, "; ")
 				}
 				kv := fields[i]
-				fmt.Fprintf(b, "%s: %s", kv.key, logfmtValue(stringValue(kv.key == "cid", false, kv.value)))
+				fmt.Fprintf(b, "%s: %s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
 			}
 			fmt.Fprint(b, ")")
 		}
@@ -373,10 +373,10 @@ func (l *Log) match(level Level) (bool, Level) {
 	seen := false
 	var high Level
 	for _, kv := range l.fields {
-		if kv.key != "pkg" {
+		if kv.Key != "pkg" {
 			continue
 		}
-		pkg, ok := kv.value.(string)
+		pkg, ok := kv.Value.(string)
 		if !ok {
 			continue
 		}

mtastsdb/db.go

@@ -60,22 +60,23 @@ var (
 	ErrBackoff = errors.New("mtastsdb: policy fetch failed recently")
 )
 
-var mtastsDB *bstore.DB
+var DBTypes = []any{PolicyRecord{}} // Types stored in DB.
+var DB *bstore.DB                   // Exported for backups.
 var mutex sync.Mutex
 
 func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
 	mutex.Lock()
 	defer mutex.Unlock()
-	if mtastsDB == nil {
+	if DB == nil {
 		p := mox.DataDirPath("mtasts.db")
 		os.MkdirAll(filepath.Dir(p), 0770)
-		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, PolicyRecord{})
+		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
 		if err != nil {
 			return nil, err
 		}
-		mtastsDB = db
+		DB = db
 	}
-	return mtastsDB, nil
+	return DB, nil
 }
 
 // Init opens the database and starts a goroutine that refreshes policies in
@@ -98,10 +99,10 @@ func Init(refresher bool) error {
 func Close() {
 	mutex.Lock()
 	defer mutex.Unlock()
-	if mtastsDB != nil {
+	if DB != nil {
-		err := mtastsDB.Close()
+		err := DB.Close()
 		xlog.Check(err, "closing database")
-		mtastsDB = nil
+		DB = nil
 	}
 }
 

queue/queue.go

@@ -70,7 +70,8 @@ var dial = func(ctx context.Context, timeout time.Duration, addr string, laddr n
 
 var jitter = mox.NewRand()
 
-var queueDB *bstore.DB
+var DBTypes = []any{Msg{}} // Types stored in DB.
+var DB *bstore.DB          // Exported for making backups.
 
 // Set for mox localserve, to prevent queueing.
 var Localserve bool
@@ -122,7 +123,7 @@ func Init() error {
 	}
 
 	var err error
-	queueDB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, Msg{})
+	DB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
 	if err != nil {
 		if isNew {
 			os.Remove(qpath)
@@ -134,15 +135,15 @@ func Init() error {
 
 // Shutdown closes the queue database. The delivery process isn't stopped. For tests only.
 func Shutdown() {
-	err := queueDB.Close()
+	err := DB.Close()
 	xlog.Check(err, "closing queue db")
-	queueDB = nil
+	DB = nil
 }
 
 // List returns all messages in the delivery queue.
 // Ordered by earliest delivery attempt first.
 func List(ctx context.Context) ([]Msg, error) {
-	qmsgs, err := bstore.QueryDB[Msg](ctx, queueDB).List()
+	qmsgs, err := bstore.QueryDB[Msg](ctx, DB).List()
 	if err != nil {
 		return nil, err
 	}
@@ -166,7 +167,7 @@ func List(ctx context.Context) ([]Msg, error) {
 
 // Count returns the number of messages in the delivery queue.
 func Count(ctx context.Context) (int, error) {
-	return bstore.QueryDB[Msg](ctx, queueDB).Count()
+	return bstore.QueryDB[Msg](ctx, DB).Count()
 }
 
 // Add a new message to the queue. The queue is kicked immediately to start a
@@ -187,7 +188,7 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
 		return fmt.Errorf("no queuing with localserve")
 	}
 
-	tx, err := queueDB.Begin(ctx, true)
+	tx, err := DB.Begin(ctx, true)
 	if err != nil {
 		return fmt.Errorf("begin transaction: %w", err)
 	}
@@ -288,7 +289,7 @@ func queuekick() {
 // are zero, all messages are kicked.
 // Returns number of messages queued for immediate delivery.
 func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
-	q := bstore.QueryDB[Msg](ctx, queueDB)
+	q := bstore.QueryDB[Msg](ctx, DB)
 	if ID > 0 {
 		q.FilterID(ID)
 	}
@@ -312,7 +313,7 @@ func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int
 // If all parameters are zero, all messages are removed.
 // Returns number of messages removed.
 func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
-	q := bstore.QueryDB[Msg](ctx, queueDB)
+	q := bstore.QueryDB[Msg](ctx, DB)
 	if ID > 0 {
 		q.FilterID(ID)
 	}
@@ -347,7 +348,7 @@ type ReadReaderAtCloser interface {
 // OpenMessage opens a message present in the queue.
 func OpenMessage(ctx context.Context, id int64) (ReadReaderAtCloser, error) {
 	qm := Msg{ID: id}
-	err := queueDB.Get(ctx, &qm)
+	err := DB.Get(ctx, &qm)
 	if err != nil {
 		return nil, err
 	}
@@ -397,7 +398,7 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
 }
 
 func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration {
-	q := bstore.QueryDB[Msg](ctx, queueDB)
+	q := bstore.QueryDB[Msg](ctx, DB)
 	if len(busyDomains) > 0 {
 		var doms []any
 		for d := range busyDomains {
@@ -418,7 +419,7 @@ func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duratio
 }
 
 func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
-	q := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
+	q := bstore.QueryDB[Msg](mox.Shutdown, DB)
 	q.FilterLessEqual("NextAttempt", time.Now())
 	q.SortAsc("NextAttempt")
 	q.Limit(maxConcurrentDeliveries)
@@ -445,7 +446,7 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
 
 // Remove message from queue in database and file system.
 func queueDelete(ctx context.Context, msgID int64) error {
-	if err := queueDB.Delete(ctx, &Msg{ID: msgID}); err != nil {
+	if err := DB.Delete(ctx, &Msg{ID: msgID}); err != nil {
 		return err
 	}
 	// If removing from database fails, we'll also leave the file in the file system.
@@ -491,7 +492,7 @@ func deliver(resolver dns.Resolver, m Msg) {
 	now := time.Now()
 	m.LastAttempt = &now
 	m.NextAttempt = now.Add(backoff)
-	qup := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
+	qup := bstore.QueryDB[Msg](mox.Shutdown, DB)
 	qup.FilterID(m.ID)
 	update := Msg{Attempts: m.Attempts, NextAttempt: m.NextAttempt, LastAttempt: m.LastAttempt}
 	if _, err := qup.UpdateNonzero(update); err != nil {
@@ -510,7 +511,7 @@ func deliver(resolver dns.Resolver, m Msg) {
 			return
 		}
 
-		qup := bstore.QueryDB[Msg](context.Background(), queueDB)
+		qup := bstore.QueryDB[Msg](context.Background(), DB)
 		qup.FilterID(m.ID)
 		if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil {
 			qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg))

queue/queue_test.go

@@ -138,7 +138,7 @@ func TestQueue(t *testing.T) {
 	case <-dialed:
 		i := 0
 		for {
-			m, err := bstore.QueryDB[Msg](ctxbg, queueDB).Get()
+			m, err := bstore.QueryDB[Msg](ctxbg, DB).Get()
 			tcheck(t, err, "get")
 			if m.Attempts == 1 {
 				break
@@ -288,7 +288,7 @@ func TestQueue(t *testing.T) {
 	for i := 1; i < 8; i++ {
 		go func() { <-deliveryResult }() // Deliver sends here.
 		deliver(resolver, msg)
-		err = queueDB.Get(ctxbg, &msg)
+		err = DB.Get(ctxbg, &msg)
 		tcheck(t, err, "get msg")
 		if msg.Attempts != i {
 			t.Fatalf("got attempt %d, expected %d", msg.Attempts, i)
@@ -311,7 +311,7 @@ func TestQueue(t *testing.T) {
 	// Trigger final failure.
 	go func() { <-deliveryResult }() // Deliver sends here.
 	deliver(resolver, msg)
-	err = queueDB.Get(ctxbg, &msg)
+	err = DB.Get(ctxbg, &msg)
 	if err != bstore.ErrAbsent {
 		t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err)
 	}

serve.go

@@ -292,7 +292,7 @@ requested, other TLS certificates are requested on demand.
 				}
 			}()
 			m := &store.Message{Received: time.Now(), Flags: store.Flags{Flagged: true}}
-			n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this is install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n"))
+			n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this is install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nRemember to make a backup with \"mox backup\" before upgrading.\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n"))
 			if err != nil {
 				log.Infox("writing temporary message file for changelog delivery", err)
 				return next

smtpserver/reputation_test.go

@@ -102,7 +102,7 @@ func TestReputation(t *testing.T) {
 		p := "../testdata/smtpserver-reputation.db"
 		defer os.Remove(p)
 
-		db, err := bstore.Open(ctxbg, p, &bstore.Options{Timeout: 5 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{})
+		db, err := bstore.Open(ctxbg, p, &bstore.Options{Timeout: 5 * time.Second}, store.DBTypes...)
 		tcheck(t, err, "open db")
 		defer db.Close()
 

store/account.go

@@ -377,6 +377,9 @@ type Outgoing struct {
 	Submitted time.Time `bstore:"nonzero,default now"`
 }
 
+// Types stored in DB.
+var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}}
+
 // Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
 type Account struct {
 	Name   string     // Name, according to configuration.
@@ -455,7 +458,7 @@ func openAccount(name string) (a *Account, rerr error) {
 		os.MkdirAll(dir, 0770)
 	}
 
-	db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{})
+	db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
 	if err != nil {
 		return nil, err
 	}

test-upgrade.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+# todo: should we also test with mox.conf and domains.conf files? should "mox backup" and "mox gentestdata" add them, and "mox verifydata" use them?
+
+set -e
+# set -x
+
+(rm -r testdata/upgrade 2>/dev/null || exit 0)
+mkdir testdata/upgrade
+cd testdata/upgrade
+
+# Check that we can upgrade what we currently generate.
+../../mox gentestdata data
+../../mox verifydata data
+rm -r data
+
+# For each historic release (i.e. all tagged versions) except the first few that
+# didn't have the gentestdata command, we generate a data directory for testing
+# and simulate upgrade to currently checked out version.
+# The awk command reverses the tags, so we try the previous release first since
+# it is the most likely to fail.
+tagsrev=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }')
+if test "$tagsrev" = ""; then exit 0; fi
+for tag in $tagsrev; do
+	echo "Testing upgrade from $tag to current."
+	mkdir $tag
+	(CGO_ENABLED=0 GOBIN=$PWD/$tag go install github.com/mjl-/mox@$tag)
+	# Generate with historic release.
+	./$tag/mox gentestdata $tag/data
+	# Verify with current code.
+	../../mox verifydata $tag/data
+	rm -r $tag/data
+done
+
+# Also go step-wise through each released version. Having upgraded step by step
+# can have added more schema upgrades to the database files.
+tags=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | cat)
+first=yes
+for tag in $tags; do
+	if test "$first" = yes; then
+		echo "Starting with test data for $tag."
+		./$tag/mox gentestdata stepdata
+		first=
+	else
+		echo "Upgrade data to $tag."
+		./$tag/mox verifydata stepdata
+	fi
+done
+echo "Testing final upgrade to current."
+../../mox verifydata stepdata
+rm -r stepdata
+rm */mox
+cd ../..
+rmdir testdata/upgrade/* testdata/upgrade

tlsrptdb/db.go

@@ -23,8 +23,9 @@ import (
 var (
 	xlog = mlog.New("tlsrptdb")
 
-	tlsrptDB *bstore.DB
+	DBTypes = []any{TLSReportRecord{}}
-	mutex    sync.Mutex
+	DB      *bstore.DB
+	mutex   sync.Mutex
 
 	metricSession = promauto.NewCounterVec(
 		prometheus.CounterOpts{
@@ -64,16 +65,16 @@ type TLSReportRecord struct {
 func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
 	mutex.Lock()
 	defer mutex.Unlock()
-	if tlsrptDB == nil {
+	if DB == nil {
 		p := mox.DataDirPath("tlsrpt.db")
 		os.MkdirAll(filepath.Dir(p), 0770)
-		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, TLSReportRecord{})
+		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
 		if err != nil {
 			return nil, err
 		}
-		tlsrptDB = db
+		DB = db
 	}
-	return tlsrptDB, nil
+	return DB, nil
 }
 
 // Init opens and possibly initializes the database.
@@ -86,10 +87,10 @@ func Init() error {
 func Close() {
 	mutex.Lock()
 	defer mutex.Unlock()
-	if tlsrptDB != nil {
+	if DB != nil {
-		err := tlsrptDB.Close()
+		err := DB.Close()
 		xlog.Check(err, "closing database")
-		tlsrptDB = nil
+		DB = nil
 	}
 }
 

verifydata.go

@@ -0,0 +1,347 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	bolt "go.etcd.io/bbolt"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/dmarcdb"
+	"github.com/mjl-/mox/junk"
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/mtastsdb"
+	"github.com/mjl-/mox/queue"
+	"github.com/mjl-/mox/store"
+	"github.com/mjl-/mox/tlsrptdb"
+)
+
+func cmdVerifydata(c *cmd) {
+	c.params = "data-dir"
+	c.help = `Verify the contents of a data directory, typically of a backup.
+
+Verifydata checks all database files to see if they are valid BoltDB/bstore
+databases. It checks that all messages in the database have a corresponding
+on-disk message file and there are no unrecognized files. If option -fix is
+specified, unrecognized message files are moved away. This may be needed after
+a restore, because messages enqueued or delivered in the future may get those
+message sequence numbers assigned and writing the message file would fail.
+
+Because verifydata opens the database files, schema upgrades may automatically
+be applied. This can happen if you use a new mox release. It is useful to run
+"mox verifydata" with a new binary before attempting an upgrade, but only on a
+copy of the database files, as made with "mox backup". Before upgrading, make a
+new backup again since "mox verifydata" may have upgraded the database files,
+possibly making them potentially no longer readable by the previous version.
+`
+	var fix bool
+	c.flag.BoolVar(&fix, "fix", false, "fix fixable problems, such as moving away message files not referenced by their database")
+	args := c.Parse()
+	if len(args) != 1 {
+		c.Usage()
+	}
+
+	dataDir := filepath.Clean(args[0])
+
+	ctxbg := context.Background()
+
+	// Check whether file exists, or rather, that it doesn't not exist. Other errors
+	// will return true as well, so the triggered check can give the details.
+	exists := func(path string) bool {
+		_, err := os.Stat(path)
+		return err == nil || !os.IsNotExist(err)
+	}
+
+	// Check for error. If so, write a log line, including the path, and set fail so we
+	// can warn at the end.
+	var fail bool
+	checkf := func(err error, path, format string, args ...any) {
+		if err == nil {
+			return
+		}
+		fail = true
+		log.Printf("error: %s: %s: %v", path, fmt.Sprintf(format, args...), err)
+	}
+
+	// When we fix problems, we may have to move files/dirs. We need to ensure the
+	// directory of the destination path exists before we move. We keep track of
+	// created dirs so we don't try to create the same directory all the time.
+	createdDirs := map[string]struct{}{}
+	ensureDir := func(path string) {
+		dir := filepath.Dir(path)
+		if _, ok := createdDirs[dir]; ok {
+			return
+		}
+		err := os.MkdirAll(dir, 0770)
+		checkf(err, dir, "creating directory")
+		createdDirs[dir] = struct{}{}
+	}
+
+	// Check a database file by opening it with BoltDB and bstore and lightly checking
+	// its contents.
+	checkDB := func(path string, types []any) {
+		_, err := os.Stat(path)
+		checkf(err, path, "checking if file exists")
+		if err != nil {
+			return
+		}
+		bdb, err := bolt.Open(path, 0600, nil)
+		checkf(err, path, "open database with bolt")
+		if err != nil {
+			return
+		}
+		// Check BoltDB consistency.
+		err = bdb.View(func(tx *bolt.Tx) error {
+			for err := range tx.Check() {
+				checkf(err, path, "bolt database problem")
+			}
+			return nil
+		})
+		checkf(err, path, "reading bolt database")
+		bdb.Close()
+
+		db, err := bstore.Open(ctxbg, path, nil, types...)
+		checkf(err, path, "open database with bstore")
+		if err != nil {
+			return
+		}
+		defer db.Close()
+
+		err = db.Read(ctxbg, func(tx *bstore.Tx) error {
+			// Check bstore consistency, if it can export all records for all types. This is a
+			// quick way to get bstore to parse all records.
+			types, err := tx.Types()
+			checkf(err, path, "getting bstore types from database")
+			if err != nil {
+				return nil
+			}
+			for _, t := range types {
+				var fields []string
+				err := tx.Records(t, &fields, func(m map[string]any) error {
+					return nil
+				})
+				checkf(err, path, "parsing record for type %q", t)
+			}
+			return nil
+		})
+		checkf(err, path, "checking database file")
+	}
+
+	checkFile := func(path string) {
+		_, err := os.Stat(path)
+		checkf(err, path, "checking if file exists")
+	}
+
+	checkQueue := func() {
+		dbpath := filepath.Join(dataDir, "queue/index.db")
+		checkDB(dbpath, queue.DBTypes)
+
+		// Check that all messages present in the database also exist on disk.
+		seen := map[string]struct{}{}
+		db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, queue.DBTypes...)
+		checkf(err, dbpath, "opening queue database to check messages")
+		if err == nil {
+			err := bstore.QueryDB[queue.Msg](ctxbg, db).ForEach(func(m queue.Msg) error {
+				mp := store.MessagePath(m.ID)
+				seen[mp] = struct{}{}
+				p := filepath.Join(dataDir, "queue", mp)
+				checkFile(p)
+				return nil
+			})
+			checkf(err, dbpath, "reading messages in queue database to check files")
+		}
+
+		// Check that there are no files that could be treated as a message.
+		qdir := filepath.Join(dataDir, "queue")
+		err = filepath.WalkDir(qdir, func(qpath string, d fs.DirEntry, err error) error {
+			checkf(err, qpath, "walk")
+			if err != nil {
+				return nil
+			}
+			if d.IsDir() {
+				return nil
+			}
+			p := qpath[len(qdir)+1:]
+			if p == "index.db" {
+				return nil
+			}
+			if _, ok := seen[p]; ok {
+				return nil
+			}
+			l := strings.Split(p, string(filepath.Separator))
+			if len(l) == 1 {
+				log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)
+				return nil
+			}
+			// If it doesn't look like a message number, there is no risk of it being the name
+			// of a message enqueued in the future.
+			if len(l) >= 3 {
+				if _, err := strconv.ParseInt(l[1], 10, 64); err != nil {
+					log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)
+					return nil
+				}
+			}
+			if !fix {
+				checkf(errors.New("may interfere with messages enqueued in the future"), qpath, "unrecognized file in queue directory (use the -fix flag to move it away)")
+				return nil
+			}
+			npath := filepath.Join(dataDir, "moved", "queue", p)
+			ensureDir(npath)
+			err = os.Rename(qpath, npath)
+			checkf(err, qpath, "moving queue message file away")
+			if err == nil {
+				log.Printf("warning: moved %s to %s", qpath, npath)
+			}
+			return nil
+		})
+		checkf(err, qdir, "walking queue directory")
+	}
+
+	// Check an account, with its database file and messages.
+	checkAccount := func(name string) {
+		accdir := filepath.Join(dataDir, "accounts", name)
+		checkDB(filepath.Join(accdir, "index.db"), store.DBTypes)
+
+		jfdbpath := filepath.Join(accdir, "junkfilter.db")
+		jfbloompath := filepath.Join(accdir, "junkfilter.bloom")
+		if exists(jfdbpath) || exists(jfbloompath) {
+			checkDB(jfdbpath, junk.DBTypes)
+		}
+		// todo: add some kind of check for the bloom filter?
+
+		// Check that all messages in the database have a message file on disk.
+		seen := map[string]struct{}{}
+		dbpath := filepath.Join(accdir, "index.db")
+		db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, store.DBTypes...)
+		checkf(err, dbpath, "opening account database to check messages")
+		if err == nil {
+			err := bstore.QueryDB[store.Message](ctxbg, db).ForEach(func(m store.Message) error {
+				mp := store.MessagePath(m.ID)
+				seen[mp] = struct{}{}
+				p := filepath.Join(accdir, "msg", mp)
+				checkFile(p)
+				return nil
+			})
+			checkf(err, dbpath, "reading messages in account database to check files")
+		}
+
+		// Walk through all files in the msg directory. Warn about files that weren't in
+		// the database as message file. Possibly move away files that could cause trouble.
+		msgdir := filepath.Join(accdir, "msg")
+		if !exists(msgdir) {
+			// New accounts with messages don't have a msg directory.
+			return
+		}
+		err = filepath.WalkDir(msgdir, func(msgpath string, d fs.DirEntry, err error) error {
+			checkf(err, msgpath, "walk")
+			if err != nil {
+				return nil
+			}
+			if d.IsDir() {
+				return nil
+			}
+			p := msgpath[len(msgdir)+1:]
+			if _, ok := seen[p]; ok {
+				return nil
+			}
+			l := strings.Split(p, string(filepath.Separator))
+			if len(l) == 1 {
+				log.Printf("warning: %s: unrecognized file in message directory, ignoring", msgpath)
+				return nil
+			}
+			if !fix {
+				checkf(errors.New("may interfere with future account messages"), msgpath, "unrecognized file in account message directory (use the -fix flag to move it away)")
+				return nil
+			}
+			npath := filepath.Join(dataDir, "moved", "accounts", name, "msg", p)
+			ensureDir(npath)
+			err = os.Rename(msgpath, npath)
+			checkf(err, msgpath, "moving account message file away")
+			if err == nil {
+				log.Printf("warning: moved %s to %s", msgpath, npath)
+			}
+			return nil
+		})
+		checkf(err, msgdir, "walking account message directory")
+	}
+
+	// Check everything in the "accounts" directory.
+	checkAccounts := func() {
+		accountsDir := filepath.Join(dataDir, "accounts")
+		entries, err := os.ReadDir(accountsDir)
+		checkf(err, accountsDir, "reading accounts directory")
+		for _, e := range entries {
+			// We treat all directories as accounts. When we were backing up, we only verified
+			// accounts from the config and made regular file copies of all other files
+			// (perhaps an old account, but at least not with an open database file). It may
+			// turn out that that account was/is not valid, generating warnings. Better safe
+			// than sorry. It should hopefully get the admin to move away such an old account.
+			if e.IsDir() {
+				checkAccount(e.Name())
+			} else {
+				log.Printf("warning: %s: unrecognized file in accounts directory, ignoring", filepath.Join("accounts", e.Name()))
+			}
+		}
+	}
+
+	// Check all files, skipping the known files, queue and accounts directories. Warn
+	// about unknown files. Skip a "tmp" directory. And a "moved" directory, we
+	// probably created it ourselves.
+	backupmoxversion := "(unknown)"
+	checkOther := func() {
+		err := filepath.WalkDir(dataDir, func(dpath string, d fs.DirEntry, err error) error {
+			checkf(err, dpath, "walk")
+			if err != nil {
+				return nil
+			}
+			if dpath == dataDir {
+				return nil
+			}
+			p := dpath
+			if dataDir != "." {
+				p = p[len(dataDir)+1:]
+			}
+			switch p {
+			case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion":
+				return nil
+			case "acme", "queue", "accounts", "tmp", "moved":
+				return fs.SkipDir
+			case "moxversion":
+				buf, err := os.ReadFile(dpath)
+				checkf(err, dpath, "reading moxversion")
+				if err == nil {
+					backupmoxversion = string(buf)
+				}
+				return nil
+			}
+			log.Printf("warning: %s: unrecognized other file, ignoring", dpath)
+			return nil
+		})
+		checkf(err, dataDir, "walking data directory")
+	}
+
+	checkDB(filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.DBTypes)
+	checkDB(filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)
+	checkDB(filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes)
+	checkQueue()
+	checkAccounts()
+	checkOther()
+
+	if fail {
+		log.Fatalf("errors were found")
+	} else {
+		log.Printf("%s: OK", dataDir)
+	}
+
+	if backupmoxversion != moxvar.Version {
+		log.Printf("NOTE: The backup was made with mox version %q, while verifydata was run with mox version %q. Database files have probably been modified by running mox verifydata. Make a fresh backup before upgrading.", backupmoxversion, moxvar.Version)
+	}
+}