diff --git a/Makefile b/Makefile index 11134e8..1a52f65 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ test-race: CGO_ENABLED=1 go test -race -shuffle=on -covermode atomic -coverprofile cover.out ./... go tool cover -html=cover.out -o cover.html +test-upgrade: + ./test-upgrade.sh + check: staticcheck ./... staticcheck -tags integration diff --git a/README.md b/README.md index 1aeaee3..748c8cd 100644 --- a/README.md +++ b/README.md @@ -282,3 +282,34 @@ Mox also has an "admin" web interface where the mox instance administrator can make changes, e.g. add/remove/modify domains/accounts/addresses. Mox does not have a webmail yet, so there are no screenshots of actual email. + +## How do I upgrade my mox installation? + +We try to make upgrades effortless and you can typically just put a new binary +in place and restart. If manual actions are required, the release notes mention +them. Check the release notes of all version between your current installation +and the release you're upgrading to. + +Before upgrading, make a backup of the data directory with `mox backup +`. This writes consistent snapshots of the database files, and +duplicates message files from the queue and accounts. Using the new mox +binary, run `mox verifydata ` (do NOT use the "live" data +directory!) for a dry run. If this fails, an upgrade will probably fail too. +Important: verifydata with the new mox binary can modify the database files +(due to automatic schema upgrades). So make a fresh backup again before the +actual upgrade. See the help output of the "backup" and "verifydata" commands +for more details. + +During backup, message files are hardlinked if possible. Using a destination +directory like `data/tmp/backup` increases the odds hardlinking succeeds: the +default systemd service file specifically mounts the data directory, causing +attempts to outside it to fail with an error about cross-device linking. + +If an upgrade fails and you have to restore (parts) of the data directory, you +should run `mox verifydata ` (with the original binary) on the +restored directory before starting mox again. If problematic files are found, +for example queue or account message files that are not in the database, run +`mox verifydata -fix ` to move away those files. After a restore, you may +also want to run `mox bumpuidvalidity ` for each account for which +messages in a mailbox changed, to force IMAP clients to synchronize mailbox +state. diff --git a/backup.go b/backup.go new file mode 100644 index 0000000..3e68880 --- /dev/null +++ b/backup.go @@ -0,0 +1,556 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/dmarcdb" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/moxvar" + "github.com/mjl-/mox/mtastsdb" + "github.com/mjl-/mox/queue" + "github.com/mjl-/mox/store" + "github.com/mjl-/mox/tlsrptdb" +) + +func backupctl(ctx context.Context, ctl *ctl) { + /* protocol: + > "backup" + > destdir + > "verbose" or "" + < stream + < "ok" or error + */ + + // Convention in this function: variables containing "src" or "dst" are file system + // paths that can be passed to os.Open and such. Variables with dirs/paths without + // "src" or "dst" are incomplete paths relative to the source or destination data + // directories. + + dstDataDir := ctl.xread() + verbose := ctl.xread() == "verbose" + + // Set when an error is encountered. At the end, we warn if set. + var incomplete bool + + // We'll be writing output, and logging both to mox and the ctl stream. + writer := ctl.writer() + + // Format easily readable output for the user. + formatLog := func(prefix, text string, err error, fields ...mlog.Pair) []byte { + var b bytes.Buffer + fmt.Fprint(&b, prefix) + fmt.Fprint(&b, text) + if err != nil { + fmt.Fprint(&b, ": "+err.Error()) + } + for _, f := range fields { + fmt.Fprintf(&b, "; %s=%v", f.Key, f.Value) + } + fmt.Fprint(&b, "\n") + return b.Bytes() + } + + // Log an error to both the mox service as the user running "mox backup". + xlogx := func(prefix, text string, err error, fields ...mlog.Pair) { + ctl.log.Errorx(text, err, fields...) + + _, werr := writer.Write(formatLog(prefix, text, err, fields...)) + ctl.xcheck(werr, "write to ctl") + } + + // Log an error but don't mark backup as failed. + xwarnx := func(text string, err error, fields ...mlog.Pair) { + xlogx("warning: ", text, err, fields...) + } + + // Log an error that causes the backup to be marked as failed. We typically + // continue processing though. + xerrx := func(text string, err error, fields ...mlog.Pair) { + incomplete = true + xlogx("error: ", text, err, fields...) + } + + // If verbose is enabled, log to the cli command. Always log as info level. + xvlog := func(text string, fields ...mlog.Pair) { + ctl.log.Info(text, fields...) + if verbose { + _, werr := writer.Write(formatLog("", text, nil, fields...)) + ctl.xcheck(werr, "write to ctl") + } + } + + if _, err := os.Stat(dstDataDir); err == nil { + xwarnx("destination data directory already exists", nil, mlog.Field("dir", dstDataDir)) + } + + srcDataDir := filepath.Clean(mox.DataDirPath(".")) + + // When creating a file in the destination, we first ensure its directory exists. + // We track which directories we created, to prevent needless syscalls. + createdDirs := map[string]struct{}{} + ensureDestDir := func(dstpath string) { + dstdir := filepath.Dir(dstpath) + if _, ok := createdDirs[dstdir]; !ok { + err := os.MkdirAll(dstdir, 0770) + if err != nil { + xerrx("creating directory", err) + } + createdDirs[dstdir] = struct{}{} + } + } + + // Backup a single file by copying (never hardlinking, the file may change). + backupFile := func(path string) { + tmFile := time.Now() + srcpath := filepath.Join(srcDataDir, path) + dstpath := filepath.Join(dstDataDir, path) + + sf, err := os.Open(srcpath) + if err != nil { + xerrx("open source file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + return + } + defer sf.Close() + + ensureDestDir(dstpath) + df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) + if err != nil { + xerrx("creating destination file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + return + } + defer func() { + if df != nil { + df.Close() + } + }() + if _, err := io.Copy(df, sf); err != nil { + xerrx("copying file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + return + } + err = df.Close() + df = nil + if err != nil { + xerrx("closing destination file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + return + } + xvlog("backed up file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmFile))) + } + + // Back up the files in a directory (by copying). + backupDir := func(dir string) { + tmDir := time.Now() + srcdir := filepath.Join(srcDataDir, dir) + dstdir := filepath.Join(dstDataDir, dir) + err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error { + if err != nil { + xerrx("walking file (not backed up)", err, mlog.Field("srcpath", srcpath)) + return nil + } + if d.IsDir() { + return nil + } + backupFile(srcpath[len(srcDataDir)+1:]) + return nil + }) + if err != nil { + xerrx("copying directory (not backed up properly)", err, mlog.Field("srcdir", srcdir), mlog.Field("dstdir", dstdir), mlog.Field("duration", time.Since(tmDir))) + return + } + xvlog("backed up directory", mlog.Field("dir", dir), mlog.Field("duration", time.Since(tmDir))) + } + + // Backup a database by copying it in a readonly transaction. + // Always logs on error, so caller doesn't have to, but also returns the error so + // callers can see result. + backupDB := func(db *bstore.DB, path string) (rerr error) { + defer func() { + if rerr != nil { + xerrx("backing up database", rerr, mlog.Field("path", path)) + } + }() + + tmDB := time.Now() + + dstpath := filepath.Join(dstDataDir, path) + ensureDestDir(dstpath) + df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) + if err != nil { + return fmt.Errorf("creating destination file: %v", err) + } + defer func() { + if df != nil { + df.Close() + } + }() + err = db.Read(ctx, func(tx *bstore.Tx) error { + // Using regular WriteTo seems fine, and fast. It just copies pages. + // + // bolt.Compact is slower, it writes all key/value pairs, building up new data + // structures. My compacted test database was ~60% of original size. Lz4 on the + // uncompacted database got it to 14%. Lz4 on the compacted database got it to 13%. + // Backups are likely archived somewhere with compression, so we don't compact. + // + // Tests with WriteTo and os.O_DIRECT were slower than without O_DIRECT, but + // probably because everything fit in the page cache. It may be better to use + // O_DIRECT when copying many large or inactive databases. + _, err := tx.WriteTo(df) + return err + }) + if err != nil { + return fmt.Errorf("copying database: %v", err) + } + err = df.Close() + df = nil + if err != nil { + return fmt.Errorf("closing destination database after copy: %v", err) + } + xvlog("backed up database file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmDB))) + return nil + } + + // Try to create a hardlink. Fall back to copying the file (e.g. when on different file system). + warnedHardlink := false // We warn once about failing to hardlink. + linkOrCopy := func(srcpath, dstpath string) (bool, error) { + ensureDestDir(dstpath) + + if err := os.Link(srcpath, dstpath); err == nil { + return true, nil + } else if os.IsNotExist(err) { + // No point in trying with regular copy, we would warn twice. + return false, err + } else if !warnedHardlink { + xwarnx("creating hardlink to message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + warnedHardlink = true + } + + // Fall back to copying. + sf, err := os.Open(srcpath) + if err != nil { + return false, fmt.Errorf("open source path %s: %v", srcpath, err) + } + defer sf.Close() + + df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) + if err != nil { + return false, fmt.Errorf("open destination path %s: %v", dstpath, err) + } + defer func() { + if df != nil { + df.Close() + } + }() + if _, err := io.Copy(df, sf); err != nil { + return false, fmt.Errorf("coping: %v", err) + } + err = df.Close() + df = nil + if err != nil { + return false, fmt.Errorf("close: %v", err) + } + return false, nil + } + + // Start making the backup. + tmStart := time.Now() + + ctl.log.Print("making backup", mlog.Field("destdir", dstDataDir)) + + err := os.MkdirAll(dstDataDir, 0770) + if err != nil { + xerrx("creating destination data directory", err) + } + + if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil { + xerrx("writing moxversion", err) + } + backupDB(dmarcdb.DB, "dmarcrpt.db") + backupDB(mtastsdb.DB, "mtasts.db") + backupDB(tlsrptdb.DB, "tlsrpt.db") + backupFile("receivedid.key") + + // Acme directory is optional. + srcAcmeDir := filepath.Join(srcDataDir, "acme") + if _, err := os.Stat(srcAcmeDir); err == nil { + backupDir("acme") + } else if err != nil && !os.IsNotExist(err) { + xerrx("copying acme/", err) + } + + // Copy the queue database and all message files. + backupQueue := func(path string) { + tmQueue := time.Now() + + if err := backupDB(queue.DB, path); err != nil { + xerrx("queue not backed up", err, mlog.Field("path", path), mlog.Field("duration", time.Since(tmQueue))) + return + } + + dstdbpath := filepath.Join(dstDataDir, path) + db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, queue.DBTypes...) + if err != nil { + xerrx("open copied queue database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmQueue))) + return + } + + defer func() { + if db != nil { + err := db.Close() + ctl.log.Check(err, "closing new queue db") + } + }() + + // Link/copy known message files. Warn if files are missing or unexpected + // (though a message file could have been removed just now due to delivery, or a + // new message may have been queued). + tmMsgs := time.Now() + seen := map[string]struct{}{} + var nlinked, ncopied int + err = bstore.QueryDB[queue.Msg](ctx, db).ForEach(func(m queue.Msg) error { + mp := store.MessagePath(m.ID) + seen[mp] = struct{}{} + srcpath := filepath.Join(srcDataDir, "queue", mp) + dstpath := filepath.Join(dstDataDir, "queue", mp) + if linked, err := linkOrCopy(srcpath, dstpath); err != nil { + xerrx("linking/copying queue message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + } else if linked { + nlinked++ + } else { + ncopied++ + } + return nil + }) + if err != nil { + xerrx("processing queue messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs))) + } else { + xvlog("queue message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs))) + } + + // Read through all files in queue directory and warn about anything we haven't handled yet. + tmWalk := time.Now() + srcqdir := filepath.Join(srcDataDir, "queue") + err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error { + if err != nil { + xerrx("walking files in queue", err, mlog.Field("srcpath", srcqpath)) + return nil + } + if d.IsDir() { + return nil + } + p := srcqpath[len(srcqdir)+1:] + if _, ok := seen[p]; ok { + return nil + } + if p == "index.db" { + return nil + } + qp := filepath.Join("queue", p) + xwarnx("backing up unrecognized file in queue directory", nil, mlog.Field("path", qp)) + backupFile(qp) + return nil + }) + if err != nil { + xerrx("walking queue directory (not backed up properly)", err, mlog.Field("dir", "queue"), mlog.Field("duration", time.Since(tmWalk))) + } else { + xvlog("walked queue directory", mlog.Field("duration", time.Since(tmWalk))) + } + + xvlog("queue backed finished", mlog.Field("duration", time.Since(tmQueue))) + } + backupQueue("queue/index.db") + + backupAccount := func(acc *store.Account) { + defer acc.Close() + + tmAccount := time.Now() + + // Copy database file. + dbpath := filepath.Join("accounts", acc.Name, "index.db") + err := backupDB(acc.DB, dbpath) + if err != nil { + xerrx("copying account database", err, mlog.Field("path", dbpath), mlog.Field("duration", time.Since(tmAccount))) + } + + // todo: should document/check not taking a rlock on account. + + // Copy junkfilter files, if configured. + if jf, _, err := acc.OpenJunkFilter(ctx, ctl.log); err != nil { + if !errors.Is(err, store.ErrNoJunkFilter) { + xerrx("opening junk filter for account (not backed up)", err) + } + } else { + db := jf.DB() + jfpath := filepath.Join("accounts", acc.Name, "junkfilter.db") + backupDB(db, jfpath) + bloompath := filepath.Join("accounts", acc.Name, "junkfilter.bloom") + backupFile(bloompath) + db = nil + err := jf.Close() + ctl.log.Check(err, "closing junkfilter") + } + + dstdbpath := filepath.Join(dstDataDir, dbpath) + db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, store.DBTypes...) + if err != nil { + xerrx("open copied account database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmAccount))) + return + } + + defer func() { + if db != nil { + err := db.Close() + ctl.log.Check(err, "close account database") + } + }() + + // Link/copy known message files. Warn if files are missing or unexpected (though a + // message file could have been added just now due to delivery, or a message have + // been removed). + tmMsgs := time.Now() + seen := map[string]struct{}{} + var nlinked, ncopied int + err = bstore.QueryDB[store.Message](ctx, db).ForEach(func(m store.Message) error { + mp := store.MessagePath(m.ID) + seen[mp] = struct{}{} + amp := filepath.Join("accounts", acc.Name, "msg", mp) + srcpath := filepath.Join(srcDataDir, amp) + dstpath := filepath.Join(dstDataDir, amp) + if linked, err := linkOrCopy(srcpath, dstpath); err != nil { + xerrx("linking/copying account message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath)) + } else if linked { + nlinked++ + } else { + ncopied++ + } + return nil + }) + if err != nil { + xerrx("processing account messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs))) + } else { + xvlog("account message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs))) + } + + // Read through all files in account directory and warn about anything we haven't handled yet. + tmWalk := time.Now() + srcadir := filepath.Join(srcDataDir, "accounts", acc.Name) + err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error { + if err != nil { + xerrx("walking files in account", err, mlog.Field("srcpath", srcapath)) + return nil + } + if d.IsDir() { + return nil + } + p := srcapath[len(srcadir)+1:] + l := strings.Split(p, string(filepath.Separator)) + if l[0] == "msg" { + mp := filepath.Join(l[1:]...) + if _, ok := seen[mp]; ok { + return nil + } + } + switch p { + case "index.db", "junkfilter.db", "junkfilter.bloom": + return nil + } + ap := filepath.Join("accounts", acc.Name, p) + if strings.HasPrefix(p, "msg/") { + xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, mlog.Field("path", ap)) + } else { + xwarnx("backing up unrecognized file in account directory", nil, mlog.Field("path", ap)) + } + backupFile(ap) + return nil + }) + if err != nil { + xerrx("walking account directory (not backed up properly)", err, mlog.Field("srcdir", srcadir), mlog.Field("duration", time.Since(tmWalk))) + } else { + xvlog("walked account directory", mlog.Field("duration", time.Since(tmWalk))) + } + + xvlog("account backup finished", mlog.Field("dir", filepath.Join("accounts", acc.Name)), mlog.Field("duration", time.Since(tmAccount))) + } + + // For each configured account, open it, make a copy of the database and + // hardlink/copy the messages. We track the accounts we handled, and skip the + // account directories when handling "all other files" below. + accounts := map[string]struct{}{} + for _, accName := range mox.Conf.Accounts() { + acc, err := store.OpenAccount(accName) + if err != nil { + xerrx("opening account for copying (will try to copy as regular files later)", err, mlog.Field("account", accName)) + continue + } + accounts[accName] = struct{}{} + backupAccount(acc) + } + + // Copy all other files, that aren't part of the known files, databases, queue or accounts. + tmWalk := time.Now() + err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error { + if err != nil { + xerrx("walking path", err, mlog.Field("path", srcpath)) + return nil + } + + if srcpath == srcDataDir { + return nil + } + p := srcpath[len(srcDataDir)+1:] + if p == "queue" || p == "acme" || p == "tmp" { + if p == "tmp" { + xwarnx("skipping entire tmp directory", nil, mlog.Field("path", p)) + } + return fs.SkipDir + } + l := strings.Split(p, string(filepath.Separator)) + if len(l) >= 2 && l[0] == "accounts" { + name := l[1] + if _, ok := accounts[name]; ok { + return fs.SkipDir + } + } + + // Only files are explicitly backed up. + if d.IsDir() { + return nil + } + + switch p { + case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl": + // Already handled. + return nil + case "lastknownversion": // Optional file, not yet handled. + default: + xwarnx("backing up unrecognized file", nil, mlog.Field("path", p)) + } + backupFile(p) + return nil + }) + if err != nil { + xerrx("walking other files (not backed up properly)", err, mlog.Field("duration", time.Since(tmWalk))) + } else { + xvlog("walking other files finished", mlog.Field("duration", time.Since(tmWalk))) + } + + xvlog("backup finished", mlog.Field("duration", time.Since(tmStart))) + + writer.xclose() + + if incomplete { + ctl.xwrite("errors were encountered during backup") + } else { + ctl.xwriteok() + } +} diff --git a/ctl.go b/ctl.go index f9c9ff7..a890088 100644 --- a/ctl.go +++ b/ctl.go @@ -30,6 +30,7 @@ import ( // ctl represents a connection to the ctl unix domain socket of a running mox instance. // ctl provides functions to read/write commands/responses/data streams. type ctl struct { + cmd string // Set for server-side of commands. conn net.Conn r *bufio.Reader // Set for first reader. x any // If set, errors are handled by calling panic(x) instead of log.Fatal. @@ -57,7 +58,7 @@ func (c *ctl) xerror(msg string) { if c.x == nil { log.Fatalln(msg) } - c.log.Debugx("ctl error", fmt.Errorf("%s", msg)) + c.log.Debugx("ctl error", fmt.Errorf("%s", msg), mlog.Field("cmd", c.cmd)) c.xwrite(msg) panic(c.x) } @@ -72,7 +73,7 @@ func (c *ctl) xcheck(err error, msg string) { if c.x == nil { log.Fatalf("%s: %s", msg, err) } - c.log.Debugx(msg, err) + c.log.Debugx(msg, err, mlog.Field("cmd", c.cmd)) fmt.Fprintf(c.conn, "%s: %s\n", msg, err) panic(c.x) } @@ -124,7 +125,7 @@ func (c *ctl) xstreamfrom(src io.Reader) { // When done writing, caller must call xclose to signal the end of the stream. // Behaviour of "x" is copied from ctl. func (c *ctl) writer() *ctlwriter { - return &ctlwriter{conn: c.conn, x: c.x, log: c.log} + return &ctlwriter{cmd: c.cmd, conn: c.conn, x: c.x, log: c.log} } // Reader returns an io.Reader for a data stream from ctl. @@ -133,7 +134,7 @@ func (c *ctl) reader() *ctlreader { if c.r == nil { c.r = bufio.NewReader(c.conn) } - return &ctlreader{conn: c.conn, r: c.r, x: c.x, log: c.log} + return &ctlreader{cmd: c.cmd, conn: c.conn, r: c.r, x: c.x, log: c.log} } /* @@ -154,6 +155,7 @@ Followed by a end of stream indicated by zero data bytes message: */ type ctlwriter struct { + cmd string // Set for server-side of commands. conn net.Conn // Ctl socket from which messages are read. buf []byte // Scratch buffer, for reading response. x any // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal. @@ -181,7 +183,7 @@ func (s *ctlwriter) xerror(msg string) { if s.x == nil { log.Fatalln(msg) } else { - s.log.Debugx("error", fmt.Errorf("%s", msg)) + s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd)) panic(s.x) } } @@ -193,7 +195,7 @@ func (s *ctlwriter) xcheck(err error, msg string) { if s.x == nil { log.Fatalf("%s: %s", msg, err) } else { - s.log.Debugx(msg, err) + s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd)) panic(s.x) } } @@ -204,6 +206,7 @@ func (s *ctlwriter) xclose() { } type ctlreader struct { + cmd string // Set for server-side of command. conn net.Conn // For writing "ok" after reading. r *bufio.Reader // Buffered ctl socket. err error // If set, returned for each read. can also be io.EOF. @@ -246,7 +249,7 @@ func (s *ctlreader) xerror(msg string) { if s.x == nil { log.Fatalln(msg) } else { - s.log.Debugx("error", fmt.Errorf("%s", msg)) + s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd)) panic(s.x) } } @@ -258,7 +261,7 @@ func (s *ctlreader) xcheck(err error, msg string) { if s.x == nil { log.Fatalf("%s: %s", msg, err) } else { - s.log.Debugx(msg, err) + s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd)) panic(s.x) } } @@ -267,33 +270,30 @@ func (s *ctlreader) xcheck(err error, msg string) { func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()) { log.Debug("ctl connection") - var cmd string - var stop = struct{}{} // Sentinel value for panic and recover. + ctl := &ctl{conn: conn, x: stop, log: log} defer func() { x := recover() if x == nil || x == stop { return } - log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", cmd)) + log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", ctl.cmd)) debug.PrintStack() metrics.PanicInc("ctl") }() defer conn.Close() - ctl := &ctl{conn: conn, x: stop, log: log} ctl.xwrite("ctlv0") - for { - servectlcmd(ctx, log, ctl, &cmd, shutdown) + servectlcmd(ctx, log, ctl, shutdown) } } -func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shutdown func()) { +func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, shutdown func()) { cmd := ctl.xread() + ctl.cmd = cmd log.Info("ctl command", mlog.Field("cmd", cmd)) - *xcmd = cmd switch cmd { case "stop": shutdown() @@ -641,6 +641,9 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu ctl.xwriteok() + case "backup": + backupctl(ctx, ctl) + default: log.Info("unrecognized command", mlog.Field("cmd", cmd)) ctl.xwrite("unrecognized command") diff --git a/develop.txt b/develop.txt index 2017bd4..e12f218 100644 --- a/develop.txt +++ b/develop.txt @@ -167,6 +167,7 @@ testing purposes. - Write release notes, use instructions from updating.txt. - Build and run tests with previous major Go release. - Run all (integration) tests, including with race detector. +- Test upgrades. - Run fuzzing tests for a while. - Deploy to test environment. Test the update instructions. - Generate a config with quickstart, check if it results in a working setup. diff --git a/dmarcdb/db.go b/dmarcdb/db.go index 3120106..bb3835d 100644 --- a/dmarcdb/db.go +++ b/dmarcdb/db.go @@ -28,7 +28,8 @@ import ( var xlog = mlog.New("dmarcdb") var ( - dmarcDB *bstore.DB + DBTypes = []any{DomainFeedback{}} // Types stored in DB. + DB *bstore.DB // Exported for backups. mutex sync.Mutex ) @@ -70,16 +71,16 @@ type DomainFeedback struct { func database(ctx context.Context) (rdb *bstore.DB, rerr error) { mutex.Lock() defer mutex.Unlock() - if dmarcDB == nil { + if DB == nil { p := mox.DataDirPath("dmarcrpt.db") os.MkdirAll(filepath.Dir(p), 0770) - db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DomainFeedback{}) + db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) if err != nil { return nil, err } - dmarcDB = db + DB = db } - return dmarcDB, nil + return DB, nil } // Init opens the database. diff --git a/doc.go b/doc.go index 4f758e3..0dc36d3 100644 --- a/doc.go +++ b/doc.go @@ -29,6 +29,8 @@ low-maintenance self-hosted email. mox export mbox dst-dir account-path [mailbox] mox localserve mox help [command ...] + mox backup dest-dir + mox verifydata data-dir mox config test mox config dnscheck domain mox config dnsrecords domain @@ -331,6 +333,69 @@ If a single command matches, its usage and full help text is printed. usage: mox help [command ...] +# mox backup + +Creates a backup of the data directory. + +Backup creates consistent snapshots of the databases and message files and +copies other files in the data directory. Empty directories are not copied. +These files can then be stored elsewhere for long-term storage, or used to fall +back to should an upgrade fail. Simply copying files in the data directory +while mox is running can result in unusable database files. + +Message files never change (they are read-only, though can be removed) and are +hardlinked so they don't consume additional space. If hardlinking fails, for +example when the backup destination directory is on a different file system, a +regular copy is made. Using a destination directory like "data/tmp/backup" +increases the odds hardlinking succeeds: the default systemd service file +specifically mounts the data directory, causing attempts to hardlink outside it +to fail with an error about cross-device linking. + +All files in the data directory that aren't recognized (i.e. other than known +database files, message files, an acme directory, etc), are stored, but with a +warning. + +A clean successful backup does not print any output by default. Use the +-verbose flag for details, including timing. + +To restore a backup, first shut down mox, move away the old data directory and +move an earlier backed up directory in its place, run "mox verifydata", +possibly with the "-fix" option, and restart mox. After the restore, you may +also want to run "mox bumpuidvalidity" for each account for which messages in a +mailbox changed, to force IMAP clients to synchronize mailbox state. + +Before upgrading, to check if the upgrade will likely succeed, first make a +backup, then use the new mox binary to run "mox verifydata" on the backup. This +can change the backup files (e.g. upgrade database files, move away +unrecognized message files), so you should make a new backup before actually +upgrading. + + usage: mox backup dest-dir + -verbose + print progress + +# mox verifydata + +Verify the contents of a data directory, typically of a backup. + +Verifydata checks all database files to see if they are valid BoltDB/bstore +databases. It checks that all messages in the database have a corresponding +on-disk message file and there are no unrecognized files. If option -fix is +specified, unrecognized message files are moved away. This may be needed after +a restore, because messages enqueued or delivered in the future may get those +message sequence numbers assigned and writing the message file would fail. + +Because verifydata opens the database files, schema upgrades may automatically +be applied. This can happen if you use a new mox release. It is useful to run +"mox verifydata" with a new binary before attempting an upgrade, but only on a +copy of the database files, as made with "mox backup". Before upgrading, make a +new backup again since "mox verifydata" may have upgraded the database files, +possibly making them potentially no longer readable by the previous version. + + usage: mox verifydata data-dir + -fix + fix fixable problems, such as moving away message files not referenced by their database + # mox config test Parses and validates the configuration files. diff --git a/export.go b/export.go index a284ef5..3f331a4 100644 --- a/export.go +++ b/export.go @@ -57,7 +57,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) { } dbpath := filepath.Join(accountDir, "index.db") - db, err := bstore.Open(context.Background(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{}) + db, err := bstore.Open(context.Background(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.DBTypes...) xcheckf(err, "open database %q", dbpath) defer func() { if err := db.Close(); err != nil { diff --git a/gentestdata.go b/gentestdata.go new file mode 100644 index 0000000..b09f7c3 --- /dev/null +++ b/gentestdata.go @@ -0,0 +1,361 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/mjl-/bstore" + "github.com/mjl-/sconf" + + "github.com/mjl-/mox/config" + "github.com/mjl-/mox/dmarcdb" + "github.com/mjl-/mox/dmarcrpt" + "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/moxvar" + "github.com/mjl-/mox/mtasts" + "github.com/mjl-/mox/mtastsdb" + "github.com/mjl-/mox/queue" + "github.com/mjl-/mox/smtp" + "github.com/mjl-/mox/store" + "github.com/mjl-/mox/tlsrpt" + "github.com/mjl-/mox/tlsrptdb" +) + +func cmdGentestdata(c *cmd) { + c.unlisted = true + c.params = "dest-dir" + c.help = `Generate a data directory populated, for testing upgrades.` + args := c.Parse() + if len(args) != 1 { + c.Usage() + } + + destDataDir, err := filepath.Abs(args[0]) + xcheckf(err, "making destination directory an absolute path") + + if _, err := os.Stat(destDataDir); err == nil { + log.Fatalf("destination directory already exists, refusing to generate test data") + } + err = os.MkdirAll(destDataDir, 0770) + xcheckf(err, "creating destination data directory") + err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770) + xcheckf(err, "creating tmp directory") + + tempfile := func() *os.File { + f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp") + xcheckf(err, "creating temp file") + return f + } + + ctxbg := context.Background() + mox.Shutdown = ctxbg + mox.Context = ctxbg + mox.Conf.Log[""] = mlog.LevelInfo + mlog.SetConfig(mox.Conf.Log) + + const domainsConf = ` +Domains: + mox.example: nil + ☺.example: nil +Accounts: + test0: + Domain: mox.example + Destinations: + test0@mox.example: nil + test1: + Domain: mox.example + Destinations: + test1@mox.example: nil + test2: + Domain: ☺.example + Destinations: + ☹@☺.example: nil + JunkFilter: + Threshold: 0.95 + Params: + Twograms: true + MaxPower: 0.1 + TopWords: 10 + IgnoreWords: 0.1 +` + + mox.ConfigStaticPath = "/tmp/mox-bogus/mox.conf" + mox.ConfigDynamicPath = "/tmp/mox-bogus/domains.conf" + mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning. + mox.Conf.Static = config.Static{ + DataDir: destDataDir, + } + err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic) + xcheckf(err, "parsing domains config") + + const dmarcReport = ` + + + google.com + noreply-dmarc-support@google.com + https://support.google.com/a/answer/2466580 + 10051505501689795560 + + 1596412800 + 1596499199 + + + + mox.example + r + r +

reject

+ reject + 100 +
+ + + 127.0.0.1 + 1 + + none + pass + pass + + + + example.org + + + + example.org + pass + example + + + example.org + pass + + + +
+` + + const tlsReport = `{ + "organization-name": "Company-X", + "date-range": { + "start-datetime": "2016-04-01T00:00:00Z", + "end-datetime": "2016-04-01T23:59:59Z" + }, + "contact-info": "sts-reporting@company-x.example", + "report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be", + "policies": [{ + "policy": { + "policy-type": "sts", + "policy-string": ["version: STSv1","mode: testing", + "mx: *.mail.company-y.example","max_age: 86400"], + "policy-domain": "mox.example", + "mx-host": ["*.mail.company-y.example"] + }, + "summary": { + "total-successful-session-count": 5326, + "total-failure-session-count": 303 + }, + "failure-details": [{ + "result-type": "certificate-expired", + "sending-mta-ip": "2001:db8:abcd:0012::1", + "receiving-mx-hostname": "mx1.mail.company-y.example", + "failed-session-count": 100 + }, { + "result-type": "starttls-not-supported", + "sending-mta-ip": "2001:db8:abcd:0013::1", + "receiving-mx-hostname": "mx2.mail.company-y.example", + "receiving-ip": "203.0.113.56", + "failed-session-count": 200, + "additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported " + }, { + "result-type": "validation-failure", + "sending-mta-ip": "198.51.100.62", + "receiving-ip": "203.0.113.58", + "receiving-mx-hostname": "mx-backup.mail.company-y.example", + "failed-session-count": 3, + "failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED" + }] + }] + }` + + err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660) + xcheckf(err, "writing moxversion") + + // Populate dmarc.db. + err = dmarcdb.Init() + xcheckf(err, "dmarcdb init") + report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport)) + xcheckf(err, "parsing dmarc report") + err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"}) + xcheckf(err, "adding dmarc report") + + // Populate mtasts.db. + err = mtastsdb.Init(false) + xcheckf(err, "mtastsdb init") + mtastsPolicy := mtasts.Policy{ + Version: "STSv1", + Mode: mtasts.ModeTesting, + MX: []mtasts.STSMX{ + {Domain: dns.Domain{ASCII: "mx1.example.com"}}, + {Domain: dns.Domain{ASCII: "mx2.example.com"}}, + {Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true}, + }, + MaxAgeSeconds: 1296000, + } + err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy) + xcheckf(err, "adding mtastsdb report") + + // Populate tlsrpt.db. + err = tlsrptdb.Init() + xcheckf(err, "tlsrptdb init") + tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport)) + xcheckf(err, "parsing tls report") + err = tlsrptdb.AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", tlsr) + xcheckf(err, "adding tls report") + + // Populate queue, with a message. + err = queue.Init() + xcheckf(err, "queue init") + mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}} + rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}} + prefix := []byte{} + mf := tempfile() + xcheckf(err, "temp file for queue message") + defer mf.Close() + const qmsg = "From: \r\nTo: \r\nSubject: test\r\n\r\nthe message...\r\n" + _, err = fmt.Fprint(mf, qmsg) + xcheckf(err, "writing message") + err = queue.Add(ctxbg, mlog.New("gentestdata"), "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), prefix, mf, nil, true) + xcheckf(err, "enqueue message") + + // Create three accounts. + // First account without messages. + accTest0, err := store.OpenAccount("test0") + xcheckf(err, "open account test0") + err = accTest0.Close() + xcheckf(err, "close account") + + // Second account with one message. + accTest1, err := store.OpenAccount("test1") + xcheckf(err, "open account test1") + err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error { + inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() + xcheckf(err, "looking up inbox") + const msg = "From: \r\nTo: \r\nSubject: test\r\n\r\nthe message...\r\n" + m := store.Message{ + MailboxID: inbox.ID, + MailboxOrigID: inbox.ID, + MailboxDestinedID: inbox.ID, + RemoteIP: "1.2.3.4", + RemoteIPMasked1: "1.2.3.4", + RemoteIPMasked2: "1.2.3.0", + RemoteIPMasked3: "1.2.0.0", + EHLODomain: "other.example", + MailFrom: "other@remote.example", + MailFromLocalpart: smtp.Localpart("other"), + MailFromDomain: "remote.example", + RcptToLocalpart: "test1", + RcptToDomain: "mox.example", + MsgFromLocalpart: "other", + MsgFromDomain: "remote.example", + MsgFromOrgDomain: "remote.example", + EHLOValidated: true, + MailFromValidated: true, + MsgFromValidated: true, + EHLOValidation: store.ValidationStrict, + MailFromValidation: store.ValidationPass, + MsgFromValidation: store.ValidationStrict, + DKIMDomains: []string{"other.example"}, + Size: int64(len(msg)), + } + mf := tempfile() + xcheckf(err, "creating temp file for delivery") + _, err = fmt.Fprint(mf, msg) + xcheckf(err, "writing deliver message to file") + err = accTest1.DeliverMessage(mlog.New("gentestdata"), tx, &m, mf, true, false, false, true) + xcheckf(err, "add message to account test1") + err = mf.Close() + xcheckf(err, "closing file") + return nil + }) + xcheckf(err, "write transaction with new message") + err = accTest1.Close() + xcheckf(err, "close account") + + // Third account with two messages and junkfilter. + accTest2, err := store.OpenAccount("test2") + xcheckf(err, "open account test2") + err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error { + inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() + xcheckf(err, "looking up inbox") + const msg0 = "From: \r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n" + m0 := store.Message{ + MailboxID: inbox.ID, + MailboxOrigID: inbox.ID, + MailboxDestinedID: inbox.ID, + RemoteIP: "::1", + RemoteIPMasked1: "::", + RemoteIPMasked2: "::", + RemoteIPMasked3: "::", + EHLODomain: "other.example", + MailFrom: "other@remote.example", + MailFromLocalpart: smtp.Localpart("other"), + MailFromDomain: "remote.example", + RcptToLocalpart: "☹", + RcptToDomain: "☺.example", + MsgFromLocalpart: "other", + MsgFromDomain: "remote.example", + MsgFromOrgDomain: "remote.example", + EHLOValidated: true, + MailFromValidated: true, + MsgFromValidated: true, + EHLOValidation: store.ValidationStrict, + MailFromValidation: store.ValidationPass, + MsgFromValidation: store.ValidationStrict, + DKIMDomains: []string{"other.example"}, + Size: int64(len(msg0)), + } + mf0 := tempfile() + xcheckf(err, "creating temp file for delivery") + _, err = fmt.Fprint(mf0, msg0) + xcheckf(err, "writing deliver message to file") + err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m0, mf0, true, false, false, false) + xcheckf(err, "add message to account test2") + err = mf0.Close() + xcheckf(err, "closing file") + + sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get() + xcheckf(err, "looking up inbox") + const prefix1 = "Extra: test\r\n" + const msg1 = "From: \r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n" + m1 := store.Message{ + MailboxID: sent.ID, + MailboxOrigID: sent.ID, + MailboxDestinedID: sent.ID, + Flags: store.Flags{Seen: true, Junk: true}, + Size: int64(len(prefix1) + len(msg1)), + MsgPrefix: []byte(prefix), + } + mf1 := tempfile() + xcheckf(err, "creating temp file for delivery") + _, err = fmt.Fprint(mf1, msg1) + xcheckf(err, "writing deliver message to file") + err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m1, mf1, true, true, false, false) + xcheckf(err, "add message to account test2") + err = mf1.Close() + xcheckf(err, "closing file") + + return nil + }) + xcheckf(err, "write transaction with new message") + err = accTest2.Close() + xcheckf(err, "close account") +} diff --git a/integration_test.go b/integration_test.go index a5479d7..6dd62e6 100644 --- a/integration_test.go +++ b/integration_test.go @@ -82,7 +82,7 @@ func TestDeliver(t *testing.T) { latestMsgID := func(username string) int64 { // We open the account index database created by mox for the test user. And we keep looking for the email we sent. dbpath := fmt.Sprintf("testdata/integration/data/accounts/%s/index.db", username) - db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{}, store.Password{}) + db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.DBTypes...) if err != nil && errors.Is(err, bolt.ErrTimeout) { log.Printf("db open timeout (normal delay for new sender with account and db file kept open)") return 0 diff --git a/junk/filter.go b/junk/filter.go index f8bc0a5..c927c44 100644 --- a/junk/filter.go +++ b/junk/filter.go @@ -56,6 +56,8 @@ type Params struct { RareWords int `sconf:"optional" sconf-doc:"Occurrences in word database until a word is considered rare and its influence in calculating probability reduced. E.g. 1 or 2."` } +var DBTypes = []any{wordscore{}} // Stored in DB. + type Filter struct { Params @@ -228,7 +230,7 @@ func newDB(ctx context.Context, log *mlog.Log, path string) (db *bstore.DB, rerr } }() - db, err := bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{}) + db, err := bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) if err != nil { return nil, fmt.Errorf("open new database: %w", err) } @@ -239,7 +241,7 @@ func openDB(ctx context.Context, path string) (*bstore.DB, error) { if _, err := os.Stat(path); err != nil { return nil, fmt.Errorf("stat db file: %w", err) } - return bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{}) + return bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) } // Save stores modifications, e.g. from training, to the database and bloom @@ -744,3 +746,8 @@ func (f *Filter) fileSize(p string) int { } return int(fi.Size()) } + +// DB returns the database, for backups. +func (f *Filter) DB() *bstore.DB { + return f.db +} diff --git a/main.go b/main.go index 68ebef4..8bb8251 100644 --- a/main.go +++ b/main.go @@ -88,6 +88,8 @@ var commands = []struct { {"export mbox", cmdExportMbox}, {"localserve", cmdLocalserve}, {"help", cmdHelp}, + {"backup", cmdBackup}, + {"verifydata", cmdVerifydata}, {"config test", cmdConfigTest}, {"config dnscheck", cmdConfigDNSCheck}, @@ -146,6 +148,7 @@ var commands = []struct { {"updates pubkey", cmdUpdatesPubkey}, {"updates serve", cmdUpdatesServe}, {"updates verify", cmdUpdatesVerify}, + {"gentestdata", cmdGentestdata}, } var cmds []cmd @@ -961,6 +964,67 @@ new mail deliveries. fmt.Println("mox stopped") } +func cmdBackup(c *cmd) { + c.params = "dest-dir" + c.help = `Creates a backup of the data directory. + +Backup creates consistent snapshots of the databases and message files and +copies other files in the data directory. Empty directories are not copied. +These files can then be stored elsewhere for long-term storage, or used to fall +back to should an upgrade fail. Simply copying files in the data directory +while mox is running can result in unusable database files. + +Message files never change (they are read-only, though can be removed) and are +hardlinked so they don't consume additional space. If hardlinking fails, for +example when the backup destination directory is on a different file system, a +regular copy is made. Using a destination directory like "data/tmp/backup" +increases the odds hardlinking succeeds: the default systemd service file +specifically mounts the data directory, causing attempts to hardlink outside it +to fail with an error about cross-device linking. + +All files in the data directory that aren't recognized (i.e. other than known +database files, message files, an acme directory, etc), are stored, but with a +warning. + +A clean successful backup does not print any output by default. Use the +-verbose flag for details, including timing. + +To restore a backup, first shut down mox, move away the old data directory and +move an earlier backed up directory in its place, run "mox verifydata", +possibly with the "-fix" option, and restart mox. After the restore, you may +also want to run "mox bumpuidvalidity" for each account for which messages in a +mailbox changed, to force IMAP clients to synchronize mailbox state. + +Before upgrading, to check if the upgrade will likely succeed, first make a +backup, then use the new mox binary to run "mox verifydata" on the backup. This +can change the backup files (e.g. upgrade database files, move away +unrecognized message files), so you should make a new backup before actually +upgrading. +` + + var verbose bool + c.flag.BoolVar(&verbose, "verbose", false, "print progress") + args := c.Parse() + if len(args) != 1 { + c.Usage() + } + mustLoadConfig() + + dstDataDir, err := filepath.Abs(args[0]) + xcheckf(err, "making path absolute") + + ctl := xctl() + ctl.xwrite("backup") + ctl.xwrite(dstDataDir) + if verbose { + ctl.xwrite("verbose") + } else { + ctl.xwrite("") + } + ctl.xstreamto(os.Stdout) + ctl.xreadok() +} + func cmdSetadminpassword(c *cmd) { c.help = `Set a new admin password, for the web interface. diff --git a/mlog/log.go b/mlog/log.go index 0e1da8c..0c5be6a 100644 --- a/mlog/log.go +++ b/mlog/log.go @@ -96,8 +96,8 @@ func SetConfig(c map[string]Level) { // Pair is a field/value pair, for use in logged lines. type Pair struct { - key string - value any + Key string + Value any } // Field is a shorthand for making a Pair. @@ -339,7 +339,7 @@ func (l *Log) plog(level Level, err error, text string, fields ...Pair) { } for i := 0; i < len(fields); i++ { kv := fields[i] - fmt.Fprintf(b, " %s=%s", kv.key, logfmtValue(stringValue(kv.key == "cid", false, kv.value))) + fmt.Fprintf(b, " %s=%s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value))) } b.WriteString("\n") } else { @@ -354,7 +354,7 @@ func (l *Log) plog(level Level, err error, text string, fields ...Pair) { fmt.Fprint(b, "; ") } kv := fields[i] - fmt.Fprintf(b, "%s: %s", kv.key, logfmtValue(stringValue(kv.key == "cid", false, kv.value))) + fmt.Fprintf(b, "%s: %s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value))) } fmt.Fprint(b, ")") } @@ -373,10 +373,10 @@ func (l *Log) match(level Level) (bool, Level) { seen := false var high Level for _, kv := range l.fields { - if kv.key != "pkg" { + if kv.Key != "pkg" { continue } - pkg, ok := kv.value.(string) + pkg, ok := kv.Value.(string) if !ok { continue } diff --git a/mtastsdb/db.go b/mtastsdb/db.go index 27b2d77..d649b60 100644 --- a/mtastsdb/db.go +++ b/mtastsdb/db.go @@ -60,22 +60,23 @@ var ( ErrBackoff = errors.New("mtastsdb: policy fetch failed recently") ) -var mtastsDB *bstore.DB +var DBTypes = []any{PolicyRecord{}} // Types stored in DB. +var DB *bstore.DB // Exported for backups. var mutex sync.Mutex func database(ctx context.Context) (rdb *bstore.DB, rerr error) { mutex.Lock() defer mutex.Unlock() - if mtastsDB == nil { + if DB == nil { p := mox.DataDirPath("mtasts.db") os.MkdirAll(filepath.Dir(p), 0770) - db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, PolicyRecord{}) + db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) if err != nil { return nil, err } - mtastsDB = db + DB = db } - return mtastsDB, nil + return DB, nil } // Init opens the database and starts a goroutine that refreshes policies in @@ -98,10 +99,10 @@ func Init(refresher bool) error { func Close() { mutex.Lock() defer mutex.Unlock() - if mtastsDB != nil { - err := mtastsDB.Close() + if DB != nil { + err := DB.Close() xlog.Check(err, "closing database") - mtastsDB = nil + DB = nil } } diff --git a/queue/queue.go b/queue/queue.go index 9527fa2..6059ea6 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -70,7 +70,8 @@ var dial = func(ctx context.Context, timeout time.Duration, addr string, laddr n var jitter = mox.NewRand() -var queueDB *bstore.DB +var DBTypes = []any{Msg{}} // Types stored in DB. +var DB *bstore.DB // Exported for making backups. // Set for mox localserve, to prevent queueing. var Localserve bool @@ -122,7 +123,7 @@ func Init() error { } var err error - queueDB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, Msg{}) + DB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) if err != nil { if isNew { os.Remove(qpath) @@ -134,15 +135,15 @@ func Init() error { // Shutdown closes the queue database. The delivery process isn't stopped. For tests only. func Shutdown() { - err := queueDB.Close() + err := DB.Close() xlog.Check(err, "closing queue db") - queueDB = nil + DB = nil } // List returns all messages in the delivery queue. // Ordered by earliest delivery attempt first. func List(ctx context.Context) ([]Msg, error) { - qmsgs, err := bstore.QueryDB[Msg](ctx, queueDB).List() + qmsgs, err := bstore.QueryDB[Msg](ctx, DB).List() if err != nil { return nil, err } @@ -166,7 +167,7 @@ func List(ctx context.Context) ([]Msg, error) { // Count returns the number of messages in the delivery queue. func Count(ctx context.Context) (int, error) { - return bstore.QueryDB[Msg](ctx, queueDB).Count() + return bstore.QueryDB[Msg](ctx, DB).Count() } // Add a new message to the queue. The queue is kicked immediately to start a @@ -187,7 +188,7 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp return fmt.Errorf("no queuing with localserve") } - tx, err := queueDB.Begin(ctx, true) + tx, err := DB.Begin(ctx, true) if err != nil { return fmt.Errorf("begin transaction: %w", err) } @@ -288,7 +289,7 @@ func queuekick() { // are zero, all messages are kicked. // Returns number of messages queued for immediate delivery. func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) { - q := bstore.QueryDB[Msg](ctx, queueDB) + q := bstore.QueryDB[Msg](ctx, DB) if ID > 0 { q.FilterID(ID) } @@ -312,7 +313,7 @@ func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int // If all parameters are zero, all messages are removed. // Returns number of messages removed. func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) { - q := bstore.QueryDB[Msg](ctx, queueDB) + q := bstore.QueryDB[Msg](ctx, DB) if ID > 0 { q.FilterID(ID) } @@ -347,7 +348,7 @@ type ReadReaderAtCloser interface { // OpenMessage opens a message present in the queue. func OpenMessage(ctx context.Context, id int64) (ReadReaderAtCloser, error) { qm := Msg{ID: id} - err := queueDB.Get(ctx, &qm) + err := DB.Get(ctx, &qm) if err != nil { return nil, err } @@ -397,7 +398,7 @@ func Start(resolver dns.Resolver, done chan struct{}) error { } func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration { - q := bstore.QueryDB[Msg](ctx, queueDB) + q := bstore.QueryDB[Msg](ctx, DB) if len(busyDomains) > 0 { var doms []any for d := range busyDomains { @@ -418,7 +419,7 @@ func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duratio } func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int { - q := bstore.QueryDB[Msg](mox.Shutdown, queueDB) + q := bstore.QueryDB[Msg](mox.Shutdown, DB) q.FilterLessEqual("NextAttempt", time.Now()) q.SortAsc("NextAttempt") q.Limit(maxConcurrentDeliveries) @@ -445,7 +446,7 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int { // Remove message from queue in database and file system. func queueDelete(ctx context.Context, msgID int64) error { - if err := queueDB.Delete(ctx, &Msg{ID: msgID}); err != nil { + if err := DB.Delete(ctx, &Msg{ID: msgID}); err != nil { return err } // If removing from database fails, we'll also leave the file in the file system. @@ -491,7 +492,7 @@ func deliver(resolver dns.Resolver, m Msg) { now := time.Now() m.LastAttempt = &now m.NextAttempt = now.Add(backoff) - qup := bstore.QueryDB[Msg](mox.Shutdown, queueDB) + qup := bstore.QueryDB[Msg](mox.Shutdown, DB) qup.FilterID(m.ID) update := Msg{Attempts: m.Attempts, NextAttempt: m.NextAttempt, LastAttempt: m.LastAttempt} if _, err := qup.UpdateNonzero(update); err != nil { @@ -510,7 +511,7 @@ func deliver(resolver dns.Resolver, m Msg) { return } - qup := bstore.QueryDB[Msg](context.Background(), queueDB) + qup := bstore.QueryDB[Msg](context.Background(), DB) qup.FilterID(m.ID) if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil { qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg)) diff --git a/queue/queue_test.go b/queue/queue_test.go index fd9e2b8..70df178 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -138,7 +138,7 @@ func TestQueue(t *testing.T) { case <-dialed: i := 0 for { - m, err := bstore.QueryDB[Msg](ctxbg, queueDB).Get() + m, err := bstore.QueryDB[Msg](ctxbg, DB).Get() tcheck(t, err, "get") if m.Attempts == 1 { break @@ -288,7 +288,7 @@ func TestQueue(t *testing.T) { for i := 1; i < 8; i++ { go func() { <-deliveryResult }() // Deliver sends here. deliver(resolver, msg) - err = queueDB.Get(ctxbg, &msg) + err = DB.Get(ctxbg, &msg) tcheck(t, err, "get msg") if msg.Attempts != i { t.Fatalf("got attempt %d, expected %d", msg.Attempts, i) @@ -311,7 +311,7 @@ func TestQueue(t *testing.T) { // Trigger final failure. go func() { <-deliveryResult }() // Deliver sends here. deliver(resolver, msg) - err = queueDB.Get(ctxbg, &msg) + err = DB.Get(ctxbg, &msg) if err != bstore.ErrAbsent { t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err) } diff --git a/serve.go b/serve.go index 40dc5f1..00a657a 100644 --- a/serve.go +++ b/serve.go @@ -292,7 +292,7 @@ requested, other TLS certificates are requested on demand. } }() m := &store.Message{Received: time.Now(), Flags: store.Flags{Flagged: true}} - n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this is install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n")) + n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this is install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nRemember to make a backup with \"mox backup\" before upgrading.\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n")) if err != nil { log.Infox("writing temporary message file for changelog delivery", err) return next diff --git a/smtpserver/reputation_test.go b/smtpserver/reputation_test.go index 58856ee..52a9dce 100644 --- a/smtpserver/reputation_test.go +++ b/smtpserver/reputation_test.go @@ -102,7 +102,7 @@ func TestReputation(t *testing.T) { p := "../testdata/smtpserver-reputation.db" defer os.Remove(p) - db, err := bstore.Open(ctxbg, p, &bstore.Options{Timeout: 5 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{}) + db, err := bstore.Open(ctxbg, p, &bstore.Options{Timeout: 5 * time.Second}, store.DBTypes...) tcheck(t, err, "open db") defer db.Close() diff --git a/store/account.go b/store/account.go index 4152a0c..f1a98d3 100644 --- a/store/account.go +++ b/store/account.go @@ -377,6 +377,9 @@ type Outgoing struct { Submitted time.Time `bstore:"nonzero,default now"` } +// Types stored in DB. +var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}} + // Account holds the information about a user, includings mailboxes, messages, imap subscriptions. type Account struct { Name string // Name, according to configuration. @@ -455,7 +458,7 @@ func openAccount(name string) (a *Account, rerr error) { os.MkdirAll(dir, 0770) } - db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}) + db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) if err != nil { return nil, err } diff --git a/test-upgrade.sh b/test-upgrade.sh new file mode 100755 index 0000000..dbd7565 --- /dev/null +++ b/test-upgrade.sh @@ -0,0 +1,54 @@ +#!/bin/sh + +# todo: should we also test with mox.conf and domains.conf files? should "mox backup" and "mox gentestdata" add them, and "mox verifydata" use them? + +set -e +# set -x + +(rm -r testdata/upgrade 2>/dev/null || exit 0) +mkdir testdata/upgrade +cd testdata/upgrade + +# Check that we can upgrade what we currently generate. +../../mox gentestdata data +../../mox verifydata data +rm -r data + +# For each historic release (i.e. all tagged versions) except the first few that +# didn't have the gentestdata command, we generate a data directory for testing +# and simulate upgrade to currently checked out version. +# The awk command reverses the tags, so we try the previous release first since +# it is the most likely to fail. +tagsrev=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }') +if test "$tagsrev" = ""; then exit 0; fi +for tag in $tagsrev; do + echo "Testing upgrade from $tag to current." + mkdir $tag + (CGO_ENABLED=0 GOBIN=$PWD/$tag go install github.com/mjl-/mox@$tag) + # Generate with historic release. + ./$tag/mox gentestdata $tag/data + # Verify with current code. + ../../mox verifydata $tag/data + rm -r $tag/data +done + +# Also go step-wise through each released version. Having upgraded step by step +# can have added more schema upgrades to the database files. +tags=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | cat) +first=yes +for tag in $tags; do + if test "$first" = yes; then + echo "Starting with test data for $tag." + ./$tag/mox gentestdata stepdata + first= + else + echo "Upgrade data to $tag." + ./$tag/mox verifydata stepdata + fi +done +echo "Testing final upgrade to current." +../../mox verifydata stepdata +rm -r stepdata +rm */mox +cd ../.. +rmdir testdata/upgrade/* testdata/upgrade diff --git a/tlsrptdb/db.go b/tlsrptdb/db.go index 572cab7..cb340a5 100644 --- a/tlsrptdb/db.go +++ b/tlsrptdb/db.go @@ -23,8 +23,9 @@ import ( var ( xlog = mlog.New("tlsrptdb") - tlsrptDB *bstore.DB - mutex sync.Mutex + DBTypes = []any{TLSReportRecord{}} + DB *bstore.DB + mutex sync.Mutex metricSession = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -64,16 +65,16 @@ type TLSReportRecord struct { func database(ctx context.Context) (rdb *bstore.DB, rerr error) { mutex.Lock() defer mutex.Unlock() - if tlsrptDB == nil { + if DB == nil { p := mox.DataDirPath("tlsrpt.db") os.MkdirAll(filepath.Dir(p), 0770) - db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, TLSReportRecord{}) + db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) if err != nil { return nil, err } - tlsrptDB = db + DB = db } - return tlsrptDB, nil + return DB, nil } // Init opens and possibly initializes the database. @@ -86,10 +87,10 @@ func Init() error { func Close() { mutex.Lock() defer mutex.Unlock() - if tlsrptDB != nil { - err := tlsrptDB.Close() + if DB != nil { + err := DB.Close() xlog.Check(err, "closing database") - tlsrptDB = nil + DB = nil } } diff --git a/verifydata.go b/verifydata.go new file mode 100644 index 0000000..a51a732 --- /dev/null +++ b/verifydata.go @@ -0,0 +1,347 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "strconv" + "strings" + + bolt "go.etcd.io/bbolt" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/dmarcdb" + "github.com/mjl-/mox/junk" + "github.com/mjl-/mox/moxvar" + "github.com/mjl-/mox/mtastsdb" + "github.com/mjl-/mox/queue" + "github.com/mjl-/mox/store" + "github.com/mjl-/mox/tlsrptdb" +) + +func cmdVerifydata(c *cmd) { + c.params = "data-dir" + c.help = `Verify the contents of a data directory, typically of a backup. + +Verifydata checks all database files to see if they are valid BoltDB/bstore +databases. It checks that all messages in the database have a corresponding +on-disk message file and there are no unrecognized files. If option -fix is +specified, unrecognized message files are moved away. This may be needed after +a restore, because messages enqueued or delivered in the future may get those +message sequence numbers assigned and writing the message file would fail. + +Because verifydata opens the database files, schema upgrades may automatically +be applied. This can happen if you use a new mox release. It is useful to run +"mox verifydata" with a new binary before attempting an upgrade, but only on a +copy of the database files, as made with "mox backup". Before upgrading, make a +new backup again since "mox verifydata" may have upgraded the database files, +possibly making them potentially no longer readable by the previous version. +` + var fix bool + c.flag.BoolVar(&fix, "fix", false, "fix fixable problems, such as moving away message files not referenced by their database") + args := c.Parse() + if len(args) != 1 { + c.Usage() + } + + dataDir := filepath.Clean(args[0]) + + ctxbg := context.Background() + + // Check whether file exists, or rather, that it doesn't not exist. Other errors + // will return true as well, so the triggered check can give the details. + exists := func(path string) bool { + _, err := os.Stat(path) + return err == nil || !os.IsNotExist(err) + } + + // Check for error. If so, write a log line, including the path, and set fail so we + // can warn at the end. + var fail bool + checkf := func(err error, path, format string, args ...any) { + if err == nil { + return + } + fail = true + log.Printf("error: %s: %s: %v", path, fmt.Sprintf(format, args...), err) + } + + // When we fix problems, we may have to move files/dirs. We need to ensure the + // directory of the destination path exists before we move. We keep track of + // created dirs so we don't try to create the same directory all the time. + createdDirs := map[string]struct{}{} + ensureDir := func(path string) { + dir := filepath.Dir(path) + if _, ok := createdDirs[dir]; ok { + return + } + err := os.MkdirAll(dir, 0770) + checkf(err, dir, "creating directory") + createdDirs[dir] = struct{}{} + } + + // Check a database file by opening it with BoltDB and bstore and lightly checking + // its contents. + checkDB := func(path string, types []any) { + _, err := os.Stat(path) + checkf(err, path, "checking if file exists") + if err != nil { + return + } + bdb, err := bolt.Open(path, 0600, nil) + checkf(err, path, "open database with bolt") + if err != nil { + return + } + // Check BoltDB consistency. + err = bdb.View(func(tx *bolt.Tx) error { + for err := range tx.Check() { + checkf(err, path, "bolt database problem") + } + return nil + }) + checkf(err, path, "reading bolt database") + bdb.Close() + + db, err := bstore.Open(ctxbg, path, nil, types...) + checkf(err, path, "open database with bstore") + if err != nil { + return + } + defer db.Close() + + err = db.Read(ctxbg, func(tx *bstore.Tx) error { + // Check bstore consistency, if it can export all records for all types. This is a + // quick way to get bstore to parse all records. + types, err := tx.Types() + checkf(err, path, "getting bstore types from database") + if err != nil { + return nil + } + for _, t := range types { + var fields []string + err := tx.Records(t, &fields, func(m map[string]any) error { + return nil + }) + checkf(err, path, "parsing record for type %q", t) + } + return nil + }) + checkf(err, path, "checking database file") + } + + checkFile := func(path string) { + _, err := os.Stat(path) + checkf(err, path, "checking if file exists") + } + + checkQueue := func() { + dbpath := filepath.Join(dataDir, "queue/index.db") + checkDB(dbpath, queue.DBTypes) + + // Check that all messages present in the database also exist on disk. + seen := map[string]struct{}{} + db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, queue.DBTypes...) + checkf(err, dbpath, "opening queue database to check messages") + if err == nil { + err := bstore.QueryDB[queue.Msg](ctxbg, db).ForEach(func(m queue.Msg) error { + mp := store.MessagePath(m.ID) + seen[mp] = struct{}{} + p := filepath.Join(dataDir, "queue", mp) + checkFile(p) + return nil + }) + checkf(err, dbpath, "reading messages in queue database to check files") + } + + // Check that there are no files that could be treated as a message. + qdir := filepath.Join(dataDir, "queue") + err = filepath.WalkDir(qdir, func(qpath string, d fs.DirEntry, err error) error { + checkf(err, qpath, "walk") + if err != nil { + return nil + } + if d.IsDir() { + return nil + } + p := qpath[len(qdir)+1:] + if p == "index.db" { + return nil + } + if _, ok := seen[p]; ok { + return nil + } + l := strings.Split(p, string(filepath.Separator)) + if len(l) == 1 { + log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath) + return nil + } + // If it doesn't look like a message number, there is no risk of it being the name + // of a message enqueued in the future. + if len(l) >= 3 { + if _, err := strconv.ParseInt(l[1], 10, 64); err != nil { + log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath) + return nil + } + } + if !fix { + checkf(errors.New("may interfere with messages enqueued in the future"), qpath, "unrecognized file in queue directory (use the -fix flag to move it away)") + return nil + } + npath := filepath.Join(dataDir, "moved", "queue", p) + ensureDir(npath) + err = os.Rename(qpath, npath) + checkf(err, qpath, "moving queue message file away") + if err == nil { + log.Printf("warning: moved %s to %s", qpath, npath) + } + return nil + }) + checkf(err, qdir, "walking queue directory") + } + + // Check an account, with its database file and messages. + checkAccount := func(name string) { + accdir := filepath.Join(dataDir, "accounts", name) + checkDB(filepath.Join(accdir, "index.db"), store.DBTypes) + + jfdbpath := filepath.Join(accdir, "junkfilter.db") + jfbloompath := filepath.Join(accdir, "junkfilter.bloom") + if exists(jfdbpath) || exists(jfbloompath) { + checkDB(jfdbpath, junk.DBTypes) + } + // todo: add some kind of check for the bloom filter? + + // Check that all messages in the database have a message file on disk. + seen := map[string]struct{}{} + dbpath := filepath.Join(accdir, "index.db") + db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, store.DBTypes...) + checkf(err, dbpath, "opening account database to check messages") + if err == nil { + err := bstore.QueryDB[store.Message](ctxbg, db).ForEach(func(m store.Message) error { + mp := store.MessagePath(m.ID) + seen[mp] = struct{}{} + p := filepath.Join(accdir, "msg", mp) + checkFile(p) + return nil + }) + checkf(err, dbpath, "reading messages in account database to check files") + } + + // Walk through all files in the msg directory. Warn about files that weren't in + // the database as message file. Possibly move away files that could cause trouble. + msgdir := filepath.Join(accdir, "msg") + if !exists(msgdir) { + // New accounts with messages don't have a msg directory. + return + } + err = filepath.WalkDir(msgdir, func(msgpath string, d fs.DirEntry, err error) error { + checkf(err, msgpath, "walk") + if err != nil { + return nil + } + if d.IsDir() { + return nil + } + p := msgpath[len(msgdir)+1:] + if _, ok := seen[p]; ok { + return nil + } + l := strings.Split(p, string(filepath.Separator)) + if len(l) == 1 { + log.Printf("warning: %s: unrecognized file in message directory, ignoring", msgpath) + return nil + } + if !fix { + checkf(errors.New("may interfere with future account messages"), msgpath, "unrecognized file in account message directory (use the -fix flag to move it away)") + return nil + } + npath := filepath.Join(dataDir, "moved", "accounts", name, "msg", p) + ensureDir(npath) + err = os.Rename(msgpath, npath) + checkf(err, msgpath, "moving account message file away") + if err == nil { + log.Printf("warning: moved %s to %s", msgpath, npath) + } + return nil + }) + checkf(err, msgdir, "walking account message directory") + } + + // Check everything in the "accounts" directory. + checkAccounts := func() { + accountsDir := filepath.Join(dataDir, "accounts") + entries, err := os.ReadDir(accountsDir) + checkf(err, accountsDir, "reading accounts directory") + for _, e := range entries { + // We treat all directories as accounts. When we were backing up, we only verified + // accounts from the config and made regular file copies of all other files + // (perhaps an old account, but at least not with an open database file). It may + // turn out that that account was/is not valid, generating warnings. Better safe + // than sorry. It should hopefully get the admin to move away such an old account. + if e.IsDir() { + checkAccount(e.Name()) + } else { + log.Printf("warning: %s: unrecognized file in accounts directory, ignoring", filepath.Join("accounts", e.Name())) + } + } + } + + // Check all files, skipping the known files, queue and accounts directories. Warn + // about unknown files. Skip a "tmp" directory. And a "moved" directory, we + // probably created it ourselves. + backupmoxversion := "(unknown)" + checkOther := func() { + err := filepath.WalkDir(dataDir, func(dpath string, d fs.DirEntry, err error) error { + checkf(err, dpath, "walk") + if err != nil { + return nil + } + if dpath == dataDir { + return nil + } + p := dpath + if dataDir != "." { + p = p[len(dataDir)+1:] + } + switch p { + case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion": + return nil + case "acme", "queue", "accounts", "tmp", "moved": + return fs.SkipDir + case "moxversion": + buf, err := os.ReadFile(dpath) + checkf(err, dpath, "reading moxversion") + if err == nil { + backupmoxversion = string(buf) + } + return nil + } + log.Printf("warning: %s: unrecognized other file, ignoring", dpath) + return nil + }) + checkf(err, dataDir, "walking data directory") + } + + checkDB(filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.DBTypes) + checkDB(filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes) + checkDB(filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes) + checkQueue() + checkAccounts() + checkOther() + + if fail { + log.Fatalf("errors were found") + } else { + log.Printf("%s: OK", dataDir) + } + + if backupmoxversion != moxvar.Version { + log.Printf("NOTE: The backup was made with mox version %q, while verifydata was run with mox version %q. Database files have probably been modified by running mox verifydata. Make a fresh backup before upgrading.", backupmoxversion, moxvar.Version) + } +}