serv/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2024-12-26 00:13:47 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

add a "backup" subcommand to make consistent backups, and a "verifydata" subcommand to verify a backup before restoring, and add tests for future upgrades

Browse source 
the backup command will make consistent snapshots of all the database files. i
had been copying the db files before, and it usually works. but if the file is
modified during the backup, it is inconsistent and is likely to generate errors
when reading (can be at any moment in the future, when reading some db page).
"mox backup" opens the database file and writes out a copy in a transaction.
it also duplicates the message files.

before doing a restore, you could run "mox verifydata" on the to-be-restored
"data" directory. it check the database files, and compares the message files
with the database.

the new "gentestdata" subcommand generates a basic "data" directory, with a
queue and a few accounts. we will use it in the future along with "verifydata"
to test upgrades from old version to the latest version. both when going to the
next version, and when skipping several versions. the script test-upgrades.sh
executes these tests and doesn't do anything at the moment, because no releases
have this subcommand yet.

inspired by a failed upgrade attempt of a pre-release version.
This commit is contained in:
Mechiel Lukkien 2023-05-26 19:26:51 +02:00
parent 753ec56b3d
commit aad5a5bcb9
No known key found for this signature in database
22 changed files with 1568 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Makefile
View file

@ -18,6 +18,9 @@ test-race:
CGO_ENABLED=1 go test -race -shuffle=on -covermode atomic -coverprofile cover.out ./...
go tool cover -html=cover.out -o cover.html
test-upgrade:
./test-upgrade.sh
check:
staticcheck ./...
staticcheck -tags integration

31
README.md
View file

@ -282,3 +282,34 @@ Mox also has an "admin" web interface where the mox instance administrator can
make changes, e.g. add/remove/modify domains/accounts/addresses.
Mox does not have a webmail yet, so there are no screenshots of actual email.
## How do I upgrade my mox installation?
We try to make upgrades effortless and you can typically just put a new binary
in place and restart. If manual actions are required, the release notes mention
them. Check the release notes of all version between your current installation
and the release you're upgrading to.
Before upgrading, make a backup of the data directory with `mox backup
<destdir>`. This writes consistent snapshots of the database files, and
duplicates message files from the queue and accounts. Using the new mox
binary, run `mox verifydata <backupdir>` (do NOT use the "live" data
directory!) for a dry run. If this fails, an upgrade will probably fail too.
Important: verifydata with the new mox binary can modify the database files
(due to automatic schema upgrades). So make a fresh backup again before the
actual upgrade. See the help output of the "backup" and "verifydata" commands
for more details.
During backup, message files are hardlinked if possible. Using a destination
directory like `data/tmp/backup` increases the odds hardlinking succeeds: the
default systemd service file specifically mounts the data directory, causing
attempts to outside it to fail with an error about cross-device linking.
If an upgrade fails and you have to restore (parts) of the data directory, you
should run `mox verifydata <datadir>` (with the original binary) on the
restored directory before starting mox again. If problematic files are found,
for example queue or account message files that are not in the database, run
`mox verifydata -fix <datadir>` to move away those files. After a restore, you may
also want to run `mox bumpuidvalidity <account>` for each account for which
messages in a mailbox changed, to force IMAP clients to synchronize mailbox
state.

556
backup.go Normal file
View file

@ -0,0 +1,556 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dmarcdb"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/mtastsdb"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/store"
"github.com/mjl-/mox/tlsrptdb"
)
func backupctl(ctx context.Context, ctl *ctl) {
/* protocol:
> "backup"
> destdir
> "verbose" or ""
< stream
< "ok" or error
*/
// Convention in this function: variables containing "src" or "dst" are file system
// paths that can be passed to os.Open and such. Variables with dirs/paths without
// "src" or "dst" are incomplete paths relative to the source or destination data
// directories.
dstDataDir := ctl.xread()
verbose := ctl.xread() == "verbose"
// Set when an error is encountered. At the end, we warn if set.
var incomplete bool
// We'll be writing output, and logging both to mox and the ctl stream.
writer := ctl.writer()
// Format easily readable output for the user.
formatLog := func(prefix, text string, err error, fields ...mlog.Pair) []byte {
var b bytes.Buffer
fmt.Fprint(&b, prefix)
fmt.Fprint(&b, text)
if err != nil {
fmt.Fprint(&b, ": "+err.Error())
}
for _, f := range fields {
fmt.Fprintf(&b, "; %s=%v", f.Key, f.Value)
}
fmt.Fprint(&b, "\n")
return b.Bytes()
}
// Log an error to both the mox service as the user running "mox backup".
xlogx := func(prefix, text string, err error, fields ...mlog.Pair) {
ctl.log.Errorx(text, err, fields...)
_, werr := writer.Write(formatLog(prefix, text, err, fields...))
ctl.xcheck(werr, "write to ctl")
}
// Log an error but don't mark backup as failed.
xwarnx := func(text string, err error, fields ...mlog.Pair) {
xlogx("warning: ", text, err, fields...)
}
// Log an error that causes the backup to be marked as failed. We typically
// continue processing though.
xerrx := func(text string, err error, fields ...mlog.Pair) {
incomplete = true
xlogx("error: ", text, err, fields...)
}
// If verbose is enabled, log to the cli command. Always log as info level.
xvlog := func(text string, fields ...mlog.Pair) {
ctl.log.Info(text, fields...)
if verbose {
_, werr := writer.Write(formatLog("", text, nil, fields...))
ctl.xcheck(werr, "write to ctl")
}
}
if _, err := os.Stat(dstDataDir); err == nil {
xwarnx("destination data directory already exists", nil, mlog.Field("dir", dstDataDir))
}
srcDataDir := filepath.Clean(mox.DataDirPath("."))
// When creating a file in the destination, we first ensure its directory exists.
// We track which directories we created, to prevent needless syscalls.
createdDirs := map[string]struct{}{}
ensureDestDir := func(dstpath string) {
dstdir := filepath.Dir(dstpath)
if _, ok := createdDirs[dstdir]; !ok {
err := os.MkdirAll(dstdir, 0770)
if err != nil {
xerrx("creating directory", err)
}
createdDirs[dstdir] = struct{}{}
}
}
// Backup a single file by copying (never hardlinking, the file may change).
backupFile := func(path string) {
tmFile := time.Now()
srcpath := filepath.Join(srcDataDir, path)
dstpath := filepath.Join(dstDataDir, path)
sf, err := os.Open(srcpath)
if err != nil {
xerrx("open source file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
return
}
defer sf.Close()
ensureDestDir(dstpath)
df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
if err != nil {
xerrx("creating destination file (not backed up)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
return
}
defer func() {
if df != nil {
df.Close()
}
}()
if _, err := io.Copy(df, sf); err != nil {
xerrx("copying file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
return
}
err = df.Close()
df = nil
if err != nil {
xerrx("closing destination file (not backed up properly)", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
return
}
xvlog("backed up file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmFile)))
}
// Back up the files in a directory (by copying).
backupDir := func(dir string) {
tmDir := time.Now()
srcdir := filepath.Join(srcDataDir, dir)
dstdir := filepath.Join(dstDataDir, dir)
err := filepath.WalkDir(srcdir, func(srcpath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking file (not backed up)", err, mlog.Field("srcpath", srcpath))
return nil
}
if d.IsDir() {
return nil
}
backupFile(srcpath[len(srcDataDir)+1:])
return nil
})
if err != nil {
xerrx("copying directory (not backed up properly)", err, mlog.Field("srcdir", srcdir), mlog.Field("dstdir", dstdir), mlog.Field("duration", time.Since(tmDir)))
return
}
xvlog("backed up directory", mlog.Field("dir", dir), mlog.Field("duration", time.Since(tmDir)))
}
// Backup a database by copying it in a readonly transaction.
// Always logs on error, so caller doesn't have to, but also returns the error so
// callers can see result.
backupDB := func(db *bstore.DB, path string) (rerr error) {
defer func() {
if rerr != nil {
xerrx("backing up database", rerr, mlog.Field("path", path))
}
}()
tmDB := time.Now()
dstpath := filepath.Join(dstDataDir, path)
ensureDestDir(dstpath)
df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
if err != nil {
return fmt.Errorf("creating destination file: %v", err)
}
defer func() {
if df != nil {
df.Close()
}
}()
err = db.Read(ctx, func(tx *bstore.Tx) error {
// Using regular WriteTo seems fine, and fast. It just copies pages.
//
// bolt.Compact is slower, it writes all key/value pairs, building up new data
// structures. My compacted test database was ~60% of original size. Lz4 on the
// uncompacted database got it to 14%. Lz4 on the compacted database got it to 13%.
// Backups are likely archived somewhere with compression, so we don't compact.
//
// Tests with WriteTo and os.O_DIRECT were slower than without O_DIRECT, but
// probably because everything fit in the page cache. It may be better to use
// O_DIRECT when copying many large or inactive databases.
_, err := tx.WriteTo(df)
return err
})
if err != nil {
return fmt.Errorf("copying database: %v", err)
}
err = df.Close()
df = nil
if err != nil {
return fmt.Errorf("closing destination database after copy: %v", err)
}
xvlog("backed up database file", mlog.Field("path", path), mlog.Field("duration", time.Since(tmDB)))
return nil
}
// Try to create a hardlink. Fall back to copying the file (e.g. when on different file system).
warnedHardlink := false // We warn once about failing to hardlink.
linkOrCopy := func(srcpath, dstpath string) (bool, error) {
ensureDestDir(dstpath)
if err := os.Link(srcpath, dstpath); err == nil {
return true, nil
} else if os.IsNotExist(err) {
// No point in trying with regular copy, we would warn twice.
return false, err
} else if !warnedHardlink {
xwarnx("creating hardlink to message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
warnedHardlink = true
}
// Fall back to copying.
sf, err := os.Open(srcpath)
if err != nil {
return false, fmt.Errorf("open source path %s: %v", srcpath, err)
}
defer sf.Close()
df, err := os.OpenFile(dstpath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
if err != nil {
return false, fmt.Errorf("open destination path %s: %v", dstpath, err)
}
defer func() {
if df != nil {
df.Close()
}
}()
if _, err := io.Copy(df, sf); err != nil {
return false, fmt.Errorf("coping: %v", err)
}
err = df.Close()
df = nil
if err != nil {
return false, fmt.Errorf("close: %v", err)
}
return false, nil
}
// Start making the backup.
tmStart := time.Now()
ctl.log.Print("making backup", mlog.Field("destdir", dstDataDir))
err := os.MkdirAll(dstDataDir, 0770)
if err != nil {
xerrx("creating destination data directory", err)
}
if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil {
xerrx("writing moxversion", err)
}
backupDB(dmarcdb.DB, "dmarcrpt.db")
backupDB(mtastsdb.DB, "mtasts.db")
backupDB(tlsrptdb.DB, "tlsrpt.db")
backupFile("receivedid.key")
// Acme directory is optional.
srcAcmeDir := filepath.Join(srcDataDir, "acme")
if _, err := os.Stat(srcAcmeDir); err == nil {
backupDir("acme")
} else if err != nil && !os.IsNotExist(err) {
xerrx("copying acme/", err)
}
// Copy the queue database and all message files.
backupQueue := func(path string) {
tmQueue := time.Now()
if err := backupDB(queue.DB, path); err != nil {
xerrx("queue not backed up", err, mlog.Field("path", path), mlog.Field("duration", time.Since(tmQueue)))
return
}
dstdbpath := filepath.Join(dstDataDir, path)
db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, queue.DBTypes...)
if err != nil {
xerrx("open copied queue database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmQueue)))
return
}
defer func() {
if db != nil {
err := db.Close()
ctl.log.Check(err, "closing new queue db")
}
}()
// Link/copy known message files. Warn if files are missing or unexpected
// (though a message file could have been removed just now due to delivery, or a
// new message may have been queued).
tmMsgs := time.Now()
seen := map[string]struct{}{}
var nlinked, ncopied int
err = bstore.QueryDB[queue.Msg](ctx, db).ForEach(func(m queue.Msg) error {
mp := store.MessagePath(m.ID)
seen[mp] = struct{}{}
srcpath := filepath.Join(srcDataDir, "queue", mp)
dstpath := filepath.Join(dstDataDir, "queue", mp)
if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
xerrx("linking/copying queue message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
} else if linked {
nlinked++
} else {
ncopied++
}
return nil
})
if err != nil {
xerrx("processing queue messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs)))
} else {
xvlog("queue message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs)))
}
// Read through all files in queue directory and warn about anything we haven't handled yet.
tmWalk := time.Now()
srcqdir := filepath.Join(srcDataDir, "queue")
err = filepath.WalkDir(srcqdir, func(srcqpath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking files in queue", err, mlog.Field("srcpath", srcqpath))
return nil
}
if d.IsDir() {
return nil
}
p := srcqpath[len(srcqdir)+1:]
if _, ok := seen[p]; ok {
return nil
}
if p == "index.db" {
return nil
}
qp := filepath.Join("queue", p)
xwarnx("backing up unrecognized file in queue directory", nil, mlog.Field("path", qp))
backupFile(qp)
return nil
})
if err != nil {
xerrx("walking queue directory (not backed up properly)", err, mlog.Field("dir", "queue"), mlog.Field("duration", time.Since(tmWalk)))
} else {
xvlog("walked queue directory", mlog.Field("duration", time.Since(tmWalk)))
}
xvlog("queue backed finished", mlog.Field("duration", time.Since(tmQueue)))
}
backupQueue("queue/index.db")
backupAccount := func(acc *store.Account) {
defer acc.Close()
tmAccount := time.Now()
// Copy database file.
dbpath := filepath.Join("accounts", acc.Name, "index.db")
err := backupDB(acc.DB, dbpath)
if err != nil {
xerrx("copying account database", err, mlog.Field("path", dbpath), mlog.Field("duration", time.Since(tmAccount)))
}
// todo: should document/check not taking a rlock on account.
// Copy junkfilter files, if configured.
if jf, _, err := acc.OpenJunkFilter(ctx, ctl.log); err != nil {
if !errors.Is(err, store.ErrNoJunkFilter) {
xerrx("opening junk filter for account (not backed up)", err)
}
} else {
db := jf.DB()
jfpath := filepath.Join("accounts", acc.Name, "junkfilter.db")
backupDB(db, jfpath)
bloompath := filepath.Join("accounts", acc.Name, "junkfilter.bloom")
backupFile(bloompath)
db = nil
err := jf.Close()
ctl.log.Check(err, "closing junkfilter")
}
dstdbpath := filepath.Join(dstDataDir, dbpath)
db, err := bstore.Open(ctx, dstdbpath, &bstore.Options{MustExist: true}, store.DBTypes...)
if err != nil {
xerrx("open copied account database", err, mlog.Field("dstpath", dstdbpath), mlog.Field("duration", time.Since(tmAccount)))
return
}
defer func() {
if db != nil {
err := db.Close()
ctl.log.Check(err, "close account database")
}
}()
// Link/copy known message files. Warn if files are missing or unexpected (though a
// message file could have been added just now due to delivery, or a message have
// been removed).
tmMsgs := time.Now()
seen := map[string]struct{}{}
var nlinked, ncopied int
err = bstore.QueryDB[store.Message](ctx, db).ForEach(func(m store.Message) error {
mp := store.MessagePath(m.ID)
seen[mp] = struct{}{}
amp := filepath.Join("accounts", acc.Name, "msg", mp)
srcpath := filepath.Join(srcDataDir, amp)
dstpath := filepath.Join(dstDataDir, amp)
if linked, err := linkOrCopy(srcpath, dstpath); err != nil {
xerrx("linking/copying account message", err, mlog.Field("srcpath", srcpath), mlog.Field("dstpath", dstpath))
} else if linked {
nlinked++
} else {
ncopied++
}
return nil
})
if err != nil {
xerrx("processing account messages (not backed up properly)", err, mlog.Field("duration", time.Since(tmMsgs)))
} else {
xvlog("account message files linked/copied", mlog.Field("linked", nlinked), mlog.Field("copied", ncopied), mlog.Field("duration", time.Since(tmMsgs)))
}
// Read through all files in account directory and warn about anything we haven't handled yet.
tmWalk := time.Now()
srcadir := filepath.Join(srcDataDir, "accounts", acc.Name)
err = filepath.WalkDir(srcadir, func(srcapath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking files in account", err, mlog.Field("srcpath", srcapath))
return nil
}
if d.IsDir() {
return nil
}
p := srcapath[len(srcadir)+1:]
l := strings.Split(p, string(filepath.Separator))
if l[0] == "msg" {
mp := filepath.Join(l[1:]...)
if _, ok := seen[mp]; ok {
return nil
}
}
switch p {
case "index.db", "junkfilter.db", "junkfilter.bloom":
return nil
}
ap := filepath.Join("accounts", acc.Name, p)
if strings.HasPrefix(p, "msg/") {
xwarnx("backing up unrecognized file in account message directory (should be moved away)", nil, mlog.Field("path", ap))
} else {
xwarnx("backing up unrecognized file in account directory", nil, mlog.Field("path", ap))
}
backupFile(ap)
return nil
})
if err != nil {
xerrx("walking account directory (not backed up properly)", err, mlog.Field("srcdir", srcadir), mlog.Field("duration", time.Since(tmWalk)))
} else {
xvlog("walked account directory", mlog.Field("duration", time.Since(tmWalk)))
}
xvlog("account backup finished", mlog.Field("dir", filepath.Join("accounts", acc.Name)), mlog.Field("duration", time.Since(tmAccount)))
}
// For each configured account, open it, make a copy of the database and
// hardlink/copy the messages. We track the accounts we handled, and skip the
// account directories when handling "all other files" below.
accounts := map[string]struct{}{}
for _, accName := range mox.Conf.Accounts() {
acc, err := store.OpenAccount(accName)
if err != nil {
xerrx("opening account for copying (will try to copy as regular files later)", err, mlog.Field("account", accName))
continue
}
accounts[accName] = struct{}{}
backupAccount(acc)
}
// Copy all other files, that aren't part of the known files, databases, queue or accounts.
tmWalk := time.Now()
err = filepath.WalkDir(srcDataDir, func(srcpath string, d fs.DirEntry, err error) error {
if err != nil {
xerrx("walking path", err, mlog.Field("path", srcpath))
return nil
}
if srcpath == srcDataDir {
return nil
}
p := srcpath[len(srcDataDir)+1:]
if p == "queue" || p == "acme" || p == "tmp" {
if p == "tmp" {
xwarnx("skipping entire tmp directory", nil, mlog.Field("path", p))
}
return fs.SkipDir
}
l := strings.Split(p, string(filepath.Separator))
if len(l) >= 2 && l[0] == "accounts" {
name := l[1]
if _, ok := accounts[name]; ok {
return fs.SkipDir
}
}
// Only files are explicitly backed up.
if d.IsDir() {
return nil
}
switch p {
case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
// Already handled.
return nil
case "lastknownversion": // Optional file, not yet handled.
default:
xwarnx("backing up unrecognized file", nil, mlog.Field("path", p))
}
backupFile(p)
return nil
})
if err != nil {
xerrx("walking other files (not backed up properly)", err, mlog.Field("duration", time.Since(tmWalk)))
} else {
xvlog("walking other files finished", mlog.Field("duration", time.Since(tmWalk)))
}
xvlog("backup finished", mlog.Field("duration", time.Since(tmStart)))
writer.xclose()
if incomplete {
ctl.xwrite("errors were encountered during backup")
} else {
ctl.xwriteok()
}
}

35
ctl.go
View file

@ -30,6 +30,7 @@ import (
// ctl represents a connection to the ctl unix domain socket of a running mox instance.
// ctl provides functions to read/write commands/responses/data streams.
type ctl struct {
cmd string // Set for server-side of commands.
conn net.Conn
r *bufio.Reader // Set for first reader.
x any // If set, errors are handled by calling panic(x) instead of log.Fatal.
@ -57,7 +58,7 @@ func (c *ctl) xerror(msg string) {
if c.x == nil {
log.Fatalln(msg)
}
c.log.Debugx("ctl error", fmt.Errorf("%s", msg))
c.log.Debugx("ctl error", fmt.Errorf("%s", msg), mlog.Field("cmd", c.cmd))
c.xwrite(msg)
panic(c.x)
}
@ -72,7 +73,7 @@ func (c *ctl) xcheck(err error, msg string) {
if c.x == nil {
log.Fatalf("%s: %s", msg, err)
}
c.log.Debugx(msg, err)
c.log.Debugx(msg, err, mlog.Field("cmd", c.cmd))
fmt.Fprintf(c.conn, "%s: %s\n", msg, err)
panic(c.x)
}
@ -124,7 +125,7 @@ func (c *ctl) xstreamfrom(src io.Reader) {
// When done writing, caller must call xclose to signal the end of the stream.
// Behaviour of "x" is copied from ctl.
func (c *ctl) writer() *ctlwriter {
return &ctlwriter{conn: c.conn, x: c.x, log: c.log}
return &ctlwriter{cmd: c.cmd, conn: c.conn, x: c.x, log: c.log}
}
// Reader returns an io.Reader for a data stream from ctl.
@ -133,7 +134,7 @@ func (c *ctl) reader() *ctlreader {
if c.r == nil {
c.r = bufio.NewReader(c.conn)
}
return &ctlreader{conn: c.conn, r: c.r, x: c.x, log: c.log}
return &ctlreader{cmd: c.cmd, conn: c.conn, r: c.r, x: c.x, log: c.log}
}
/*
@ -154,6 +155,7 @@ Followed by a end of stream indicated by zero data bytes message:
*/
type ctlwriter struct {
cmd string // Set for server-side of commands.
conn net.Conn // Ctl socket from which messages are read.
buf []byte // Scratch buffer, for reading response.
x any // If not nil, errors in Write and xcheckf are handled with panic(x), otherwise with a log.Fatal.
@ -181,7 +183,7 @@ func (s *ctlwriter) xerror(msg string) {
if s.x == nil {
log.Fatalln(msg)
} else {
s.log.Debugx("error", fmt.Errorf("%s", msg))
s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd))
panic(s.x)
}
}
@ -193,7 +195,7 @@ func (s *ctlwriter) xcheck(err error, msg string) {
if s.x == nil {
log.Fatalf("%s: %s", msg, err)
} else {
s.log.Debugx(msg, err)
s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd))
panic(s.x)
}
}
@ -204,6 +206,7 @@ func (s *ctlwriter) xclose() {
}
type ctlreader struct {
cmd string // Set for server-side of command.
conn net.Conn // For writing "ok" after reading.
r *bufio.Reader // Buffered ctl socket.
err error // If set, returned for each read. can also be io.EOF.
@ -246,7 +249,7 @@ func (s *ctlreader) xerror(msg string) {
if s.x == nil {
log.Fatalln(msg)
} else {
s.log.Debugx("error", fmt.Errorf("%s", msg))
s.log.Debugx("error", fmt.Errorf("%s", msg), mlog.Field("cmd", s.cmd))
panic(s.x)
}
}
@ -258,7 +261,7 @@ func (s *ctlreader) xcheck(err error, msg string) {
if s.x == nil {
log.Fatalf("%s: %s", msg, err)
} else {
s.log.Debugx(msg, err)
s.log.Debugx(msg, err, mlog.Field("cmd", s.cmd))
panic(s.x)
}
}
@ -267,33 +270,30 @@ func (s *ctlreader) xcheck(err error, msg string) {
func servectl(ctx context.Context, log *mlog.Log, conn net.Conn, shutdown func()) {
log.Debug("ctl connection")
var cmd string
var stop = struct{}{} // Sentinel value for panic and recover.
ctl := &ctl{conn: conn, x: stop, log: log}
defer func() {
x := recover()
if x == nil || x == stop {
return
}
log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", cmd))
log.Error("servectl panic", mlog.Field("err", x), mlog.Field("cmd", ctl.cmd))
debug.PrintStack()
metrics.PanicInc("ctl")
}()
defer conn.Close()
ctl := &ctl{conn: conn, x: stop, log: log}
ctl.xwrite("ctlv0")
for {
servectlcmd(ctx, log, ctl, &cmd, shutdown)
servectlcmd(ctx, log, ctl, shutdown)
}
}
func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shutdown func()) {
func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, shutdown func()) {
cmd := ctl.xread()
ctl.cmd = cmd
log.Info("ctl command", mlog.Field("cmd", cmd))
*xcmd = cmd
switch cmd {
case "stop":
shutdown()
@ -641,6 +641,9 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
ctl.xwriteok()
case "backup":
backupctl(ctx, ctl)
default:
log.Info("unrecognized command", mlog.Field("cmd", cmd))
ctl.xwrite("unrecognized command")

1
develop.txt
View file

@ -167,6 +167,7 @@ testing purposes.
- Write release notes, use instructions from updating.txt.
- Build and run tests with previous major Go release.
- Run all (integration) tests, including with race detector.
- Test upgrades.
- Run fuzzing tests for a while.
- Deploy to test environment. Test the update instructions.
- Generate a config with quickstart, check if it results in a working setup.

11
dmarcdb/db.go
View file

@ -28,7 +28,8 @@ import (
var xlog = mlog.New("dmarcdb")
var (
dmarcDB *bstore.DB
DBTypes = []any{DomainFeedback{}} // Types stored in DB.
DB *bstore.DB // Exported for backups.
mutex sync.Mutex
)
@ -70,16 +71,16 @@ type DomainFeedback struct {
func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
mutex.Lock()
defer mutex.Unlock()
if dmarcDB == nil {
if DB == nil {
p := mox.DataDirPath("dmarcrpt.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DomainFeedback{})
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
if err != nil {
return nil, err
}
dmarcDB = db
DB = db
}
return dmarcDB, nil
return DB, nil
}
// Init opens the database.

65
doc.go
View file

@ -29,6 +29,8 @@ low-maintenance self-hosted email.
mox export mbox dst-dir account-path [mailbox]
mox localserve
mox help [command ...]
mox backup dest-dir
mox verifydata data-dir
mox config test
mox config dnscheck domain
mox config dnsrecords domain
@ -331,6 +333,69 @@ If a single command matches, its usage and full help text is printed.
usage: mox help [command ...]
# mox backup
Creates a backup of the data directory.
Backup creates consistent snapshots of the databases and message files and
copies other files in the data directory. Empty directories are not copied.
These files can then be stored elsewhere for long-term storage, or used to fall
back to should an upgrade fail. Simply copying files in the data directory
while mox is running can result in unusable database files.
Message files never change (they are read-only, though can be removed) and are
hardlinked so they don't consume additional space. If hardlinking fails, for
example when the backup destination directory is on a different file system, a
regular copy is made. Using a destination directory like "data/tmp/backup"
increases the odds hardlinking succeeds: the default systemd service file
specifically mounts the data directory, causing attempts to hardlink outside it
to fail with an error about cross-device linking.
All files in the data directory that aren't recognized (i.e. other than known
database files, message files, an acme directory, etc), are stored, but with a
warning.
A clean successful backup does not print any output by default. Use the
-verbose flag for details, including timing.
To restore a backup, first shut down mox, move away the old data directory and
move an earlier backed up directory in its place, run "mox verifydata",
possibly with the "-fix" option, and restart mox. After the restore, you may
also want to run "mox bumpuidvalidity" for each account for which messages in a
mailbox changed, to force IMAP clients to synchronize mailbox state.
Before upgrading, to check if the upgrade will likely succeed, first make a
backup, then use the new mox binary to run "mox verifydata" on the backup. This
can change the backup files (e.g. upgrade database files, move away
unrecognized message files), so you should make a new backup before actually
upgrading.
usage: mox backup dest-dir
-verbose
print progress
# mox verifydata
Verify the contents of a data directory, typically of a backup.
Verifydata checks all database files to see if they are valid BoltDB/bstore
databases. It checks that all messages in the database have a corresponding
on-disk message file and there are no unrecognized files. If option -fix is
specified, unrecognized message files are moved away. This may be needed after
a restore, because messages enqueued or delivered in the future may get those
message sequence numbers assigned and writing the message file would fail.
Because verifydata opens the database files, schema upgrades may automatically
be applied. This can happen if you use a new mox release. It is useful to run
"mox verifydata" with a new binary before attempting an upgrade, but only on a
copy of the database files, as made with "mox backup". Before upgrading, make a
new backup again since "mox verifydata" may have upgraded the database files,
possibly making them potentially no longer readable by the previous version.
usage: mox verifydata data-dir
-fix
fix fixable problems, such as moving away message files not referenced by their database
# mox config test
Parses and validates the configuration files.

2
export.go
View file

@ -57,7 +57,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) {
}
dbpath := filepath.Join(accountDir, "index.db")
db, err := bstore.Open(context.Background(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{})
db, err := bstore.Open(context.Background(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.DBTypes...)
xcheckf(err, "open database %q", dbpath)
defer func() {
if err := db.Close(); err != nil {

361
gentestdata.go Normal file
View file

@ -0,0 +1,361 @@
package main
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/mjl-/bstore"
"github.com/mjl-/sconf"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dmarcdb"
"github.com/mjl-/mox/dmarcrpt"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/mtasts"
"github.com/mjl-/mox/mtastsdb"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/store"
"github.com/mjl-/mox/tlsrpt"
"github.com/mjl-/mox/tlsrptdb"
)
func cmdGentestdata(c *cmd) {
c.unlisted = true
c.params = "dest-dir"
c.help = `Generate a data directory populated, for testing upgrades.`
args := c.Parse()
if len(args) != 1 {
c.Usage()
}
destDataDir, err := filepath.Abs(args[0])
xcheckf(err, "making destination directory an absolute path")
if _, err := os.Stat(destDataDir); err == nil {
log.Fatalf("destination directory already exists, refusing to generate test data")
}
err = os.MkdirAll(destDataDir, 0770)
xcheckf(err, "creating destination data directory")
err = os.MkdirAll(filepath.Join(destDataDir, "tmp"), 0770)
xcheckf(err, "creating tmp directory")
tempfile := func() *os.File {
f, err := os.CreateTemp(filepath.Join(destDataDir, "tmp"), "temp")
xcheckf(err, "creating temp file")
return f
}
ctxbg := context.Background()
mox.Shutdown = ctxbg
mox.Context = ctxbg
mox.Conf.Log[""] = mlog.LevelInfo
mlog.SetConfig(mox.Conf.Log)
const domainsConf = `
Domains:
mox.example: nil
☺.example: nil
Accounts:
test0:
Domain: mox.example
Destinations:
test0@mox.example: nil
test1:
Domain: mox.example
Destinations:
test1@mox.example: nil
test2:
Domain: ☺.example
Destinations:
☹@☺.example: nil
JunkFilter:
Threshold: 0.95
Params:
Twograms: true
MaxPower: 0.1
TopWords: 10
IgnoreWords: 0.1
`
mox.ConfigStaticPath = "/tmp/mox-bogus/mox.conf"
mox.ConfigDynamicPath = "/tmp/mox-bogus/domains.conf"
mox.Conf.DynamicLastCheck = time.Now() // Should prevent warning.
mox.Conf.Static = config.Static{
DataDir: destDataDir,
}
err = sconf.Parse(strings.NewReader(domainsConf), &mox.Conf.Dynamic)
xcheckf(err, "parsing domains config")
const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
<feedback>
<report_metadata>
<org_name>google.com</org_name>
<email>noreply-dmarc-support@google.com</email>
<extra_contact_info>https://support.google.com/a/answer/2466580</extra_contact_info>
<report_id>10051505501689795560</report_id>
<date_range>
<begin>1596412800</begin>
<end>1596499199</end>
</date_range>
</report_metadata>
<policy_published>
<domain>mox.example</domain>
<adkim>r</adkim>
<aspf>r</aspf>
<p>reject</p>
<sp>reject</sp>
<pct>100</pct>
</policy_published>
<record>
<row>
<source_ip>127.0.0.1</source_ip>
<count>1</count>
<policy_evaluated>
<disposition>none</disposition>
<dkim>pass</dkim>
<spf>pass</spf>
</policy_evaluated>
</row>
<identifiers>
<header_from>example.org</header_from>
</identifiers>
<auth_results>
<dkim>
<domain>example.org</domain>
<result>pass</result>
<selector>example</selector>
</dkim>
<spf>
<domain>example.org</domain>
<result>pass</result>
</spf>
</auth_results>
</record>
</feedback>
`
const tlsReport = `{
"organization-name": "Company-X",
"date-range": {
"start-datetime": "2016-04-01T00:00:00Z",
"end-datetime": "2016-04-01T23:59:59Z"
},
"contact-info": "sts-reporting@company-x.example",
"report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
"policies": [{
"policy": {
"policy-type": "sts",
"policy-string": ["version: STSv1","mode: testing",
"mx: *.mail.company-y.example","max_age: 86400"],
"policy-domain": "mox.example",
"mx-host": ["*.mail.company-y.example"]
},
"summary": {
"total-successful-session-count": 5326,
"total-failure-session-count": 303
},
"failure-details": [{
"result-type": "certificate-expired",
"sending-mta-ip": "2001:db8:abcd:0012::1",
"receiving-mx-hostname": "mx1.mail.company-y.example",
"failed-session-count": 100
}, {
"result-type": "starttls-not-supported",
"sending-mta-ip": "2001:db8:abcd:0013::1",
"receiving-mx-hostname": "mx2.mail.company-y.example",
"receiving-ip": "203.0.113.56",
"failed-session-count": 200,
"additional-information": "https://reports.company-x.example/report_info ? id = 5065427 c - 23 d3# StarttlsNotSupported "
}, {
"result-type": "validation-failure",
"sending-mta-ip": "198.51.100.62",
"receiving-ip": "203.0.113.58",
"receiving-mx-hostname": "mx-backup.mail.company-y.example",
"failed-session-count": 3,
"failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
}]
}]
}`
err = os.WriteFile(filepath.Join(destDataDir, "moxversion"), []byte(moxvar.Version), 0660)
xcheckf(err, "writing moxversion")
// Populate dmarc.db.
err = dmarcdb.Init()
xcheckf(err, "dmarcdb init")
report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
xcheckf(err, "parsing dmarc report")
err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
xcheckf(err, "adding dmarc report")
// Populate mtasts.db.
err = mtastsdb.Init(false)
xcheckf(err, "mtastsdb init")
mtastsPolicy := mtasts.Policy{
Version: "STSv1",
Mode: mtasts.ModeTesting,
MX: []mtasts.STSMX{
{Domain: dns.Domain{ASCII: "mx1.example.com"}},
{Domain: dns.Domain{ASCII: "mx2.example.com"}},
{Domain: dns.Domain{ASCII: "backup-example.com"}, Wildcard: true},
},
MaxAgeSeconds: 1296000,
}
err = mtastsdb.Upsert(ctxbg, dns.Domain{ASCII: "mox.example"}, "123", &mtastsPolicy)
xcheckf(err, "adding mtastsdb report")
// Populate tlsrpt.db.
err = tlsrptdb.Init()
xcheckf(err, "tlsrptdb init")
tlsr, err := tlsrpt.Parse(strings.NewReader(tlsReport))
xcheckf(err, "parsing tls report")
err = tlsrptdb.AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", tlsr)
xcheckf(err, "adding tls report")
// Populate queue, with a message.
err = queue.Init()
xcheckf(err, "queue init")
mailfrom := smtp.Path{Localpart: "other", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "other.example"}}}
rcptto := smtp.Path{Localpart: "test0", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
prefix := []byte{}
mf := tempfile()
xcheckf(err, "temp file for queue message")
defer mf.Close()
const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
_, err = fmt.Fprint(mf, qmsg)
xcheckf(err, "writing message")
err = queue.Add(ctxbg, mlog.New("gentestdata"), "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), prefix, mf, nil, true)
xcheckf(err, "enqueue message")
// Create three accounts.
// First account without messages.
accTest0, err := store.OpenAccount("test0")
xcheckf(err, "open account test0")
err = accTest0.Close()
xcheckf(err, "close account")
// Second account with one message.
accTest1, err := store.OpenAccount("test1")
xcheckf(err, "open account test1")
err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
xcheckf(err, "looking up inbox")
const msg = "From: <other@remote.example>\r\nTo: <test1@mox.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
m := store.Message{
MailboxID: inbox.ID,
MailboxOrigID: inbox.ID,
MailboxDestinedID: inbox.ID,
RemoteIP: "1.2.3.4",
RemoteIPMasked1: "1.2.3.4",
RemoteIPMasked2: "1.2.3.0",
RemoteIPMasked3: "1.2.0.0",
EHLODomain: "other.example",
MailFrom: "other@remote.example",
MailFromLocalpart: smtp.Localpart("other"),
MailFromDomain: "remote.example",
RcptToLocalpart: "test1",
RcptToDomain: "mox.example",
MsgFromLocalpart: "other",
MsgFromDomain: "remote.example",
MsgFromOrgDomain: "remote.example",
EHLOValidated: true,
MailFromValidated: true,
MsgFromValidated: true,
EHLOValidation: store.ValidationStrict,
MailFromValidation: store.ValidationPass,
MsgFromValidation: store.ValidationStrict,
DKIMDomains: []string{"other.example"},
Size: int64(len(msg)),
}
mf := tempfile()
xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf, msg)
xcheckf(err, "writing deliver message to file")
err = accTest1.DeliverMessage(mlog.New("gentestdata"), tx, &m, mf, true, false, false, true)
xcheckf(err, "add message to account test1")
err = mf.Close()
xcheckf(err, "closing file")
return nil
})
xcheckf(err, "write transaction with new message")
err = accTest1.Close()
xcheckf(err, "close account")
// Third account with two messages and junkfilter.
accTest2, err := store.OpenAccount("test2")
xcheckf(err, "open account test2")
err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
xcheckf(err, "looking up inbox")
const msg0 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
m0 := store.Message{
MailboxID: inbox.ID,
MailboxOrigID: inbox.ID,
MailboxDestinedID: inbox.ID,
RemoteIP: "::1",
RemoteIPMasked1: "::",
RemoteIPMasked2: "::",
RemoteIPMasked3: "::",
EHLODomain: "other.example",
MailFrom: "other@remote.example",
MailFromLocalpart: smtp.Localpart("other"),
MailFromDomain: "remote.example",
RcptToLocalpart: "☹",
RcptToDomain: "☺.example",
MsgFromLocalpart: "other",
MsgFromDomain: "remote.example",
MsgFromOrgDomain: "remote.example",
EHLOValidated: true,
MailFromValidated: true,
MsgFromValidated: true,
EHLOValidation: store.ValidationStrict,
MailFromValidation: store.ValidationPass,
MsgFromValidation: store.ValidationStrict,
DKIMDomains: []string{"other.example"},
Size: int64(len(msg0)),
}
mf0 := tempfile()
xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf0, msg0)
xcheckf(err, "writing deliver message to file")
err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m0, mf0, true, false, false, false)
xcheckf(err, "add message to account test2")
err = mf0.Close()
xcheckf(err, "closing file")
sent, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Sent"}).Get()
xcheckf(err, "looking up inbox")
const prefix1 = "Extra: test\r\n"
const msg1 = "From: <other@remote.example>\r\nTo: <☹@xn--74h.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
m1 := store.Message{
MailboxID: sent.ID,
MailboxOrigID: sent.ID,
MailboxDestinedID: sent.ID,
Flags: store.Flags{Seen: true, Junk: true},
Size: int64(len(prefix1) + len(msg1)),
MsgPrefix: []byte(prefix),
}
mf1 := tempfile()
xcheckf(err, "creating temp file for delivery")
_, err = fmt.Fprint(mf1, msg1)
xcheckf(err, "writing deliver message to file")
err = accTest2.DeliverMessage(mlog.New("gentestdata"), tx, &m1, mf1, true, true, false, false)
xcheckf(err, "add message to account test2")
err = mf1.Close()
xcheckf(err, "closing file")
return nil
})
xcheckf(err, "write transaction with new message")
err = accTest2.Close()
xcheckf(err, "close account")
}

2
integration_test.go
View file

@ -82,7 +82,7 @@ func TestDeliver(t *testing.T) {
latestMsgID := func(username string) int64 {
// We open the account index database created by mox for the test user. And we keep looking for the email we sent.
dbpath := fmt.Sprintf("testdata/integration/data/accounts/%s/index.db", username)
db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{}, store.Password{})
db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.DBTypes...)
if err != nil && errors.Is(err, bolt.ErrTimeout) {
log.Printf("db open timeout (normal delay for new sender with account and db file kept open)")
return 0

11
junk/filter.go
View file

@ -56,6 +56,8 @@ type Params struct {
RareWords int `sconf:"optional" sconf-doc:"Occurrences in word database until a word is considered rare and its influence in calculating probability reduced. E.g. 1 or 2."`
}
var DBTypes = []any{wordscore{}} // Stored in DB.
type Filter struct {
Params
@ -228,7 +230,7 @@ func newDB(ctx context.Context, log *mlog.Log, path string) (db *bstore.DB, rerr
}
}()
db, err := bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
db, err := bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
if err != nil {
return nil, fmt.Errorf("open new database: %w", err)
}
@ -239,7 +241,7 @@ func openDB(ctx context.Context, path string) (*bstore.DB, error) {
if _, err := os.Stat(path); err != nil {
return nil, fmt.Errorf("stat db file: %w", err)
}
return bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
return bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
}
// Save stores modifications, e.g. from training, to the database and bloom
@ -744,3 +746,8 @@ func (f *Filter) fileSize(p string) int {
}
return int(fi.Size())
}
// DB returns the database, for backups.
func (f *Filter) DB() *bstore.DB {
return f.db
}

64
main.go
View file

@ -88,6 +88,8 @@ var commands = []struct {
{"export mbox", cmdExportMbox},
{"localserve", cmdLocalserve},
{"help", cmdHelp},
{"backup", cmdBackup},
{"verifydata", cmdVerifydata},
{"config test", cmdConfigTest},
{"config dnscheck", cmdConfigDNSCheck},
@ -146,6 +148,7 @@ var commands = []struct {
{"updates pubkey", cmdUpdatesPubkey},
{"updates serve", cmdUpdatesServe},
{"updates verify", cmdUpdatesVerify},
{"gentestdata", cmdGentestdata},
}
var cmds []cmd
@ -961,6 +964,67 @@ new mail deliveries.
fmt.Println("mox stopped")
}
func cmdBackup(c *cmd) {
c.params = "dest-dir"
c.help = `Creates a backup of the data directory.
Backup creates consistent snapshots of the databases and message files and
copies other files in the data directory. Empty directories are not copied.
These files can then be stored elsewhere for long-term storage, or used to fall
back to should an upgrade fail. Simply copying files in the data directory
while mox is running can result in unusable database files.
Message files never change (they are read-only, though can be removed) and are
hardlinked so they don't consume additional space. If hardlinking fails, for
example when the backup destination directory is on a different file system, a
regular copy is made. Using a destination directory like "data/tmp/backup"
increases the odds hardlinking succeeds: the default systemd service file
specifically mounts the data directory, causing attempts to hardlink outside it
to fail with an error about cross-device linking.
All files in the data directory that aren't recognized (i.e. other than known
database files, message files, an acme directory, etc), are stored, but with a
warning.
A clean successful backup does not print any output by default. Use the
-verbose flag for details, including timing.
To restore a backup, first shut down mox, move away the old data directory and
move an earlier backed up directory in its place, run "mox verifydata",
possibly with the "-fix" option, and restart mox. After the restore, you may
also want to run "mox bumpuidvalidity" for each account for which messages in a
mailbox changed, to force IMAP clients to synchronize mailbox state.
Before upgrading, to check if the upgrade will likely succeed, first make a
backup, then use the new mox binary to run "mox verifydata" on the backup. This
can change the backup files (e.g. upgrade database files, move away
unrecognized message files), so you should make a new backup before actually
upgrading.
`
var verbose bool
c.flag.BoolVar(&verbose, "verbose", false, "print progress")
args := c.Parse()
if len(args) != 1 {
c.Usage()
}
mustLoadConfig()
dstDataDir, err := filepath.Abs(args[0])
xcheckf(err, "making path absolute")
ctl := xctl()
ctl.xwrite("backup")
ctl.xwrite(dstDataDir)
if verbose {
ctl.xwrite("verbose")
} else {
ctl.xwrite("")
}
ctl.xstreamto(os.Stdout)
ctl.xreadok()
}
func cmdSetadminpassword(c *cmd) {
c.help = `Set a new admin password, for the web interface.

12
mlog/log.go
View file

@ -96,8 +96,8 @@ func SetConfig(c map[string]Level) {
// Pair is a field/value pair, for use in logged lines.
type Pair struct {
key string
value any
Key string
Value any
}
// Field is a shorthand for making a Pair.
@ -339,7 +339,7 @@ func (l *Log) plog(level Level, err error, text string, fields ...Pair) {
}
for i := 0; i < len(fields); i++ {
kv := fields[i]
fmt.Fprintf(b, " %s=%s", kv.key, logfmtValue(stringValue(kv.key == "cid", false, kv.value)))
fmt.Fprintf(b, " %s=%s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
}
b.WriteString("\n")
} else {
@ -354,7 +354,7 @@ func (l *Log) plog(level Level, err error, text string, fields ...Pair) {
fmt.Fprint(b, "; ")
}
kv := fields[i]
fmt.Fprintf(b, "%s: %s", kv.key, logfmtValue(stringValue(kv.key == "cid", false, kv.value)))
fmt.Fprintf(b, "%s: %s", kv.Key, logfmtValue(stringValue(kv.Key == "cid", false, kv.Value)))
}
fmt.Fprint(b, ")")
}
@ -373,10 +373,10 @@ func (l *Log) match(level Level) (bool, Level) {
seen := false
var high Level
for _, kv := range l.fields {
if kv.key != "pkg" {
if kv.Key != "pkg" {
continue
}
pkg, ok := kv.value.(string)
pkg, ok := kv.Value.(string)
if !ok {
continue
}

17
mtastsdb/db.go
View file

@ -60,22 +60,23 @@ var (
ErrBackoff = errors.New("mtastsdb: policy fetch failed recently")
)
var mtastsDB *bstore.DB
var DBTypes = []any{PolicyRecord{}} // Types stored in DB.
var DB *bstore.DB // Exported for backups.
var mutex sync.Mutex
func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
mutex.Lock()
defer mutex.Unlock()
if mtastsDB == nil {
if DB == nil {
p := mox.DataDirPath("mtasts.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, PolicyRecord{})
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
if err != nil {
return nil, err
}
mtastsDB = db
DB = db
}
return mtastsDB, nil
return DB, nil
}
// Init opens the database and starts a goroutine that refreshes policies in
@ -98,10 +99,10 @@ func Init(refresher bool) error {
func Close() {
mutex.Lock()
defer mutex.Unlock()
if mtastsDB != nil {
err := mtastsDB.Close()
if DB != nil {
err := DB.Close()
xlog.Check(err, "closing database")
mtastsDB = nil
DB = nil
}
}

31
queue/queue.go
View file

@ -70,7 +70,8 @@ var dial = func(ctx context.Context, timeout time.Duration, addr string, laddr n
var jitter = mox.NewRand()
var queueDB *bstore.DB
var DBTypes = []any{Msg{}} // Types stored in DB.
var DB *bstore.DB // Exported for making backups.
// Set for mox localserve, to prevent queueing.
var Localserve bool
@ -122,7 +123,7 @@ func Init() error {
}
var err error
queueDB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, Msg{})
DB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
if err != nil {
if isNew {
os.Remove(qpath)
@ -134,15 +135,15 @@ func Init() error {
// Shutdown closes the queue database. The delivery process isn't stopped. For tests only.
func Shutdown() {
err := queueDB.Close()
err := DB.Close()
xlog.Check(err, "closing queue db")
queueDB = nil
DB = nil
}
// List returns all messages in the delivery queue.
// Ordered by earliest delivery attempt first.
func List(ctx context.Context) ([]Msg, error) {
qmsgs, err := bstore.QueryDB[Msg](ctx, queueDB).List()
qmsgs, err := bstore.QueryDB[Msg](ctx, DB).List()
if err != nil {
return nil, err
}
@ -166,7 +167,7 @@ func List(ctx context.Context) ([]Msg, error) {
// Count returns the number of messages in the delivery queue.
func Count(ctx context.Context) (int, error) {
return bstore.QueryDB[Msg](ctx, queueDB).Count()
return bstore.QueryDB[Msg](ctx, DB).Count()
}
// Add a new message to the queue. The queue is kicked immediately to start a
@ -187,7 +188,7 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
return fmt.Errorf("no queuing with localserve")
}
tx, err := queueDB.Begin(ctx, true)
tx, err := DB.Begin(ctx, true)
if err != nil {
return fmt.Errorf("begin transaction: %w", err)
}
@ -288,7 +289,7 @@ func queuekick() {
// are zero, all messages are kicked.
// Returns number of messages queued for immediate delivery.
func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](ctx, queueDB)
q := bstore.QueryDB[Msg](ctx, DB)
if ID > 0 {
q.FilterID(ID)
}
@ -312,7 +313,7 @@ func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int
// If all parameters are zero, all messages are removed.
// Returns number of messages removed.
func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
q := bstore.QueryDB[Msg](ctx, queueDB)
q := bstore.QueryDB[Msg](ctx, DB)
if ID > 0 {
q.FilterID(ID)
}
@ -347,7 +348,7 @@ type ReadReaderAtCloser interface {
// OpenMessage opens a message present in the queue.
func OpenMessage(ctx context.Context, id int64) (ReadReaderAtCloser, error) {
qm := Msg{ID: id}
err := queueDB.Get(ctx, &qm)
err := DB.Get(ctx, &qm)
if err != nil {
return nil, err
}
@ -397,7 +398,7 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
}
func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration {
q := bstore.QueryDB[Msg](ctx, queueDB)
q := bstore.QueryDB[Msg](ctx, DB)
if len(busyDomains) > 0 {
var doms []any
for d := range busyDomains {
@ -418,7 +419,7 @@ func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duratio
}
func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
q := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
q := bstore.QueryDB[Msg](mox.Shutdown, DB)
q.FilterLessEqual("NextAttempt", time.Now())
q.SortAsc("NextAttempt")
q.Limit(maxConcurrentDeliveries)
@ -445,7 +446,7 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
// Remove message from queue in database and file system.
func queueDelete(ctx context.Context, msgID int64) error {
if err := queueDB.Delete(ctx, &Msg{ID: msgID}); err != nil {
if err := DB.Delete(ctx, &Msg{ID: msgID}); err != nil {
return err
}
// If removing from database fails, we'll also leave the file in the file system.
@ -491,7 +492,7 @@ func deliver(resolver dns.Resolver, m Msg) {
now := time.Now()
m.LastAttempt = &now
m.NextAttempt = now.Add(backoff)
qup := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
qup := bstore.QueryDB[Msg](mox.Shutdown, DB)
qup.FilterID(m.ID)
update := Msg{Attempts: m.Attempts, NextAttempt: m.NextAttempt, LastAttempt: m.LastAttempt}
if _, err := qup.UpdateNonzero(update); err != nil {
@ -510,7 +511,7 @@ func deliver(resolver dns.Resolver, m Msg) {
return
}
qup := bstore.QueryDB[Msg](context.Background(), queueDB)
qup := bstore.QueryDB[Msg](context.Background(), DB)
qup.FilterID(m.ID)
if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil {
qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg))

6
queue/queue_test.go
View file

@ -138,7 +138,7 @@ func TestQueue(t *testing.T) {
case <-dialed:
i := 0
for {
m, err := bstore.QueryDB[Msg](ctxbg, queueDB).Get()
m, err := bstore.QueryDB[Msg](ctxbg, DB).Get()
tcheck(t, err, "get")
if m.Attempts == 1 {
break
@ -288,7 +288,7 @@ func TestQueue(t *testing.T) {
for i := 1; i < 8; i++ {
go func() { <-deliveryResult }() // Deliver sends here.
deliver(resolver, msg)
err = queueDB.Get(ctxbg, &msg)
err = DB.Get(ctxbg, &msg)
tcheck(t, err, "get msg")
if msg.Attempts != i {
t.Fatalf("got attempt %d, expected %d", msg.Attempts, i)
@ -311,7 +311,7 @@ func TestQueue(t *testing.T) {
// Trigger final failure.
go func() { <-deliveryResult }() // Deliver sends here.
deliver(resolver, msg)
err = queueDB.Get(ctxbg, &msg)
err = DB.Get(ctxbg, &msg)
if err != bstore.ErrAbsent {
t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err)
}

2
serve.go
View file

@ -292,7 +292,7 @@ requested, other TLS certificates are requested on demand.
}
}()
m := &store.Message{Received: time.Now(), Flags: store.Flags{Flagged: true}}
n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this is install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n"))
n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this is install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nRemember to make a backup with \"mox backup\" before upgrading.\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n"))
if err != nil {
log.Infox("writing temporary message file for changelog delivery", err)
return next

2
smtpserver/reputation_test.go
View file

@ -102,7 +102,7 @@ func TestReputation(t *testing.T) {
p := "../testdata/smtpserver-reputation.db"
defer os.Remove(p)
db, err := bstore.Open(ctxbg, p, &bstore.Options{Timeout: 5 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{})
db, err := bstore.Open(ctxbg, p, &bstore.Options{Timeout: 5 * time.Second}, store.DBTypes...)
tcheck(t, err, "open db")
defer db.Close()

5
store/account.go
View file

@ -377,6 +377,9 @@ type Outgoing struct {
Submitted time.Time `bstore:"nonzero,default now"`
}
// Types stored in DB.
var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}}
// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
type Account struct {
Name string // Name, according to configuration.
@ -455,7 +458,7 @@ func openAccount(name string) (a *Account, rerr error) {
os.MkdirAll(dir, 0770)
}
db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{})
db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
if err != nil {
return nil, err
}

54
test-upgrade.sh Executable file
View file

@ -0,0 +1,54 @@
#!/bin/sh
# todo: should we also test with mox.conf and domains.conf files? should "mox backup" and "mox gentestdata" add them, and "mox verifydata" use them?
set -e
# set -x
(rm -r testdata/upgrade 2>/dev/null || exit 0)
mkdir testdata/upgrade
cd testdata/upgrade
# Check that we can upgrade what we currently generate.
../../mox gentestdata data
../../mox verifydata data
rm -r data
# For each historic release (i.e. all tagged versions) except the first few that
# didn't have the gentestdata command, we generate a data directory for testing
# and simulate upgrade to currently checked out version.
# The awk command reverses the tags, so we try the previous release first since
# it is the most likely to fail.
tagsrev=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }')
if test "$tagsrev" = ""; then exit 0; fi
for tag in $tagsrev; do
echo "Testing upgrade from $tag to current."
mkdir $tag
(CGO_ENABLED=0 GOBIN=$PWD/$tag go install github.com/mjl-/mox@$tag)
# Generate with historic release.
./$tag/mox gentestdata $tag/data
# Verify with current code.
../../mox verifydata $tag/data
rm -r $tag/data
done
# Also go step-wise through each released version. Having upgraded step by step
# can have added more schema upgrades to the database files.
tags=$(git tag --sort creatordate | grep -v '^v0\.0\.[123]$' | cat)
first=yes
for tag in $tags; do
if test "$first" = yes; then
echo "Starting with test data for $tag."
./$tag/mox gentestdata stepdata
first=
else
echo "Upgrade data to $tag."
./$tag/mox verifydata stepdata
fi
done
echo "Testing final upgrade to current."
../../mox verifydata stepdata
rm -r stepdata
rm */mox
cd ../..
rmdir testdata/upgrade/* testdata/upgrade

19
tlsrptdb/db.go
View file

@ -23,8 +23,9 @@ import (
var (
xlog = mlog.New("tlsrptdb")
tlsrptDB *bstore.DB
mutex sync.Mutex
DBTypes = []any{TLSReportRecord{}}
DB *bstore.DB
mutex sync.Mutex
metricSession = promauto.NewCounterVec(
prometheus.CounterOpts{
@ -64,16 +65,16 @@ type TLSReportRecord struct {
func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
mutex.Lock()
defer mutex.Unlock()
if tlsrptDB == nil {
if DB == nil {
p := mox.DataDirPath("tlsrpt.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, TLSReportRecord{})
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
if err != nil {
return nil, err
}
tlsrptDB = db
DB = db
}
return tlsrptDB, nil
return DB, nil
}
// Init opens and possibly initializes the database.
@ -86,10 +87,10 @@ func Init() error {
func Close() {
mutex.Lock()
defer mutex.Unlock()
if tlsrptDB != nil {
err := tlsrptDB.Close()
if DB != nil {
err := DB.Close()
xlog.Check(err, "closing database")
tlsrptDB = nil
DB = nil
}
}

347
verifydata.go Normal file
View file

@ -0,0 +1,347 @@
package main
import (
"context"
"errors"
"fmt"
"io/fs"
"log"
"os"
"path/filepath"
"strconv"
"strings"
bolt "go.etcd.io/bbolt"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/dmarcdb"
"github.com/mjl-/mox/junk"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/mtastsdb"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/store"
"github.com/mjl-/mox/tlsrptdb"
)
func cmdVerifydata(c *cmd) {
c.params = "data-dir"
c.help = `Verify the contents of a data directory, typically of a backup.
Verifydata checks all database files to see if they are valid BoltDB/bstore
databases. It checks that all messages in the database have a corresponding
on-disk message file and there are no unrecognized files. If option -fix is
specified, unrecognized message files are moved away. This may be needed after
a restore, because messages enqueued or delivered in the future may get those
message sequence numbers assigned and writing the message file would fail.
Because verifydata opens the database files, schema upgrades may automatically
be applied. This can happen if you use a new mox release. It is useful to run
"mox verifydata" with a new binary before attempting an upgrade, but only on a
copy of the database files, as made with "mox backup". Before upgrading, make a
new backup again since "mox verifydata" may have upgraded the database files,
possibly making them potentially no longer readable by the previous version.
`
var fix bool
c.flag.BoolVar(&fix, "fix", false, "fix fixable problems, such as moving away message files not referenced by their database")
args := c.Parse()
if len(args) != 1 {
c.Usage()
}
dataDir := filepath.Clean(args[0])
ctxbg := context.Background()
// Check whether file exists, or rather, that it doesn't not exist. Other errors
// will return true as well, so the triggered check can give the details.
exists := func(path string) bool {
_, err := os.Stat(path)
return err == nil || !os.IsNotExist(err)
}
// Check for error. If so, write a log line, including the path, and set fail so we
// can warn at the end.
var fail bool
checkf := func(err error, path, format string, args ...any) {
if err == nil {
return
}
fail = true
log.Printf("error: %s: %s: %v", path, fmt.Sprintf(format, args...), err)
}
// When we fix problems, we may have to move files/dirs. We need to ensure the
// directory of the destination path exists before we move. We keep track of
// created dirs so we don't try to create the same directory all the time.
createdDirs := map[string]struct{}{}
ensureDir := func(path string) {
dir := filepath.Dir(path)
if _, ok := createdDirs[dir]; ok {
return
}
err := os.MkdirAll(dir, 0770)
checkf(err, dir, "creating directory")
createdDirs[dir] = struct{}{}
}
// Check a database file by opening it with BoltDB and bstore and lightly checking
// its contents.
checkDB := func(path string, types []any) {
_, err := os.Stat(path)
checkf(err, path, "checking if file exists")
if err != nil {
return
}
bdb, err := bolt.Open(path, 0600, nil)
checkf(err, path, "open database with bolt")
if err != nil {
return
}
// Check BoltDB consistency.
err = bdb.View(func(tx *bolt.Tx) error {
for err := range tx.Check() {
checkf(err, path, "bolt database problem")
}
return nil
})
checkf(err, path, "reading bolt database")
bdb.Close()
db, err := bstore.Open(ctxbg, path, nil, types...)
checkf(err, path, "open database with bstore")
if err != nil {
return
}
defer db.Close()
err = db.Read(ctxbg, func(tx *bstore.Tx) error {
// Check bstore consistency, if it can export all records for all types. This is a
// quick way to get bstore to parse all records.
types, err := tx.Types()
checkf(err, path, "getting bstore types from database")
if err != nil {
return nil
}
for _, t := range types {
var fields []string
err := tx.Records(t, &fields, func(m map[string]any) error {
return nil
})
checkf(err, path, "parsing record for type %q", t)
}
return nil
})
checkf(err, path, "checking database file")
}
checkFile := func(path string) {
_, err := os.Stat(path)
checkf(err, path, "checking if file exists")
}
checkQueue := func() {
dbpath := filepath.Join(dataDir, "queue/index.db")
checkDB(dbpath, queue.DBTypes)
// Check that all messages present in the database also exist on disk.
seen := map[string]struct{}{}
db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, queue.DBTypes...)
checkf(err, dbpath, "opening queue database to check messages")
if err == nil {
err := bstore.QueryDB[queue.Msg](ctxbg, db).ForEach(func(m queue.Msg) error {
mp := store.MessagePath(m.ID)
seen[mp] = struct{}{}
p := filepath.Join(dataDir, "queue", mp)
checkFile(p)
return nil
})
checkf(err, dbpath, "reading messages in queue database to check files")
}
// Check that there are no files that could be treated as a message.
qdir := filepath.Join(dataDir, "queue")
err = filepath.WalkDir(qdir, func(qpath string, d fs.DirEntry, err error) error {
checkf(err, qpath, "walk")
if err != nil {
return nil
}
if d.IsDir() {
return nil
}
p := qpath[len(qdir)+1:]
if p == "index.db" {
return nil
}
if _, ok := seen[p]; ok {
return nil
}
l := strings.Split(p, string(filepath.Separator))
if len(l) == 1 {
log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)
return nil
}
// If it doesn't look like a message number, there is no risk of it being the name
// of a message enqueued in the future.
if len(l) >= 3 {
if _, err := strconv.ParseInt(l[1], 10, 64); err != nil {
log.Printf("warning: %s: unrecognized file in queue directory, ignoring", qpath)
return nil
}
}
if !fix {
checkf(errors.New("may interfere with messages enqueued in the future"), qpath, "unrecognized file in queue directory (use the -fix flag to move it away)")
return nil
}
npath := filepath.Join(dataDir, "moved", "queue", p)
ensureDir(npath)
err = os.Rename(qpath, npath)
checkf(err, qpath, "moving queue message file away")
if err == nil {
log.Printf("warning: moved %s to %s", qpath, npath)
}
return nil
})
checkf(err, qdir, "walking queue directory")
}
// Check an account, with its database file and messages.
checkAccount := func(name string) {
accdir := filepath.Join(dataDir, "accounts", name)
checkDB(filepath.Join(accdir, "index.db"), store.DBTypes)
jfdbpath := filepath.Join(accdir, "junkfilter.db")
jfbloompath := filepath.Join(accdir, "junkfilter.bloom")
if exists(jfdbpath) || exists(jfbloompath) {
checkDB(jfdbpath, junk.DBTypes)
}
// todo: add some kind of check for the bloom filter?
// Check that all messages in the database have a message file on disk.
seen := map[string]struct{}{}
dbpath := filepath.Join(accdir, "index.db")
db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{MustExist: true}, store.DBTypes...)
checkf(err, dbpath, "opening account database to check messages")
if err == nil {
err := bstore.QueryDB[store.Message](ctxbg, db).ForEach(func(m store.Message) error {
mp := store.MessagePath(m.ID)
seen[mp] = struct{}{}
p := filepath.Join(accdir, "msg", mp)
checkFile(p)
return nil
})
checkf(err, dbpath, "reading messages in account database to check files")
}
// Walk through all files in the msg directory. Warn about files that weren't in
// the database as message file. Possibly move away files that could cause trouble.
msgdir := filepath.Join(accdir, "msg")
if !exists(msgdir) {
// New accounts with messages don't have a msg directory.
return
}
err = filepath.WalkDir(msgdir, func(msgpath string, d fs.DirEntry, err error) error {
checkf(err, msgpath, "walk")
if err != nil {
return nil
}
if d.IsDir() {
return nil
}
p := msgpath[len(msgdir)+1:]
if _, ok := seen[p]; ok {
return nil
}
l := strings.Split(p, string(filepath.Separator))
if len(l) == 1 {
log.Printf("warning: %s: unrecognized file in message directory, ignoring", msgpath)
return nil
}
if !fix {
checkf(errors.New("may interfere with future account messages"), msgpath, "unrecognized file in account message directory (use the -fix flag to move it away)")
return nil
}
npath := filepath.Join(dataDir, "moved", "accounts", name, "msg", p)
ensureDir(npath)
err = os.Rename(msgpath, npath)
checkf(err, msgpath, "moving account message file away")
if err == nil {
log.Printf("warning: moved %s to %s", msgpath, npath)
}
return nil
})
checkf(err, msgdir, "walking account message directory")
}
// Check everything in the "accounts" directory.
checkAccounts := func() {
accountsDir := filepath.Join(dataDir, "accounts")
entries, err := os.ReadDir(accountsDir)
checkf(err, accountsDir, "reading accounts directory")
for _, e := range entries {
// We treat all directories as accounts. When we were backing up, we only verified
// accounts from the config and made regular file copies of all other files
// (perhaps an old account, but at least not with an open database file). It may
// turn out that that account was/is not valid, generating warnings. Better safe
// than sorry. It should hopefully get the admin to move away such an old account.
if e.IsDir() {
checkAccount(e.Name())
} else {
log.Printf("warning: %s: unrecognized file in accounts directory, ignoring", filepath.Join("accounts", e.Name()))
}
}
}
// Check all files, skipping the known files, queue and accounts directories. Warn
// about unknown files. Skip a "tmp" directory. And a "moved" directory, we
// probably created it ourselves.
backupmoxversion := "(unknown)"
checkOther := func() {
err := filepath.WalkDir(dataDir, func(dpath string, d fs.DirEntry, err error) error {
checkf(err, dpath, "walk")
if err != nil {
return nil
}
if dpath == dataDir {
return nil
}
p := dpath
if dataDir != "." {
p = p[len(dataDir)+1:]
}
switch p {
case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion":
return nil
case "acme", "queue", "accounts", "tmp", "moved":
return fs.SkipDir
case "moxversion":
buf, err := os.ReadFile(dpath)
checkf(err, dpath, "reading moxversion")
if err == nil {
backupmoxversion = string(buf)
}
return nil
}
log.Printf("warning: %s: unrecognized other file, ignoring", dpath)
return nil
})
checkf(err, dataDir, "walking data directory")
}
checkDB(filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.DBTypes)
checkDB(filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)
checkDB(filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes)
checkQueue()
checkAccounts()
checkOther()
if fail {
log.Fatalf("errors were found")
} else {
log.Printf("%s: OK", dataDir)
}
if backupmoxversion != moxvar.Version {
log.Printf("NOTE: The backup was made with mox version %q, while verifydata was run with mox version %q. Database files have probably been modified by running mox verifydata. Make a fresh backup before upgrading.", backupmoxversion, moxvar.Version)
}
}