export more imap flags (eg $Junk/$NotJunk/$Forwarded) with maildirs, in dovecot-keywords file

and let the subcommand "export" use the same export code as the accounts page.

doc.go

@@ -26,8 +26,8 @@ low-maintenance self-hosted email.
 	mox queue dump id
 	mox import maildir accountname mailboxname maildir
 	mox import mbox accountname mailboxname mbox
-	mox export maildir dst-path account-path [mailbox]
-	mox export mbox dst-path account-path [mailbox]
+	mox export maildir dst-dir account-path [mailbox]
+	mox export mbox dst-dir account-path [mailbox]
 	mox help [command ...]
 	mox config test
 	mox config dnscheck domain
@@ -266,7 +266,7 @@ database file directly. This may block if a running mox instance also has the
 database open, e.g. for IMAP connections. To export from a running instance, use
 the accounts web page.
 
-	usage: mox export maildir dst-path account-path [mailbox]
+	usage: mox export maildir dst-dir account-path [mailbox]
 
 # mox export mbox
 
@@ -279,11 +279,11 @@ database file directly. This may block if a running mox instance also has the
 database open, e.g. for IMAP connections. To export from a running instance, use
 the accounts web page.
 
-For mbox export, we use "mboxrd" where message lines starting with the magic
-"From " string are escaped by prepending a >. We escape all ">*From ",
+For mbox export, "mboxrd" is used where message lines starting with the magic
+"From " string are escaped by prepending a >. All ">*From " are escaped,
 otherwise reconstructing the original could lose a ">".
 
-	usage: mox export mbox dst-path account-path [mailbox]
+	usage: mox export mbox dst-dir account-path [mailbox]
 
 # mox help
 

export.go

@@ -1,23 +1,17 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"io"
-	"log"
-	"os"
 	"path/filepath"
-	"sort"
 	"time"
 
 	"github.com/mjl-/bstore"
 
+	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/store"
 )
 
 func cmdExportMaildir(c *cmd) {
-	c.params = "dst-path account-path [mailbox]"
+	c.params = "dst-dir account-path [mailbox]"
 	c.help = `Export one or all mailboxes from an account in maildir format.
 
 Export bypasses a running mox instance. It opens the account mailbox/message
@@ -30,7 +24,7 @@ the accounts web page.
 }
 
 func cmdExportMbox(c *cmd) {
-	c.params = "dst-path account-path [mailbox]"
+	c.params = "dst-dir account-path [mailbox]"
 	c.help = `Export messages from one or all mailboxes in an account in mbox format.
 
 Using mbox is not recommended. Maildir is a better format.
@@ -40,8 +34,8 @@ database file directly. This may block if a running mox instance also has the
 database open, e.g. for IMAP connections. To export from a running instance, use
 the accounts web page.
 
-For mbox export, we use "mboxrd" where message lines starting with the magic
-"From " string are escaped by prepending a >. We escape all ">*From ",
+For mbox export, "mboxrd" is used where message lines starting with the magic
+"From " string are escaped by prepending a >. All ">*From " are escaped,
 otherwise reconstructing the original could lose a ">".
 `
 	args := c.Parse()
@@ -63,204 +57,11 @@ func xcmdExport(mbox bool, args []string, c *cmd) {
 	dbpath := filepath.Join(accountDir, "index.db")
 	db, err := bstore.Open(dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{})
 	xcheckf(err, "open database %q", dbpath)
+	defer db.Close()
 
-	err = db.Read(func(tx *bstore.Tx) error {
-		exporttx(tx, mbox, dst, accountDir, mailbox)
-		return nil
-	})
-	xcheckf(err, "transaction")
-}
-
-func exporttx(tx *bstore.Tx, mbox bool, dst, accountDir, mailbox string) {
-	id2name := map[int64]string{}
-	name2id := map[string]int64{}
-
-	mailboxes, err := bstore.QueryTx[store.Mailbox](tx).List()
-	xcheckf(err, "query mailboxes")
-	for _, mb := range mailboxes {
-		id2name[mb.ID] = mb.Name
-		name2id[mb.Name] = mb.ID
-	}
-
-	var mailboxID int64
-	if mailbox != "" {
-		var ok bool
-		mailboxID, ok = name2id[mailbox]
-		if !ok {
-			log.Fatalf("mailbox %q not found", mailbox)
-		}
-	}
-
-	mboxes := map[string]*os.File{}
-
-	// Open mbox files or create dirs.
-	var names []string
-	for _, name := range id2name {
-		if mailbox != "" && name != mailbox {
-			continue
-		}
-		names = append(names, name)
-	}
-	// We need to sort the names because maildirs can create subdirs. Ranging over
-	// id2name directly would randomize the directory names, we would create a sub
-	// maildir before the parent, and fail with "dir exists" when creating the parent
-	// dir.
-	sort.Slice(names, func(i, j int) bool {
-		return names[i] < names[j]
-	})
-	for _, name := range names {
-		p := dst
-		if mailbox == "" {
-			p = filepath.Join(p, name)
-		}
-
-		os.MkdirAll(filepath.Dir(p), 0770)
-		if mbox {
-			mbp := p
-			if mailbox == "" {
-				mbp += ".mbox"
-			}
-			f, err := os.OpenFile(mbp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
-			xcheckf(err, "creating mbox file")
-			log.Printf("creating mbox file %s", mbp)
-			mboxes[name] = f
-		} else {
-			err = os.Mkdir(p, 0770)
-			xcheckf(err, "making maildir")
-			log.Printf("creating maildir %s", p)
-			subdirs := []string{"new", "cur", "tmp"}
-			for _, subdir := range subdirs {
-				err = os.Mkdir(filepath.Join(p, subdir), 0770)
-				xcheckf(err, "making maildir subdir")
-			}
-		}
-	}
-
-	q := bstore.QueryTx[store.Message](tx)
-	if mailboxID > 0 {
-		q.FilterNonzero(store.Message{MailboxID: mailboxID})
-	}
-	defer q.Close()
-	for {
-		m, err := q.Next()
-		if err == bstore.ErrAbsent {
-			break
-		}
-		xcheckf(err, "next message")
-
-		mbname := id2name[m.MailboxID]
-
-		p := dst
-		if mailbox == "" {
-			p = filepath.Join(p, mbname)
-		}
-
-		mp := filepath.Join(accountDir, "msg", store.MessagePath(m.ID))
-		var mr io.ReadCloser
-		if m.Size == int64(len(m.MsgPrefix)) {
-			log.Printf("message size is prefix size for m id %d", m.ID)
-			mr = io.NopCloser(bytes.NewReader(m.MsgPrefix))
-		} else {
-			mpf, err := os.Open(mp)
-			xcheckf(err, "open message file")
-			st, err := mpf.Stat()
-			xcheckf(err, "stat message file")
-			size := st.Size() + int64(len(m.MsgPrefix))
-			if size != m.Size {
-				log.Fatalf("message size mismatch, database has %d, size is %d+%d=%d", m.Size, len(m.MsgPrefix), st.Size(), size)
-			}
-			mr = store.FileMsgReader(m.MsgPrefix, mpf)
-		}
-
-		if mbox {
-			// todo: should we put status flags in Status or X-Status header inside the message?
-			// todo: should we do anything with Content-Length headers? changing the escaping could invalidate those. is anything checking that field?
-
-			f := mboxes[mbname]
-			mailfrom := "mox"
-			if m.MailFrom != "" {
-				mailfrom = m.MailFrom
-			}
-			_, err := fmt.Fprintf(f, "From %s %s\n", mailfrom, m.Received.Format(time.ANSIC))
-			xcheckf(err, "writing from header")
-			r := bufio.NewReader(mr)
-			for {
-				line, rerr := r.ReadBytes('\n')
-				if rerr != io.EOF {
-					xcheckf(rerr, "reading from message")
-				}
-				if len(line) > 0 {
-					if bytes.HasSuffix(line, []byte("\r\n")) {
-						line = line[:len(line)-1]
-						line[len(line)-1] = '\n'
-					}
-					if bytes.HasPrefix(bytes.TrimLeft(line, ">"), []byte("From ")) {
-						_, err = fmt.Fprint(f, ">")
-						xcheckf(err, "writing escaping >")
-					}
-					_, err = f.Write(line)
-					xcheckf(err, "writing line")
-				}
-				if rerr == io.EOF {
-					break
-				}
-			}
-			_, err = fmt.Fprint(f, "\n")
-			xcheckf(err, "writing end of message newline")
-		} else {
-			if m.Flags.Seen {
-				p = filepath.Join(p, "cur")
-			} else {
-				p = filepath.Join(p, "new")
-			}
-			name := fmt.Sprintf("%d.%d.mox:2,", m.Received.Unix(), m.ID)
-			// todo: more flags? forwarded, (non)junk, phishing, mdnsent would be nice. but what is the convention. dovecot-keywords sounds non-standard.
-			if m.Flags.Seen {
-				name += "S"
-			}
-			if m.Flags.Answered {
-				name += "R"
-			}
-			if m.Flags.Flagged {
-				name += "F"
-			}
-			if m.Flags.Draft {
-				name += "D"
-			}
-			p = filepath.Join(p, name)
-			f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660)
-			xcheckf(err, "creating message file in maildir")
-
-			r := bufio.NewReader(mr)
-			for {
-				line, rerr := r.ReadBytes('\n')
-				if rerr != io.EOF {
-					xcheckf(rerr, "reading from message")
-				}
-				if len(line) > 0 {
-					if bytes.HasSuffix(line, []byte("\r\n")) {
-						line = line[:len(line)-1]
-						line[len(line)-1] = '\n'
-					}
-					_, err = f.Write(line)
-					xcheckf(err, "writing line")
-				}
-				if rerr == io.EOF {
-					break
-				}
-			}
-			mr.Close()
-			err = f.Close()
-			xcheckf(err, "closing new file in maildir")
-		}
-
-		mr.Close()
-	}
-
-	if mbox {
-		for _, f := range mboxes {
-			err = f.Close()
-			xcheckf(err, "closing mbox file")
-		}
-	}
+	a := store.DirArchiver{Dir: dst}
+	err = store.ExportMessages(mlog.New("export"), db, accountDir, a, !mbox, mailbox)
+	xcheckf(err, "exporting messages")
+	err = a.Close()
+	xcheckf(err, "closing archiver")
 }

http/account.go

@@ -161,7 +161,7 @@ func accountHandle(w http.ResponseWriter, r *http.Request) {
 				log.Errorx("exporting mail close", err)
 			}
 		}()
-		if err := acc.ExportMessages(log, archiver, maildir, ""); err != nil {
+		if err := store.ExportMessages(log, acc.DB, acc.Dir, archiver, maildir, ""); err != nil {
 			log.Errorx("exporting mail", err)
 		}
 

store/export.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/mjl-/bstore"
@@ -19,7 +20,9 @@ import (
 
 // Archiver can archive multiple mailboxes and their messages.
 type Archiver interface {
-	Create(name string, size int64, mtime time.Time) (io.Writer, error)
+	// Add file to archive. If name ends with a slash, it is created as a directory and
+	// the returned io.WriteCloser can be ignored.
+	Create(name string, size int64, mtime time.Time) (io.WriteCloser, error)
 	Close() error
 }
 
@@ -29,18 +32,18 @@ type TarArchiver struct {
 }
 
 // Create adds a file header to the tar file.
-func (a TarArchiver) Create(name string, size int64, mtime time.Time) (io.Writer, error) {
+func (a TarArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {
 	hdr := tar.Header{
 		Name:    name,
 		Size:    size,
-		Mode:    0600,
+		Mode:    0660,
 		ModTime: mtime,
 		Format:  tar.FormatPAX,
 	}
 	if err := a.WriteHeader(&hdr); err != nil {
 		return nil, err
 	}
-	return a, nil
+	return nopCloser{a}, nil
 }
 
 // ZipArchiver is an Archiver that writes to a zip file.
@@ -49,14 +52,49 @@ type ZipArchiver struct {
 }
 
 // Create adds a file header to the zip file.
-func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.Writer, error) {
+func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {
 	hdr := zip.FileHeader{
 		Name:               name,
 		Method:             zip.Deflate,
 		Modified:           mtime,
 		UncompressedSize64: uint64(size),
 	}
-	return a.CreateHeader(&hdr)
+	w, err := a.CreateHeader(&hdr)
+	if err != nil {
+		return nil, err
+	}
+	return nopCloser{w}, nil
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+// Close does nothing.
+func (nopCloser) Close() error {
+	return nil
+}
+
+// DirArchiver is an Archiver that writes to a directory.
+type DirArchiver struct {
+	Dir string
+}
+
+// Create create name in the file system, in dir.
+func (a DirArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) {
+	isdir := strings.HasSuffix(name, "/")
+	name = strings.TrimSuffix(name, "/")
+	p := filepath.Join(a.Dir, name)
+	os.MkdirAll(filepath.Dir(p), 0770)
+	if isdir {
+		return nil, os.Mkdir(p, 0770)
+	}
+	return os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0660)
+}
+
+// Close on a dir does nothing.
+func (a DirArchiver) Close() error {
+	return nil
 }
 
 // ExportMessages writes messages to archiver. Either in maildir format, or otherwise in
@@ -66,11 +104,11 @@ func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.Writer
 // Some errors are not fatal and result in skipped messages. In that happens, a
 // file "errors.txt" is added to the archive describing the errors. The goal is to
 // let users export (hopefully) most messages even in the face of errors.
-func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, mailboxOpt string) error {
+func ExportMessages(log *mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error {
 	// Start transaction without closure, we are going to close it early, but don't
 	// want to deal with declaring many variables now to be able to assign them in a
 	// closure and use them afterwards.
-	tx, err := a.DB.Begin(false)
+	tx, err := db.Begin(false)
 	if err != nil {
 		return fmt.Errorf("transaction: %v", err)
 	}
@@ -173,7 +211,48 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 		}
 	}()
 
-	finishMbox := func() error {
+	// For dovecot-keyword-style flags not in standard maildir.
+	maildirFlags := map[string]int{}
+	var maildirFlaglist []string
+	maildirFlag := func(flag string) string {
+		i, ok := maildirFlags[flag]
+		if !ok {
+			if len(maildirFlags) >= 26 {
+				// Max 26 flag characters.
+				return ""
+			}
+			i = len(maildirFlags)
+			maildirFlags[flag] = i
+			maildirFlaglist = append(maildirFlaglist, flag)
+		}
+		return string(rune('a' + i))
+	}
+
+	finishMailbox := func() error {
+		if maildir {
+			if len(maildirFlags) == 0 {
+				return nil
+			}
+
+			var b bytes.Buffer
+			for i, flag := range maildirFlaglist {
+				if _, err := fmt.Fprintf(&b, "%d %s\n", i, flag); err != nil {
+					return err
+				}
+			}
+			w, err := archiver.Create(curMailbox+"/dovecot-keywords", int64(b.Len()), start)
+			if err != nil {
+				return fmt.Errorf("adding dovecot-keywords: %v", err)
+			}
+			if _, err := w.Write(b.Bytes()); err != nil {
+				w.Close()
+				return fmt.Errorf("writing dovecot-keywords: %v", err)
+			}
+			maildirFlags = map[string]int{}
+			maildirFlaglist = nil
+			return w.Close()
+		}
+
 		if mboxtmp == nil {
 			return nil
 		}
@@ -193,8 +272,12 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 			return fmt.Errorf("add mbox to archive: %v", err)
 		}
 		if _, err := io.Copy(w, mboxtmp); err != nil {
+			w.Close()
 			return fmt.Errorf("copying temp mbox file to archive: %v", err)
 		}
+		if err := w.Close(); err != nil {
+			return fmt.Errorf("closing message file: %v", err)
+		}
 		if err := mboxtmp.Close(); err != nil {
 			log.Errorx("closing temporary mbox file", err)
 			// Continue, not fatal.
@@ -205,7 +288,7 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 	}
 
 	exportMessage := func(m Message) error {
-		mp := a.MessagePath(m.ID)
+		mp := filepath.Join(accountDir, "msg", MessagePath(m.ID))
 		var mr io.ReadCloser
 		if m.Size == int64(len(m.MsgPrefix)) {
 			mr = io.NopCloser(bytes.NewReader(m.MsgPrefix))
@@ -236,19 +319,41 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 				p = filepath.Join(p, "new")
 			}
 			name := fmt.Sprintf("%d.%d.mox:2,", m.Received.Unix(), m.ID)
-			// todo: more flags? forwarded, (non)junk, phishing, mdnsent would be nice. but what is the convention. dovecot-keywords sounds non-standard.
-			if m.Flags.Seen {
-				name += "S"
-			}
-			if m.Flags.Answered {
-				name += "R"
+
+			// Standard flags. May need to be sorted.
+			if m.Flags.Draft {
+				name += "D"
 			}
 			if m.Flags.Flagged {
 				name += "F"
 			}
-			if m.Flags.Draft {
-				name += "D"
+			if m.Flags.Answered {
+				name += "R"
 			}
+			if m.Flags.Seen {
+				name += "S"
+			}
+			if m.Flags.Deleted {
+				name += "T"
+			}
+
+			// Non-standard flag. We set them with a dovecot-keywords file.
+			if m.Flags.Forwarded {
+				name += maildirFlag("$Forwarded")
+			}
+			if m.Flags.Junk {
+				name += maildirFlag("$Junk")
+			}
+			if m.Flags.Notjunk {
+				name += maildirFlag("$NotJunk")
+			}
+			if m.Flags.Phishing {
+				name += maildirFlag("$Phishing")
+			}
+			if m.Flags.MDNSent {
+				name += maildirFlag("$MDNSent")
+			}
+
 			p = filepath.Join(p, name)
 
 			// We store messages with \r\n, maildir needs without. But we need to know the
@@ -281,9 +386,10 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 				return fmt.Errorf("adding message to archive: %v", err)
 			}
 			if _, err := io.Copy(w, &dst); err != nil {
+				w.Close()
 				return fmt.Errorf("copying message to archive: %v", err)
 			}
-			return nil
+			return w.Close()
 		}
 
 		// todo: should we put status flags in Status or X-Status header inside the message?
@@ -327,7 +433,7 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 
 	for _, m := range msgs {
 		if m.MailboxID != curMailboxID {
-			if err := finishMbox(); err != nil {
+			if err := finishMailbox(); err != nil {
 				return err
 			}
 
@@ -362,7 +468,7 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 			return err
 		}
 	}
-	if err := finishMbox(); err != nil {
+	if err := finishMailbox(); err != nil {
 		return err
 	}
 
@@ -373,9 +479,13 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool,
 			return err
 		}
 		if _, err := w.Write([]byte(errors)); err != nil {
+			w.Close()
 			log.Errorx("writing errors.txt to archive", err)
 			return err
 		}
+		if err := w.Close(); err != nil {
+			return err
+		}
 	}
 
 	return nil

store/export_test.go

@@ -5,7 +5,9 @@ import (
 	"archive/zip"
 	"bytes"
 	"io"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -47,16 +49,21 @@ func TestExport(t *testing.T) {
 
 	archive := func(archiver Archiver, maildir bool) {
 		t.Helper()
-		err = acc.ExportMessages(log, archiver, maildir, "")
+		err = ExportMessages(log, acc.DB, acc.Dir, archiver, maildir, "")
 		tcheck(t, err, "export messages")
 		err = archiver.Close()
 		tcheck(t, err, "archiver close")
 	}
 
+	os.RemoveAll("../testdata/exportmaildir")
+	os.RemoveAll("../testdata/exportmbox")
+
 	archive(ZipArchiver{zip.NewWriter(&maildirZip)}, true)
 	archive(ZipArchiver{zip.NewWriter(&mboxZip)}, false)
 	archive(TarArchiver{tar.NewWriter(&maildirTar)}, true)
 	archive(TarArchiver{tar.NewWriter(&mboxTar)}, false)
+	archive(DirArchiver{"../testdata/exportmaildir"}, true)
+	archive(DirArchiver{"../testdata/exportmbox"}, false)
 
 	if r, err := zip.NewReader(bytes.NewReader(maildirZip.Bytes()), int64(maildirZip.Len())); err != nil {
 		t.Fatalf("reading maildir zip: %v", err)
@@ -86,11 +93,29 @@ func TestExport(t *testing.T) {
 			_, err = io.Copy(io.Discard, tr)
 			tcheck(t, err, "copy")
 		}
-		if n != have {
-			t.Fatalf("got %d files, expected %d", n, have)
+		if have != n {
+			t.Fatalf("got %d files, expected %d", have, n)
 		}
 	}
 
 	checkTarFiles(&maildirTar, 2*3+2)
 	checkTarFiles(&mboxTar, 2)
+
+	checkDirFiles := func(dir string, n int) {
+		t.Helper()
+		have := 0
+		err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+			if err == nil && !d.IsDir() {
+				have++
+			}
+			return nil
+		})
+		tcheck(t, err, "walkdir")
+		if n != have {
+			t.Fatalf("got %d files, expected %d", have, n)
+		}
+	}
+
+	checkDirFiles("../testdata/exportmaildir", 2)
+	checkDirFiles("../testdata/exportmbox", 2)
 }