From 4a58b8f434b492e6956c6de61c4fec0fd2eefb75 Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Mon, 13 Feb 2023 22:37:25 +0100 Subject: [PATCH] export more imap flags (eg $Junk/$NotJunk/$Forwarded) with maildirs, in dovecot-keywords file and let the subcommand "export" use the same export code as the accounts page. --- doc.go | 12 +-- export.go | 221 +++---------------------------------------- http/account.go | 2 +- store/export.go | 152 +++++++++++++++++++++++++---- store/export_test.go | 31 +++++- 5 files changed, 177 insertions(+), 241 deletions(-) diff --git a/doc.go b/doc.go index 3a94335..0ee0b90 100644 --- a/doc.go +++ b/doc.go @@ -26,8 +26,8 @@ low-maintenance self-hosted email. mox queue dump id mox import maildir accountname mailboxname maildir mox import mbox accountname mailboxname mbox - mox export maildir dst-path account-path [mailbox] - mox export mbox dst-path account-path [mailbox] + mox export maildir dst-dir account-path [mailbox] + mox export mbox dst-dir account-path [mailbox] mox help [command ...] mox config test mox config dnscheck domain @@ -266,7 +266,7 @@ database file directly. This may block if a running mox instance also has the database open, e.g. for IMAP connections. To export from a running instance, use the accounts web page. - usage: mox export maildir dst-path account-path [mailbox] + usage: mox export maildir dst-dir account-path [mailbox] # mox export mbox @@ -279,11 +279,11 @@ database file directly. This may block if a running mox instance also has the database open, e.g. for IMAP connections. To export from a running instance, use the accounts web page. -For mbox export, we use "mboxrd" where message lines starting with the magic -"From " string are escaped by prepending a >. We escape all ">*From ", +For mbox export, "mboxrd" is used where message lines starting with the magic +"From " string are escaped by prepending a >. All ">*From " are escaped, otherwise reconstructing the original could lose a ">". - usage: mox export mbox dst-path account-path [mailbox] + usage: mox export mbox dst-dir account-path [mailbox] # mox help diff --git a/export.go b/export.go index 8795378..d0f196e 100644 --- a/export.go +++ b/export.go @@ -1,23 +1,17 @@ package main import ( - "bufio" - "bytes" - "fmt" - "io" - "log" - "os" "path/filepath" - "sort" "time" "github.com/mjl-/bstore" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/store" ) func cmdExportMaildir(c *cmd) { - c.params = "dst-path account-path [mailbox]" + c.params = "dst-dir account-path [mailbox]" c.help = `Export one or all mailboxes from an account in maildir format. Export bypasses a running mox instance. It opens the account mailbox/message @@ -30,7 +24,7 @@ the accounts web page. } func cmdExportMbox(c *cmd) { - c.params = "dst-path account-path [mailbox]" + c.params = "dst-dir account-path [mailbox]" c.help = `Export messages from one or all mailboxes in an account in mbox format. Using mbox is not recommended. Maildir is a better format. @@ -40,8 +34,8 @@ database file directly. This may block if a running mox instance also has the database open, e.g. for IMAP connections. To export from a running instance, use the accounts web page. -For mbox export, we use "mboxrd" where message lines starting with the magic -"From " string are escaped by prepending a >. We escape all ">*From ", +For mbox export, "mboxrd" is used where message lines starting with the magic +"From " string are escaped by prepending a >. All ">*From " are escaped, otherwise reconstructing the original could lose a ">". ` args := c.Parse() @@ -63,204 +57,11 @@ func xcmdExport(mbox bool, args []string, c *cmd) { dbpath := filepath.Join(accountDir, "index.db") db, err := bstore.Open(dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{}) xcheckf(err, "open database %q", dbpath) + defer db.Close() - err = db.Read(func(tx *bstore.Tx) error { - exporttx(tx, mbox, dst, accountDir, mailbox) - return nil - }) - xcheckf(err, "transaction") -} - -func exporttx(tx *bstore.Tx, mbox bool, dst, accountDir, mailbox string) { - id2name := map[int64]string{} - name2id := map[string]int64{} - - mailboxes, err := bstore.QueryTx[store.Mailbox](tx).List() - xcheckf(err, "query mailboxes") - for _, mb := range mailboxes { - id2name[mb.ID] = mb.Name - name2id[mb.Name] = mb.ID - } - - var mailboxID int64 - if mailbox != "" { - var ok bool - mailboxID, ok = name2id[mailbox] - if !ok { - log.Fatalf("mailbox %q not found", mailbox) - } - } - - mboxes := map[string]*os.File{} - - // Open mbox files or create dirs. - var names []string - for _, name := range id2name { - if mailbox != "" && name != mailbox { - continue - } - names = append(names, name) - } - // We need to sort the names because maildirs can create subdirs. Ranging over - // id2name directly would randomize the directory names, we would create a sub - // maildir before the parent, and fail with "dir exists" when creating the parent - // dir. - sort.Slice(names, func(i, j int) bool { - return names[i] < names[j] - }) - for _, name := range names { - p := dst - if mailbox == "" { - p = filepath.Join(p, name) - } - - os.MkdirAll(filepath.Dir(p), 0770) - if mbox { - mbp := p - if mailbox == "" { - mbp += ".mbox" - } - f, err := os.OpenFile(mbp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) - xcheckf(err, "creating mbox file") - log.Printf("creating mbox file %s", mbp) - mboxes[name] = f - } else { - err = os.Mkdir(p, 0770) - xcheckf(err, "making maildir") - log.Printf("creating maildir %s", p) - subdirs := []string{"new", "cur", "tmp"} - for _, subdir := range subdirs { - err = os.Mkdir(filepath.Join(p, subdir), 0770) - xcheckf(err, "making maildir subdir") - } - } - } - - q := bstore.QueryTx[store.Message](tx) - if mailboxID > 0 { - q.FilterNonzero(store.Message{MailboxID: mailboxID}) - } - defer q.Close() - for { - m, err := q.Next() - if err == bstore.ErrAbsent { - break - } - xcheckf(err, "next message") - - mbname := id2name[m.MailboxID] - - p := dst - if mailbox == "" { - p = filepath.Join(p, mbname) - } - - mp := filepath.Join(accountDir, "msg", store.MessagePath(m.ID)) - var mr io.ReadCloser - if m.Size == int64(len(m.MsgPrefix)) { - log.Printf("message size is prefix size for m id %d", m.ID) - mr = io.NopCloser(bytes.NewReader(m.MsgPrefix)) - } else { - mpf, err := os.Open(mp) - xcheckf(err, "open message file") - st, err := mpf.Stat() - xcheckf(err, "stat message file") - size := st.Size() + int64(len(m.MsgPrefix)) - if size != m.Size { - log.Fatalf("message size mismatch, database has %d, size is %d+%d=%d", m.Size, len(m.MsgPrefix), st.Size(), size) - } - mr = store.FileMsgReader(m.MsgPrefix, mpf) - } - - if mbox { - // todo: should we put status flags in Status or X-Status header inside the message? - // todo: should we do anything with Content-Length headers? changing the escaping could invalidate those. is anything checking that field? - - f := mboxes[mbname] - mailfrom := "mox" - if m.MailFrom != "" { - mailfrom = m.MailFrom - } - _, err := fmt.Fprintf(f, "From %s %s\n", mailfrom, m.Received.Format(time.ANSIC)) - xcheckf(err, "writing from header") - r := bufio.NewReader(mr) - for { - line, rerr := r.ReadBytes('\n') - if rerr != io.EOF { - xcheckf(rerr, "reading from message") - } - if len(line) > 0 { - if bytes.HasSuffix(line, []byte("\r\n")) { - line = line[:len(line)-1] - line[len(line)-1] = '\n' - } - if bytes.HasPrefix(bytes.TrimLeft(line, ">"), []byte("From ")) { - _, err = fmt.Fprint(f, ">") - xcheckf(err, "writing escaping >") - } - _, err = f.Write(line) - xcheckf(err, "writing line") - } - if rerr == io.EOF { - break - } - } - _, err = fmt.Fprint(f, "\n") - xcheckf(err, "writing end of message newline") - } else { - if m.Flags.Seen { - p = filepath.Join(p, "cur") - } else { - p = filepath.Join(p, "new") - } - name := fmt.Sprintf("%d.%d.mox:2,", m.Received.Unix(), m.ID) - // todo: more flags? forwarded, (non)junk, phishing, mdnsent would be nice. but what is the convention. dovecot-keywords sounds non-standard. - if m.Flags.Seen { - name += "S" - } - if m.Flags.Answered { - name += "R" - } - if m.Flags.Flagged { - name += "F" - } - if m.Flags.Draft { - name += "D" - } - p = filepath.Join(p, name) - f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0660) - xcheckf(err, "creating message file in maildir") - - r := bufio.NewReader(mr) - for { - line, rerr := r.ReadBytes('\n') - if rerr != io.EOF { - xcheckf(rerr, "reading from message") - } - if len(line) > 0 { - if bytes.HasSuffix(line, []byte("\r\n")) { - line = line[:len(line)-1] - line[len(line)-1] = '\n' - } - _, err = f.Write(line) - xcheckf(err, "writing line") - } - if rerr == io.EOF { - break - } - } - mr.Close() - err = f.Close() - xcheckf(err, "closing new file in maildir") - } - - mr.Close() - } - - if mbox { - for _, f := range mboxes { - err = f.Close() - xcheckf(err, "closing mbox file") - } - } + a := store.DirArchiver{Dir: dst} + err = store.ExportMessages(mlog.New("export"), db, accountDir, a, !mbox, mailbox) + xcheckf(err, "exporting messages") + err = a.Close() + xcheckf(err, "closing archiver") } diff --git a/http/account.go b/http/account.go index b686a9d..1fd4899 100644 --- a/http/account.go +++ b/http/account.go @@ -161,7 +161,7 @@ func accountHandle(w http.ResponseWriter, r *http.Request) { log.Errorx("exporting mail close", err) } }() - if err := acc.ExportMessages(log, archiver, maildir, ""); err != nil { + if err := store.ExportMessages(log, acc.DB, acc.Dir, archiver, maildir, ""); err != nil { log.Errorx("exporting mail", err) } diff --git a/store/export.go b/store/export.go index e5e4a20..384e80b 100644 --- a/store/export.go +++ b/store/export.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "sort" + "strings" "time" "github.com/mjl-/bstore" @@ -19,7 +20,9 @@ import ( // Archiver can archive multiple mailboxes and their messages. type Archiver interface { - Create(name string, size int64, mtime time.Time) (io.Writer, error) + // Add file to archive. If name ends with a slash, it is created as a directory and + // the returned io.WriteCloser can be ignored. + Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) Close() error } @@ -29,18 +32,18 @@ type TarArchiver struct { } // Create adds a file header to the tar file. -func (a TarArchiver) Create(name string, size int64, mtime time.Time) (io.Writer, error) { +func (a TarArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) { hdr := tar.Header{ Name: name, Size: size, - Mode: 0600, + Mode: 0660, ModTime: mtime, Format: tar.FormatPAX, } if err := a.WriteHeader(&hdr); err != nil { return nil, err } - return a, nil + return nopCloser{a}, nil } // ZipArchiver is an Archiver that writes to a zip file. @@ -49,14 +52,49 @@ type ZipArchiver struct { } // Create adds a file header to the zip file. -func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.Writer, error) { +func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) { hdr := zip.FileHeader{ Name: name, Method: zip.Deflate, Modified: mtime, UncompressedSize64: uint64(size), } - return a.CreateHeader(&hdr) + w, err := a.CreateHeader(&hdr) + if err != nil { + return nil, err + } + return nopCloser{w}, nil +} + +type nopCloser struct { + io.Writer +} + +// Close does nothing. +func (nopCloser) Close() error { + return nil +} + +// DirArchiver is an Archiver that writes to a directory. +type DirArchiver struct { + Dir string +} + +// Create create name in the file system, in dir. +func (a DirArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) { + isdir := strings.HasSuffix(name, "/") + name = strings.TrimSuffix(name, "/") + p := filepath.Join(a.Dir, name) + os.MkdirAll(filepath.Dir(p), 0770) + if isdir { + return nil, os.Mkdir(p, 0770) + } + return os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0660) +} + +// Close on a dir does nothing. +func (a DirArchiver) Close() error { + return nil } // ExportMessages writes messages to archiver. Either in maildir format, or otherwise in @@ -66,11 +104,11 @@ func (a ZipArchiver) Create(name string, size int64, mtime time.Time) (io.Writer // Some errors are not fatal and result in skipped messages. In that happens, a // file "errors.txt" is added to the archive describing the errors. The goal is to // let users export (hopefully) most messages even in the face of errors. -func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, mailboxOpt string) error { +func ExportMessages(log *mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error { // Start transaction without closure, we are going to close it early, but don't // want to deal with declaring many variables now to be able to assign them in a // closure and use them afterwards. - tx, err := a.DB.Begin(false) + tx, err := db.Begin(false) if err != nil { return fmt.Errorf("transaction: %v", err) } @@ -173,7 +211,48 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, } }() - finishMbox := func() error { + // For dovecot-keyword-style flags not in standard maildir. + maildirFlags := map[string]int{} + var maildirFlaglist []string + maildirFlag := func(flag string) string { + i, ok := maildirFlags[flag] + if !ok { + if len(maildirFlags) >= 26 { + // Max 26 flag characters. + return "" + } + i = len(maildirFlags) + maildirFlags[flag] = i + maildirFlaglist = append(maildirFlaglist, flag) + } + return string(rune('a' + i)) + } + + finishMailbox := func() error { + if maildir { + if len(maildirFlags) == 0 { + return nil + } + + var b bytes.Buffer + for i, flag := range maildirFlaglist { + if _, err := fmt.Fprintf(&b, "%d %s\n", i, flag); err != nil { + return err + } + } + w, err := archiver.Create(curMailbox+"/dovecot-keywords", int64(b.Len()), start) + if err != nil { + return fmt.Errorf("adding dovecot-keywords: %v", err) + } + if _, err := w.Write(b.Bytes()); err != nil { + w.Close() + return fmt.Errorf("writing dovecot-keywords: %v", err) + } + maildirFlags = map[string]int{} + maildirFlaglist = nil + return w.Close() + } + if mboxtmp == nil { return nil } @@ -193,8 +272,12 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, return fmt.Errorf("add mbox to archive: %v", err) } if _, err := io.Copy(w, mboxtmp); err != nil { + w.Close() return fmt.Errorf("copying temp mbox file to archive: %v", err) } + if err := w.Close(); err != nil { + return fmt.Errorf("closing message file: %v", err) + } if err := mboxtmp.Close(); err != nil { log.Errorx("closing temporary mbox file", err) // Continue, not fatal. @@ -205,7 +288,7 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, } exportMessage := func(m Message) error { - mp := a.MessagePath(m.ID) + mp := filepath.Join(accountDir, "msg", MessagePath(m.ID)) var mr io.ReadCloser if m.Size == int64(len(m.MsgPrefix)) { mr = io.NopCloser(bytes.NewReader(m.MsgPrefix)) @@ -236,19 +319,41 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, p = filepath.Join(p, "new") } name := fmt.Sprintf("%d.%d.mox:2,", m.Received.Unix(), m.ID) - // todo: more flags? forwarded, (non)junk, phishing, mdnsent would be nice. but what is the convention. dovecot-keywords sounds non-standard. - if m.Flags.Seen { - name += "S" - } - if m.Flags.Answered { - name += "R" + + // Standard flags. May need to be sorted. + if m.Flags.Draft { + name += "D" } if m.Flags.Flagged { name += "F" } - if m.Flags.Draft { - name += "D" + if m.Flags.Answered { + name += "R" } + if m.Flags.Seen { + name += "S" + } + if m.Flags.Deleted { + name += "T" + } + + // Non-standard flag. We set them with a dovecot-keywords file. + if m.Flags.Forwarded { + name += maildirFlag("$Forwarded") + } + if m.Flags.Junk { + name += maildirFlag("$Junk") + } + if m.Flags.Notjunk { + name += maildirFlag("$NotJunk") + } + if m.Flags.Phishing { + name += maildirFlag("$Phishing") + } + if m.Flags.MDNSent { + name += maildirFlag("$MDNSent") + } + p = filepath.Join(p, name) // We store messages with \r\n, maildir needs without. But we need to know the @@ -281,9 +386,10 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, return fmt.Errorf("adding message to archive: %v", err) } if _, err := io.Copy(w, &dst); err != nil { + w.Close() return fmt.Errorf("copying message to archive: %v", err) } - return nil + return w.Close() } // todo: should we put status flags in Status or X-Status header inside the message? @@ -327,7 +433,7 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, for _, m := range msgs { if m.MailboxID != curMailboxID { - if err := finishMbox(); err != nil { + if err := finishMailbox(); err != nil { return err } @@ -362,7 +468,7 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, return err } } - if err := finishMbox(); err != nil { + if err := finishMailbox(); err != nil { return err } @@ -373,9 +479,13 @@ func (a *Account) ExportMessages(log *mlog.Log, archiver Archiver, maildir bool, return err } if _, err := w.Write([]byte(errors)); err != nil { + w.Close() log.Errorx("writing errors.txt to archive", err) return err } + if err := w.Close(); err != nil { + return err + } } return nil diff --git a/store/export_test.go b/store/export_test.go index da9d647..f00321a 100644 --- a/store/export_test.go +++ b/store/export_test.go @@ -5,7 +5,9 @@ import ( "archive/zip" "bytes" "io" + "io/fs" "os" + "path/filepath" "testing" "time" @@ -47,16 +49,21 @@ func TestExport(t *testing.T) { archive := func(archiver Archiver, maildir bool) { t.Helper() - err = acc.ExportMessages(log, archiver, maildir, "") + err = ExportMessages(log, acc.DB, acc.Dir, archiver, maildir, "") tcheck(t, err, "export messages") err = archiver.Close() tcheck(t, err, "archiver close") } + os.RemoveAll("../testdata/exportmaildir") + os.RemoveAll("../testdata/exportmbox") + archive(ZipArchiver{zip.NewWriter(&maildirZip)}, true) archive(ZipArchiver{zip.NewWriter(&mboxZip)}, false) archive(TarArchiver{tar.NewWriter(&maildirTar)}, true) archive(TarArchiver{tar.NewWriter(&mboxTar)}, false) + archive(DirArchiver{"../testdata/exportmaildir"}, true) + archive(DirArchiver{"../testdata/exportmbox"}, false) if r, err := zip.NewReader(bytes.NewReader(maildirZip.Bytes()), int64(maildirZip.Len())); err != nil { t.Fatalf("reading maildir zip: %v", err) @@ -86,11 +93,29 @@ func TestExport(t *testing.T) { _, err = io.Copy(io.Discard, tr) tcheck(t, err, "copy") } - if n != have { - t.Fatalf("got %d files, expected %d", n, have) + if have != n { + t.Fatalf("got %d files, expected %d", have, n) } } checkTarFiles(&maildirTar, 2*3+2) checkTarFiles(&mboxTar, 2) + + checkDirFiles := func(dir string, n int) { + t.Helper() + have := 0 + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() { + have++ + } + return nil + }) + tcheck(t, err, "walkdir") + if n != have { + t.Fatalf("got %d files, expected %d", have, n) + } + } + + checkDirFiles("../testdata/exportmaildir", 2) + checkDirFiles("../testdata/exportmbox", 2) }