diff --git a/ctl_test.go b/ctl_test.go index aaa01bb..ff03e78 100644 --- a/ctl_test.go +++ b/ctl_test.go @@ -306,8 +306,8 @@ func TestCtl(t *testing.T) { }) // Export data, import it again - xcmdExport(true, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog}) - xcmdExport(false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog}) + xcmdExport(true, false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog}) + xcmdExport(false, false, []string{filepath.FromSlash("testdata/ctl/data/tmp/export/maildir/"), filepath.FromSlash("testdata/ctl/data/accounts/mjl")}, &cmd{log: pkglog}) testctl(func(ctl *ctl) { ctlcmdImport(ctl, true, "mjl", "inbox", filepath.FromSlash("testdata/ctl/data/tmp/export/mbox/Inbox.mbox")) }) diff --git a/doc.go b/doc.go index 0e513ec..3dcb845 100644 --- a/doc.go +++ b/doc.go @@ -51,8 +51,8 @@ any parameters. Followed by the help and usage information for each command. mox queue webhook retired print id mox import maildir accountname mailboxname maildir mox import mbox accountname mailboxname mbox - mox export maildir dst-dir account-path [mailbox] - mox export mbox dst-dir account-path [mailbox] + mox export maildir [-single] dst-dir account-path [mailbox] + mox export mbox [-single] dst-dir account-path [mailbox] mox localserve mox help [command ...] mox backup dest-dir @@ -724,9 +724,11 @@ Export one or all mailboxes from an account in maildir format. Export bypasses a running mox instance. It opens the account mailbox/message database file directly. This may block if a running mox instance also has the database open, e.g. for IMAP connections. To export from a running instance, use -the accounts web page. +the accounts web page or webmail. - usage: mox export maildir dst-dir account-path [mailbox] + usage: mox export maildir [-single] dst-dir account-path [mailbox] + -single + export single mailbox, without any children. disabled if mailbox isn't specified. # mox export mbox @@ -737,13 +739,15 @@ Using mbox is not recommended. Maildir is a better format. Export bypasses a running mox instance. It opens the account mailbox/message database file directly. This may block if a running mox instance also has the database open, e.g. for IMAP connections. To export from a running instance, use -the accounts web page. +the accounts web page or webmail. For mbox export, "mboxrd" is used where message lines starting with the magic "From " string are escaped by prepending a >. All ">*From " are escaped, otherwise reconstructing the original could lose a ">". - usage: mox export mbox dst-dir account-path [mailbox] + usage: mox export mbox [-single] dst-dir account-path [mailbox] + -single + export single mailbox, without any children. disabled if mailbox isn't specified. # mox localserve diff --git a/export.go b/export.go index 0368a2c..1be5c5f 100644 --- a/export.go +++ b/export.go @@ -12,20 +12,22 @@ import ( ) func cmdExportMaildir(c *cmd) { - c.params = "dst-dir account-path [mailbox]" + c.params = "[-single] dst-dir account-path [mailbox]" c.help = `Export one or all mailboxes from an account in maildir format. Export bypasses a running mox instance. It opens the account mailbox/message database file directly. This may block if a running mox instance also has the database open, e.g. for IMAP connections. To export from a running instance, use -the accounts web page. +the accounts web page or webmail. ` + var single bool + c.flag.BoolVar(&single, "single", false, "export single mailbox, without any children. disabled if mailbox isn't specified.") args := c.Parse() - xcmdExport(false, args, c) + xcmdExport(false, single, args, c) } func cmdExportMbox(c *cmd) { - c.params = "dst-dir account-path [mailbox]" + c.params = "[-single] dst-dir account-path [mailbox]" c.help = `Export messages from one or all mailboxes in an account in mbox format. Using mbox is not recommended. Maildir is a better format. @@ -33,17 +35,19 @@ Using mbox is not recommended. Maildir is a better format. Export bypasses a running mox instance. It opens the account mailbox/message database file directly. This may block if a running mox instance also has the database open, e.g. for IMAP connections. To export from a running instance, use -the accounts web page. +the accounts web page or webmail. For mbox export, "mboxrd" is used where message lines starting with the magic "From " string are escaped by prepending a >. All ">*From " are escaped, otherwise reconstructing the original could lose a ">". ` + var single bool + c.flag.BoolVar(&single, "single", false, "export single mailbox, without any children. disabled if mailbox isn't specified.") args := c.Parse() - xcmdExport(true, args, c) + xcmdExport(true, single, args, c) } -func xcmdExport(mbox bool, args []string, c *cmd) { +func xcmdExport(mbox, single bool, args []string, c *cmd) { if len(args) != 2 && len(args) != 3 { c.Usage() } @@ -53,6 +57,8 @@ func xcmdExport(mbox bool, args []string, c *cmd) { var mailbox string if len(args) == 3 { mailbox = args[2] + } else { + single = false } dbpath := filepath.Join(accountDir, "index.db") @@ -65,7 +71,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) { }() a := store.DirArchiver{Dir: dst} - err = store.ExportMessages(context.Background(), c.log, db, accountDir, a, !mbox, mailbox) + err = store.ExportMessages(context.Background(), c.log, db, accountDir, a, !mbox, mailbox, !single) xcheckf(err, "exporting messages") err = a.Close() xcheckf(err, "closing archiver") diff --git a/store/export.go b/store/export.go index 80f3229..447bd58 100644 --- a/store/export.go +++ b/store/export.go @@ -10,8 +10,8 @@ import ( "io" "log/slog" "os" + "path" "path/filepath" - "sort" "strings" "time" @@ -28,7 +28,7 @@ type Archiver interface { Close() error } -// TarArchiver is an Archiver that writes to a tar ifle. +// TarArchiver is an Archiver that writes to a tar file. type TarArchiver struct { *tar.Writer } @@ -82,7 +82,7 @@ type DirArchiver struct { Dir string } -// Create create name in the file system, in dir. +// Create creates name in the file system, in dir. // name must always use forwarded slashes. func (a DirArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) { isdir := strings.HasSuffix(name, "/") @@ -100,6 +100,28 @@ func (a DirArchiver) Close() error { return nil } +// MboxArchive fakes being an archiver to which a single mbox file can be written. +// It returns an error when a second file is added. It returns its writer for the +// first file to be written, leaving parameters unused. +type MboxArchiver struct { + Writer io.Writer + have bool +} + +// Create returns the underlying writer for the first call, and an error on later calls. +func (a *MboxArchiver) Create(name string, size int64, mtime time.Time) (io.WriteCloser, error) { + if a.have { + return nil, fmt.Errorf("cannot export multiple files with mbox") + } + a.have = true + return nopCloser{a.Writer}, nil +} + +// Close on an mbox archiver does nothing. +func (a *MboxArchiver) Close() error { + return nil +} + // ExportMessages writes messages to archiver. Either in maildir format, or otherwise in // mbox. If mailboxOpt is empty, all mailboxes are exported, otherwise only the // named mailbox. @@ -107,7 +129,7 @@ func (a DirArchiver) Close() error { // Some errors are not fatal and result in skipped messages. In that happens, a // file "errors.txt" is added to the archive describing the errors. The goal is to // let users export (hopefully) most messages even in the face of errors. -func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error { +func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string, recursive bool) error { // todo optimize: should prepare next file to add to archive (can be an mbox with many messages) while writing a file to the archive (which typically compresses, which takes time). // Start transaction without closure, we are going to close it early, but don't @@ -118,89 +140,12 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir return fmt.Errorf("transaction: %v", err) } defer func() { - if tx != nil { - err := tx.Rollback() - log.Check(err, "transaction rollback after export error") - } + err := tx.Rollback() + log.Check(err, "transaction rollback") }() start := time.Now() - // Set up mailbox names and ids. - id2name := map[int64]string{} - name2id := map[string]int64{} - - mailboxes, err := bstore.QueryTx[Mailbox](tx).List() - if err != nil { - return fmt.Errorf("query mailboxes: %w", err) - } - for _, mb := range mailboxes { - id2name[mb.ID] = mb.Name - name2id[mb.Name] = mb.ID - } - - var mailboxID int64 - if mailboxOpt != "" { - var ok bool - mailboxID, ok = name2id[mailboxOpt] - if !ok { - return fmt.Errorf("mailbox not found") - } - } - - var names []string - for _, name := range id2name { - if mailboxOpt != "" && name != mailboxOpt { - continue - } - names = append(names, name) - } - // We need to sort the names because maildirs can create subdirs. Ranging over - // id2name directly would randomize the directory names, we would create a sub - // maildir before the parent, and fail with "dir exists" when creating the parent - // dir. - sort.Slice(names, func(i, j int) bool { - return names[i] < names[j] - }) - - mailboxOrder := map[int64]int{} - for i, name := range names { - mbID := name2id[name] - mailboxOrder[mbID] = i - } - - // Fetch all messages. This can take quite a bit of memory if the mailbox is large. - q := bstore.QueryTx[Message](tx) - if mailboxID > 0 { - q.FilterNonzero(Message{MailboxID: mailboxID}) - } - msgs, err := q.List() - if err != nil { - return fmt.Errorf("listing messages: %v", err) - } - - // Close transaction. We don't want to hold it for too long. We are now at risk - // that a message is be removed while we export, or flags changed. At least the - // size won't change. If we cannot open the message later on, we'll skip it and add - // an error message to an errors.txt file in the output archive. - if err := tx.Rollback(); err != nil { - return fmt.Errorf("closing transaction: %v", err) - } - tx = nil - - // Order the messages by mailbox, received time and finally message ID. - sort.Slice(msgs, func(i, j int) bool { - iid := msgs[i].MailboxID - jid := msgs[j].MailboxID - if iid != jid { - return mailboxOrder[iid] < mailboxOrder[jid] - } - if !msgs[i].Received.Equal(msgs[j].Received) { - return msgs[i].Received.Before(msgs[j].Received) - } - return msgs[i].ID < msgs[j].ID - }) - // We keep track of errors reading message files. We continue exporting and add an // errors.txt file to the archive. In case of errors, the user can get (hopefully) // most of their emails, and see something went wrong. For other errors, like @@ -208,8 +153,55 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir // continue with useless work. var errors string - var curMailboxID int64 // Used to set curMailbox and finish a previous mbox file. - var curMailbox string + // Process mailboxes sorted by name, so submaildirs come after their parent. + prefix := mailboxOpt + "/" + var trimPrefix string + if mailboxOpt != "" { + // If exporting a specific mailbox, trim its parent path from stored file names. + trimPrefix = path.Dir(mailboxOpt) + "/" + } + q := bstore.QueryTx[Mailbox](tx) + q.FilterFn(func(mb Mailbox) bool { + return mailboxOpt == "" || mb.Name == mailboxOpt || recursive && strings.HasPrefix(mb.Name, prefix) + }) + q.SortAsc("Name") + err = q.ForEach(func(mb Mailbox) error { + mailboxName := mb.Name + if trimPrefix != "" { + mailboxName = strings.TrimPrefix(mailboxName, trimPrefix) + } + errmsgs, err := exportMailbox(log, tx, accountDir, mb.ID, mailboxName, archiver, maildir, start) + if err != nil { + return err + } + errors += errmsgs + return nil + }) + if err != nil { + return fmt.Errorf("query mailboxes: %w", err) + } + + if errors != "" { + w, err := archiver.Create("errors.txt", int64(len(errors)), time.Now()) + if err != nil { + log.Errorx("adding errors.txt to archive", err) + return err + } + if _, err := w.Write([]byte(errors)); err != nil { + log.Errorx("writing errors.txt to archive", err) + xerr := w.Close() + log.Check(xerr, "closing errors.txt after error") + return err + } + if err := w.Close(); err != nil { + return err + } + } + return nil +} + +func exportMailbox(log mlog.Log, tx *bstore.Tx, accountDir string, mailboxID int64, mailboxName string, archiver Archiver, maildir bool, start time.Time) (string, error) { + var errors string var mboxtmp *os.File var mboxwriter *bufio.Writer @@ -248,7 +240,7 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir return err } } - w, err := archiver.Create(curMailbox+"/dovecot-keywords", int64(b.Len()), start) + w, err := archiver.Create(mailboxName+"/dovecot-keywords", int64(b.Len()), start) if err != nil { return fmt.Errorf("adding dovecot-keywords: %v", err) } @@ -262,10 +254,6 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir return w.Close() } - if mboxtmp == nil { - return nil - } - if err := mboxwriter.Flush(); err != nil { return fmt.Errorf("flush mbox writer: %v", err) } @@ -276,7 +264,7 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir if _, err := mboxtmp.Seek(0, 0); err != nil { return fmt.Errorf("seek to start of temporary mbox file") } - w, err := archiver.Create(curMailbox+".mbox", fi.Size(), fi.ModTime()) + w, err := archiver.Create(mailboxName+".mbox", fi.Size(), fi.ModTime()) if err != nil { return fmt.Errorf("add mbox to archive: %v", err) } @@ -326,7 +314,7 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir } if maildir { - p := curMailbox + p := mailboxName if m.Flags.Seen { p = filepath.Join(p, "cur") } else { @@ -378,7 +366,7 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir for { line, rerr := r.ReadBytes('\n') if rerr != io.EOF && rerr != nil { - errors += fmt.Sprintf("reading from message for id %d: %v (message skipped)\n", m.ID, err) + errors += fmt.Sprintf("reading from message for id %d: %v (message skipped)\n", m.ID, rerr) return nil } if len(line) > 0 { @@ -386,7 +374,7 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir line = line[:len(line)-1] line[len(line)-1] = '\n' } - if _, err = dst.Write(line); err != nil { + if _, err := dst.Write(line); err != nil { return fmt.Errorf("writing message: %v", err) } } @@ -466,7 +454,7 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir for { line, rerr := r.ReadBytes('\n') if rerr != io.EOF && rerr != nil { - return fmt.Errorf("reading message: %v", err) + return fmt.Errorf("reading message: %v", rerr) } if len(line) > 0 { if bytes.HasSuffix(line, []byte("\r\n")) { @@ -503,59 +491,40 @@ func ExportMessages(ctx context.Context, log mlog.Log, db *bstore.DB, accountDir return nil } - for _, m := range msgs { - if m.MailboxID != curMailboxID { - if err := finishMailbox(); err != nil { - return err - } - - curMailbox = id2name[m.MailboxID] - curMailboxID = m.MailboxID - if maildir { - // Create the directories that show this is a maildir. - if _, err := archiver.Create(curMailbox+"/new/", 0, start); err != nil { - return fmt.Errorf("adding maildir new directory: %v", err) - } - if _, err := archiver.Create(curMailbox+"/cur/", 0, start); err != nil { - return fmt.Errorf("adding maildir cur directory: %v", err) - } - if _, err := archiver.Create(curMailbox+"/tmp/", 0, start); err != nil { - return fmt.Errorf("adding maildir tmp directory: %v", err) - } - } else { - - mboxtmp, err = os.CreateTemp("", "mox-mail-export-mbox") - if err != nil { - return fmt.Errorf("creating temp mbox file: %v", err) - } - mboxwriter = bufio.NewWriter(mboxtmp) - } + if maildir { + // Create the directories that show this is a maildir. + if _, err := archiver.Create(mailboxName+"/new/", 0, start); err != nil { + return errors, fmt.Errorf("adding maildir new directory: %v", err) } - - if err := exportMessage(m); err != nil { - return err + if _, err := archiver.Create(mailboxName+"/cur/", 0, start); err != nil { + return errors, fmt.Errorf("adding maildir cur directory: %v", err) } + if _, err := archiver.Create(mailboxName+"/tmp/", 0, start); err != nil { + return errors, fmt.Errorf("adding maildir tmp directory: %v", err) + } + } else { + var err error + mboxtmp, err = os.CreateTemp("", "mox-mail-export-mbox") + if err != nil { + return errors, fmt.Errorf("creating temp mbox file: %v", err) + } + mboxwriter = bufio.NewWriter(mboxtmp) + } + + // Fetch all messages for mailbox. + q := bstore.QueryTx[Message](tx) + q.FilterNonzero(Message{MailboxID: mailboxID}) + q.FilterEqual("Expunged", false) + q.SortAsc("Received", "ID") + err := q.ForEach(func(m Message) error { + return exportMessage(m) + }) + if err != nil { + return errors, err } if err := finishMailbox(); err != nil { - return err + return errors, err } - if errors != "" { - w, err := archiver.Create("errors.txt", int64(len(errors)), time.Now()) - if err != nil { - log.Errorx("adding errors.txt to archive", err) - return err - } - if _, err := w.Write([]byte(errors)); err != nil { - log.Errorx("writing errors.txt to archive", err) - xerr := w.Close() - log.Check(xerr, "closing errors.txt after error") - return err - } - if err := w.Close(); err != nil { - return err - } - } - - return nil + return errors, nil } diff --git a/store/export_test.go b/store/export_test.go index 3afb3d1..e5c50f3 100644 --- a/store/export_test.go +++ b/store/export_test.go @@ -19,18 +19,20 @@ func TestExport(t *testing.T) { // Set up an account, add 2 messages to different 2 mailboxes. export as tar/zip // and maildir/mbox. check there are 2 files in the repo, no errors.txt. + log := mlog.New("export", nil) + os.RemoveAll("../testdata/store/data") mox.ConfigStaticPath = filepath.FromSlash("../testdata/store/mox.conf") mox.MustLoadConfig(true, false) acc, err := OpenAccount(pkglog, "mjl") tcheck(t, err, "open account") defer func() { - acc.Close() + err := acc.Close() + log.Check(err, "closing account") acc.CheckClosed() }() defer Switchboard()() - log := mlog.New("export", nil) msgFile, err := CreateMessageTemp(pkglog, "mox-test-export") tcheck(t, err, "create temp") @@ -52,7 +54,7 @@ func TestExport(t *testing.T) { archive := func(archiver Archiver, maildir bool) { t.Helper() - err = ExportMessages(ctxbg, log, acc.DB, acc.Dir, archiver, maildir, "") + err = ExportMessages(ctxbg, log, acc.DB, acc.Dir, archiver, maildir, "", true) tcheck(t, err, "export messages") err = archiver.Close() tcheck(t, err, "archiver close") @@ -68,16 +70,17 @@ func TestExport(t *testing.T) { archive(DirArchiver{filepath.FromSlash("../testdata/exportmaildir")}, true) archive(DirArchiver{filepath.FromSlash("../testdata/exportmbox")}, false) + const defaultMailboxes = 6 // Inbox, Drafts, etc if r, err := zip.NewReader(bytes.NewReader(maildirZip.Bytes()), int64(maildirZip.Len())); err != nil { t.Fatalf("reading maildir zip: %v", err) - } else if len(r.File) != 2*3+2 { - t.Fatalf("maildir zip, expected 2*3 dirs, and 2 files, got %d files", len(r.File)) + } else if len(r.File) != defaultMailboxes*3+2 { + t.Fatalf("maildir zip, expected %d*3 dirs, and 2 files, got %d files", defaultMailboxes, len(r.File)) } if r, err := zip.NewReader(bytes.NewReader(mboxZip.Bytes()), int64(mboxZip.Len())); err != nil { t.Fatalf("reading mbox zip: %v", err) - } else if len(r.File) != 2 { - t.Fatalf("maildir zip, 2 files, got %d files", len(r.File)) + } else if len(r.File) != defaultMailboxes { + t.Fatalf("maildir zip, expected %d files, got %d files", defaultMailboxes, len(r.File)) } checkTarFiles := func(r io.Reader, n int) { @@ -101,8 +104,8 @@ func TestExport(t *testing.T) { } } - checkTarFiles(&maildirTar, 2*3+2) - checkTarFiles(&mboxTar, 2) + checkTarFiles(&maildirTar, defaultMailboxes*3+2) + checkTarFiles(&mboxTar, defaultMailboxes) checkDirFiles := func(dir string, n int) { t.Helper() @@ -120,5 +123,5 @@ func TestExport(t *testing.T) { } checkDirFiles(filepath.FromSlash("../testdata/exportmaildir"), 2) - checkDirFiles(filepath.FromSlash("../testdata/exportmbox"), 2) + checkDirFiles(filepath.FromSlash("../testdata/exportmbox"), defaultMailboxes) } diff --git a/webaccount/account.go b/webaccount/account.go index f069023..f986865 100644 --- a/webaccount/account.go +++ b/webaccount/account.go @@ -3,10 +3,7 @@ package webaccount import ( - "archive/tar" - "archive/zip" "bytes" - "compress/gzip" "context" cryptorand "crypto/rand" "encoding/base64" @@ -39,6 +36,7 @@ import ( "github.com/mjl-/mox/webapi" "github.com/mjl-/mox/webauth" "github.com/mjl-/mox/webhook" + "github.com/mjl-/mox/webops" ) var pkglog = mlog.New("webaccount", nil) @@ -218,7 +216,7 @@ func handle(apiHandler http.Handler, isForwarded bool, w http.ResponseWriter, r // All other URLs, except the login endpoint require some authentication. if r.URL.Path != "/api/LoginPrep" && r.URL.Path != "/api/Login" { var ok bool - isExport := strings.HasPrefix(r.URL.Path, "/export/") + isExport := r.URL.Path == "/export" requireCSRF := isAPI || r.URL.Path == "/import" || isExport accName, sessionToken, loginAddress, ok = webauth.Check(ctx, log, webauth.Accounts, "webaccount", isForwarded, w, r, isAPI, requireCSRF, isExport) if !ok { @@ -235,47 +233,8 @@ func handle(apiHandler http.Handler, isForwarded bool, w http.ResponseWriter, r } switch r.URL.Path { - case "/export/mail-export-maildir.tgz", "/export/mail-export-maildir.zip", "/export/mail-export-mbox.tgz", "/export/mail-export-mbox.zip": - if r.Method != "POST" { - http.Error(w, "405 - method not allowed - use post", http.StatusMethodNotAllowed) - return - } - - maildir := strings.Contains(r.URL.Path, "maildir") - tgz := strings.Contains(r.URL.Path, ".tgz") - - acc, err := store.OpenAccount(log, accName) - if err != nil { - log.Errorx("open account for export", err) - http.Error(w, "500 - internal server error", http.StatusInternalServerError) - return - } - defer func() { - err := acc.Close() - log.Check(err, "closing account") - }() - - var archiver store.Archiver - if tgz { - // Don't tempt browsers to "helpfully" decompress. - w.Header().Set("Content-Type", "application/octet-stream") - - gzw := gzip.NewWriter(w) - defer func() { - _ = gzw.Close() - }() - archiver = store.TarArchiver{Writer: tar.NewWriter(gzw)} - } else { - w.Header().Set("Content-Type", "application/zip") - archiver = store.ZipArchiver{Writer: zip.NewWriter(w)} - } - defer func() { - err := archiver.Close() - log.Check(err, "exporting mail close") - }() - if err := store.ExportMessages(r.Context(), log, acc.DB, acc.Dir, archiver, maildir, ""); err != nil { - log.Errorx("exporting mail", err) - } + case "/export": + webops.Export(log, accName, w, r) case "/import": if r.Method != "POST" { diff --git a/webaccount/account.js b/webaccount/account.js index e6ebc78..1d835b9 100644 --- a/webaccount/account.js +++ b/webaccount/account.js @@ -1233,9 +1233,6 @@ const index = async () => { }); }); }; - const exportForm = (filename) => { - return dom.form(attr.target('_blank'), attr.method('POST'), attr.action('export/' + filename), dom.input(attr.type('hidden'), attr.name('csrf'), attr.value(localStorageGet('webaccountcsrftoken') || '')), dom.submitbutton('Export')); - }; const authorizationPopup = (dest) => { let username; let password; @@ -1474,7 +1471,7 @@ const index = async () => { }), dom.table(dom.thead(dom.tr(dom.th('Address', attr.title('Address that caused this entry to be added to the list. The title (shown on hover) displays an address with a fictional simplified localpart, with lower-cased, dots removed, only first part before "+" or "-" (typicaly catchall separators). When checking if an address is on the suppression list, it is checked against this address.')), dom.th('Manual', attr.title('Whether suppression was added manually, instead of automatically based on bounces.')), dom.th('Reason'), dom.th('Since'), dom.th('Action'))), dom.tbody((suppressions || []).length === 0 ? dom.tr(dom.td(attr.colspan('5'), '(None)')) : [], (suppressions || []).map(s => dom.tr(dom.td(s.OriginalAddress, attr.title(s.BaseAddress)), dom.td(s.Manual ? '✓' : ''), dom.td(s.Reason), dom.td(age(s.Created)), dom.td(dom.clickbutton('Remove', async function click(e) { await check(e.target, client.SuppressionRemove(s.OriginalAddress)); window.location.reload(); // todo: reload less - }))))), dom.tfoot(dom.tr(dom.td(suppressionAddress = dom.input(attr.type('required'), attr.form('suppressionAdd'))), dom.td(), dom.td(suppressionReason = dom.input(style({ width: '100%' }), attr.form('suppressionAdd'))), dom.td(), dom.td(dom.submitbutton('Add suppression', attr.form('suppressionAdd')))))), dom.br(), dom.h2('Export'), dom.p('Export all messages in all mailboxes. In maildir or mbox format, as .zip or .tgz file.'), dom.table(dom._class('slim'), dom.tr(dom.td('Maildirs in .tgz'), dom.td(exportForm('mail-export-maildir.tgz'))), dom.tr(dom.td('Maildirs in .zip'), dom.td(exportForm('mail-export-maildir.zip'))), dom.tr(dom.td('Mbox files in .tgz'), dom.td(exportForm('mail-export-mbox.tgz'))), dom.tr(dom.td('Mbox files in .zip'), dom.td(exportForm('mail-export-mbox.zip')))), dom.br(), dom.h2('Import'), dom.p('Import messages from a .zip or .tgz file with maildirs and/or mbox files.'), importForm = dom.form(async function submit(e) { + }))))), dom.tfoot(dom.tr(dom.td(suppressionAddress = dom.input(attr.type('required'), attr.form('suppressionAdd'))), dom.td(), dom.td(suppressionReason = dom.input(style({ width: '100%' }), attr.form('suppressionAdd'))), dom.td(), dom.td(dom.submitbutton('Add suppression', attr.form('suppressionAdd')))))), dom.br(), dom.h2('Export'), dom.p('Export all messages in all mailboxes.'), dom.form(attr.target('_blank'), attr.method('POST'), attr.action('export'), dom.input(attr.type('hidden'), attr.name('csrf'), attr.value(localStorageGet('webaccountcsrftoken') || '')), dom.input(attr.type('hidden'), attr.name('mailbox'), attr.value('')), dom.input(attr.type('hidden'), attr.name('recursive'), attr.value('on')), dom.div(style({ display: 'flex', flexDirection: 'column', gap: '.5ex' }), dom.div(dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('maildir'), attr.checked('')), ' Maildir'), ' ', dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('mbox')), ' Mbox')), dom.div(dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tar')), ' Tar'), ' ', dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tgz'), attr.checked('')), ' Tgz'), ' ', dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('zip')), ' Zip'), ' '), dom.div(style({ marginTop: '1ex' }), dom.submitbutton('Export')))), dom.br(), dom.h2('Import'), dom.p('Import messages from a .zip or .tgz file with maildirs and/or mbox files.'), importForm = dom.form(async function submit(e) { e.preventDefault(); e.stopPropagation(); const request = async () => { diff --git a/webaccount/account.ts b/webaccount/account.ts index 7f279f1..302b2b5 100644 --- a/webaccount/account.ts +++ b/webaccount/account.ts @@ -477,14 +477,6 @@ const index = async () => { }) } - const exportForm = (filename: string) => { - return dom.form( - attr.target('_blank'), attr.method('POST'), attr.action('export/'+filename), - dom.input(attr.type('hidden'), attr.name('csrf'), attr.value(localStorageGet('webaccountcsrftoken') || '')), - dom.submitbutton('Export'), - ) - } - const authorizationPopup = (dest: HTMLInputElement) => { let username: HTMLInputElement let password: HTMLInputElement @@ -1148,23 +1140,24 @@ const index = async () => { dom.br(), dom.h2('Export'), - dom.p('Export all messages in all mailboxes. In maildir or mbox format, as .zip or .tgz file.'), - dom.table(dom._class('slim'), - dom.tr( - dom.td('Maildirs in .tgz'), - dom.td(exportForm('mail-export-maildir.tgz')), - ), - dom.tr( - dom.td('Maildirs in .zip'), - dom.td(exportForm('mail-export-maildir.zip')), - ), - dom.tr( - dom.td('Mbox files in .tgz'), - dom.td(exportForm('mail-export-mbox.tgz')), - ), - dom.tr( - dom.td('Mbox files in .zip'), - dom.td(exportForm('mail-export-mbox.zip')), + dom.p('Export all messages in all mailboxes.'), + dom.form( + attr.target('_blank'), attr.method('POST'), attr.action('export'), + dom.input(attr.type('hidden'), attr.name('csrf'), attr.value(localStorageGet('webaccountcsrftoken') || '')), + dom.input(attr.type('hidden'), attr.name('mailbox'), attr.value('')), + dom.input(attr.type('hidden'), attr.name('recursive'), attr.value('on')), + + dom.div(style({display: 'flex', flexDirection: 'column', gap: '.5ex'}), + dom.div( + dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('maildir'), attr.checked('')), ' Maildir'), ' ', + dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('mbox')), ' Mbox'), + ), + dom.div( + dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tar')), ' Tar'), ' ', + dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tgz'), attr.checked('')), ' Tgz'), ' ', + dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('zip')), ' Zip'), ' ', + ), + dom.div(style({marginTop: '1ex'}), dom.submitbutton('Export')), ), ), dom.br(), diff --git a/webaccount/account_test.go b/webaccount/account_test.go index 232cdbb..277163f 100644 --- a/webaccount/account_test.go +++ b/webaccount/account_test.go @@ -214,12 +214,9 @@ func TestAccount(t *testing.T) { testHTTP("POST", "/import", httpHeaders{}, http.StatusForbidden, nil, nil) testHTTP("POST", "/import", httpHeaders{hdrSessionBad}, http.StatusForbidden, nil, nil) - testHTTP("GET", "/export/mail-export-maildir.tgz", httpHeaders{}, http.StatusForbidden, nil, nil) - testHTTP("GET", "/export/mail-export-maildir.tgz", httpHeaders{hdrSessionBad}, http.StatusForbidden, nil, nil) - testHTTP("GET", "/export/mail-export-maildir.tgz", httpHeaders{hdrSessionOK}, http.StatusForbidden, nil, nil) - testHTTP("GET", "/export/mail-export-maildir.zip", httpHeaders{}, http.StatusForbidden, nil, nil) - testHTTP("GET", "/export/mail-export-mbox.tgz", httpHeaders{}, http.StatusForbidden, nil, nil) - testHTTP("GET", "/export/mail-export-mbox.zip", httpHeaders{}, http.StatusForbidden, nil, nil) + testHTTP("GET", "/export", httpHeaders{}, http.StatusForbidden, nil, nil) + testHTTP("GET", "/export", httpHeaders{hdrSessionBad}, http.StatusForbidden, nil, nil) + testHTTP("GET", "/export", httpHeaders{hdrSessionOK}, http.StatusForbidden, nil, nil) // SetPassword needs the token. sessionToken := store.SessionToken(strings.SplitN(sessionCookie.Value, " ", 2)[0]) @@ -336,11 +333,17 @@ func TestAccount(t *testing.T) { return nil }) - testExport := func(httppath string, iszip bool, expectFiles int) { + testExport := func(format, archive string, expectFiles int) { t.Helper() - fields := url.Values{"csrf": []string{string(csrfToken)}} - r := httptest.NewRequest("POST", httppath, strings.NewReader(fields.Encode())) + fields := url.Values{ + "csrf": []string{string(csrfToken)}, + "format": []string{format}, + "archive": []string{archive}, + "mailbox": []string{""}, + "recursive": []string{"on"}, + } + r := httptest.NewRequest("POST", "/export", strings.NewReader(fields.Encode())) r.Header.Set("Content-Type", "application/x-www-form-urlencoded") r.Header.Add("Cookie", cookieOK.String()) w := httptest.NewRecorder() @@ -349,7 +352,7 @@ func TestAccount(t *testing.T) { t.Fatalf("export, got status code %d, expected 200: %s", w.Code, w.Body.Bytes()) } var count int - if iszip { + if archive == "zip" { buf := w.Body.Bytes() zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf))) tcheck(t, err, "reading zip") @@ -359,9 +362,13 @@ func TestAccount(t *testing.T) { } } } else { - gzr, err := gzip.NewReader(w.Body) - tcheck(t, err, "gzip reader") - tr := tar.NewReader(gzr) + var src io.Reader = w.Body + if archive == "tgz" { + gzr, err := gzip.NewReader(src) + tcheck(t, err, "gzip reader") + src = gzr + } + tr := tar.NewReader(src) for { h, err := tr.Next() if err == io.EOF { @@ -380,10 +387,10 @@ func TestAccount(t *testing.T) { } } - testExport("/export/mail-export-maildir.tgz", false, 6) // 2 mailboxes, each with 2 messages and a dovecot-keyword file - testExport("/export/mail-export-maildir.zip", true, 6) - testExport("/export/mail-export-mbox.tgz", false, 2) - testExport("/export/mail-export-mbox.zip", true, 2) + testExport("maildir", "tgz", 6) // 2 mailboxes, each with 2 messages and a dovecot-keyword file + testExport("maildir", "zip", 6) + testExport("mbox", "tar", 2+6) // 2 imported plus 6 default mailboxes (Inbox, Draft, etc) + testExport("mbox", "zip", 2+6) sl := api.SuppressionList(ctx) tcompare(t, len(sl), 0) diff --git a/webmail/webmail.go b/webmail/webmail.go index 2d6f3eb..d900201 100644 --- a/webmail/webmail.go +++ b/webmail/webmail.go @@ -39,6 +39,7 @@ import ( "github.com/mjl-/mox/moxio" "github.com/mjl-/mox/store" "github.com/mjl-/mox/webauth" + "github.com/mjl-/mox/webops" ) var pkglog = mlog.New("webmail", nil) @@ -259,7 +260,9 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt // All other URLs, except the login endpoint require some authentication. if r.URL.Path != "/api/LoginPrep" && r.URL.Path != "/api/Login" { var ok bool - accName, sessionToken, loginAddress, ok = webauth.Check(ctx, log, webauth.Accounts, "webmail", isForwarded, w, r, isAPI, isAPI, false) + isExport := r.URL.Path == "/export" + requireCSRF := isAPI || isExport + accName, sessionToken, loginAddress, ok = webauth.Check(ctx, log, webauth.Accounts, "webmail", isForwarded, w, r, isAPI, requireCSRF, isExport) if !ok { // Response has been written already. return @@ -289,10 +292,16 @@ func handle(apiHandler http.Handler, isForwarded bool, accountPath string, w htt } // We are now expecting the following URLs: + // .../export // .../msg//{attachments.zip,parsedmessage.js,raw} // .../msg//{,msg}{text,html,htmlexternal} // .../msg//{view,viewtext,download}/ + if r.URL.Path == "/export" { + webops.Export(log, accName, w, r) + return + } + if !strings.HasPrefix(r.URL.Path, "/msg/") { http.NotFound(w, r) return diff --git a/webmail/webmail.js b/webmail/webmail.js index ab6f973..4c0863f 100644 --- a/webmail/webmail.js +++ b/webmail/webmail.js @@ -1332,7 +1332,7 @@ Enable consistency checking in UI updates: - todo: buttons/mechanism to operate on all messages in a mailbox/search query, without having to list and select all messages. e.g. clearing flags/labels. - todo: can we detect if browser supports proper CSP? if not, refuse to load html messages? - todo: more search criteria? Date header field (instead of time received), text vs html (only, either or both), attachment filenames and sizes -- todo: integrate more of the account page into webmail? importing/exporting messages, configuring delivery rules (possibly with sieve). +- todo: import messages into specific mailbox? - todo: configurable keyboard shortcuts? we use strings like "ctrl p" which we already generate and match on, add a mapping from command name to cmd* functions, and have a map of keys to command names. the commands for up/down with shift/ctrl modifiers may need special attention. - todo: consider composing messages with bcc headers that are sent as message Bcc headers to the bcc-addressees, optionally with checkbox. - todo: improve accessibility @@ -5194,6 +5194,12 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p }; return mlv; }; +const popoverExport = (reference, mailboxName) => { + const removeExport = popover(reference, {}, dom.h1('Export ', mailboxName || 'all mailboxes'), dom.form(function submit() { + // If we would remove the popup immediately, the form would be deleted too and never submitted. + window.setTimeout(() => removeExport(), 100); + }, attr.target('_blank'), attr.method('POST'), attr.action('export'), dom.input(attr.type('hidden'), attr.name('csrf'), attr.value(localStorageGet('webmailcsrftoken') || '')), dom.input(attr.type('hidden'), attr.name('mailbox'), attr.value(mailboxName)), dom.div(style({ display: 'flex', flexDirection: 'column', gap: '.5ex' }), dom.div(dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('maildir'), attr.checked('')), ' Maildir'), ' ', dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('mbox')), ' Mbox')), dom.div(dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tar')), ' Tar'), ' ', dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tgz'), attr.checked('')), ' Tgz'), ' ', dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('zip')), ' Zip'), ' ', dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('none')), ' None')), dom.div(dom.label(dom.input(attr.type('checkbox'), attr.checked(''), attr.name('recursive'), attr.value('on')), ' Recursive')), dom.div(style({ marginTop: '1ex' }), dom.submitbutton('Export'))))); +}; const newMailboxView = (xmb, mailboxlistView, otherMailbox) => { const plusbox = '⊞'; const minusbox = '⊟'; @@ -5260,6 +5266,9 @@ const newMailboxView = (xmb, mailboxlistView, otherMailbox) => { await withStatus('Marking mailbox as special use', client.MailboxSetSpecialUse(mb)); }; popover(actionBtn, { transparent: true }, dom.div(style({ display: 'flex', flexDirection: 'column', gap: '.5ex' }), dom.div(dom.clickbutton('Archive', async function click() { await setUse((mb) => { mb.Archive = true; }); })), dom.div(dom.clickbutton('Draft', async function click() { await setUse((mb) => { mb.Draft = true; }); })), dom.div(dom.clickbutton('Junk', async function click() { await setUse((mb) => { mb.Junk = true; }); })), dom.div(dom.clickbutton('Sent', async function click() { await setUse((mb) => { mb.Sent = true; }); })), dom.div(dom.clickbutton('Trash', async function click() { await setUse((mb) => { mb.Trash = true; }); })))); + })), dom.div(dom.clickbutton('Export', function click() { + popoverExport(actionBtn, mbv.mailbox.Name); + remove(); })))); }; // Keep track of dragenter/dragleave ourselves, we don't get a neat 1 enter and 1 @@ -5483,13 +5492,23 @@ const newMailboxlistView = (msglistView, requestNewView, updatePageTitle, setLoc }; const root = dom.div(); const mailboxesElem = dom.div(); - dom._kids(root, dom.div(attr.role('region'), attr.arialabel('Mailboxes'), dom.div(dom.h1('Mailboxes', style({ display: 'inline', fontSize: 'inherit' })), ' ', dom.clickbutton('+', attr.arialabel('Create new mailbox.'), attr.title('Create new mailbox.'), style({ padding: '0 .25em' }), function click(e) { - let fieldset, name; - const remove = popover(e.target, {}, dom.form(async function submit(e) { - e.preventDefault(); - await withStatus('Creating mailbox', client.MailboxCreate(name.value), fieldset); + dom._kids(root, dom.div(attr.role('region'), attr.arialabel('Mailboxes'), dom.div(dom.h1('Mailboxes', style({ display: 'inline', fontSize: 'inherit' })), ' ', dom.clickbutton('...', attr.arialabel('Mailboxes actions'), attr.title('Actions on mailboxes like creating a new mailbox or exporting all email.'), function click(e) { + e.stopPropagation(); + const remove = popover(e.target, { transparent: true }, dom.div(style({ display: 'flex', flexDirection: 'column', gap: '.5ex' }), dom.div(dom.clickbutton('Create mailbox', attr.arialabel('Create new mailbox.'), attr.title('Create new mailbox.'), style({ padding: '0 .25em' }), function click(e) { + let fieldset; + let name; + const ref = e.target; + const removeCreate = popover(ref, {}, dom.form(async function submit(e) { + e.preventDefault(); + await withStatus('Creating mailbox', client.MailboxCreate(name.value), fieldset); + removeCreate(); + }, fieldset = dom.fieldset(dom.label('Name ', name = dom.input(attr.required('yes'), focusPlaceholder('Lists/Go/Nuts'))), ' ', dom.submitbutton('Create')))); remove(); - }, fieldset = dom.fieldset(dom.label('Name ', name = dom.input(attr.required('yes'), focusPlaceholder('Lists/Go/Nuts'))), ' ', dom.submitbutton('Create')))); + })), dom.div(dom.clickbutton('Export', function click(e) { + const ref = e.target; + popoverExport(ref, ''); + remove(); + })))); })), mailboxesElem)); const loadMailboxes = (mailboxes, mbnameOpt) => { mailboxViews = mailboxes.map(mb => newMailboxView(mb, mblv, otherMailbox)); diff --git a/webmail/webmail.ts b/webmail/webmail.ts index 4927eb9..9a0f763 100644 --- a/webmail/webmail.ts +++ b/webmail/webmail.ts @@ -79,7 +79,7 @@ Enable consistency checking in UI updates: - todo: buttons/mechanism to operate on all messages in a mailbox/search query, without having to list and select all messages. e.g. clearing flags/labels. - todo: can we detect if browser supports proper CSP? if not, refuse to load html messages? - todo: more search criteria? Date header field (instead of time received), text vs html (only, either or both), attachment filenames and sizes -- todo: integrate more of the account page into webmail? importing/exporting messages, configuring delivery rules (possibly with sieve). +- todo: import messages into specific mailbox? - todo: configurable keyboard shortcuts? we use strings like "ctrl p" which we already generate and match on, add a mapping from command name to cmd* functions, and have a map of keys to command names. the commands for up/down with shift/ctrl modifiers may need special attention. - todo: consider composing messages with bcc headers that are sent as message Bcc headers to the bcc-addressees, optionally with checkbox. - todo: improve accessibility @@ -4960,6 +4960,36 @@ interface MailboxView { setKeywords: (keywords: string[]) => void } +const popoverExport = (reference: HTMLElement, mailboxName: string) => { + const removeExport=popover(reference, {}, + dom.h1('Export ', mailboxName || 'all mailboxes'), + dom.form( + function submit() { + // If we would remove the popup immediately, the form would be deleted too and never submitted. + window.setTimeout(() => removeExport(), 100) + }, + attr.target('_blank'), attr.method('POST'), attr.action('export'), + dom.input(attr.type('hidden'), attr.name('csrf'), attr.value(localStorageGet('webmailcsrftoken') || '')), + dom.input(attr.type('hidden'), attr.name('mailbox'), attr.value(mailboxName)), + + dom.div(style({display: 'flex', flexDirection: 'column', gap: '.5ex'}), + dom.div( + dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('maildir'), attr.checked('')), ' Maildir'), ' ', + dom.label(dom.input(attr.type('radio'), attr.name('format'), attr.value('mbox')), ' Mbox'), + ), + dom.div( + dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tar')), ' Tar'), ' ', + dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('tgz'), attr.checked('')), ' Tgz'), ' ', + dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('zip')), ' Zip'), ' ', + dom.label(dom.input(attr.type('radio'), attr.name('archive'), attr.value('none')), ' None'), + ), + dom.div(dom.label(dom.input(attr.type('checkbox'), attr.checked(''), attr.name('recursive'), attr.value('on')), ' Recursive')), + dom.div(style({marginTop: '1ex'}), dom.submitbutton('Export')), + ), + ), + ) +} + const newMailboxView = (xmb: api.Mailbox, mailboxlistView: MailboxlistView, otherMailbox: otherMailbox): MailboxView => { const plusbox = '⊞' const minusbox = '⊟' @@ -5069,6 +5099,12 @@ const newMailboxView = (xmb: api.Mailbox, mailboxlistView: MailboxlistView, othe ) }), ), + dom.div( + dom.clickbutton('Export', function click() { + popoverExport(actionBtn, mbv.mailbox.Name) + remove() + }), + ), ), ) } @@ -5356,27 +5392,52 @@ const newMailboxlistView = (msglistView: MsglistView, requestNewView: requestNew dom.div( dom.h1('Mailboxes', style({display: 'inline', fontSize: 'inherit'})), ' ', - dom.clickbutton('+', attr.arialabel('Create new mailbox.'), attr.title('Create new mailbox.'), style({padding: '0 .25em'}), function click(e: MouseEvent) { - let fieldset: HTMLFieldSetElement, name: HTMLInputElement - const remove = popover(e.target! as HTMLElement, {}, - dom.form( - async function submit(e: SubmitEvent) { - e.preventDefault() - await withStatus('Creating mailbox', client.MailboxCreate(name.value), fieldset) - remove() - }, - fieldset=dom.fieldset( - dom.label( - 'Name ', - name=dom.input(attr.required('yes'), focusPlaceholder('Lists/Go/Nuts')), + dom.clickbutton( + '...', + attr.arialabel('Mailboxes actions'), + attr.title('Actions on mailboxes like creating a new mailbox or exporting all email.'), + function click(e: MouseEvent) { + e.stopPropagation() + + const remove = popover(e.target! as HTMLElement, {transparent: true}, + dom.div(style({display: 'flex', flexDirection: 'column', gap: '.5ex'}), + dom.div( + dom.clickbutton('Create mailbox', attr.arialabel('Create new mailbox.'), attr.title('Create new mailbox.'), style({padding: '0 .25em'}), function click(e: MouseEvent) { + let fieldset: HTMLFieldSetElement + let name: HTMLInputElement + const ref = e.target! as HTMLElement + const removeCreate = popover(ref, {}, + dom.form( + async function submit(e: SubmitEvent) { + e.preventDefault() + await withStatus('Creating mailbox', client.MailboxCreate(name.value), fieldset) + removeCreate() + }, + fieldset=dom.fieldset( + dom.label( + 'Name ', + name=dom.input(attr.required('yes'), focusPlaceholder('Lists/Go/Nuts')), + ), + ' ', + dom.submitbutton('Create'), + ), + ), + ) + remove() + }), ), - ' ', - dom.submitbutton('Create'), - ), - ), - ) - }), + dom.div( + dom.clickbutton('Export', function click(e: MouseEvent) { + const ref = e.target! as HTMLElement + popoverExport(ref, '') + remove() + }), + ), + ) + ) + }, + ), ), mailboxesElem, ), diff --git a/webmail/webmail_test.go b/webmail/webmail_test.go index 7f3757b..0349c61 100644 --- a/webmail/webmail_test.go +++ b/webmail/webmail_test.go @@ -1,8 +1,10 @@ package webmail import ( + "archive/tar" "archive/zip" "bytes" + "compress/gzip" "context" "encoding/json" "fmt" @@ -11,6 +13,7 @@ import ( "net/http" "net/http/httptest" "net/textproto" + "net/url" "os" "path/filepath" "reflect" @@ -462,6 +465,74 @@ func TestWebmail(t *testing.T) { // Unknown. testHTTP("GET", "/other", httpHeaders{}, http.StatusForbidden, nil, nil) + // Export. + testHTTP("GET", "/export", httpHeaders{}, http.StatusForbidden, nil, nil) + testHTTP("GET", "/export", httpHeaders{hdrSessionBad}, http.StatusForbidden, nil, nil) + testHTTP("GET", "/export", httpHeaders{hdrSessionOK}, http.StatusForbidden, nil, nil) + + testExport := func(format, archive, mailbox string, recursive bool, expectFiles int) { + t.Helper() + + fields := url.Values{ + "csrf": []string{string(csrfToken)}, + "format": []string{format}, + "archive": []string{archive}, + "mailbox": []string{mailbox}, + } + if recursive { + fields.Add("recursive", "on") + } + r := httptest.NewRequest("POST", "/export", strings.NewReader(fields.Encode())) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + r.Header.Add("Cookie", cookieOK.String()) + w := httptest.NewRecorder() + handle(apiHandler, false, "", w, r) + if w.Code != http.StatusOK { + t.Fatalf("export, got status code %d, expected 200: %s", w.Code, w.Body.Bytes()) + } + var count int + if archive == "zip" { + buf := w.Body.Bytes() + zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf))) + tcheck(t, err, "reading zip") + for _, f := range zr.File { + if !strings.HasSuffix(f.Name, "/") { + count++ + } + } + } else { + var src io.Reader = w.Body + if archive == "tgz" { + gzr, err := gzip.NewReader(src) + tcheck(t, err, "gzip reader") + src = gzr + } + tr := tar.NewReader(src) + for { + h, err := tr.Next() + if err == io.EOF { + break + } + tcheck(t, err, "next file in tar") + if !strings.HasSuffix(h.Name, "/") { + count++ + } + _, err = io.Copy(io.Discard, tr) + tcheck(t, err, "reading from tar") + } + } + if count != expectFiles { + t.Fatalf("export, has %d files, expected %d", count, expectFiles) + } + } + + testExport("maildir", "tgz", "", true, 8+1) // 8 messages, 1 flags file + testExport("maildir", "zip", "", true, 8+1) + testExport("mbox", "tar", "", true, 6+5) // 6 default mailboxes, 5 created + testExport("mbox", "zip", "", true, 6+5) + testExport("mbox", "zip", "Lists", true, 3) + testExport("mbox", "zip", "Lists", false, 1) + // HTTP message, generic testHTTP("GET", fmt.Sprintf("/msg/%v/attachments.zip", inboxMinimal.ID), nil, http.StatusForbidden, nil, nil) testHTTP("GET", fmt.Sprintf("/msg/%v/attachments.zip", inboxMinimal.ID), httpHeaders{hdrCSRFBad}, http.StatusForbidden, nil, nil) diff --git a/webops/export.go b/webops/export.go new file mode 100644 index 0000000..af82725 --- /dev/null +++ b/webops/export.go @@ -0,0 +1,96 @@ +package webops + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "fmt" + "mime" + "net/http" + "strings" + "time" + + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/store" +) + +// Export is used by webmail and webaccount to export messages of one or +// multiple mailboxes, in maildir or mbox format, in a tar/tgz/zip archive or +// direct mbox. +func Export(log mlog.Log, accName string, w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "405 - method not allowed - use post", http.StatusMethodNotAllowed) + return + } + + mailbox := r.FormValue("mailbox") // Empty means all. + format := r.FormValue("format") + archive := r.FormValue("archive") + recursive := r.FormValue("recursive") != "" + switch format { + case "maildir", "mbox": + default: + http.Error(w, "400 - bad request - unknown format", http.StatusBadRequest) + return + } + switch archive { + case "none", "tar", "tgz", "zip": + default: + http.Error(w, "400 - bad request - unknown archive", http.StatusBadRequest) + return + } + if archive == "none" && (format != "mbox" || recursive) { + http.Error(w, "400 - bad request - archive none can only be used with non-recursive mbox", http.StatusBadRequest) + return + } + + acc, err := store.OpenAccount(log, accName) + if err != nil { + log.Errorx("open account for export", err) + http.Error(w, "500 - internal server error", http.StatusInternalServerError) + return + } + defer func() { + err := acc.Close() + log.Check(err, "closing account") + }() + + name := strings.ReplaceAll(mailbox, "/", "-") + if name == "" { + name = "all" + } + filename := fmt.Sprintf("mailexport-%s-%s", name, time.Now().Format("20060102-150405")) + filename += "." + format + var archiver store.Archiver + if archive == "none" { + w.Header().Set("Content-Type", "application/mbox") + archiver = &store.MboxArchiver{Writer: w} + } else if archive == "tar" { + // Don't tempt browsers to "helpfully" decompress. + w.Header().Set("Content-Type", "application/x-tar") + archiver = store.TarArchiver{Writer: tar.NewWriter(w)} + filename += ".tar" + } else if archive == "tgz" { + // Don't tempt browsers to "helpfully" decompress. + w.Header().Set("Content-Type", "application/octet-stream") + + gzw := gzip.NewWriter(w) + defer func() { + _ = gzw.Close() + }() + archiver = store.TarArchiver{Writer: tar.NewWriter(gzw)} + filename += ".tgz" + } else { + w.Header().Set("Content-Type", "application/zip") + archiver = store.ZipArchiver{Writer: zip.NewWriter(w)} + filename += ".zip" + } + defer func() { + err := archiver.Close() + log.Check(err, "exporting mail close") + }() + w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": filename})) + if err := store.ExportMessages(r.Context(), log, acc.DB, acc.Dir, archiver, format == "maildir", mailbox, recursive); err != nil { + log.Errorx("exporting mail", err) + } +}