diff --git a/.gitignore b/.gitignore index 8abe8aa..60403d2 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,7 @@ /cover.html /.go/ /node_modules/ +/upgrade-verifydata.cpu.pprof +/upgrade-verifydata.mem.pprof +/upgrade-openaccounts.cpu.pprof +/upgrade-openaccounts.mem.pprof diff --git a/README.md b/README.md index aa1e6a5..9815bb9 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,16 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili ## Roadmap -- IMAP THREAD extension -- Prepare data storage for JMAP - DANE and DNSSEC - Sending DMARC and TLS reports (currently only receiving) +- Require TLS SMTP extension (RFC 8689) +- Prepare data storage for JMAP - Calendaring - Add special IMAP mailbox ("Queue?") that contains queued but not-yet-delivered messages - OAUTH2 support, for single sign on - Sieve for filtering (for now see Rulesets in the account config) +- Expose threading through IMAP extension - Privilege separation, isolating parts of the application to more restricted sandbox (e.g. new unauthenticated connections) - Using mox as backup MX diff --git a/ctl.go b/ctl.go index 3e90951..1283c0f 100644 --- a/ctl.go +++ b/ctl.go @@ -932,6 +932,62 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { } w.xclose() + case "reassignthreads": + /* protocol: + > "reassignthreads" + > account or empty + < "ok" or error + < stream + */ + + accountOpt := ctl.xread() + ctl.xwriteok() + w := ctl.writer() + + xreassignThreads := func(accName string) { + acc, err := store.OpenAccount(accName) + ctl.xcheck(err, "open account") + defer func() { + err := acc.Close() + log.Check(err, "closing account after reassigning threads") + }() + + // We don't want to step on an existing upgrade process. + err = acc.ThreadingWait(ctl.log) + ctl.xcheck(err, "waiting for threading upgrade to finish") + // todo: should we try to continue if the threading upgrade failed? only if there is a chance it will succeed this time... + + // todo: reassigning isn't atomic (in a single transaction), ideally it would be (bstore would need to be able to handle large updates). + const batchSize = 50000 + total, err := acc.ResetThreading(ctx, ctl.log, batchSize, true) + ctl.xcheck(err, "resetting threading fields") + _, err = fmt.Fprintf(w, "New thread base subject assigned to %d message(s), starting to reassign threads...\n", total) + ctl.xcheck(err, "write") + + // Assign threads again. Ideally we would do this in a single transaction, but + // bstore/boltdb cannot handle so many pending changes, so we set a high batchsize. + err = acc.AssignThreads(ctx, ctl.log, nil, 0, 50000, w) + ctl.xcheck(err, "reassign threads") + + _, err = fmt.Fprintf(w, "Threads reassigned. You should invalidate messages stored at imap clients with the \"mox bumpuidvalidity account [mailbox]\" command.\n") + ctl.xcheck(err, "write") + } + + if accountOpt != "" { + xreassignThreads(accountOpt) + } else { + for i, accName := range mox.Conf.Accounts() { + var line string + if i > 0 { + line = "\n" + } + _, err := fmt.Fprintf(w, "%sReassigning threads for account %s...\n", line, accName) + ctl.xcheck(err, "write") + xreassignThreads(accName) + } + } + w.xclose() + case "backup": backupctl(ctx, ctl) diff --git a/ctl_test.go b/ctl_test.go index 3f957e7..c56c330 100644 --- a/ctl_test.go +++ b/ctl_test.go @@ -217,6 +217,14 @@ func TestCtl(t *testing.T) { ctlcmdReparse(ctl, "") }) + // "reassignthreads" + testctl(func(ctl *ctl) { + ctlcmdReassignthreads(ctl, "mjl") + }) + testctl(func(ctl *ctl) { + ctlcmdReassignthreads(ctl, "") + }) + // "backup", backup account. err = dmarcdb.Init() tcheck(t, err, "dmarcdb init") diff --git a/doc.go b/doc.go index e325f91..4fb19c1 100644 --- a/doc.go +++ b/doc.go @@ -77,6 +77,7 @@ low-maintenance self-hosted email. mox ensureparsed account mox recalculatemailboxcounts account mox message parse message.eml + mox reassignthreads [account] Many commands talk to a running mox instance, through the ctl file in the data directory. Specify the configuration file (that holds the path to the data @@ -882,6 +883,40 @@ incorrect. This command will find, fix and print them. Parse message, print JSON representation. usage: mox message parse message.eml + +# mox reassignthreads + +Reassign message threads. + +For all accounts, or optionally only the specified account. + +Threading for all messages in an account is first reset, and new base subject +and normalized message-id saved with the message. Then all messages are +evaluated and matched against their parents/ancestors. + +Messages are matched based on the References header, with a fall-back to an +In-Reply-To header, and if neither is present/valid, based only on base +subject. + +A References header typically points to multiple previous messages in a +hierarchy. From oldest ancestor to most recent parent. An In-Reply-To header +would have only a message-id of the parent message. + +A message is only linked to a parent/ancestor if their base subject is the +same. This ensures unrelated replies, with a new subject, are placed in their +own thread. + +The base subject is lower cased, has whitespace collapsed to a single +space, and some components removed: leading "Re:", "Fwd:", "Fw:", or bracketed +tag (that mailing lists often add, e.g. "[listname]"), trailing "(fwd)", or +enclosing "[fwd: ...]". + +Messages are linked to all their ancestors. If an intermediate parent/ancestor +message is deleted in the future, the message can still be linked to the earlier +ancestors. If the direct parent already wasn't available while matching, this is +stored as the message having a "missing link" to its stored ancestors. + + usage: mox reassignthreads [account] */ package main diff --git a/gentestdata.go b/gentestdata.go index 4dbb1ba..d74ae5d 100644 --- a/gentestdata.go +++ b/gentestdata.go @@ -56,8 +56,6 @@ func cmdGentestdata(c *cmd) { log := mlog.New("gentestdata") ctxbg := context.Background() - mox.Shutdown = ctxbg - mox.Context = ctxbg mox.Conf.Log[""] = mlog.LevelInfo mlog.SetConfig(mox.Conf.Log) @@ -241,12 +239,16 @@ Accounts: // First account without messages. accTest0, err := store.OpenAccount("test0") xcheckf(err, "open account test0") + err = accTest0.ThreadingWait(log) + xcheckf(err, "wait for threading to finish") err = accTest0.Close() xcheckf(err, "close account") // Second account with one message. accTest1, err := store.OpenAccount("test1") xcheckf(err, "open account test1") + err = accTest1.ThreadingWait(log) + xcheckf(err, "wait for threading to finish") err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error { inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() xcheckf(err, "looking up inbox") @@ -281,7 +283,7 @@ Accounts: xcheckf(err, "creating temp file for delivery") _, err = fmt.Fprint(mf, msg) xcheckf(err, "writing deliver message to file") - err = accTest1.DeliverMessage(log, tx, &m, mf, true, false, true) + err = accTest1.DeliverMessage(log, tx, &m, mf, true, false, true, false) xcheckf(err, "add message to account test1") err = mf.Close() xcheckf(err, "closing file") @@ -301,6 +303,8 @@ Accounts: // Third account with two messages and junkfilter. accTest2, err := store.OpenAccount("test2") xcheckf(err, "open account test2") + err = accTest2.ThreadingWait(log) + xcheckf(err, "wait for threading to finish") err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error { inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get() xcheckf(err, "looking up inbox") @@ -335,7 +339,7 @@ Accounts: xcheckf(err, "creating temp file for delivery") _, err = fmt.Fprint(mf0, msg0) xcheckf(err, "writing deliver message to file") - err = accTest2.DeliverMessage(log, tx, &m0, mf0, true, false, false) + err = accTest2.DeliverMessage(log, tx, &m0, mf0, true, false, false, false) xcheckf(err, "add message to account test2") err = mf0.Close() xcheckf(err, "closing file") @@ -362,7 +366,7 @@ Accounts: xcheckf(err, "creating temp file for delivery") _, err = fmt.Fprint(mf1, msg1) xcheckf(err, "writing deliver message to file") - err = accTest2.DeliverMessage(log, tx, &m1, mf1, true, false, false) + err = accTest2.DeliverMessage(log, tx, &m1, mf1, true, false, false, false) xcheckf(err, "add message to account test2") err = mf1.Close() xcheckf(err, "closing file") diff --git a/imapserver/server.go b/imapserver/server.go index f6a34c8..e675033 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -31,7 +31,7 @@ not in IMAP4rev2). ../rfc/3501:964 - todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64? - todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes? - todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command. -- future: more extensions: STATUS=SIZE, OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD, CREATE-SPECIAL-USE. +- todo future: more extensions: STATUS=SIZE, OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD, CREATE-SPECIAL-USE. */ import ( @@ -1199,7 +1199,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription: n = append(n, change) continue - case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords: + case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread: default: panic(fmt.Errorf("missing case for %#v", change)) } @@ -2740,7 +2740,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { err = tx.Update(&mb) xcheckf(err, "updating mailbox counts") - err := c.account.DeliverMessage(c.log, tx, &m, msgFile, true, true, false) + err := c.account.DeliverMessage(c.log, tx, &m, msgFile, true, true, false, false) xcheckf(err, "delivering message") }) diff --git a/import.go b/import.go index b3d42fa..8383349 100644 --- a/import.go +++ b/import.go @@ -185,6 +185,20 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { var mdnewf, mdcurf *os.File var msgreader store.MsgSource + // Open account, creating a database file if it doesn't exist yet. It must be known + // in the configuration file. + a, err := store.OpenAccount(account) + ctl.xcheck(err, "opening account") + defer func() { + if a != nil { + err := a.Close() + ctl.log.Check(err, "closing account after import") + } + }() + + err = a.ThreadingWait(ctl.log) + ctl.xcheck(err, "waiting for account thread upgrade") + defer func() { if mboxf != nil { err := mboxf.Close() @@ -200,17 +214,6 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { } }() - // Open account, creating a database file if it doesn't exist yet. It must be known - // in the configuration file. - a, err := store.OpenAccount(account) - ctl.xcheck(err, "opening account") - defer func() { - if a != nil { - err := a.Close() - ctl.log.Check(err, "closing account after import") - } - }() - // Messages don't always have a junk flag set. We'll assume anything in a mailbox // starting with junk or spam is junk mail. @@ -277,7 +280,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { const consumeFile = true const sync = false const notrain = true - err := a.DeliverMessage(ctl.log, tx, m, mf, consumeFile, sync, notrain) + const nothreads = true + err := a.DeliverMessage(ctl.log, tx, m, mf, consumeFile, sync, notrain, nothreads) ctl.xcheck(err, "delivering message") deliveredIDs = append(deliveredIDs, m.ID) ctl.log.Debug("delivered message", mlog.Field("id", m.ID)) @@ -332,6 +336,11 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { m.ParsedBuf, err = json.Marshal(p) ctl.xcheck(err, "marshal parsed message structure") + // Set fields needed for future threading. By doing it now, DeliverMessage won't + // have to parse the Part again. + p.SetReaderAt(store.FileMsgReader(m.MsgPrefix, msgf)) + m.PrepareThreading(ctl.log, &p) + if m.Received.IsZero() { if p.Envelope != nil && !p.Envelope.Date.IsZero() { m.Received = p.Envelope.Date @@ -385,6 +394,12 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { process(m, msgf, origPath) } + // Match threads. + if len(deliveredIDs) > 0 { + err = a.AssignThreads(ctx, ctl.log, tx, deliveredIDs[0], 0, io.Discard) + ctl.xcheck(err, "assigning messages to threads") + } + // Get mailbox again, uidnext is likely updated. mc := mb.MailboxCounts err = tx.Get(&mb) diff --git a/main.go b/main.go index 3eea02f..6f1b409 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "net/url" "os" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -37,6 +38,7 @@ import ( "github.com/mjl-/mox/message" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/mtasts" "github.com/mjl-/mox/publicsuffix" @@ -143,6 +145,7 @@ var commands = []struct { {"ensureparsed", cmdEnsureParsed}, {"recalculatemailboxcounts", cmdRecalculateMailboxCounts}, {"message parse", cmdMessageParse}, + {"reassignthreads", cmdReassignthreads}, // Not listed. {"helpall", cmdHelpall}, @@ -162,6 +165,7 @@ var commands = []struct { {"ximport maildir", cmdXImportMaildir}, {"ximport mbox", cmdXImportMbox}, {"openaccounts", cmdOpenaccounts}, + {"readmessages", cmdReadmessages}, } var cmds []cmd @@ -389,6 +393,10 @@ func main() { // flag. store.CheckConsistencyOnClose = false + ctxbg := context.Background() + mox.Shutdown = ctxbg + mox.Context = ctxbg + log.SetFlags(0) // If invoked as sendmail, e.g. /usr/sbin/sendmail, we do enough so cron can get a @@ -2086,6 +2094,7 @@ func cmdVersion(c *cmd) { fmt.Println(moxvar.Version) } +// todo: should make it possible to run this command against a running mox. it should disconnect existing clients for accounts with a bumped uidvalidity, so they will reconnect and refetch the data. func cmdBumpUIDValidity(c *cmd) { c.params = "account [mailbox]" c.help = `Change the IMAP UID validity of the mailbox, causing IMAP clients to refetch messages. @@ -2476,13 +2485,193 @@ Opens database files directly, not going through a running mox instance. c.Usage() } + clog := mlog.New("openaccounts") + dataDir := filepath.Clean(args[0]) for _, accName := range args[1:] { accDir := filepath.Join(dataDir, "accounts", accName) - log.Printf("opening account %s...", filepath.Join(accDir, accName)) + log.Printf("opening account %s...", accDir) a, err := store.OpenAccountDB(accDir, accName) xcheckf(err, "open account %s", accName) + err = a.ThreadingWait(clog) + xcheckf(err, "wait for threading upgrade to complete for %s", accName) err = a.Close() xcheckf(err, "close account %s", accName) } } + +func cmdReassignthreads(c *cmd) { + c.params = "[account]" + c.help = `Reassign message threads. + +For all accounts, or optionally only the specified account. + +Threading for all messages in an account is first reset, and new base subject +and normalized message-id saved with the message. Then all messages are +evaluated and matched against their parents/ancestors. + +Messages are matched based on the References header, with a fall-back to an +In-Reply-To header, and if neither is present/valid, based only on base +subject. + +A References header typically points to multiple previous messages in a +hierarchy. From oldest ancestor to most recent parent. An In-Reply-To header +would have only a message-id of the parent message. + +A message is only linked to a parent/ancestor if their base subject is the +same. This ensures unrelated replies, with a new subject, are placed in their +own thread. + +The base subject is lower cased, has whitespace collapsed to a single +space, and some components removed: leading "Re:", "Fwd:", "Fw:", or bracketed +tag (that mailing lists often add, e.g. "[listname]"), trailing "(fwd)", or +enclosing "[fwd: ...]". + +Messages are linked to all their ancestors. If an intermediate parent/ancestor +message is deleted in the future, the message can still be linked to the earlier +ancestors. If the direct parent already wasn't available while matching, this is +stored as the message having a "missing link" to its stored ancestors. +` + args := c.Parse() + if len(args) > 1 { + c.Usage() + } + + mustLoadConfig() + var account string + if len(args) == 1 { + account = args[0] + } + ctlcmdReassignthreads(xctl(), account) +} + +func ctlcmdReassignthreads(ctl *ctl, account string) { + ctl.xwrite("reassignthreads") + ctl.xwrite(account) + ctl.xreadok() + ctl.xstreamto(os.Stdout) +} + +func cmdReadmessages(c *cmd) { + c.unlisted = true + c.params = "datadir account ..." + c.help = `Open account, parse several headers for all messages. + +For performance testing. + +Opens database files directly, not going through a running mox instance. +` + + gomaxprocs := runtime.GOMAXPROCS(0) + var procs, workqueuesize, limit int + c.flag.IntVar(&procs, "procs", gomaxprocs, "number of goroutines for reading messages") + c.flag.IntVar(&workqueuesize, "workqueuesize", 2*gomaxprocs, "number of messages to keep in work queue") + c.flag.IntVar(&limit, "limit", 0, "number of messages to process if greater than zero") + args := c.Parse() + if len(args) <= 1 { + c.Usage() + } + + type threadPrep struct { + references []string + inReplyTo []string + } + + threadingFields := [][]byte{ + []byte("references"), + []byte("in-reply-to"), + } + + dataDir := filepath.Clean(args[0]) + for _, accName := range args[1:] { + accDir := filepath.Join(dataDir, "accounts", accName) + log.Printf("opening account %s...", accDir) + a, err := store.OpenAccountDB(accDir, accName) + xcheckf(err, "open account %s", accName) + + prepareMessages := func(in, out chan moxio.Work[store.Message, threadPrep]) { + headerbuf := make([]byte, 8*1024) + scratch := make([]byte, 4*1024) + for { + w, ok := <-in + if !ok { + return + } + + m := w.In + var partialPart struct { + HeaderOffset int64 + BodyOffset int64 + } + if err := json.Unmarshal(m.ParsedBuf, &partialPart); err != nil { + w.Err = fmt.Errorf("unmarshal part: %v", err) + } else { + size := partialPart.BodyOffset - partialPart.HeaderOffset + if int(size) > len(headerbuf) { + headerbuf = make([]byte, size) + } + if size > 0 { + buf := headerbuf[:int(size)] + err := func() error { + mr := a.MessageReader(m) + defer mr.Close() + + // ReadAt returns whole buffer or error. Single read should be fast. + n, err := mr.ReadAt(buf, partialPart.HeaderOffset) + if err != nil || n != len(buf) { + return fmt.Errorf("read header: %v", err) + } + return nil + }() + if err != nil { + w.Err = err + } else if h, err := message.ParseHeaderFields(buf, scratch, threadingFields); err != nil { + w.Err = err + } else { + w.Out.references = h["References"] + w.Out.inReplyTo = h["In-Reply-To"] + } + } + } + + out <- w + } + } + + n := 0 + t := time.Now() + t0 := t + + processMessage := func(m store.Message, prep threadPrep) error { + if n%100000 == 0 { + log.Printf("%d messages (delta %s)", n, time.Since(t)) + t = time.Now() + } + n++ + return nil + } + + wq := moxio.NewWorkQueue[store.Message, threadPrep](procs, workqueuesize, prepareMessages, processMessage) + + err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error { + q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("Expunged", false) + q.SortAsc("ID") + if limit > 0 { + q.Limit(limit) + } + err = q.ForEach(wq.Add) + if err == nil { + err = wq.Finish() + } + wq.Stop() + + return err + }) + xcheckf(err, "processing message") + + err = a.Close() + xcheckf(err, "close account %s", accName) + log.Printf("account %s, total time %s", accName, time.Since(t0)) + } +} diff --git a/message/messageid.go b/message/messageid.go new file mode 100644 index 0000000..aba06d0 --- /dev/null +++ b/message/messageid.go @@ -0,0 +1,53 @@ +package message + +import ( + "errors" + "fmt" + "strings" + + "github.com/mjl-/mox/moxvar" + "github.com/mjl-/mox/smtp" +) + +var errBadMessageID = errors.New("not a message-id") + +// MessageIDCanonical parses the Message-ID, returning a canonical value that is +// lower-cased, without <>, and no unneeded quoting. For matching in threading, +// with References/In-Reply-To. If the message-id is invalid (e.g. no <>), an error +// is returned. If the message-id could not be parsed as address (localpart "@" +// domain), the raw value and the bool return parameter true is returned. It is +// quite common that message-id's don't adhere to the localpart @ domain +// syntax. +func MessageIDCanonical(s string) (string, bool, error) { + // ../rfc/5322:1383 + + s = strings.TrimSpace(s) + if !strings.HasPrefix(s, "<") { + return "", false, fmt.Errorf("%w: missing <", errBadMessageID) + } + s = s[1:] + // Seen in practice: Message-ID: (added by postmaster@some.example) + // Doesn't seem valid, but we allow it. + s, rem, have := strings.Cut(s, ">") + if !have || (rem != "" && (moxvar.Pedantic || !strings.HasPrefix(rem, " "))) { + return "", false, fmt.Errorf("%w: missing >", errBadMessageID) + } + // We canonicalize the Message-ID: lower-case, no unneeded quoting. + s = strings.ToLower(s) + if s == "" { + return "", false, fmt.Errorf("%w: empty message-id", errBadMessageID) + } + addr, err := smtp.ParseAddress(s) + if err != nil { + // Common reasons for not being an address: + // 1. underscore in hostname. + // 2. ip literal instead of domain. + // 3. two @'s, perhaps intended as time-separator + // 4. no @'s, so no domain/host + return s, true, nil + } + // We preserve the unicode-ness of domain. + t := strings.Split(s, "@") + s = addr.Localpart.String() + "@" + t[len(t)-1] + return s, false, nil +} diff --git a/message/messageid_test.go b/message/messageid_test.go new file mode 100644 index 0000000..0ea1875 --- /dev/null +++ b/message/messageid_test.go @@ -0,0 +1,29 @@ +package message + +import ( + "errors" + "testing" +) + +func TestMessageIDCanonical(t *testing.T) { + check := func(s string, expID string, expRaw bool, expErr error) { + t.Helper() + + id, raw, err := MessageIDCanonical(s) + if id != expID || raw != expRaw || (expErr == nil) != (err == nil) || err != nil && !errors.Is(err, expErr) { + t.Fatalf("got message-id %q, raw %v, err %v, expected %q %v %v, for message-id %q", id, raw, err, expID, expRaw, expErr, s) + } + } + + check("bogus", "", false, errBadMessageID) + check("", "", false, errBadMessageID) + check("<>", "", false, errBadMessageID) + check("", "user@domain", false, nil) + check("", "user@domain", false, nil) + check("", "user@[10.0.0.1]", true, nil) + check(" (added by postmaster@isp.example)", "user@domain", false, nil) + check(" other", "user@domain", false, nil) + check("", "user@domain@time", true, nil) + check("", "user", true, nil) +} diff --git a/message/parseheaderfields.go b/message/parseheaderfields.go new file mode 100644 index 0000000..4ddff66 --- /dev/null +++ b/message/parseheaderfields.go @@ -0,0 +1,77 @@ +package message + +import ( + "bytes" + "fmt" + "net/mail" + "net/textproto" +) + +// ParseHeaderFields parses only the header fields in "fields" from the complete +// header buffer "header", while using "scratch" as temporary space, prevent lots +// of unneeded allocations when only a few headers are needed. +func ParseHeaderFields(header []byte, scratch []byte, fields [][]byte) (textproto.MIMEHeader, error) { + // todo: should not use mail.ReadMessage, it allocates a bufio.Reader. should implement header parsing ourselves. + + // Gather the raw lines for the fields, with continuations, without the other + // headers. Put them in a byte slice and only parse those headers. For now, use + // mail.ReadMessage without letting it do allocations for all headers. + scratch = scratch[:0] + var keepcontinuation bool + for len(header) > 0 { + if header[0] == ' ' || header[0] == '\t' { + // Continuation. + i := bytes.IndexByte(header, '\n') + if i < 0 { + i = len(header) + } else { + i++ + } + if keepcontinuation { + scratch = append(scratch, header[:i]...) + } + header = header[i:] + continue + } + i := bytes.IndexByte(header, ':') + if i < 0 || i > 0 && (header[i-1] == ' ' || header[i-1] == '\t') { + i = bytes.IndexByte(header, '\n') + if i < 0 { + break + } + header = header[i+1:] + keepcontinuation = false + continue + } + k := header[:i] + keepcontinuation = false + for _, f := range fields { + if bytes.EqualFold(k, f) { + keepcontinuation = true + break + } + } + i = bytes.IndexByte(header, '\n') + if i < 0 { + i = len(header) + } else { + i++ + } + if keepcontinuation { + scratch = append(scratch, header[:i]...) + } + header = header[i:] + } + + if len(scratch) == 0 { + return nil, nil + } + + scratch = append(scratch, "\r\n"...) + + msg, err := mail.ReadMessage(bytes.NewReader(scratch)) + if err != nil { + return nil, fmt.Errorf("reading message header") + } + return textproto.MIMEHeader(msg.Header), nil +} diff --git a/message/parseheaderfields_test.go b/message/parseheaderfields_test.go new file mode 100644 index 0000000..fbba9a4 --- /dev/null +++ b/message/parseheaderfields_test.go @@ -0,0 +1,40 @@ +package message + +import ( + "net/textproto" + "reflect" + "strings" + "testing" +) + +func TestParseHeaderFields(t *testing.T) { + check := func(headers string, fields []string, expHdrs textproto.MIMEHeader, expErr error) { + t.Helper() + + buffields := [][]byte{} + for _, f := range fields { + buffields = append(buffields, []byte(f)) + } + + scratches := [][]byte{ + make([]byte, 0), + make([]byte, 4*1024), + } + for _, scratch := range scratches { + hdrs, err := ParseHeaderFields([]byte(strings.ReplaceAll(headers, "\n", "\r\n")), scratch, buffields) + if !reflect.DeepEqual(hdrs, expHdrs) || !reflect.DeepEqual(err, expErr) { + t.Fatalf("got %v %v, expected %v %v", hdrs, err, expHdrs, expErr) + } + } + } + + check("", []string{"subject"}, textproto.MIMEHeader(nil), nil) + check("Subject: test\n", []string{"subject"}, textproto.MIMEHeader{"Subject": []string{"test"}}, nil) + check("References: \nOther: ignored\nSubject: first\nSubject: test\n\tcontinuation\n", []string{"subject", "REFERENCES"}, textproto.MIMEHeader{"References": []string{""}, "Subject": []string{"first", "test continuation"}}, nil) + check(":\n", []string{"subject"}, textproto.MIMEHeader(nil), nil) + check("bad\n", []string{"subject"}, textproto.MIMEHeader(nil), nil) + check("subject: test\n continuation without end\n", []string{"subject"}, textproto.MIMEHeader{"Subject": []string{"test continuation without end"}}, nil) + check("subject: test\n", []string{"subject"}, textproto.MIMEHeader{"Subject": []string{"test"}}, nil) + check("subject \t: test\n", []string{"subject"}, textproto.MIMEHeader(nil), nil) // Note: In go1.20, this would be interpreted as valid "Subject" header. Not in go1.21. + // note: in go1.20, missing end of line would cause it to be ignored, in go1.21 it is used. +} diff --git a/message/part.go b/message/part.go index e965997..7c6c949 100644 --- a/message/part.go +++ b/message/part.go @@ -88,7 +88,7 @@ type Part struct { // Envelope holds the basic/common message headers as used in IMAP4. type Envelope struct { Date time.Time - Subject string + Subject string // Q/B-word-decoded. From []Address Sender []Address ReplyTo []Address diff --git a/message/referencedids.go b/message/referencedids.go new file mode 100644 index 0000000..2847d64 --- /dev/null +++ b/message/referencedids.go @@ -0,0 +1,74 @@ +package message + +import ( + "strings" + + "github.com/mjl-/mox/smtp" +) + +// ReferencedIDs returns the Message-IDs referenced from the References header(s), +// with a fallback to the In-Reply-To header(s). The ids are canonicalized for +// thread-matching, like with MessageIDCanonical. Empty message-id's are skipped. +func ReferencedIDs(references []string, inReplyTo []string) ([]string, error) { + var refids []string // In thread-canonical form. + + // parse and add 0 or 1 reference, returning the remaining refs string for a next attempt. + parse1 := func(refs string, one bool) string { + refs = strings.TrimLeft(refs, " \t\r\n") + if !strings.HasPrefix(refs, "<") { + // To make progress, we skip to next space or >. + i := strings.IndexAny(refs, " >") + if i < 0 { + return "" + } + return refs[i+1:] + } + refs = refs[1:] + // Look for the ending > or next <. If < is before >, this entry is truncated. + i := strings.IndexAny(refs, "<>") + if i < 0 { + return "" + } + if refs[i] == '<' { + // Truncated entry, we ignore it. + return refs[i:] + } + ref := strings.ToLower(refs[:i]) + // Some MUAs wrap References line in the middle of message-id's, and others + // recombine them. Take out bare WSP in message-id's. + ref = strings.ReplaceAll(ref, " ", "") + ref = strings.ReplaceAll(ref, "\t", "") + refs = refs[i+1:] + // Canonicalize the quotedness of the message-id. + addr, err := smtp.ParseAddress(ref) + if err == nil { + // Leave the hostname form intact. + t := strings.Split(ref, "@") + ref = addr.Localpart.String() + "@" + t[len(t)-1] + } + // log.Errorx("assigning threads: bad reference in references header, using raw value", err, mlog.Field("msgid", mid), mlog.Field("reference", ref)) + if ref != "" { + refids = append(refids, ref) + } + return refs + } + + // References is the modern way (for a long time already) to reference ancestors. + // The direct parent is typically at the end of the list. + for _, refs := range references { + for refs != "" { + refs = parse1(refs, false) + } + } + // We only look at the In-Reply-To header if we didn't find any References. + if len(refids) == 0 { + for _, s := range inReplyTo { + parse1(s, true) + if len(refids) > 0 { + break + } + } + } + + return refids, nil +} diff --git a/message/referencedids_test.go b/message/referencedids_test.go new file mode 100644 index 0000000..cbeab0b --- /dev/null +++ b/message/referencedids_test.go @@ -0,0 +1,35 @@ +package message + +import ( + "strings" + "testing" +) + +func TestReferencedIDs(t *testing.T) { + check := func(msg string, expRefs []string) { + t.Helper() + + p, err := Parse(xlog, true, strings.NewReader(msg)) + tcheck(t, err, "parsing message") + + h, err := p.Header() + tcheck(t, err, "parsing header") + + refs, err := ReferencedIDs(h["References"], h["In-Reply-To"]) + tcheck(t, err, "parsing references/in-reply-to") + tcompare(t, refs, expRefs) + } + + check("References: bogus\r\n", nil) + check("References: \r\n", []string{"user@host"}) + check("References: \r\n", []string{"user@tést.example"}) + check("References: \r\n", []string{"user@xn--tst-bma.example"}) + check("References: \r\n", []string{"user@bad_label.domain"}) + check("References: \r\n", []string{"user@host"}) + check("References: \r\n", []string{"previouslywrapped@host"}) + check("References: \r\n", []string{"user1@host", "user2@other.example"}) + check("References: \r\n", []string{"missinghost"}) + check("References: \r\n", []string{"user@host@time"}) + check("References: bogus bad \r\n", []string{"user@host"}) + check("In-Reply-To: more stuff\r\nReferences: bogus bad\r\n", []string{"user@host"}) +} diff --git a/message/threadsubject.go b/message/threadsubject.go new file mode 100644 index 0000000..3a67e84 --- /dev/null +++ b/message/threadsubject.go @@ -0,0 +1,124 @@ +package message + +import ( + "strings" +) + +// ThreadSubject returns the base subject to use for matching against other +// messages, to see if they belong to the same thread. A matching subject is +// always required to match to an existing thread, both if +// References/In-Reply-To header(s) are present, and if not. +// +// Subject should already be q/b-word-decoded. +// +// If allowNull is true, base subjects with a \0 can be returned. If not set, +// an empty string is returned if a base subject would have a \0. +func ThreadSubject(subject string, allowNull bool) (threadSubject string, isResponse bool) { + subject = strings.ToLower(subject) + + // ../rfc/5256:101, Step 1. + var s string + for _, c := range subject { + if c == '\r' { + continue + } else if c == ' ' || c == '\n' || c == '\t' { + if !strings.HasSuffix(s, " ") { + s += " " + } + } else { + s += string(c) + } + } + + // ../rfc/5256:107 ../rfc/5256:811, removing mailing list tag "[...]" and reply/forward "re"/"fwd" prefix. + removeBlob := func(s string) string { + for i, c := range s { + if i == 0 { + if c != '[' { + return s + } + } else if c == '[' { + return s + } else if c == ']' { + s = s[i+1:] // Past [...]. + s = strings.TrimRight(s, " \t") // *WSP + return s + } + } + return s + } + // ../rfc/5256:107 ../rfc/5256:811 + removeLeader := func(s string) string { + if strings.HasPrefix(s, " ") || strings.HasPrefix(s, "\t") { + s = s[1:] // WSP + } + + orig := s + + // Remove zero or more subj-blob + for { + prevs := s + s = removeBlob(s) + if prevs == s { + break + } + } + + if strings.HasPrefix(s, "re") { + s = s[2:] + } else if strings.HasPrefix(s, "fwd") { + s = s[3:] + } else if strings.HasPrefix(s, "fw") { + s = s[2:] + } else { + return orig + } + s = strings.TrimLeft(s, " \t") // *WSP + s = removeBlob(s) + if !strings.HasPrefix(s, ":") { + return orig + } + s = s[1:] + isResponse = true + return s + } + + for { + // ../rfc/5256:104 ../rfc/5256:817, remove trailing "(fwd)" or WSP, Step 2. + for { + prevs := s + s = strings.TrimRight(s, " \t") + if strings.HasSuffix(s, "(fwd)") { + s = strings.TrimSuffix(s, "(fwd)") + isResponse = true + } + if s == prevs { + break + } + } + + for { + prevs := s + s = removeLeader(s) // Step 3. + if ns := removeBlob(s); ns != "" { + s = ns // Step 4. + } + // Step 5, ../rfc/5256:123 + if s == prevs { + break + } + } + + // Step 6. ../rfc/5256:128 ../rfc/5256:805 + if strings.HasPrefix(s, "[fwd:") && strings.HasSuffix(s, "]") { + s = s[len("[fwd:") : len(s)-1] + isResponse = true + continue // From step 2 again. + } + break + } + if !allowNull && strings.ContainsRune(s, 0) { + s = "" + } + return s, isResponse +} diff --git a/message/threadsubject_test.go b/message/threadsubject_test.go new file mode 100644 index 0000000..92351b3 --- /dev/null +++ b/message/threadsubject_test.go @@ -0,0 +1,35 @@ +package message + +import ( + "testing" +) + +func TestThreadSubject(t *testing.T) { + check := func(s, expBase string, expResp bool) { + t.Helper() + + base, isResp := ThreadSubject(s, false) + if base != expBase || isResp != expResp { + t.Fatalf("got base %q, resp %v, expected %q %v for subject %q", base, isResp, expBase, expResp, s) + } + } + + check("test", "test", false) + check(" a b\tc\r\n d\t", "a b c d", false) + check("test (fwd) (fwd) ", "test", true) + check("re: test", "test", true) + check("fw: test", "test", true) + check("fwd: test", "test", true) + check("fwd [tag] Test", "fwd [tag] test", false) + check("[list] re: a b c\t", "a b c", true) + check("[list] fw: a b c", "a b c", true) + check("[tag1][tag2] [tag3]\t re: a b c", "a b c", true) + check("[tag1][tag2] [tag3]\t re: a \u0000b c", "", true) + check("[list] fw:[tag] a b c", "a b c", true) + check("[list] re: [list] fwd: a b c\t", "a b c", true) + check("[fwd: a b c]", "a b c", true) + check("[fwd: [fwd: a b c]]", "a b c", true) + check("[fwd: [list] re: a b c]", "a b c", true) + check("[nonlist]", "[nonlist]", false) + check("fwd [list]:", "", true) +} diff --git a/moxio/workq.go b/moxio/workq.go new file mode 100644 index 0000000..3a0268c --- /dev/null +++ b/moxio/workq.go @@ -0,0 +1,131 @@ +package moxio + +import ( + "sync" +) + +// Work is a slot for work that needs to be done. +type Work[T, R any] struct { + In T + Err error + Out R + + i int + done bool +} + +// WorkQueue can be used to execute a work load where many items are processed +// with a slow step and where a pool of workers goroutines to execute the slow +// step helps. Reading messages from the database file is fast and cannot be +// easily done concurrently, but reading the message file from disk and parsing +// the headers is the bottleneck. The workqueue can manage the goroutines that +// read the message file from disk and parse. +type WorkQueue[T, R any] struct { + max int + ring []Work[T, R] + start int + n int + + wg sync.WaitGroup // For waiting for workers to stop. + work chan Work[T, R] + done chan Work[T, R] + + process func(T, R) error +} + +// NewWorkQueue creates a new work queue with "procs" goroutines, and a total work +// queue size of "size" (e.g. 2*procs). The worker goroutines run "preparer", which +// should be a loop receiving work from "in" and sending the work result (with Err +// or Out set) on "out". The preparer function should return when the "in" channel +// is closed, the signal to stop. WorkQueue processes the results in the order they +// went in, so prepared work that was scheduled after earlier work that is not yet +// prepared will wait and be queued. +func NewWorkQueue[T, R any](procs, size int, preparer func(in, out chan Work[T, R]), process func(T, R) error) *WorkQueue[T, R] { + wq := &WorkQueue[T, R]{ + max: size, + ring: make([]Work[T, R], size), + work: make(chan Work[T, R], size), // Ensure scheduling never blocks for main goroutine. + done: make(chan Work[T, R], size), // Ensure sending result never blocks for worker goroutine. + process: process, + } + + wq.wg.Add(procs) + for i := 0; i < procs; i++ { + go func() { + defer wq.wg.Done() + preparer(wq.work, wq.done) + }() + } + + return wq +} + +// Add adds new work to be prepared to the queue. If the queue is full, it +// waits until space becomes available, i.e. when the head of the queue has +// work that becomes prepared. Add processes the prepared items to make space +// available. +func (wq *WorkQueue[T, R]) Add(in T) error { + // Schedule the new work if we can. + if wq.n < wq.max { + wq.work <- Work[T, R]{i: (wq.start + wq.n) % wq.max, done: true, In: in} + wq.n++ + return nil + } + + // We cannot schedule new work. Wait for finished work until start is done. + for { + w := <-wq.done + wq.ring[w.i] = w + if w.i == wq.start { + break + } + } + + // Process as much finished work as possible. Will be at least 1. + if err := wq.processHead(); err != nil { + return err + } + + // Schedule this message as new work. + wq.work <- Work[T, R]{i: (wq.start + wq.n) % wq.max, done: true, In: in} + wq.n++ + return nil +} + +// processHead processes the work at the head of the queue by calling process +// on the work. +func (wq *WorkQueue[T, R]) processHead() error { + for wq.n > 0 && wq.ring[wq.start].done { + wq.ring[wq.start].done = false + w := wq.ring[wq.start] + wq.start = (wq.start + 1) % len(wq.ring) + wq.n -= 1 + + if w.Err != nil { + return w.Err + } + if err := wq.process(w.In, w.Out); err != nil { + return err + } + } + return nil +} + +// Finish waits for the remaining work to be prepared and processes the work. +func (wq *WorkQueue[T, R]) Finish() error { + var err error + for wq.n > 0 && err == nil { + w := <-wq.done + wq.ring[w.i] = w + + err = wq.processHead() + } + return err +} + +// Stop shuts down the worker goroutines and waits until they have returned. +// Stop must always be called on a WorkQueue, otherwise the goroutines never stop. +func (wq *WorkQueue[T, R]) Stop() { + close(wq.work) + wq.wg.Wait() +} diff --git a/smtpserver/rejects.go b/smtpserver/rejects.go index 0ff9273..fd53c17 100644 --- a/smtpserver/rejects.go +++ b/smtpserver/rejects.go @@ -23,7 +23,10 @@ func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m * } else if header, err := p.Header(); err != nil { log.Infox("parsing reject message header for message-id", err) } else { - msgID = header.Get("Message-Id") + msgID, _, err = message.MessageIDCanonical(header.Get("Message-Id")) + if err != nil { + log.Debugx("parsing message-id for reject", err, mlog.Field("messageid", header.Get("Message-Id"))) + } } // We must not read MsgPrefix, it will likely change for subsequent deliveries. diff --git a/smtpserver/server.go b/smtpserver/server.go index e458275..500337f 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -2327,7 +2327,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW if !a.accept { conf, _ := acc.Conf() if conf.RejectsMailbox != "" { - present, messageid, messagehash, err := rejectPresent(log, acc, conf.RejectsMailbox, m, dataFile) + present, _, messagehash, err := rejectPresent(log, acc, conf.RejectsMailbox, m, dataFile) if err != nil { log.Errorx("checking whether reject is already present", err) } else if !present { @@ -2336,7 +2336,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW // Regular automatic junk flags configuration applies to these messages. The // default is to treat these as neutral, so they won't cause outright rejections // due to reputation for later delivery attempts. - m.MessageID = messageid m.MessageHash = messagehash acc.WithWLock(func() { hasSpace := true @@ -2390,7 +2389,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW } } - // If a forwarded message and this is a first-time sender, wait before actually + // If this is a first-time sender and not a forwarded message, wait before actually // delivering. If this turns out to be a spammer, we've kept one of their // connections busy. if delayFirstTime && !m.IsForward && a.reason == reasonNoBadSignals && c.firstTimeSenderDelay > 0 { @@ -2432,8 +2431,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW log.Info("incoming message delivered", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom)) conf, _ := acc.Conf() - if conf.RejectsMailbox != "" && messageID != "" { - if err := acc.RejectsRemove(log, conf.RejectsMailbox, messageID); err != nil { + if conf.RejectsMailbox != "" && m.MessageID != "" { + if err := acc.RejectsRemove(log, conf.RejectsMailbox, m.MessageID); err != nil { log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID)) } } diff --git a/store/account.go b/store/account.go index 08cbe55..960e479 100644 --- a/store/account.go +++ b/store/account.go @@ -34,7 +34,9 @@ import ( "io" "os" "path/filepath" + "runtime/debug" "sort" + "strconv" "strings" "sync" "time" @@ -48,6 +50,7 @@ import ( "github.com/mjl-/mox/config" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/message" + "github.com/mjl-/mox/metrics" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/moxio" @@ -444,14 +447,41 @@ type Message struct { OrigEHLODomain string OrigDKIMDomains []string - // Value of Message-Id header. Only set for messages that were - // delivered to the rejects mailbox. For ensuring such messages are - // delivered only once. Value includes <>. + // Canonicalized Message-Id, always lower-case and normalized quoting, without + // <>'s. Empty if missing. Used for matching message threads, and to prevent + // duplicate reject delivery. MessageID string `bstore:"index"` + // lower-case: ../rfc/5256:495 - // Hash of message. For rejects delivery, so optional like MessageID. + // For matching threads in case there is no References/In-Reply-To header. It is + // lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed. + SubjectBase string `bstore:"index"` + // ../rfc/5256:90 + + // Hash of message. For rejects delivery in case there is no Message-ID, only set + // when delivered as reject. MessageHash []byte + // ID of message starting this thread. + ThreadID int64 `bstore:"index"` + // IDs of parent messages, from closest parent to the root message. Parent messages + // may be in a different mailbox, or may no longer exist. ThreadParentIDs must + // never contain the message id itself (a cycle), and parent messages must + // reference the same ancestors. + ThreadParentIDs []int64 + // ThreadMissingLink is true if there is no match with a direct parent. E.g. first + // ID in ThreadParentIDs is not the direct ancestor (an intermediate message may + // have been deleted), or subject-based matching was done. + ThreadMissingLink bool + // If set, newly delivered child messages are automatically marked as read. This + // field is copied to new child messages. Changes are propagated to the webmail + // client. + ThreadMuted bool + // If set, this (sub)thread is collapsed in the webmail client, for threading mode + // "on" (mode "unread" ignores it). This field is copied to new child message. + // Changes are propagated to the webmail client. + ThreadCollapsed bool + Flags // For keywords other than system flags or the basic well-known $-flags. Only in // "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case @@ -498,6 +528,10 @@ func (m Message) ChangeFlags(orig Flags) ChangeFlags { return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords} } +func (m Message) ChangeThread() ChangeThread { + return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed} +} + // ModSeq represents a modseq as stored in the database. ModSeq 0 in the // database is sent to the client as 1, because modseq 0 is special in IMAP. // ModSeq coming from the client are of type int64. @@ -533,6 +567,22 @@ func (m *Message) PrepareExpunge() { } } +// PrepareThreading sets MessageID and SubjectBase (used in threading) based on the +// envelope in part. +func (m *Message) PrepareThreading(log *mlog.Log, part *message.Part) { + if part.Envelope == nil { + return + } + messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID) + if err != nil { + log.Debugx("parsing message-id, ignoring", err, mlog.Field("messageid", part.Envelope.MessageID)) + } else if raw { + log.Debug("could not parse message-id as address, continuing with raw value", mlog.Field("messageid", part.Envelope.MessageID)) + } + m.MessageID = messageID + m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false) +} + // LoadPart returns a message.Part by reading from m.ParsedBuf. func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) { if m.ParsedBuf == nil { @@ -614,7 +664,7 @@ type Outgoing struct { } // Types stored in DB. -var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}} +var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}, Upgrade{}} // Account holds the information about a user, includings mailboxes, messages, imap subscriptions. type Account struct { @@ -623,6 +673,13 @@ type Account struct { DBPath string // Path to database with mailboxes, messages, etc. DB *bstore.DB // Open database connection. + // Channel that is closed if/when account has/gets "threads" accounting (see + // Upgrade.Threads). + threadsCompleted chan struct{} + // If threads upgrade completed with error, this is set. Used for warning during + // delivery, or aborting when importing. + threadsErr error + // Write lock must be held for account/mailbox modifications including message delivery. // Read lock for reading mailboxes/messages. // When making changes to mailboxes/messages, changes must be broadcasted before @@ -632,6 +689,11 @@ type Account struct { nused int // Reference count, while >0, this account is alive and shared. } +type Upgrade struct { + ID byte + Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed. +} + // InitialUIDValidity returns a UIDValidity used for initializing an account. // It can be replaced during tests with a predictable value. var InitialUIDValidity = func() uint32 { @@ -650,6 +712,7 @@ func closeAccount(acc *Account) (rerr error) { acc.nused-- defer openAccounts.Unlock() if acc.nused == 0 { + // threadsCompleted must be closed now because it increased nused. rerr = acc.DB.Close() acc.DB = nil delete(openAccounts.names, acc.Name) @@ -714,46 +777,127 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) { } }() + acc := &Account{ + Name: accountName, + Dir: accountDir, + DBPath: dbpath, + DB: db, + nused: 1, + threadsCompleted: make(chan struct{}), + } + if isNew { if err := initAccount(db); err != nil { return nil, fmt.Errorf("initializing account: %v", err) } - } else { - // Ensure mailbox counts are set. - var mentioned bool - err := db.Write(context.TODO(), func(tx *bstore.Tx) error { - return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error { - if !mentioned { - mentioned = true - xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", accountName)) - } - mc, err := mb.CalculateCounts(tx) - if err != nil { - return err - } - mb.HaveCounts = true - mb.MailboxCounts = mc - return tx.Update(&mb) - }) - }) - if err != nil { - return nil, fmt.Errorf("calculating counts for mailbox: %v", err) - } + close(acc.threadsCompleted) + return acc, nil } - return &Account{ - Name: accountName, - Dir: accountDir, - DBPath: dbpath, - DB: db, - nused: 1, - }, nil + // Ensure mailbox counts are set. + var mentioned bool + err = db.Write(context.TODO(), func(tx *bstore.Tx) error { + return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error { + if !mentioned { + mentioned = true + xlog.Info("first calculation of mailbox counts for account", mlog.Field("account", accountName)) + } + mc, err := mb.CalculateCounts(tx) + if err != nil { + return err + } + mb.HaveCounts = true + mb.MailboxCounts = mc + return tx.Update(&mb) + }) + }) + if err != nil { + return nil, fmt.Errorf("calculating counts for mailbox: %v", err) + } + + // Start adding threading if needed. + up := Upgrade{ID: 1} + err = db.Write(context.TODO(), func(tx *bstore.Tx) error { + err := tx.Get(&up) + if err == bstore.ErrAbsent { + if err := tx.Insert(&up); err != nil { + return fmt.Errorf("inserting initial upgrade record: %v", err) + } + err = nil + } + return err + }) + if err != nil { + return nil, fmt.Errorf("checking message threading: %v", err) + } + if up.Threads == 2 { + close(acc.threadsCompleted) + return acc, nil + } + + // Increase account use before holding on to account in background. + // Caller holds the lock. The goroutine below decreases nused by calling + // closeAccount. + acc.nused++ + + // Ensure all messages have a MessageID and SubjectBase, which are needed when + // matching threads. + // Then assign messages to threads, in the same way we do during imports. + xlog.Info("upgrading account for threading, in background", mlog.Field("account", acc.Name)) + go func() { + defer func() { + err := closeAccount(acc) + xlog.Check(err, "closing use of account after upgrading account storage for threads", mlog.Field("account", a.Name)) + }() + + defer func() { + x := recover() // Should not happen, but don't take program down if it does. + if x != nil { + xlog.Error("upgradeThreads panic", mlog.Field("err", x)) + debug.PrintStack() + metrics.PanicInc("upgradeThreads") + acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x) + } + + // Mark that upgrade has finished, possibly error is indicated in threadsErr. + close(acc.threadsCompleted) + }() + + err := upgradeThreads(mox.Shutdown, acc, &up) + if err != nil { + a.threadsErr = err + xlog.Errorx("upgrading account for threading, aborted", err, mlog.Field("account", a.Name)) + } else { + xlog.Info("upgrading account for threading, completed", mlog.Field("account", a.Name)) + } + }() + return acc, nil +} + +// ThreadingWait blocks until the one-time account threading upgrade for the +// account has completed, and returns an error if not successful. +// +// To be used before starting an import of messages. +func (a *Account) ThreadingWait(log *mlog.Log) error { + select { + case <-a.threadsCompleted: + return a.threadsErr + default: + } + log.Debug("waiting for account upgrade to complete") + + <-a.threadsCompleted + return a.threadsErr } func initAccount(db *bstore.DB) error { return db.Write(context.TODO(), func(tx *bstore.Tx) error { uidvalidity := InitialUIDValidity() + if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil { + return err + } + if len(mox.Conf.Static.DefaultMailboxes) > 0 { // Deprecated in favor of InitialMailboxes. defaultMailboxes := mox.Conf.Static.DefaultMailboxes @@ -862,10 +1006,14 @@ func (a *Account) Close() error { // - Message with UID >= mailbox uid next. // - Mailbox uidvalidity >= account uid validity. // - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq. +// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail. func (a *Account) CheckConsistency() error { - var uiderrors []string // With a limit, could be many. - var modseqerrors []string // With limit. - var fileerrors []string // With limit. + var uidErrors []string // With a limit, could be many. + var modseqErrors []string // With limit. + var fileErrors []string // With limit. + var threadidErrors []string // With limit. + var threadParentErrors []string // With limit. + var threadAncestorErrors []string // With limit. var errors []string err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error { @@ -897,13 +1045,13 @@ func (a *Account) CheckConsistency() error { mb := mailboxes[m.MailboxID] - if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqerrors) < 20 { + if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 { modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq) - modseqerrors = append(modseqerrors, modseqerr) + modseqErrors = append(modseqErrors, modseqerr) } - if m.UID >= mb.UIDNext && len(uiderrors) < 20 { + if m.UID >= mb.UIDNext && len(uidErrors) < 20 { uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext) - uiderrors = append(uiderrors, uiderr) + uidErrors = append(uidErrors, uiderr) } if m.Expunged { return nil @@ -912,10 +1060,32 @@ func (a *Account) CheckConsistency() error { st, err := os.Stat(p) if err != nil { existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err) - fileerrors = append(fileerrors, existserr) - } else if len(fileerrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() { + fileErrors = append(fileErrors, existserr) + } else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() { sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size()) - fileerrors = append(fileerrors, sizeerr) + fileErrors = append(fileErrors, sizeerr) + } + + if m.ThreadID <= 0 && len(threadidErrors) < 20 { + err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID) + threadidErrors = append(threadidErrors, err) + } + if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 { + err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID) + threadParentErrors = append(threadParentErrors, err) + } + for i, pid := range m.ThreadParentIDs { + am := Message{ID: pid} + if err := tx.Get(&am); err == bstore.ErrAbsent { + continue + } else if err != nil { + return fmt.Errorf("get ancestor message: %v", err) + } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 { + err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs) + threadAncestorErrors = append(threadAncestorErrors, err) + } else { + break + } } return nil }) @@ -938,9 +1108,12 @@ func (a *Account) CheckConsistency() error { if err != nil { return err } - errors = append(errors, uiderrors...) - errors = append(errors, modseqerrors...) - errors = append(errors, fileerrors...) + errors = append(errors, uidErrors...) + errors = append(errors, modseqErrors...) + errors = append(errors, fileErrors...) + errors = append(errors, threadidErrors...) + errors = append(errors, threadParentErrors...) + errors = append(errors, threadAncestorErrors...) if len(errors) > 0 { return fmt.Errorf("%s", strings.Join(errors, "; ")) } @@ -1031,7 +1204,7 @@ func (a *Account) WithRLock(fn func()) { // Caller must broadcast new message. // // Caller must update mailbox counts. -func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, consumeFile, sync, notrain bool) error { +func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, consumeFile, sync, notrain, nothreads bool) error { if m.Expunged { return fmt.Errorf("cannot deliver expunged message") } @@ -1049,9 +1222,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi conf, _ := a.Conf() m.JunkFlagsForMailbox(mb.Name, conf) + mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile. var part *message.Part if m.ParsedBuf == nil { - mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile. p, err := message.EnsurePart(log, false, mr, m.Size) if err != nil { log.Infox("parsing delivered message", err, mlog.Field("parse", ""), mlog.Field("message", m.ID)) @@ -1063,6 +1236,13 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi return fmt.Errorf("marshal parsed message: %w", err) } m.ParsedBuf = buf + } else { + var p message.Part + if err := json.Unmarshal(m.ParsedBuf, &p); err != nil { + log.Errorx("unmarshal parsed message, continuing", err, mlog.Field("parse", "")) + } else { + part = &p + } } // If we are delivering to the originally intended mailbox, no need to store the mailbox ID again. @@ -1078,52 +1258,73 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi m.ModSeq = modseq } + if part != nil && m.MessageID == "" && m.SubjectBase == "" { + m.PrepareThreading(log, part) + } + + // Assign to thread (if upgrade has completed). + noThreadID := nothreads + if m.ThreadID == 0 && !nothreads && part != nil { + select { + case <-a.threadsCompleted: + if a.threadsErr != nil { + log.Info("not assigning threads for new delivery, upgrading to threads failed") + noThreadID = true + } else { + if err := assignThread(log, tx, m, part); err != nil { + return fmt.Errorf("assigning thread: %w", err) + } + } + default: + // note: since we have a write transaction to get here, we can't wait for the + // thread upgrade to finish. + // If we don't assign a threadid the upgrade process will do it. + log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message") + noThreadID = true + } + } + if err := tx.Insert(m); err != nil { return fmt.Errorf("inserting message: %w", err) } + if !noThreadID && m.ThreadID == 0 { + m.ThreadID = m.ID + if err := tx.Update(m); err != nil { + return fmt.Errorf("updating message for its own thread id: %w", err) + } + } // todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's. for webmail, we could insert the recipients directly. - if mb.Sent { - // Attempt to parse the message for its To/Cc/Bcc headers, which we insert into Recipient. - if part == nil { - var p message.Part - if err := json.Unmarshal(m.ParsedBuf, &p); err != nil { - log.Errorx("unmarshal parsed message for its to,cc,bcc headers, continuing", err, mlog.Field("parse", "")) - } else { - part = &p - } + if mb.Sent && part != nil && part.Envelope != nil { + e := part.Envelope + sent := e.Date + if sent.IsZero() { + sent = m.Received } - if part != nil && part.Envelope != nil { - e := part.Envelope - sent := e.Date - if sent.IsZero() { - sent = m.Received + if sent.IsZero() { + sent = time.Now() + } + addrs := append(append(e.To, e.CC...), e.BCC...) + for _, addr := range addrs { + if addr.User == "" { + // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero. + log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", mlog.Field("address", addr)) + continue } - if sent.IsZero() { - sent = time.Now() + d, err := dns.ParseDomain(addr.Host) + if err != nil { + log.Debugx("parsing domain in to/cc/bcc address", err, mlog.Field("address", addr)) + continue } - addrs := append(append(e.To, e.CC...), e.BCC...) - for _, addr := range addrs { - if addr.User == "" { - // Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero. - log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", mlog.Field("address", addr)) - continue - } - d, err := dns.ParseDomain(addr.Host) - if err != nil { - log.Debugx("parsing domain in to/cc/bcc address", err, mlog.Field("address", addr)) - continue - } - mr := Recipient{ - MessageID: m.ID, - Localpart: smtp.Localpart(addr.User), - Domain: d.Name(), - OrgDomain: publicsuffix.Lookup(context.TODO(), d).Name(), - Sent: sent, - } - if err := tx.Insert(&mr); err != nil { - return fmt.Errorf("inserting sent message recipients: %w", err) - } + mr := Recipient{ + MessageID: m.ID, + Localpart: smtp.Localpart(addr.User), + Domain: d.Name(), + OrgDomain: publicsuffix.Lookup(context.TODO(), d).Name(), + Sent: sent, + } + if err := tx.Insert(&mr); err != nil { + return fmt.Errorf("inserting sent message recipients: %w", err) } } } @@ -1440,7 +1641,7 @@ ruleset: // MessagePath returns the file system path of a message. func (a *Account) MessagePath(messageID int64) string { - return filepath.Join(a.Dir, "msg", MessagePath(messageID)) + return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), "/") } // MessageReader opens a message for reading, transparently combining the @@ -1489,7 +1690,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF return fmt.Errorf("updating mailbox for delivery: %w", err) } - if err := a.DeliverMessage(log, tx, m, msgFile, consumeFile, true, false); err != nil { + if err := a.DeliverMessage(log, tx, m, msgFile, consumeFile, true, false, false); err != nil { return err } @@ -1773,6 +1974,12 @@ const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVW // MessagePath returns the filename of the on-disk filename, relative to the containing directory such as /msg or queue. // Returns names like "AB/1". func MessagePath(messageID int64) string { + return strings.Join(messagePathElems(messageID), "/") +} + +// messagePathElems returns the elems, for a single join without intermediate +// string allocations. +func messagePathElems(messageID int64) []string { v := messageID >> 13 // 8k files per directory. dir := "" for { @@ -1782,7 +1989,7 @@ func MessagePath(messageID int64) string { break } } - return fmt.Sprintf("%s/%d", dir, messageID) + return []string{dir, strconv.FormatInt(messageID, 10)} } // Set returns a copy of f, with each flag that is true in mask set to the diff --git a/store/account_test.go b/store/account_test.go index 2bb4d00..b208f50 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -58,6 +58,8 @@ func TestMailbox(t *testing.T) { MsgPrefix: msgPrefix, } msent := m + m.ThreadMuted = true + m.ThreadCollapsed = true var mbsent Mailbox mbrejects := Mailbox{Name: "Rejects", UIDValidity: 1, UIDNext: 1, HaveCounts: true} mreject := m @@ -77,8 +79,11 @@ func TestMailbox(t *testing.T) { tcheck(t, err, "sent mailbox") msent.MailboxID = mbsent.ID msent.MailboxOrigID = mbsent.ID - err = acc.DeliverMessage(xlog, tx, &msent, msgFile, false, true, false) + err = acc.DeliverMessage(xlog, tx, &msent, msgFile, false, true, false, false) tcheck(t, err, "deliver message") + if !msent.ThreadMuted || !msent.ThreadCollapsed { + t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m") + } err = tx.Get(&mbsent) tcheck(t, err, "get mbsent") @@ -90,7 +95,7 @@ func TestMailbox(t *testing.T) { tcheck(t, err, "insert rejects mailbox") mreject.MailboxID = mbrejects.ID mreject.MailboxOrigID = mbrejects.ID - err = acc.DeliverMessage(xlog, tx, &mreject, msgFile, false, true, false) + err = acc.DeliverMessage(xlog, tx, &mreject, msgFile, false, true, false, false) tcheck(t, err, "deliver message") err = tx.Get(&mbrejects) diff --git a/store/state.go b/store/state.go index 817f157..dac92be 100644 --- a/store/state.go +++ b/store/state.go @@ -50,6 +50,13 @@ type ChangeFlags struct { Keywords []string // Non-system/well-known flags/keywords/labels. } +// ChangeThread is sent when muted/collapsed changes. +type ChangeThread struct { + MessageIDs []int64 + Muted bool + Collapsed bool +} + // ChangeRemoveMailbox is sent for a removed mailbox. type ChangeRemoveMailbox struct { MailboxID int64 diff --git a/store/threads.go b/store/threads.go new file mode 100644 index 0000000..5c69ce0 --- /dev/null +++ b/store/threads.go @@ -0,0 +1,775 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "runtime" + "sort" + "time" + + "golang.org/x/exp/slices" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/message" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/moxio" +) + +// Assign a new/incoming message to a thread. Message does not yet have an ID. If +// this isn't a response, ThreadID should remain 0 (unless this is a message with +// existing message-id) and the caller must set ThreadID to ID. +// If the account is still busy upgrading messages with threadids in the background, parents +// may have a threadid 0. That results in this message getting threadid 0, which +// will handled by the background upgrade process assigning a threadid when it gets +// to this message. +func assignThread(log *mlog.Log, tx *bstore.Tx, m *Message, part *message.Part) error { + if m.MessageID != "" { + // Match against existing different message with same Message-ID. + q := bstore.QueryTx[Message](tx) + q.FilterNonzero(Message{MessageID: m.MessageID}) + q.FilterEqual("Expunged", false) + q.FilterNotEqual("ID", m.ID) + q.FilterNotEqual("ThreadID", int64(0)) + q.SortAsc("ID") + q.Limit(1) + em, err := q.Get() + if err != nil && err != bstore.ErrAbsent { + return fmt.Errorf("looking up existing message with message-id: %v", err) + } else if err == nil { + assignParent(m, em, true) + return nil + } + } + + h, err := part.Header() + if err != nil { + log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID)) + } + messageIDs, err := message.ReferencedIDs(h.Values("References"), h.Values("In-Reply-To")) + if err != nil { + log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID)) + } + for i := len(messageIDs) - 1; i >= 0; i-- { + messageID := messageIDs[i] + if messageID == m.MessageID { + continue + } + tm, _, err := lookupThreadMessage(tx, m.ID, messageID, m.SubjectBase) + if err != nil { + return fmt.Errorf("looking up thread message for new message: %v", err) + } else if tm != nil { + assignParent(m, *tm, true) + return nil + } + m.ThreadMissingLink = true + } + if len(messageIDs) > 0 { + return nil + } + + var isResp bool + if part != nil && part.Envelope != nil { + m.SubjectBase, isResp = message.ThreadSubject(part.Envelope.Subject, false) + } + if !isResp || m.SubjectBase == "" { + return nil + } + m.ThreadMissingLink = true + tm, err := lookupThreadMessageSubject(tx, *m, m.SubjectBase) + if err != nil { + return fmt.Errorf("looking up thread message by subject: %v", err) + } else if tm != nil { + assignParent(m, *tm, true) + } + return nil +} + +// assignParent assigns threading fields to m that make it a child of parent message pm. +// updateSeen indicates if m.Seen should be cleared if pm is thread-muted. +func assignParent(m *Message, pm Message, updateSeen bool) { + if pm.ThreadID == 0 { + panic(fmt.Sprintf("assigning message id %d/d%q to parent message id %d/%q which has threadid 0", m.ID, m.MessageID, pm.ID, pm.MessageID)) + } + if m.ID == pm.ID { + panic(fmt.Sprintf("trying to make message id %d/%q its own parent", m.ID, m.MessageID)) + } + m.ThreadID = pm.ThreadID + // Make sure we don't add cycles. + if !slices.Contains(pm.ThreadParentIDs, m.ID) { + m.ThreadParentIDs = append([]int64{pm.ID}, pm.ThreadParentIDs...) + } else if pm.ID != m.ID { + m.ThreadParentIDs = []int64{pm.ID} + } else { + m.ThreadParentIDs = nil + } + if m.MessageID != "" && m.MessageID == pm.MessageID { + m.ThreadMissingLink = true + } + m.ThreadMuted = pm.ThreadMuted + m.ThreadCollapsed = pm.ThreadCollapsed + if updateSeen && m.ThreadMuted { + m.Seen = true + } +} + +// ResetThreading resets the MessageID and SubjectBase fields for all messages in +// the account. If clearIDs is true, all Thread* fields are also cleared. Changes +// are made in transactions of batchSize changes. The total number of updated +// messages is returned. +// +// ModSeq is not changed. Calles should bump the uid validity of the mailboxes +// to propagate the changes to IMAP clients. +func (a *Account) ResetThreading(ctx context.Context, log *mlog.Log, batchSize int, clearIDs bool) (int, error) { + // todo: should this send Change events for ThreadMuted and ThreadCollapsed? worth it? + + var lastID int64 + total := 0 + for { + n := 0 + + prepareMessages := func(in, out chan moxio.Work[Message, Message]) { + for { + w, ok := <-in + if !ok { + return + } + + m := w.In + + // We have the Message-ID and Subject headers in ParsedBuf. We use a partial part + // struct so we don't generate so much garbage for the garbage collector to sift + // through. + var part struct { + Envelope *message.Envelope + } + if err := json.Unmarshal(m.ParsedBuf, &part); err != nil { + log.Errorx("unmarshal json parsedbuf for setting message-id, skipping", err, mlog.Field("msgid", m.ID)) + } else { + m.MessageID = "" + if part.Envelope != nil && part.Envelope.MessageID != "" { + s, _, err := message.MessageIDCanonical(part.Envelope.MessageID) + if err != nil { + log.Debugx("parsing message-id, skipping", err, mlog.Field("msgid", m.ID), mlog.Field("messageid", part.Envelope.MessageID)) + } + m.MessageID = s + } + if part.Envelope != nil { + m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false) + } + } + w.Out = m + + out <- w + } + } + + err := a.DB.Write(ctx, func(tx *bstore.Tx) error { + processMessage := func(in, m Message) error { + if clearIDs { + m.ThreadID = 0 + m.ThreadParentIDs = nil + m.ThreadMissingLink = false + } + return tx.Update(&m) + } + + // JSON parsing is relatively heavy, we benefit from multiple goroutines. + procs := runtime.GOMAXPROCS(0) + wq := moxio.NewWorkQueue[Message, Message](procs, 2*procs, prepareMessages, processMessage) + + q := bstore.QueryTx[Message](tx) + q.FilterEqual("Expunged", false) + q.FilterGreater("ID", lastID) + q.SortAsc("ID") + err := q.ForEach(func(m Message) error { + // We process in batches so we don't block other operations for a long time. + if n >= batchSize { + return bstore.StopForEach + } + // Update starting point for next batch. + lastID = m.ID + + n++ + return wq.Add(m) + }) + if err == nil { + err = wq.Finish() + } + wq.Stop() + return err + }) + if err != nil { + return total, fmt.Errorf("upgrading account to threads storage, step 1/2: %w", err) + } + total += n + if n == 0 { + break + } + } + return total, nil +} + +// AssignThreads assigns thread-related fields to messages with ID >= +// startMessageID. Changes are committed each batchSize changes if txOpt is nil +// (i.e. during automatic account upgrade, we don't want to block database access +// for a long time). If txOpt is not nil, all changes are made in that +// transaction. +// +// When resetting thread assignments, the caller must first clear the existing +// thread fields. +// +// Messages are processed in order of ID, so when added to the account, not +// necessarily by received/date. Most threaded messages can immediately be matched +// to their parent message. If not, we keep track of the missing message-id and +// resolve as soon as we encounter it. At the end, we resolve all remaining +// messages, they start with a cycle. +// +// Does not set Seen flag for muted threads. +// +// Progress is written to progressWriter, every 100k messages. +func (a *Account) AssignThreads(ctx context.Context, log *mlog.Log, txOpt *bstore.Tx, startMessageID int64, batchSize int, progressWriter io.Writer) error { + // We use a more basic version of the thread-matching algorithm describe in: + // ../rfc/5256:443 + // The algorithm assumes you'll select messages, then group into threads. We normally do + // thread-calculation when messages are delivered. Here, we assign threads as soon + // as we can, but will queue messages that reference known ancestors and resolve as + // soon as we process them. We can handle large number of messages, but not very + // quickly because we make lots of database queries. + + type childMsg struct { + ID int64 // This message will be fetched and updated with the threading fields once the parent is resolved. + MessageID string // Of child message. Once child is resolved, its own children can be resolved too. + ThreadMissingLink bool + } + // Messages that have a References/In-Reply-To that we want to set as parent, but + // where the parent doesn't have a ThreadID yet are added to pending. The key is + // the normalized MessageID of the parent, and the value is a list of messages that + // can get resolved once the parent gets its ThreadID. The kids will get the same + // ThreadIDs, and they themselves may be parents to kids, and so on. + // For duplicate messages (messages with identical Message-ID), the second + // Message-ID to be added to pending is added under its own message-id, so it gets + // its original as parent. + pending := map[string][]childMsg{} + + // Current tx. If not equal to txOpt, we clean it up before we leave. + var tx *bstore.Tx + defer func() { + if tx != nil && tx != txOpt { + err := tx.Rollback() + log.Check(err, "rolling back transaction") + } + }() + + // Set thread-related fields for a single message. Caller must save the message, + // only if not an error and not added to the pending list. + assign := func(m *Message, references, inReplyTo []string, subject string) (pend bool, rerr error) { + if m.MessageID != "" { + // Attempt to match against existing different message with same Message-ID that + // already has a threadid. + // If there are multiple messages for a message-id a future call to assign may use + // its threadid, or it may end up in pending and we resolve it when we need to. + q := bstore.QueryTx[Message](tx) + q.FilterNonzero(Message{MessageID: m.MessageID}) + q.FilterEqual("Expunged", false) + q.FilterLess("ID", m.ID) + q.SortAsc("ID") + q.Limit(1) + em, err := q.Get() + if err != nil && err != bstore.ErrAbsent { + return false, fmt.Errorf("looking up existing message with message-id: %v", err) + } else if err == nil { + if em.ThreadID == 0 { + pending[em.MessageID] = append(pending[em.MessageID], childMsg{m.ID, m.MessageID, true}) + return true, nil + } else { + assignParent(m, em, false) + return false, nil + } + } + } + + refids, err := message.ReferencedIDs(references, inReplyTo) + if err != nil { + log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID)) + } + + for i := len(refids) - 1; i >= 0; i-- { + messageID := refids[i] + if messageID == m.MessageID { + continue + } + tm, exists, err := lookupThreadMessage(tx, m.ID, messageID, m.SubjectBase) + if err != nil { + return false, fmt.Errorf("lookup up thread by message-id %s for message id %d: %w", messageID, m.ID, err) + } else if tm != nil { + assignParent(m, *tm, false) + return false, nil + } else if exists { + pending[messageID] = append(pending[messageID], childMsg{m.ID, m.MessageID, i < len(refids)-1}) + return true, nil + } + } + + var subjectBase string + var isResp bool + if subject != "" { + subjectBase, isResp = message.ThreadSubject(subject, false) + } + if len(refids) > 0 || !isResp || subjectBase == "" { + m.ThreadID = m.ID + m.ThreadMissingLink = len(refids) > 0 + return false, nil + } + + // No references to use. If this is a reply/forward (based on subject), we'll match + // against base subject, at most 4 weeks back so we don't match against ancient + // messages and 1 day ahead so we can match against delayed deliveries. + tm, err := lookupThreadMessageSubject(tx, *m, subjectBase) + if err != nil { + return false, fmt.Errorf("looking up recent messages by base subject %q: %w", subjectBase, err) + } else if tm != nil { + m.ThreadID = tm.ThreadID + m.ThreadParentIDs = []int64{tm.ThreadID} // Always under root message with subject-match. + m.ThreadMissingLink = true + m.ThreadMuted = tm.ThreadMuted + m.ThreadCollapsed = tm.ThreadCollapsed + } else { + m.ThreadID = m.ID + } + return false, nil + } + + npendingResolved := 0 + + // Resolve pending messages that wait on m.MessageID to be resolved, recursively. + var resolvePending func(tm Message, cyclic bool) error + resolvePending = func(tm Message, cyclic bool) error { + if tm.MessageID == "" { + return nil + } + l := pending[tm.MessageID] + delete(pending, tm.MessageID) + for _, mi := range l { + m := Message{ID: mi.ID} + if err := tx.Get(&m); err != nil { + return fmt.Errorf("get message %d for resolving pending thread for message-id %s, %d: %w", mi.ID, tm.MessageID, tm.ID, err) + } + if m.ThreadID != 0 { + // ThreadID already set because this is a cyclic message. If we would assign a + // parent again, we would create a cycle. + if m.MessageID != tm.MessageID && !cyclic { + panic(fmt.Sprintf("threadid already set (%d) while handling non-cyclic message id %d/%q and with different message-id %q as parent message id %d", m.ThreadID, m.ID, m.MessageID, tm.MessageID, tm.ID)) + } + continue + } + assignParent(&m, tm, false) + m.ThreadMissingLink = mi.ThreadMissingLink + if err := tx.Update(&m); err != nil { + return fmt.Errorf("update message %d for resolving pending thread for message-id %s, %d: %w", mi.ID, tm.MessageID, tm.ID, err) + } + if err := resolvePending(m, cyclic); err != nil { + return err + } + npendingResolved++ + } + return nil + } + + // Output of the worker goroutines. + type threadPrep struct { + references []string + inReplyTo []string + subject string + } + + // Single allocation. + threadingFields := [][]byte{ + []byte("references"), + []byte("in-reply-to"), + []byte("subject"), + } + + // Worker goroutine function. We start with a reasonably large buffer for reading + // the header into. And we have scratch space to copy the needed headers into. That + // means we normally won't allocate any more buffers. + prepareMessages := func(in, out chan moxio.Work[Message, threadPrep]) { + headerbuf := make([]byte, 8*1024) + scratch := make([]byte, 4*1024) + for { + w, ok := <-in + if !ok { + return + } + + m := w.In + var partialPart struct { + HeaderOffset int64 + BodyOffset int64 + } + if err := json.Unmarshal(m.ParsedBuf, &partialPart); err != nil { + w.Err = fmt.Errorf("unmarshal part: %v", err) + } else { + size := partialPart.BodyOffset - partialPart.HeaderOffset + if int(size) > len(headerbuf) { + headerbuf = make([]byte, size) + } + if size > 0 { + buf := headerbuf[:int(size)] + err := func() error { + mr := a.MessageReader(m) + defer mr.Close() + + // ReadAt returns whole buffer or error. Single read should be fast. + n, err := mr.ReadAt(buf, partialPart.HeaderOffset) + if err != nil || n != len(buf) { + return fmt.Errorf("read header: %v", err) + } + return nil + }() + if err != nil { + w.Err = err + } else if h, err := message.ParseHeaderFields(buf, scratch, threadingFields); err != nil { + w.Err = err + } else { + w.Out.references = h["References"] + w.Out.inReplyTo = h["In-Reply-To"] + l := h["Subject"] + if len(l) > 0 { + w.Out.subject = l[0] + } + } + } + } + + out <- w + } + } + + // Assign threads to messages, possibly in batches. + nassigned := 0 + for { + n := 0 + tx = txOpt + if tx == nil { + var err error + tx, err = a.DB.Begin(ctx, true) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + } + + processMessage := func(m Message, prep threadPrep) error { + pend, err := assign(&m, prep.references, prep.inReplyTo, prep.subject) + if err != nil { + return fmt.Errorf("for msgid %d: %w", m.ID, err) + } else if pend { + return nil + } + if m.ThreadID == 0 { + panic(fmt.Sprintf("no threadid after assign of message id %d/%q", m.ID, m.MessageID)) + } + // Fields have been set, store in database and resolve messages waiting for this MessageID. + if slices.Contains(m.ThreadParentIDs, m.ID) { + panic(fmt.Sprintf("message id %d/%q contains itself in parent ids %v", m.ID, m.MessageID, m.ThreadParentIDs)) + } + if err := tx.Update(&m); err != nil { + return err + } + if err := resolvePending(m, false); err != nil { + return fmt.Errorf("resolving pending message-id: %v", err) + } + return nil + } + + // Use multiple worker goroutines to read parse headers from on-disk messages. + procs := runtime.GOMAXPROCS(0) + wq := moxio.NewWorkQueue[Message, threadPrep](2*procs, 4*procs, prepareMessages, processMessage) + + // We assign threads in order by ID, so messages delivered in between our + // transaction will get assigned threads too: they'll have the highest id's. + q := bstore.QueryTx[Message](tx) + q.FilterGreaterEqual("ID", startMessageID) + q.FilterEqual("Expunged", false) + q.SortAsc("ID") + err := q.ForEach(func(m Message) error { + // Batch number of changes, so we give other users of account a change to run. + if txOpt == nil && n >= batchSize { + return bstore.StopForEach + } + // Starting point for next batch. + startMessageID = m.ID + 1 + // Don't process again. Can happen when earlier upgrade was aborted. + if m.ThreadID != 0 { + return nil + } + + n++ + return wq.Add(m) + }) + if err == nil { + err = wq.Finish() + } + wq.Stop() + + if err == nil && txOpt == nil { + err = tx.Commit() + tx = nil + } + if err != nil { + return fmt.Errorf("assigning threads: %w", err) + } + if n == 0 { + break + } + nassigned += n + if nassigned%100000 == 0 { + log.Debug("assigning threads, progress", mlog.Field("count", nassigned), mlog.Field("unresolved", len(pending))) + if _, err := fmt.Fprintf(progressWriter, "assigning threads, progress: %d messages\n", nassigned); err != nil { + return fmt.Errorf("writing progress: %v", err) + } + } + } + if _, err := fmt.Fprintf(progressWriter, "assigning threads, done: %d messages\n", nassigned); err != nil { + return fmt.Errorf("writing progress: %v", err) + } + + log.Debug("assigning threads, mostly done, finishing with resolving of cyclic messages", mlog.Field("count", nassigned), mlog.Field("unresolved", len(pending))) + + if _, err := fmt.Fprintf(progressWriter, "assigning threads, resolving %d cyclic pending message-ids\n", len(pending)); err != nil { + return fmt.Errorf("writing progress: %v", err) + } + + // Remaining messages in pending have cycles and possibly tails. The cycle is at + // the head of the thread. Once we resolve that, the rest of the thread can be + // resolved too. Ignoring self-references (duplicate messages), there can only be + // one cycle, and it is at the head. So we look for cycles, ignoring + // self-references, and resolve a message as soon as we see the cycle. + + parent := map[string]string{} // Child Message-ID pointing to the parent Message-ID, excluding self-references. + pendlist := []string{} + for pmsgid, l := range pending { + pendlist = append(pendlist, pmsgid) + for _, k := range l { + if k.MessageID == pmsgid { + // No self-references for duplicate messages. + continue + } + if _, ok := parent[k.MessageID]; !ok { + parent[k.MessageID] = pmsgid + } + // else, this message should be resolved by following pending. + } + } + sort.Strings(pendlist) + + tx = txOpt + if tx == nil { + var err error + tx, err = a.DB.Begin(ctx, true) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + } + + // We walk through all messages of pendlist, but some will already have been + // resolved by the time we get to them. + done := map[string]bool{} + for _, msgid := range pendlist { + if done[msgid] { + continue + } + + // We walk up to parent, until we see a message-id we've already seen, a cycle. + seen := map[string]bool{} + for { + pmsgid, ok := parent[msgid] + if !ok { + panic(fmt.Sprintf("missing parent message-id %q, not a cycle?", msgid)) + } + if !seen[pmsgid] { + seen[pmsgid] = true + msgid = pmsgid + continue + } + + // Cycle detected. Make this message-id the thread root. + q := bstore.QueryTx[Message](tx) + q.FilterNonzero(Message{MessageID: msgid}) + q.FilterEqual("ThreadID", int64(0)) + q.FilterEqual("Expunged", false) + q.SortAsc("ID") + l, err := q.List() + if err == nil && len(l) == 0 { + err = errors.New("no messages") + } + if err != nil { + return fmt.Errorf("list message by message-id for cyclic thread root: %v", err) + } + for i, m := range l { + m.ThreadID = l[0].ID + m.ThreadMissingLink = true + if i == 0 { + m.ThreadParentIDs = nil + l[0] = m // For resolvePending below. + } else { + assignParent(&m, l[0], false) + } + if slices.Contains(m.ThreadParentIDs, m.ID) { + panic(fmt.Sprintf("message id %d/%q contains itself in parents %v", m.ID, m.MessageID, m.ThreadParentIDs)) + } + if err := tx.Update(&m); err != nil { + return fmt.Errorf("assigning threadid to cyclic thread root: %v", err) + } + } + + // Mark all children as done so we don't process these messages again. + walk := map[string]struct{}{msgid: {}} + for len(walk) > 0 { + for msgid := range walk { + delete(walk, msgid) + if done[msgid] { + continue + } + done[msgid] = true + for _, mi := range pending[msgid] { + if !done[mi.MessageID] { + walk[mi.MessageID] = struct{}{} + } + } + } + } + + // Resolve all messages in this thread. + if err := resolvePending(l[0], true); err != nil { + return fmt.Errorf("resolving cyclic children of cyclic thread root: %v", err) + } + + break + } + } + + // Check that there are no more messages without threadid. + q := bstore.QueryTx[Message](tx) + q.FilterEqual("ThreadID", int64(0)) + q.FilterEqual("Expunged", false) + l, err := q.List() + if err == nil && len(l) > 0 { + err = errors.New("found messages without threadid") + } + if err != nil { + return fmt.Errorf("listing messages without threadid: %v", err) + } + + if txOpt == nil { + err := tx.Commit() + tx = nil + if err != nil { + return fmt.Errorf("commit resolving cyclic thread roots: %v", err) + } + } + return nil +} + +// lookupThreadMessage tries to find the parent message with messageID that must +// have a matching subjectBase. +// +// If the message isn't present (with a valid thread id), a nil message and nil +// error is returned. The bool return value indicates if a message with the +// message-id exists at all. +func lookupThreadMessage(tx *bstore.Tx, mID int64, messageID, subjectBase string) (*Message, bool, error) { + q := bstore.QueryTx[Message](tx) + q.FilterNonzero(Message{MessageID: messageID}) + q.FilterEqual("SubjectBase", subjectBase) + q.FilterEqual("Expunged", false) + q.FilterNotEqual("ID", mID) + q.SortAsc("ID") + l, err := q.List() + if err != nil { + return nil, false, fmt.Errorf("message-id %s: %w", messageID, err) + } + exists := len(l) > 0 + for _, tm := range l { + if tm.ThreadID != 0 { + return &tm, true, nil + } + } + return nil, exists, nil +} + +// lookupThreadMessageSubject looks up a parent/ancestor message for the message +// thread based on a matching subject. The message must have been delivered to the same mailbox originally. +// +// If no message (with a threadid) is found a nil message and nil error is returned. +func lookupThreadMessageSubject(tx *bstore.Tx, m Message, subjectBase string) (*Message, error) { + q := bstore.QueryTx[Message](tx) + q.FilterGreater("Received", m.Received.Add(-4*7*24*time.Hour)) + q.FilterLess("Received", m.Received.Add(1*24*time.Hour)) + q.FilterNonzero(Message{SubjectBase: subjectBase, MailboxOrigID: m.MailboxOrigID}) + q.FilterEqual("Expunged", false) + q.FilterNotEqual("ID", m.ID) + q.FilterNotEqual("ThreadID", int64(0)) + q.SortDesc("Received") + q.Limit(1) + tm, err := q.Get() + if err == bstore.ErrAbsent { + return nil, nil + } else if err != nil { + return nil, err + } + return &tm, nil +} + +func upgradeThreads(ctx context.Context, acc *Account, up *Upgrade) error { + log := xlog.Fields(mlog.Field("account", acc.Name)) + + if up.Threads == 0 { + // Step 1 in the threads upgrade is storing the canonicalized Message-ID for each + // message and the base subject for thread matching. This allows efficient thread + // lookup in the second step. + + log.Info("upgrading account for threading, step 1/2: updating all messages with message-id and base subject") + t0 := time.Now() + + const batchSize = 10000 + total, err := acc.ResetThreading(ctx, xlog, batchSize, true) + if err != nil { + return fmt.Errorf("resetting message threading fields: %v", err) + } + + up.Threads = 1 + if err := acc.DB.Update(ctx, up); err != nil { + up.Threads = 0 + return fmt.Errorf("saving upgrade process while upgrading account to threads storage, step 1/2: %w", err) + } + log.Info("upgrading account for threading, step 1/2: completed", mlog.Field("duration", time.Since(t0)), mlog.Field("messages", total)) + } + + if up.Threads == 1 { + // Step 2 of the upgrade is going through all messages and assigning threadid's. + // Lookup of messageid and base subject is now fast through indexed database + // access. + + log.Info("upgrading account for threading, step 2/2: matching messages to threads") + t0 := time.Now() + + const batchSize = 10000 + if err := acc.AssignThreads(ctx, xlog, nil, 1, batchSize, io.Discard); err != nil { + return fmt.Errorf("upgrading to threads storage, step 2/2: %w", err) + } + up.Threads = 2 + if err := acc.DB.Update(ctx, up); err != nil { + up.Threads = 1 + return fmt.Errorf("saving upgrade process for thread storage, step 2/2: %w", err) + } + log.Info("upgrading account for threading, step 2/2: completed", mlog.Field("duration", time.Since(t0))) + } + + // Note: Not bumping uidvalidity or setting modseq. Clients haven't been able to + // use threadid's before, so there is nothing to be out of date. + + return nil +} diff --git a/store/threads_test.go b/store/threads_test.go new file mode 100644 index 0000000..606f8e1 --- /dev/null +++ b/store/threads_test.go @@ -0,0 +1,208 @@ +package store + +import ( + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" +) + +func TestThreadingUpgrade(t *testing.T) { + os.RemoveAll("../testdata/store/data") + mox.ConfigStaticPath = "../testdata/store/mox.conf" + mox.MustLoadConfig(true, false) + acc, err := OpenAccount("mjl") + tcheck(t, err, "open account") + defer func() { + err = acc.Close() + tcheck(t, err, "closing account") + }() + defer Switchboard()() + + log := mlog.New("store") + + // New account already has threading. Add some messages, check the threading. + deliver := func(recv time.Time, s string, expThreadID int64) Message { + t.Helper() + f, err := CreateMessageTemp("account-test") + tcheck(t, err, "temp file") + defer f.Close() + + s = strings.ReplaceAll(s, "\n", "\r\n") + m := Message{ + Size: int64(len(s)), + MsgPrefix: []byte(s), + Received: recv, + } + err = acc.DeliverMailbox(log, "Inbox", &m, f, true) + tcheck(t, err, "deliver") + if expThreadID == 0 { + expThreadID = m.ID + } + if m.ThreadID != expThreadID { + t.Fatalf("got threadid %d, expected %d", m.ThreadID, expThreadID) + } + return m + } + + now := time.Now() + + m0 := deliver(now, "Message-ID: \nSubject: test1\n\ntest\n", 0) + m1 := deliver(now, "Message-ID: \nReferences: \nSubject: test1\n\ntest\n", m0.ID) // References. + m2 := deliver(now, "Message-ID: \nReferences: \nSubject: other\n\ntest\n", 0) // References, but different subject. + m3 := deliver(now, "Message-ID: \nIn-Reply-To: \nSubject: test1\n\ntest\n", m0.ID) // In-Reply-To. + m4 := deliver(now, "Message-ID: \nSubject: re: test1\n\ntest\n", m0.ID) // Subject. + m5 := deliver(now, "Message-ID: \nSubject: test1 (fwd)\n\ntest\n", m0.ID) // Subject. + m6 := deliver(now, "Message-ID: \nSubject: [fwd: test1]\n\ntest\n", m0.ID) // Subject. + m7 := deliver(now, "Message-ID: \nSubject: test1\n\ntest\n", 0) // Only subject, but not a response. + + // Thread with a cyclic head, a self-referencing message. + c1 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", 0) // Head cycle with m8. + c2 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", c1.ID) // Head cycle with c1. + c3 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", c1.ID) // Connected to one of the cycle elements. + c4 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", c1.ID) // Connected to other cycle element. + c5 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", c1.ID) + c5b := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", c1.ID) // Duplicate, e.g. Sent item, internal cycle during upgrade. + c6 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", c1.ID) + c7 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle0\n\ntest\n", c1.ID) // Self-referencing message that also points to actual parent. + + // More than 2 messages to make a cycle. + d0 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle1\n\ntest\n", 0) + d1 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle1\n\ntest\n", d0.ID) + d2 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle1\n\ntest\n", d0.ID) + + // Cycle with messages delivered later. During import/upgrade, they will all be one thread. + e0 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle2\n\ntest\n", 0) + e1 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle2\n\ntest\n", 0) + e2 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle2\n\ntest\n", e0.ID) + + // Three messages in a cycle (f1, f2, f3), with one with an additional ancestor (f4) which is ignored due to the cycle. Has different threads during import. + f0 := deliver(now, "Message-ID: \nSubject: cycle3\n\ntest\n", 0) + f1 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle3\n\ntest\n", f0.ID) + f2 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle3\n\ntest\n", 0) + f3 := deliver(now, "Message-ID: \nReferences: \nSubject: cycle3\n\ntest\n", f0.ID) + + // Duplicate single message (no larger thread). + g0 := deliver(now, "Message-ID: \nSubject: dup\n\ntest\n", 0) + g0b := deliver(now, "Message-ID: \nSubject: dup\n\ntest\n", g0.ID) + + // Duplicate message with a child message. + h0 := deliver(now, "Message-ID: \nSubject: dup2\n\ntest\n", 0) + h0b := deliver(now, "Message-ID: \nSubject: dup2\n\ntest\n", h0.ID) + h1 := deliver(now, "Message-ID: \nReferences: \nSubject: dup2\n\ntest\n", h0.ID) + + // Message has itself as reference. + s0 := deliver(now, "Message-ID: \nReferences: \nSubject: self-referencing message\n\ntest\n", 0) + + // Message with \0 in subject, should get an empty base subject. + b0 := deliver(now, "Message-ID: \nSubject: bad\u0000subject\n\ntest\n", 0) + b1 := deliver(now, "Message-ID: \nSubject: bad\u0000subject\n\ntest\n", 0) // Not matched. + + // Interleaved duplicate threaded messages. First child, then parent, then duplicate parent, then duplicat child again. + i0 := deliver(now, "Message-ID: \nReferences: \nSubject: interleaved duplicate\n\ntest\n", 0) + i1 := deliver(now, "Message-ID: \nSubject: interleaved duplicate\n\ntest\n", 0) + i2 := deliver(now, "Message-ID: \nSubject: interleaved duplicate\n\ntest\n", i1.ID) + i3 := deliver(now, "Message-ID: \nReferences: \nSubject: interleaved duplicate\n\ntest\n", i0.ID) + + j0 := deliver(now, "Message-ID: \nReferences: <>\nSubject: empty id in references\n\ntest\n", 0) + + dbpath := acc.DBPath + err = acc.Close() + tcheck(t, err, "close account") + + // Now clear the threading upgrade, and the threading fields and close the account. + // We open the database file directly, so we don't trigger the consistency checker. + db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) + err = db.Write(ctxbg, func(tx *bstore.Tx) error { + up := Upgrade{ID: 1} + err := tx.Delete(&up) + tcheck(t, err, "delete upgrade") + + q := bstore.QueryTx[Message](tx) + _, err = q.UpdateFields(map[string]any{ + "MessageID": "", + "SubjectBase": "", + "ThreadID": int64(0), + "ThreadParentIDs": []int64(nil), + "ThreadMissingLink": false, + }) + return err + }) + tcheck(t, err, "reset threading fields") + err = db.Close() + tcheck(t, err, "closing db") + + // Open the account again, that should get the account upgraded. Wait for upgrade to finish. + acc, err = OpenAccount("mjl") + tcheck(t, err, "open account") + err = acc.ThreadingWait(log) + tcheck(t, err, "wait for threading") + + check := func(id int64, expThreadID int64, expParentIDs []int64, expMissingLink bool) { + t.Helper() + + m := Message{ID: id} + err := acc.DB.Get(ctxbg, &m) + tcheck(t, err, "get message") + if m.ThreadID != expThreadID || !reflect.DeepEqual(m.ThreadParentIDs, expParentIDs) || m.ThreadMissingLink != expMissingLink { + t.Fatalf("got thread id %d, parent ids %v, missing link %v, expected %d %v %v", m.ThreadID, m.ThreadParentIDs, m.ThreadMissingLink, expThreadID, expParentIDs, expMissingLink) + } + } + + parents0 := []int64{m0.ID} + check(m0.ID, m0.ID, nil, false) + check(m1.ID, m0.ID, parents0, false) + check(m2.ID, m2.ID, nil, true) + check(m3.ID, m0.ID, parents0, false) + check(m4.ID, m0.ID, parents0, true) + check(m5.ID, m0.ID, parents0, true) + check(m6.ID, m0.ID, parents0, true) + check(m7.ID, m7.ID, nil, false) + + check(c1.ID, c1.ID, nil, true) // Head of cycle, hence missing link + check(c2.ID, c1.ID, []int64{c1.ID}, false) + check(c3.ID, c1.ID, []int64{c1.ID}, false) + check(c4.ID, c1.ID, []int64{c2.ID, c1.ID}, false) + check(c5.ID, c1.ID, []int64{c4.ID, c2.ID, c1.ID}, false) + check(c5b.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true) + check(c6.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, false) + check(c7.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true) + + check(d0.ID, d0.ID, nil, true) + check(d1.ID, d0.ID, []int64{d0.ID}, false) + check(d2.ID, d0.ID, []int64{d1.ID, d0.ID}, false) + + check(e0.ID, e0.ID, nil, true) + check(e1.ID, e0.ID, []int64{e2.ID, e0.ID}, false) + check(e2.ID, e0.ID, []int64{e0.ID}, false) + + check(f0.ID, f0.ID, nil, false) + check(f1.ID, f1.ID, nil, true) + check(f2.ID, f1.ID, []int64{f3.ID, f1.ID}, false) + check(f3.ID, f1.ID, []int64{f1.ID}, false) + + check(g0.ID, g0.ID, nil, false) + check(g0b.ID, g0.ID, []int64{g0.ID}, true) + + check(h0.ID, h0.ID, nil, false) + check(h0b.ID, h0.ID, []int64{h0.ID}, true) + check(h1.ID, h0.ID, []int64{h0.ID}, false) + + check(s0.ID, s0.ID, nil, true) + + check(b0.ID, b0.ID, nil, false) + check(b1.ID, b1.ID, nil, false) + + check(i0.ID, i1.ID, []int64{i1.ID}, false) + check(i1.ID, i1.ID, nil, false) + check(i2.ID, i1.ID, []int64{i1.ID}, true) + check(i3.ID, i1.ID, []int64{i0.ID, i1.ID}, true) + + check(j0.ID, j0.ID, nil, false) +} diff --git a/test-upgrade.sh b/test-upgrade.sh index c82138f..6b81bca 100755 --- a/test-upgrade.sh +++ b/test-upgrade.sh @@ -81,8 +81,8 @@ for tag in $tags; do fi done echo "Testing final upgrade to current." -time ../../mox verifydata -skip-size-check stepdata -time ../../mox openaccounts stepdata test0 test1 test2 +time ../../mox -cpuprof ../../upgrade-verifydata.cpu.pprof -memprof ../../upgrade-verifydata.mem.pprof verifydata -skip-size-check stepdata +time ../../mox -loglevel info -cpuprof ../../upgrade-openaccounts.cpu.pprof -memprof ../../upgrade-openaccounts.mem.pprof openaccounts stepdata test0 test1 test2 rm -r stepdata rm */mox cd ../.. diff --git a/verifydata.go b/verifydata.go index 3479646..f04583d 100644 --- a/verifydata.go +++ b/verifydata.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" + "golang.org/x/exp/slices" + bolt "go.etcd.io/bbolt" "github.com/mjl-/bstore" @@ -241,6 +243,13 @@ possibly making them potentially no longer readable by the previous version. checkf(err, dbpath, "missing nextuidvalidity") } + up := store.Upgrade{ID: 1} + if err := db.Get(ctxbg, &up); err != nil { + log.Printf("warning: getting upgrade record (continuing, but not checking message threading): %v", err) + } else if up.Threads != 2 { + log.Printf("warning: no message threading in database, skipping checks for threading consistency") + } + mailboxes := map[int64]store.Mailbox{} err := bstore.QueryDB[store.Mailbox](ctxbg, db).ForEach(func(mb store.Mailbox) error { mailboxes[mb.ID] = mb @@ -270,10 +279,37 @@ possibly making them potentially no longer readable by the previous version. if m.Expunged { return nil } + mp := store.MessagePath(m.ID) seen[mp] = struct{}{} p := filepath.Join(accdir, "msg", mp) checkFile(dbpath, p, len(m.MsgPrefix), m.Size) + + if up.Threads != 2 { + return nil + } + + if m.ThreadID <= 0 { + checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has bad threadid", m.ID, m.ThreadID, mb.Name, mb.ID) + } + if len(m.ThreadParentIDs) == 0 { + return nil + } + if slices.Contains(m.ThreadParentIDs, m.ID) { + checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has itself as thread parent", m.ID, m.ThreadID, mb.Name, mb.ID) + } + for i, pid := range m.ThreadParentIDs { + am := store.Message{ID: pid} + if err := db.Get(ctxbg, &am); err == bstore.ErrAbsent { + continue + } else if err != nil { + return fmt.Errorf("get ancestor message: %v", err) + } else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) { + checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs) + } else { + break + } + } return nil }) checkf(err, dbpath, "reading messages in account database to check files") diff --git a/webaccount/account.html b/webaccount/account.html index 5260b1e..df7d9fe 100644 --- a/webaccount/account.html +++ b/webaccount/account.html @@ -235,6 +235,11 @@ const index = async () => { } problems.appendChild(dom.div(box(yellow, data.Message))) }) + eventSource.addEventListener('step', (e) => { + const data = JSON.parse(e.data) // {Title: ...} + console.log('import step event', {e, data}) + importProgress.appendChild(dom.div(dom.br(), box(blue, 'Step: '+data.Title))) + }) eventSource.addEventListener('done', (e) => { console.log('import done event', {e}) importProgress.appendChild(dom.div(dom.br(), box(blue, 'Import finished'))) @@ -475,7 +480,7 @@ const index = async () => { ), dom.div( dom.button('Upload and import'), - dom.p(style({fontStyle: 'italic', marginTop: '.5ex'}), 'The file is uploaded first, then its messages are imported. Importing is done in a transaction, you can abort the entire import before it is finished.'), + dom.p(style({fontStyle: 'italic', marginTop: '.5ex'}), 'The file is uploaded first, then its messages are imported, finally messages are matched for threading. Importing is done in a transaction, you can abort the entire import before it is finished.'), ), ), ), diff --git a/webaccount/account_test.go b/webaccount/account_test.go index b5f0b6a..d7fdb0d 100644 --- a/webaccount/account_test.go +++ b/webaccount/account_test.go @@ -131,6 +131,7 @@ func TestAccount(t *testing.T) { count += x.Count case importProblem: t.Fatalf("unexpected problem: %q", x.Message) + case importStep: case importDone: break loop case importAborted: diff --git a/webaccount/import.go b/webaccount/import.go index 800a690..b7489a4 100644 --- a/webaccount/import.go +++ b/webaccount/import.go @@ -193,6 +193,9 @@ type importProblem struct { } type importDone struct{} type importAborted struct{} +type importStep struct { + Title string +} // importStart prepare the import and launches the goroutine to actually import. // importStart is responsible for closing f. @@ -284,12 +287,6 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store // ID's of delivered messages. If we have to rollback, we have to remove this files. var deliveredIDs []int64 - ximportcheckf := func(err error, format string, args ...any) { - if err != nil { - panic(importError{fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), err)}) - } - } - sendEvent := func(kind string, v any) { buf, err := json.Marshal(v) if err != nil { @@ -300,11 +297,6 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store importers.Events <- importEvent{token, []byte(ssemsg), v, nil} } - problemf := func(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - sendEvent("problem", importProblem{Message: msg}) - } - canceled := func() bool { select { case <-ctx.Done(): @@ -315,6 +307,11 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store } } + problemf := func(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + sendEvent("problem", importProblem{Message: msg}) + } + defer func() { err := f.Close() log.Check(err, "closing uploaded messages file") @@ -349,6 +346,15 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store } }() + ximportcheckf := func(err error, format string, args ...any) { + if err != nil { + panic(importError{fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), err)}) + } + } + + err := acc.ThreadingWait(log) + ximportcheckf(err, "waiting for account thread upgrade") + conf, _ := acc.Conf() jf, _, err := acc.OpenJunkFilter(ctx, log) @@ -515,6 +521,11 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store m.ParsedBuf, err = json.Marshal(p) ximportcheckf(err, "marshal parsed message structure") + // Set fields needed for future threading. By doing it now, DeliverMessage won't + // have to parse the Part again. + p.SetReaderAt(store.FileMsgReader(m.MsgPrefix, f)) + m.PrepareThreading(log, &p) + if m.Received.IsZero() { if p.Envelope != nil && !p.Envelope.Date.IsZero() { m.Received = p.Envelope.Date @@ -534,7 +545,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store const consumeFile = true const sync = false const notrain = true - if err := acc.DeliverMessage(log, tx, m, f, consumeFile, sync, notrain); err != nil { + const nothreads = true + if err := acc.DeliverMessage(log, tx, m, f, consumeFile, sync, notrain, nothreads); err != nil { problemf("delivering message %s: %s (continuing)", pos, err) return } @@ -797,13 +809,20 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store for _, count := range messages { total += count } - log.Debug("message imported", mlog.Field("total", total)) + log.Debug("messages imported", mlog.Field("total", total)) // Send final update for count of last-imported mailbox. if prevMailbox != "" { sendEvent("count", importCount{prevMailbox, messages[prevMailbox]}) } + // Match threads. + if len(deliveredIDs) > 0 { + sendEvent("step", importStep{"matching messages with threads"}) + err = acc.AssignThreads(ctx, log, tx, deliveredIDs[0], 0, io.Discard) + ximportcheckf(err, "assigning messages to threads") + } + // Update mailboxes with counts and keywords. for mbID, mc := range destMailboxCounts { mb := store.Mailbox{ID: mbID} diff --git a/webmail/api.go b/webmail/api.go index 9e450b3..b6f37fa 100644 --- a/webmail/api.go +++ b/webmail/api.go @@ -748,7 +748,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { err = tx.Update(&sentmb) xcheckf(ctx, err, "updating sent mailbox for counts") - err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, true, false) + err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, true, false, false) if err != nil { metricSubmission.WithLabelValues("storesenterror").Inc() metricked = true @@ -1488,7 +1488,140 @@ func (Webmail) MailboxSetSpecialUse(ctx context.Context, mb store.Mailbox) { }) } +// ThreadCollapse saves the ThreadCollapse field for the messages and its +// children. The messageIDs are typically thread roots. But not all roots +// (without parent) of a thread need to have the same collapsed state. +func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse bool) { + log := xlog.WithContext(ctx) + reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) + acc, err := store.OpenAccount(reqInfo.AccountName) + xcheckf(ctx, err, "open account") + defer func() { + err := acc.Close() + log.Check(err, "closing account") + }() + + if len(messageIDs) == 0 { + xcheckuserf(ctx, errors.New("no messages"), "setting collapse") + } + + acc.WithWLock(func() { + changes := make([]store.Change, 0, len(messageIDs)) + xdbwrite(ctx, acc, func(tx *bstore.Tx) { + // Gather ThreadIDs to list all potential messages, for a way to get all potential + // (child) messages. Further refined in FilterFn. + threadIDs := map[int64]struct{}{} + msgIDs := map[int64]struct{}{} + for _, id := range messageIDs { + m := store.Message{ID: id} + err := tx.Get(&m) + if err == bstore.ErrAbsent { + xcheckuserf(ctx, err, "get message") + } + xcheckf(ctx, err, "get message") + threadIDs[m.ThreadID] = struct{}{} + msgIDs[id] = struct{}{} + } + + var updated []store.Message + q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...) + q.FilterNotEqual("ThreadCollapsed", collapse) + q.FilterFn(func(tm store.Message) bool { + for _, id := range tm.ThreadParentIDs { + if _, ok := msgIDs[id]; ok { + return true + } + } + _, ok := msgIDs[tm.ID] + return ok + }) + q.Gather(&updated) + q.SortAsc("ID") // Consistent order for testing. + _, err = q.UpdateFields(map[string]any{"ThreadCollapsed": collapse}) + xcheckf(ctx, err, "updating collapse in database") + + for _, m := range updated { + changes = append(changes, m.ChangeThread()) + } + }) + store.BroadcastChanges(acc, changes) + }) +} + +// ThreadMute saves the ThreadMute field for the messages and their children. +// If messages are muted, they are also marked collapsed. +func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) { + log := xlog.WithContext(ctx) + reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) + acc, err := store.OpenAccount(reqInfo.AccountName) + xcheckf(ctx, err, "open account") + defer func() { + err := acc.Close() + log.Check(err, "closing account") + }() + + if len(messageIDs) == 0 { + xcheckuserf(ctx, errors.New("no messages"), "setting mute") + } + + acc.WithWLock(func() { + changes := make([]store.Change, 0, len(messageIDs)) + xdbwrite(ctx, acc, func(tx *bstore.Tx) { + threadIDs := map[int64]struct{}{} + msgIDs := map[int64]struct{}{} + for _, id := range messageIDs { + m := store.Message{ID: id} + err := tx.Get(&m) + if err == bstore.ErrAbsent { + xcheckuserf(ctx, err, "get message") + } + xcheckf(ctx, err, "get message") + threadIDs[m.ThreadID] = struct{}{} + msgIDs[id] = struct{}{} + } + + var updated []store.Message + + q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...) + q.FilterFn(func(tm store.Message) bool { + if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) { + return false + } + for _, id := range tm.ThreadParentIDs { + if _, ok := msgIDs[id]; ok { + return true + } + } + _, ok := msgIDs[tm.ID] + return ok + }) + q.Gather(&updated) + fields := map[string]any{"ThreadMuted": mute} + if mute { + fields["ThreadCollapsed"] = true + } + _, err = q.UpdateFields(fields) + xcheckf(ctx, err, "updating mute in database") + + for _, m := range updated { + changes = append(changes, m.ChangeThread()) + } + }) + store.BroadcastChanges(acc, changes) + }) +} + +func slicesAny[T any](l []T) []any { + r := make([]any, len(l)) + for i, v := range l { + r[i] = v + } + return r +} + // SSETypes exists to ensure the generated API contains the types, for use in SSE events. -func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) { +func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, msgThread ChangeMsgThread, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) { return } diff --git a/webmail/api.json b/webmail/api.json index a3ff4cf..abeac44 100644 --- a/webmail/api.json +++ b/webmail/api.json @@ -235,6 +235,46 @@ ], "Returns": [] }, + { + "Name": "ThreadCollapse", + "Docs": "ThreadCollapse saves the ThreadCollapse field for the messages and its\nchildren. The messageIDs are typically thread roots. But not all roots\n(without parent) of a thread need to have the same collapsed state.", + "Params": [ + { + "Name": "messageIDs", + "Typewords": [ + "[]", + "int64" + ] + }, + { + "Name": "collapse", + "Typewords": [ + "bool" + ] + } + ], + "Returns": [] + }, + { + "Name": "ThreadMute", + "Docs": "ThreadMute saves the ThreadMute field for the messages and their children.\nIf messages are muted, they are also marked collapsed.", + "Params": [ + { + "Name": "messageIDs", + "Typewords": [ + "[]", + "int64" + ] + }, + { + "Name": "mute", + "Typewords": [ + "bool" + ] + } + ], + "Returns": [] + }, { "Name": "SSETypes", "Docs": "SSETypes exists to ensure the generated API contains the types, for use in SSE events.", @@ -288,6 +328,12 @@ "ChangeMsgFlags" ] }, + { + "Name": "msgThread", + "Typewords": [ + "ChangeMsgThread" + ] + }, { "Name": "mailboxRemove", "Typewords": [ @@ -394,6 +440,13 @@ "bool" ] }, + { + "Name": "Threading", + "Docs": "", + "Typewords": [ + "ThreadMode" + ] + }, { "Name": "Filter", "Docs": "", @@ -783,7 +836,7 @@ }, { "Name": "Subject", - "Docs": "", + "Docs": "Q/B-word-decoded.", "Typewords": [ "string" ] @@ -1319,8 +1372,9 @@ }, { "Name": "MessageItems", - "Docs": "If empty, this was the last message for the request.", + "Docs": "If empty, this was the last message for the request. If non-empty, a list of thread messages. Each with the first message being the reason this thread is included and can be used as AnchorID in followup requests. If the threading mode is \"off\" in the query, there will always be only a single message. If a thread is sent, all messages in the thread are sent, including those that don't match the query (e.g. from another mailbox). Threads can be displayed based on the ThreadParentIDs field, with possibly slightly different display based on field ThreadMissingLink.", "Typewords": [ + "[]", "[]", "MessageItem" ] @@ -1388,6 +1442,13 @@ "Typewords": [ "string" ] + }, + { + "Name": "MatchQuery", + "Docs": "If message does not match query, it can still be included because of threading.", + "Typewords": [ + "bool" + ] } ] }, @@ -1630,19 +1691,62 @@ }, { "Name": "MessageID", - "Docs": "Value of Message-Id header. Only set for messages that were delivered to the rejects mailbox. For ensuring such messages are delivered only once. Value includes \u003c\u003e.", + "Docs": "Canonicalized Message-Id, always lower-case and normalized quoting, without \u003c\u003e's. Empty if missing. Used for matching message threads, and to prevent duplicate reject delivery.", + "Typewords": [ + "string" + ] + }, + { + "Name": "SubjectBase", + "Docs": "For matching threads in case there is no References/In-Reply-To header. It is lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.", "Typewords": [ "string" ] }, { "Name": "MessageHash", - "Docs": "Hash of message. For rejects delivery, so optional like MessageID.", + "Docs": "Hash of message. For rejects delivery in case there is no Message-ID, only set when delivered as reject.", "Typewords": [ "[]", "uint8" ] }, + { + "Name": "ThreadID", + "Docs": "ID of message starting this thread.", + "Typewords": [ + "int64" + ] + }, + { + "Name": "ThreadParentIDs", + "Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.", + "Typewords": [ + "[]", + "int64" + ] + }, + { + "Name": "ThreadMissingLink", + "Docs": "ThreadMissingLink is true if there is no match with a direct parent. E.g. first ID in ThreadParentIDs is not the direct ancestor (an intermediate message may have been deleted), or subject-based matching was done.", + "Typewords": [ + "bool" + ] + }, + { + "Name": "ThreadMuted", + "Docs": "If set, newly delivered child messages are automatically marked as read. This field is copied to new child messages. Changes are propagated to the webmail client.", + "Typewords": [ + "bool" + ] + }, + { + "Name": "ThreadCollapsed", + "Docs": "If set, this (sub)thread is collapsed in the webmail client, for threading mode \"on\" (mode \"unread\" ignores it). This field is copied to new child message. Changes are propagated to the webmail client.", + "Typewords": [ + "bool" + ] + }, { "Name": "Seen", "Docs": "", @@ -1888,7 +1992,7 @@ }, { "Name": "ChangeMsgAdd", - "Docs": "ChangeMsgAdd adds a new message to the view.", + "Docs": "ChangeMsgAdd adds a new message and possibly its thread to the view.", "Fields": [ { "Name": "MailboxID", @@ -1927,9 +2031,10 @@ ] }, { - "Name": "MessageItem", + "Name": "MessageItems", "Docs": "", "Typewords": [ + "[]", "MessageItem" ] } @@ -2088,6 +2193,34 @@ } ] }, + { + "Name": "ChangeMsgThread", + "Docs": "ChangeMsgThread updates muted/collapsed fields for one message.", + "Fields": [ + { + "Name": "MessageIDs", + "Docs": "", + "Typewords": [ + "[]", + "int64" + ] + }, + { + "Name": "Muted", + "Docs": "", + "Typewords": [ + "bool" + ] + }, + { + "Name": "Collapsed", + "Docs": "", + "Typewords": [ + "bool" + ] + } + ] + }, { "Name": "ChangeMailboxRemove", "Docs": "ChangeMailboxRemove indicates a mailbox was removed, including all its messages.", @@ -2382,6 +2515,27 @@ } ], "Strings": [ + { + "Name": "ThreadMode", + "Docs": "", + "Values": [ + { + "Name": "ThreadOff", + "Value": "off", + "Docs": "" + }, + { + "Name": "ThreadOn", + "Value": "on", + "Docs": "" + }, + { + "Name": "ThreadUnread", + "Value": "unread", + "Docs": "" + } + ] + }, { "Name": "AttachmentType", "Docs": "AttachmentType is for filtering by attachment type.", diff --git a/webmail/api.ts b/webmail/api.ts index 7e12e6a..c6a50ae 100644 --- a/webmail/api.ts +++ b/webmail/api.ts @@ -16,6 +16,7 @@ export interface Request { // Query is a request for messages that match filters, in a given order. export interface Query { OrderAsc: boolean // Order by received ascending or desending. + Threading: ThreadMode Filter: Filter NotFilter: NotFilter } @@ -91,7 +92,7 @@ export interface Part { // Envelope holds the basic/common message headers as used in IMAP4. export interface Envelope { Date: Date - Subject: string + Subject: string // Q/B-word-decoded. From?: Address[] | null Sender?: Address[] | null ReplyTo?: Address[] | null @@ -218,7 +219,7 @@ export interface EventViewReset { export interface EventViewMsgs { ViewID: number RequestID: number - MessageItems?: MessageItem[] | null // If empty, this was the last message for the request. + MessageItems?: (MessageItem[] | null)[] | null // If empty, this was the last message for the request. If non-empty, a list of thread messages. Each with the first message being the reason this thread is included and can be used as AnchorID in followup requests. If the threading mode is "off" in the query, there will always be only a single message. If a thread is sent, all messages in the thread are sent, including those that don't match the query (e.g. from another mailbox). Threads can be displayed based on the ThreadParentIDs field, with possibly slightly different display based on field ThreadMissingLink. ParsedMessage?: ParsedMessage | null // If set, will match the target page.DestMessageID from the request. ViewEnd: boolean // If set, there are no more messages in this view at this moment. Messages can be added, typically via Change messages, e.g. for new deliveries. } @@ -233,6 +234,7 @@ export interface MessageItem { IsSigned: boolean IsEncrypted: boolean FirstLine: string // Of message body, for showing as preview. + MatchQuery: boolean // If message does not match query, it can still be included because of threading. } // Message stored in database and per-message file on disk. @@ -276,8 +278,14 @@ export interface Message { DKIMDomains?: string[] | null // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a DKIM domain that matched a ruleset's verified domain is left out, but included in OrigDKIMDomains. OrigEHLODomain: string // For forwarded messages, OrigDKIMDomains?: string[] | null - MessageID: string // Value of Message-Id header. Only set for messages that were delivered to the rejects mailbox. For ensuring such messages are delivered only once. Value includes <>. - MessageHash?: string | null // Hash of message. For rejects delivery, so optional like MessageID. + MessageID: string // Canonicalized Message-Id, always lower-case and normalized quoting, without <>'s. Empty if missing. Used for matching message threads, and to prevent duplicate reject delivery. + SubjectBase: string // For matching threads in case there is no References/In-Reply-To header. It is lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed. + MessageHash?: string | null // Hash of message. For rejects delivery in case there is no Message-ID, only set when delivered as reject. + ThreadID: number // ID of message starting this thread. + ThreadParentIDs?: number[] | null // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors. + ThreadMissingLink: boolean // ThreadMissingLink is true if there is no match with a direct parent. E.g. first ID in ThreadParentIDs is not the direct ancestor (an intermediate message may have been deleted), or subject-based matching was done. + ThreadMuted: boolean // If set, newly delivered child messages are automatically marked as read. This field is copied to new child messages. Changes are propagated to the webmail client. + ThreadCollapsed: boolean // If set, this (sub)thread is collapsed in the webmail client, for threading mode "on" (mode "unread" ignores it). This field is copied to new child message. Changes are propagated to the webmail client. Seen: boolean Answered: boolean Flagged: boolean @@ -326,14 +334,14 @@ export interface EventViewChanges { Changes?: (any[] | null)[] | null // The first field of [2]any is a string, the second of the Change types below. } -// ChangeMsgAdd adds a new message to the view. +// ChangeMsgAdd adds a new message and possibly its thread to the view. export interface ChangeMsgAdd { MailboxID: number UID: UID ModSeq: ModSeq Flags: Flags // System flags. Keywords?: string[] | null // Other flags. - MessageItem: MessageItem + MessageItems?: MessageItem[] | null } // Flags for a mail message. @@ -367,6 +375,13 @@ export interface ChangeMsgFlags { Keywords?: string[] | null // Non-system/well-known flags/keywords/labels. } +// ChangeMsgThread updates muted/collapsed fields for one message. +export interface ChangeMsgThread { + MessageIDs?: number[] | null + Muted: boolean + Collapsed: boolean +} + // ChangeMailboxRemove indicates a mailbox was removed, including all its messages. export interface ChangeMailboxRemove { MailboxID: number @@ -446,6 +461,12 @@ export enum Validation { ValidationNone = 10, // E.g. No records. } +export enum ThreadMode { + ThreadOff = "off", + ThreadOn = "on", + ThreadUnread = "unread", +} + // AttachmentType is for filtering by attachment type. export enum AttachmentType { AttachmentIndifferent = "", @@ -464,12 +485,12 @@ export enum AttachmentType { // An empty string can be a valid localpart. export type Localpart = string -export const structTypes: {[typename: string]: boolean} = {"Address":true,"Attachment":true,"ChangeMailboxAdd":true,"ChangeMailboxCounts":true,"ChangeMailboxKeywords":true,"ChangeMailboxRemove":true,"ChangeMailboxRename":true,"ChangeMailboxSpecialUse":true,"ChangeMsgAdd":true,"ChangeMsgFlags":true,"ChangeMsgRemove":true,"Domain":true,"DomainAddressConfig":true,"Envelope":true,"EventStart":true,"EventViewChanges":true,"EventViewErr":true,"EventViewMsgs":true,"EventViewReset":true,"File":true,"Filter":true,"Flags":true,"ForwardAttachments":true,"Mailbox":true,"Message":true,"MessageAddress":true,"MessageEnvelope":true,"MessageItem":true,"NotFilter":true,"Page":true,"ParsedMessage":true,"Part":true,"Query":true,"Request":true,"SpecialUse":true,"SubmitMessage":true} -export const stringsTypes: {[typename: string]: boolean} = {"AttachmentType":true,"Localpart":true} +export const structTypes: {[typename: string]: boolean} = {"Address":true,"Attachment":true,"ChangeMailboxAdd":true,"ChangeMailboxCounts":true,"ChangeMailboxKeywords":true,"ChangeMailboxRemove":true,"ChangeMailboxRename":true,"ChangeMailboxSpecialUse":true,"ChangeMsgAdd":true,"ChangeMsgFlags":true,"ChangeMsgRemove":true,"ChangeMsgThread":true,"Domain":true,"DomainAddressConfig":true,"Envelope":true,"EventStart":true,"EventViewChanges":true,"EventViewErr":true,"EventViewMsgs":true,"EventViewReset":true,"File":true,"Filter":true,"Flags":true,"ForwardAttachments":true,"Mailbox":true,"Message":true,"MessageAddress":true,"MessageEnvelope":true,"MessageItem":true,"NotFilter":true,"Page":true,"ParsedMessage":true,"Part":true,"Query":true,"Request":true,"SpecialUse":true,"SubmitMessage":true} +export const stringsTypes: {[typename: string]: boolean} = {"AttachmentType":true,"Localpart":true,"ThreadMode":true} export const intsTypes: {[typename: string]: boolean} = {"ModSeq":true,"UID":true,"Validation":true} export const types: TypenameMap = { "Request": {"Name":"Request","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"SSEID","Docs":"","Typewords":["int64"]},{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Cancel","Docs":"","Typewords":["bool"]},{"Name":"Query","Docs":"","Typewords":["Query"]},{"Name":"Page","Docs":"","Typewords":["Page"]}]}, - "Query": {"Name":"Query","Docs":"","Fields":[{"Name":"OrderAsc","Docs":"","Typewords":["bool"]},{"Name":"Filter","Docs":"","Typewords":["Filter"]},{"Name":"NotFilter","Docs":"","Typewords":["NotFilter"]}]}, + "Query": {"Name":"Query","Docs":"","Fields":[{"Name":"OrderAsc","Docs":"","Typewords":["bool"]},{"Name":"Threading","Docs":"","Typewords":["ThreadMode"]},{"Name":"Filter","Docs":"","Typewords":["Filter"]},{"Name":"NotFilter","Docs":"","Typewords":["NotFilter"]}]}, "Filter": {"Name":"Filter","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxChildrenIncluded","Docs":"","Typewords":["bool"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"Words","Docs":"","Typewords":["[]","string"]},{"Name":"From","Docs":"","Typewords":["[]","string"]},{"Name":"To","Docs":"","Typewords":["[]","string"]},{"Name":"Oldest","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"Newest","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"Subject","Docs":"","Typewords":["[]","string"]},{"Name":"Attachments","Docs":"","Typewords":["AttachmentType"]},{"Name":"Labels","Docs":"","Typewords":["[]","string"]},{"Name":"Headers","Docs":"","Typewords":["[]","[]","string"]},{"Name":"SizeMin","Docs":"","Typewords":["int64"]},{"Name":"SizeMax","Docs":"","Typewords":["int64"]}]}, "NotFilter": {"Name":"NotFilter","Docs":"","Fields":[{"Name":"Words","Docs":"","Typewords":["[]","string"]},{"Name":"From","Docs":"","Typewords":["[]","string"]},{"Name":"To","Docs":"","Typewords":["[]","string"]},{"Name":"Subject","Docs":"","Typewords":["[]","string"]},{"Name":"Attachments","Docs":"","Typewords":["AttachmentType"]},{"Name":"Labels","Docs":"","Typewords":["[]","string"]}]}, "Page": {"Name":"Page","Docs":"","Fields":[{"Name":"AnchorMessageID","Docs":"","Typewords":["int64"]},{"Name":"Count","Docs":"","Typewords":["int32"]},{"Name":"DestMessageID","Docs":"","Typewords":["int64"]}]}, @@ -487,16 +508,17 @@ export const types: TypenameMap = { "DomainAddressConfig": {"Name":"DomainAddressConfig","Docs":"","Fields":[{"Name":"LocalpartCatchallSeparator","Docs":"","Typewords":["string"]},{"Name":"LocalpartCaseSensitive","Docs":"","Typewords":["bool"]}]}, "EventViewErr": {"Name":"EventViewErr","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"Err","Docs":"","Typewords":["string"]}]}, "EventViewReset": {"Name":"EventViewReset","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]}]}, - "EventViewMsgs": {"Name":"EventViewMsgs","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]},{"Name":"ParsedMessage","Docs":"","Typewords":["nullable","ParsedMessage"]},{"Name":"ViewEnd","Docs":"","Typewords":["bool"]}]}, - "MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"FirstLine","Docs":"","Typewords":["string"]}]}, - "Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]}, + "EventViewMsgs": {"Name":"EventViewMsgs","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","[]","MessageItem"]},{"Name":"ParsedMessage","Docs":"","Typewords":["nullable","ParsedMessage"]},{"Name":"ViewEnd","Docs":"","Typewords":["bool"]}]}, + "MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"FirstLine","Docs":"","Typewords":["string"]},{"Name":"MatchQuery","Docs":"","Typewords":["bool"]}]}, + "Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"SubjectBase","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"ThreadID","Docs":"","Typewords":["int64"]},{"Name":"ThreadParentIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"ThreadMissingLink","Docs":"","Typewords":["bool"]},{"Name":"ThreadMuted","Docs":"","Typewords":["bool"]},{"Name":"ThreadCollapsed","Docs":"","Typewords":["bool"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]}, "MessageEnvelope": {"Name":"MessageEnvelope","Docs":"","Fields":[{"Name":"Date","Docs":"","Typewords":["timestamp"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"From","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"Sender","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"ReplyTo","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"To","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"CC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"BCC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"InReplyTo","Docs":"","Typewords":["string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]}]}, "Attachment": {"Name":"Attachment","Docs":"","Fields":[{"Name":"Path","Docs":"","Typewords":["[]","int32"]},{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"Part","Docs":"","Typewords":["Part"]}]}, "EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]}, - "ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItem","Docs":"","Typewords":["MessageItem"]}]}, + "ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]}]}, "Flags": {"Name":"Flags","Docs":"","Fields":[{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]}]}, "ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]}, "ChangeMsgFlags": {"Name":"ChangeMsgFlags","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Mask","Docs":"","Typewords":["Flags"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]}, + "ChangeMsgThread": {"Name":"ChangeMsgThread","Docs":"","Fields":[{"Name":"MessageIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"Muted","Docs":"","Typewords":["bool"]},{"Name":"Collapsed","Docs":"","Typewords":["bool"]}]}, "ChangeMailboxRemove": {"Name":"ChangeMailboxRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]}]}, "ChangeMailboxAdd": {"Name":"ChangeMailboxAdd","Docs":"","Fields":[{"Name":"Mailbox","Docs":"","Typewords":["Mailbox"]}]}, "ChangeMailboxRename": {"Name":"ChangeMailboxRename","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"OldName","Docs":"","Typewords":["string"]},{"Name":"NewName","Docs":"","Typewords":["string"]},{"Name":"Flags","Docs":"","Typewords":["[]","string"]}]}, @@ -507,6 +529,7 @@ export const types: TypenameMap = { "UID": {"Name":"UID","Docs":"","Values":null}, "ModSeq": {"Name":"ModSeq","Docs":"","Values":null}, "Validation": {"Name":"Validation","Docs":"","Values":[{"Name":"ValidationUnknown","Value":0,"Docs":""},{"Name":"ValidationStrict","Value":1,"Docs":""},{"Name":"ValidationDMARC","Value":2,"Docs":""},{"Name":"ValidationRelaxed","Value":3,"Docs":""},{"Name":"ValidationPass","Value":4,"Docs":""},{"Name":"ValidationNeutral","Value":5,"Docs":""},{"Name":"ValidationTemperror","Value":6,"Docs":""},{"Name":"ValidationPermerror","Value":7,"Docs":""},{"Name":"ValidationFail","Value":8,"Docs":""},{"Name":"ValidationSoftfail","Value":9,"Docs":""},{"Name":"ValidationNone","Value":10,"Docs":""}]}, + "ThreadMode": {"Name":"ThreadMode","Docs":"","Values":[{"Name":"ThreadOff","Value":"off","Docs":""},{"Name":"ThreadOn","Value":"on","Docs":""},{"Name":"ThreadUnread","Value":"unread","Docs":""}]}, "AttachmentType": {"Name":"AttachmentType","Docs":"","Values":[{"Name":"AttachmentIndifferent","Value":"","Docs":""},{"Name":"AttachmentNone","Value":"none","Docs":""},{"Name":"AttachmentAny","Value":"any","Docs":""},{"Name":"AttachmentImage","Value":"image","Docs":""},{"Name":"AttachmentPDF","Value":"pdf","Docs":""},{"Name":"AttachmentArchive","Value":"archive","Docs":""},{"Name":"AttachmentSpreadsheet","Value":"spreadsheet","Docs":""},{"Name":"AttachmentDocument","Value":"document","Docs":""},{"Name":"AttachmentPresentation","Value":"presentation","Docs":""}]}, "Localpart": {"Name":"Localpart","Docs":"","Values":null}, } @@ -541,6 +564,7 @@ export const parser = { Flags: (v: any) => parse("Flags", v) as Flags, ChangeMsgRemove: (v: any) => parse("ChangeMsgRemove", v) as ChangeMsgRemove, ChangeMsgFlags: (v: any) => parse("ChangeMsgFlags", v) as ChangeMsgFlags, + ChangeMsgThread: (v: any) => parse("ChangeMsgThread", v) as ChangeMsgThread, ChangeMailboxRemove: (v: any) => parse("ChangeMailboxRemove", v) as ChangeMailboxRemove, ChangeMailboxAdd: (v: any) => parse("ChangeMailboxAdd", v) as ChangeMailboxAdd, ChangeMailboxRename: (v: any) => parse("ChangeMailboxRename", v) as ChangeMailboxRename, @@ -551,6 +575,7 @@ export const parser = { UID: (v: any) => parse("UID", v) as UID, ModSeq: (v: any) => parse("ModSeq", v) as ModSeq, Validation: (v: any) => parse("Validation", v) as Validation, + ThreadMode: (v: any) => parse("ThreadMode", v) as ThreadMode, AttachmentType: (v: any) => parse("AttachmentType", v) as AttachmentType, Localpart: (v: any) => parse("Localpart", v) as Localpart, } @@ -712,13 +737,34 @@ export class Client { return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as void } + // ThreadCollapse saves the ThreadCollapse field for the messages and its + // children. The messageIDs are typically thread roots. But not all roots + // (without parent) of a thread need to have the same collapsed state. + async ThreadCollapse(messageIDs: number[] | null, collapse: boolean): Promise { + const fn: string = "ThreadCollapse" + const paramTypes: string[][] = [["[]","int64"],["bool"]] + const returnTypes: string[][] = [] + const params: any[] = [messageIDs, collapse] + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as void + } + + // ThreadMute saves the ThreadMute field for the messages and their children. + // If messages are muted, they are also marked collapsed. + async ThreadMute(messageIDs: number[] | null, mute: boolean): Promise { + const fn: string = "ThreadMute" + const paramTypes: string[][] = [["[]","int64"],["bool"]] + const returnTypes: string[][] = [] + const params: any[] = [messageIDs, mute] + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as void + } + // SSETypes exists to ensure the generated API contains the types, for use in SSE events. - async SSETypes(): Promise<[EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags]> { + async SSETypes(): Promise<[EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMsgThread, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags]> { const fn: string = "SSETypes" const paramTypes: string[][] = [] - const returnTypes: string[][] = [["EventStart"],["EventViewErr"],["EventViewReset"],["EventViewMsgs"],["EventViewChanges"],["ChangeMsgAdd"],["ChangeMsgRemove"],["ChangeMsgFlags"],["ChangeMailboxRemove"],["ChangeMailboxAdd"],["ChangeMailboxRename"],["ChangeMailboxCounts"],["ChangeMailboxSpecialUse"],["ChangeMailboxKeywords"],["Flags"]] + const returnTypes: string[][] = [["EventStart"],["EventViewErr"],["EventViewReset"],["EventViewMsgs"],["EventViewChanges"],["ChangeMsgAdd"],["ChangeMsgRemove"],["ChangeMsgFlags"],["ChangeMsgThread"],["ChangeMailboxRemove"],["ChangeMailboxAdd"],["ChangeMailboxRename"],["ChangeMailboxCounts"],["ChangeMailboxSpecialUse"],["ChangeMailboxKeywords"],["Flags"]] const params: any[] = [] - return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as [EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags] + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as [EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMsgThread, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags] } } diff --git a/webmail/message.go b/webmail/message.go index f9fb77e..5ff23cb 100644 --- a/webmail/message.go +++ b/webmail/message.go @@ -26,7 +26,7 @@ func messageItem(log *mlog.Log, m store.Message, state *msgState) (MessageItem, // Clear largish unused data. m.MsgPrefix = nil m.ParsedBuf = nil - return MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine}, nil + return MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine, true}, nil } // formatFirstLine returns a line the client can display next to the subject line diff --git a/webmail/msg.js b/webmail/msg.js index d072364..8767f1c 100644 --- a/webmail/msg.js +++ b/webmail/msg.js @@ -17,6 +17,12 @@ var api; Validation[Validation["ValidationSoftfail"] = 9] = "ValidationSoftfail"; Validation[Validation["ValidationNone"] = 10] = "ValidationNone"; })(Validation = api.Validation || (api.Validation = {})); + let ThreadMode; + (function (ThreadMode) { + ThreadMode["ThreadOff"] = "off"; + ThreadMode["ThreadOn"] = "on"; + ThreadMode["ThreadUnread"] = "unread"; + })(ThreadMode = api.ThreadMode || (api.ThreadMode = {})); // AttachmentType is for filtering by attachment type. let AttachmentType; (function (AttachmentType) { @@ -30,12 +36,12 @@ var api; AttachmentType["AttachmentDocument"] = "document"; AttachmentType["AttachmentPresentation"] = "presentation"; })(AttachmentType = api.AttachmentType || (api.AttachmentType = {})); - api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true }; - api.stringsTypes = { "AttachmentType": true, "Localpart": true }; + api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true }; + api.stringsTypes = { "AttachmentType": true, "Localpart": true, "ThreadMode": true }; api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true }; api.types = { "Request": { "Name": "Request", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Cancel", "Docs": "", "Typewords": ["bool"] }, { "Name": "Query", "Docs": "", "Typewords": ["Query"] }, { "Name": "Page", "Docs": "", "Typewords": ["Page"] }] }, - "Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] }, + "Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Threading", "Docs": "", "Typewords": ["ThreadMode"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] }, "Filter": { "Name": "Filter", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxChildrenIncluded", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Oldest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Newest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Headers", "Docs": "", "Typewords": ["[]", "[]", "string"] }, { "Name": "SizeMin", "Docs": "", "Typewords": ["int64"] }, { "Name": "SizeMax", "Docs": "", "Typewords": ["int64"] }] }, "NotFilter": { "Name": "NotFilter", "Docs": "", "Fields": [{ "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }] }, "Page": { "Name": "Page", "Docs": "", "Fields": [{ "Name": "AnchorMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Count", "Docs": "", "Typewords": ["int32"] }, { "Name": "DestMessageID", "Docs": "", "Typewords": ["int64"] }] }, @@ -53,16 +59,17 @@ var api; "DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] }, "EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] }, "EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] }, - "EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] }, - "MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }] }, - "Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] }, + "EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] }, + "MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }] }, + "Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] }, "MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] }, "Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] }, "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, - "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItem", "Docs": "", "Typewords": ["MessageItem"] }] }, + "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, + "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }] }, "ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] }, "ChangeMailboxRename": { "Name": "ChangeMailboxRename", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "OldName", "Docs": "", "Typewords": ["string"] }, { "Name": "NewName", "Docs": "", "Typewords": ["string"] }, { "Name": "Flags", "Docs": "", "Typewords": ["[]", "string"] }] }, @@ -73,6 +80,7 @@ var api; "UID": { "Name": "UID", "Docs": "", "Values": null }, "ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null }, "Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] }, + "ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] }, "AttachmentType": { "Name": "AttachmentType", "Docs": "", "Values": [{ "Name": "AttachmentIndifferent", "Value": "", "Docs": "" }, { "Name": "AttachmentNone", "Value": "none", "Docs": "" }, { "Name": "AttachmentAny", "Value": "any", "Docs": "" }, { "Name": "AttachmentImage", "Value": "image", "Docs": "" }, { "Name": "AttachmentPDF", "Value": "pdf", "Docs": "" }, { "Name": "AttachmentArchive", "Value": "archive", "Docs": "" }, { "Name": "AttachmentSpreadsheet", "Value": "spreadsheet", "Docs": "" }, { "Name": "AttachmentDocument", "Value": "document", "Docs": "" }, { "Name": "AttachmentPresentation", "Value": "presentation", "Docs": "" }] }, "Localpart": { "Name": "Localpart", "Docs": "", "Values": null }, }; @@ -106,6 +114,7 @@ var api; Flags: (v) => api.parse("Flags", v), ChangeMsgRemove: (v) => api.parse("ChangeMsgRemove", v), ChangeMsgFlags: (v) => api.parse("ChangeMsgFlags", v), + ChangeMsgThread: (v) => api.parse("ChangeMsgThread", v), ChangeMailboxRemove: (v) => api.parse("ChangeMailboxRemove", v), ChangeMailboxAdd: (v) => api.parse("ChangeMailboxAdd", v), ChangeMailboxRename: (v) => api.parse("ChangeMailboxRename", v), @@ -116,6 +125,7 @@ var api; UID: (v) => api.parse("UID", v), ModSeq: (v) => api.parse("ModSeq", v), Validation: (v) => api.parse("Validation", v), + ThreadMode: (v) => api.parse("ThreadMode", v), AttachmentType: (v) => api.parse("AttachmentType", v), Localpart: (v) => api.parse("Localpart", v), }; @@ -261,11 +271,30 @@ var api; const params = [mb]; return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); } + // ThreadCollapse saves the ThreadCollapse field for the messages and its + // children. The messageIDs are typically thread roots. But not all roots + // (without parent) of a thread need to have the same collapsed state. + async ThreadCollapse(messageIDs, collapse) { + const fn = "ThreadCollapse"; + const paramTypes = [["[]", "int64"], ["bool"]]; + const returnTypes = []; + const params = [messageIDs, collapse]; + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); + } + // ThreadMute saves the ThreadMute field for the messages and their children. + // If messages are muted, they are also marked collapsed. + async ThreadMute(messageIDs, mute) { + const fn = "ThreadMute"; + const paramTypes = [["[]", "int64"], ["bool"]]; + const returnTypes = []; + const params = [messageIDs, mute]; + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); + } // SSETypes exists to ensure the generated API contains the types, for use in SSE events. async SSETypes() { const fn = "SSETypes"; const paramTypes = []; - const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]]; + const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMsgThread"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]]; const params = []; return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); } diff --git a/webmail/text.js b/webmail/text.js index 5a0f78c..9c36c00 100644 --- a/webmail/text.js +++ b/webmail/text.js @@ -17,6 +17,12 @@ var api; Validation[Validation["ValidationSoftfail"] = 9] = "ValidationSoftfail"; Validation[Validation["ValidationNone"] = 10] = "ValidationNone"; })(Validation = api.Validation || (api.Validation = {})); + let ThreadMode; + (function (ThreadMode) { + ThreadMode["ThreadOff"] = "off"; + ThreadMode["ThreadOn"] = "on"; + ThreadMode["ThreadUnread"] = "unread"; + })(ThreadMode = api.ThreadMode || (api.ThreadMode = {})); // AttachmentType is for filtering by attachment type. let AttachmentType; (function (AttachmentType) { @@ -30,12 +36,12 @@ var api; AttachmentType["AttachmentDocument"] = "document"; AttachmentType["AttachmentPresentation"] = "presentation"; })(AttachmentType = api.AttachmentType || (api.AttachmentType = {})); - api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true }; - api.stringsTypes = { "AttachmentType": true, "Localpart": true }; + api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true }; + api.stringsTypes = { "AttachmentType": true, "Localpart": true, "ThreadMode": true }; api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true }; api.types = { "Request": { "Name": "Request", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Cancel", "Docs": "", "Typewords": ["bool"] }, { "Name": "Query", "Docs": "", "Typewords": ["Query"] }, { "Name": "Page", "Docs": "", "Typewords": ["Page"] }] }, - "Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] }, + "Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Threading", "Docs": "", "Typewords": ["ThreadMode"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] }, "Filter": { "Name": "Filter", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxChildrenIncluded", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Oldest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Newest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Headers", "Docs": "", "Typewords": ["[]", "[]", "string"] }, { "Name": "SizeMin", "Docs": "", "Typewords": ["int64"] }, { "Name": "SizeMax", "Docs": "", "Typewords": ["int64"] }] }, "NotFilter": { "Name": "NotFilter", "Docs": "", "Fields": [{ "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }] }, "Page": { "Name": "Page", "Docs": "", "Fields": [{ "Name": "AnchorMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Count", "Docs": "", "Typewords": ["int32"] }, { "Name": "DestMessageID", "Docs": "", "Typewords": ["int64"] }] }, @@ -53,16 +59,17 @@ var api; "DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] }, "EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] }, "EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] }, - "EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] }, - "MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }] }, - "Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] }, + "EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] }, + "MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }] }, + "Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] }, "MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] }, "Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] }, "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, - "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItem", "Docs": "", "Typewords": ["MessageItem"] }] }, + "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, + "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }] }, "ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] }, "ChangeMailboxRename": { "Name": "ChangeMailboxRename", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "OldName", "Docs": "", "Typewords": ["string"] }, { "Name": "NewName", "Docs": "", "Typewords": ["string"] }, { "Name": "Flags", "Docs": "", "Typewords": ["[]", "string"] }] }, @@ -73,6 +80,7 @@ var api; "UID": { "Name": "UID", "Docs": "", "Values": null }, "ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null }, "Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] }, + "ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] }, "AttachmentType": { "Name": "AttachmentType", "Docs": "", "Values": [{ "Name": "AttachmentIndifferent", "Value": "", "Docs": "" }, { "Name": "AttachmentNone", "Value": "none", "Docs": "" }, { "Name": "AttachmentAny", "Value": "any", "Docs": "" }, { "Name": "AttachmentImage", "Value": "image", "Docs": "" }, { "Name": "AttachmentPDF", "Value": "pdf", "Docs": "" }, { "Name": "AttachmentArchive", "Value": "archive", "Docs": "" }, { "Name": "AttachmentSpreadsheet", "Value": "spreadsheet", "Docs": "" }, { "Name": "AttachmentDocument", "Value": "document", "Docs": "" }, { "Name": "AttachmentPresentation", "Value": "presentation", "Docs": "" }] }, "Localpart": { "Name": "Localpart", "Docs": "", "Values": null }, }; @@ -106,6 +114,7 @@ var api; Flags: (v) => api.parse("Flags", v), ChangeMsgRemove: (v) => api.parse("ChangeMsgRemove", v), ChangeMsgFlags: (v) => api.parse("ChangeMsgFlags", v), + ChangeMsgThread: (v) => api.parse("ChangeMsgThread", v), ChangeMailboxRemove: (v) => api.parse("ChangeMailboxRemove", v), ChangeMailboxAdd: (v) => api.parse("ChangeMailboxAdd", v), ChangeMailboxRename: (v) => api.parse("ChangeMailboxRename", v), @@ -116,6 +125,7 @@ var api; UID: (v) => api.parse("UID", v), ModSeq: (v) => api.parse("ModSeq", v), Validation: (v) => api.parse("Validation", v), + ThreadMode: (v) => api.parse("ThreadMode", v), AttachmentType: (v) => api.parse("AttachmentType", v), Localpart: (v) => api.parse("Localpart", v), }; @@ -261,11 +271,30 @@ var api; const params = [mb]; return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); } + // ThreadCollapse saves the ThreadCollapse field for the messages and its + // children. The messageIDs are typically thread roots. But not all roots + // (without parent) of a thread need to have the same collapsed state. + async ThreadCollapse(messageIDs, collapse) { + const fn = "ThreadCollapse"; + const paramTypes = [["[]", "int64"], ["bool"]]; + const returnTypes = []; + const params = [messageIDs, collapse]; + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); + } + // ThreadMute saves the ThreadMute field for the messages and their children. + // If messages are muted, they are also marked collapsed. + async ThreadMute(messageIDs, mute) { + const fn = "ThreadMute"; + const paramTypes = [["[]", "int64"], ["bool"]]; + const returnTypes = []; + const params = [messageIDs, mute]; + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); + } // SSETypes exists to ensure the generated API contains the types, for use in SSE events. async SSETypes() { const fn = "SSETypes"; const paramTypes = []; - const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]]; + const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMsgThread"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]]; const params = []; return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); } diff --git a/webmail/view.go b/webmail/view.go index 47e3f60..1c40e61 100644 --- a/webmail/view.go +++ b/webmail/view.go @@ -52,9 +52,18 @@ type Request struct { Page Page } +type ThreadMode string + +const ( + ThreadOff ThreadMode = "off" + ThreadOn ThreadMode = "on" + ThreadUnread ThreadMode = "unread" +) + // Query is a request for messages that match filters, in a given order. type Query struct { OrderAsc bool // Order by received ascending or desending. + Threading ThreadMode Filter Filter NotFilter NotFilter } @@ -163,6 +172,7 @@ type MessageItem struct { IsSigned bool IsEncrypted bool FirstLine string // Of message body, for showing as preview. + MatchQuery bool // If message does not match query, it can still be included because of threading. } // ParsedMessage has more parsed/derived information about a message, intended @@ -219,8 +229,18 @@ type EventViewMsgs struct { ViewID int64 RequestID int64 - MessageItems []MessageItem // If empty, this was the last message for the request. - ParsedMessage *ParsedMessage // If set, will match the target page.DestMessageID from the request. + // If empty, this was the last message for the request. If non-empty, a list of + // thread messages. Each with the first message being the reason this thread is + // included and can be used as AnchorID in followup requests. If the threading mode + // is "off" in the query, there will always be only a single message. If a thread + // is sent, all messages in the thread are sent, including those that don't match + // the query (e.g. from another mailbox). Threads can be displayed based on the + // ThreadParentIDs field, with possibly slightly different display based on field + // ThreadMissingLink. + MessageItems [][]MessageItem + + // If set, will match the target page.DestMessageID from the request. + ParsedMessage *ParsedMessage // If set, there are no more messages in this view at this moment. Messages can be // added, typically via Change messages, e.g. for new deliveries. @@ -253,10 +273,10 @@ type EventViewChanges struct { Changes [][2]any // The first field of [2]any is a string, the second of the Change types below. } -// ChangeMsgAdd adds a new message to the view. +// ChangeMsgAdd adds a new message and possibly its thread to the view. type ChangeMsgAdd struct { store.ChangeAddUID - MessageItem MessageItem + MessageItems []MessageItem } // ChangeMsgRemove removes one or more messages from the view. @@ -269,6 +289,11 @@ type ChangeMsgFlags struct { store.ChangeFlags } +// ChangeMsgThread updates muted/collapsed fields for one message. +type ChangeMsgThread struct { + store.ChangeThread +} + // ChangeMailboxRemove indicates a mailbox was removed, including all its messages. type ChangeMailboxRemove struct { store.ChangeRemoveMailbox @@ -308,9 +333,9 @@ type ChangeMailboxKeywords struct { type view struct { Request Request - // Last message we sent to the client. We use it to decide if a newly delivered - // message is within the view and the client should get a notification. - LastMessage store.Message + // Received of last message we sent to the client. We use it to decide if a newly + // delivered message is within the view and the client should get a notification. + LastMessageReceived time.Time // If set, the last message in the query view has been sent. There is no need to do // another query, it will not return more data. Used to decide if an event for a @@ -322,6 +347,12 @@ type view struct { // Mailboxes to match, can be multiple, for matching children. If empty, there is // no filter on mailboxes. mailboxIDs map[int64]bool + + // Threads sent to client. New messages for this thread are also sent, regardless + // of regular query matching, so also for other mailboxes. If the user (re)moved + // all messages of a thread, they may still receive events for the thread. Only + // filled when query with threading not off. + threadIDs map[int64]struct{} } // sses tracks all sse connections, and access to them. @@ -513,6 +544,9 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h http.Error(w, "400 - bad request - request cannot have Page.Count 0", http.StatusBadRequest) return } + if req.Query.Threading == "" { + req.Query.Threading = ThreadOff + } var writer *eventWriter @@ -699,7 +733,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h // Start a view, it determines if we send a change to the client. And start an // implicit query for messages, we'll send the messages to the client which can // fill its ui with messages. - v := view{req, store.Message{}, false, matchMailboxes, mailboxIDs} + v := view{req, time.Time{}, false, matchMailboxes, mailboxIDs, map[int64]struct{}{}} go viewRequestTx(reqctx, log, acc, qtx, v, viewMsgsc, viewErrc, viewResetc, donec) qtx = nil // viewRequestTx closes qtx @@ -764,7 +798,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h // Return uids that are within range in view. Because the end has been reached, or // because the UID is not after the last message. - xchangedUIDs := func(mailboxID int64, uids []store.UID) (changedUIDs []store.UID) { + xchangedUIDs := func(mailboxID int64, uids []store.UID, isRemove bool) (changedUIDs []store.UID) { uidsAny := make([]any, len(uids)) for i, uid := range uids { uidsAny[i] = uid @@ -774,8 +808,10 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h q := bstore.QueryTx[store.Message](xtx) q.FilterNonzero(store.Message{MailboxID: mailboxID}) q.FilterEqual("UID", uidsAny...) + mbOK := v.matchesMailbox(mailboxID) err = q.ForEach(func(m store.Message) error { - if v.inRange(m) { + _, thread := v.threadIDs[m.ThreadID] + if thread || mbOK && (v.inRange(m) || isRemove && m.Expunged) { changedUIDs = append(changedUIDs, m.UID) } return nil @@ -788,33 +824,40 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h for _, change := range changes { switch c := change.(type) { case store.ChangeAddUID: - if ok, err := v.matches(log, acc, true, 0, c.MailboxID, c.UID, c.Flags, c.Keywords, getmsg); err != nil { - xcheckf(ctx, err, "matching new message against view") - } else if !ok { - continue - } + ok, err := v.matches(log, acc, true, 0, c.MailboxID, c.UID, c.Flags, c.Keywords, getmsg) + xcheckf(ctx, err, "matching new message against view") m, err := getmsg(0, c.MailboxID, c.UID) xcheckf(ctx, err, "get message") + _, thread := v.threadIDs[m.ThreadID] + if !ok && !thread { + continue + } state := msgState{acc: acc} mi, err := messageItem(log, m, &state) state.clear() xcheckf(ctx, err, "make messageitem") - taggedChanges = append(taggedChanges, [2]any{"ChangeMsgAdd", ChangeMsgAdd{c, mi}}) + mi.MatchQuery = ok + + mil := []MessageItem{mi} + if !thread && req.Query.Threading != ThreadOff { + err := ensureTx() + xcheckf(ctx, err, "transaction") + more, _, err := gatherThread(log, xtx, acc, v, m, 0) + xcheckf(ctx, err, "gathering thread messages for id %d, thread %d", m.ID, m.ThreadID) + mil = append(mil, more...) + v.threadIDs[m.ThreadID] = struct{}{} + } + + taggedChanges = append(taggedChanges, [2]any{"ChangeMsgAdd", ChangeMsgAdd{c, mil}}) // If message extends the view, store it as such. - if !v.Request.Query.OrderAsc && m.Received.Before(v.LastMessage.Received) || v.Request.Query.OrderAsc && m.Received.After(v.LastMessage.Received) { - v.LastMessage = m + if !v.Request.Query.OrderAsc && m.Received.Before(v.LastMessageReceived) || v.Request.Query.OrderAsc && m.Received.After(v.LastMessageReceived) { + v.LastMessageReceived = m.Received } case store.ChangeRemoveUIDs: - // We do a quick filter over changes, not sending UID updates for unselected - // mailboxes or when the message is outside the range of the view. But we still may - // send messages that don't apply to the filter. If client doesn't recognize the - // messages, that's fine. - if !v.matchesMailbox(c.MailboxID) { - continue - } - changedUIDs := xchangedUIDs(c.MailboxID, c.UIDs) + // We may send changes for uids the client doesn't know, that's fine. + changedUIDs := xchangedUIDs(c.MailboxID, c.UIDs, true) if len(changedUIDs) == 0 { continue } @@ -823,11 +866,8 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h taggedChanges = append(taggedChanges, [2]any{"ChangeMsgRemove", ch}) case store.ChangeFlags: - // As with ChangeRemoveUIDs above, we send more changes than strictly needed. - if !v.matchesMailbox(c.MailboxID) { - continue - } - changedUIDs := xchangedUIDs(c.MailboxID, []store.UID{c.UID}) + // We may send changes for uids the client doesn't know, that's fine. + changedUIDs := xchangedUIDs(c.MailboxID, []store.UID{c.UID}, false) if len(changedUIDs) == 0 { continue } @@ -835,6 +875,10 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h ch.UID = changedUIDs[0] taggedChanges = append(taggedChanges, [2]any{"ChangeMsgFlags", ch}) + case store.ChangeThread: + // Change in muted/collaped state, just always ship it. + taggedChanges = append(taggedChanges, [2]any{"ChangeMsgThread", ChangeMsgThread{c}}) + case store.ChangeRemoveMailbox: taggedChanges = append(taggedChanges, [2]any{"ChangeMailboxRemove", ChangeMailboxRemove{c}}) @@ -910,7 +954,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h v.End = true } if len(vm.MessageItems) > 0 { - v.LastMessage = vm.MessageItems[len(vm.MessageItems)-1].Message + v.LastMessageReceived = vm.MessageItems[len(vm.MessageItems)-1][0].Message.Received } writer.xsendEvent(ctx, log, "viewMsgs", vm) @@ -948,7 +992,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h cancelDrain() } if req.Cancel { - v = view{req, store.Message{}, false, false, nil} + v = view{req, time.Time{}, false, false, nil, nil} continue } @@ -987,7 +1031,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h if req.Query.Filter.MailboxChildrenIncluded { xgatherMailboxIDs(ctx, rtx, mailboxIDs, mailboxPrefixes) } - v = view{req, store.Message{}, false, matchMailboxes, mailboxIDs} + v = view{req, time.Time{}, false, matchMailboxes, mailboxIDs, map[int64]struct{}{}} } else { v.Request = req } @@ -1067,7 +1111,7 @@ func (v view) matchesMailbox(mailboxID int64) bool { // inRange returns whether m is within the range for the view, whether a change for // this message should be sent to the client so it can update its state. func (v view) inRange(m store.Message) bool { - return v.End || !v.Request.Query.OrderAsc && !m.Received.Before(v.LastMessage.Received) || v.Request.Query.OrderAsc && !m.Received.After(v.LastMessage.Received) + return v.End || !v.Request.Query.OrderAsc && !m.Received.Before(v.LastMessageReceived) || v.Request.Query.OrderAsc && !m.Received.After(v.LastMessageReceived) } // matches checks if the message, identified by either messageID or mailboxID+UID, @@ -1152,7 +1196,7 @@ type msgResp struct { err error // If set, an error happened and fields below are not set. reset bool // If set, the anchor message does not exist (anymore?) and we are sending messages from the start, fields below not set. viewEnd bool // If set, the last message for the view was seen, no more should be requested, fields below not set. - mi MessageItem // If none of the cases above apply, the message that was found matching the query. + mil []MessageItem // If none of the cases above apply, the messages that was found matching the query. First message was reason the thread is returned, for use as AnchorID in followup request. pm *ParsedMessage // If m was the target page.DestMessageID, or this is the first match, this is the parsed message of mi. } @@ -1176,7 +1220,7 @@ func viewRequestTx(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b } }() - var msgitems []MessageItem // Gathering for 300ms, then flushing. + var msgitems [][]MessageItem // Gathering for 300ms, then flushing. var parsedMessage *ParsedMessage var viewEnd bool @@ -1225,7 +1269,7 @@ func viewRequestTx(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b return } - msgitems = append(msgitems, mr.mi) + msgitems = append(msgitems, mr.mil) if mr.pm != nil { parsedMessage = mr.pm } @@ -1406,6 +1450,12 @@ func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b end = false return bstore.StopForEach } + + if _, ok := v.threadIDs[m.ThreadID]; ok { + // Message was already returned as part of a thread. + return nil + } + var pm *ParsedMessage if m.ID == page.DestMessageID || page.DestMessageID == 0 && have == 0 && page.AnchorMessageID == 0 { found = true @@ -1415,12 +1465,60 @@ func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b } pm = &xpm } + mi, err := messageItem(log, m, &state) if err != nil { return fmt.Errorf("making messageitem for message %d: %v", m.ID, err) } - mrc <- msgResp{mi: mi, pm: pm} - have++ + mil := []MessageItem{mi} + if query.Threading != ThreadOff { + more, xpm, err := gatherThread(log, tx, acc, v, m, page.DestMessageID) + if err != nil { + return fmt.Errorf("gathering thread messages for id %d, thread %d: %v", m.ID, m.ThreadID, err) + } + if xpm != nil { + pm = xpm + found = true + } + mil = append(mil, more...) + v.threadIDs[m.ThreadID] = struct{}{} + + // Calculate how many messages the frontend is going to show, and only count those as returned. + collapsed := map[int64]bool{} + for _, mi := range mil { + collapsed[mi.Message.ID] = mi.Message.ThreadCollapsed + } + unread := map[int64]bool{} // Propagated to thread root. + if query.Threading == ThreadUnread { + for _, mi := range mil { + m := mi.Message + if m.Seen { + continue + } + unread[m.ID] = true + for _, id := range m.ThreadParentIDs { + unread[id] = true + } + } + } + for _, mi := range mil { + m := mi.Message + threadRoot := true + rootID := m.ID + for _, id := range m.ThreadParentIDs { + if _, ok := collapsed[id]; ok { + threadRoot = false + rootID = id + } + } + if threadRoot || (query.Threading == ThreadOn && !collapsed[rootID] || query.Threading == ThreadUnread && unread[rootID]) { + have++ + } + } + } else { + have++ + } + mrc <- msgResp{mil: mil, pm: pm} return nil }) // Check for an error in one of the filters again. Check in ForEach would not @@ -1437,6 +1535,57 @@ func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b } } +func gatherThread(log *mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m store.Message, destMessageID int64) ([]MessageItem, *ParsedMessage, error) { + if m.ThreadID == 0 { + // If we would continue, FilterNonzero would fail because there are no non-zero fields. + return nil, nil, fmt.Errorf("message has threadid 0, account is probably still being upgraded, try turning threading off until the upgrade is done") + } + + // Fetch other messages for this thread. + qt := bstore.QueryTx[store.Message](tx) + qt.FilterNonzero(store.Message{ThreadID: m.ThreadID}) + qt.FilterEqual("Expunged", false) + qt.FilterNotEqual("ID", m.ID) + tml, err := qt.List() + if err != nil { + return nil, nil, fmt.Errorf("listing other messages in thread for message %d, thread %d: %v", m.ID, m.ThreadID, err) + } + + var mil []MessageItem + var pm *ParsedMessage + for _, tm := range tml { + err := func() error { + xstate := msgState{acc: acc} + defer xstate.clear() + + mi, err := messageItem(log, tm, &xstate) + if err != nil { + return fmt.Errorf("making messageitem for message %d, for thread %d: %v", tm.ID, m.ThreadID, err) + } + mi.MatchQuery, err = v.matches(log, acc, false, tm.ID, tm.MailboxID, tm.UID, tm.Flags, tm.Keywords, func(int64, int64, store.UID) (store.Message, error) { + return tm, nil + }) + if err != nil { + return fmt.Errorf("matching thread message %d against view query: %v", tm.ID, err) + } + mil = append(mil, mi) + + if tm.ID == destMessageID { + xpm, err := parsedMessage(log, tm, &xstate, true, false) + if err != nil { + return fmt.Errorf("parsing thread message %d: %v", tm.ID, err) + } + pm = &xpm + } + return nil + }() + if err != nil { + return nil, nil, err + } + } + return mil, pm, nil +} + // While checking the filters on a message, we may need to get more message // details as each filter passes. We check the filters that need the basic // information first, and load and cache more details for the next filters. diff --git a/webmail/view_test.go b/webmail/view_test.go index 63d2463..11fd494 100644 --- a/webmail/view_test.go +++ b/webmail/view_test.go @@ -50,8 +50,10 @@ func TestView(t *testing.T) { listsGoNutsMinimal = &testmsg{"Lists/Go/Nuts", store.Flags{}, nil, msgMinimal, zerom, 0} trashMinimal = &testmsg{"Trash", store.Flags{}, nil, msgMinimal, zerom, 0} junkMinimal = &testmsg{"Trash", store.Flags{}, nil, msgMinimal, zerom, 0} + trashAlt = &testmsg{"Trash", store.Flags{}, nil, msgAlt, zerom, 0} + inboxAltReply = &testmsg{"Inbox", store.Flags{}, nil, msgAltReply, zerom, 0} ) - var testmsgs = []*testmsg{inboxMinimal, inboxFlags, listsMinimal, listsGoNutsMinimal, trashMinimal, junkMinimal} + var testmsgs = []*testmsg{inboxMinimal, inboxFlags, listsMinimal, listsGoNutsMinimal, trashMinimal, junkMinimal, trashAlt, inboxAltReply} for _, tm := range testmsgs { tdeliver(t, acc, tm) } @@ -116,10 +118,10 @@ func TestView(t *testing.T) { evr.Get("start", &start) var viewMsgs EventViewMsgs evr.Get("viewMsgs", &viewMsgs) - tcompare(t, len(viewMsgs.MessageItems), 2) + tcompare(t, len(viewMsgs.MessageItems), 3) tcompare(t, viewMsgs.ViewEnd, true) - var inbox, archive, lists store.Mailbox + var inbox, archive, lists, trash store.Mailbox for _, mb := range start.Mailboxes { if mb.Archive { archive = mb @@ -127,6 +129,8 @@ func TestView(t *testing.T) { inbox = mb } else if mb.Name == "Lists" { lists = mb + } else if mb.Name == "Trash" { + trash = mb } } @@ -161,7 +165,7 @@ func TestView(t *testing.T) { testConn(api.Token(ctx), "&waitMinMsec=1&waitMaxMsec=2", waitReq, func(start EventStart, evr eventReader) { var vm EventViewMsgs evr.Get("viewMsgs", &vm) - tcompare(t, len(vm.MessageItems), 2) + tcompare(t, len(vm.MessageItems), 3) }) // Connection with DestMessageID. @@ -174,7 +178,7 @@ func TestView(t *testing.T) { testConn(tokens[len(tokens)-3], "", destMsgReq, func(start EventStart, evr eventReader) { var vm EventViewMsgs evr.Get("viewMsgs", &vm) - tcompare(t, len(vm.MessageItems), 2) + tcompare(t, len(vm.MessageItems), 3) tcompare(t, vm.ParsedMessage.ID, destMsgReq.Page.DestMessageID) }) // todo: destmessageid past count, needs large mailbox @@ -189,7 +193,7 @@ func TestView(t *testing.T) { testConn(api.Token(ctx), "", badDestMsgReq, func(start EventStart, evr eventReader) { var vm EventViewMsgs evr.Get("viewMsgs", &vm) - tcompare(t, len(vm.MessageItems), 2) + tcompare(t, len(vm.MessageItems), 3) }) // Connection with missing unknown AnchorMessageID, resets view. @@ -205,7 +209,7 @@ func TestView(t *testing.T) { var vm EventViewMsgs evr.Get("viewMsgs", &vm) - tcompare(t, len(vm.MessageItems), 2) + tcompare(t, len(vm.MessageItems), 3) }) // Connection that starts with a filter, without mailbox. @@ -219,12 +223,12 @@ func TestView(t *testing.T) { var vm EventViewMsgs evr.Get("viewMsgs", &vm) tcompare(t, len(vm.MessageItems), 1) - tcompare(t, vm.MessageItems[0].Message.ID, inboxFlags.ID) + tcompare(t, vm.MessageItems[0][0].Message.ID, inboxFlags.ID) }) // Paginate from previous last element. There is nothing new. var viewID int64 = 1 - api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}}, Page: Page{Count: 10, AnchorMessageID: viewMsgs.MessageItems[len(viewMsgs.MessageItems)-1].Message.ID}}) + api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}}, Page: Page{Count: 10, AnchorMessageID: viewMsgs.MessageItems[len(viewMsgs.MessageItems)-1][0].Message.ID}}) evr.Get("viewMsgs", &viewMsgs) tcompare(t, len(viewMsgs.MessageItems), 0) @@ -235,6 +239,36 @@ func TestView(t *testing.T) { tcompare(t, len(viewMsgs.MessageItems), 0) tcompare(t, viewMsgs.ViewEnd, true) + threadlen := func(mil [][]MessageItem) int { + n := 0 + for _, l := range mil { + n += len(l) + } + return n + } + + // Request with threading, should also include parent message from Trash mailbox (trashAlt). + viewID++ + api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}, Threading: "unread"}, Page: Page{Count: 10}}) + evr.Get("viewMsgs", &viewMsgs) + tcompare(t, len(viewMsgs.MessageItems), 3) + tcompare(t, threadlen(viewMsgs.MessageItems), 3+1) + tcompare(t, viewMsgs.ViewEnd, true) + // And likewise when querying Trash, should also include child message in Inbox (inboxAltReply). + viewID++ + api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: trash.ID}, Threading: "on"}, Page: Page{Count: 10}}) + evr.Get("viewMsgs", &viewMsgs) + tcompare(t, len(viewMsgs.MessageItems), 3) + tcompare(t, threadlen(viewMsgs.MessageItems), 3+1) + tcompare(t, viewMsgs.ViewEnd, true) + // Without threading, the inbox has just 3 messages. + viewID++ + api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}, Threading: "off"}, Page: Page{Count: 10}}) + evr.Get("viewMsgs", &viewMsgs) + tcompare(t, len(viewMsgs.MessageItems), 3) + tcompare(t, threadlen(viewMsgs.MessageItems), 3) + tcompare(t, viewMsgs.ViewEnd, true) + testFilter := func(orderAsc bool, f Filter, nf NotFilter, expIDs []int64) { t.Helper() viewID++ @@ -242,7 +276,7 @@ func TestView(t *testing.T) { evr.Get("viewMsgs", &viewMsgs) ids := make([]int64, len(viewMsgs.MessageItems)) for i, mi := range viewMsgs.MessageItems { - ids[i] = mi.Message.ID + ids[i] = mi[0].Message.ID } tcompare(t, ids, expIDs) tcompare(t, viewMsgs.ViewEnd, true) @@ -250,32 +284,32 @@ func TestView(t *testing.T) { // Test filtering. var znf NotFilter - testFilter(false, Filter{MailboxID: lists.ID, MailboxChildrenIncluded: true}, znf, []int64{listsGoNutsMinimal.ID, listsMinimal.ID}) // Mailbox and sub mailbox. - testFilter(true, Filter{MailboxID: lists.ID, MailboxChildrenIncluded: true}, znf, []int64{listsMinimal.ID, listsGoNutsMinimal.ID}) // Oldest first first. - testFilter(false, Filter{MailboxID: -1}, znf, []int64{listsGoNutsMinimal.ID, listsMinimal.ID, inboxFlags.ID, inboxMinimal.ID}) // All except trash/junk/rejects. + testFilter(false, Filter{MailboxID: lists.ID, MailboxChildrenIncluded: true}, znf, []int64{listsGoNutsMinimal.ID, listsMinimal.ID}) // Mailbox and sub mailbox. + testFilter(true, Filter{MailboxID: lists.ID, MailboxChildrenIncluded: true}, znf, []int64{listsMinimal.ID, listsGoNutsMinimal.ID}) // Oldest first first. + testFilter(false, Filter{MailboxID: -1}, znf, []int64{inboxAltReply.ID, listsGoNutsMinimal.ID, listsMinimal.ID, inboxFlags.ID, inboxMinimal.ID}) // All except trash/junk/rejects. testFilter(false, Filter{Labels: []string{`\seen`}}, znf, []int64{inboxFlags.ID}) - testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`\seen`}}, []int64{inboxMinimal.ID}) + testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`\seen`}}, []int64{inboxAltReply.ID, inboxMinimal.ID}) testFilter(false, Filter{Labels: []string{`testlabel`}}, znf, []int64{inboxFlags.ID}) - testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`testlabel`}}, []int64{inboxMinimal.ID}) - testFilter(false, Filter{MailboxID: inbox.ID, Oldest: &inboxFlags.m.Received}, znf, []int64{inboxFlags.ID}) + testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`testlabel`}}, []int64{inboxAltReply.ID, inboxMinimal.ID}) + testFilter(false, Filter{MailboxID: inbox.ID, Oldest: &inboxFlags.m.Received}, znf, []int64{inboxAltReply.ID, inboxFlags.ID}) testFilter(false, Filter{MailboxID: inbox.ID, Newest: &inboxMinimal.m.Received}, znf, []int64{inboxMinimal.ID}) testFilter(false, Filter{MailboxID: inbox.ID, SizeMin: inboxFlags.m.Size}, znf, []int64{inboxFlags.ID}) testFilter(false, Filter{MailboxID: inbox.ID, SizeMax: inboxMinimal.m.Size}, znf, []int64{inboxMinimal.ID}) testFilter(false, Filter{From: []string{"mjl+altrel@mox.example"}}, znf, []int64{inboxFlags.ID}) - testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{From: []string{"mjl+altrel@mox.example"}}, []int64{inboxMinimal.ID}) + testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{From: []string{"mjl+altrel@mox.example"}}, []int64{inboxAltReply.ID, inboxMinimal.ID}) testFilter(false, Filter{To: []string{"mox+altrel@other.example"}}, znf, []int64{inboxFlags.ID}) - testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{To: []string{"mox+altrel@other.example"}}, []int64{inboxMinimal.ID}) + testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{To: []string{"mox+altrel@other.example"}}, []int64{inboxAltReply.ID, inboxMinimal.ID}) testFilter(false, Filter{From: []string{"mjl+altrel@mox.example", "bogus"}}, znf, []int64{}) testFilter(false, Filter{To: []string{"mox+altrel@other.example", "bogus"}}, znf, []int64{}) testFilter(false, Filter{Subject: []string{"test", "alt", "rel"}}, znf, []int64{inboxFlags.ID}) - testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Subject: []string{"alt"}}, []int64{inboxMinimal.ID}) + testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Subject: []string{"alt"}}, []int64{inboxAltReply.ID, inboxMinimal.ID}) testFilter(false, Filter{MailboxID: inbox.ID, Words: []string{"the text body", "body", "the "}}, znf, []int64{inboxFlags.ID}) - testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Words: []string{"the text body"}}, []int64{inboxMinimal.ID}) + testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Words: []string{"the text body"}}, []int64{inboxAltReply.ID, inboxMinimal.ID}) testFilter(false, Filter{Headers: [][2]string{{"X-Special", ""}}}, znf, []int64{inboxFlags.ID}) testFilter(false, Filter{Headers: [][2]string{{"X-Special", "testing"}}}, znf, []int64{inboxFlags.ID}) testFilter(false, Filter{Headers: [][2]string{{"X-Special", "other"}}}, znf, []int64{}) testFilter(false, Filter{Attachments: AttachmentImage}, znf, []int64{inboxFlags.ID}) - testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Attachments: AttachmentImage}, []int64{inboxMinimal.ID}) + testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Attachments: AttachmentImage}, []int64{inboxAltReply.ID, inboxMinimal.ID}) // Test changes. getChanges := func(changes ...any) { @@ -341,13 +375,13 @@ func TestView(t *testing.T) { var chmbcounts ChangeMailboxCounts getChanges(&chmsgadd, &chmbcounts) tcompare(t, chmsgadd.ChangeAddUID.MailboxID, inbox.ID) - tcompare(t, chmsgadd.MessageItem.Message.ID, inboxNew.ID) + tcompare(t, chmsgadd.MessageItems[0].Message.ID, inboxNew.ID) chmbcounts.Size = 0 tcompare(t, chmbcounts, ChangeMailboxCounts{ ChangeMailboxCounts: store.ChangeMailboxCounts{ MailboxID: inbox.ID, MailboxName: inbox.Name, - MailboxCounts: store.MailboxCounts{Total: 3, Unread: 2, Unseen: 2}, + MailboxCounts: store.MailboxCounts{Total: 4, Unread: 3, Unseen: 3}, }, }) @@ -369,7 +403,7 @@ func TestView(t *testing.T) { ChangeMailboxCounts: store.ChangeMailboxCounts{ MailboxID: inbox.ID, MailboxName: inbox.Name, - MailboxCounts: store.MailboxCounts{Total: 3, Unread: 1, Unseen: 1}, + MailboxCounts: store.MailboxCounts{Total: 4, Unread: 2, Unseen: 2}, }, }) @@ -384,10 +418,40 @@ func TestView(t *testing.T) { ChangeMailboxCounts: store.ChangeMailboxCounts{ MailboxID: inbox.ID, MailboxName: inbox.Name, - MailboxCounts: store.MailboxCounts{Total: 1}, + MailboxCounts: store.MailboxCounts{Total: 2, Unread: 1, Unseen: 1}, }, }) + // ChangeMsgThread + api.ThreadCollapse(ctx, []int64{inboxAltReply.ID}, true) + var chmsgthread ChangeMsgThread + getChanges(&chmsgthread) + tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: false, Collapsed: true}) + + // Now collapsing the thread root, the child is already collapsed so no change. + api.ThreadCollapse(ctx, []int64{trashAlt.ID}, true) + getChanges(&chmsgthread) + tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: false, Collapsed: true}) + + // Expand thread root, including change for child. + api.ThreadCollapse(ctx, []int64{trashAlt.ID}, false) + var chmsgthread2 ChangeMsgThread + getChanges(&chmsgthread, &chmsgthread2) + tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: false, Collapsed: false}) + tcompare(t, chmsgthread2.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: false, Collapsed: false}) + + // Mute thread, including child, also collapses. + api.ThreadMute(ctx, []int64{trashAlt.ID}, true) + getChanges(&chmsgthread, &chmsgthread2) + tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: true, Collapsed: true}) + tcompare(t, chmsgthread2.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: true, Collapsed: true}) + + // And unmute Mute thread, including child. Messages are not expanded. + api.ThreadMute(ctx, []int64{trashAlt.ID}, false) + getChanges(&chmsgthread, &chmsgthread2) + tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: false, Collapsed: true}) + tcompare(t, chmsgthread2.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: false, Collapsed: true}) + // todo: check move operations and their changes, e.g. MailboxDelete, MailboxEmpty, MessageRemove. } diff --git a/webmail/webmail.go b/webmail/webmail.go index 9eed01c..0f4e4a6 100644 --- a/webmail/webmail.go +++ b/webmail/webmail.go @@ -758,7 +758,7 @@ func handle(apiHandler http.Handler, w http.ResponseWriter, r *http.Request) { m.MsgPrefix = nil m.ParsedBuf = nil - mi := MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine} + mi := MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine, false} mijson, err := json.Marshal(mi) xcheckf(ctx, err, "marshal messageitem") diff --git a/webmail/webmail.html b/webmail/webmail.html index aa21f18..9067cc5 100644 --- a/webmail/webmail.html +++ b/webmail/webmail.html @@ -34,20 +34,24 @@ iframe { border: 0; } .msgitemcell { padding: 2px 4px; } /* note: we assign widths to .msgitemflags, .msgitemfrom, .msgitemsubject, .msgitemage, and offsets through a stylesheet created in js */ .msgitemage { text-align: right; } +.msgitemfrom { position: relative; } .msgitemfromtext { white-space: nowrap; overflow: hidden; } +.msgitemfromthreadbar { position: absolute; border-right: 2px solid #666; right: 0; top: 0; bottom: 0; /* top or bottom set with inline style for first & last */ } .msgitemsubjecttext { white-space: nowrap; overflow: hidden; } .msgitemsubjectsnippet { font-weight: normal; color: #666; } -.msgitemmailbox { background-color: #999; color: white; border: 1px solid #777; padding: 0 .15em; margin-left: .15em; border-radius: .15em; font-weight: normal; font-size: .9em; white-space: nowrap; } +.msgitemmailbox { background: #999; color: white; border: 1px solid #777; padding: 0 .15em; margin-left: .15em; border-radius: .15em; font-weight: normal; font-size: .9em; white-space: nowrap; } +.msgitemmailbox.msgitemmailboxcollapsed { background: #eee; color: #333; } .msgitemidentity { background-color: #999; color: white; border: 1px solid #777; padding: 0 .15em; margin-left: .15em; border-radius: .15em; font-weight: normal; font-size: .9em; white-space: nowrap; } .topbar, .mailboxesbar { background-color: #fdfdf1; } .msglist { background-color: #f5ffff; } table.search td { padding: .25em; } .keyword { background-color: gold; color: black; border: 1px solid #8c7600; padding: 0 .15em; border-radius: .15em; font-weight: normal; font-size: .9em; margin: 0 .15em; white-space: nowrap; } +.keyword.keywordcollapsed { background-color: #ffeb7e; color: #333; } .mailbox { padding: .15em .25em; } .mailboxitem { cursor: pointer; border-radius: .15em; } -.mailboxitem.dropping { background-color: gold !important; } -.mailboxitem:hover { background-color: #eee; } +.mailboxitem.dropping { background: gold !important; } +.mailboxitem:hover { background: #eee; } .mailboxitem.active { background: linear-gradient(135deg, #ffc7ab 0%, #ffdeab 100%); } .mailboxhoveronly { visibility: hidden; } .mailboxitem:hover .mailboxhoveronly, .mailboxitem:focus .mailboxhoveronly { visibility: visible; } @@ -59,6 +63,7 @@ table.search td { padding: .25em; } .msgitem.active { background: linear-gradient(135deg, #8bc8ff 0%, #8ee5ff 100%); } .msgitemsubject { position: relative; } .msgitemflag { margin-right: 1px; font-weight: normal; font-size: .9em; } +.msgitemflag.msgitemflagcollapsed { color: #666; } .quoted1 { color: #03828f; } .quoted2 { color: #c7445c; } .quoted3 { color: #417c10; } diff --git a/webmail/webmail.js b/webmail/webmail.js index ee7f586..0c3d846 100644 --- a/webmail/webmail.js +++ b/webmail/webmail.js @@ -17,6 +17,12 @@ var api; Validation[Validation["ValidationSoftfail"] = 9] = "ValidationSoftfail"; Validation[Validation["ValidationNone"] = 10] = "ValidationNone"; })(Validation = api.Validation || (api.Validation = {})); + let ThreadMode; + (function (ThreadMode) { + ThreadMode["ThreadOff"] = "off"; + ThreadMode["ThreadOn"] = "on"; + ThreadMode["ThreadUnread"] = "unread"; + })(ThreadMode = api.ThreadMode || (api.ThreadMode = {})); // AttachmentType is for filtering by attachment type. let AttachmentType; (function (AttachmentType) { @@ -30,12 +36,12 @@ var api; AttachmentType["AttachmentDocument"] = "document"; AttachmentType["AttachmentPresentation"] = "presentation"; })(AttachmentType = api.AttachmentType || (api.AttachmentType = {})); - api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true }; - api.stringsTypes = { "AttachmentType": true, "Localpart": true }; + api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true }; + api.stringsTypes = { "AttachmentType": true, "Localpart": true, "ThreadMode": true }; api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true }; api.types = { "Request": { "Name": "Request", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Cancel", "Docs": "", "Typewords": ["bool"] }, { "Name": "Query", "Docs": "", "Typewords": ["Query"] }, { "Name": "Page", "Docs": "", "Typewords": ["Page"] }] }, - "Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] }, + "Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Threading", "Docs": "", "Typewords": ["ThreadMode"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] }, "Filter": { "Name": "Filter", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxChildrenIncluded", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Oldest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Newest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Headers", "Docs": "", "Typewords": ["[]", "[]", "string"] }, { "Name": "SizeMin", "Docs": "", "Typewords": ["int64"] }, { "Name": "SizeMax", "Docs": "", "Typewords": ["int64"] }] }, "NotFilter": { "Name": "NotFilter", "Docs": "", "Fields": [{ "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }] }, "Page": { "Name": "Page", "Docs": "", "Fields": [{ "Name": "AnchorMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Count", "Docs": "", "Typewords": ["int32"] }, { "Name": "DestMessageID", "Docs": "", "Typewords": ["int64"] }] }, @@ -53,16 +59,17 @@ var api; "DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] }, "EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] }, "EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] }, - "EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] }, - "MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }] }, - "Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] }, + "EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] }, + "MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }] }, + "Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] }, "MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] }, "Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] }, "EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] }, - "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItem", "Docs": "", "Typewords": ["MessageItem"] }] }, + "ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] }, "Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] }, "ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] }, + "ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] }, "ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }] }, "ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] }, "ChangeMailboxRename": { "Name": "ChangeMailboxRename", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "OldName", "Docs": "", "Typewords": ["string"] }, { "Name": "NewName", "Docs": "", "Typewords": ["string"] }, { "Name": "Flags", "Docs": "", "Typewords": ["[]", "string"] }] }, @@ -73,6 +80,7 @@ var api; "UID": { "Name": "UID", "Docs": "", "Values": null }, "ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null }, "Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] }, + "ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] }, "AttachmentType": { "Name": "AttachmentType", "Docs": "", "Values": [{ "Name": "AttachmentIndifferent", "Value": "", "Docs": "" }, { "Name": "AttachmentNone", "Value": "none", "Docs": "" }, { "Name": "AttachmentAny", "Value": "any", "Docs": "" }, { "Name": "AttachmentImage", "Value": "image", "Docs": "" }, { "Name": "AttachmentPDF", "Value": "pdf", "Docs": "" }, { "Name": "AttachmentArchive", "Value": "archive", "Docs": "" }, { "Name": "AttachmentSpreadsheet", "Value": "spreadsheet", "Docs": "" }, { "Name": "AttachmentDocument", "Value": "document", "Docs": "" }, { "Name": "AttachmentPresentation", "Value": "presentation", "Docs": "" }] }, "Localpart": { "Name": "Localpart", "Docs": "", "Values": null }, }; @@ -106,6 +114,7 @@ var api; Flags: (v) => api.parse("Flags", v), ChangeMsgRemove: (v) => api.parse("ChangeMsgRemove", v), ChangeMsgFlags: (v) => api.parse("ChangeMsgFlags", v), + ChangeMsgThread: (v) => api.parse("ChangeMsgThread", v), ChangeMailboxRemove: (v) => api.parse("ChangeMailboxRemove", v), ChangeMailboxAdd: (v) => api.parse("ChangeMailboxAdd", v), ChangeMailboxRename: (v) => api.parse("ChangeMailboxRename", v), @@ -116,6 +125,7 @@ var api; UID: (v) => api.parse("UID", v), ModSeq: (v) => api.parse("ModSeq", v), Validation: (v) => api.parse("Validation", v), + ThreadMode: (v) => api.parse("ThreadMode", v), AttachmentType: (v) => api.parse("AttachmentType", v), Localpart: (v) => api.parse("Localpart", v), }; @@ -261,11 +271,30 @@ var api; const params = [mb]; return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); } + // ThreadCollapse saves the ThreadCollapse field for the messages and its + // children. The messageIDs are typically thread roots. But not all roots + // (without parent) of a thread need to have the same collapsed state. + async ThreadCollapse(messageIDs, collapse) { + const fn = "ThreadCollapse"; + const paramTypes = [["[]", "int64"], ["bool"]]; + const returnTypes = []; + const params = [messageIDs, collapse]; + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); + } + // ThreadMute saves the ThreadMute field for the messages and their children. + // If messages are muted, they are also marked collapsed. + async ThreadMute(messageIDs, mute) { + const fn = "ThreadMute"; + const paramTypes = [["[]", "int64"], ["bool"]]; + const returnTypes = []; + const params = [messageIDs, mute]; + return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); + } // SSETypes exists to ensure the generated API contains the types, for use in SSE events. async SSETypes() { const fn = "SSETypes"; const paramTypes = []; - const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]]; + const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMsgThread"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]]; const params = []; return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params); } @@ -970,12 +999,21 @@ different color. Browsers to test with: Firefox, Chromium, Safari, Edge. To simulate slow API calls and SSE events: -window.localStorage.setItem('sherpats-debug', JSON.stringify({waitMinMsec: 2000, waitMaxMsec: 4000})) + + localStorage.setItem('sherpats-debug', JSON.stringify({waitMinMsec: 2000, waitMaxMsec: 4000})) Show additional headers of messages: -settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id']}) -- todo: threading (needs support in mox first) + settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id', 'List-Id', 'List-Post', 'X-Mox-Reason']}) + +Enable logging and reload afterwards: + + localStorage.setItem('log', 'yes') + +Enable consistency checking in UI updates: + + settingsPut({...settings, checkConsistency: true}) + - todo: in msglistView, show names of people we have sent to, and address otherwise. - todo: implement settings stored in the server, such as mailboxCollapsed, keyboard shortcuts. also new settings for displaying email as html by default for configured sender address or domain. name to use for "From", optional default Reply-To and Bcc addresses, signatures (per address), configured labels/keywords with human-readable name, colors and toggling with shortcut keys 1-9. - todo: in msglist, if our address is in the from header, list addresses in the to/cc/bcc, it's likely a sent folder @@ -991,7 +1029,6 @@ settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id']} - todo: only show orange underline where it could be a problem? in addresses and anchor texts. we may be lighting up a christmas tree now, desensitizing users. - todo: saved searches that are displayed below list of mailboxes, for quick access to preset view - todo: when search on free-form text is active, highlight the searched text in the message view. -- todo: when reconnecting, request only the changes to the current state/msglist, passing modseq query string parameter - todo: composeView: save as draft, periodically and when closing. - todo: forwarding of html parts, including inline attachments, so the html version can be rendered like the original by the receiver. - todo: buttons/mechanism to operate on all messages in a mailbox/search query, without having to list and select all messages. e.g. clearing flags/labels. @@ -1002,14 +1039,15 @@ settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id']} - todo: nicer address input fields like other mail clients do. with tab to autocomplete and turn input into a box and delete removing of the entire address. - todo: consider composing messages with bcc headers that are kept as message Bcc headers, optionally with checkbox. - todo: improve accessibility +- todo: threading mode where we don't show messages in Trash/Sent in thread? - todo: msglistView: preload next message? - todo: previews of zip files - todo: undo? -- todo: mute threads? - todo: mobile-friendly version. should perhaps be a completely different app, because it is so different. -- todo: msglistView: for mailbox views (which are fast to list the results of), should we ask the full number of messages, set the height of the scroll div based on the number of messages, then request messages when user scrolls, putting the messages in place. not sure if worth the trouble. - todo: basic vim key bindings in textarea/input. or just let users use a browser plugin. */ +class ConsistencyError extends Error { +} const zindexes = { splitter: '1', compose: '2', @@ -1023,7 +1061,7 @@ const zindexes = { // All logging goes through log() instead of console.log, except "should not happen" logging. let log = () => { }; try { - if (localStorage.getItem('log')) { + if (localStorage.getItem('log') || location.hostname === 'localhost') { log = console.log; } } @@ -1043,7 +1081,9 @@ const defaultSettings = { showHTML: false, mailboxCollapsed: {}, showAllHeaders: false, - showHeaders: [], // Additional message headers to show. + showHeaders: [], + threading: api.ThreadMode.ThreadUnread, + checkConsistency: location.hostname === 'localhost', // Enable UI update consistency checks, default only for local development. }; const parseSettings = () => { try { @@ -1095,6 +1135,8 @@ const parseSettings = () => { mailboxCollapsed: mailboxCollapsed, showAllHeaders: getBool('showAllHeaders'), showHeaders: getStringArray('showHeaders'), + threading: getString('threading', api.ThreadMode.ThreadOff, api.ThreadMode.ThreadOn, api.ThreadMode.ThreadUnread), + checkConsistency: getBool('checkConsistency'), }; } catch (err) { @@ -1157,7 +1199,7 @@ const showShortcut = (c) => { const shortcutCmd = async (cmdfn, shortcuts) => { let shortcut = ''; for (const k in shortcuts) { - if (shortcuts[k] == cmdfn) { + if (shortcuts[k] === cmdfn) { shortcut = k; break; } @@ -1297,7 +1339,7 @@ const parseSearchTokens = (s) => { add(); } } - else if (quoted && c == '"') { + else if (quoted && c === '"') { quoteend = true; } else if (c === '"') { @@ -1361,7 +1403,7 @@ const parseSearch = (searchquery, mailboxlistView) => { } return; } - else if (tag == 'submb') { + else if (tag === 'submb') { fpos.MailboxChildrenIncluded = true; return; } @@ -1486,34 +1528,41 @@ const newAddressComplete = () => { } }; }; -// Characters we display in the message list for flags set for a message. -// todo: icons would be nice to have instead. -const flagchars = { - Replied: 'r', - Flagged: '!', - Forwarded: 'f', - Junk: 'j', - Deleted: 'D', - Draft: 'd', - Phishing: 'p', -}; -const flagList = (m, mi) => { - let l = []; +const flagList = (miv) => { + const msgflags = []; // Flags for message in miv. + const othermsgflags = []; // Flags for descendant messages if miv is collapsed. Only flags not in msgflags. + let l = msgflags; + const seen = new Set(); const flag = (v, char, name) => { - if (v) { + if (v && !seen.has(name)) { l.push([name, char]); + seen.add(name); } }; - flag(m.Answered, 'r', 'Replied/answered'); - flag(m.Flagged, '!', 'Flagged'); - flag(m.Forwarded, 'f', 'Forwarded'); - flag(m.Junk, 'j', 'Junk'); - flag(m.Deleted, 'D', 'Deleted, used in IMAP, message will likely be removed soon.'); - flag(m.Draft, 'd', 'Draft'); - flag(m.Phishing, 'p', 'Phishing'); - flag(!m.Junk && !m.Notjunk, '?', 'Unclassified, neither junk nor not junk: message does not contribute to spam classification of new incoming messages'); - flag(mi.Attachments && mi.Attachments.length > 0 ? true : false, 'a', 'Has at least one attachment'); - return l.map(t => dom.span(dom._class('msgitemflag'), t[1], attr.title(t[0]))); + const addFlags = (mi) => { + const m = mi.Message; + flag(m.Answered, 'r', 'Replied/answered'); + flag(m.Flagged, '!', 'Flagged'); + flag(m.Forwarded, 'f', 'Forwarded'); + flag(m.Junk, 'j', 'Junk'); + flag(m.Deleted, 'D', 'Deleted, used in IMAP, message will likely be removed soon.'); + flag(m.Draft, 'd', 'Draft'); + flag(m.Phishing, 'p', 'Phishing'); + flag(!m.Junk && !m.Notjunk, '?', 'Unclassified, neither junk nor not junk: message does not contribute to spam classification of new incoming messages'); + flag(mi.Attachments && mi.Attachments.length > 0 ? true : false, 'a', 'Has at least one attachment'); + if (m.ThreadMuted) { + flag(true, 'm', 'Muted, new messages are automatically marked as read.'); + } + }; + addFlags(miv.messageitem); + if (miv.isCollapsedThreadRoot()) { + l = othermsgflags; + for (miv of miv.descendants()) { + addFlags(miv.messageitem); + } + } + return msgflags.map(t => dom.span(dom._class('msgitemflag'), t[1], attr.title(t[0]))) + .concat(othermsgflags.map(t => dom.span(dom._class('msgitemflag'), dom._class('msgitemflagcollapsed'), t[1], attr.title(t[0])))); }; // Turn filters from the search bar into filters with the refine filters (buttons // above message list) applied, to send to the server in a request. The original @@ -1778,7 +1827,7 @@ const cmdHelp = async () => { ['i', 'open inbox'], ['?', 'help'], ['ctrl ?', 'tooltip for focused element'], - ['M', 'focus message'], + ['ctrl m', 'focus message'], ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))), dom.tr(dom.td(attr.colspan('2'), dom.h2('Mailbox', style({ margin: '0' })))), [ ['←', 'collapse'], ['→', 'expand'], @@ -1795,20 +1844,26 @@ const cmdHelp = async () => { ['d, Delete', 'move to trash folder'], ['D', 'delete permanently'], ['q', 'move to junk folder'], - ['n', 'mark not junk'], + ['Q', 'mark not junk'], ['a', 'move to archive folder'], - ['u', 'mark unread'], + ['M', 'mark unread'], ['m', 'mark read'], - ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))), dom.tr(dom.td(attr.colspan('2'), dom.h2('Compose', style({ margin: '1ex 0 0 0' })))), [ + ['u', 'to next unread message'], + ['p', 'to root of thread or previous thread'], + ['n', 'to root of next thread'], + ['S', 'select thread messages'], + ['C', 'toggle thread collapse'], + ['X', 'toggle thread mute, automatically marking new messages as read'], + ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))))), dom.div(style({ width: '40em' }), dom.table(dom.tr(dom.td(attr.colspan('2'), dom.h2('Compose', style({ margin: '0' })))), [ ['ctrl Enter', 'send message'], ['ctrl w', 'cancel message'], - ['ctlr O', 'add To'], + ['ctrl O', 'add To'], ['ctrl C', 'add Cc'], ['ctrl B', 'add Bcc'], ['ctrl Y', 'add Reply-To'], ['ctrl -', 'remove current address'], ['ctrl +', 'add address of same type'], - ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))))), dom.div(style({ width: '40em' }), dom.table(dom.tr(dom.td(attr.colspan('2'), dom.h2('Message', style({ margin: '0' })))), [ + ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))), dom.tr(dom.td(attr.colspan('2'), dom.h2('Message', style({ margin: '1ex 0 0 0' })))), [ ['r', 'reply or list reply'], ['R', 'reply all'], ['f', 'forward message'], @@ -1914,7 +1969,7 @@ const compose = (opts) => { const fr = new window.FileReader(); fr.addEventListener('load', () => { l.push({ Filename: f.name, DataURI: fr.result }); - if (attachments.files && l.length == attachments.files.length) { + if (attachments.files && l.length === attachments.files.length) { resolve(l); } }); @@ -2050,7 +2105,7 @@ const compose = (opts) => { // Find own address matching the specified address, taking wildcards, localpart // separators and case-sensitivity into account. const addressSelf = (addr) => { - return accountAddresses.find(a => a.Domain.ASCII === addr.Domain.ASCII && (a.User === '' || normalizeUser(a) == normalizeUser(addr))); + return accountAddresses.find(a => a.Domain.ASCII === addr.Domain.ASCII && (a.User === '' || normalizeUser(a) === normalizeUser(addr))); }; let haveFrom = false; const fromOptions = accountAddresses.map(a => { @@ -2168,10 +2223,9 @@ const movePopover = (e, mailboxes, msgs) => { remove(); }))))); }; -// Make new MsgitemView, to be added to the list. othermb is set when this msgitem -// is displayed in a msglistView for other/multiple mailboxes, the mailbox name -// should be shown. -const newMsgitemView = (mi, msglistView, othermb) => { +// Make new MsgitemView, to be added to the list. +const newMsgitemView = (mi, msglistView, otherMailbox, listMailboxes, receivedTime, initialCollapsed) => { + // note: mi may be replaced. // Timer to update the age of the message. let ageTimer = 0; // Show with a tag if we are in the cc/bcc headers, or - if none. @@ -2189,41 +2243,6 @@ const newMsgitemView = (mi, msglistView, othermb) => { identityHeader.push(identityTag('-', 'You are not in any To, From, CC, BCC header. Could message to a mailing list or Bcc without Bcc message header.')); } } - // If mailbox of message is not specified in filter (i.e. for mailbox list or - // search on the mailbox), we show it on the right-side of the subject. - const mailboxtag = []; - if (othermb) { - let name = othermb.Name; - if (name.length > 8 + 1 + 3 + 1 + 8 + 4) { - const t = name.split('/'); - const first = t[0]; - const last = t[t.length - 1]; - if (first.length + last.length <= 8 + 8) { - name = first + '/.../' + last; - } - else { - name = first.substring(0, 8) + '/.../' + last.substring(0, 8); - } - } - const e = dom.span(dom._class('msgitemmailbox'), name === othermb.Name ? [] : attr.title(othermb.Name), name); - mailboxtag.push(e); - } - const updateFlags = (modseq, mask, flags, keywords) => { - msgitemView.messageitem.Message.ModSeq = modseq; - const maskobj = mask; - const flagsobj = flags; - const mobj = msgitemView.messageitem.Message; - for (const k in maskobj) { - if (maskobj[k]) { - mobj[k] = flagsobj[k]; - } - } - msgitemView.messageitem.Message.Keywords = keywords; - const elem = render(); - msgitemView.root.replaceWith(elem); - msgitemView.root = elem; - msglistView.redraw(msgitemView); - }; const remove = () => { msgitemView.root.remove(); if (ageTimer) { @@ -2252,7 +2271,7 @@ const newMsgitemView = (mi, msglistView, othermb) => { let nextSecs = 0; for (let i = 0; i < periods.length; i++) { const p = periods[i]; - if (t >= 2 * p || i == periods.length - 1) { + if (t >= 2 * p || i === periods.length - 1) { const n = Math.round(t / p); s = '' + n + suffix[i]; const prev = Math.floor(t / p); @@ -2278,28 +2297,239 @@ const newMsgitemView = (mi, msglistView, othermb) => { return r; }; const render = () => { + const mi = msgitemView.messageitem; + const m = mi.Message; // Set by calling age(). if (ageTimer) { window.clearTimeout(ageTimer); ageTimer = 0; } - const m = msgitemView.messageitem.Message; + // Keywords are normally shown per message. For collapsed threads, we show the + // keywords of the thread root message as normal, and any additional keywords from + // children in a way that draws less attention. const keywords = (m.Keywords || []).map(kw => dom.span(dom._class('keyword'), kw)); - return dom.div(dom._class('msgitem'), attr.draggable('true'), function dragstart(e) { - e.dataTransfer.setData('application/vnd.mox.messages', JSON.stringify(msglistView.selected().map(miv => miv.messageitem.Message.ID))); - }, m.Seen ? [] : style({ fontWeight: 'bold' }), dom.div(dom._class('msgitemcell', 'msgitemflags'), flagList(m, msgitemView.messageitem)), dom.div(dom._class('msgitemcell', 'msgitemfrom'), dom.div(style({ display: 'flex', justifyContent: 'space-between' }), dom.div(dom._class('msgitemfromtext', 'silenttitle'), attr.title((mi.Envelope.From || []).map(a => formatAddressFull(a)).join(', ')), join((mi.Envelope.From || []).map(a => formatAddressShort(a)), () => ', ')), identityHeader)), dom.div(dom._class('msgitemcell', 'msgitemsubject'), dom.div(style({ display: 'flex', justifyContent: 'space-between', position: 'relative' }), dom.div(dom._class('msgitemsubjecttext'), mi.Envelope.Subject || '(no subject)', dom.span(dom._class('msgitemsubjectsnippet'), ' ' + mi.FirstLine)), dom.div(keywords, mailboxtag))), dom.div(dom._class('msgitemcell', 'msgitemage'), age(m.Received)), function click(e) { + if (msgitemView.isCollapsedThreadRoot()) { + const keywordsSeen = new Set(); + for (const kw of (m.Keywords || [])) { + keywordsSeen.add(kw); + } + for (const miv of msgitemView.descendants()) { + for (const kw of (miv.messageitem.Message.Keywords || [])) { + if (!keywordsSeen.has(kw)) { + keywordsSeen.add(kw); + keywords.push(dom.span(dom._class('keyword'), dom._class('keywordcollapsed'), kw)); + } + } + } + } + let threadIndent = 0; + for (let miv = msgitemView; miv.parent; miv = miv.parent) { + threadIndent++; + } + // For threaded messages, we draw the subject/first-line indented, and with a + // charactering indicating the relationship. + // todo: show different arrow is message is a forward? we can tell by the message flag, it will likely be a message the user sent. + let threadChar = ''; + let threadCharTitle = ''; + if (msgitemView.parent) { + threadChar = '↳'; // Down-right arrow for direct response (reply/forward). + if (msgitemView.parent.messageitem.Message.MessageID === mi.Message.MessageID) { + // Approximately equal, for duplicate message-id, typically in Sent and incoming + // from mailing list or when sending to self. + threadChar = '≈'; + threadCharTitle = 'Same Message-ID.'; + } + else if (mi.Message.ThreadMissingLink || (mi.Message.ThreadParentIDs || []).length > 0 && (mi.Message.ThreadParentIDs || [])[0] !== msgitemView.parent.messageitem.Message.ID) { + // Zigzag arrow, e.g. if immediate parent is missing, or when matching was done + // based on subject. + threadChar = '↯'; + threadCharTitle = 'Immediate parent message is missing.'; + } + } + // Message is unread if it itself is unread, or it is collapsed and has an unread child message. + const isUnread = () => !mi.Message.Seen || msgitemView.isCollapsedThreadRoot() && !!msgitemView.findDescendant(miv => !miv.messageitem.Message.Seen); + const isRelevant = () => !mi.Message.ThreadMuted && mi.MatchQuery || (msgitemView.isCollapsedThreadRoot() && msgitemView.findDescendant(miv => !miv.messageitem.Message.ThreadMuted && miv.messageitem.MatchQuery)); + // Effective receive time to display. For collapsed thread roots, we show the time + // of the newest or oldest message, depending on whether you're viewing + // newest-first or oldest-first messages. + const received = () => { + let r = mi.Message.Received; + if (!msgitemView.isCollapsedThreadRoot()) { + return r; + } + msgitemView.descendants().forEach(dmiv => { + if (settings.orderAsc && dmiv.messageitem.Message.Received.getTime() < r.getTime()) { + r = dmiv.messageitem.Message.Received; + } + else if (!settings.orderAsc && dmiv.messageitem.Message.Received.getTime() > r.getTime()) { + r = dmiv.messageitem.Message.Received; + } + }); + return r; + }; + // For drawing half a thread bar for the last message in the thread. + const isThreadLast = () => { + let miv = msgitemView.threadRoot(); + while (miv.kids.length > 0) { + miv = miv.kids[miv.kids.length - 1]; + } + return miv === msgitemView; + }; + // If mailbox of message is not specified in filter (i.e. for a regular mailbox + // view, or search on a mailbox), we show it on the right-side of the subject. For + // collapsed thread roots, we show all additional mailboxes of descendants with + // different style. + const mailboxtags = []; + const mailboxIDs = new Set(); + const addMailboxTag = (mb, isCollapsedKid) => { + let name = mb.Name; + mailboxIDs.add(mb.ID); + if (name.length > 8 + 1 + 3 + 1 + 8 + 4) { + const t = name.split('/'); + const first = t[0]; + const last = t[t.length - 1]; + if (first.length + last.length <= 8 + 8) { + name = first + '/.../' + last; + } + else { + name = first.substring(0, 8) + '/.../' + last.substring(0, 8); + } + } + const e = dom.span(dom._class('msgitemmailbox'), isCollapsedKid ? dom._class('msgitemmailboxcollapsed') : [], name === mb.Name ? [] : attr.title(mb.Name), name); + mailboxtags.push(e); + }; + const othermb = otherMailbox(m.MailboxID); + if (othermb) { + addMailboxTag(othermb, false); + } + if (msgitemView.isCollapsedThreadRoot()) { + for (const miv of msgitemView.descendants()) { + const m = miv.messageitem.Message; + if (!mailboxIDs.has(m.MailboxID) && otherMailbox(m.MailboxID)) { + const mb = listMailboxes().find(mb => mb.ID === m.MailboxID); + if (!mb) { + throw new ConsistencyError('missing mailbox for message in thread'); + } + addMailboxTag(mb, true); + } + } + } + // When rerendering, we remember active & focus states. So we don't have to make + // the caller also call redraw on MsglistView. + const active = msgitemView.root && msgitemView.root.classList.contains('active'); + const focus = msgitemView.root && msgitemView.root.classList.contains('focus'); + const elem = dom.div(dom._class('msgitem'), active ? dom._class('active') : [], focus ? dom._class('focus') : [], attr.draggable('true'), function dragstart(e) { + // We send the Message.ID and MailboxID, so we can decide based on the destination + // mailbox whether to move. We don't move messages already in the destination + // mailbox, and also skip messages in the Sent mailbox when there are also messages + // from other mailboxes. + e.dataTransfer.setData('application/vnd.mox.messages', JSON.stringify(msglistView.selected().map(miv => [miv.messageitem.Message.MailboxID, miv.messageitem.Message.ID]))); + }, + // Thread root with kids can be collapsed/expanded with double click. + settings.threading !== api.ThreadMode.ThreadOff && !msgitemView.parent && msgitemView.kids.length > 0 ? + function dblclick(e) { + e.stopPropagation(); // Prevent selection. + if (settings.threading === api.ThreadMode.ThreadOn) { + // No await, we don't wait for the result. + withStatus('Saving thread expand/collapse', client.ThreadCollapse([msgitemView.messageitem.Message.ID], !msgitemView.collapsed)); + } + if (msgitemView.collapsed) { + msglistView.threadExpand(msgitemView); + } + else { + msglistView.threadCollapse(msgitemView); + msglistView.viewportEnsureMessages(); + } + } : [], isUnread() ? style({ fontWeight: 'bold' }) : [], + // Relevant means not muted and matching the query. + isRelevant() ? [] : style({ opacity: '.4' }), dom.div(dom._class('msgitemcell', 'msgitemflags'), dom.div(style({ display: 'flex', justifyContent: 'space-between' }), dom.div(flagList(msgitemView)), !msgitemView.parent && msgitemView.kids.length > 0 && msgitemView.collapsed ? + dom.clickbutton('' + (1 + msgitemView.descendants().length), attr.tabindex('-1'), attr.title('Expand thread.'), attr.arialabel('Expand thread.'), function click(e) { + e.stopPropagation(); // Prevent selection. + if (settings.threading === api.ThreadMode.ThreadOn) { + withStatus('Saving thread expanded', client.ThreadCollapse([msgitemView.messageitem.Message.ID], false)); + } + msglistView.threadExpand(msgitemView); + }) : [], !msgitemView.parent && msgitemView.kids.length > 0 && !msgitemView.collapsed ? + dom.clickbutton('-', style({ width: '1em' }), attr.tabindex('-1'), attr.title('Collapse thread.'), attr.arialabel('Collapse thread.'), function click(e) { + e.stopPropagation(); // Prevent selection. + if (settings.threading === api.ThreadMode.ThreadOn) { + withStatus('Saving thread expanded', client.ThreadCollapse([msgitemView.messageitem.Message.ID], true)); + } + msglistView.threadCollapse(msgitemView); + msglistView.viewportEnsureMessages(); + }) : [])), dom.div(dom._class('msgitemcell', 'msgitemfrom'), dom.div(style({ display: 'flex', justifyContent: 'space-between' }), dom.div(dom._class('msgitemfromtext', 'silenttitle'), + // todo: for collapsed messages, show all participants in thread? + attr.title((mi.Envelope.From || []).map(a => formatAddressFull(a)).join(', ')), join((mi.Envelope.From || []).map(a => formatAddressShort(a)), () => ', ')), identityHeader), + // Thread messages are connected by a vertical bar. The first and last message are + // only half the height of the item, to indicate start/end, and so it stands out + // from any thread above/below. + ((msgitemView.parent || msgitemView.kids.length > 0) && !msgitemView.threadRoot().collapsed) ? + dom.div(dom._class('msgitemfromthreadbar'), !msgitemView.parent ? style({ top: '50%', bottom: '-1px' }) : (isThreadLast() ? + style({ top: '-1px', bottom: '50%' }) : + style({ top: '-1px', bottom: '-1px' }))) : []), dom.div(dom._class('msgitemcell', 'msgitemsubject'), dom.div(style({ display: 'flex', justifyContent: 'space-between', position: 'relative' }), dom.div(dom._class('msgitemsubjecttext'), threadIndent > 0 ? dom.span(threadChar, style({ paddingLeft: (threadIndent / 2) + 'em', color: '#444', fontWeight: 'normal' }), threadCharTitle ? attr.title(threadCharTitle) : []) : [], msgitemView.parent ? [] : mi.Envelope.Subject || '(no subject)', dom.span(dom._class('msgitemsubjectsnippet'), ' ' + mi.FirstLine)), dom.div(keywords, mailboxtags))), dom.div(dom._class('msgitemcell', 'msgitemage'), age(received())), function click(e) { e.preventDefault(); e.stopPropagation(); msglistView.click(msgitemView, e.ctrlKey, e.shiftKey); }); + msgitemView.root.replaceWith(elem); + msgitemView.root = elem; }; const msgitemView = { root: dom.div(), messageitem: mi, - updateFlags: updateFlags, + receivedTime: receivedTime, + kids: [], + parent: null, + collapsed: initialCollapsed, + threadRoot: () => { + let miv = msgitemView; + while (miv.parent) { + miv = miv.parent; + } + return miv; + }, + isCollapsedThreadRoot: () => !msgitemView.parent && msgitemView.collapsed && msgitemView.kids.length > 0, + descendants: () => { + let l = []; + const walk = (miv) => { + for (const kmiv of miv.kids) { + l.push(kmiv); + walk(kmiv); + } + }; + walk(msgitemView); + return l; + }, + // We often just need to know if a descendant with certain properties exist. No + // need to create an array, then call find on it. + findDescendant: (matchfn) => { + const walk = (miv) => { + if (matchfn(miv)) { + return miv; + } + for (const kmiv of miv.kids) { + const r = walk(kmiv); + if (r) { + return r; + } + } + return null; + }; + return walk(msgitemView); + }, + lastDescendant: () => { + let l = msgitemView; + if (l.kids.length === 0) { + return null; + } + while (l.kids.length > 0) { + l = l.kids[l.kids.length - 1]; + } + return l; + }, remove: remove, + render: render, }; - msgitemView.root = render(); return msgitemView; }; // If attachmentView is open, keyboard shortcuts go there. @@ -2465,9 +2695,9 @@ const newMsgView = (miv, msglistView, listMailboxes, possibleLabels, messageLoad d: msglistView.cmdTrash, D: msglistView.cmdDelete, q: msglistView.cmdJunk, - n: msglistView.cmdMarkNotJunk, - u: msglistView.cmdMarkUnread, + Q: msglistView.cmdMarkNotJunk, m: msglistView.cmdMarkRead, + M: msglistView.cmdMarkUnread, }; let urlType; // text, html, htmlexternal; for opening in new tab/print let msgbuttonElem, msgheaderElem, msgattachmentElem, msgmodeElem; @@ -2489,8 +2719,10 @@ const newMsgView = (miv, msglistView, listMailboxes, possibleLabels, messageLoad popover(e.target, { transparent: true }, dom.div(style({ display: 'flex', flexDirection: 'column', gap: '.5ex', textAlign: 'right' }), [ dom.clickbutton('Print', attr.title('Print message, opens in new tab and opens print dialog.'), clickCmd(cmdPrint, shortcuts)), dom.clickbutton('Mark Not Junk', attr.title('Mark as not junk, causing this message to be used in spam classification of new incoming messages.'), clickCmd(msglistView.cmdMarkNotJunk, shortcuts)), - dom.clickbutton('Mark as read', clickCmd(msglistView.cmdMarkRead, shortcuts)), - dom.clickbutton('Mark as unread', clickCmd(msglistView.cmdMarkUnread, shortcuts)), + dom.clickbutton('Mark Read', clickCmd(msglistView.cmdMarkRead, shortcuts)), + dom.clickbutton('Mark Unread', clickCmd(msglistView.cmdMarkUnread, shortcuts)), + dom.clickbutton('Mute thread', clickCmd(msglistView.cmdMute, shortcuts)), + dom.clickbutton('Unmute thread', clickCmd(msglistView.cmdUnmute, shortcuts)), dom.clickbutton('Open in new tab', clickCmd(cmdOpenNewTab, shortcuts)), dom.clickbutton('Show raw original message in new tab', clickCmd(cmdOpenRaw, shortcuts)), dom.clickbutton('Show internals in popup', clickCmd(cmdShowInternals, shortcuts)), @@ -2719,23 +2951,48 @@ const newMsgView = (miv, msglistView, listMailboxes, possibleLabels, messageLoad })(); return mv; }; -const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, possibleLabels, scrollElemHeight, refineKeyword) => { - // These contain one msgitemView or an array of them. - // Zero or more selected msgitemViews. If there is a single message, its content is - // shown. If there are multiple, just the count is shown. These are in order of - // being added, not in order of how they are shown in the list. This is needed to - // handle selection changes with the shift key. +const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, possibleLabels, scrollElemHeight, refineKeyword, viewportEnsureMessages) => { + // msgitemViews holds all visible item views: All thread roots, and kids only if + // the thread is expanded, in order of descendants. All descendants of a collapsed + // root are in collapsedMsgitemViews, unsorted. Having msgitemViews as a list is + // convenient for reasoning about the visible items, and handling changes to the + // selected messages. + // When messages for a thread are all non-matching the query, we no longer show it + // (e.g. when moving a thread to Archive), but we keep the messages around in + // oldThreadMessageItems, so an update to the thread (e.g. new delivery) can + // resurrect the messages. + let msgitemViews = []; // Only visible msgitems, in order on screen. + let collapsedMsgitemViews = []; // Invisible messages because collapsed, unsorted. + let oldThreadMessageItems = []; // Messages from threads removed from view. + // selected holds the messages that are selected, zero or more. If there is a + // single message, its content is shown. If there are multiple, just the count is + // shown. These are in order of being added, not in order of how they are shown in + // the list. This is needed to handle selection changes with the shift key. For + // collapsed thread roots, only that root will be in this list. The effective + // selection must always expand descendants, use mlv.selected() to gather all. let selected = []; - // MsgitemView last interacted with, or the first when messages are loaded. Always - // set when there is a message. Used for shift+click to expand selection. + // Focus is the message last interacted with, or the first when messages are + // loaded. Always set when there is a message. Used for shift+click to expand + // selection. let focus = null; - let msgitemViews = []; let msgView = null; + // Messages for actions like "archive", "trash", "move to...". We skip messages + // that are (already) in skipMBID. And we skip messages that are in the designated + // Sent mailbox, unless there is only one selected message or the view is for the + // Sent mailbox, then it must be intentional. + const moveActionMsgIDs = (skipMBID) => { + const sentMailboxID = listMailboxes().find(mb => mb.Sent)?.ID; + const effselected = mlv.selected(); + return effselected + .filter(miv => miv.messageitem.Message.MailboxID !== skipMBID) + .map(miv => miv.messageitem.Message) + .filter(m => effselected.length === 1 || !sentMailboxID || m.MailboxID !== sentMailboxID || !otherMailbox(sentMailboxID)) + .map(m => m.ID); + }; const cmdArchive = async () => { const mb = listMailboxes().find(mb => mb.Archive); if (mb) { - const msgIDs = selected.filter(miv => miv.messageitem.Message.MailboxID !== mb.ID).map(miv => miv.messageitem.Message.ID); - await withStatus('Moving to archive mailbox', client.MessageMove(msgIDs, mb.ID)); + await withStatus('Moving to archive mailbox', client.MessageMove(moveActionMsgIDs(mb.ID), mb.ID)); } else { window.alert('No mailbox configured for archiving yet.'); @@ -2745,13 +3002,12 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p if (!confirm('Are you sure you want to permanently delete?')) { return; } - await withStatus('Permanently deleting messages', client.MessageDelete(selected.map(miv => miv.messageitem.Message.ID))); + await withStatus('Permanently deleting messages', client.MessageDelete(mlv.selected().map(miv => miv.messageitem.Message.ID))); }; const cmdTrash = async () => { const mb = listMailboxes().find(mb => mb.Trash); if (mb) { - const msgIDs = selected.filter(miv => miv.messageitem.Message.MailboxID !== mb.ID).map(miv => miv.messageitem.Message.ID); - await withStatus('Moving to trash mailbox', client.MessageMove(msgIDs, mb.ID)); + await withStatus('Moving to trash mailbox', client.MessageMove(moveActionMsgIDs(mb.ID), mb.ID)); } else { window.alert('No mailbox configured for trash yet.'); @@ -2760,30 +3016,211 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p const cmdJunk = async () => { const mb = listMailboxes().find(mb => mb.Junk); if (mb) { - const msgIDs = selected.filter(miv => miv.messageitem.Message.MailboxID !== mb.ID).map(miv => miv.messageitem.Message.ID); - await withStatus('Moving to junk mailbox', client.MessageMove(msgIDs, mb.ID)); + await withStatus('Moving to junk mailbox', client.MessageMove(moveActionMsgIDs(mb.ID), mb.ID)); } else { window.alert('No mailbox configured for junk yet.'); } }; - const cmdMarkNotJunk = async () => { await withStatus('Marking as not junk', client.FlagsAdd(selected.map(miv => miv.messageitem.Message.ID), ['$notjunk'])); }; - const cmdMarkRead = async () => { await withStatus('Marking as read', client.FlagsAdd(selected.map(miv => miv.messageitem.Message.ID), ['\\seen'])); }; - const cmdMarkUnread = async () => { await withStatus('Marking as not read', client.FlagsClear(selected.map(miv => miv.messageitem.Message.ID), ['\\seen'])); }; + const cmdMarkNotJunk = async () => { await withStatus('Marking as not junk', client.FlagsAdd(mlv.selected().map(miv => miv.messageitem.Message.ID), ['$notjunk'])); }; + const cmdMarkRead = async () => { await withStatus('Marking as read', client.FlagsAdd(mlv.selected().map(miv => miv.messageitem.Message.ID), ['\\seen'])); }; + const cmdMarkUnread = async () => { await withStatus('Marking as not read', client.FlagsClear(mlv.selected().map(miv => miv.messageitem.Message.ID), ['\\seen'])); }; + const cmdMute = async () => { + const l = mlv.selected(); + await withStatus('Muting thread', client.ThreadMute(l.map(miv => miv.messageitem.Message.ID), true)); + const oldstate = state(); + for (const miv of l) { + if (!miv.parent && miv.kids.length > 0 && !miv.collapsed) { + threadCollapse(miv, false); + } + } + updateState(oldstate); + viewportEnsureMessages(); + }; + const cmdUnmute = async () => { await withStatus('Unmuting thread', client.ThreadMute(mlv.selected().map(miv => miv.messageitem.Message.ID), false)); }; + const seletedRoots = () => { + const mivs = []; + mlv.selected().forEach(miv => { + const mivroot = miv.threadRoot(); + if (!mivs.includes(mivroot)) { + mivs.push(mivroot); + } + }); + return mivs; + }; + const cmdToggleMute = async () => { + if (settings.threading === api.ThreadMode.ThreadOff) { + alert('Toggle muting threads is only available when threading is enabled.'); + return; + } + const rootmivs = seletedRoots(); + const unmuted = !!rootmivs.find(miv => !miv.messageitem.Message.ThreadMuted); + await withStatus(unmuted ? 'Muting' : 'Unmuting', client.ThreadMute(rootmivs.map(miv => miv.messageitem.Message.ID), unmuted ? true : false)); + if (unmuted) { + const oldstate = state(); + rootmivs.forEach(miv => { + if (!miv.collapsed) { + threadCollapse(miv, false); + } + }); + updateState(oldstate); + viewportEnsureMessages(); + } + }; + const cmdToggleCollapse = async () => { + if (settings.threading === api.ThreadMode.ThreadOff) { + alert('Toggling thread collapse/expand is only available when threading is enabled.'); + return; + } + const rootmivs = seletedRoots(); + const collapse = !!rootmivs.find(miv => !miv.collapsed); + const oldstate = state(); + if (collapse) { + rootmivs.forEach(miv => { + if (!miv.collapsed) { + threadCollapse(miv, false); + } + }); + selected = rootmivs; + if (focus) { + focus = focus.threadRoot(); + } + viewportEnsureMessages(); + } + else { + rootmivs.forEach(miv => { + if (miv.collapsed) { + threadExpand(miv, false); + } + }); + } + updateState(oldstate); + if (settings.threading === api.ThreadMode.ThreadOn) { + const action = collapse ? 'Collapsing' : 'Expanding'; + await withStatus(action, client.ThreadCollapse(rootmivs.map(miv => miv.messageitem.Message.ID), collapse)); + } + }; + const cmdSelectThread = async () => { + if (!focus) { + return; + } + const oldstate = state(); + selected = msgitemViews.filter(miv => miv.messageitem.Message.ThreadID === focus.messageitem.Message.ThreadID); + updateState(oldstate); + }; const shortcuts = { d: cmdTrash, Delete: cmdTrash, D: cmdDelete, - q: cmdJunk, a: cmdArchive, - n: cmdMarkNotJunk, - u: cmdMarkUnread, + q: cmdJunk, + Q: cmdMarkNotJunk, m: cmdMarkRead, + M: cmdMarkUnread, + X: cmdToggleMute, + C: cmdToggleCollapse, + S: cmdSelectThread, + }; + // After making changes, this function looks through the data structure for + // inconsistencies. Useful during development. + const checkConsistency = (checkSelection) => { + if (!settings.checkConsistency) { + return; + } + // Check for duplicates in msgitemViews. + const mivseen = new Set(); + const threadActive = new Set(); + for (const miv of msgitemViews) { + const id = miv.messageitem.Message.ID; + if (mivseen.has(id)) { + log('duplicate Message.ID', { id: id, mivseenSize: mivseen.size }); + throw new ConsistencyError('duplicate Message.ID in msgitemViews'); + } + mivseen.add(id); + if (!miv.root.parentNode) { + throw new ConsistencyError('msgitemView.root not in dom'); + } + threadActive.add(miv.messageitem.Message.ThreadID); + } + // Check for duplicates in collapsedMsgitemViews, and whether also in msgitemViews. + const colseen = new Set(); + for (const miv of collapsedMsgitemViews) { + const id = miv.messageitem.Message.ID; + if (colseen.has(id)) { + throw new ConsistencyError('duplicate Message.ID in collapsedMsgitemViews'); + } + colseen.add(id); + if (mivseen.has(id)) { + throw new ConsistencyError('Message.ID in both collapsedMsgitemViews and msgitemViews'); + } + threadActive.add(miv.messageitem.Message.ThreadID); + } + if (settings.threading !== api.ThreadMode.ThreadOff) { + const oldseen = new Set(); + for (const mi of oldThreadMessageItems) { + const id = mi.Message.ID; + if (oldseen.has(id)) { + throw new ConsistencyError('duplicate Message.ID in oldThreadMessageItems'); + } + oldseen.add(id); + if (mivseen.has(id)) { + throw new ConsistencyError('Message.ID in both msgitemViews and oldThreadMessageItems'); + } + if (colseen.has(id)) { + throw new ConsistencyError('Message.ID in both collapsedMsgitemViews and oldThreadMessageItems'); + } + if (threadActive.has(mi.Message.ThreadID)) { + throw new ConsistencyError('threadid both in active and in old thread list'); + } + } + } + // Walk all (collapsed) msgitemViews, check each and their descendants are in + // msgitemViews at the correct position, or in collapsedmsgitemViews. + msgitemViews.forEach((miv, i) => { + if (miv.collapsed) { + for (const dmiv of miv.descendants()) { + if (!colseen.has(dmiv.messageitem.Message.ID)) { + throw new ConsistencyError('descendant message id missing from collapsedMsgitemViews'); + } + } + return; + } + for (const dmiv of miv.descendants()) { + i++; + if (!mivseen.has(dmiv.messageitem.Message.ID)) { + throw new ConsistencyError('descendant missing from msgitemViews'); + } + if (msgitemViews[i] !== dmiv) { + throw new ConsistencyError('descendant not at expected position in msgitemViews'); + } + } + }); + if (!checkSelection) { + return; + } + // Check all selected & focus exists. + const selseen = new Set(); + for (const miv of selected) { + const id = miv.messageitem.Message.ID; + if (selseen.has(id)) { + throw new ConsistencyError('duplicate miv in selected'); + } + selseen.add(id); + if (!mivseen.has(id)) { + throw new ConsistencyError('selected id not in msgitemViews'); + } + } + if (focus) { + const id = focus.messageitem.Message.ID; + if (!mivseen.has(id)) { + throw new ConsistencyError('focus set to unknown miv'); + } + } }; // Return active & focus state, and update the UI after changing state. const state = () => { const active = {}; - for (const miv of selected) { + for (const miv of mlv.selected()) { active[miv.messageitem.Message.ID] = miv; } return { active: active, focus: focus }; @@ -2813,38 +3250,40 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p activeChanged = true; } } - if (initial && selected.length === 1) { - mlv.redraw(selected[0]); + const effselected = mlv.selected(); + if (initial && effselected.length === 1) { + mlv.redraw(effselected[0]); } - if (activeChanged) { - if (msgView) { - msgView.aborter.abort(); - } - msgView = null; - if (selected.length === 0) { - dom._kids(msgElem); - } - else if (selected.length === 1) { - msgElem.classList.toggle('loading', true); - const loaded = () => { msgElem.classList.toggle('loading', false); }; - msgView = newMsgView(selected[0], mlv, listMailboxes, possibleLabels, loaded, refineKeyword, parsedMessageOpt); - dom._kids(msgElem, msgView); - } - else { - const trashMailboxID = listMailboxes().find(mb => mb.Trash)?.ID; - const allTrash = trashMailboxID && !selected.find(miv => miv.messageitem.Message.MailboxID !== trashMailboxID); - dom._kids(msgElem, dom.div(attr.role('region'), attr.arialabel('Buttons for multiple messages'), style({ position: 'absolute', top: 0, right: 0, bottom: 0, left: 0, display: 'flex', alignItems: 'center', justifyContent: 'center' }), dom.div(style({ padding: '4ex', backgroundColor: 'white', borderRadius: '.25em', border: '1px solid #ccc' }), dom.div(style({ textAlign: 'center', marginBottom: '4ex' }), '' + selected.length + ' messages selected'), dom.div(dom.clickbutton('Archive', attr.title('Move to the Archive mailbox.'), clickCmd(cmdArchive, shortcuts)), ' ', allTrash ? - dom.clickbutton('Delete', attr.title('Permanently delete messages.'), clickCmd(cmdDelete, shortcuts)) : - dom.clickbutton('Trash', attr.title('Move to the Trash mailbox.'), clickCmd(cmdTrash, shortcuts)), ' ', dom.clickbutton('Junk', attr.title('Move to Junk mailbox, marking as junk and causing this message to be used in spam classification of new incoming messages.'), clickCmd(cmdJunk, shortcuts)), ' ', dom.clickbutton('Move to...', function click(e) { - movePopover(e, listMailboxes(), selected.map(miv => miv.messageitem.Message)); - }), ' ', dom.clickbutton('Labels...', attr.title('Add/remove labels ...'), function click(e) { - labelsPopover(e, selected.map(miv => miv.messageitem.Message), possibleLabels); - }), ' ', dom.clickbutton('Mark Not Junk', attr.title('Mark as not junk, causing this message to be used in spam classification of new incoming messages.'), clickCmd(cmdMarkNotJunk, shortcuts)), ' ', dom.clickbutton('Mark read', clickCmd(cmdMarkRead, shortcuts)), ' ', dom.clickbutton('Mark unread', clickCmd(cmdMarkUnread, shortcuts)))))); - } + checkConsistency(true); + if (!activeChanged) { + return; } - if (activeChanged) { - setLocationHash(); + if (msgView) { + msgView.aborter.abort(); } + msgView = null; + if (effselected.length === 0) { + dom._kids(msgElem); + } + else if (effselected.length === 1) { + msgElem.classList.toggle('loading', true); + const loaded = () => { msgElem.classList.toggle('loading', false); }; + msgView = newMsgView(effselected[0], mlv, listMailboxes, possibleLabels, loaded, refineKeyword, parsedMessageOpt); + dom._kids(msgElem, msgView); + } + else { + const trashMailboxID = listMailboxes().find(mb => mb.Trash)?.ID; + const allTrash = trashMailboxID && !effselected.find(miv => miv.messageitem.Message.MailboxID !== trashMailboxID); + dom._kids(msgElem, dom.div(attr.role('region'), attr.arialabel('Buttons for multiple messages'), style({ position: 'absolute', top: 0, right: 0, bottom: 0, left: 0, display: 'flex', alignItems: 'center', justifyContent: 'center' }), dom.div(style({ padding: '4ex', backgroundColor: 'white', borderRadius: '.25em', border: '1px solid #ccc' }), dom.div(style({ textAlign: 'center', marginBottom: '4ex' }), '' + effselected.length + ' messages selected'), dom.div(dom.clickbutton('Archive', attr.title('Move to the Archive mailbox. Messages in the designated Sent mailbox are only moved if a single message is selected, or the current mailbox is the Sent mailbox.'), clickCmd(cmdArchive, shortcuts)), ' ', allTrash ? + dom.clickbutton('Delete', attr.title('Permanently delete messages.'), clickCmd(cmdDelete, shortcuts)) : + dom.clickbutton('Trash', attr.title('Move to the Trash mailbox. Messages in the designated Sent mailbox are only moved if a single message is selected, or the current mailbox is the Sent mailbox.'), clickCmd(cmdTrash, shortcuts)), ' ', dom.clickbutton('Junk', attr.title('Move to Junk mailbox, marking as junk and causing this message to be used in spam classification of new incoming messages. Messages in the designated Sent mailbox are only moved if a single message is selected, or the current mailbox is the Sent mailbox.'), clickCmd(cmdJunk, shortcuts)), ' ', dom.clickbutton('Move to...', function click(e) { + const sentMailboxID = listMailboxes().find(mb => mb.Sent)?.ID; + movePopover(e, listMailboxes(), effselected.map(miv => miv.messageitem.Message).filter(m => effselected.length === 1 || !sentMailboxID || m.MailboxID !== sentMailboxID || !otherMailbox(sentMailboxID))); + }), ' ', dom.clickbutton('Labels...', attr.title('Add/remove labels ...'), function click(e) { + labelsPopover(e, effselected.map(miv => miv.messageitem.Message), possibleLabels); + }), ' ', dom.clickbutton('Mark Not Junk', attr.title('Mark as not junk, causing this message to be used in spam classification of new incoming messages.'), clickCmd(cmdMarkNotJunk, shortcuts)), ' ', dom.clickbutton('Mark Read', clickCmd(cmdMarkRead, shortcuts)), ' ', dom.clickbutton('Mark Unread', clickCmd(cmdMarkUnread, shortcuts)), ' ', dom.clickbutton('Mute thread', clickCmd(cmdMute, shortcuts)), ' ', dom.clickbutton('Unmute thread', clickCmd(cmdUnmute, shortcuts)))))); + } + setLocationHash(); }; // Moves the currently focused msgitemView, without changing selection. const moveFocus = (miv) => { @@ -2852,95 +3291,682 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p focus = miv; updateState(oldstate); }; + const threadExpand = (miv, changeState) => { + if (miv.parent) { + throw new ConsistencyError('cannot expand non-root'); + } + const oldstate = state(); + miv.collapsed = false; + const mivl = miv.descendants(); + miv.render(); + mivl.forEach(dmiv => dmiv.render()); + for (const miv of mivl) { + collapsedMsgitemViews.splice(collapsedMsgitemViews.indexOf(miv), 1); + } + const pi = msgitemViews.indexOf(miv); + msgitemViews.splice(pi + 1, 0, ...mivl); + const next = miv.root.nextSibling; + for (const miv of mivl) { + mlv.root.insertBefore(miv.root, next); + } + if (changeState) { + updateState(oldstate); + } + }; + const threadCollapse = (miv, changeState) => { + if (miv.parent) { + throw new ConsistencyError('cannot expand non-root'); + } + const oldstate = state(); + miv.collapsed = true; + const mivl = miv.descendants(); + collapsedMsgitemViews.push(...mivl); + // If miv or any child was selected, ensure collapsed thread root is also selected. + let select = [miv, ...mivl].find(xmiv => selected.indexOf(xmiv) >= 0); + let seli = selected.length; // Track first index of already selected miv, which is where we insert the thread root if needed, to keep order. + msgitemViews.splice(msgitemViews.indexOf(miv) + 1, mivl.length); + for (const dmiv of mivl) { + dmiv.remove(); + if (focus === dmiv) { + focus = miv; + } + const si = selected.indexOf(dmiv); + if (si >= 0) { + if (si < seli) { + seli = si; + } + selected.splice(si, 1); + } + } + if (select) { + const si = selected.indexOf(miv); + if (si < 0) { + selected.splice(seli, 0, miv); + } + } + // Selected messages may have changed. + if (changeState) { + updateState(oldstate); + } + // Render remaining thread root, with tree size, effective received age/unread state. + miv.render(); + }; + const threadToggle = () => { + const oldstate = state(); + const roots = msgitemViews.filter(miv => !miv.parent && miv.kids.length > 0); + roots.forEach(miv => { + let wantCollapsed = miv.messageitem.Message.ThreadCollapsed; + if (settings.threading === api.ThreadMode.ThreadUnread) { + wantCollapsed = !miv.messageitem.Message.Seen && !miv.findDescendant(miv => !miv.messageitem.Message.Seen); + } + if (miv.collapsed === wantCollapsed) { + return; + } + if (wantCollapsed) { + threadCollapse(miv, false); + } + else { + threadExpand(miv, false); + } + }); + updateState(oldstate); + viewportEnsureMessages(); + }; + const removeSelected = (miv) => { + const si = selected.indexOf(miv); + if (si >= 0) { + selected.splice(si, 1); + } + if (focus === miv) { + const i = msgitemViews.indexOf(miv); + if (i > 0) { + focus = msgitemViews[i - 1]; + } + else if (i + 1 < msgitemViews.length) { + focus = msgitemViews[i + 1]; + } + else { + focus = null; + } + } + }; + // Removes message from either msgitemViews, collapsedMsgitemViews, + // oldThreadMessageItems, and updates UI. + // Returns ThreadID of removed message if active (expanded or collapsed), or 0 otherwise. + const removeUID = (mailboxID, uid) => { + const match = (miv) => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid; + const ci = collapsedMsgitemViews.findIndex(match); + if (ci >= 0) { + const miv = collapsedMsgitemViews[ci]; + removeCollapsed(ci); + return miv.messageitem.Message.ThreadID; + } + const i = msgitemViews.findIndex(match); + if (i >= 0) { + const miv = msgitemViews[i]; + removeExpanded(i); + return miv.messageitem.Message.ThreadID; + } + const ti = oldThreadMessageItems.findIndex(mi => mi.Message.MailboxID === mailboxID && mi.Message.UID === uid); + if (ti >= 0) { + oldThreadMessageItems.splice(ti, 1); + } + return 0; + }; + // Removes message from collapsedMsgitemView and UI at given index, placing + // messages in oldThreadMessageItems. + const removeCollapsed = (ci) => { + // Message is collapsed. That means it isn't visible, and neither are its children, + // and it has a parent. So we just merge the kids with those of the parent. + const miv = collapsedMsgitemViews[ci]; + collapsedMsgitemViews.splice(ci, 1); + removeSelected(miv); + const trmiv = miv.threadRoot(); // To rerender, below. + const pmiv = miv.parent; + if (!pmiv) { + throw new ConsistencyError('removing collapsed miv, but has no parent'); + } + miv.parent = null; // Strict cleanup. + const pki = pmiv.kids.indexOf(miv); + if (pki < 0) { + throw new ConsistencyError('miv not in parent.kids'); + } + pmiv.kids.splice(pki, 1, ...miv.kids); // In parent, replace miv with its kids. + miv.kids.forEach(kmiv => kmiv.parent = pmiv); // Give kids their new parent. + miv.kids = []; // Strict cleanup. + pmiv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()); // Sort new list of kids. + trmiv.render(); // For count, unread state. + return; + }; + // Remove message from msgitemViews and UI at the index i. + const removeExpanded = (i) => { + log('removeExpanded', { i }); + // Note: If we remove a message we may be left with only messages from another + // mailbox. We'll leave it, new messages could be delivered for that thread. It + // would be strange to see the remaining messages of the thread disappear. + const miv = msgitemViews[i]; + removeSelected(miv); + const pmiv = miv.parent; + miv.parent = null; + if (miv.kids.length === 0) { + // No kids, easy case, just remove this leaf message. + miv.remove(); + msgitemViews.splice(i, 1); + if (pmiv) { + const pki = pmiv.kids.indexOf(miv); + if (pki < 0) { + throw new ConsistencyError('miv not in parent.kids'); + } + pmiv.kids.splice(pki, 1); // Remove miv from parent's kids. + miv.parent = null; // Strict cleanup. + pmiv.render(); // Update counts. + } + return; + } + if (!pmiv) { + // If the kids no longer have a parent and become thread roots we leave them in + // their original location. + const next = miv.root.nextSibling; + miv.remove(); + msgitemViews.splice(i, 1); + if (miv.collapsed) { + msgitemViews.splice(i, 0, ...miv.kids); + for (const kmiv of miv.kids) { + const pki = collapsedMsgitemViews.indexOf(kmiv); + if (pki < 0) { + throw new ConsistencyError('cannot find collapsed kid in collapsedMsgitemViews'); + } + collapsedMsgitemViews.splice(pki, 1); + kmiv.collapsed = true; + kmiv.parent = null; + kmiv.render(); + mlv.root.insertBefore(kmiv.root, next); + } + } + else { + // Note: if not collapsed, we leave the kids in the original position in msgitemViews. + miv.kids.forEach(kmiv => { + kmiv.collapsed = false; + kmiv.parent = null; + kmiv.render(); + const lastDesc = kmiv.lastDescendant(); + if (lastDesc) { + // Update end of thread bar. + lastDesc.render(); + } + }); + } + miv.kids = []; // Strict cleanup. + return; + } + // If the kids will have a parent, we insert them at the expected location in + // between parent's existing kids. It is easiest just to take out all kids, add the + // new ones, sort kids, and add back the subtree. + const odmivs = pmiv.descendants(); // Old direct descendants of parent. This includes miv and kids, and other kids, and miv siblings. + const pi = msgitemViews.indexOf(pmiv); + if (pi < 0) { + throw new ConsistencyError('cannot find parent of removed miv'); + } + msgitemViews.splice(pi + 1, odmivs.length); // Remove all old descendants, we'll add an updated list later. + const pki = pmiv.kids.indexOf(miv); + if (pki < 0) { + throw new Error('did not find miv in parent.kids'); + } + pmiv.kids.splice(pki, 1); // Remove miv from parent's kids. + pmiv.kids.push(...miv.kids); // Add miv.kids to parent's kids. + miv.kids.forEach(kmiv => { kmiv.parent = pmiv; }); // Set new parent for miv kids. + miv.kids = []; // Strict cleanup. + pmiv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()); + const ndmivs = pmiv.descendants(); // Excludes miv, that we are removing. + if (ndmivs.length !== odmivs.length - 1) { + throw new ConsistencyError('unexpected new descendants counts during remove'); + } + msgitemViews.splice(pi + 1, 0, ...ndmivs); // Add all new/current descedants. There is one less than in odmivs. + odmivs.forEach(ndimv => ndimv.remove()); + const next = pmiv.root.nextSibling; + for (const ndmiv of ndmivs) { + mlv.root.insertBefore(ndmiv.root, next); + } + pmiv.render(); + ndmivs.forEach(dmiv => dmiv.render()); + }; + // If there are no query-matching messages left for this thread, remove the + // remaining messages from view and keep them around for future deliveries for the + // thread. + const possiblyTakeoutOldThreads = (threadIDs) => { + const hasMatch = (mivs, threadID) => mivs.find(miv => miv.messageitem.Message.ThreadID === threadID && miv.messageitem.MatchQuery); + const takeoutOldThread = (mivs, threadID, visible) => { + let i = 0; + while (i < mivs.length) { + const miv = mivs[i]; + const mi = miv.messageitem; + const m = mi.Message; + if (threadID !== m.ThreadID) { + i++; + continue; + } + mivs.splice(i, 1); + if (visible) { + miv.remove(); + } + if (focus === miv) { + focus = null; + if (i < mivs.length) { + focus = mivs[i]; + } + else if (i > 0) { + focus = mivs[i - 1]; + } + const si = selected.indexOf(miv); + if (si >= 0) { + selected.splice(si, 1); + } + } + // Strict cleanup. + miv.parent = null; + miv.kids = []; + oldThreadMessageItems.push(mi); + log('took out old thread message', { mi }); + } + }; + for (const threadID of threadIDs) { + if (hasMatch(msgitemViews, threadID) || hasMatch(collapsedMsgitemViews, threadID)) { + log('still have query-matching message for thread', { threadID }); + continue; + } + takeoutOldThread(msgitemViews, threadID, true); + takeoutOldThread(collapsedMsgitemViews, threadID, false); + } + }; const mlv = { root: dom.div(), updateFlags: (mailboxID, uid, modseq, mask, flags, keywords) => { + const updateMessageFlags = (m) => { + m.ModSeq = modseq; + const maskobj = mask; + const flagsobj = flags; + const mobj = m; + for (const k in maskobj) { + if (maskobj[k]) { + mobj[k] = flagsobj[k]; + } + } + m.Keywords = keywords; + }; // todo optimize: keep mapping of uid to msgitemView for performance. instead of using Array.find - const miv = msgitemViews.find(miv => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid); + let miv = msgitemViews.find(miv => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid); if (!miv) { - // Happens for messages outside of view. - log('could not find msgitemView for uid', uid); + miv = collapsedMsgitemViews.find(miv => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid); + } + if (miv) { + updateMessageFlags(miv.messageitem.Message); + miv.render(); + if (miv.parent) { + const tr = miv.threadRoot(); + if (tr.collapsed) { + tr.render(); + } + } + if (msgView && msgView.messageitem.Message.ID === miv.messageitem.Message.ID) { + msgView.updateKeywords(modseq, keywords); + } return; } - miv.updateFlags(modseq, mask, flags, keywords); - if (msgView && msgView.messageitem.Message.ID === miv.messageitem.Message.ID) { - msgView.updateKeywords(modseq, keywords); + const mi = oldThreadMessageItems.find(mi => mi.Message.MailboxID === mailboxID && mi.Message.UID === uid); + if (mi) { + updateMessageFlags(mi.Message); + } + else { + // Happens for messages outside of view. + log('could not find msgitemView for uid', uid); } }, - addMessageItems: (messageItems) => { + // Add messages to view, either messages to fill the view with complete threads, or + // individual messages delivered later. + addMessageItems: (messageItems, isChange, requestMsgID) => { if (messageItems.length === 0) { return; } - messageItems.forEach(mi => { - const miv = newMsgitemView(mi, mlv, otherMailbox(mi.Message.MailboxID)); - const orderNewest = !settings.orderAsc; - const tm = mi.Message.Received.getTime(); - const nextmivindex = msgitemViews.findIndex(miv => { - const vtm = miv.messageitem.Message.Received.getTime(); - return orderNewest && vtm <= tm || !orderNewest && tm <= vtm; - }); - if (nextmivindex < 0) { - mlv.root.appendChild(miv.root); - msgitemViews.push(miv); + // Each "mil" is a thread, possibly with multiple thread roots. The thread may + // already be present. + messageItems.forEach(mil => { + if (!mil) { + return; // For types, should not happen. + } + const threadID = mil[0].Message.ThreadID; + const hasMatch = !!mil.find(mi => mi.MatchQuery); + if (hasMatch) { + // This may be a message for a thread that had query-matching matches at some + // point, but then no longer, causing its messages to have been moved to + // oldThreadMessageItems. We add back those messages. + let i = 0; + while (i < oldThreadMessageItems.length) { + const omi = oldThreadMessageItems[i]; + if (omi.Message.ThreadID === threadID) { + oldThreadMessageItems.splice(i, 1); + if (!mil.find(mi => mi.Message.ID === omi.Message.ID)) { + mil.push(omi); + log('resurrected old message'); + } + else { + log('dropped old thread message'); + } + } + else { + i++; + } + } } else { - mlv.root.insertBefore(miv.root, msgitemViews[nextmivindex].root); - msgitemViews.splice(nextmivindex, 0, miv); + // New message(s) are not matching query. If there are no "active" messages for + // this thread, update/add oldThreadMessageItems. + const match = (miv) => miv.messageitem.Message.ThreadID === threadID; + if (!msgitemViews.find(match) && !collapsedMsgitemViews.find(match)) { + log('adding new message(s) to oldTheadMessageItems'); + for (const mi of mil) { + const ti = oldThreadMessageItems.findIndex(tmi => tmi.Message.ID === mi.Message.ID); + if (ti) { + oldThreadMessageItems[ti] = mi; + } + else { + oldThreadMessageItems.push(mi); + } + } + return; + } + } + if (isChange) { + // This could be an "add" for a message from another mailbox that we are already + // displaying because of threads. If so, it may have new properties such as the + // mailbox, so update it. + const threadIDs = new Set(); + let i = 0; + while (i < mil.length) { + const mi = mil[i]; + let miv = msgitemViews.find(miv => miv.messageitem.Message.ID === mi.Message.ID); + if (!miv) { + miv = collapsedMsgitemViews.find(miv => miv.messageitem.Message.ID === mi.Message.ID); + } + if (miv) { + miv.messageitem = mi; + miv.render(); + mil.splice(i, 1); + miv.threadRoot().render(); + threadIDs.add(mi.Message.ThreadID); + } + else { + i++; + } + } + log('processed changes for messages with thread', { threadIDs, mil }); + if (mil.length === 0) { + const oldstate = state(); + possiblyTakeoutOldThreads(threadIDs); + updateState(oldstate); + return; + } + } + // Find effective receive time for messages. We'll insert at that point. + let receivedTime = mil[0].Message.Received.getTime(); + const tmiv = msgitemViews.find(miv => miv.messageitem.Message.ThreadID === mil[0].Message.ThreadID); + if (tmiv) { + receivedTime = tmiv.receivedTime; + } + else { + for (const mi of mil) { + const t = mi.Message.Received.getTime(); + if (settings.orderAsc && t < receivedTime || !settings.orderAsc && t > receivedTime) { + receivedTime = t; + } + } + } + // Create new MsgitemViews. + const m = new Map(); + for (const mi of mil) { + m.set(mi.Message.ID, newMsgitemView(mi, mlv, otherMailbox, listMailboxes, receivedTime, false)); + } + // Assign miv's to parents or add them to the potential roots. + let roots = []; + if (settings.threading === api.ThreadMode.ThreadOff) { + roots = [...m.values()]; + } + else { + nextmiv: for (const [_, miv] of m) { + for (const pid of (miv.messageitem.Message.ThreadParentIDs || [])) { + const pmiv = m.get(pid); + if (pmiv) { + pmiv.kids.push(miv); + miv.parent = pmiv; + continue nextmiv; + } + } + roots.push(miv); + } + } + // Ensure all kids are properly sorted, always ascending by time received. + for (const [_, miv] of m) { + miv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()); + } + // Add the potential roots as kids to existing parents, if they exist. Only with threading enabled. + if (settings.threading !== api.ThreadMode.ThreadOff) { + nextroot: for (let i = 0; i < roots.length;) { + const miv = roots[i]; + for (const pid of (miv.messageitem.Message.ThreadParentIDs || [])) { + const pi = msgitemViews.findIndex(xmiv => xmiv.messageitem.Message.ID === pid); + let parentmiv; + let collapsed; + if (pi >= 0) { + parentmiv = msgitemViews[pi]; + collapsed = parentmiv.collapsed; + log('found parent', { pi }); + } + else { + parentmiv = collapsedMsgitemViews.find(xmiv => xmiv.messageitem.Message.ID === pid); + collapsed = true; + } + if (!parentmiv) { + log('no parentmiv', pid); + continue; + } + const trmiv = parentmiv.threadRoot(); + if (collapsed !== trmiv.collapsed) { + log('collapsed mismatch', { collapsed: collapsed, 'trmiv.collapsed': trmiv.collapsed, trmiv: trmiv }); + throw new ConsistencyError('mismatch between msgitemViews/collapsedMsgitemViews and threadroot collapsed'); + } + let prevLastDesc = null; + if (!trmiv.collapsed) { + // Remove current parent, we'll insert again after linking parent/kids. + const ndesc = parentmiv.descendants().length; + log('removing descendants temporarily', { ndesc }); + prevLastDesc = parentmiv.lastDescendant(); + msgitemViews.splice(pi + 1, ndesc); + } + // Link parent & kid, sort kids. + miv.parent = parentmiv; + parentmiv.kids.push(miv); + parentmiv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()); + if (trmiv.collapsed) { + // Thread root is collapsed. + collapsedMsgitemViews.push(miv, ...miv.descendants()); + // Ensure mivs have a root. + miv.render(); + miv.descendants().forEach(miv => miv.render()); + // Update count/unread status. + trmiv.render(); + } + else { + const desc = parentmiv.descendants(); + log('inserting parent descendants again', { pi, desc }); + msgitemViews.splice(pi + 1, 0, ...desc); // We had removed the old tree, now adding the updated tree. + // Insert at correct position in dom. + const i = msgitemViews.indexOf(miv); + if (i < 0) { + throw new ConsistencyError('cannot find miv just inserted'); + } + const l = [miv, ...miv.descendants()]; + // Ensure mivs have valid root. + l.forEach(miv => miv.render()); + const next = i + 1 < msgitemViews.length ? msgitemViews[i + 1].root : null; + log('inserting l before next, or appending', { next, l }); + if (next) { + for (const miv of l) { + log('inserting miv', { root: miv.root, before: next }); + mlv.root.insertBefore(miv.root, next); + } + } + else { + mlv.root.append(...l.map(e => e.root)); + } + // For beginning/end of thread bar. + msgitemViews[i - 1].render(); + if (prevLastDesc) { + prevLastDesc.render(); + } + } + roots.splice(i, 1); + continue nextroot; + } + i++; + } + } + // Sort the remaining new roots by their receive times. + const sign = settings.threading === api.ThreadMode.ThreadOff && settings.orderAsc ? -1 : 1; + roots.sort((miva, mivb) => sign * (mivb.messageitem.Message.Received.getTime() - miva.messageitem.Message.Received.getTime())); + // Find place to insert, based on thread receive time. + let nextmivindex; + if (tmiv) { + nextmivindex = msgitemViews.indexOf(tmiv.threadRoot()); + } + else { + nextmivindex = msgitemViews.findIndex(miv => !settings.orderAsc && miv.receivedTime <= receivedTime || settings.orderAsc && receivedTime <= miv.receivedTime); + } + for (const miv of roots) { + miv.collapsed = settings.threading === api.ThreadMode.ThreadOn && miv.messageitem.Message.ThreadCollapsed; + if (settings.threading === api.ThreadMode.ThreadUnread) { + miv.collapsed = miv.messageitem.Message.Seen && !miv.findDescendant(dmiv => !dmiv.messageitem.Message.Seen); + } + if (requestMsgID > 0 && miv.collapsed) { + miv.collapsed = !miv.findDescendant(dmiv => dmiv.messageitem.Message.ID === requestMsgID); + } + const takeThreadRoot = (xmiv) => { + log('taking threadRoot', { id: xmiv.messageitem.Message.ID }); + // Remove subtree from dom. + const xdmiv = xmiv.descendants(); + xdmiv.forEach(xdmiv => xdmiv.remove()); + xmiv.remove(); + // Link to new parent. + miv.kids.push(xmiv); + xmiv.parent = miv; + miv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()); + return 1 + xdmiv.length; + }; + if (settings.threading !== api.ThreadMode.ThreadOff) { + // We may have to take out existing threadroots and place them under this new root. + // Because when we move a threadroot, we first remove it, then add it again. + for (let i = 0; i < msgitemViews.length;) { + const xmiv = msgitemViews[i]; + if (!xmiv.parent && xmiv.messageitem.Message.ThreadID === miv.messageitem.Message.ThreadID && (xmiv.messageitem.Message.ThreadParentIDs || []).includes(miv.messageitem.Message.ID)) { + msgitemViews.splice(i, takeThreadRoot(xmiv)); + nextmivindex = i; + } + else { + i++; + } + } + for (let i = 0; i < collapsedMsgitemViews.length;) { + const xmiv = collapsedMsgitemViews[i]; + if (!xmiv.parent && xmiv.messageitem.Message.ThreadID === miv.messageitem.Message.ThreadID && (xmiv.messageitem.Message.ThreadParentIDs || []).includes(miv.messageitem.Message.ID)) { + takeThreadRoot(xmiv); + collapsedMsgitemViews.splice(i, 1); + } + else { + i++; + } + } + } + let l = miv.descendants(); + miv.render(); + l.forEach(kmiv => kmiv.render()); + if (miv.collapsed) { + collapsedMsgitemViews.push(...l); + l = [miv]; + } + else { + l = [miv, ...l]; + } + if (nextmivindex < 0) { + mlv.root.append(...l.map(miv => miv.root)); + msgitemViews.push(...l); + } + else { + const next = msgitemViews[nextmivindex].root; + for (const miv of l) { + mlv.root.insertBefore(miv.root, next); + } + msgitemViews.splice(nextmivindex, 0, ...l); + } } }); + if (!isChange) { + return; + } const oldstate = state(); if (!focus) { focus = msgitemViews[0]; } if (selected.length === 0) { - selected = [msgitemViews[0]]; + if (focus) { + selected = [focus]; + } + else if (msgitemViews.length > 0) { + selected = [msgitemViews[0]]; + } } updateState(oldstate); }, + // Remove messages, they can be in different threads. removeUIDs: (mailboxID, uids) => { - const uidmap = {}; - uids.forEach(uid => uidmap['' + mailboxID + ',' + uid] = true); // todo: we would like messageID here. - const key = (miv) => '' + miv.messageitem.Message.MailboxID + ',' + miv.messageitem.Message.UID; const oldstate = state(); - selected = selected.filter(miv => !uidmap[key(miv)]); - if (focus && uidmap[key(focus)]) { - const index = msgitemViews.indexOf(focus); - var nextmiv; - for (let i = index + 1; i < msgitemViews.length; i++) { - if (!uidmap[key(msgitemViews[i])]) { - nextmiv = msgitemViews[i]; - break; - } + const hadSelected = selected.length > 0; + const threadIDs = new Set(); + uids.forEach(uid => { + const threadID = removeUID(mailboxID, uid); + log('removed message with thread', { threadID }); + if (threadID) { + threadIDs.add(threadID); } - if (!nextmiv) { - for (let i = index - 1; i >= 0; i--) { - if (!uidmap[key(msgitemViews[i])]) { - nextmiv = msgitemViews[i]; - break; - } - } - } - if (nextmiv) { - focus = nextmiv; - } - else { - focus = null; - } - } - if (selected.length === 0 && focus) { + }); + possiblyTakeoutOldThreads(threadIDs); + if (hadSelected && focus && selected.length === 0) { selected = [focus]; } updateState(oldstate); - let i = 0; - while (i < msgitemViews.length) { - const miv = msgitemViews[i]; - const k = '' + miv.messageitem.Message.MailboxID + ',' + miv.messageitem.Message.UID; - if (!uidmap[k]) { - i++; - continue; + }, + // Set new muted/collapsed flags for messages in thread. + updateMessageThreadFields: (messageIDs, muted, collapsed) => { + for (const id of messageIDs) { + let miv = msgitemViews.find(miv => miv.messageitem.Message.ID === id); + if (!miv) { + miv = collapsedMsgitemViews.find(miv => miv.messageitem.Message.ID === id); + } + if (miv) { + miv.messageitem.Message.ThreadMuted = muted; + miv.messageitem.Message.ThreadCollapsed = collapsed; + const mivthr = miv.threadRoot(); + if (mivthr.collapsed) { + mivthr.render(); + } + else { + miv.render(); + } + } + else { + const mi = oldThreadMessageItems.find(mi => mi.Message.ID === id); + if (mi) { + mi.Message.ThreadMuted = muted; + mi.Message.ThreadCollapsed = collapsed; + } } - miv.remove(); - msgitemViews.splice(i, 1); } }, // For location hash. @@ -2949,15 +3975,12 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p miv.root.classList.toggle('focus', miv === focus); miv.root.classList.toggle('active', selected.indexOf(miv) >= 0); }, - anchorMessageID: () => msgitemViews[msgitemViews.length - 1].messageitem.Message.ID, - addMsgitemViews: (mivs) => { - mlv.root.append(...mivs.map(v => v.root)); - msgitemViews.push(...mivs); - }, clear: () => { dom._kids(mlv.root); msgitemViews.forEach(miv => miv.remove()); msgitemViews = []; + collapsedMsgitemViews = []; + oldThreadMessageItems = []; focus = null; selected = []; dom._kids(msgElem); @@ -2974,12 +3997,27 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p selected = [miv]; updateState(oldstate); }, - selected: () => selected, - openMessage: (miv, initial, parsedMessageOpt) => { + selected: () => { + const l = []; + for (const miv of selected) { + l.push(miv); + if (miv.collapsed) { + l.push(...miv.descendants()); + } + } + return l; + }, + openMessage: (parsedMessage) => { + let miv = msgitemViews.find(miv => miv.messageitem.Message.ID === parsedMessage.ID); + if (!miv) { + // todo: could move focus to the nearest expanded message in this thread, if any? + return false; + } const oldstate = state(); focus = miv; selected = [miv]; - updateState(oldstate, initial, parsedMessageOpt); + updateState(oldstate, true, parsedMessage); + return true; }, click: (miv, ctrl, shift) => { if (msgitemViews.length === 0) { @@ -3045,6 +4083,9 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p 'k', 'K', 'Home', ',', '<', 'End', '.', '>', + 'n', 'N', + 'p', 'P', + 'u', 'U', ]; if (!e.altKey && moveKeys.includes(e.key)) { const moveclick = (index, clip) => { @@ -3076,7 +4117,7 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p else if (e.key === 'ArrowDown' || e.key === 'j' || e.key === 'J') { moveclick(i + 1, e.key === 'J'); } - else if (e.key === 'PageUp' || e.key === 'h' || e.key == 'H' || e.key === 'PageDown' || e.key === 'l' || e.key === 'L') { + else if (e.key === 'PageUp' || e.key === 'h' || e.key === 'H' || e.key === 'PageDown' || e.key === 'l' || e.key === 'L') { if (msgitemViews.length > 0) { let n = Math.max(1, Math.floor(scrollElemHeight() / mlv.itemHeight()) - 1); if (e.key === 'PageUp' || e.key === 'h' || e.key === 'H') { @@ -3091,6 +4132,42 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p else if (e.key === 'End' || e.key === '.' || e.key === '>') { moveclick(msgitemViews.length - 1, true); } + else if (e.key === 'n' || e.key === 'N') { + if (i < 0) { + moveclick(0, true); + } + else { + const tid = msgitemViews[i].messageitem.Message.ThreadID; + for (; i < msgitemViews.length; i++) { + if (msgitemViews[i].messageitem.Message.ThreadID !== tid) { + moveclick(i, true); + break; + } + } + } + } + else if (e.key === 'p' || e.key === 'P') { + if (i < 0) { + moveclick(0, true); + } + else { + let thrmiv = msgitemViews[i].threadRoot(); + if (thrmiv === msgitemViews[i]) { + if (i - 1 >= 0) { + thrmiv = msgitemViews[i - 1].threadRoot(); + } + } + moveclick(msgitemViews.indexOf(thrmiv), true); + } + } + else if (e.key === 'u' || e.key === 'U') { + for (i = i < 0 ? 0 : i + 1; i < msgitemViews.length; i += 1) { + if (!msgitemViews[i].messageitem.Message.Seen || msgitemViews[i].collapsed && msgitemViews[i].findDescendant(miv => !miv.messageitem.Message.Seen)) { + moveclick(i, true); + break; + } + } + } e.preventDefault(); e.stopPropagation(); return; @@ -3110,6 +4187,10 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p }, mailboxes: () => listMailboxes(), itemHeight: () => msgitemViews.length > 0 ? msgitemViews[0].root.getBoundingClientRect().height : 25, + threadExpand: (miv) => threadExpand(miv, true), + threadCollapse: (miv) => threadCollapse(miv, true), + threadToggle: threadToggle, + viewportEnsureMessages: viewportEnsureMessages, cmdArchive: cmdArchive, cmdTrash: cmdTrash, cmdDelete: cmdDelete, @@ -3117,10 +4198,12 @@ const newMsglistView = (msgElem, listMailboxes, setLocationHash, otherMailbox, p cmdMarkNotJunk: cmdMarkNotJunk, cmdMarkRead: cmdMarkRead, cmdMarkUnread: cmdMarkUnread, + cmdMute: cmdMute, + cmdUnmute: cmdUnmute, }; return mlv; }; -const newMailboxView = (xmb, mailboxlistView) => { +const newMailboxView = (xmb, mailboxlistView, otherMailbox) => { const plusbox = '⊞'; const minusbox = '⊟'; const cmdCollapse = async () => { @@ -3230,7 +4313,12 @@ const newMailboxView = (xmb, mailboxlistView) => { }, async function drop(e) { e.preventDefault(); mbv.root.classList.toggle('dropping', false); - const msgIDs = JSON.parse(e.dataTransfer.getData('application/vnd.mox.messages')); + const sentMailboxID = mailboxlistView.mailboxes().find(mb => mb.Sent)?.ID; + const mailboxMsgIDs = JSON.parse(e.dataTransfer.getData('application/vnd.mox.messages')); + const msgIDs = mailboxMsgIDs + .filter(mbMsgID => mbMsgID[0] !== xmb.ID) + .filter(mbMsgID => mailboxMsgIDs.length === 1 || !sentMailboxID || mbMsgID[0] !== sentMailboxID || !otherMailbox(sentMailboxID)) + .map(mbMsgID => mbMsgID[1]); await withStatus('Moving to ' + xmb.Name, client.MessageMove(msgIDs, xmb.ID)); }, dom.div(dom._class('mailbox'), style({ display: 'flex', justifyContent: 'space-between' }), name = dom.div(style({ whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis' })), dom.div(style({ whiteSpace: 'nowrap' }), actionBtn = dom.clickbutton(dom._class('mailboxhoveronly'), '...', attr.tabindex('-1'), // Without, tab breaks because this disappears when mailbox loses focus. attr.arialabel('Mailbox actions'), attr.title('Actions on mailbox, like deleting, emptying, renaming.'), function click(e) { @@ -3290,7 +4378,7 @@ const newMailboxView = (xmb, mailboxlistView) => { }; return mbv; }; -const newMailboxlistView = (msglistView, requestNewView, updatePageTitle, setLocationHash, unloadSearch) => { +const newMailboxlistView = (msglistView, requestNewView, updatePageTitle, setLocationHash, unloadSearch, otherMailbox) => { let mailboxViews = []; let mailboxViewActive; // Reorder mailboxes and assign new short names and indenting. Called after changing the list. @@ -3398,7 +4486,7 @@ const newMailboxlistView = (msglistView, requestNewView, updatePageTitle, setLoc }, fieldset = dom.fieldset(dom.label('Name ', name = dom.input(attr.required('yes'), focusPlaceholder('Lists/Go/Nuts'))), ' ', dom.submitbutton('Create')))); })), mailboxesElem)); const loadMailboxes = (mailboxes, mbnameOpt) => { - mailboxViews = mailboxes.map(mb => newMailboxView(mb, mblv)); + mailboxViews = mailboxes.map(mb => newMailboxView(mb, mblv, otherMailbox)); updateMailboxNames(); if (mbnameOpt) { const mbv = mailboxViews.find(mbv => mbv.mailbox.Name === mbnameOpt); @@ -3460,7 +4548,7 @@ const newMailboxlistView = (msglistView, requestNewView, updatePageTitle, setLoc } }, addMailbox: (mb) => { - const mbv = newMailboxView(mb, mblv); + const mbv = newMailboxView(mb, mblv, otherMailbox); mailboxViews.push(mbv); updateMailboxNames(); }, @@ -3841,6 +4929,7 @@ const init = async () => { }; let requestSequence = 0; // Counter for assigning requestID. let requestID = 0; // Current request, server will mirror it in SSE data. If we get data for a different id, we ignore it. + let requestAnchorMessageID = 0; // For pagination. let requestViewEnd = false; // If true, there is no more data to fetch, no more page needed for this view. let requestFilter = newFilter(); let requestNotFilter = newNotFilter(); @@ -3910,14 +4999,15 @@ const init = async () => { requestFilter = filterOpt; requestNotFilter = notFilterOpt || newNotFilter(); } + requestAnchorMessageID = 0; requestViewEnd = false; const bounds = msglistscrollElem.getBoundingClientRect(); - await requestMessages(bounds, 0, requestMsgID); + await requestMessages(bounds, requestMsgID); }; - const requestMessages = async (scrollBounds, anchorMessageID, destMessageID) => { + const requestMessages = async (scrollBounds, destMessageID) => { const fetchCount = Math.max(50, 3 * Math.ceil(scrollBounds.height / msglistView.itemHeight())); const page = { - AnchorMessageID: anchorMessageID, + AnchorMessageID: requestAnchorMessageID, Count: fetchCount, DestMessageID: destMessageID, }; @@ -3926,6 +5016,7 @@ const init = async () => { const [f, notf] = refineFilters(requestFilter, requestNotFilter); const query = { OrderAsc: settings.orderAsc, + Threading: settings.threading, Filter: f, NotFilter: notf, }; @@ -3967,10 +5058,20 @@ const init = async () => { dom._kids(refineLabelBtn, 'Label: ' + kw); await withStatus('Requesting messages', requestNewView(false)); }; + const viewportEnsureMessages = async () => { + // We know how many entries we have, and how many screenfulls. So we know when we + // only have 2 screen fulls left. That's when we request the next data. + const bounds = msglistscrollElem.getBoundingClientRect(); + if (msglistscrollElem.scrollTop < msglistscrollElem.scrollHeight - 3 * bounds.height) { + return; + } + // log('new request for scroll') + await withStatus('Requesting more messages', requestMessages(bounds, 0)); + }; const otherMailbox = (mailboxID) => requestFilter.MailboxID !== mailboxID ? (mailboxlistView.findMailboxByID(mailboxID) || null) : null; const listMailboxes = () => mailboxlistView.mailboxes(); - const msglistView = newMsglistView(msgElem, listMailboxes, setLocationHash, otherMailbox, possibleLabels, () => msglistscrollElem ? msglistscrollElem.getBoundingClientRect().height : 0, refineKeyword); - const mailboxlistView = newMailboxlistView(msglistView, requestNewView, updatePageTitle, setLocationHash, unloadSearch); + const msglistView = newMsglistView(msgElem, listMailboxes, setLocationHash, otherMailbox, possibleLabels, () => msglistscrollElem ? msglistscrollElem.getBoundingClientRect().height : 0, refineKeyword, viewportEnsureMessages); + const mailboxlistView = newMailboxlistView(msglistView, requestNewView, updatePageTitle, setLocationHash, unloadSearch, otherMailbox); let refineUnreadBtn, refineReadBtn, refineAttachmentsBtn, refineLabelBtn; const refineToggleActive = (btn) => { for (const e of [refineUnreadBtn, refineReadBtn, refineAttachmentsBtn, refineLabelBtn]) { @@ -3980,6 +5081,7 @@ const init = async () => { dom._kids(refineLabelBtn, 'Label'); } }; + let threadMode; let msglistElem = dom.div(dom._class('msglist'), style({ position: 'absolute', left: '0', right: 0, top: 0, bottom: 0, display: 'flex', flexDirection: 'column' }), dom.div(attr.role('region'), attr.arialabel('Filter and sorting buttons for message list'), style({ display: 'flex', justifyContent: 'space-between', backgroundColor: '#f8f8f8', borderBottom: '1px solid #ccc', padding: '.25em .5em' }), dom.div(dom.h1('Refine:', style({ fontWeight: 'normal', fontSize: 'inherit', display: 'inline', margin: 0 }), attr.title('Refine message listing with quick filters. These refinement filters are in addition to any search criteria, but the refine attachment filter overrides a search attachment criteria.')), ' ', dom.span(dom._class('btngroup'), refineUnreadBtn = dom.clickbutton(settings.refine === 'unread' ? dom._class('active') : [], 'Unread', attr.title('Only show messages marked as unread.'), async function click(e) { settingsPut({ ...settings, refine: 'unread' }); refineToggleActive(e.target); @@ -4010,7 +5112,17 @@ const init = async () => { settingsPut({ ...settings, refine: '' }); refineToggleActive(e.target); await withStatus('Requesting messages', requestNewView(false)); - })), dom.div(queryactivityElem = dom.span(), ' ', dom.clickbutton('↑↓', attr.title('Toggle sorting by date received.'), settings.orderAsc ? dom._class('invert') : [], async function click(e) { + })), dom.div(queryactivityElem = dom.span(), ' ', threadMode = dom.select(attr.arialabel('Thread modes.'), attr.title('Off: Threading disabled, messages are shown individually.\nOn: Group messages in threads, expanded by default except when (previously) manually collapsed.\nUnread: Only expand thread with unread messages, ignoring and not saving whether they were manually collapsed.'), dom.option('Threads: Off', attr.value(api.ThreadMode.ThreadOff), settings.threading === api.ThreadMode.ThreadOff ? attr.selected('') : []), dom.option('Threads: On', attr.value(api.ThreadMode.ThreadOn), settings.threading === api.ThreadMode.ThreadOn ? attr.selected('') : []), dom.option('Threads: Unread', attr.value(api.ThreadMode.ThreadUnread), settings.threading === api.ThreadMode.ThreadUnread ? attr.selected('') : []), async function change() { + let reset = settings.threading === api.ThreadMode.ThreadOff; + settingsPut({ ...settings, threading: threadMode.value }); + reset = reset || settings.threading === api.ThreadMode.ThreadOff; + if (reset) { + await withStatus('Requesting messages', requestNewView(false)); + } + else { + msglistView.threadToggle(); + } + }), ' ', dom.clickbutton('↑↓', attr.title('Toggle sorting by date received.'), settings.orderAsc ? dom._class('invert') : [], async function click(e) { settingsPut({ ...settings, orderAsc: !settings.orderAsc }); e.target.classList.toggle('invert', settings.orderAsc); // We don't want to include the currently selected message because it could cause a @@ -4044,15 +5156,7 @@ const init = async () => { if (!sseID || requestViewEnd || requestID) { return; } - // We know how many entries we have, and how many screenfulls. So we know when we - // only have 2 screen fulls left. That's when we request the next data. - const bounds = msglistscrollElem.getBoundingClientRect(); - if (msglistscrollElem.scrollTop < msglistscrollElem.scrollHeight - 3 * bounds.height) { - return; - } - // log('new request for scroll') - const reqAnchor = msglistView.anchorMessageID(); - await withStatus('Requesting more messages', requestMessages(bounds, reqAnchor, 0)); + await viewportEnsureMessages(); }, dom.div(style({ width: '100%', borderSpacing: '0' }), msglistView)))); let searchbarElem; // Input field for search // Called by searchView when user executes the search. @@ -4143,7 +5247,7 @@ const init = async () => { '?': cmdHelp, 'ctrl ?': cmdTooltip, c: cmdCompose, - M: cmdFocusMsg, + 'ctrl m': cmdFocusMsg, }; const webmailroot = dom.div(style({ display: 'flex', flexDirection: 'column', alignContent: 'stretch', height: '100dvh' }), dom.div(dom._class('topbar'), style({ display: 'flex' }), attr.role('region'), attr.arialabel('Top bar'), topcomposeboxElem = dom.div(dom._class('pad'), style({ width: settings.mailboxesWidth + 'px', textAlign: 'center' }), dom.clickbutton('Compose', attr.title('Compose new email message.'), function click() { shortcutCmd(cmdCompose, shortcuts); @@ -4317,7 +5421,7 @@ const init = async () => { let lastflagswidth, lastagewidth; let rulesInserted = false; const updateMsglistWidths = () => { - const width = msglistscrollElem.clientWidth; + const width = msglistscrollElem.clientWidth - 2; // Borders. lastmsglistwidth = width; let flagswidth = settings.msglistflagsWidth; let agewidth = settings.msglistageWidth; @@ -4445,6 +5549,7 @@ const init = async () => { const fetchCount = Math.max(50, 3 * Math.ceil(msglistscrollElem.getBoundingClientRect().height / msglistView.itemHeight())); const query = { OrderAsc: settings.orderAsc, + Threading: settings.threading, Filter: f, NotFilter: notf, }; @@ -4458,6 +5563,7 @@ const init = async () => { // We get an implicit query for the automatically selected mailbox or query. requestSequence++; requestID = requestSequence; + requestAnchorMessageID = 0; requestViewEnd = false; clearList(); const request = { @@ -4548,7 +5654,7 @@ const init = async () => { if (formatEmailASCII(b) === loginAddr) { return 1; } - if (a.Domain.ASCII != b.Domain.ASCII) { + if (a.Domain.ASCII !== b.Domain.ASCII) { return a.Domain.ASCII < b.Domain.ASCII ? -1 : 1; } return a.User < b.User ? -1 : 1; @@ -4580,7 +5686,7 @@ const init = async () => { eventSource.addEventListener('viewErr', async (e) => { const viewErr = checkParse(() => api.parser.EventViewErr(JSON.parse(e.data))); log('event viewErr', viewErr); - if (viewErr.ViewID != viewID || viewErr.RequestID !== requestID) { + if (viewErr.ViewID !== viewID || viewErr.RequestID !== requestID) { log('received viewErr for other viewID or requestID', { expected: { viewID, requestID }, got: { viewID: viewErr.ViewID, requestID: viewErr.RequestID } }); return; } @@ -4596,7 +5702,7 @@ const init = async () => { eventSource.addEventListener('viewReset', async (e) => { const viewReset = checkParse(() => api.parser.EventViewReset(JSON.parse(e.data))); log('event viewReset', viewReset); - if (viewReset.ViewID != viewID || viewReset.RequestID !== requestID) { + if (viewReset.ViewID !== viewID || viewReset.RequestID !== requestID) { log('received viewReset for other viewID or requestID', { expected: { viewID, requestID }, got: { viewID: viewReset.ViewID, requestID: viewReset.RequestID } }); return; } @@ -4608,28 +5714,25 @@ const init = async () => { eventSource.addEventListener('viewMsgs', async (e) => { const viewMsgs = checkParse(() => api.parser.EventViewMsgs(JSON.parse(e.data))); log('event viewMsgs', viewMsgs); - if (viewMsgs.ViewID != viewID || viewMsgs.RequestID !== requestID) { + if (viewMsgs.ViewID !== viewID || viewMsgs.RequestID !== requestID) { log('received viewMsgs for other viewID or requestID', { expected: { viewID, requestID }, got: { viewID: viewMsgs.ViewID, requestID: viewMsgs.RequestID } }); return; } msglistView.root.classList.toggle('loading', false); - const extramsgitemViews = (viewMsgs.MessageItems || []).map(mi => { - const othermb = requestFilter.MailboxID !== mi.Message.MailboxID ? mailboxlistView.findMailboxByID(mi.Message.MailboxID) : undefined; - return newMsgitemView(mi, msglistView, othermb || null); - }); - msglistView.addMsgitemViews(extramsgitemViews); + if (viewMsgs.MessageItems) { + msglistView.addMessageItems(viewMsgs.MessageItems || [], false, requestMsgID); + } if (viewMsgs.ParsedMessage) { - const msgID = viewMsgs.ParsedMessage.ID; - const miv = extramsgitemViews.find(miv => miv.messageitem.Message.ID === msgID); - if (miv) { - msglistView.openMessage(miv, true, viewMsgs.ParsedMessage); - } - else { + const ok = msglistView.openMessage(viewMsgs.ParsedMessage); + if (!ok) { // Should not happen, server would be sending a parsedmessage while not including the message itself. requestMsgID = 0; setLocationHash(); } } + if (viewMsgs.MessageItems && viewMsgs.MessageItems.length > 0) { + requestAnchorMessageID = viewMsgs.MessageItems[viewMsgs.MessageItems.length - 1][0].Message.ID; + } requestViewEnd = viewMsgs.ViewEnd; if (requestViewEnd) { msglistscrollElem.appendChild(listendElem); @@ -4647,7 +5750,7 @@ const init = async () => { eventSource.addEventListener('viewChanges', async (e) => { const viewChanges = checkParse(() => api.parser.EventViewChanges(JSON.parse(e.data))); log('event viewChanges', viewChanges); - if (viewChanges.ViewID != viewID) { + if (viewChanges.ViewID !== viewID) { log('received viewChanges for other viewID', { expected: viewID, got: viewChanges.ViewID }); return; } @@ -4671,7 +5774,7 @@ const init = async () => { } else if (tag === 'ChangeMsgAdd') { const c = api.parser.ChangeMsgAdd(x); - msglistView.addMessageItems([c.MessageItem]); + msglistView.addMessageItems([c.MessageItems || []], true, 0); } else if (tag === 'ChangeMsgRemove') { const c = api.parser.ChangeMsgRemove(x); @@ -4681,6 +5784,12 @@ const init = async () => { const c = api.parser.ChangeMsgFlags(x); msglistView.updateFlags(c.MailboxID, c.UID, c.ModSeq, c.Mask, c.Flags, c.Keywords || []); } + else if (tag === 'ChangeMsgThread') { + const c = api.parser.ChangeMsgThread(x); + if (c.MessageIDs) { + msglistView.updateMessageThreadFields(c.MessageIDs, c.Muted, c.Collapsed); + } + } else if (tag === 'ChangeMailboxRemove') { const c = api.parser.ChangeMailboxRemove(x); mailboxlistView.removeMailbox(c.MailboxID); @@ -4767,7 +5876,7 @@ window.addEventListener('unhandledrejection', (e) => { return; } const err = e.reason; - if (err instanceof EvalError || err instanceof RangeError || err instanceof ReferenceError || err instanceof SyntaxError || err instanceof TypeError || err instanceof URIError) { + if (err instanceof EvalError || err instanceof RangeError || err instanceof ReferenceError || err instanceof SyntaxError || err instanceof TypeError || err instanceof URIError || err instanceof ConsistencyError) { showUnhandledError(err, 0, 0); } else { diff --git a/webmail/webmail.ts b/webmail/webmail.ts index f49231f..9c4b036 100644 --- a/webmail/webmail.ts +++ b/webmail/webmail.ts @@ -47,12 +47,21 @@ different color. Browsers to test with: Firefox, Chromium, Safari, Edge. To simulate slow API calls and SSE events: -window.localStorage.setItem('sherpats-debug', JSON.stringify({waitMinMsec: 2000, waitMaxMsec: 4000})) + + localStorage.setItem('sherpats-debug', JSON.stringify({waitMinMsec: 2000, waitMaxMsec: 4000})) Show additional headers of messages: -settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id']}) -- todo: threading (needs support in mox first) + settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id', 'List-Id', 'List-Post', 'X-Mox-Reason']}) + +Enable logging and reload afterwards: + + localStorage.setItem('log', 'yes') + +Enable consistency checking in UI updates: + + settingsPut({...settings, checkConsistency: true}) + - todo: in msglistView, show names of people we have sent to, and address otherwise. - todo: implement settings stored in the server, such as mailboxCollapsed, keyboard shortcuts. also new settings for displaying email as html by default for configured sender address or domain. name to use for "From", optional default Reply-To and Bcc addresses, signatures (per address), configured labels/keywords with human-readable name, colors and toggling with shortcut keys 1-9. - todo: in msglist, if our address is in the from header, list addresses in the to/cc/bcc, it's likely a sent folder @@ -68,7 +77,6 @@ settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id']} - todo: only show orange underline where it could be a problem? in addresses and anchor texts. we may be lighting up a christmas tree now, desensitizing users. - todo: saved searches that are displayed below list of mailboxes, for quick access to preset view - todo: when search on free-form text is active, highlight the searched text in the message view. -- todo: when reconnecting, request only the changes to the current state/msglist, passing modseq query string parameter - todo: composeView: save as draft, periodically and when closing. - todo: forwarding of html parts, including inline attachments, so the html version can be rendered like the original by the receiver. - todo: buttons/mechanism to operate on all messages in a mailbox/search query, without having to list and select all messages. e.g. clearing flags/labels. @@ -79,15 +87,17 @@ settingsPut({...settings, showHeaders: ['User-Agent', 'X-Mailer', 'Message-Id']} - todo: nicer address input fields like other mail clients do. with tab to autocomplete and turn input into a box and delete removing of the entire address. - todo: consider composing messages with bcc headers that are kept as message Bcc headers, optionally with checkbox. - todo: improve accessibility +- todo: threading mode where we don't show messages in Trash/Sent in thread? - todo: msglistView: preload next message? - todo: previews of zip files - todo: undo? -- todo: mute threads? - todo: mobile-friendly version. should perhaps be a completely different app, because it is so different. -- todo: msglistView: for mailbox views (which are fast to list the results of), should we ask the full number of messages, set the height of the scroll div based on the number of messages, then request messages when user scrolls, putting the messages in place. not sure if worth the trouble. - todo: basic vim key bindings in textarea/input. or just let users use a browser plugin. */ +class ConsistencyError extends Error { +} + const zindexes = { splitter: '1', compose: '2', @@ -106,7 +116,7 @@ declare let moxversion: string // All logging goes through log() instead of console.log, except "should not happen" logging. let log: (...args: any[]) => void = () => {} try { - if (localStorage.getItem('log')) { + if (localStorage.getItem('log') || location.hostname === 'localhost') { log = console.log } } catch (err) {} @@ -127,6 +137,8 @@ const defaultSettings = { mailboxCollapsed: {} as {[mailboxID: number]: boolean}, // Mailboxes that are collapsed. showAllHeaders: false, // Whether to show all message headers. showHeaders: [] as string[], // Additional message headers to show. + threading: api.ThreadMode.ThreadUnread, + checkConsistency: location.hostname === 'localhost', // Enable UI update consistency checks, default only for local development. } const parseSettings = (): typeof defaultSettings => { try { @@ -179,6 +191,8 @@ const parseSettings = (): typeof defaultSettings => { mailboxCollapsed: mailboxCollapsed, showAllHeaders: getBool('showAllHeaders'), showHeaders: getStringArray('showHeaders'), + threading: getString('threading', api.ThreadMode.ThreadOff, api.ThreadMode.ThreadOn, api.ThreadMode.ThreadUnread) as api.ThreadMode, + checkConsistency: getBool('checkConsistency'), } } catch (err) { console.log('getting settings from localstorage', err) @@ -252,7 +266,7 @@ type command = () => Promise const shortcutCmd = async (cmdfn: command, shortcuts: {[key: string]: command}) => { let shortcut = '' for (const k in shortcuts) { - if (shortcuts[k] == cmdfn) { + if (shortcuts[k] === cmdfn) { shortcut = k break } @@ -294,7 +308,7 @@ const keyHandler = (shortcuts: {[key: string]: command}) => { } // For attachment sizes. -const formatSize = (size: number) => size > 1024*1024 ? (size/(1024*1024)).toFixed(1)+'mb' : Math.ceil(size/1024)+'kb' +const formatSize = (size: number) => size > 1024*1024 ? (size/(1024*1024)).toFixed(1)+'mb' : Math.ceil(size/1024)+'kb' // Parse size as used in minsize: and maxsize: in the search bar. const parseSearchSize = (s: string): [string, number] => { @@ -404,7 +418,7 @@ const parseSearchTokens = (s: string): Token[] => { } else if (t) { add() } - } else if (quoted && c == '"') { + } else if (quoted && c === '"') { quoteend = true } else if (c === '"') { quoted = true @@ -474,7 +488,7 @@ const parseSearch = (searchquery: string, mailboxlistView: MailboxlistView): [ap fpos.MailboxID = 0 } return - } else if (tag == 'submb') { + } else if (tag === 'submb') { fpos.MailboxChildrenIncluded = true return } else if (tag === 'start') { @@ -593,35 +607,43 @@ const newAddressComplete = (): any => { } } -// Characters we display in the message list for flags set for a message. -// todo: icons would be nice to have instead. -const flagchars: {[key: string]: string} = { - Replied: 'r', - Flagged: '!', - Forwarded: 'f', - Junk: 'j', - Deleted: 'D', - Draft: 'd', - Phishing: 'p', -} -const flagList = (m: api.Message, mi: api.MessageItem): HTMLElement[] => { - let l: [string, string][] = [] +const flagList = (miv: MsgitemView): HTMLElement[] => { + const msgflags: [string, string][] = [] // Flags for message in miv. + const othermsgflags: [string, string][] = [] // Flags for descendant messages if miv is collapsed. Only flags not in msgflags. + let l = msgflags + const seen = new Set() const flag = (v: boolean, char: string, name: string) => { - if (v) { + if (v && !seen.has(name)) { l.push([name, char]) + seen.add(name) } } - flag(m.Answered, 'r', 'Replied/answered') - flag(m.Flagged, '!', 'Flagged') - flag(m.Forwarded, 'f', 'Forwarded') - flag(m.Junk, 'j', 'Junk') - flag(m.Deleted, 'D', 'Deleted, used in IMAP, message will likely be removed soon.') - flag(m.Draft, 'd', 'Draft') - flag(m.Phishing, 'p', 'Phishing') - flag(!m.Junk && !m.Notjunk, '?', 'Unclassified, neither junk nor not junk: message does not contribute to spam classification of new incoming messages') - flag(mi.Attachments && mi.Attachments.length > 0 ? true : false, 'a', 'Has at least one attachment') - return l.map(t => dom.span(dom._class('msgitemflag'), t[1], attr.title(t[0]))) + const addFlags = (mi: api.MessageItem) => { + const m = mi.Message + flag(m.Answered, 'r', 'Replied/answered') + flag(m.Flagged, '!', 'Flagged') + flag(m.Forwarded, 'f', 'Forwarded') + flag(m.Junk, 'j', 'Junk') + flag(m.Deleted, 'D', 'Deleted, used in IMAP, message will likely be removed soon.') + flag(m.Draft, 'd', 'Draft') + flag(m.Phishing, 'p', 'Phishing') + flag(!m.Junk && !m.Notjunk, '?', 'Unclassified, neither junk nor not junk: message does not contribute to spam classification of new incoming messages') + flag(mi.Attachments && mi.Attachments.length > 0 ? true : false, 'a', 'Has at least one attachment') + if (m.ThreadMuted) { + flag(true, 'm', 'Muted, new messages are automatically marked as read.') + } + } + addFlags(miv.messageitem) + if (miv.isCollapsedThreadRoot()) { + l = othermsgflags + for (miv of miv.descendants()) { + addFlags(miv.messageitem) + } + } + + return msgflags.map(t => dom.span(dom._class('msgitemflag'), t[1], attr.title(t[0]))) + .concat(othermsgflags.map(t => dom.span(dom._class('msgitemflag'), dom._class('msgitemflagcollapsed'), t[1], attr.title(t[0])))) } // Turn filters from the search bar into filters with the refine filters (buttons @@ -927,7 +949,7 @@ const cmdHelp = async () => { ['i', 'open inbox'], ['?', 'help'], ['ctrl ?', 'tooltip for focused element'], - ['M', 'focus message'], + ['ctrl m', 'focus message'], ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))), dom.tr(dom.td(attr.colspan('2'), dom.h2('Mailbox', style({margin: '0'})))), @@ -957,22 +979,16 @@ const cmdHelp = async () => { ['d, Delete', 'move to trash folder'], ['D', 'delete permanently'], ['q', 'move to junk folder'], - ['n', 'mark not junk'], + ['Q', 'mark not junk'], ['a', 'move to archive folder'], - ['u', 'mark unread'], + ['M', 'mark unread'], ['m', 'mark read'], - ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))), - - dom.tr(dom.td(attr.colspan('2'), dom.h2('Compose', style({margin: '1ex 0 0 0'})))), - [ - ['ctrl Enter', 'send message'], - ['ctrl w', 'cancel message'], - ['ctlr O', 'add To'], - ['ctrl C', 'add Cc'], - ['ctrl B', 'add Bcc'], - ['ctrl Y', 'add Reply-To'], - ['ctrl -', 'remove current address'], - ['ctrl +', 'add address of same type'], + ['u', 'to next unread message'], + ['p', 'to root of thread or previous thread'], + ['n', 'to root of next thread'], + ['S', 'select thread messages'], + ['C', 'toggle thread collapse'], + ['X', 'toggle thread mute, automatically marking new messages as read'], ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))), ), ), @@ -980,7 +996,19 @@ const cmdHelp = async () => { style({width: '40em'}), dom.table( - dom.tr(dom.td(attr.colspan('2'), dom.h2('Message', style({margin: '0'})))), + dom.tr(dom.td(attr.colspan('2'), dom.h2('Compose', style({margin: '0'})))), + [ + ['ctrl Enter', 'send message'], + ['ctrl w', 'cancel message'], + ['ctrl O', 'add To'], + ['ctrl C', 'add Cc'], + ['ctrl B', 'add Bcc'], + ['ctrl Y', 'add Reply-To'], + ['ctrl -', 'remove current address'], + ['ctrl +', 'add address of same type'], + ].map(t => dom.tr(dom.td(t[0]), dom.td(t[1]))), + + dom.tr(dom.td(attr.colspan('2'), dom.h2('Message', style({margin: '1ex 0 0 0'})))), [ ['r', 'reply or list reply'], ['R', 'reply all'], @@ -1155,7 +1183,7 @@ const compose = (opts: ComposeOptions) => { const fr = new window.FileReader() fr.addEventListener('load', () => { l.push({Filename: f.name, DataURI: fr.result as string}) - if (attachments.files && l.length == attachments.files.length) { + if (attachments.files && l.length === attachments.files.length) { resolve(l) } }) @@ -1313,7 +1341,7 @@ const compose = (opts: ComposeOptions) => { // Find own address matching the specified address, taking wildcards, localpart // separators and case-sensitivity into account. const addressSelf = (addr: api.MessageAddress) => { - return accountAddresses.find(a => a.Domain.ASCII === addr.Domain.ASCII && (a.User === '' || normalizeUser(a) == normalizeUser(addr))) + return accountAddresses.find(a => a.Domain.ASCII === addr.Domain.ASCII && (a.User === '' || normalizeUser(a) === normalizeUser(addr))) } let haveFrom = false @@ -1374,7 +1402,7 @@ const compose = (opts: ComposeOptions) => { ), ), toRow=dom.tr( - dom.td('To:', style({textAlign: 'right', color: '#555'})), + dom.td('To:', style({textAlign: 'right', color: '#555'})), toCell=dom.td(style({width: '100%'})), ), replyToRow=dom.tr( @@ -1551,19 +1579,47 @@ const movePopover = (e: MouseEvent, mailboxes: api.Mailbox[], msgs: api.Message[ // MsgitemView is a message-line in the list of messages. Selecting it loads and displays the message, a MsgView. interface MsgitemView { root: HTMLElement // MsglistView toggles active/focus classes on the root element. - messageitem: api.MessageItem - // Called when flags/keywords change for a message. - updateFlags: (modseq: number, mask: api.Flags, flags: api.Flags, keywords: string[]) => void + messageitem: api.MessageItem // Can be replaced with an updated version, e.g. with message with different mailbox. - // Must be called when MsgitemView is no longer needed. Typically through - // msglistView.clear(). This cleans up the timer that updates the message age. + // Fields for threading. + // + // Effective received time. When sorting ascending, the oldest of all children. + // When sorting descending, the newest of all. Does not change after creating + // MsgitemView, we don't move threads around when a new message is delivered to a + // thread. + receivedTime: number + // Parent message in thread. May not be the direct replied/forwarded message, e.g. + // if the direct parent was permanently removed. Thread roots don't have a parent. + parent: MsgitemView | null + // Sub messages in thread. Can be further descendants, when an intermediate message + // is missing. + kids: MsgitemView[] + // Whether this thread root is collapsed. If so, the root is visible, all descedants + // are not. Value is only valid if this is a thread root. + collapsed: boolean + + // Root MsgitemView for this subtree. Does not necessarily contain all messages in + // a thread, there can be multiple visible roots. A MsgitemView is visible if it is + // the threadRoot or otherwise if its threadRoot isn't collapsed. + threadRoot: () => MsgitemView + isCollapsedThreadRoot: () => boolean + descendants: () => MsgitemView[] // Flattened list of all descendents. + findDescendant: (match: (dmiv: MsgitemView) => boolean) => MsgitemView | null + lastDescendant: () => MsgitemView | null + + // Removes msgitem from the DOM and cleans up the timer that updates the message + // age. Must be called when MsgitemView is no longer needed. Typically through + // msglistView.clear(). remove: () => void + + // Must be called after initializing kids/parent field for proper rendering. + render: () => void } -// Make new MsgitemView, to be added to the list. othermb is set when this msgitem -// is displayed in a msglistView for other/multiple mailboxes, the mailbox name -// should be shown. -const newMsgitemView = (mi: api.MessageItem, msglistView: MsglistView, othermb: api.Mailbox | null): MsgitemView => { +// Make new MsgitemView, to be added to the list. +const newMsgitemView = (mi: api.MessageItem, msglistView: MsglistView, otherMailbox: otherMailbox, listMailboxes: listMailboxes, receivedTime: number, initialCollapsed: boolean): MsgitemView => { + // note: mi may be replaced. + // Timer to update the age of the message. let ageTimer = 0 @@ -1583,45 +1639,6 @@ const newMsgitemView = (mi: api.MessageItem, msglistView: MsglistView, othermb: } } - // If mailbox of message is not specified in filter (i.e. for mailbox list or - // search on the mailbox), we show it on the right-side of the subject. - const mailboxtag: HTMLElement[] = [] - if (othermb) { - let name = othermb.Name - if (name.length > 8+1+3+1+8+4) { - const t = name.split('/') - const first = t[0] - const last = t[t.length-1] - if (first.length + last.length <= 8+8) { - name = first+'/.../'+last - } else { - name = first.substring(0, 8) + '/.../' + last.substring(0, 8) - } - } - const e = dom.span(dom._class('msgitemmailbox'), - name === othermb.Name ? [] : attr.title(othermb.Name), - name, - ) - mailboxtag.push(e) - } - - const updateFlags = (modseq: number, mask: api.Flags, flags: api.Flags, keywords: string[]) => { - msgitemView.messageitem.Message.ModSeq = modseq - const maskobj = mask as unknown as {[key: string]: boolean} - const flagsobj = flags as unknown as {[key: string]: boolean} - const mobj = msgitemView.messageitem.Message as unknown as {[key: string]: boolean} - for (const k in maskobj) { - if (maskobj[k]) { - mobj[k] = flagsobj[k] - } - } - msgitemView.messageitem.Message.Keywords = keywords - const elem = render() - msgitemView.root.replaceWith(elem) - msgitemView.root = elem - msglistView.redraw(msgitemView) - } - const remove = (): void => { msgitemView.root.remove() if (ageTimer) { @@ -1652,7 +1669,7 @@ const newMsgitemView = (mi: api.MessageItem, msglistView: MsglistView, othermb: let nextSecs = 0 for (let i = 0; i < periods.length; i++) { const p = periods[i] - if (t >= 2*p || i == periods.length-1) { + if (t >= 2*p || i === periods.length-1) { const n = Math.round(t/p) s = '' + n + suffix[i] const prev = Math.floor(t/p) @@ -1680,59 +1697,295 @@ const newMsgitemView = (mi: api.MessageItem, msglistView: MsglistView, othermb: } const render = () => { + const mi = msgitemView.messageitem + const m = mi.Message + // Set by calling age(). if (ageTimer) { window.clearTimeout(ageTimer) ageTimer = 0 } - const m = msgitemView.messageitem.Message + // Keywords are normally shown per message. For collapsed threads, we show the + // keywords of the thread root message as normal, and any additional keywords from + // children in a way that draws less attention. const keywords = (m.Keywords || []).map(kw => dom.span(dom._class('keyword'), kw)) + if (msgitemView.isCollapsedThreadRoot()) { + const keywordsSeen = new Set() + for (const kw of (m.Keywords || [])) { + keywordsSeen.add(kw) + } + for (const miv of msgitemView.descendants()) { + for (const kw of (miv.messageitem.Message.Keywords || [])) { + if (!keywordsSeen.has(kw)) { + keywordsSeen.add(kw) + keywords.push(dom.span(dom._class('keyword'), dom._class('keywordcollapsed'), kw)) + } + } + } + } - return dom.div(dom._class('msgitem'), + let threadIndent = 0 + for (let miv = msgitemView; miv.parent; miv = miv.parent) { + threadIndent++ + } + + // For threaded messages, we draw the subject/first-line indented, and with a + // charactering indicating the relationship. + // todo: show different arrow is message is a forward? we can tell by the message flag, it will likely be a message the user sent. + let threadChar = '' + let threadCharTitle = '' + if (msgitemView.parent) { + threadChar = '↳' // Down-right arrow for direct response (reply/forward). + if (msgitemView.parent.messageitem.Message.MessageID === mi.Message.MessageID) { + // Approximately equal, for duplicate message-id, typically in Sent and incoming + // from mailing list or when sending to self. + threadChar = '≈' + threadCharTitle = 'Same Message-ID.' + } else if (mi.Message.ThreadMissingLink || (mi.Message.ThreadParentIDs || []).length > 0 && (mi.Message.ThreadParentIDs || [])[0] !== msgitemView.parent.messageitem.Message.ID) { + // Zigzag arrow, e.g. if immediate parent is missing, or when matching was done + // based on subject. + threadChar = '↯' + threadCharTitle = 'Immediate parent message is missing.' + } + } + + // Message is unread if it itself is unread, or it is collapsed and has an unread child message. + const isUnread = () => !mi.Message.Seen || msgitemView.isCollapsedThreadRoot() && !!msgitemView.findDescendant(miv => !miv.messageitem.Message.Seen) + + const isRelevant = () => !mi.Message.ThreadMuted && mi.MatchQuery || (msgitemView.isCollapsedThreadRoot() && msgitemView.findDescendant(miv => !miv.messageitem.Message.ThreadMuted && miv.messageitem.MatchQuery)) + + // Effective receive time to display. For collapsed thread roots, we show the time + // of the newest or oldest message, depending on whether you're viewing + // newest-first or oldest-first messages. + const received = () => { + let r = mi.Message.Received + if (!msgitemView.isCollapsedThreadRoot()) { + return r + } + msgitemView.descendants().forEach(dmiv => { + if (settings.orderAsc && dmiv.messageitem.Message.Received.getTime() < r.getTime()) { + r = dmiv.messageitem.Message.Received + } else if (!settings.orderAsc && dmiv.messageitem.Message.Received.getTime() > r.getTime()) { + r = dmiv.messageitem.Message.Received + } + }) + return r + } + + // For drawing half a thread bar for the last message in the thread. + const isThreadLast = () => { + let miv = msgitemView.threadRoot() + while (miv.kids.length > 0) { + miv = miv.kids[miv.kids.length-1] + } + return miv === msgitemView + } + + // If mailbox of message is not specified in filter (i.e. for a regular mailbox + // view, or search on a mailbox), we show it on the right-side of the subject. For + // collapsed thread roots, we show all additional mailboxes of descendants with + // different style. + const mailboxtags: HTMLElement[] = [] + const mailboxIDs = new Set() + const addMailboxTag = (mb: api.Mailbox, isCollapsedKid: boolean) => { + let name = mb.Name + mailboxIDs.add(mb.ID) + if (name.length > 8+1+3+1+8+4) { + const t = name.split('/') + const first = t[0] + const last = t[t.length-1] + if (first.length + last.length <= 8+8) { + name = first+'/.../'+last + } else { + name = first.substring(0, 8) + '/.../' + last.substring(0, 8) + } + } + const e = dom.span(dom._class('msgitemmailbox'), isCollapsedKid ? dom._class('msgitemmailboxcollapsed') : [], + name === mb.Name ? [] : attr.title(mb.Name), + name, + ) + mailboxtags.push(e) + } + const othermb = otherMailbox(m.MailboxID) + if (othermb) { + addMailboxTag(othermb, false) + } + if (msgitemView.isCollapsedThreadRoot()) { + for (const miv of msgitemView.descendants()) { + const m = miv.messageitem.Message + if (!mailboxIDs.has(m.MailboxID) && otherMailbox(m.MailboxID)) { + const mb = listMailboxes().find(mb => mb.ID === m.MailboxID) + if (!mb) { + throw new ConsistencyError('missing mailbox for message in thread') + } + addMailboxTag(mb, true) + } + } + } + + // When rerendering, we remember active & focus states. So we don't have to make + // the caller also call redraw on MsglistView. + const active = msgitemView.root && msgitemView.root.classList.contains('active') + const focus = msgitemView.root && msgitemView.root.classList.contains('focus') + const elem = dom.div(dom._class('msgitem'), + active ? dom._class('active') : [], + focus ? dom._class('focus') : [], attr.draggable('true'), function dragstart(e: DragEvent) { - e.dataTransfer!.setData('application/vnd.mox.messages', JSON.stringify(msglistView.selected().map(miv => miv.messageitem.Message.ID))) + // We send the Message.ID and MailboxID, so we can decide based on the destination + // mailbox whether to move. We don't move messages already in the destination + // mailbox, and also skip messages in the Sent mailbox when there are also messages + // from other mailboxes. + e.dataTransfer!.setData('application/vnd.mox.messages', JSON.stringify(msglistView.selected().map(miv => [miv.messageitem.Message.MailboxID, miv.messageitem.Message.ID]))) }, - m.Seen ? [] : style({fontWeight: 'bold'}), - dom.div(dom._class('msgitemcell', 'msgitemflags'), flagList(m, msgitemView.messageitem)), + // Thread root with kids can be collapsed/expanded with double click. + settings.threading !== api.ThreadMode.ThreadOff && !msgitemView.parent && msgitemView.kids.length > 0 ? + function dblclick(e: MouseEvent) { + e.stopPropagation() // Prevent selection. + if (settings.threading === api.ThreadMode.ThreadOn) { + // No await, we don't wait for the result. + withStatus('Saving thread expand/collapse', client.ThreadCollapse([msgitemView.messageitem.Message.ID], !msgitemView.collapsed)) + } + if (msgitemView.collapsed) { + msglistView.threadExpand(msgitemView) + } else { + msglistView.threadCollapse(msgitemView) + msglistView.viewportEnsureMessages() + } + } : [], + isUnread() ? style({fontWeight: 'bold'}) : [], + // Relevant means not muted and matching the query. + isRelevant() ? [] : style({opacity: '.4'}), + dom.div(dom._class('msgitemcell', 'msgitemflags'), + dom.div(style({display: 'flex', justifyContent: 'space-between'}), + dom.div(flagList(msgitemView)), + !msgitemView.parent && msgitemView.kids.length > 0 && msgitemView.collapsed ? + dom.clickbutton('' + (1+msgitemView.descendants().length), attr.tabindex('-1'), attr.title('Expand thread.'), attr.arialabel('Expand thread.'), function click(e: MouseEvent) { + e.stopPropagation() // Prevent selection. + if (settings.threading === api.ThreadMode.ThreadOn) { + withStatus('Saving thread expanded', client.ThreadCollapse([msgitemView.messageitem.Message.ID], false)) + } + msglistView.threadExpand(msgitemView) + }) : [], + !msgitemView.parent && msgitemView.kids.length > 0 && !msgitemView.collapsed ? + dom.clickbutton('-', style({width: '1em'}), attr.tabindex('-1'), attr.title('Collapse thread.'), attr.arialabel('Collapse thread.'), function click(e: MouseEvent) { + e.stopPropagation() // Prevent selection. + if (settings.threading === api.ThreadMode.ThreadOn) { + withStatus('Saving thread expanded', client.ThreadCollapse([msgitemView.messageitem.Message.ID], true)) + } + msglistView.threadCollapse(msgitemView) + msglistView.viewportEnsureMessages() + }) : [], + ), + ), dom.div(dom._class('msgitemcell', 'msgitemfrom'), dom.div(style({display: 'flex', justifyContent: 'space-between'}), dom.div(dom._class('msgitemfromtext', 'silenttitle'), + // todo: for collapsed messages, show all participants in thread? attr.title((mi.Envelope.From || []).map(a => formatAddressFull(a)).join(', ')), join((mi.Envelope.From || []).map(a => formatAddressShort(a)), () => ', ') ), identityHeader, ), + // Thread messages are connected by a vertical bar. The first and last message are + // only half the height of the item, to indicate start/end, and so it stands out + // from any thread above/below. + ((msgitemView.parent || msgitemView.kids.length > 0) && !msgitemView.threadRoot().collapsed) ? + dom.div(dom._class('msgitemfromthreadbar'), + !msgitemView.parent ? style({top: '50%', bottom: '-1px'}) : ( + isThreadLast() ? + style({top: '-1px', bottom: '50%'}) : + style({top: '-1px', bottom: '-1px'}) + ) + ) : [] ), dom.div(dom._class('msgitemcell', 'msgitemsubject'), dom.div(style({display: 'flex', justifyContent: 'space-between', position: 'relative'}), dom.div(dom._class('msgitemsubjecttext'), - mi.Envelope.Subject || '(no subject)', + threadIndent > 0 ? dom.span(threadChar, style({paddingLeft: (threadIndent/2)+'em', color: '#444', fontWeight: 'normal'}), threadCharTitle ? attr.title(threadCharTitle) : []) : [], + msgitemView.parent ? [] : mi.Envelope.Subject || '(no subject)', dom.span(dom._class('msgitemsubjectsnippet'), ' '+mi.FirstLine), ), dom.div( keywords, - mailboxtag, + mailboxtags, ), ), ), - dom.div(dom._class('msgitemcell', 'msgitemage'), age(m.Received)), + dom.div(dom._class('msgitemcell', 'msgitemage'), age(received())), function click(e: MouseEvent) { e.preventDefault() e.stopPropagation() msglistView.click(msgitemView, e.ctrlKey, e.shiftKey) } ) + msgitemView.root.replaceWith(elem) + msgitemView.root = elem } const msgitemView: MsgitemView = { root: dom.div(), messageitem: mi, - updateFlags: updateFlags, + receivedTime: receivedTime, + kids: [], + parent: null, + collapsed: initialCollapsed, + + threadRoot: () => { + let miv = msgitemView + while (miv.parent) { + miv = miv.parent + } + return miv + }, + + isCollapsedThreadRoot: () => !msgitemView.parent && msgitemView.collapsed && msgitemView.kids.length > 0, + + descendants: () => { + let l: MsgitemView[] = [] + const walk = (miv: MsgitemView) => { + for (const kmiv of miv.kids) { + l.push(kmiv) + walk(kmiv) + } + } + walk(msgitemView) + return l + }, + + // We often just need to know if a descendant with certain properties exist. No + // need to create an array, then call find on it. + findDescendant: (matchfn) => { + const walk = (miv: MsgitemView): MsgitemView | null => { + if (matchfn(miv)) { + return miv + } + for (const kmiv of miv.kids) { + const r = walk(kmiv) + if (r) { + return r + } + } + return null + } + return walk(msgitemView) + }, + + lastDescendant: () => { + let l = msgitemView + if (l.kids.length === 0) { + return null + } + while(l.kids.length > 0) { + l = l.kids[l.kids.length-1] + } + return l + }, + remove: remove, + render: render, } - msgitemView.root = render() return msgitemView } @@ -1901,8 +2154,8 @@ const newMsgView = (miv: MsgitemView, msglistView: MsglistView, listMailboxes: l ) } - const cmdUp = async () => { msgscrollElem.scrollTo({top: msgscrollElem.scrollTop - 3*msgscrollElem.getBoundingClientRect().height / 4, behavior: 'smooth'}) } - const cmdDown = async () => { msgscrollElem.scrollTo({top: msgscrollElem.scrollTop + 3*msgscrollElem.getBoundingClientRect().height / 4, behavior: 'smooth'}) } + const cmdUp = async () => { msgscrollElem.scrollTo({top: msgscrollElem.scrollTop - 3*msgscrollElem.getBoundingClientRect().height / 4, behavior: 'smooth'}) } + const cmdDown = async () => { msgscrollElem.scrollTo({top: msgscrollElem.scrollTop + 3*msgscrollElem.getBoundingClientRect().height / 4, behavior: 'smooth'}) } const cmdHome = async () => { msgscrollElem.scrollTo({top: 0 }) } const cmdEnd = async () => { msgscrollElem.scrollTo({top: msgscrollElem.scrollHeight}) } @@ -1931,9 +2184,9 @@ const newMsgView = (miv: MsgitemView, msglistView: MsglistView, listMailboxes: l d: msglistView.cmdTrash, D: msglistView.cmdDelete, q: msglistView.cmdJunk, - n: msglistView.cmdMarkNotJunk, - u: msglistView.cmdMarkUnread, + Q: msglistView.cmdMarkNotJunk, m: msglistView.cmdMarkRead, + M: msglistView.cmdMarkUnread, } let urlType: string // text, html, htmlexternal; for opening in new tab/print @@ -1991,8 +2244,10 @@ const newMsgView = (miv: MsgitemView, msglistView: MsglistView, listMailboxes: l [ dom.clickbutton('Print', attr.title('Print message, opens in new tab and opens print dialog.'), clickCmd(cmdPrint, shortcuts)), dom.clickbutton('Mark Not Junk', attr.title('Mark as not junk, causing this message to be used in spam classification of new incoming messages.'), clickCmd(msglistView.cmdMarkNotJunk, shortcuts)), - dom.clickbutton('Mark as read', clickCmd(msglistView.cmdMarkRead, shortcuts)), - dom.clickbutton('Mark as unread', clickCmd(msglistView.cmdMarkUnread, shortcuts)), + dom.clickbutton('Mark Read', clickCmd(msglistView.cmdMarkRead, shortcuts)), + dom.clickbutton('Mark Unread', clickCmd(msglistView.cmdMarkUnread, shortcuts)), + dom.clickbutton('Mute thread', clickCmd(msglistView.cmdMute, shortcuts)), + dom.clickbutton('Unmute thread', clickCmd(msglistView.cmdUnmute, shortcuts)), dom.clickbutton('Open in new tab', clickCmd(cmdOpenNewTab, shortcuts)), dom.clickbutton('Show raw original message in new tab', clickCmd(cmdOpenRaw, shortcuts)), dom.clickbutton('Show internals in popup', clickCmd(cmdShowInternals, shortcuts)), @@ -2039,7 +2294,7 @@ const newMsgView = (miv: MsgitemView, msglistView: MsglistView, listMailboxes: l 'image/apng', 'image/svg+xml', ] - const isImage = (a: api.Attachment) => imageTypes.includes((a.Part.MediaType + '/' + a.Part.MediaSubType).toLowerCase()) + const isImage = (a: api.Attachment) => imageTypes.includes((a.Part.MediaType + '/' + a.Part.MediaSubType).toLowerCase()) const isPDF = (a: api.Attachment) => (a.Part.MediaType+'/'+a.Part.MediaSubType).toLowerCase() === 'application/pdf' const isViewable = (a: api.Attachment) => isImage(a) || isPDF(a) const attachments: api.Attachment[] = (mi.Attachments || []) @@ -2359,26 +2614,50 @@ const newMsgView = (miv: MsgitemView, msglistView: MsglistView, listMailboxes: l // archive/trash/junk. Focus is typically on the last clicked message, but can be // changed with keyboard interaction without changing selected messages. // -// We just have one MsglistView, that is updated when a -// different mailbox/search query is opened. +// With threading enabled, we show the messages in a thread below each other. A +// thread can have multiple "thread roots": messages without a parent message. This +// can occur if a parent message with multiple kids is permanently removed. We also +// show messages from the same thread but a different mailbox. A thread root can be +// collapsed, independently of collapsed state of other thread roots. We order +// thread roots, and kids/siblings, by received timestamp. +// +// For incoming changes (add/remove of messages), we update the thread view in a +// way that resembles a fresh mailbox load as much as possible. Exceptions: If a +// message is removed, and there are thread messages remaining, but they are all in +// other mailboxes (or don't match the search query), we still show the remaining +// messages. If you would load the mailbox/search query again, you would not see +// those remaining messages. Also, if a new message is delivered to a thread, the +// thread isn't moved. After a refresh, the thread would be the most recent (at the +// top for the default sorting). +// +// When updating the UI for threaded messages, we often take this simple approach: +// Remove a subtree of messages from the UI, sort their data structures, and add +// them to the UI again. That saves tricky code that would need to make just the +// exact changes needed. +// +// We just have one MsglistView, that is updated when a different mailbox/search +// query is opened. interface MsglistView { root: HTMLElement updateFlags: (mailboxID: number, uid: number, modseq: number, mask: api.Flags, flags: api.Flags, keywords: string[]) => void - addMessageItems: (messageItems: api.MessageItem[]) => void + addMessageItems: (messageItems: (api.MessageItem[] | null)[], isChange: boolean, requestMsgID: number) => void removeUIDs: (mailboxID: number, uids: number[]) => void + updateMessageThreadFields: (messageIDs: number[], muted: boolean, collapsed: boolean) => void activeMessageID: () => number // For single message selected, otherwise returns 0. redraw: (miv: MsgitemView) => void // To be called after updating flags or focus/active state, rendering it again. - anchorMessageID: () => number // For next request, for more messages. - addMsgitemViews: (mivs: MsgitemView[]) => void clear: () => void // Clear all messages, reset focus/active state. unselect: () => void select: (miv: MsgitemView) => void selected: () => MsgitemView[] - openMessage: (miv: MsgitemView, initial: boolean, parsedMessageOpt?: api.ParsedMessage) => void + openMessage: (parsedMessage: api.ParsedMessage) => boolean click: (miv: MsgitemView, ctrl: boolean, shift: boolean) => void key: (k: string, e: KeyboardEvent) => void mailboxes: () => api.Mailbox[] itemHeight: () => number // For calculating how many messageitems to request to load next view. + threadExpand: (miv: MsgitemView) => void + threadCollapse: (miv: MsgitemView) => void + threadToggle: () => void // Toggle threads based on state. + viewportEnsureMessages: () => Promise // Load more messages if last message is near the end of the viewport. // Exported for MsgView. cmdArchive: () => Promise @@ -2388,28 +2667,57 @@ interface MsglistView { cmdMarkNotJunk: () => Promise cmdMarkRead: () => Promise cmdMarkUnread: () => Promise + cmdMute: () => Promise + cmdUnmute: () => Promise } -const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setLocationHash: setLocationHash, otherMailbox: otherMailbox, possibleLabels: possibleLabels, scrollElemHeight: () => number, refineKeyword: (kw: string) => Promise): MsglistView => { - // These contain one msgitemView or an array of them. - // Zero or more selected msgitemViews. If there is a single message, its content is - // shown. If there are multiple, just the count is shown. These are in order of - // being added, not in order of how they are shown in the list. This is needed to - // handle selection changes with the shift key. +const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setLocationHash: setLocationHash, otherMailbox: otherMailbox, possibleLabels: possibleLabels, scrollElemHeight: () => number, refineKeyword: (kw: string) => Promise, viewportEnsureMessages: () => Promise): MsglistView => { + // msgitemViews holds all visible item views: All thread roots, and kids only if + // the thread is expanded, in order of descendants. All descendants of a collapsed + // root are in collapsedMsgitemViews, unsorted. Having msgitemViews as a list is + // convenient for reasoning about the visible items, and handling changes to the + // selected messages. + // When messages for a thread are all non-matching the query, we no longer show it + // (e.g. when moving a thread to Archive), but we keep the messages around in + // oldThreadMessageItems, so an update to the thread (e.g. new delivery) can + // resurrect the messages. + let msgitemViews: MsgitemView[] = [] // Only visible msgitems, in order on screen. + let collapsedMsgitemViews: MsgitemView[] = [] // Invisible messages because collapsed, unsorted. + let oldThreadMessageItems: api.MessageItem[] = [] // Messages from threads removed from view. + + // selected holds the messages that are selected, zero or more. If there is a + // single message, its content is shown. If there are multiple, just the count is + // shown. These are in order of being added, not in order of how they are shown in + // the list. This is needed to handle selection changes with the shift key. For + // collapsed thread roots, only that root will be in this list. The effective + // selection must always expand descendants, use mlv.selected() to gather all. let selected: MsgitemView[] = [] - // MsgitemView last interacted with, or the first when messages are loaded. Always - // set when there is a message. Used for shift+click to expand selection. + // Focus is the message last interacted with, or the first when messages are + // loaded. Always set when there is a message. Used for shift+click to expand + // selection. let focus: MsgitemView | null = null - let msgitemViews: MsgitemView[] = [] let msgView: MsgView | null = null + // Messages for actions like "archive", "trash", "move to...". We skip messages + // that are (already) in skipMBID. And we skip messages that are in the designated + // Sent mailbox, unless there is only one selected message or the view is for the + // Sent mailbox, then it must be intentional. + const moveActionMsgIDs = (skipMBID: number) => { + const sentMailboxID = listMailboxes().find(mb => mb.Sent)?.ID + const effselected = mlv.selected() + return effselected + .filter(miv => miv.messageitem.Message.MailboxID !== skipMBID) + .map(miv => miv.messageitem.Message) + .filter(m => effselected.length === 1 || !sentMailboxID || m.MailboxID !== sentMailboxID || !otherMailbox(sentMailboxID)) + .map(m => m.ID) + } + const cmdArchive = async () => { const mb = listMailboxes().find(mb => mb.Archive) if (mb) { - const msgIDs = selected.filter(miv => miv.messageitem.Message.MailboxID !== mb.ID).map(miv => miv.messageitem.Message.ID) - await withStatus('Moving to archive mailbox', client.MessageMove(msgIDs, mb.ID)) + await withStatus('Moving to archive mailbox', client.MessageMove(moveActionMsgIDs(mb.ID), mb.ID)) } else { window.alert('No mailbox configured for archiving yet.') } @@ -2418,13 +2726,12 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL if (!confirm('Are you sure you want to permanently delete?')) { return } - await withStatus('Permanently deleting messages', client.MessageDelete(selected.map(miv => miv.messageitem.Message.ID))) + await withStatus('Permanently deleting messages', client.MessageDelete(mlv.selected().map(miv => miv.messageitem.Message.ID))) } const cmdTrash = async () => { const mb = listMailboxes().find(mb => mb.Trash) if (mb) { - const msgIDs = selected.filter(miv => miv.messageitem.Message.MailboxID !== mb.ID).map(miv => miv.messageitem.Message.ID) - await withStatus('Moving to trash mailbox', client.MessageMove(msgIDs, mb.ID)) + await withStatus('Moving to trash mailbox', client.MessageMove(moveActionMsgIDs(mb.ID), mb.ID)) } else { window.alert('No mailbox configured for trash yet.') } @@ -2432,25 +2739,221 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL const cmdJunk = async () => { const mb = listMailboxes().find(mb => mb.Junk) if (mb) { - const msgIDs = selected.filter(miv => miv.messageitem.Message.MailboxID !== mb.ID).map(miv => miv.messageitem.Message.ID) - await withStatus('Moving to junk mailbox', client.MessageMove(msgIDs, mb.ID)) + await withStatus('Moving to junk mailbox', client.MessageMove(moveActionMsgIDs(mb.ID), mb.ID)) } else { window.alert('No mailbox configured for junk yet.') } } - const cmdMarkNotJunk = async () => { await withStatus('Marking as not junk', client.FlagsAdd(selected.map(miv => miv.messageitem.Message.ID), ['$notjunk'])) } - const cmdMarkRead = async () => { await withStatus('Marking as read', client.FlagsAdd(selected.map(miv => miv.messageitem.Message.ID), ['\\seen'])) } - const cmdMarkUnread = async () => { await withStatus('Marking as not read', client.FlagsClear(selected.map(miv => miv.messageitem.Message.ID), ['\\seen'])) } + const cmdMarkNotJunk = async () => { await withStatus('Marking as not junk', client.FlagsAdd(mlv.selected().map(miv => miv.messageitem.Message.ID), ['$notjunk'])) } + const cmdMarkRead = async () => { await withStatus('Marking as read', client.FlagsAdd(mlv.selected().map(miv => miv.messageitem.Message.ID), ['\\seen'])) } + const cmdMarkUnread = async () => { await withStatus('Marking as not read', client.FlagsClear(mlv.selected().map(miv => miv.messageitem.Message.ID), ['\\seen'])) } + const cmdMute = async () => { + const l = mlv.selected() + await withStatus('Muting thread', client.ThreadMute(l.map(miv => miv.messageitem.Message.ID), true)) + const oldstate = state() + for (const miv of l) { + if (!miv.parent && miv.kids.length > 0 && !miv.collapsed) { + threadCollapse(miv, false) + } + } + updateState(oldstate) + viewportEnsureMessages() + } + const cmdUnmute = async () => { await withStatus('Unmuting thread', client.ThreadMute(mlv.selected().map(miv => miv.messageitem.Message.ID), false)) } + + const seletedRoots = () => { + const mivs: MsgitemView[] = [] + mlv.selected().forEach(miv => { + const mivroot = miv.threadRoot() + if (!mivs.includes(mivroot)) { + mivs.push(mivroot) + } + }) + return mivs + } + + const cmdToggleMute = async () => { + if (settings.threading === api.ThreadMode.ThreadOff) { + alert('Toggle muting threads is only available when threading is enabled.') + return + } + const rootmivs = seletedRoots() + const unmuted = !!rootmivs.find(miv => !miv.messageitem.Message.ThreadMuted) + await withStatus(unmuted ? 'Muting' : 'Unmuting', client.ThreadMute(rootmivs.map(miv => miv.messageitem.Message.ID), unmuted ? true : false)) + if (unmuted) { + const oldstate = state() + rootmivs.forEach(miv => { + if (!miv.collapsed) { + threadCollapse(miv, false) + } + }) + updateState(oldstate) + viewportEnsureMessages() + } + } + + const cmdToggleCollapse = async () => { + if (settings.threading === api.ThreadMode.ThreadOff) { + alert('Toggling thread collapse/expand is only available when threading is enabled.') + return + } + + const rootmivs = seletedRoots() + const collapse = !!rootmivs.find(miv => !miv.collapsed) + + const oldstate = state() + if (collapse) { + rootmivs.forEach(miv => { + if (!miv.collapsed) { + threadCollapse(miv, false) + } + }) + selected = rootmivs + if (focus) { + focus = focus.threadRoot() + } + viewportEnsureMessages() + } else { + rootmivs.forEach(miv => { + if (miv.collapsed) { + threadExpand(miv, false) + } + }) + } + updateState(oldstate) + + if (settings.threading === api.ThreadMode.ThreadOn) { + const action = collapse ? 'Collapsing' : 'Expanding' + await withStatus(action, client.ThreadCollapse(rootmivs.map(miv => miv.messageitem.Message.ID), collapse)) + } + } + + const cmdSelectThread = async () => { + if (!focus) { + return + } + + const oldstate = state() + selected = msgitemViews.filter(miv => miv.messageitem.Message.ThreadID === focus!.messageitem.Message.ThreadID) + updateState(oldstate) + } const shortcuts: {[key: string]: command} = { d: cmdTrash, Delete: cmdTrash, D: cmdDelete, - q: cmdJunk, a: cmdArchive, - n: cmdMarkNotJunk, - u: cmdMarkUnread, + q: cmdJunk, + Q: cmdMarkNotJunk, m: cmdMarkRead, + M: cmdMarkUnread, + X: cmdToggleMute, + C: cmdToggleCollapse, + S: cmdSelectThread, + } + + // After making changes, this function looks through the data structure for + // inconsistencies. Useful during development. + const checkConsistency = (checkSelection: boolean) => { + if (!settings.checkConsistency) { + return + } + + // Check for duplicates in msgitemViews. + const mivseen = new Set() + const threadActive = new Set() + for (const miv of msgitemViews) { + const id = miv.messageitem.Message.ID + if (mivseen.has(id)) { + log('duplicate Message.ID', {id: id, mivseenSize: mivseen.size}) + throw new ConsistencyError('duplicate Message.ID in msgitemViews') + } + mivseen.add(id) + if (!miv.root.parentNode) { + throw new ConsistencyError('msgitemView.root not in dom') + } + threadActive.add(miv.messageitem.Message.ThreadID) + } + + // Check for duplicates in collapsedMsgitemViews, and whether also in msgitemViews. + const colseen = new Set() + for (const miv of collapsedMsgitemViews) { + const id = miv.messageitem.Message.ID + if (colseen.has(id)) { + throw new ConsistencyError('duplicate Message.ID in collapsedMsgitemViews') + } + colseen.add(id) + if (mivseen.has(id)) { + throw new ConsistencyError('Message.ID in both collapsedMsgitemViews and msgitemViews') + } + threadActive.add(miv.messageitem.Message.ThreadID) + } + + if (settings.threading !== api.ThreadMode.ThreadOff) { + const oldseen = new Set() + for (const mi of oldThreadMessageItems) { + const id = mi.Message.ID + if (oldseen.has(id)) { + throw new ConsistencyError('duplicate Message.ID in oldThreadMessageItems') + } + oldseen.add(id) + if (mivseen.has(id)) { + throw new ConsistencyError('Message.ID in both msgitemViews and oldThreadMessageItems') + } + if (colseen.has(id)) { + throw new ConsistencyError('Message.ID in both collapsedMsgitemViews and oldThreadMessageItems') + } + + if (threadActive.has(mi.Message.ThreadID)) { + throw new ConsistencyError('threadid both in active and in old thread list') + } + } + } + + // Walk all (collapsed) msgitemViews, check each and their descendants are in + // msgitemViews at the correct position, or in collapsedmsgitemViews. + msgitemViews.forEach((miv, i) => { + if (miv.collapsed) { + for (const dmiv of miv.descendants()) { + if (!colseen.has(dmiv.messageitem.Message.ID)) { + throw new ConsistencyError('descendant message id missing from collapsedMsgitemViews') + } + } + return + } + for (const dmiv of miv.descendants()) { + i++ + if (!mivseen.has(dmiv.messageitem.Message.ID)) { + throw new ConsistencyError('descendant missing from msgitemViews') + } + if (msgitemViews[i] !== dmiv) { + throw new ConsistencyError('descendant not at expected position in msgitemViews') + } + } + }) + + if (!checkSelection) { + return + } + + // Check all selected & focus exists. + const selseen = new Set() + for (const miv of selected) { + const id = miv.messageitem.Message.ID + if (selseen.has(id)) { + throw new ConsistencyError('duplicate miv in selected') + } + selseen.add(id) + if (!mivseen.has(id)) { + throw new ConsistencyError('selected id not in msgitemViews') + } + } + if (focus) { + const id = focus.messageitem.Message.ID + if (!mivseen.has(id)) { + throw new ConsistencyError('focus set to unknown miv') + } + } } type state = { @@ -2461,7 +2964,7 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL // Return active & focus state, and update the UI after changing state. const state = (): state => { const active: {[key: string]: MsgitemView} = {} - for (const miv of selected) { + for (const miv of mlv.selected()) { active[miv.messageitem.Message.ID] = miv } return {active: active, focus: focus} @@ -2492,61 +2995,66 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL } } - if (initial && selected.length === 1) { - mlv.redraw(selected[0]) + const effselected = mlv.selected() + if (initial && effselected.length === 1) { + mlv.redraw(effselected[0]) } - if (activeChanged) { - if (msgView) { - msgView.aborter.abort() - } - msgView = null + checkConsistency(true) - if (selected.length === 0) { - dom._kids(msgElem) - } else if (selected.length === 1) { - msgElem.classList.toggle('loading', true) - const loaded = () => { msgElem.classList.toggle('loading', false) } - msgView = newMsgView(selected[0], mlv, listMailboxes, possibleLabels, loaded, refineKeyword, parsedMessageOpt) - dom._kids(msgElem, msgView) - } else { - const trashMailboxID = listMailboxes().find(mb => mb.Trash)?.ID - const allTrash = trashMailboxID && !selected.find(miv => miv.messageitem.Message.MailboxID !== trashMailboxID) - dom._kids(msgElem, + if (!activeChanged) { + return + } + if (msgView) { + msgView.aborter.abort() + } + msgView = null + + if (effselected.length === 0) { + dom._kids(msgElem) + } else if (effselected.length === 1) { + msgElem.classList.toggle('loading', true) + const loaded = () => { msgElem.classList.toggle('loading', false) } + msgView = newMsgView(effselected[0], mlv, listMailboxes, possibleLabels, loaded, refineKeyword, parsedMessageOpt) + dom._kids(msgElem, msgView) + } else { + const trashMailboxID = listMailboxes().find(mb => mb.Trash)?.ID + const allTrash = trashMailboxID && !effselected.find(miv => miv.messageitem.Message.MailboxID !== trashMailboxID) + dom._kids(msgElem, + dom.div( + attr.role('region'), attr.arialabel('Buttons for multiple messages'), + style({position: 'absolute', top: 0, right: 0, bottom: 0, left: 0, display: 'flex', alignItems: 'center', justifyContent: 'center'}), dom.div( - attr.role('region'), attr.arialabel('Buttons for multiple messages'), - style({position: 'absolute', top: 0, right: 0, bottom: 0, left: 0, display: 'flex', alignItems: 'center', justifyContent: 'center'}), + style({padding: '4ex', backgroundColor: 'white', borderRadius: '.25em', border: '1px solid #ccc'}), dom.div( - style({padding: '4ex', backgroundColor: 'white', borderRadius: '.25em', border: '1px solid #ccc'}), - dom.div( - style({textAlign: 'center', marginBottom: '4ex'}), - ''+selected.length+' messages selected', - ), - dom.div( - dom.clickbutton('Archive', attr.title('Move to the Archive mailbox.'), clickCmd(cmdArchive, shortcuts)), ' ', - allTrash ? - dom.clickbutton('Delete', attr.title('Permanently delete messages.'), clickCmd(cmdDelete, shortcuts)) : - dom.clickbutton('Trash', attr.title('Move to the Trash mailbox.'), clickCmd(cmdTrash, shortcuts)), - ' ', - dom.clickbutton('Junk', attr.title('Move to Junk mailbox, marking as junk and causing this message to be used in spam classification of new incoming messages.'), clickCmd(cmdJunk, shortcuts)), ' ', - dom.clickbutton('Move to...', function click(e: MouseEvent) { - movePopover(e, listMailboxes(), selected.map(miv => miv.messageitem.Message)) - }), ' ', - dom.clickbutton('Labels...', attr.title('Add/remove labels ...'), function click(e: MouseEvent) { - labelsPopover(e, selected.map(miv => miv.messageitem.Message), possibleLabels) - }), ' ', - dom.clickbutton('Mark Not Junk', attr.title('Mark as not junk, causing this message to be used in spam classification of new incoming messages.'), clickCmd(cmdMarkNotJunk, shortcuts)), ' ', - dom.clickbutton('Mark read', clickCmd(cmdMarkRead, shortcuts)), ' ', - dom.clickbutton('Mark unread', clickCmd(cmdMarkUnread, shortcuts)), - ), + style({textAlign: 'center', marginBottom: '4ex'}), + ''+effselected.length+' messages selected', + ), + dom.div( + dom.clickbutton('Archive', attr.title('Move to the Archive mailbox. Messages in the designated Sent mailbox are only moved if a single message is selected, or the current mailbox is the Sent mailbox.'), clickCmd(cmdArchive, shortcuts)), ' ', + allTrash ? + dom.clickbutton('Delete', attr.title('Permanently delete messages.'), clickCmd(cmdDelete, shortcuts)) : + dom.clickbutton('Trash', attr.title('Move to the Trash mailbox. Messages in the designated Sent mailbox are only moved if a single message is selected, or the current mailbox is the Sent mailbox.'), clickCmd(cmdTrash, shortcuts)), + ' ', + dom.clickbutton('Junk', attr.title('Move to Junk mailbox, marking as junk and causing this message to be used in spam classification of new incoming messages. Messages in the designated Sent mailbox are only moved if a single message is selected, or the current mailbox is the Sent mailbox.'), clickCmd(cmdJunk, shortcuts)), ' ', + dom.clickbutton('Move to...', function click(e: MouseEvent) { + const sentMailboxID = listMailboxes().find(mb => mb.Sent)?.ID + movePopover(e, listMailboxes(), effselected.map(miv => miv.messageitem.Message).filter(m => effselected.length === 1 || !sentMailboxID || m.MailboxID !== sentMailboxID || !otherMailbox(sentMailboxID))) + }), ' ', + dom.clickbutton('Labels...', attr.title('Add/remove labels ...'), function click(e: MouseEvent) { + labelsPopover(e, effselected.map(miv => miv.messageitem.Message), possibleLabels) + }), ' ', + dom.clickbutton('Mark Not Junk', attr.title('Mark as not junk, causing this message to be used in spam classification of new incoming messages.'), clickCmd(cmdMarkNotJunk, shortcuts)), ' ', + dom.clickbutton('Mark Read', clickCmd(cmdMarkRead, shortcuts)), ' ', + dom.clickbutton('Mark Unread', clickCmd(cmdMarkUnread, shortcuts)), ' ', + dom.clickbutton('Mute thread', clickCmd(cmdMute, shortcuts)), ' ', + dom.clickbutton('Unmute thread', clickCmd(cmdUnmute, shortcuts)), ), ), - ) - } - } - if (activeChanged) { - setLocationHash() + ), + ) } + setLocationHash() } // Moves the currently focused msgitemView, without changing selection. @@ -2556,100 +3064,715 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL updateState(oldstate) } + const threadExpand = (miv: MsgitemView, changeState: boolean) => { + if (miv.parent) { + throw new ConsistencyError('cannot expand non-root') + } + + const oldstate = state() + + miv.collapsed = false + const mivl = miv.descendants() + miv.render() + mivl.forEach(dmiv => dmiv.render()) + for (const miv of mivl) { + collapsedMsgitemViews.splice(collapsedMsgitemViews.indexOf(miv), 1) + } + const pi = msgitemViews.indexOf(miv) + msgitemViews.splice(pi+1, 0, ...mivl) + const next = miv.root.nextSibling + for (const miv of mivl) { + mlv.root.insertBefore(miv.root, next) + } + + if (changeState) { + updateState(oldstate) + } + } + const threadCollapse = (miv: MsgitemView, changeState: boolean) => { + if (miv.parent) { + throw new ConsistencyError('cannot expand non-root') + } + const oldstate = state() + + miv.collapsed = true + const mivl = miv.descendants() + + collapsedMsgitemViews.push(...mivl) + // If miv or any child was selected, ensure collapsed thread root is also selected. + let select = [miv, ...mivl].find(xmiv => selected.indexOf(xmiv) >= 0) + let seli = selected.length // Track first index of already selected miv, which is where we insert the thread root if needed, to keep order. + msgitemViews.splice(msgitemViews.indexOf(miv)+1, mivl.length) + for (const dmiv of mivl) { + dmiv.remove() + + if (focus === dmiv) { + focus = miv + } + const si = selected.indexOf(dmiv) + if (si >= 0) { + if (si < seli) { + seli = si + } + selected.splice(si, 1) + } + } + if (select) { + const si = selected.indexOf(miv) + if (si < 0) { + selected.splice(seli, 0, miv) + } + } + + // Selected messages may have changed. + if (changeState) { + updateState(oldstate) + } + + // Render remaining thread root, with tree size, effective received age/unread state. + miv.render() + } + + const threadToggle = () => { + const oldstate = state() + const roots = msgitemViews.filter(miv => !miv.parent && miv.kids.length > 0) + roots.forEach(miv => { + let wantCollapsed = miv.messageitem.Message.ThreadCollapsed + if (settings.threading === api.ThreadMode.ThreadUnread) { + wantCollapsed = !miv.messageitem.Message.Seen && !miv.findDescendant(miv => !miv.messageitem.Message.Seen) + } + if (miv.collapsed === wantCollapsed) { + return + } + if (wantCollapsed) { + threadCollapse(miv, false) + } else { + threadExpand(miv, false) + } + }) + updateState(oldstate) + viewportEnsureMessages() + } + + const removeSelected = (miv: MsgitemView) => { + const si = selected.indexOf(miv) + if (si >= 0) { + selected.splice(si, 1) + } + if (focus === miv) { + const i = msgitemViews.indexOf(miv) + if (i > 0) { + focus = msgitemViews[i-1] + } else if (i+1 < msgitemViews.length) { + focus = msgitemViews[i+1] + } else { + focus = null + } + } + } + + // Removes message from either msgitemViews, collapsedMsgitemViews, + // oldThreadMessageItems, and updates UI. + // Returns ThreadID of removed message if active (expanded or collapsed), or 0 otherwise. + const removeUID = (mailboxID: number, uid: number) => { + const match = (miv: MsgitemView) => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid + + const ci = collapsedMsgitemViews.findIndex(match) + if (ci >= 0) { + const miv = collapsedMsgitemViews[ci] + removeCollapsed(ci) + return miv.messageitem.Message.ThreadID + } + + const i = msgitemViews.findIndex(match) + if (i >= 0) { + const miv = msgitemViews[i] + removeExpanded(i) + return miv.messageitem.Message.ThreadID + } + + const ti = oldThreadMessageItems.findIndex(mi => mi.Message.MailboxID === mailboxID && mi.Message.UID === uid) + if (ti >= 0) { + oldThreadMessageItems.splice(ti, 1) + } + return 0 + } + + // Removes message from collapsedMsgitemView and UI at given index, placing + // messages in oldThreadMessageItems. + const removeCollapsed = (ci: number) => { + // Message is collapsed. That means it isn't visible, and neither are its children, + // and it has a parent. So we just merge the kids with those of the parent. + const miv = collapsedMsgitemViews[ci] + collapsedMsgitemViews.splice(ci, 1) + removeSelected(miv) + const trmiv = miv.threadRoot() // To rerender, below. + const pmiv = miv.parent + if (!pmiv) { + throw new ConsistencyError('removing collapsed miv, but has no parent') + } + miv.parent = null // Strict cleanup. + const pki = pmiv.kids.indexOf(miv) + if (pki < 0) { + throw new ConsistencyError('miv not in parent.kids') + } + pmiv.kids.splice(pki, 1, ...miv.kids) // In parent, replace miv with its kids. + miv.kids.forEach(kmiv => kmiv.parent = pmiv) // Give kids their new parent. + miv.kids = [] // Strict cleanup. + pmiv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()) // Sort new list of kids. + trmiv.render() // For count, unread state. + return + } + + // Remove message from msgitemViews and UI at the index i. + const removeExpanded = (i: number) => { + log('removeExpanded', {i}) + // Note: If we remove a message we may be left with only messages from another + // mailbox. We'll leave it, new messages could be delivered for that thread. It + // would be strange to see the remaining messages of the thread disappear. + + const miv = msgitemViews[i] + removeSelected(miv) + const pmiv = miv.parent + miv.parent = null + if (miv.kids.length === 0) { + // No kids, easy case, just remove this leaf message. + miv.remove() + msgitemViews.splice(i, 1) + if (pmiv) { + const pki = pmiv.kids.indexOf(miv) + if (pki < 0) { + throw new ConsistencyError('miv not in parent.kids') + } + pmiv.kids.splice(pki, 1) // Remove miv from parent's kids. + miv.parent = null // Strict cleanup. + pmiv.render() // Update counts. + } + return + } + if (!pmiv) { + // If the kids no longer have a parent and become thread roots we leave them in + // their original location. + const next = miv.root.nextSibling + miv.remove() + msgitemViews.splice(i, 1) + if (miv.collapsed) { + msgitemViews.splice(i, 0, ...miv.kids) + for (const kmiv of miv.kids) { + const pki = collapsedMsgitemViews.indexOf(kmiv) + if (pki < 0) { + throw new ConsistencyError('cannot find collapsed kid in collapsedMsgitemViews') + } + collapsedMsgitemViews.splice(pki, 1) + kmiv.collapsed = true + kmiv.parent = null + kmiv.render() + mlv.root.insertBefore(kmiv.root, next) + } + } else { + // Note: if not collapsed, we leave the kids in the original position in msgitemViews. + miv.kids.forEach(kmiv => { + kmiv.collapsed = false + kmiv.parent = null + kmiv.render() + const lastDesc = kmiv.lastDescendant() + if (lastDesc) { + // Update end of thread bar. + lastDesc.render() + } + }) + } + miv.kids = [] // Strict cleanup. + return + } + + // If the kids will have a parent, we insert them at the expected location in + // between parent's existing kids. It is easiest just to take out all kids, add the + // new ones, sort kids, and add back the subtree. + const odmivs = pmiv.descendants() // Old direct descendants of parent. This includes miv and kids, and other kids, and miv siblings. + const pi = msgitemViews.indexOf(pmiv) + if (pi < 0) { + throw new ConsistencyError('cannot find parent of removed miv') + } + msgitemViews.splice(pi+1, odmivs.length) // Remove all old descendants, we'll add an updated list later. + const pki = pmiv.kids.indexOf(miv) + if (pki < 0) { + throw new Error('did not find miv in parent.kids') + } + pmiv.kids.splice(pki, 1) // Remove miv from parent's kids. + pmiv.kids.push(...miv.kids) // Add miv.kids to parent's kids. + miv.kids.forEach(kmiv => { kmiv.parent = pmiv }) // Set new parent for miv kids. + miv.kids = [] // Strict cleanup. + pmiv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()) + const ndmivs = pmiv.descendants() // Excludes miv, that we are removing. + if (ndmivs.length !== odmivs.length-1) { + throw new ConsistencyError('unexpected new descendants counts during remove') + } + msgitemViews.splice(pi+1, 0, ...ndmivs) // Add all new/current descedants. There is one less than in odmivs. + odmivs.forEach(ndimv => ndimv.remove()) + const next = pmiv.root.nextSibling + for (const ndmiv of ndmivs) { + mlv.root.insertBefore(ndmiv.root, next) + } + pmiv.render() + ndmivs.forEach(dmiv => dmiv.render()) + } + + + // If there are no query-matching messages left for this thread, remove the + // remaining messages from view and keep them around for future deliveries for the + // thread. + const possiblyTakeoutOldThreads = (threadIDs: Set) => { + const hasMatch = (mivs: MsgitemView[], threadID: number) => mivs.find(miv => miv.messageitem.Message.ThreadID === threadID && miv.messageitem.MatchQuery) + const takeoutOldThread = (mivs: MsgitemView[], threadID: number, visible: boolean) => { + let i = 0 + while (i < mivs.length) { + const miv = mivs[i] + const mi = miv.messageitem + const m = mi.Message + if (threadID !== m.ThreadID) { + i++ + continue + } + mivs.splice(i, 1) + if (visible) { + miv.remove() + } + if (focus === miv) { + focus = null + if (i < mivs.length) { + focus = mivs[i] + } else if (i > 0) { + focus = mivs[i-1] + } + const si = selected.indexOf(miv) + if (si >= 0) { + selected.splice(si, 1) + } + } + // Strict cleanup. + miv.parent = null + miv.kids = [] + oldThreadMessageItems.push(mi) + log('took out old thread message', {mi}) + } + } + + for (const threadID of threadIDs) { + if (hasMatch(msgitemViews, threadID) || hasMatch(collapsedMsgitemViews, threadID)) { + log('still have query-matching message for thread', {threadID}) + continue + } + takeoutOldThread(msgitemViews, threadID, true) + takeoutOldThread(collapsedMsgitemViews, threadID, false) + } + } + const mlv: MsglistView = { root: dom.div(), updateFlags: (mailboxID: number, uid: number, modseq: number, mask: api.Flags, flags: api.Flags, keywords: string[]) => { + const updateMessageFlags = (m: api.Message) => { + m.ModSeq = modseq + const maskobj = mask as unknown as {[key: string]: boolean} + const flagsobj = flags as unknown as {[key: string]: boolean} + const mobj = m as unknown as {[key: string]: boolean} + for (const k in maskobj) { + if (maskobj[k]) { + mobj[k] = flagsobj[k] + } + } + m.Keywords = keywords + } + // todo optimize: keep mapping of uid to msgitemView for performance. instead of using Array.find - const miv = msgitemViews.find(miv => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid) + let miv = msgitemViews.find(miv => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid) if (!miv) { - // Happens for messages outside of view. - log('could not find msgitemView for uid', uid) + miv = collapsedMsgitemViews.find(miv => miv.messageitem.Message.MailboxID === mailboxID && miv.messageitem.Message.UID === uid) + } + if (miv) { + updateMessageFlags(miv.messageitem.Message) + miv.render() + if (miv.parent) { + const tr = miv.threadRoot() + if (tr.collapsed) { + tr.render() + } + } + if (msgView && msgView.messageitem.Message.ID === miv.messageitem.Message.ID) { + msgView.updateKeywords(modseq, keywords) + } return } - miv.updateFlags(modseq, mask, flags, keywords) - if (msgView && msgView.messageitem.Message.ID === miv.messageitem.Message.ID) { - msgView.updateKeywords(modseq, keywords) + const mi = oldThreadMessageItems.find(mi => mi.Message.MailboxID === mailboxID && mi.Message.UID === uid) + if (mi) { + updateMessageFlags(mi.Message) + } else { + // Happens for messages outside of view. + log('could not find msgitemView for uid', uid) } }, - addMessageItems: (messageItems: api.MessageItem[]) => { + // Add messages to view, either messages to fill the view with complete threads, or + // individual messages delivered later. + addMessageItems: (messageItems: (api.MessageItem[] | null)[], isChange: boolean, requestMsgID: number) => { if (messageItems.length === 0) { return } - messageItems.forEach(mi => { - const miv = newMsgitemView(mi, mlv, otherMailbox(mi.Message.MailboxID)) - const orderNewest = !settings.orderAsc - const tm = mi.Message.Received.getTime() - const nextmivindex = msgitemViews.findIndex(miv => { - const vtm = miv.messageitem.Message.Received.getTime() - return orderNewest && vtm <= tm || !orderNewest && tm <= vtm - }) - if (nextmivindex < 0) { - mlv.root.appendChild(miv.root) - msgitemViews.push(miv) + + // Each "mil" is a thread, possibly with multiple thread roots. The thread may + // already be present. + messageItems.forEach(mil => { + if (!mil) { + return // For types, should not happen. + } + + const threadID = mil[0].Message.ThreadID + + const hasMatch = !!mil.find(mi => mi.MatchQuery) + if (hasMatch) { + // This may be a message for a thread that had query-matching matches at some + // point, but then no longer, causing its messages to have been moved to + // oldThreadMessageItems. We add back those messages. + let i = 0 + while (i < oldThreadMessageItems.length) { + const omi = oldThreadMessageItems[i] + if (omi.Message.ThreadID === threadID) { + oldThreadMessageItems.splice(i, 1) + if (!mil.find(mi => mi.Message.ID === omi.Message.ID)) { + mil.push(omi) + log('resurrected old message') + } else { + log('dropped old thread message') + } + } else { + i++ + } + } } else { - mlv.root.insertBefore(miv.root, msgitemViews[nextmivindex].root) - msgitemViews.splice(nextmivindex, 0, miv) + // New message(s) are not matching query. If there are no "active" messages for + // this thread, update/add oldThreadMessageItems. + const match = (miv: MsgitemView) => miv.messageitem.Message.ThreadID === threadID + if (!msgitemViews.find(match) && !collapsedMsgitemViews.find(match)) { + log('adding new message(s) to oldTheadMessageItems') + for (const mi of mil) { + const ti = oldThreadMessageItems.findIndex(tmi => tmi.Message.ID === mi.Message.ID) + if (ti) { + oldThreadMessageItems[ti] = mi + } else { + oldThreadMessageItems.push(mi) + } + } + return + } + } + + if (isChange) { + // This could be an "add" for a message from another mailbox that we are already + // displaying because of threads. If so, it may have new properties such as the + // mailbox, so update it. + const threadIDs = new Set() + let i = 0 + while (i < mil.length) { + const mi = mil[i] + let miv = msgitemViews.find(miv => miv.messageitem.Message.ID === mi.Message.ID) + if (!miv) { + miv = collapsedMsgitemViews.find(miv => miv.messageitem.Message.ID === mi.Message.ID) + } + if (miv) { + miv.messageitem = mi + miv.render() + mil.splice(i, 1) + miv.threadRoot().render() + threadIDs.add(mi.Message.ThreadID) + } else { + i++ + } + } + log('processed changes for messages with thread', {threadIDs, mil}) + if (mil.length === 0) { + const oldstate = state() + possiblyTakeoutOldThreads(threadIDs) + updateState(oldstate) + return + } + } + + // Find effective receive time for messages. We'll insert at that point. + let receivedTime = mil[0].Message.Received.getTime() + const tmiv = msgitemViews.find(miv => miv.messageitem.Message.ThreadID === mil[0].Message.ThreadID) + if (tmiv) { + receivedTime = tmiv.receivedTime + } else { + for (const mi of mil) { + const t = mi.Message.Received.getTime() + if (settings.orderAsc && t < receivedTime || !settings.orderAsc && t > receivedTime) { + receivedTime = t + } + } + } + + // Create new MsgitemViews. + const m = new Map() + for (const mi of mil) { + m.set(mi.Message.ID, newMsgitemView(mi, mlv, otherMailbox, listMailboxes, receivedTime, false)) + } + + // Assign miv's to parents or add them to the potential roots. + let roots: MsgitemView[] = [] + if (settings.threading === api.ThreadMode.ThreadOff) { + roots = [...m.values()] + } else { + nextmiv: + for (const [_, miv] of m) { + for (const pid of (miv.messageitem.Message.ThreadParentIDs || [])) { + const pmiv = m.get(pid) + if (pmiv) { + pmiv.kids.push(miv) + miv.parent = pmiv + continue nextmiv + } + } + roots.push(miv) + } + } + + // Ensure all kids are properly sorted, always ascending by time received. + for (const [_, miv] of m) { + miv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()) + } + + // Add the potential roots as kids to existing parents, if they exist. Only with threading enabled. + if (settings.threading !== api.ThreadMode.ThreadOff) { + nextroot: + for (let i = 0; i < roots.length; ) { + const miv = roots[i] + for (const pid of (miv.messageitem.Message.ThreadParentIDs || [])) { + const pi = msgitemViews.findIndex(xmiv => xmiv.messageitem.Message.ID === pid) + let parentmiv: MsgitemView | undefined + let collapsed: boolean + if (pi >= 0) { + parentmiv = msgitemViews[pi] + collapsed = parentmiv.collapsed + log('found parent', {pi}) + } else { + parentmiv = collapsedMsgitemViews.find(xmiv => xmiv.messageitem.Message.ID === pid) + collapsed = true + } + if (!parentmiv) { + log('no parentmiv', pid) + continue + } + + const trmiv = parentmiv.threadRoot() + if (collapsed !== trmiv.collapsed) { + log('collapsed mismatch', {collapsed: collapsed, 'trmiv.collapsed': trmiv.collapsed, trmiv: trmiv}) + throw new ConsistencyError('mismatch between msgitemViews/collapsedMsgitemViews and threadroot collapsed') + } + let prevLastDesc: MsgitemView | null = null + if (!trmiv.collapsed) { + // Remove current parent, we'll insert again after linking parent/kids. + const ndesc = parentmiv.descendants().length + log('removing descendants temporarily', {ndesc}) + prevLastDesc = parentmiv.lastDescendant() + msgitemViews.splice(pi+1, ndesc) + } + + // Link parent & kid, sort kids. + miv.parent = parentmiv + parentmiv.kids.push(miv) + parentmiv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()) + + if (trmiv.collapsed) { + // Thread root is collapsed. + collapsedMsgitemViews.push(miv, ...miv.descendants()) + + // Ensure mivs have a root. + miv.render() + miv.descendants().forEach(miv => miv.render()) + + // Update count/unread status. + trmiv.render() + } else { + const desc = parentmiv.descendants() + log('inserting parent descendants again', {pi, desc}) + msgitemViews.splice(pi+1, 0, ...desc) // We had removed the old tree, now adding the updated tree. + + // Insert at correct position in dom. + const i = msgitemViews.indexOf(miv) + if (i < 0) { + throw new ConsistencyError('cannot find miv just inserted') + } + const l = [miv, ...miv.descendants()] + // Ensure mivs have valid root. + l.forEach(miv => miv.render()) + const next = i+1 < msgitemViews.length ? msgitemViews[i+1].root : null + log('inserting l before next, or appending', {next, l}) + if (next) { + for (const miv of l) { + log('inserting miv', {root: miv.root, before: next}) + mlv.root.insertBefore(miv.root, next) + } + } else { + mlv.root.append(...l.map(e => e.root)) + } + // For beginning/end of thread bar. + msgitemViews[i-1].render() + if (prevLastDesc) { + prevLastDesc.render() + } + } + roots.splice(i, 1) + continue nextroot + } + i++ + } + } + + // Sort the remaining new roots by their receive times. + const sign = settings.threading === api.ThreadMode.ThreadOff && settings.orderAsc ? -1 : 1 + roots.sort((miva, mivb) => sign * (mivb.messageitem.Message.Received.getTime() - miva.messageitem.Message.Received.getTime())) + + // Find place to insert, based on thread receive time. + let nextmivindex: number + if (tmiv) { + nextmivindex = msgitemViews.indexOf(tmiv.threadRoot()) + } else { + nextmivindex = msgitemViews.findIndex(miv => !settings.orderAsc && miv.receivedTime <= receivedTime || settings.orderAsc && receivedTime <= miv.receivedTime) + } + + for (const miv of roots) { + miv.collapsed = settings.threading === api.ThreadMode.ThreadOn && miv.messageitem.Message.ThreadCollapsed + if (settings.threading === api.ThreadMode.ThreadUnread) { + miv.collapsed = miv.messageitem.Message.Seen && !miv.findDescendant(dmiv => !dmiv.messageitem.Message.Seen) + } + if (requestMsgID > 0 && miv.collapsed) { + miv.collapsed = !miv.findDescendant(dmiv => dmiv.messageitem.Message.ID === requestMsgID) + } + + const takeThreadRoot = (xmiv: MsgitemView): number => { + log('taking threadRoot', {id: xmiv.messageitem.Message.ID}) + // Remove subtree from dom. + const xdmiv = xmiv.descendants() + xdmiv.forEach(xdmiv => xdmiv.remove()) + xmiv.remove() + // Link to new parent. + miv.kids.push(xmiv) + xmiv.parent = miv + miv.kids.sort((miva, mivb) => miva.messageitem.Message.Received.getTime() - mivb.messageitem.Message.Received.getTime()) + return 1+xdmiv.length + } + + if (settings.threading !== api.ThreadMode.ThreadOff) { + // We may have to take out existing threadroots and place them under this new root. + // Because when we move a threadroot, we first remove it, then add it again. + for (let i = 0; i < msgitemViews.length; ) { + const xmiv = msgitemViews[i] + if (!xmiv.parent && xmiv.messageitem.Message.ThreadID === miv.messageitem.Message.ThreadID && (xmiv.messageitem.Message.ThreadParentIDs || []).includes(miv.messageitem.Message.ID)) { + msgitemViews.splice(i, takeThreadRoot(xmiv)) + nextmivindex = i + } else { + i++ + } + } + for (let i = 0; i < collapsedMsgitemViews.length; ) { + const xmiv = collapsedMsgitemViews[i] + if (!xmiv.parent && xmiv.messageitem.Message.ThreadID === miv.messageitem.Message.ThreadID && (xmiv.messageitem.Message.ThreadParentIDs || []).includes(miv.messageitem.Message.ID)) { + takeThreadRoot(xmiv) + collapsedMsgitemViews.splice(i, 1) + } else { + i++ + } + } + } + + let l = miv.descendants() + + miv.render() + l.forEach(kmiv => kmiv.render()) + + if (miv.collapsed) { + collapsedMsgitemViews.push(...l) + l = [miv] + } else { + l = [miv, ...l] + } + + if (nextmivindex < 0) { + mlv.root.append(...l.map(miv => miv.root)) + msgitemViews.push(...l) + } else { + const next = msgitemViews[nextmivindex].root + for (const miv of l) { + mlv.root.insertBefore(miv.root, next) + } + msgitemViews.splice(nextmivindex, 0, ...l) + } } }) + + if (!isChange) { + return + } + const oldstate = state() if (!focus) { focus = msgitemViews[0] } if (selected.length === 0) { - selected = [msgitemViews[0]] + if (focus) { + selected = [focus] + } else if (msgitemViews.length > 0) { + selected = [msgitemViews[0]] + } } updateState(oldstate) }, + + // Remove messages, they can be in different threads. removeUIDs: (mailboxID: number, uids: number[]) => { - const uidmap: {[key: string]: boolean} = {} - uids.forEach(uid => uidmap[''+mailboxID+','+uid] = true) // todo: we would like messageID here. - - const key = (miv: MsgitemView) => ''+miv.messageitem.Message.MailboxID+','+miv.messageitem.Message.UID - const oldstate = state() - selected = selected.filter(miv => !uidmap[key(miv)]) - if (focus && uidmap[key(focus)]) { - const index = msgitemViews.indexOf(focus) - var nextmiv - for (let i = index+1; i < msgitemViews.length; i++) { - if (!uidmap[key(msgitemViews[i])]) { - nextmiv = msgitemViews[i] - break - } - } - if (!nextmiv) { - for (let i = index-1; i >= 0; i--) { - if (!uidmap[key(msgitemViews[i])]) { - nextmiv = msgitemViews[i] - break - } - } + const hadSelected = selected.length > 0 + const threadIDs = new Set() + uids.forEach(uid => { + const threadID = removeUID(mailboxID, uid) + log('removed message with thread', {threadID}) + if (threadID) { + threadIDs.add(threadID) } + }) - if (nextmiv) { - focus = nextmiv - } else { - focus = null - } - } + possiblyTakeoutOldThreads(threadIDs) - if (selected.length === 0 && focus) { + if (hadSelected && focus && selected.length === 0) { selected = [focus] } updateState(oldstate) + }, - let i = 0 - while (i < msgitemViews.length) { - const miv = msgitemViews[i] - const k = ''+miv.messageitem.Message.MailboxID+','+miv.messageitem.Message.UID - if (!uidmap[k]) { - i++ - continue + // Set new muted/collapsed flags for messages in thread. + updateMessageThreadFields: (messageIDs: number[], muted: boolean, collapsed: boolean) => { + for (const id of messageIDs) { + let miv = msgitemViews.find(miv => miv.messageitem.Message.ID === id) + if (!miv) { + miv = collapsedMsgitemViews.find(miv => miv.messageitem.Message.ID === id) + } + if (miv) { + miv.messageitem.Message.ThreadMuted = muted + miv.messageitem.Message.ThreadCollapsed = collapsed + const mivthr = miv.threadRoot() + if (mivthr.collapsed) { + mivthr.render() + } else { + miv.render() + } + } else { + const mi = oldThreadMessageItems.find(mi => mi.Message.ID === id) + if (mi) { + mi.Message.ThreadMuted = muted + mi.Message.ThreadCollapsed = collapsed + } } - miv.remove() - msgitemViews.splice(i, 1) } }, @@ -2661,17 +3784,12 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL miv.root.classList.toggle('active', selected.indexOf(miv) >= 0) }, - anchorMessageID: () => msgitemViews[msgitemViews.length-1].messageitem.Message.ID, - - addMsgitemViews: (mivs: MsgitemView[]) => { - mlv.root.append(...mivs.map(v => v.root)) - msgitemViews.push(...mivs) - }, - clear: (): void => { dom._kids(mlv.root) msgitemViews.forEach(miv => miv.remove()) msgitemViews = [] + collapsedMsgitemViews = [] + oldThreadMessageItems = [] focus = null selected = [] dom._kids(msgElem) @@ -2690,12 +3808,27 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL selected = [miv] updateState(oldstate) }, - selected: () => selected, - openMessage: (miv: MsgitemView, initial: boolean, parsedMessageOpt?: api.ParsedMessage) => { + selected: () => { + const l = [] + for (const miv of selected) { + l.push(miv) + if (miv.collapsed) { + l.push(...miv.descendants()) + } + } + return l + }, + openMessage: (parsedMessage: api.ParsedMessage) => { + let miv = msgitemViews.find(miv => miv.messageitem.Message.ID === parsedMessage.ID) + if (!miv) { + // todo: could move focus to the nearest expanded message in this thread, if any? + return false + } const oldstate = state() focus = miv selected = [miv] - updateState(oldstate, initial, parsedMessageOpt) + updateState(oldstate, true, parsedMessage) + return true }, click: (miv: MsgitemView, ctrl: boolean, shift: boolean) => { @@ -2757,6 +3890,9 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL 'k', 'K', 'Home', ',', '<', 'End', '.', '>', + 'n', 'N', + 'p', 'P', + 'u', 'U', ] if (!e.altKey && moveKeys.includes(e.key)) { const moveclick = (index: number, clip: boolean) => { @@ -2765,7 +3901,7 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL } else if (clip && index >= msgitemViews.length) { index = msgitemViews.length-1 } - if (index < 0 || index >= msgitemViews.length) { + if (index < 0 || index >= msgitemViews.length) { return } if (e.ctrlKey) { @@ -2784,7 +3920,7 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL moveclick(i-1, e.key === 'K') } else if (e.key === 'ArrowDown' || e.key === 'j' || e.key === 'J') { moveclick(i+1, e.key === 'J') - } else if (e.key === 'PageUp' || e.key === 'h' || e.key == 'H' || e.key === 'PageDown' || e.key === 'l' || e.key === 'L') { + } else if (e.key === 'PageUp' || e.key === 'h' || e.key === 'H' || e.key === 'PageDown' || e.key === 'l' || e.key === 'L') { if (msgitemViews.length > 0) { let n = Math.max(1, Math.floor(scrollElemHeight()/mlv.itemHeight())-1) if (e.key === 'PageUp' || e.key === 'h' || e.key === 'H') { @@ -2796,6 +3932,37 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL moveclick(0, true) } else if (e.key === 'End' || e.key === '.' || e.key === '>') { moveclick(msgitemViews.length-1, true) + } else if (e.key === 'n' || e.key === 'N') { + if (i < 0) { + moveclick(0, true) + } else { + const tid = msgitemViews[i].messageitem.Message.ThreadID + for (; i < msgitemViews.length; i++) { + if (msgitemViews[i].messageitem.Message.ThreadID !== tid) { + moveclick(i, true) + break + } + } + } + } else if (e.key === 'p' || e.key === 'P') { + if (i < 0) { + moveclick(0, true) + } else { + let thrmiv = msgitemViews[i].threadRoot() + if (thrmiv === msgitemViews[i]) { + if (i-1 >= 0) { + thrmiv = msgitemViews[i-1].threadRoot() + } + } + moveclick(msgitemViews.indexOf(thrmiv), true) + } + } else if (e.key === 'u' || e.key === 'U') { + for (i = i < 0 ? 0 : i+1; i < msgitemViews.length; i += 1) { + if (!msgitemViews[i].messageitem.Message.Seen || msgitemViews[i].collapsed && msgitemViews[i].findDescendant(miv => !miv.messageitem.Message.Seen)) { + moveclick(i, true) + break + } + } } e.preventDefault() e.stopPropagation() @@ -2814,6 +3981,10 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL }, mailboxes: () => listMailboxes(), itemHeight: () => msgitemViews.length > 0 ? msgitemViews[0].root.getBoundingClientRect().height : 25, + threadExpand: (miv: MsgitemView) => threadExpand(miv, true), + threadCollapse: (miv: MsgitemView) => threadCollapse(miv, true), + threadToggle: threadToggle, + viewportEnsureMessages: viewportEnsureMessages, cmdArchive: cmdArchive, cmdTrash: cmdTrash, @@ -2822,6 +3993,8 @@ const newMsglistView = (msgElem: HTMLElement, listMailboxes: listMailboxes, setL cmdMarkNotJunk: cmdMarkNotJunk, cmdMarkRead: cmdMarkRead, cmdMarkUnread: cmdMarkUnread, + cmdMute: cmdMute, + cmdUnmute: cmdUnmute, } return mlv @@ -2847,7 +4020,7 @@ interface MailboxView { setKeywords: (keywords: string[]) => void } -const newMailboxView = (xmb: api.Mailbox, mailboxlistView: MailboxlistView): MailboxView => { +const newMailboxView = (xmb: api.Mailbox, mailboxlistView: MailboxlistView, otherMailbox: otherMailbox): MailboxView => { const plusbox = '⊞' const minusbox = '⊟' const cmdCollapse = async () => { @@ -3007,7 +4180,12 @@ const newMailboxView = (xmb: api.Mailbox, mailboxlistView: MailboxlistView): Mai async function drop(e: DragEvent) { e.preventDefault() mbv.root.classList.toggle('dropping', false) - const msgIDs = JSON.parse(e.dataTransfer!.getData('application/vnd.mox.messages')) as number[] + const sentMailboxID = mailboxlistView.mailboxes().find(mb => mb.Sent)?.ID + const mailboxMsgIDs = JSON.parse(e.dataTransfer!.getData('application/vnd.mox.messages')) as number[][] + const msgIDs = mailboxMsgIDs + .filter(mbMsgID => mbMsgID[0] !== xmb.ID) + .filter(mbMsgID => mailboxMsgIDs.length === 1 || !sentMailboxID || mbMsgID[0] !== sentMailboxID || !otherMailbox(sentMailboxID)) + .map(mbMsgID => mbMsgID[1]) await withStatus('Moving to '+xmb.Name, client.MessageMove(msgIDs, xmb.ID)) }, dom.div(dom._class('mailbox'), @@ -3114,7 +4292,7 @@ interface MailboxlistView { setMailboxKeywords: (mailboxID: number, keywords: string[]) => void } -const newMailboxlistView = (msglistView: MsglistView, requestNewView: requestNewView, updatePageTitle: updatePageTitle, setLocationHash: setLocationHash, unloadSearch: unloadSearch): MailboxlistView => { +const newMailboxlistView = (msglistView: MsglistView, requestNewView: requestNewView, updatePageTitle: updatePageTitle, setLocationHash: setLocationHash, unloadSearch: unloadSearch, otherMailbox: otherMailbox): MailboxlistView => { let mailboxViews: MailboxView[] = [] let mailboxViewActive: MailboxView | null @@ -3250,7 +4428,7 @@ const newMailboxlistView = (msglistView: MsglistView, requestNewView: requestNew ) const loadMailboxes = (mailboxes: api.Mailbox[], mbnameOpt?: string) => { - mailboxViews = mailboxes.map(mb => newMailboxView(mb, mblv)) + mailboxViews = mailboxes.map(mb => newMailboxView(mb, mblv, otherMailbox)) updateMailboxNames() if (mbnameOpt) { const mbv = mailboxViews.find(mbv => mbv.mailbox.Name === mbnameOpt) @@ -3322,7 +4500,7 @@ const newMailboxlistView = (msglistView: MsglistView, requestNewView: requestNew }, addMailbox: (mb: api.Mailbox): void => { - const mbv = newMailboxView(mb, mblv) + const mbv = newMailboxView(mb, mblv, otherMailbox) mailboxViews.push(mbv) updateMailboxNames() }, @@ -3796,7 +4974,7 @@ const newSearchView = (searchbarElem: HTMLInputElement, mailboxlistView: Mailbox updateSearchbar() }), update: () => { - v.root.style.backgroundColor = v.active === true ? '#c4ffa9' : (v.active === false ? '#ffb192' : '') + v.root.style.backgroundColor = v.active === true ? '#c4ffa9' : (v.active === false ? '#ffb192' : '') }, } return v @@ -3874,7 +5052,7 @@ const init = async () => { let queryactivityElem: HTMLElement // We show ... when a query is active and data is forthcoming. // Shown at the bottom of msglistscrollElem, immediately below the msglistView, when appropriate. - const listendElem = dom.div(style({borderTop: '1px solid #ccc', color: '#666', margin: '1ex'})) + const listendElem = dom.div(style({borderTop: '1px solid #ccc', color: '#666', margin: '1ex'})) const listloadingElem = dom.div(style({textAlign: 'center', padding: '.15em 0', color: '#333', border: '1px solid #ccc', margin: '1ex', backgroundColor: '#f8f8f8'}), 'loading...') const listerrElem = dom.div(style({textAlign: 'center', padding: '.15em 0', color: '#333', border: '1px solid #ccc', margin: '1ex', backgroundColor: '#f8f8f8'})) @@ -3887,6 +5065,7 @@ const init = async () => { } let requestSequence = 0 // Counter for assigning requestID. let requestID = 0 // Current request, server will mirror it in SSE data. If we get data for a different id, we ignore it. + let requestAnchorMessageID = 0 // For pagination. let requestViewEnd = false // If true, there is no more data to fetch, no more page needed for this view. let requestFilter = newFilter() let requestNotFilter = newNotFilter() @@ -3961,15 +5140,16 @@ const init = async () => { requestNotFilter = notFilterOpt || newNotFilter() } + requestAnchorMessageID = 0 requestViewEnd = false const bounds = msglistscrollElem.getBoundingClientRect() - await requestMessages(bounds, 0, requestMsgID) + await requestMessages(bounds, requestMsgID) } - const requestMessages = async (scrollBounds: DOMRect, anchorMessageID: number, destMessageID: number) => { + const requestMessages = async (scrollBounds: DOMRect, destMessageID: number) => { const fetchCount = Math.max(50, 3*Math.ceil(scrollBounds.height/msglistView.itemHeight())) const page = { - AnchorMessageID: anchorMessageID, + AnchorMessageID: requestAnchorMessageID, Count: fetchCount, DestMessageID: destMessageID, } @@ -3978,6 +5158,7 @@ const init = async () => { const [f, notf] = refineFilters(requestFilter, requestNotFilter) const query = { OrderAsc: settings.orderAsc, + Threading: settings.threading, Filter: f, NotFilter: notf, } @@ -4026,10 +5207,22 @@ const init = async () => { await withStatus('Requesting messages', requestNewView(false)) } + const viewportEnsureMessages = async () => { + // We know how many entries we have, and how many screenfulls. So we know when we + // only have 2 screen fulls left. That's when we request the next data. + const bounds = msglistscrollElem.getBoundingClientRect() + if (msglistscrollElem.scrollTop < msglistscrollElem.scrollHeight-3*bounds.height) { + return + } + + // log('new request for scroll') + await withStatus('Requesting more messages', requestMessages(bounds, 0)) + } + const otherMailbox = (mailboxID: number): api.Mailbox | null => requestFilter.MailboxID !== mailboxID ? (mailboxlistView.findMailboxByID(mailboxID) || null) : null const listMailboxes = () => mailboxlistView.mailboxes() - const msglistView = newMsglistView(msgElem, listMailboxes, setLocationHash, otherMailbox, possibleLabels, () => msglistscrollElem ? msglistscrollElem.getBoundingClientRect().height : 0, refineKeyword) - const mailboxlistView = newMailboxlistView(msglistView, requestNewView, updatePageTitle, setLocationHash, unloadSearch) + const msglistView = newMsglistView(msgElem, listMailboxes, setLocationHash, otherMailbox, possibleLabels, () => msglistscrollElem ? msglistscrollElem.getBoundingClientRect().height : 0, refineKeyword, viewportEnsureMessages) + const mailboxlistView = newMailboxlistView(msglistView, requestNewView, updatePageTitle, setLocationHash, unloadSearch, otherMailbox) let refineUnreadBtn: HTMLButtonElement, refineReadBtn: HTMLButtonElement, refineAttachmentsBtn: HTMLButtonElement, refineLabelBtn: HTMLButtonElement const refineToggleActive = (btn: HTMLButtonElement | null): void => { @@ -4041,6 +5234,8 @@ const init = async () => { } } + let threadMode: HTMLSelectElement + let msglistElem = dom.div(dom._class('msglist'), style({position: 'absolute', left: '0', right: 0, top: 0, bottom: 0, display: 'flex', flexDirection: 'column'}), dom.div( @@ -4120,6 +5315,24 @@ const init = async () => { dom.div( queryactivityElem=dom.span(), ' ', + threadMode=dom.select( + attr.arialabel('Thread modes.'), + attr.title('Off: Threading disabled, messages are shown individually.\nOn: Group messages in threads, expanded by default except when (previously) manually collapsed.\nUnread: Only expand thread with unread messages, ignoring and not saving whether they were manually collapsed.'), + dom.option('Threads: Off', attr.value(api.ThreadMode.ThreadOff), settings.threading === api.ThreadMode.ThreadOff ? attr.selected('') : []), + dom.option('Threads: On', attr.value(api.ThreadMode.ThreadOn), settings.threading === api.ThreadMode.ThreadOn ? attr.selected('') : []), + dom.option('Threads: Unread', attr.value(api.ThreadMode.ThreadUnread), settings.threading === api.ThreadMode.ThreadUnread ? attr.selected('') : []), + async function change() { + let reset = settings.threading === api.ThreadMode.ThreadOff + settingsPut({...settings, threading: threadMode.value as api.ThreadMode}) + reset = reset || settings.threading === api.ThreadMode.ThreadOff + if (reset) { + await withStatus('Requesting messages', requestNewView(false)) + } else { + msglistView.threadToggle() + } + }, + ), + ' ', dom.clickbutton('↑↓', attr.title('Toggle sorting by date received.'), settings.orderAsc ? dom._class('invert') : [], async function click(e: MouseEvent) { settingsPut({...settings, orderAsc: !settings.orderAsc}) ;(e.target! as HTMLButtonElement).classList.toggle('invert', settings.orderAsc) @@ -4181,16 +5394,7 @@ const init = async () => { return } - // We know how many entries we have, and how many screenfulls. So we know when we - // only have 2 screen fulls left. That's when we request the next data. - const bounds = msglistscrollElem.getBoundingClientRect() - if (msglistscrollElem.scrollTop < msglistscrollElem.scrollHeight-3*bounds.height) { - return - } - - // log('new request for scroll') - const reqAnchor = msglistView.anchorMessageID() - await withStatus('Requesting more messages', requestMessages(bounds, reqAnchor, 0)) + await viewportEnsureMessages() }, dom.div( style({width: '100%', borderSpacing: '0'}), @@ -4300,7 +5504,7 @@ const init = async () => { '?': cmdHelp, 'ctrl ?': cmdTooltip, c: cmdCompose, - M: cmdFocusMsg, + 'ctrl m': cmdFocusMsg, } const webmailroot = dom.div( @@ -4589,7 +5793,7 @@ const init = async () => { let lastflagswidth: number, lastagewidth: number let rulesInserted = false const updateMsglistWidths = () => { - const width = msglistscrollElem.clientWidth + const width = msglistscrollElem.clientWidth - 2 // Borders. lastmsglistwidth = width let flagswidth = settings.msglistflagsWidth @@ -4607,7 +5811,7 @@ const init = async () => { ['.msgitemfrom', {width: fromwidth}], ['.msgitemsubject', {width: subjectwidth}], ['.msgitemage', {width: agewidth}], - ['.msgitemflagsoffset', {left: flagswidth}], + ['.msgitemflagsoffset', {left: flagswidth}], ['.msgitemfromoffset', {left: flagswidth + fromwidth}], ['.msgitemsubjectoffset', {left: flagswidth + fromwidth + subjectwidth}], ] @@ -4734,6 +5938,7 @@ const init = async () => { const fetchCount = Math.max(50, 3*Math.ceil(msglistscrollElem.getBoundingClientRect().height/msglistView.itemHeight())) const query = { OrderAsc: settings.orderAsc, + Threading: settings.threading, Filter: f, NotFilter: notf, } @@ -4749,6 +5954,7 @@ const init = async () => { // We get an implicit query for the automatically selected mailbox or query. requestSequence++ requestID = requestSequence + requestAnchorMessageID = 0 requestViewEnd = false clearList() @@ -4843,7 +6049,7 @@ const init = async () => { if (formatEmailASCII(b) === loginAddr) { return 1 } - if (a.Domain.ASCII != b.Domain.ASCII) { + if (a.Domain.ASCII !== b.Domain.ASCII) { return a.Domain.ASCII < b.Domain.ASCII ? -1 : 1 } return a.User < b.User ? -1 : 1 @@ -4879,7 +6085,7 @@ const init = async () => { eventSource.addEventListener('viewErr', async (e: MessageEvent) => { const viewErr = checkParse(() => api.parser.EventViewErr(JSON.parse(e.data))) log('event viewErr', viewErr) - if (viewErr.ViewID != viewID || viewErr.RequestID !== requestID) { + if (viewErr.ViewID !== viewID || viewErr.RequestID !== requestID) { log('received viewErr for other viewID or requestID', {expected: {viewID, requestID}, got: {viewID: viewErr.ViewID, requestID: viewErr.RequestID}}) return } @@ -4897,7 +6103,7 @@ const init = async () => { eventSource.addEventListener('viewReset', async (e: MessageEvent) => { const viewReset = checkParse(() => api.parser.EventViewReset(JSON.parse(e.data))) log('event viewReset', viewReset) - if (viewReset.ViewID != viewID || viewReset.RequestID !== requestID) { + if (viewReset.ViewID !== viewID || viewReset.RequestID !== requestID) { log('received viewReset for other viewID or requestID', {expected: {viewID, requestID}, got: {viewID: viewReset.ViewID, requestID: viewReset.RequestID}}) return } @@ -4910,31 +6116,28 @@ const init = async () => { eventSource.addEventListener('viewMsgs', async (e: MessageEvent) => { const viewMsgs = checkParse(() => api.parser.EventViewMsgs(JSON.parse(e.data))) log('event viewMsgs', viewMsgs) - if (viewMsgs.ViewID != viewID || viewMsgs.RequestID !== requestID) { + if (viewMsgs.ViewID !== viewID || viewMsgs.RequestID !== requestID) { log('received viewMsgs for other viewID or requestID', {expected: {viewID, requestID}, got: {viewID: viewMsgs.ViewID, requestID: viewMsgs.RequestID}}) return } msglistView.root.classList.toggle('loading', false) - const extramsgitemViews = (viewMsgs.MessageItems || []).map(mi => { - const othermb = requestFilter.MailboxID !== mi.Message.MailboxID ? mailboxlistView.findMailboxByID(mi.Message.MailboxID) : undefined - return newMsgitemView(mi, msglistView, othermb || null) - }) - - msglistView.addMsgitemViews(extramsgitemViews) + if (viewMsgs.MessageItems) { + msglistView.addMessageItems(viewMsgs.MessageItems || [], false, requestMsgID) + } if (viewMsgs.ParsedMessage) { - const msgID = viewMsgs.ParsedMessage.ID - const miv = extramsgitemViews.find(miv => miv.messageitem.Message.ID === msgID) - if (miv) { - msglistView.openMessage(miv, true, viewMsgs.ParsedMessage) - } else { + const ok = msglistView.openMessage(viewMsgs.ParsedMessage) + if (!ok) { // Should not happen, server would be sending a parsedmessage while not including the message itself. requestMsgID = 0 setLocationHash() } } + if (viewMsgs.MessageItems && viewMsgs.MessageItems.length > 0) { + requestAnchorMessageID = viewMsgs.MessageItems[viewMsgs.MessageItems.length-1]![0]!.Message.ID + } requestViewEnd = viewMsgs.ViewEnd if (requestViewEnd) { msglistscrollElem.appendChild(listendElem) @@ -4952,7 +6155,7 @@ const init = async () => { eventSource.addEventListener('viewChanges', async (e: MessageEvent) => { const viewChanges = checkParse(() => api.parser.EventViewChanges(JSON.parse(e.data))) log('event viewChanges', viewChanges) - if (viewChanges.ViewID != viewID) { + if (viewChanges.ViewID !== viewID) { log('received viewChanges for other viewID', {expected: viewID, got: viewChanges.ViewID}) return } @@ -4974,13 +6177,18 @@ const init = async () => { mailboxlistView.setMailboxKeywords(c.MailboxID, c.Keywords || []) } else if (tag === 'ChangeMsgAdd') { const c = api.parser.ChangeMsgAdd(x) - msglistView.addMessageItems([c.MessageItem]) + msglistView.addMessageItems([c.MessageItems || []], true, 0) } else if (tag === 'ChangeMsgRemove') { const c = api.parser.ChangeMsgRemove(x) msglistView.removeUIDs(c.MailboxID, c.UIDs || []) } else if (tag === 'ChangeMsgFlags') { const c = api.parser.ChangeMsgFlags(x) msglistView.updateFlags(c.MailboxID, c.UID, c.ModSeq, c.Mask, c.Flags, c.Keywords || []) + } else if (tag === 'ChangeMsgThread') { + const c = api.parser.ChangeMsgThread(x) + if (c.MessageIDs) { + msglistView.updateMessageThreadFields(c.MessageIDs, c.Muted, c.Collapsed) + } } else if (tag === 'ChangeMailboxRemove') { const c = api.parser.ChangeMailboxRemove(x) mailboxlistView.removeMailbox(c.MailboxID) @@ -5091,7 +6299,7 @@ window.addEventListener('unhandledrejection', (e: PromiseRejectionEvent) => { return } const err = e.reason - if (err instanceof EvalError || err instanceof RangeError || err instanceof ReferenceError || err instanceof SyntaxError || err instanceof TypeError || err instanceof URIError) { + if (err instanceof EvalError || err instanceof RangeError || err instanceof ReferenceError || err instanceof SyntaxError || err instanceof TypeError || err instanceof URIError || err instanceof ConsistencyError) { showUnhandledError(err, 0, 0) } else { console.log('unhandled promiserejection', err, e.promise) diff --git a/webmail/webmail_test.go b/webmail/webmail_test.go index cbba192..c522bd1 100644 --- a/webmail/webmail_test.go +++ b/webmail/webmail_test.go @@ -45,6 +45,7 @@ type Message struct { From, To, Cc, Bcc, Subject, MessageID string Headers [][2]string Date time.Time + References string Part Part } @@ -84,6 +85,7 @@ func (m Message) Marshal(t *testing.T) []byte { header("Subject", m.Subject) header("Message-Id", m.MessageID) header("Date", m.Date.Format(message.RFC5322Z)) + header("References", m.References) for _, t := range m.Headers { header(t[0], t[1]) } @@ -181,10 +183,11 @@ var ( Part: Part{Type: "text/html", Content: `the body `}, } msgAlt = Message{ - From: "mjl ", - To: "mox ", - Subject: "test", - Headers: [][2]string{{"In-Reply-To", ""}}, + From: "mjl ", + To: "mox ", + Subject: "test", + MessageID: "", + Headers: [][2]string{{"In-Reply-To", ""}}, Part: Part{ Type: "multipart/alternative", Parts: []Part{ @@ -193,6 +196,11 @@ var ( }, }, } + msgAltReply = Message{ + Subject: "Re: test", + References: "", + Part: Part{Type: "text/plain", Content: "reply to alt"}, + } msgAltRel = Message{ From: "mjl ", To: "mox ",