implement message threading in backend and webmail

we match messages to their parents based on the "references" and "in-reply-to"
headers (requiring the same base subject), and in absense of those headers we
also by only base subject (against messages received max 4 weeks ago).

we store a threadid with messages. all messages in a thread have the same
threadid.  messages also have a "thread parent ids", which holds all id's of
parent messages up to the thread root.  then there is "thread missing link",
which is set when a referenced immediate parent wasn't found (but possibly
earlier ancestors can still be found and will be in thread parent ids".

threads can be muted: newly delivered messages are automatically marked as
read/seen.  threads can be marked as collapsed: if set, the webmail collapses
the thread to a single item in the basic threading view (default is to expand
threads).  the muted and collapsed fields are copied from their parent on
message delivery.

the threading is implemented in the webmail. the non-threading mode still works
as before. the new default threading mode "unread" automatically expands only
the threads with at least one unread (not seen) meessage. the basic threading
mode "on" expands all threads except when explicitly collapsed (as saved in the
thread collapsed field). new shortcuts for navigation/interaction threads have
been added, e.g. go to previous/next thread root, toggle collapse/expand of
thread (or double click), toggle mute of thread. some previous shortcuts have
changed, see the help for details.

the message threading are added with an explicit account upgrade step,
automatically started when an account is opened. the upgrade is done in the
background because it will take too long for large mailboxes to block account
operations. the upgrade takes two steps: 1. updating all message records in the
database to add a normalized message-id and thread base subject (with "re:",
"fwd:" and several other schemes stripped). 2. going through all messages in
the database again, reading the "references" and "in-reply-to" headers from
disk, and matching against their parents. this second step is also done at the
end of each import of mbox/maildir mailboxes. new deliveries are matched
immediately against other existing messages, currently no attempt is made to
rematch previously delivered messages (which could be useful for related
messages being delivered out of order).

the threading is not yet exposed over imap.

.gitignore

@@ -31,3 +31,7 @@
 /cover.html
 /.go/
 /node_modules/
+/upgrade-verifydata.cpu.pprof
+/upgrade-verifydata.mem.pprof
+/upgrade-openaccounts.cpu.pprof
+/upgrade-openaccounts.mem.pprof

README.md

@@ -109,15 +109,16 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili
 
 ## Roadmap
 
-- IMAP THREAD extension
-- Prepare data storage for JMAP
 - DANE and DNSSEC
 - Sending DMARC and TLS reports (currently only receiving)
+- Require TLS SMTP extension (RFC 8689)
+- Prepare data storage for JMAP
 - Calendaring
 - Add special IMAP mailbox ("Queue?") that contains queued but
   not-yet-delivered messages
 - OAUTH2 support, for single sign on
 - Sieve for filtering (for now see Rulesets in the account config)
+- Expose threading through IMAP extension
 - Privilege separation, isolating parts of the application to more restricted
   sandbox (e.g. new unauthenticated connections)
 - Using mox as backup MX

ctl.go

@@ -932,6 +932,62 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
 		}
 		w.xclose()
 
+	case "reassignthreads":
+		/* protocol:
+		> "reassignthreads"
+		> account or empty
+		< "ok" or error
+		< stream
+		*/
+
+		accountOpt := ctl.xread()
+		ctl.xwriteok()
+		w := ctl.writer()
+
+		xreassignThreads := func(accName string) {
+			acc, err := store.OpenAccount(accName)
+			ctl.xcheck(err, "open account")
+			defer func() {
+				err := acc.Close()
+				log.Check(err, "closing account after reassigning threads")
+			}()
+
+			// We don't want to step on an existing upgrade process.
+			err = acc.ThreadingWait(ctl.log)
+			ctl.xcheck(err, "waiting for threading upgrade to finish")
+			// todo: should we try to continue if the threading upgrade failed? only if there is a chance it will succeed this time...
+
+			// todo: reassigning isn't atomic (in a single transaction), ideally it would be (bstore would need to be able to handle large updates).
+			const batchSize = 50000
+			total, err := acc.ResetThreading(ctx, ctl.log, batchSize, true)
+			ctl.xcheck(err, "resetting threading fields")
+			_, err = fmt.Fprintf(w, "New thread base subject assigned to %d message(s), starting to reassign threads...\n", total)
+			ctl.xcheck(err, "write")
+
+			// Assign threads again. Ideally we would do this in a single transaction, but
+			// bstore/boltdb cannot handle so many pending changes, so we set a high batchsize.
+			err = acc.AssignThreads(ctx, ctl.log, nil, 0, 50000, w)
+			ctl.xcheck(err, "reassign threads")
+
+			_, err = fmt.Fprintf(w, "Threads reassigned. You should invalidate messages stored at imap clients with the \"mox bumpuidvalidity account [mailbox]\" command.\n")
+			ctl.xcheck(err, "write")
+		}
+
+		if accountOpt != "" {
+			xreassignThreads(accountOpt)
+		} else {
+			for i, accName := range mox.Conf.Accounts() {
+				var line string
+				if i > 0 {
+					line = "\n"
+				}
+				_, err := fmt.Fprintf(w, "%sReassigning threads for account %s...\n", line, accName)
+				ctl.xcheck(err, "write")
+				xreassignThreads(accName)
+			}
+		}
+		w.xclose()
+
 	case "backup":
 		backupctl(ctx, ctl)
 

ctl_test.go

@@ -217,6 +217,14 @@ func TestCtl(t *testing.T) {
 		ctlcmdReparse(ctl, "")
 	})
 
+	// "reassignthreads"
+	testctl(func(ctl *ctl) {
+		ctlcmdReassignthreads(ctl, "mjl")
+	})
+	testctl(func(ctl *ctl) {
+		ctlcmdReassignthreads(ctl, "")
+	})
+
 	// "backup", backup account.
 	err = dmarcdb.Init()
 	tcheck(t, err, "dmarcdb init")

doc.go

@@ -77,6 +77,7 @@ low-maintenance self-hosted email.
 	mox ensureparsed account
 	mox recalculatemailboxcounts account
 	mox message parse message.eml
+	mox reassignthreads [account]
 
 Many commands talk to a running mox instance, through the ctl file in the data
 directory. Specify the configuration file (that holds the path to the data
@@ -882,6 +883,40 @@ incorrect. This command will find, fix and print them.
 Parse message, print JSON representation.
 
 	usage: mox message parse message.eml
+
+# mox reassignthreads
+
+Reassign message threads.
+
+For all accounts, or optionally only the specified account.
+
+Threading for all messages in an account is first reset, and new base subject
+and normalized message-id saved with the message. Then all messages are
+evaluated and matched against their parents/ancestors.
+
+Messages are matched based on the References header, with a fall-back to an
+In-Reply-To header, and if neither is present/valid, based only on base
+subject.
+
+A References header typically points to multiple previous messages in a
+hierarchy. From oldest ancestor to most recent parent. An In-Reply-To header
+would have only a message-id of the parent message.
+
+A message is only linked to a parent/ancestor if their base subject is the
+same. This ensures unrelated replies, with a new subject, are placed in their
+own thread.
+
+The base subject is lower cased, has whitespace collapsed to a single
+space, and some components removed: leading "Re:", "Fwd:", "Fw:", or bracketed
+tag (that mailing lists often add, e.g. "[listname]"), trailing "(fwd)", or
+enclosing "[fwd: ...]".
+
+Messages are linked to all their ancestors. If an intermediate parent/ancestor
+message is deleted in the future, the message can still be linked to the earlier
+ancestors. If the direct parent already wasn't available while matching, this is
+stored as the message having a "missing link" to its stored ancestors.
+
+	usage: mox reassignthreads [account]
 */
 package main
 

gentestdata.go

@@ -56,8 +56,6 @@ func cmdGentestdata(c *cmd) {
 
 	log := mlog.New("gentestdata")
 	ctxbg := context.Background()
-	mox.Shutdown = ctxbg
-	mox.Context = ctxbg
 	mox.Conf.Log[""] = mlog.LevelInfo
 	mlog.SetConfig(mox.Conf.Log)
 
@@ -241,12 +239,16 @@ Accounts:
 	// First account without messages.
 	accTest0, err := store.OpenAccount("test0")
 	xcheckf(err, "open account test0")
+	err = accTest0.ThreadingWait(log)
+	xcheckf(err, "wait for threading to finish")
 	err = accTest0.Close()
 	xcheckf(err, "close account")
 
 	// Second account with one message.
 	accTest1, err := store.OpenAccount("test1")
 	xcheckf(err, "open account test1")
+	err = accTest1.ThreadingWait(log)
+	xcheckf(err, "wait for threading to finish")
 	err = accTest1.DB.Write(ctxbg, func(tx *bstore.Tx) error {
 		inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
 		xcheckf(err, "looking up inbox")
@@ -281,7 +283,7 @@ Accounts:
 		xcheckf(err, "creating temp file for delivery")
 		_, err = fmt.Fprint(mf, msg)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest1.DeliverMessage(log, tx, &m, mf, true, false, true)
+		err = accTest1.DeliverMessage(log, tx, &m, mf, true, false, true, false)
 		xcheckf(err, "add message to account test1")
 		err = mf.Close()
 		xcheckf(err, "closing file")
@@ -301,6 +303,8 @@ Accounts:
 	// Third account with two messages and junkfilter.
 	accTest2, err := store.OpenAccount("test2")
 	xcheckf(err, "open account test2")
+	err = accTest2.ThreadingWait(log)
+	xcheckf(err, "wait for threading to finish")
 	err = accTest2.DB.Write(ctxbg, func(tx *bstore.Tx) error {
 		inbox, err := bstore.QueryTx[store.Mailbox](tx).FilterNonzero(store.Mailbox{Name: "Inbox"}).Get()
 		xcheckf(err, "looking up inbox")
@@ -335,7 +339,7 @@ Accounts:
 		xcheckf(err, "creating temp file for delivery")
 		_, err = fmt.Fprint(mf0, msg0)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest2.DeliverMessage(log, tx, &m0, mf0, true, false, false)
+		err = accTest2.DeliverMessage(log, tx, &m0, mf0, true, false, false, false)
 		xcheckf(err, "add message to account test2")
 		err = mf0.Close()
 		xcheckf(err, "closing file")
@@ -362,7 +366,7 @@ Accounts:
 		xcheckf(err, "creating temp file for delivery")
 		_, err = fmt.Fprint(mf1, msg1)
 		xcheckf(err, "writing deliver message to file")
-		err = accTest2.DeliverMessage(log, tx, &m1, mf1, true, false, false)
+		err = accTest2.DeliverMessage(log, tx, &m1, mf1, true, false, false, false)
 		xcheckf(err, "add message to account test2")
 		err = mf1.Close()
 		xcheckf(err, "closing file")

imapserver/server.go

@@ -31,7 +31,7 @@ not in IMAP4rev2). ../rfc/3501:964
 - todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64?
 - todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes?
 - todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command.
-- future: more extensions: STATUS=SIZE, OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD, CREATE-SPECIAL-USE.
+- todo future: more extensions: STATUS=SIZE, OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD, CREATE-SPECIAL-USE.
 */
 
 import (
@@ -1199,7 +1199,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) {
 		case store.ChangeRemoveMailbox, store.ChangeAddMailbox, store.ChangeRenameMailbox, store.ChangeAddSubscription:
 			n = append(n, change)
 			continue
-		case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords:
+		case store.ChangeMailboxCounts, store.ChangeMailboxSpecialUse, store.ChangeMailboxKeywords, store.ChangeThread:
 		default:
 			panic(fmt.Errorf("missing case for %#v", change))
 		}
@@ -2740,7 +2740,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) {
 			err = tx.Update(&mb)
 			xcheckf(err, "updating mailbox counts")
 
-			err := c.account.DeliverMessage(c.log, tx, &m, msgFile, true, true, false)
+			err := c.account.DeliverMessage(c.log, tx, &m, msgFile, true, true, false, false)
 			xcheckf(err, "delivering message")
 		})
 

import.go

@@ -185,6 +185,20 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 	var mdnewf, mdcurf *os.File
 	var msgreader store.MsgSource
 
+	// Open account, creating a database file if it doesn't exist yet. It must be known
+	// in the configuration file.
+	a, err := store.OpenAccount(account)
+	ctl.xcheck(err, "opening account")
+	defer func() {
+		if a != nil {
+			err := a.Close()
+			ctl.log.Check(err, "closing account after import")
+		}
+	}()
+
+	err = a.ThreadingWait(ctl.log)
+	ctl.xcheck(err, "waiting for account thread upgrade")
+
 	defer func() {
 		if mboxf != nil {
 			err := mboxf.Close()
@@ -200,17 +214,6 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		}
 	}()
 
-	// Open account, creating a database file if it doesn't exist yet. It must be known
-	// in the configuration file.
-	a, err := store.OpenAccount(account)
-	ctl.xcheck(err, "opening account")
-	defer func() {
-		if a != nil {
-			err := a.Close()
-			ctl.log.Check(err, "closing account after import")
-		}
-	}()
-
 	// Messages don't always have a junk flag set. We'll assume anything in a mailbox
 	// starting with junk or spam is junk mail.
 
@@ -277,7 +280,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		const consumeFile = true
 		const sync = false
 		const notrain = true
-		err := a.DeliverMessage(ctl.log, tx, m, mf, consumeFile, sync, notrain)
+		const nothreads = true
+		err := a.DeliverMessage(ctl.log, tx, m, mf, consumeFile, sync, notrain, nothreads)
 		ctl.xcheck(err, "delivering message")
 		deliveredIDs = append(deliveredIDs, m.ID)
 		ctl.log.Debug("delivered message", mlog.Field("id", m.ID))
@@ -332,6 +336,11 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 			m.ParsedBuf, err = json.Marshal(p)
 			ctl.xcheck(err, "marshal parsed message structure")
 
+			// Set fields needed for future threading. By doing it now, DeliverMessage won't
+			// have to parse the Part again.
+			p.SetReaderAt(store.FileMsgReader(m.MsgPrefix, msgf))
+			m.PrepareThreading(ctl.log, &p)
+
 			if m.Received.IsZero() {
 				if p.Envelope != nil && !p.Envelope.Date.IsZero() {
 					m.Received = p.Envelope.Date
@@ -385,6 +394,12 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 			process(m, msgf, origPath)
 		}
 
+		// Match threads.
+		if len(deliveredIDs) > 0 {
+			err = a.AssignThreads(ctx, ctl.log, tx, deliveredIDs[0], 0, io.Discard)
+			ctl.xcheck(err, "assigning messages to threads")
+		}
+
 		// Get mailbox again, uidnext is likely updated.
 		mc := mb.MailboxCounts
 		err = tx.Get(&mb)

main.go

@@ -17,6 +17,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -37,6 +38,7 @@ import (
 	"github.com/mjl-/mox/message"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxio"
 	"github.com/mjl-/mox/moxvar"
 	"github.com/mjl-/mox/mtasts"
 	"github.com/mjl-/mox/publicsuffix"
@@ -143,6 +145,7 @@ var commands = []struct {
 	{"ensureparsed", cmdEnsureParsed},
 	{"recalculatemailboxcounts", cmdRecalculateMailboxCounts},
 	{"message parse", cmdMessageParse},
+	{"reassignthreads", cmdReassignthreads},
 
 	// Not listed.
 	{"helpall", cmdHelpall},
@@ -162,6 +165,7 @@ var commands = []struct {
 	{"ximport maildir", cmdXImportMaildir},
 	{"ximport mbox", cmdXImportMbox},
 	{"openaccounts", cmdOpenaccounts},
+	{"readmessages", cmdReadmessages},
 }
 
 var cmds []cmd
@@ -389,6 +393,10 @@ func main() {
 	// flag.
 	store.CheckConsistencyOnClose = false
 
+	ctxbg := context.Background()
+	mox.Shutdown = ctxbg
+	mox.Context = ctxbg
+
 	log.SetFlags(0)
 
 	// If invoked as sendmail, e.g. /usr/sbin/sendmail, we do enough so cron can get a
@@ -2086,6 +2094,7 @@ func cmdVersion(c *cmd) {
 	fmt.Println(moxvar.Version)
 }
 
+// todo: should make it possible to run this command against a running mox. it should disconnect existing clients for accounts with a bumped uidvalidity, so they will reconnect and refetch the data.
 func cmdBumpUIDValidity(c *cmd) {
 	c.params = "account [mailbox]"
 	c.help = `Change the IMAP UID validity of the mailbox, causing IMAP clients to refetch messages.
@@ -2476,13 +2485,193 @@ Opens database files directly, not going through a running mox instance.
 		c.Usage()
 	}
 
+	clog := mlog.New("openaccounts")
+
 	dataDir := filepath.Clean(args[0])
 	for _, accName := range args[1:] {
 		accDir := filepath.Join(dataDir, "accounts", accName)
-		log.Printf("opening account %s...", filepath.Join(accDir, accName))
+		log.Printf("opening account %s...", accDir)
 		a, err := store.OpenAccountDB(accDir, accName)
 		xcheckf(err, "open account %s", accName)
+		err = a.ThreadingWait(clog)
+		xcheckf(err, "wait for threading upgrade to complete for %s", accName)
 		err = a.Close()
 		xcheckf(err, "close account %s", accName)
 	}
 }
+
+func cmdReassignthreads(c *cmd) {
+	c.params = "[account]"
+	c.help = `Reassign message threads.
+
+For all accounts, or optionally only the specified account.
+
+Threading for all messages in an account is first reset, and new base subject
+and normalized message-id saved with the message. Then all messages are
+evaluated and matched against their parents/ancestors.
+
+Messages are matched based on the References header, with a fall-back to an
+In-Reply-To header, and if neither is present/valid, based only on base
+subject.
+
+A References header typically points to multiple previous messages in a
+hierarchy. From oldest ancestor to most recent parent. An In-Reply-To header
+would have only a message-id of the parent message.
+
+A message is only linked to a parent/ancestor if their base subject is the
+same. This ensures unrelated replies, with a new subject, are placed in their
+own thread.
+
+The base subject is lower cased, has whitespace collapsed to a single
+space, and some components removed: leading "Re:", "Fwd:", "Fw:", or bracketed
+tag (that mailing lists often add, e.g. "[listname]"), trailing "(fwd)", or
+enclosing "[fwd: ...]".
+
+Messages are linked to all their ancestors. If an intermediate parent/ancestor
+message is deleted in the future, the message can still be linked to the earlier
+ancestors. If the direct parent already wasn't available while matching, this is
+stored as the message having a "missing link" to its stored ancestors.
+`
+	args := c.Parse()
+	if len(args) > 1 {
+		c.Usage()
+	}
+
+	mustLoadConfig()
+	var account string
+	if len(args) == 1 {
+		account = args[0]
+	}
+	ctlcmdReassignthreads(xctl(), account)
+}
+
+func ctlcmdReassignthreads(ctl *ctl, account string) {
+	ctl.xwrite("reassignthreads")
+	ctl.xwrite(account)
+	ctl.xreadok()
+	ctl.xstreamto(os.Stdout)
+}
+
+func cmdReadmessages(c *cmd) {
+	c.unlisted = true
+	c.params = "datadir account ..."
+	c.help = `Open account, parse several headers for all messages.
+
+For performance testing.
+
+Opens database files directly, not going through a running mox instance.
+`
+
+	gomaxprocs := runtime.GOMAXPROCS(0)
+	var procs, workqueuesize, limit int
+	c.flag.IntVar(&procs, "procs", gomaxprocs, "number of goroutines for reading messages")
+	c.flag.IntVar(&workqueuesize, "workqueuesize", 2*gomaxprocs, "number of messages to keep in work queue")
+	c.flag.IntVar(&limit, "limit", 0, "number of messages to process if greater than zero")
+	args := c.Parse()
+	if len(args) <= 1 {
+		c.Usage()
+	}
+
+	type threadPrep struct {
+		references []string
+		inReplyTo  []string
+	}
+
+	threadingFields := [][]byte{
+		[]byte("references"),
+		[]byte("in-reply-to"),
+	}
+
+	dataDir := filepath.Clean(args[0])
+	for _, accName := range args[1:] {
+		accDir := filepath.Join(dataDir, "accounts", accName)
+		log.Printf("opening account %s...", accDir)
+		a, err := store.OpenAccountDB(accDir, accName)
+		xcheckf(err, "open account %s", accName)
+
+		prepareMessages := func(in, out chan moxio.Work[store.Message, threadPrep]) {
+			headerbuf := make([]byte, 8*1024)
+			scratch := make([]byte, 4*1024)
+			for {
+				w, ok := <-in
+				if !ok {
+					return
+				}
+
+				m := w.In
+				var partialPart struct {
+					HeaderOffset int64
+					BodyOffset   int64
+				}
+				if err := json.Unmarshal(m.ParsedBuf, &partialPart); err != nil {
+					w.Err = fmt.Errorf("unmarshal part: %v", err)
+				} else {
+					size := partialPart.BodyOffset - partialPart.HeaderOffset
+					if int(size) > len(headerbuf) {
+						headerbuf = make([]byte, size)
+					}
+					if size > 0 {
+						buf := headerbuf[:int(size)]
+						err := func() error {
+							mr := a.MessageReader(m)
+							defer mr.Close()
+
+							// ReadAt returns whole buffer or error. Single read should be fast.
+							n, err := mr.ReadAt(buf, partialPart.HeaderOffset)
+							if err != nil || n != len(buf) {
+								return fmt.Errorf("read header: %v", err)
+							}
+							return nil
+						}()
+						if err != nil {
+							w.Err = err
+						} else if h, err := message.ParseHeaderFields(buf, scratch, threadingFields); err != nil {
+							w.Err = err
+						} else {
+							w.Out.references = h["References"]
+							w.Out.inReplyTo = h["In-Reply-To"]
+						}
+					}
+				}
+
+				out <- w
+			}
+		}
+
+		n := 0
+		t := time.Now()
+		t0 := t
+
+		processMessage := func(m store.Message, prep threadPrep) error {
+			if n%100000 == 0 {
+				log.Printf("%d messages (delta %s)", n, time.Since(t))
+				t = time.Now()
+			}
+			n++
+			return nil
+		}
+
+		wq := moxio.NewWorkQueue[store.Message, threadPrep](procs, workqueuesize, prepareMessages, processMessage)
+
+		err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
+			q := bstore.QueryTx[store.Message](tx)
+			q.FilterEqual("Expunged", false)
+			q.SortAsc("ID")
+			if limit > 0 {
+				q.Limit(limit)
+			}
+			err = q.ForEach(wq.Add)
+			if err == nil {
+				err = wq.Finish()
+			}
+			wq.Stop()
+
+			return err
+		})
+		xcheckf(err, "processing message")
+
+		err = a.Close()
+		xcheckf(err, "close account %s", accName)
+		log.Printf("account %s, total time %s", accName, time.Since(t0))
+	}
+}

message/messageid.go

@@ -0,0 +1,53 @@
+package message
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/smtp"
+)
+
+var errBadMessageID = errors.New("not a message-id")
+
+// MessageIDCanonical parses the Message-ID, returning a canonical value that is
+// lower-cased, without <>, and no unneeded quoting. For matching in threading,
+// with References/In-Reply-To. If the message-id is invalid (e.g. no <>), an error
+// is returned. If the message-id could not be parsed as address (localpart "@"
+// domain), the raw value and the bool return parameter true is returned. It is
+// quite common that message-id's don't adhere to the localpart @ domain
+// syntax.
+func MessageIDCanonical(s string) (string, bool, error) {
+	// ../rfc/5322:1383
+
+	s = strings.TrimSpace(s)
+	if !strings.HasPrefix(s, "<") {
+		return "", false, fmt.Errorf("%w: missing <", errBadMessageID)
+	}
+	s = s[1:]
+	// Seen in practice: Message-ID: <valid@valid.example> (added by postmaster@some.example)
+	// Doesn't seem valid, but we allow it.
+	s, rem, have := strings.Cut(s, ">")
+	if !have || (rem != "" && (moxvar.Pedantic || !strings.HasPrefix(rem, " "))) {
+		return "", false, fmt.Errorf("%w: missing >", errBadMessageID)
+	}
+	// We canonicalize the Message-ID: lower-case, no unneeded quoting.
+	s = strings.ToLower(s)
+	if s == "" {
+		return "", false, fmt.Errorf("%w: empty message-id", errBadMessageID)
+	}
+	addr, err := smtp.ParseAddress(s)
+	if err != nil {
+		// Common reasons for not being an address:
+		// 1. underscore in hostname.
+		// 2. ip literal instead of domain.
+		// 3. two @'s, perhaps intended as time-separator
+		// 4. no @'s, so no domain/host
+		return s, true, nil
+	}
+	// We preserve the unicode-ness of domain.
+	t := strings.Split(s, "@")
+	s = addr.Localpart.String() + "@" + t[len(t)-1]
+	return s, false, nil
+}

message/messageid_test.go

@@ -0,0 +1,29 @@
+package message
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestMessageIDCanonical(t *testing.T) {
+	check := func(s string, expID string, expRaw bool, expErr error) {
+		t.Helper()
+
+		id, raw, err := MessageIDCanonical(s)
+		if id != expID || raw != expRaw || (expErr == nil) != (err == nil) || err != nil && !errors.Is(err, expErr) {
+			t.Fatalf("got message-id %q, raw %v, err %v, expected %q %v %v, for message-id %q", id, raw, err, expID, expRaw, expErr, s)
+		}
+	}
+
+	check("bogus", "", false, errBadMessageID)
+	check("<bogus@host", "", false, errBadMessageID)
+	check("bogus@host>", "", false, errBadMessageID)
+	check("<>", "", false, errBadMessageID)
+	check("<user@domain>", "user@domain", false, nil)
+	check("<USER@DOMAIN>", "user@domain", false, nil)
+	check("<user@[10.0.0.1]>", "user@[10.0.0.1]", true, nil)
+	check("<user@domain> (added by postmaster@isp.example)", "user@domain", false, nil)
+	check("<user@domain> other", "user@domain", false, nil)
+	check("<User@Domain@Time>", "user@domain@time", true, nil)
+	check("<User>", "user", true, nil)
+}

message/parseheaderfields.go

@@ -0,0 +1,77 @@
+package message
+
+import (
+	"bytes"
+	"fmt"
+	"net/mail"
+	"net/textproto"
+)
+
+// ParseHeaderFields parses only the header fields in "fields" from the complete
+// header buffer "header", while using "scratch" as temporary space, prevent lots
+// of unneeded allocations when only a few headers are needed.
+func ParseHeaderFields(header []byte, scratch []byte, fields [][]byte) (textproto.MIMEHeader, error) {
+	// todo: should not use mail.ReadMessage, it allocates a bufio.Reader. should implement header parsing ourselves.
+
+	// Gather the raw lines for the fields, with continuations, without the other
+	// headers. Put them in a byte slice and only parse those headers. For now, use
+	// mail.ReadMessage without letting it do allocations for all headers.
+	scratch = scratch[:0]
+	var keepcontinuation bool
+	for len(header) > 0 {
+		if header[0] == ' ' || header[0] == '\t' {
+			// Continuation.
+			i := bytes.IndexByte(header, '\n')
+			if i < 0 {
+				i = len(header)
+			} else {
+				i++
+			}
+			if keepcontinuation {
+				scratch = append(scratch, header[:i]...)
+			}
+			header = header[i:]
+			continue
+		}
+		i := bytes.IndexByte(header, ':')
+		if i < 0 || i > 0 && (header[i-1] == ' ' || header[i-1] == '\t') {
+			i = bytes.IndexByte(header, '\n')
+			if i < 0 {
+				break
+			}
+			header = header[i+1:]
+			keepcontinuation = false
+			continue
+		}
+		k := header[:i]
+		keepcontinuation = false
+		for _, f := range fields {
+			if bytes.EqualFold(k, f) {
+				keepcontinuation = true
+				break
+			}
+		}
+		i = bytes.IndexByte(header, '\n')
+		if i < 0 {
+			i = len(header)
+		} else {
+			i++
+		}
+		if keepcontinuation {
+			scratch = append(scratch, header[:i]...)
+		}
+		header = header[i:]
+	}
+
+	if len(scratch) == 0 {
+		return nil, nil
+	}
+
+	scratch = append(scratch, "\r\n"...)
+
+	msg, err := mail.ReadMessage(bytes.NewReader(scratch))
+	if err != nil {
+		return nil, fmt.Errorf("reading message header")
+	}
+	return textproto.MIMEHeader(msg.Header), nil
+}

message/parseheaderfields_test.go

@@ -0,0 +1,40 @@
+package message
+
+import (
+	"net/textproto"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParseHeaderFields(t *testing.T) {
+	check := func(headers string, fields []string, expHdrs textproto.MIMEHeader, expErr error) {
+		t.Helper()
+
+		buffields := [][]byte{}
+		for _, f := range fields {
+			buffields = append(buffields, []byte(f))
+		}
+
+		scratches := [][]byte{
+			make([]byte, 0),
+			make([]byte, 4*1024),
+		}
+		for _, scratch := range scratches {
+			hdrs, err := ParseHeaderFields([]byte(strings.ReplaceAll(headers, "\n", "\r\n")), scratch, buffields)
+			if !reflect.DeepEqual(hdrs, expHdrs) || !reflect.DeepEqual(err, expErr) {
+				t.Fatalf("got %v %v, expected %v %v", hdrs, err, expHdrs, expErr)
+			}
+		}
+	}
+
+	check("", []string{"subject"}, textproto.MIMEHeader(nil), nil)
+	check("Subject: test\n", []string{"subject"}, textproto.MIMEHeader{"Subject": []string{"test"}}, nil)
+	check("References: <id@host>\nOther: ignored\nSubject: first\nSubject: test\n\tcontinuation\n", []string{"subject", "REFERENCES"}, textproto.MIMEHeader{"References": []string{"<id@host>"}, "Subject": []string{"first", "test continuation"}}, nil)
+	check(":\n", []string{"subject"}, textproto.MIMEHeader(nil), nil)
+	check("bad\n", []string{"subject"}, textproto.MIMEHeader(nil), nil)
+	check("subject: test\n continuation without end\n", []string{"subject"}, textproto.MIMEHeader{"Subject": []string{"test continuation without end"}}, nil)
+	check("subject: test\n", []string{"subject"}, textproto.MIMEHeader{"Subject": []string{"test"}}, nil)
+	check("subject \t: test\n", []string{"subject"}, textproto.MIMEHeader(nil), nil) // Note: In go1.20, this would be interpreted as valid "Subject" header. Not in go1.21.
+	// note: in go1.20, missing end of line would cause it to be ignored, in go1.21 it is used.
+}

message/part.go

@@ -88,7 +88,7 @@ type Part struct {
 // Envelope holds the basic/common message headers as used in IMAP4.
 type Envelope struct {
 	Date      time.Time
-	Subject   string
+	Subject   string // Q/B-word-decoded.
 	From      []Address
 	Sender    []Address
 	ReplyTo   []Address

message/referencedids.go

@@ -0,0 +1,74 @@
+package message
+
+import (
+	"strings"
+
+	"github.com/mjl-/mox/smtp"
+)
+
+// ReferencedIDs returns the Message-IDs referenced from the References header(s),
+// with a fallback to the In-Reply-To header(s). The ids are canonicalized for
+// thread-matching, like with MessageIDCanonical. Empty message-id's are skipped.
+func ReferencedIDs(references []string, inReplyTo []string) ([]string, error) {
+	var refids []string // In thread-canonical form.
+
+	// parse and add 0 or 1 reference, returning the remaining refs string for a next attempt.
+	parse1 := func(refs string, one bool) string {
+		refs = strings.TrimLeft(refs, " \t\r\n")
+		if !strings.HasPrefix(refs, "<") {
+			// To make progress, we skip to next space or >.
+			i := strings.IndexAny(refs, " >")
+			if i < 0 {
+				return ""
+			}
+			return refs[i+1:]
+		}
+		refs = refs[1:]
+		// Look for the ending > or next <. If < is before >, this entry is truncated.
+		i := strings.IndexAny(refs, "<>")
+		if i < 0 {
+			return ""
+		}
+		if refs[i] == '<' {
+			// Truncated entry, we ignore it.
+			return refs[i:]
+		}
+		ref := strings.ToLower(refs[:i])
+		// Some MUAs wrap References line in the middle of message-id's, and others
+		// recombine them. Take out bare WSP in message-id's.
+		ref = strings.ReplaceAll(ref, " ", "")
+		ref = strings.ReplaceAll(ref, "\t", "")
+		refs = refs[i+1:]
+		// Canonicalize the quotedness of the message-id.
+		addr, err := smtp.ParseAddress(ref)
+		if err == nil {
+			// Leave the hostname form intact.
+			t := strings.Split(ref, "@")
+			ref = addr.Localpart.String() + "@" + t[len(t)-1]
+		}
+		// log.Errorx("assigning threads: bad reference in references header, using raw value", err, mlog.Field("msgid", mid), mlog.Field("reference", ref))
+		if ref != "" {
+			refids = append(refids, ref)
+		}
+		return refs
+	}
+
+	// References is the modern way (for a long time already) to reference ancestors.
+	// The direct parent is typically at the end of the list.
+	for _, refs := range references {
+		for refs != "" {
+			refs = parse1(refs, false)
+		}
+	}
+	// We only look at the In-Reply-To header if we didn't find any References.
+	if len(refids) == 0 {
+		for _, s := range inReplyTo {
+			parse1(s, true)
+			if len(refids) > 0 {
+				break
+			}
+		}
+	}
+
+	return refids, nil
+}

message/referencedids_test.go

@@ -0,0 +1,35 @@
+package message
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestReferencedIDs(t *testing.T) {
+	check := func(msg string, expRefs []string) {
+		t.Helper()
+
+		p, err := Parse(xlog, true, strings.NewReader(msg))
+		tcheck(t, err, "parsing message")
+
+		h, err := p.Header()
+		tcheck(t, err, "parsing header")
+
+		refs, err := ReferencedIDs(h["References"], h["In-Reply-To"])
+		tcheck(t, err, "parsing references/in-reply-to")
+		tcompare(t, refs, expRefs)
+	}
+
+	check("References: bogus\r\n", nil)
+	check("References: <User@host>\r\n", []string{"user@host"})
+	check("References: <User@tést.example>\r\n", []string{"user@tést.example"})
+	check("References: <User@xn--tst-bma.example>\r\n", []string{"user@xn--tst-bma.example"})
+	check("References: <User@bad_label.domain>\r\n", []string{"user@bad_label.domain"})
+	check("References: <truncated@hos <user@host>\r\n", []string{"user@host"})
+	check("References: <previously wrapped@host>\r\n", []string{"previouslywrapped@host"})
+	check("References: <user1@host> <user2@other.example>\r\n", []string{"user1@host", "user2@other.example"})
+	check("References: <missinghost>\r\n", []string{"missinghost"})
+	check("References: <user@host@time>\r\n", []string{"user@host@time"})
+	check("References: bogus bad <user@host>\r\n", []string{"user@host"})
+	check("In-Reply-To: <user@host> more stuff\r\nReferences: bogus bad\r\n", []string{"user@host"})
+}

message/threadsubject.go

@@ -0,0 +1,124 @@
+package message
+
+import (
+	"strings"
+)
+
+// ThreadSubject returns the base subject to use for matching against other
+// messages, to see if they belong to the same thread. A matching subject is
+// always required to match to an existing thread, both if
+// References/In-Reply-To header(s) are present, and if not.
+//
+// Subject should already be q/b-word-decoded.
+//
+// If allowNull is true, base subjects with a \0 can be returned. If not set,
+// an empty string is returned if a base subject would have a \0.
+func ThreadSubject(subject string, allowNull bool) (threadSubject string, isResponse bool) {
+	subject = strings.ToLower(subject)
+
+	// ../rfc/5256:101, Step 1.
+	var s string
+	for _, c := range subject {
+		if c == '\r' {
+			continue
+		} else if c == ' ' || c == '\n' || c == '\t' {
+			if !strings.HasSuffix(s, " ") {
+				s += " "
+			}
+		} else {
+			s += string(c)
+		}
+	}
+
+	// ../rfc/5256:107 ../rfc/5256:811, removing mailing list tag "[...]" and reply/forward "re"/"fwd" prefix.
+	removeBlob := func(s string) string {
+		for i, c := range s {
+			if i == 0 {
+				if c != '[' {
+					return s
+				}
+			} else if c == '[' {
+				return s
+			} else if c == ']' {
+				s = s[i+1:]                     // Past [...].
+				s = strings.TrimRight(s, " \t") // *WSP
+				return s
+			}
+		}
+		return s
+	}
+	// ../rfc/5256:107 ../rfc/5256:811
+	removeLeader := func(s string) string {
+		if strings.HasPrefix(s, " ") || strings.HasPrefix(s, "\t") {
+			s = s[1:] // WSP
+		}
+
+		orig := s
+
+		// Remove zero or more subj-blob
+		for {
+			prevs := s
+			s = removeBlob(s)
+			if prevs == s {
+				break
+			}
+		}
+
+		if strings.HasPrefix(s, "re") {
+			s = s[2:]
+		} else if strings.HasPrefix(s, "fwd") {
+			s = s[3:]
+		} else if strings.HasPrefix(s, "fw") {
+			s = s[2:]
+		} else {
+			return orig
+		}
+		s = strings.TrimLeft(s, " \t") // *WSP
+		s = removeBlob(s)
+		if !strings.HasPrefix(s, ":") {
+			return orig
+		}
+		s = s[1:]
+		isResponse = true
+		return s
+	}
+
+	for {
+		// ../rfc/5256:104 ../rfc/5256:817, remove trailing "(fwd)" or WSP, Step 2.
+		for {
+			prevs := s
+			s = strings.TrimRight(s, " \t")
+			if strings.HasSuffix(s, "(fwd)") {
+				s = strings.TrimSuffix(s, "(fwd)")
+				isResponse = true
+			}
+			if s == prevs {
+				break
+			}
+		}
+
+		for {
+			prevs := s
+			s = removeLeader(s) // Step 3.
+			if ns := removeBlob(s); ns != "" {
+				s = ns // Step 4.
+			}
+			// Step 5, ../rfc/5256:123
+			if s == prevs {
+				break
+			}
+		}
+
+		// Step 6. ../rfc/5256:128 ../rfc/5256:805
+		if strings.HasPrefix(s, "[fwd:") && strings.HasSuffix(s, "]") {
+			s = s[len("[fwd:") : len(s)-1]
+			isResponse = true
+			continue // From step 2 again.
+		}
+		break
+	}
+	if !allowNull && strings.ContainsRune(s, 0) {
+		s = ""
+	}
+	return s, isResponse
+}

message/threadsubject_test.go

@@ -0,0 +1,35 @@
+package message
+
+import (
+	"testing"
+)
+
+func TestThreadSubject(t *testing.T) {
+	check := func(s, expBase string, expResp bool) {
+		t.Helper()
+
+		base, isResp := ThreadSubject(s, false)
+		if base != expBase || isResp != expResp {
+			t.Fatalf("got base %q, resp %v, expected %q %v for subject %q", base, isResp, expBase, expResp, s)
+		}
+	}
+
+	check("test", "test", false)
+	check(" a  b\tc\r\n d\t", "a b c d", false)
+	check("test (fwd) (fwd) ", "test", true)
+	check("re: test", "test", true)
+	check("fw: test", "test", true)
+	check("fwd: test", "test", true)
+	check("fwd [tag] Test", "fwd [tag] test", false)
+	check("[list] re: a b c\t", "a b c", true)
+	check("[list] fw: a b c", "a b c", true)
+	check("[tag1][tag2] [tag3]\t re: a b c", "a b c", true)
+	check("[tag1][tag2] [tag3]\t re: a \u0000b c", "", true)
+	check("[list] fw:[tag] a b c", "a b c", true)
+	check("[list] re: [list] fwd: a b c\t", "a b c", true)
+	check("[fwd: a b c]", "a b c", true)
+	check("[fwd: [fwd: a b c]]", "a b c", true)
+	check("[fwd: [list] re: a b c]", "a b c", true)
+	check("[nonlist]", "[nonlist]", false)
+	check("fwd [list]:", "", true)
+}

moxio/workq.go

@@ -0,0 +1,131 @@
+package moxio
+
+import (
+	"sync"
+)
+
+// Work is a slot for work that needs to be done.
+type Work[T, R any] struct {
+	In  T
+	Err error
+	Out R
+
+	i    int
+	done bool
+}
+
+// WorkQueue can be used to execute a work load where many items are processed
+// with a slow step and where a pool of workers goroutines to execute the slow
+// step helps. Reading messages from the database file is fast and cannot be
+// easily done concurrently, but reading the message file from disk and parsing
+// the headers is the bottleneck. The workqueue can manage the goroutines that
+// read the message file from disk and parse.
+type WorkQueue[T, R any] struct {
+	max   int
+	ring  []Work[T, R]
+	start int
+	n     int
+
+	wg   sync.WaitGroup // For waiting for workers to stop.
+	work chan Work[T, R]
+	done chan Work[T, R]
+
+	process func(T, R) error
+}
+
+// NewWorkQueue creates a new work queue with "procs" goroutines, and a total work
+// queue size of "size" (e.g. 2*procs). The worker goroutines run "preparer", which
+// should be a loop receiving work from "in" and sending the work result (with Err
+// or Out set) on "out". The preparer function should return when the "in" channel
+// is closed, the signal to stop. WorkQueue processes the results in the order they
+// went in, so prepared work that was scheduled after earlier work that is not yet
+// prepared will wait and be queued.
+func NewWorkQueue[T, R any](procs, size int, preparer func(in, out chan Work[T, R]), process func(T, R) error) *WorkQueue[T, R] {
+	wq := &WorkQueue[T, R]{
+		max:     size,
+		ring:    make([]Work[T, R], size),
+		work:    make(chan Work[T, R], size), // Ensure scheduling never blocks for main goroutine.
+		done:    make(chan Work[T, R], size), // Ensure sending result never blocks for worker goroutine.
+		process: process,
+	}
+
+	wq.wg.Add(procs)
+	for i := 0; i < procs; i++ {
+		go func() {
+			defer wq.wg.Done()
+			preparer(wq.work, wq.done)
+		}()
+	}
+
+	return wq
+}
+
+// Add adds new work to be prepared to the queue. If the queue is full, it
+// waits until space becomes available, i.e. when the head of the queue has
+// work that becomes prepared. Add processes the prepared items to make space
+// available.
+func (wq *WorkQueue[T, R]) Add(in T) error {
+	// Schedule the new work if we can.
+	if wq.n < wq.max {
+		wq.work <- Work[T, R]{i: (wq.start + wq.n) % wq.max, done: true, In: in}
+		wq.n++
+		return nil
+	}
+
+	// We cannot schedule new work. Wait for finished work until start is done.
+	for {
+		w := <-wq.done
+		wq.ring[w.i] = w
+		if w.i == wq.start {
+			break
+		}
+	}
+
+	// Process as much finished work as possible. Will be at least 1.
+	if err := wq.processHead(); err != nil {
+		return err
+	}
+
+	// Schedule this message as new work.
+	wq.work <- Work[T, R]{i: (wq.start + wq.n) % wq.max, done: true, In: in}
+	wq.n++
+	return nil
+}
+
+// processHead processes the work at the head of the queue by calling process
+// on the work.
+func (wq *WorkQueue[T, R]) processHead() error {
+	for wq.n > 0 && wq.ring[wq.start].done {
+		wq.ring[wq.start].done = false
+		w := wq.ring[wq.start]
+		wq.start = (wq.start + 1) % len(wq.ring)
+		wq.n -= 1
+
+		if w.Err != nil {
+			return w.Err
+		}
+		if err := wq.process(w.In, w.Out); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Finish waits for the remaining work to be prepared and processes the work.
+func (wq *WorkQueue[T, R]) Finish() error {
+	var err error
+	for wq.n > 0 && err == nil {
+		w := <-wq.done
+		wq.ring[w.i] = w
+
+		err = wq.processHead()
+	}
+	return err
+}
+
+// Stop shuts down the worker goroutines and waits until they have returned.
+// Stop must always be called on a WorkQueue, otherwise the goroutines never stop.
+func (wq *WorkQueue[T, R]) Stop() {
+	close(wq.work)
+	wq.wg.Wait()
+}

smtpserver/rejects.go

@@ -23,7 +23,10 @@ func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m *
 	} else if header, err := p.Header(); err != nil {
 		log.Infox("parsing reject message header for message-id", err)
 	} else {
-		msgID = header.Get("Message-Id")
+		msgID, _, err = message.MessageIDCanonical(header.Get("Message-Id"))
+		if err != nil {
+			log.Debugx("parsing message-id for reject", err, mlog.Field("messageid", header.Get("Message-Id")))
+		}
 	}
 
 	// We must not read MsgPrefix, it will likely change for subsequent deliveries.

smtpserver/server.go

@@ -2327,7 +2327,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 		if !a.accept {
 			conf, _ := acc.Conf()
 			if conf.RejectsMailbox != "" {
-				present, messageid, messagehash, err := rejectPresent(log, acc, conf.RejectsMailbox, m, dataFile)
+				present, _, messagehash, err := rejectPresent(log, acc, conf.RejectsMailbox, m, dataFile)
 				if err != nil {
 					log.Errorx("checking whether reject is already present", err)
 				} else if !present {
@@ -2336,7 +2336,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 					// Regular automatic junk flags configuration applies to these messages. The
 					// default is to treat these as neutral, so they won't cause outright rejections
 					// due to reputation for later delivery attempts.
-					m.MessageID = messageid
 					m.MessageHash = messagehash
 					acc.WithWLock(func() {
 						hasSpace := true
@@ -2390,7 +2389,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 			}
 		}
 
-		// If a forwarded message and this is a first-time sender, wait before actually
+		// If this is a first-time sender and not a forwarded message, wait before actually
 		// delivering. If this turns out to be a spammer, we've kept one of their
 		// connections busy.
 		if delayFirstTime && !m.IsForward && a.reason == reasonNoBadSignals && c.firstTimeSenderDelay > 0 {
@@ -2432,8 +2431,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 			log.Info("incoming message delivered", mlog.Field("reason", a.reason), mlog.Field("msgfrom", msgFrom))
 
 			conf, _ := acc.Conf()
-			if conf.RejectsMailbox != "" && messageID != "" {
+			if conf.RejectsMailbox != "" && m.MessageID != "" {
-				if err := acc.RejectsRemove(log, conf.RejectsMailbox, messageID); err != nil {
+				if err := acc.RejectsRemove(log, conf.RejectsMailbox, m.MessageID); err != nil {
 					log.Errorx("removing message from rejects mailbox", err, mlog.Field("messageid", messageID))
 				}
 			}

store/account.go

@@ -34,7 +34,9 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime/debug"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -48,6 +50,7 @@ import (
 	"github.com/mjl-/mox/config"
 	"github.com/mjl-/mox/dns"
 	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/metrics"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/moxio"
@@ -444,14 +447,41 @@ type Message struct {
 	OrigEHLODomain  string
 	OrigDKIMDomains []string
 
-	// Value of Message-Id header. Only set for messages that were
+	// Canonicalized Message-Id, always lower-case and normalized quoting, without
-	// delivered to the rejects mailbox. For ensuring such messages are
+	// <>'s. Empty if missing. Used for matching message threads, and to prevent
-	// delivered only once. Value includes <>.
+	// duplicate reject delivery.
 	MessageID string `bstore:"index"`
+	// lower-case: ../rfc/5256:495
 
-	// Hash of message. For rejects delivery, so optional like MessageID.
+	// For matching threads in case there is no References/In-Reply-To header. It is
+	// lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.
+	SubjectBase string `bstore:"index"`
+	// ../rfc/5256:90
+
+	// Hash of message. For rejects delivery in case there is no Message-ID, only set
+	// when delivered as reject.
 	MessageHash []byte
 
+	// ID of message starting this thread.
+	ThreadID int64 `bstore:"index"`
+	// IDs of parent messages, from closest parent to the root message. Parent messages
+	// may be in a different mailbox, or may no longer exist. ThreadParentIDs must
+	// never contain the message id itself (a cycle), and parent messages must
+	// reference the same ancestors.
+	ThreadParentIDs []int64
+	// ThreadMissingLink is true if there is no match with a direct parent. E.g. first
+	// ID in ThreadParentIDs is not the direct ancestor (an intermediate message may
+	// have been deleted), or subject-based matching was done.
+	ThreadMissingLink bool
+	// If set, newly delivered child messages are automatically marked as read. This
+	// field is copied to new child messages. Changes are propagated to the webmail
+	// client.
+	ThreadMuted bool
+	// If set, this (sub)thread is collapsed in the webmail client, for threading mode
+	// "on" (mode "unread" ignores it). This field is copied to new child message.
+	// Changes are propagated to the webmail client.
+	ThreadCollapsed bool
+
 	Flags
 	// For keywords other than system flags or the basic well-known $-flags. Only in
 	// "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case
@@ -498,6 +528,10 @@ func (m Message) ChangeFlags(orig Flags) ChangeFlags {
 	return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}
 }
 
+func (m Message) ChangeThread() ChangeThread {
+	return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}
+}
+
 // ModSeq represents a modseq as stored in the database. ModSeq 0 in the
 // database is sent to the client as 1, because modseq 0 is special in IMAP.
 // ModSeq coming from the client are of type int64.
@@ -533,6 +567,22 @@ func (m *Message) PrepareExpunge() {
 	}
 }
 
+// PrepareThreading sets MessageID and SubjectBase (used in threading) based on the
+// envelope in part.
+func (m *Message) PrepareThreading(log *mlog.Log, part *message.Part) {
+	if part.Envelope == nil {
+		return
+	}
+	messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)
+	if err != nil {
+		log.Debugx("parsing message-id, ignoring", err, mlog.Field("messageid", part.Envelope.MessageID))
+	} else if raw {
+		log.Debug("could not parse message-id as address, continuing with raw value", mlog.Field("messageid", part.Envelope.MessageID))
+	}
+	m.MessageID = messageID
+	m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)
+}
+
 // LoadPart returns a message.Part by reading from m.ParsedBuf.
 func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {
 	if m.ParsedBuf == nil {
@@ -614,7 +664,7 @@ type Outgoing struct {
 }
 
 // Types stored in DB.
-var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}}
+var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}, Upgrade{}}
 
 // Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
 type Account struct {
@@ -623,6 +673,13 @@ type Account struct {
 	DBPath string     // Path to database with mailboxes, messages, etc.
 	DB     *bstore.DB // Open database connection.
 
+	// Channel that is closed if/when account has/gets "threads" accounting (see
+	// Upgrade.Threads).
+	threadsCompleted chan struct{}
+	// If threads upgrade completed with error, this is set. Used for warning during
+	// delivery, or aborting when importing.
+	threadsErr error
+
 	// Write lock must be held for account/mailbox modifications including message delivery.
 	// Read lock for reading mailboxes/messages.
 	// When making changes to mailboxes/messages, changes must be broadcasted before
@@ -632,6 +689,11 @@ type Account struct {
 	nused int // Reference count, while >0, this account is alive and shared.
 }
 
+type Upgrade struct {
+	ID      byte
+	Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.
+}
+
 // InitialUIDValidity returns a UIDValidity used for initializing an account.
 // It can be replaced during tests with a predictable value.
 var InitialUIDValidity = func() uint32 {
@@ -650,6 +712,7 @@ func closeAccount(acc *Account) (rerr error) {
 	acc.nused--
 	defer openAccounts.Unlock()
 	if acc.nused == 0 {
+		// threadsCompleted must be closed now because it increased nused.
 		rerr = acc.DB.Close()
 		acc.DB = nil
 		delete(openAccounts.names, acc.Name)
@@ -714,14 +777,26 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
 		}
 	}()
 
+	acc := &Account{
+		Name:             accountName,
+		Dir:              accountDir,
+		DBPath:           dbpath,
+		DB:               db,
+		nused:            1,
+		threadsCompleted: make(chan struct{}),
+	}
+
 	if isNew {
 		if err := initAccount(db); err != nil {
 			return nil, fmt.Errorf("initializing account: %v", err)
 		}
-	} else {
+		close(acc.threadsCompleted)
+		return acc, nil
+	}
+
 	// Ensure mailbox counts are set.
 	var mentioned bool
-		err := db.Write(context.TODO(), func(tx *bstore.Tx) error {
+	err = db.Write(context.TODO(), func(tx *bstore.Tx) error {
 		return bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {
 			if !mentioned {
 				mentioned = true
@@ -739,21 +814,90 @@ func OpenAccountDB(accountDir, accountName string) (a *Account, rerr error) {
 	if err != nil {
 		return nil, fmt.Errorf("calculating counts for mailbox: %v", err)
 	}
+
+	// Start adding threading if needed.
+	up := Upgrade{ID: 1}
+	err = db.Write(context.TODO(), func(tx *bstore.Tx) error {
+		err := tx.Get(&up)
+		if err == bstore.ErrAbsent {
+			if err := tx.Insert(&up); err != nil {
+				return fmt.Errorf("inserting initial upgrade record: %v", err)
+			}
+			err = nil
+		}
+		return err
+	})
+	if err != nil {
+		return nil, fmt.Errorf("checking message threading: %v", err)
+	}
+	if up.Threads == 2 {
+		close(acc.threadsCompleted)
+		return acc, nil
 	}
 
-	return &Account{
+	// Increase account use before holding on to account in background.
-		Name:   accountName,
+	// Caller holds the lock. The goroutine below decreases nused by calling
-		Dir:    accountDir,
+	// closeAccount.
-		DBPath: dbpath,
+	acc.nused++
-		DB:     db,
+
-		nused:  1,
+	// Ensure all messages have a MessageID and SubjectBase, which are needed when
-	}, nil
+	// matching threads.
+	// Then assign messages to threads, in the same way we do during imports.
+	xlog.Info("upgrading account for threading, in background", mlog.Field("account", acc.Name))
+	go func() {
+		defer func() {
+			err := closeAccount(acc)
+			xlog.Check(err, "closing use of account after upgrading account storage for threads", mlog.Field("account", a.Name))
+		}()
+
+		defer func() {
+			x := recover() // Should not happen, but don't take program down if it does.
+			if x != nil {
+				xlog.Error("upgradeThreads panic", mlog.Field("err", x))
+				debug.PrintStack()
+				metrics.PanicInc("upgradeThreads")
+				acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)
+			}
+
+			// Mark that upgrade has finished, possibly error is indicated in threadsErr.
+			close(acc.threadsCompleted)
+		}()
+
+		err := upgradeThreads(mox.Shutdown, acc, &up)
+		if err != nil {
+			a.threadsErr = err
+			xlog.Errorx("upgrading account for threading, aborted", err, mlog.Field("account", a.Name))
+		} else {
+			xlog.Info("upgrading account for threading, completed", mlog.Field("account", a.Name))
+		}
+	}()
+	return acc, nil
+}
+
+// ThreadingWait blocks until the one-time account threading upgrade for the
+// account has completed, and returns an error if not successful.
+//
+// To be used before starting an import of messages.
+func (a *Account) ThreadingWait(log *mlog.Log) error {
+	select {
+	case <-a.threadsCompleted:
+		return a.threadsErr
+	default:
+	}
+	log.Debug("waiting for account upgrade to complete")
+
+	<-a.threadsCompleted
+	return a.threadsErr
 }
 
 func initAccount(db *bstore.DB) error {
 	return db.Write(context.TODO(), func(tx *bstore.Tx) error {
 		uidvalidity := InitialUIDValidity()
 
+		if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {
+			return err
+		}
+
 		if len(mox.Conf.Static.DefaultMailboxes) > 0 {
 			// Deprecated in favor of InitialMailboxes.
 			defaultMailboxes := mox.Conf.Static.DefaultMailboxes
@@ -862,10 +1006,14 @@ func (a *Account) Close() error {
 // - Message with UID >= mailbox uid next.
 // - Mailbox uidvalidity >= account uid validity.
 // - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.
+// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.
 func (a *Account) CheckConsistency() error {
-	var uiderrors []string    // With a limit, could be many.
+	var uidErrors []string            // With a limit, could be many.
-	var modseqerrors []string // With limit.
+	var modseqErrors []string         // With limit.
-	var fileerrors []string   // With limit.
+	var fileErrors []string           // With limit.
+	var threadidErrors []string       // With limit.
+	var threadParentErrors []string   // With limit.
+	var threadAncestorErrors []string // With limit.
 	var errors []string
 
 	err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {
@@ -897,13 +1045,13 @@ func (a *Account) CheckConsistency() error {
 
 			mb := mailboxes[m.MailboxID]
 
-			if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqerrors) < 20 {
+			if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {
 				modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)
-				modseqerrors = append(modseqerrors, modseqerr)
+				modseqErrors = append(modseqErrors, modseqerr)
 			}
-			if m.UID >= mb.UIDNext && len(uiderrors) < 20 {
+			if m.UID >= mb.UIDNext && len(uidErrors) < 20 {
 				uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)
-				uiderrors = append(uiderrors, uiderr)
+				uidErrors = append(uidErrors, uiderr)
 			}
 			if m.Expunged {
 				return nil
@@ -912,10 +1060,32 @@ func (a *Account) CheckConsistency() error {
 			st, err := os.Stat(p)
 			if err != nil {
 				existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)
-				fileerrors = append(fileerrors, existserr)
+				fileErrors = append(fileErrors, existserr)
-			} else if len(fileerrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {
+			} else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {
 				sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())
-				fileerrors = append(fileerrors, sizeerr)
+				fileErrors = append(fileErrors, sizeerr)
+			}
+
+			if m.ThreadID <= 0 && len(threadidErrors) < 20 {
+				err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)
+				threadidErrors = append(threadidErrors, err)
+			}
+			if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {
+				err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)
+				threadParentErrors = append(threadParentErrors, err)
+			}
+			for i, pid := range m.ThreadParentIDs {
+				am := Message{ID: pid}
+				if err := tx.Get(&am); err == bstore.ErrAbsent {
+					continue
+				} else if err != nil {
+					return fmt.Errorf("get ancestor message: %v", err)
+				} else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {
+					err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)
+					threadAncestorErrors = append(threadAncestorErrors, err)
+				} else {
+					break
+				}
 			}
 			return nil
 		})
@@ -938,9 +1108,12 @@ func (a *Account) CheckConsistency() error {
 	if err != nil {
 		return err
 	}
-	errors = append(errors, uiderrors...)
+	errors = append(errors, uidErrors...)
-	errors = append(errors, modseqerrors...)
+	errors = append(errors, modseqErrors...)
-	errors = append(errors, fileerrors...)
+	errors = append(errors, fileErrors...)
+	errors = append(errors, threadidErrors...)
+	errors = append(errors, threadParentErrors...)
+	errors = append(errors, threadAncestorErrors...)
 	if len(errors) > 0 {
 		return fmt.Errorf("%s", strings.Join(errors, "; "))
 	}
@@ -1031,7 +1204,7 @@ func (a *Account) WithRLock(fn func()) {
 // Caller must broadcast new message.
 //
 // Caller must update mailbox counts.
-func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, consumeFile, sync, notrain bool) error {
+func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, consumeFile, sync, notrain, nothreads bool) error {
 	if m.Expunged {
 		return fmt.Errorf("cannot deliver expunged message")
 	}
@@ -1049,9 +1222,9 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
 	conf, _ := a.Conf()
 	m.JunkFlagsForMailbox(mb.Name, conf)
 
+	mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.
 	var part *message.Part
 	if m.ParsedBuf == nil {
-		mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.
 		p, err := message.EnsurePart(log, false, mr, m.Size)
 		if err != nil {
 			log.Infox("parsing delivered message", err, mlog.Field("parse", ""), mlog.Field("message", m.ID))
@@ -1063,6 +1236,13 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
 			return fmt.Errorf("marshal parsed message: %w", err)
 		}
 		m.ParsedBuf = buf
+	} else {
+		var p message.Part
+		if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {
+			log.Errorx("unmarshal parsed message, continuing", err, mlog.Field("parse", ""))
+		} else {
+			part = &p
+		}
 	}
 
 	// If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.
@@ -1078,22 +1258,44 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
 		m.ModSeq = modseq
 	}
 
+	if part != nil && m.MessageID == "" && m.SubjectBase == "" {
+		m.PrepareThreading(log, part)
+	}
+
+	// Assign to thread (if upgrade has completed).
+	noThreadID := nothreads
+	if m.ThreadID == 0 && !nothreads && part != nil {
+		select {
+		case <-a.threadsCompleted:
+			if a.threadsErr != nil {
+				log.Info("not assigning threads for new delivery, upgrading to threads failed")
+				noThreadID = true
+			} else {
+				if err := assignThread(log, tx, m, part); err != nil {
+					return fmt.Errorf("assigning thread: %w", err)
+				}
+			}
+		default:
+			// note: since we have a write transaction to get here, we can't wait for the
+			// thread upgrade to finish.
+			// If we don't assign a threadid the upgrade process will do it.
+			log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")
+			noThreadID = true
+		}
+	}
+
 	if err := tx.Insert(m); err != nil {
 		return fmt.Errorf("inserting message: %w", err)
 	}
+	if !noThreadID && m.ThreadID == 0 {
+		m.ThreadID = m.ID
+		if err := tx.Update(m); err != nil {
+			return fmt.Errorf("updating message for its own thread id: %w", err)
+		}
+	}
 
 	// todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's. for webmail, we could insert the recipients directly.
-	if mb.Sent {
+	if mb.Sent && part != nil && part.Envelope != nil {
-		// Attempt to parse the message for its To/Cc/Bcc headers, which we insert into Recipient.
-		if part == nil {
-			var p message.Part
-			if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {
-				log.Errorx("unmarshal parsed message for its to,cc,bcc headers, continuing", err, mlog.Field("parse", ""))
-			} else {
-				part = &p
-			}
-		}
-		if part != nil && part.Envelope != nil {
 		e := part.Envelope
 		sent := e.Date
 		if sent.IsZero() {
@@ -1126,7 +1328,6 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
 			}
 		}
 	}
-	}
 
 	msgPath := a.MessagePath(m.ID)
 	msgDir := filepath.Dir(msgPath)
@@ -1440,7 +1641,7 @@ ruleset:
 
 // MessagePath returns the file system path of a message.
 func (a *Account) MessagePath(messageID int64) string {
-	return filepath.Join(a.Dir, "msg", MessagePath(messageID))
+	return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), "/")
 }
 
 // MessageReader opens a message for reading, transparently combining the
@@ -1489,7 +1690,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF
 			return fmt.Errorf("updating mailbox for delivery: %w", err)
 		}
 
-		if err := a.DeliverMessage(log, tx, m, msgFile, consumeFile, true, false); err != nil {
+		if err := a.DeliverMessage(log, tx, m, msgFile, consumeFile, true, false, false); err != nil {
 			return err
 		}
 
@@ -1773,6 +1974,12 @@ const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVW
 // MessagePath returns the filename of the on-disk filename, relative to the containing directory such as <account>/msg or queue.
 // Returns names like "AB/1".
 func MessagePath(messageID int64) string {
+	return strings.Join(messagePathElems(messageID), "/")
+}
+
+// messagePathElems returns the elems, for a single join without intermediate
+// string allocations.
+func messagePathElems(messageID int64) []string {
 	v := messageID >> 13 // 8k files per directory.
 	dir := ""
 	for {
@@ -1782,7 +1989,7 @@ func MessagePath(messageID int64) string {
 			break
 		}
 	}
-	return fmt.Sprintf("%s/%d", dir, messageID)
+	return []string{dir, strconv.FormatInt(messageID, 10)}
 }
 
 // Set returns a copy of f, with each flag that is true in mask set to the

store/account_test.go

@@ -58,6 +58,8 @@ func TestMailbox(t *testing.T) {
 		MsgPrefix: msgPrefix,
 	}
 	msent := m
+	m.ThreadMuted = true
+	m.ThreadCollapsed = true
 	var mbsent Mailbox
 	mbrejects := Mailbox{Name: "Rejects", UIDValidity: 1, UIDNext: 1, HaveCounts: true}
 	mreject := m
@@ -77,8 +79,11 @@ func TestMailbox(t *testing.T) {
 			tcheck(t, err, "sent mailbox")
 			msent.MailboxID = mbsent.ID
 			msent.MailboxOrigID = mbsent.ID
-			err = acc.DeliverMessage(xlog, tx, &msent, msgFile, false, true, false)
+			err = acc.DeliverMessage(xlog, tx, &msent, msgFile, false, true, false, false)
 			tcheck(t, err, "deliver message")
+			if !msent.ThreadMuted || !msent.ThreadCollapsed {
+				t.Fatalf("thread muted & collapsed should have been copied from parent (duplicate message-id) m")
+			}
 
 			err = tx.Get(&mbsent)
 			tcheck(t, err, "get mbsent")
@@ -90,7 +95,7 @@ func TestMailbox(t *testing.T) {
 			tcheck(t, err, "insert rejects mailbox")
 			mreject.MailboxID = mbrejects.ID
 			mreject.MailboxOrigID = mbrejects.ID
-			err = acc.DeliverMessage(xlog, tx, &mreject, msgFile, false, true, false)
+			err = acc.DeliverMessage(xlog, tx, &mreject, msgFile, false, true, false, false)
 			tcheck(t, err, "deliver message")
 
 			err = tx.Get(&mbrejects)

store/state.go

@@ -50,6 +50,13 @@ type ChangeFlags struct {
 	Keywords  []string // Non-system/well-known flags/keywords/labels.
 }
 
+// ChangeThread is sent when muted/collapsed changes.
+type ChangeThread struct {
+	MessageIDs []int64
+	Muted      bool
+	Collapsed  bool
+}
+
 // ChangeRemoveMailbox is sent for a removed mailbox.
 type ChangeRemoveMailbox struct {
 	MailboxID int64

store/threads.go

@@ -0,0 +1,775 @@
+package store
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"runtime"
+	"sort"
+	"time"
+
+	"golang.org/x/exp/slices"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/message"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/moxio"
+)
+
+// Assign a new/incoming message to a thread. Message does not yet have an ID. If
+// this isn't a response, ThreadID should remain 0 (unless this is a message with
+// existing message-id) and the caller must set ThreadID to ID.
+// If the account is still busy upgrading messages with threadids in the background, parents
+// may have a threadid 0. That results in this message getting threadid 0, which
+// will handled by the background upgrade process assigning a threadid when it gets
+// to this message.
+func assignThread(log *mlog.Log, tx *bstore.Tx, m *Message, part *message.Part) error {
+	if m.MessageID != "" {
+		// Match against existing different message with same Message-ID.
+		q := bstore.QueryTx[Message](tx)
+		q.FilterNonzero(Message{MessageID: m.MessageID})
+		q.FilterEqual("Expunged", false)
+		q.FilterNotEqual("ID", m.ID)
+		q.FilterNotEqual("ThreadID", int64(0))
+		q.SortAsc("ID")
+		q.Limit(1)
+		em, err := q.Get()
+		if err != nil && err != bstore.ErrAbsent {
+			return fmt.Errorf("looking up existing message with message-id: %v", err)
+		} else if err == nil {
+			assignParent(m, em, true)
+			return nil
+		}
+	}
+
+	h, err := part.Header()
+	if err != nil {
+		log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID))
+	}
+	messageIDs, err := message.ReferencedIDs(h.Values("References"), h.Values("In-Reply-To"))
+	if err != nil {
+		log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID))
+	}
+	for i := len(messageIDs) - 1; i >= 0; i-- {
+		messageID := messageIDs[i]
+		if messageID == m.MessageID {
+			continue
+		}
+		tm, _, err := lookupThreadMessage(tx, m.ID, messageID, m.SubjectBase)
+		if err != nil {
+			return fmt.Errorf("looking up thread message for new message: %v", err)
+		} else if tm != nil {
+			assignParent(m, *tm, true)
+			return nil
+		}
+		m.ThreadMissingLink = true
+	}
+	if len(messageIDs) > 0 {
+		return nil
+	}
+
+	var isResp bool
+	if part != nil && part.Envelope != nil {
+		m.SubjectBase, isResp = message.ThreadSubject(part.Envelope.Subject, false)
+	}
+	if !isResp || m.SubjectBase == "" {
+		return nil
+	}
+	m.ThreadMissingLink = true
+	tm, err := lookupThreadMessageSubject(tx, *m, m.SubjectBase)
+	if err != nil {
+		return fmt.Errorf("looking up thread message by subject: %v", err)
+	} else if tm != nil {
+		assignParent(m, *tm, true)
+	}
+	return nil
+}
+
+// assignParent assigns threading fields to m that make it a child of parent message pm.
+// updateSeen indicates if m.Seen should be cleared if pm is thread-muted.
+func assignParent(m *Message, pm Message, updateSeen bool) {
+	if pm.ThreadID == 0 {
+		panic(fmt.Sprintf("assigning message id %d/d%q to parent message id %d/%q which has threadid 0", m.ID, m.MessageID, pm.ID, pm.MessageID))
+	}
+	if m.ID == pm.ID {
+		panic(fmt.Sprintf("trying to make message id %d/%q its own parent", m.ID, m.MessageID))
+	}
+	m.ThreadID = pm.ThreadID
+	// Make sure we don't add cycles.
+	if !slices.Contains(pm.ThreadParentIDs, m.ID) {
+		m.ThreadParentIDs = append([]int64{pm.ID}, pm.ThreadParentIDs...)
+	} else if pm.ID != m.ID {
+		m.ThreadParentIDs = []int64{pm.ID}
+	} else {
+		m.ThreadParentIDs = nil
+	}
+	if m.MessageID != "" && m.MessageID == pm.MessageID {
+		m.ThreadMissingLink = true
+	}
+	m.ThreadMuted = pm.ThreadMuted
+	m.ThreadCollapsed = pm.ThreadCollapsed
+	if updateSeen && m.ThreadMuted {
+		m.Seen = true
+	}
+}
+
+// ResetThreading resets the MessageID and SubjectBase fields for all messages in
+// the account. If clearIDs is true, all Thread* fields are also cleared. Changes
+// are made in transactions of batchSize changes. The total number of updated
+// messages is returned.
+//
+// ModSeq is not changed. Calles should bump the uid validity of the mailboxes
+// to propagate the changes to IMAP clients.
+func (a *Account) ResetThreading(ctx context.Context, log *mlog.Log, batchSize int, clearIDs bool) (int, error) {
+	// todo: should this send Change events for ThreadMuted and ThreadCollapsed? worth it?
+
+	var lastID int64
+	total := 0
+	for {
+		n := 0
+
+		prepareMessages := func(in, out chan moxio.Work[Message, Message]) {
+			for {
+				w, ok := <-in
+				if !ok {
+					return
+				}
+
+				m := w.In
+
+				// We have the Message-ID and Subject headers in ParsedBuf. We use a partial part
+				// struct so we don't generate so much garbage for the garbage collector to sift
+				// through.
+				var part struct {
+					Envelope *message.Envelope
+				}
+				if err := json.Unmarshal(m.ParsedBuf, &part); err != nil {
+					log.Errorx("unmarshal json parsedbuf for setting message-id, skipping", err, mlog.Field("msgid", m.ID))
+				} else {
+					m.MessageID = ""
+					if part.Envelope != nil && part.Envelope.MessageID != "" {
+						s, _, err := message.MessageIDCanonical(part.Envelope.MessageID)
+						if err != nil {
+							log.Debugx("parsing message-id, skipping", err, mlog.Field("msgid", m.ID), mlog.Field("messageid", part.Envelope.MessageID))
+						}
+						m.MessageID = s
+					}
+					if part.Envelope != nil {
+						m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)
+					}
+				}
+				w.Out = m
+
+				out <- w
+			}
+		}
+
+		err := a.DB.Write(ctx, func(tx *bstore.Tx) error {
+			processMessage := func(in, m Message) error {
+				if clearIDs {
+					m.ThreadID = 0
+					m.ThreadParentIDs = nil
+					m.ThreadMissingLink = false
+				}
+				return tx.Update(&m)
+			}
+
+			// JSON parsing is relatively heavy, we benefit from multiple goroutines.
+			procs := runtime.GOMAXPROCS(0)
+			wq := moxio.NewWorkQueue[Message, Message](procs, 2*procs, prepareMessages, processMessage)
+
+			q := bstore.QueryTx[Message](tx)
+			q.FilterEqual("Expunged", false)
+			q.FilterGreater("ID", lastID)
+			q.SortAsc("ID")
+			err := q.ForEach(func(m Message) error {
+				// We process in batches so we don't block other operations for a long time.
+				if n >= batchSize {
+					return bstore.StopForEach
+				}
+				// Update starting point for next batch.
+				lastID = m.ID
+
+				n++
+				return wq.Add(m)
+			})
+			if err == nil {
+				err = wq.Finish()
+			}
+			wq.Stop()
+			return err
+		})
+		if err != nil {
+			return total, fmt.Errorf("upgrading account to threads storage, step 1/2: %w", err)
+		}
+		total += n
+		if n == 0 {
+			break
+		}
+	}
+	return total, nil
+}
+
+// AssignThreads assigns thread-related fields to messages with ID >=
+// startMessageID. Changes are committed each batchSize changes if txOpt is nil
+// (i.e. during automatic account upgrade, we don't want to block database access
+// for a long time). If txOpt is not nil, all changes are made in that
+// transaction.
+//
+// When resetting thread assignments, the caller must first clear the existing
+// thread fields.
+//
+// Messages are processed in order of ID, so when added to the account, not
+// necessarily by received/date. Most threaded messages can immediately be matched
+// to their parent message. If not, we keep track of the missing message-id and
+// resolve as soon as we encounter it. At the end, we resolve all remaining
+// messages, they start with a cycle.
+//
+// Does not set Seen flag for muted threads.
+//
+// Progress is written to progressWriter, every 100k messages.
+func (a *Account) AssignThreads(ctx context.Context, log *mlog.Log, txOpt *bstore.Tx, startMessageID int64, batchSize int, progressWriter io.Writer) error {
+	// We use a more basic version of the thread-matching algorithm describe in:
+	// ../rfc/5256:443
+	// The algorithm assumes you'll select messages, then group into threads. We normally do
+	// thread-calculation when messages are delivered. Here, we assign threads as soon
+	// as we can, but will queue messages that reference known ancestors and resolve as
+	// soon as we process them. We can handle large number of messages, but not very
+	// quickly because we make lots of database queries.
+
+	type childMsg struct {
+		ID                int64  // This message will be fetched and updated with the threading fields once the parent is resolved.
+		MessageID         string // Of child message. Once child is resolved, its own children can be resolved too.
+		ThreadMissingLink bool
+	}
+	// Messages that have a References/In-Reply-To that we want to set as parent, but
+	// where the parent doesn't have a ThreadID yet are added to pending. The key is
+	// the normalized MessageID of the parent, and the value is a list of messages that
+	// can get resolved once the parent gets its ThreadID. The kids will get the same
+	// ThreadIDs, and they themselves may be parents to kids, and so on.
+	// For duplicate messages (messages with identical Message-ID), the second
+	// Message-ID to be added to pending is added under its own message-id, so it gets
+	// its original as parent.
+	pending := map[string][]childMsg{}
+
+	// Current tx. If not equal to txOpt, we clean it up before we leave.
+	var tx *bstore.Tx
+	defer func() {
+		if tx != nil && tx != txOpt {
+			err := tx.Rollback()
+			log.Check(err, "rolling back transaction")
+		}
+	}()
+
+	// Set thread-related fields for a single message. Caller must save the message,
+	// only if not an error and not added to the pending list.
+	assign := func(m *Message, references, inReplyTo []string, subject string) (pend bool, rerr error) {
+		if m.MessageID != "" {
+			// Attempt to match against existing different message with same Message-ID that
+			// already has a threadid.
+			// If there are multiple messages for a message-id a future call to assign may use
+			// its threadid, or it may end up in pending and we resolve it when we need to.
+			q := bstore.QueryTx[Message](tx)
+			q.FilterNonzero(Message{MessageID: m.MessageID})
+			q.FilterEqual("Expunged", false)
+			q.FilterLess("ID", m.ID)
+			q.SortAsc("ID")
+			q.Limit(1)
+			em, err := q.Get()
+			if err != nil && err != bstore.ErrAbsent {
+				return false, fmt.Errorf("looking up existing message with message-id: %v", err)
+			} else if err == nil {
+				if em.ThreadID == 0 {
+					pending[em.MessageID] = append(pending[em.MessageID], childMsg{m.ID, m.MessageID, true})
+					return true, nil
+				} else {
+					assignParent(m, em, false)
+					return false, nil
+				}
+			}
+		}
+
+		refids, err := message.ReferencedIDs(references, inReplyTo)
+		if err != nil {
+			log.Errorx("assigning threads: parsing references/in-reply-to headers, not matching by message-id", err, mlog.Field("msgid", m.ID))
+		}
+
+		for i := len(refids) - 1; i >= 0; i-- {
+			messageID := refids[i]
+			if messageID == m.MessageID {
+				continue
+			}
+			tm, exists, err := lookupThreadMessage(tx, m.ID, messageID, m.SubjectBase)
+			if err != nil {
+				return false, fmt.Errorf("lookup up thread by message-id %s for message id %d: %w", messageID, m.ID, err)
+			} else if tm != nil {
+				assignParent(m, *tm, false)
+				return false, nil
+			} else if exists {
+				pending[messageID] = append(pending[messageID], childMsg{m.ID, m.MessageID, i < len(refids)-1})
+				return true, nil
+			}
+		}
+
+		var subjectBase string
+		var isResp bool
+		if subject != "" {
+			subjectBase, isResp = message.ThreadSubject(subject, false)
+		}
+		if len(refids) > 0 || !isResp || subjectBase == "" {
+			m.ThreadID = m.ID
+			m.ThreadMissingLink = len(refids) > 0
+			return false, nil
+		}
+
+		// No references to use. If this is a reply/forward (based on subject), we'll match
+		// against base subject, at most 4 weeks back so we don't match against ancient
+		// messages and 1 day ahead so we can match against delayed deliveries.
+		tm, err := lookupThreadMessageSubject(tx, *m, subjectBase)
+		if err != nil {
+			return false, fmt.Errorf("looking up recent messages by base subject %q: %w", subjectBase, err)
+		} else if tm != nil {
+			m.ThreadID = tm.ThreadID
+			m.ThreadParentIDs = []int64{tm.ThreadID} // Always under root message with subject-match.
+			m.ThreadMissingLink = true
+			m.ThreadMuted = tm.ThreadMuted
+			m.ThreadCollapsed = tm.ThreadCollapsed
+		} else {
+			m.ThreadID = m.ID
+		}
+		return false, nil
+	}
+
+	npendingResolved := 0
+
+	// Resolve pending messages that wait on m.MessageID to be resolved, recursively.
+	var resolvePending func(tm Message, cyclic bool) error
+	resolvePending = func(tm Message, cyclic bool) error {
+		if tm.MessageID == "" {
+			return nil
+		}
+		l := pending[tm.MessageID]
+		delete(pending, tm.MessageID)
+		for _, mi := range l {
+			m := Message{ID: mi.ID}
+			if err := tx.Get(&m); err != nil {
+				return fmt.Errorf("get message %d for resolving pending thread for message-id %s, %d: %w", mi.ID, tm.MessageID, tm.ID, err)
+			}
+			if m.ThreadID != 0 {
+				// ThreadID already set because this is a cyclic message. If we would assign a
+				// parent again, we would create a cycle.
+				if m.MessageID != tm.MessageID && !cyclic {
+					panic(fmt.Sprintf("threadid already set (%d) while handling non-cyclic message id %d/%q and with different message-id %q as parent message id %d", m.ThreadID, m.ID, m.MessageID, tm.MessageID, tm.ID))
+				}
+				continue
+			}
+			assignParent(&m, tm, false)
+			m.ThreadMissingLink = mi.ThreadMissingLink
+			if err := tx.Update(&m); err != nil {
+				return fmt.Errorf("update message %d for resolving pending thread for message-id %s, %d: %w", mi.ID, tm.MessageID, tm.ID, err)
+			}
+			if err := resolvePending(m, cyclic); err != nil {
+				return err
+			}
+			npendingResolved++
+		}
+		return nil
+	}
+
+	// Output of the worker goroutines.
+	type threadPrep struct {
+		references []string
+		inReplyTo  []string
+		subject    string
+	}
+
+	// Single allocation.
+	threadingFields := [][]byte{
+		[]byte("references"),
+		[]byte("in-reply-to"),
+		[]byte("subject"),
+	}
+
+	// Worker goroutine function. We start with a reasonably large buffer for reading
+	// the header into. And we have scratch space to copy the needed headers into. That
+	// means we normally won't allocate any more buffers.
+	prepareMessages := func(in, out chan moxio.Work[Message, threadPrep]) {
+		headerbuf := make([]byte, 8*1024)
+		scratch := make([]byte, 4*1024)
+		for {
+			w, ok := <-in
+			if !ok {
+				return
+			}
+
+			m := w.In
+			var partialPart struct {
+				HeaderOffset int64
+				BodyOffset   int64
+			}
+			if err := json.Unmarshal(m.ParsedBuf, &partialPart); err != nil {
+				w.Err = fmt.Errorf("unmarshal part: %v", err)
+			} else {
+				size := partialPart.BodyOffset - partialPart.HeaderOffset
+				if int(size) > len(headerbuf) {
+					headerbuf = make([]byte, size)
+				}
+				if size > 0 {
+					buf := headerbuf[:int(size)]
+					err := func() error {
+						mr := a.MessageReader(m)
+						defer mr.Close()
+
+						// ReadAt returns whole buffer or error. Single read should be fast.
+						n, err := mr.ReadAt(buf, partialPart.HeaderOffset)
+						if err != nil || n != len(buf) {
+							return fmt.Errorf("read header: %v", err)
+						}
+						return nil
+					}()
+					if err != nil {
+						w.Err = err
+					} else if h, err := message.ParseHeaderFields(buf, scratch, threadingFields); err != nil {
+						w.Err = err
+					} else {
+						w.Out.references = h["References"]
+						w.Out.inReplyTo = h["In-Reply-To"]
+						l := h["Subject"]
+						if len(l) > 0 {
+							w.Out.subject = l[0]
+						}
+					}
+				}
+			}
+
+			out <- w
+		}
+	}
+
+	// Assign threads to messages, possibly in batches.
+	nassigned := 0
+	for {
+		n := 0
+		tx = txOpt
+		if tx == nil {
+			var err error
+			tx, err = a.DB.Begin(ctx, true)
+			if err != nil {
+				return fmt.Errorf("begin transaction: %w", err)
+			}
+		}
+
+		processMessage := func(m Message, prep threadPrep) error {
+			pend, err := assign(&m, prep.references, prep.inReplyTo, prep.subject)
+			if err != nil {
+				return fmt.Errorf("for msgid %d: %w", m.ID, err)
+			} else if pend {
+				return nil
+			}
+			if m.ThreadID == 0 {
+				panic(fmt.Sprintf("no threadid after assign of message id %d/%q", m.ID, m.MessageID))
+			}
+			// Fields have been set, store in database and resolve messages waiting for this MessageID.
+			if slices.Contains(m.ThreadParentIDs, m.ID) {
+				panic(fmt.Sprintf("message id %d/%q contains itself in parent ids %v", m.ID, m.MessageID, m.ThreadParentIDs))
+			}
+			if err := tx.Update(&m); err != nil {
+				return err
+			}
+			if err := resolvePending(m, false); err != nil {
+				return fmt.Errorf("resolving pending message-id: %v", err)
+			}
+			return nil
+		}
+
+		// Use multiple worker goroutines to read parse headers from on-disk messages.
+		procs := runtime.GOMAXPROCS(0)
+		wq := moxio.NewWorkQueue[Message, threadPrep](2*procs, 4*procs, prepareMessages, processMessage)
+
+		// We assign threads in order by ID, so messages delivered in between our
+		// transaction will get assigned threads too: they'll have the highest id's.
+		q := bstore.QueryTx[Message](tx)
+		q.FilterGreaterEqual("ID", startMessageID)
+		q.FilterEqual("Expunged", false)
+		q.SortAsc("ID")
+		err := q.ForEach(func(m Message) error {
+			// Batch number of changes, so we give other users of account a change to run.
+			if txOpt == nil && n >= batchSize {
+				return bstore.StopForEach
+			}
+			// Starting point for next batch.
+			startMessageID = m.ID + 1
+			// Don't process again. Can happen when earlier upgrade was aborted.
+			if m.ThreadID != 0 {
+				return nil
+			}
+
+			n++
+			return wq.Add(m)
+		})
+		if err == nil {
+			err = wq.Finish()
+		}
+		wq.Stop()
+
+		if err == nil && txOpt == nil {
+			err = tx.Commit()
+			tx = nil
+		}
+		if err != nil {
+			return fmt.Errorf("assigning threads: %w", err)
+		}
+		if n == 0 {
+			break
+		}
+		nassigned += n
+		if nassigned%100000 == 0 {
+			log.Debug("assigning threads, progress", mlog.Field("count", nassigned), mlog.Field("unresolved", len(pending)))
+			if _, err := fmt.Fprintf(progressWriter, "assigning threads, progress: %d messages\n", nassigned); err != nil {
+				return fmt.Errorf("writing progress: %v", err)
+			}
+		}
+	}
+	if _, err := fmt.Fprintf(progressWriter, "assigning threads, done: %d messages\n", nassigned); err != nil {
+		return fmt.Errorf("writing progress: %v", err)
+	}
+
+	log.Debug("assigning threads, mostly done, finishing with resolving of cyclic messages", mlog.Field("count", nassigned), mlog.Field("unresolved", len(pending)))
+
+	if _, err := fmt.Fprintf(progressWriter, "assigning threads, resolving %d cyclic pending message-ids\n", len(pending)); err != nil {
+		return fmt.Errorf("writing progress: %v", err)
+	}
+
+	// Remaining messages in pending have cycles and possibly tails. The cycle is at
+	// the head of the thread. Once we resolve that, the rest of the thread can be
+	// resolved too. Ignoring self-references (duplicate messages), there can only be
+	// one cycle, and it is at the head. So we look for cycles, ignoring
+	// self-references, and resolve a message as soon as we see the cycle.
+
+	parent := map[string]string{} // Child Message-ID pointing to the parent Message-ID, excluding self-references.
+	pendlist := []string{}
+	for pmsgid, l := range pending {
+		pendlist = append(pendlist, pmsgid)
+		for _, k := range l {
+			if k.MessageID == pmsgid {
+				// No self-references for duplicate messages.
+				continue
+			}
+			if _, ok := parent[k.MessageID]; !ok {
+				parent[k.MessageID] = pmsgid
+			}
+			// else, this message should be resolved by following pending.
+		}
+	}
+	sort.Strings(pendlist)
+
+	tx = txOpt
+	if tx == nil {
+		var err error
+		tx, err = a.DB.Begin(ctx, true)
+		if err != nil {
+			return fmt.Errorf("begin transaction: %w", err)
+		}
+	}
+
+	// We walk through all messages of pendlist, but some will already have been
+	// resolved by the time we get to them.
+	done := map[string]bool{}
+	for _, msgid := range pendlist {
+		if done[msgid] {
+			continue
+		}
+
+		// We walk up to parent, until we see a message-id we've already seen, a cycle.
+		seen := map[string]bool{}
+		for {
+			pmsgid, ok := parent[msgid]
+			if !ok {
+				panic(fmt.Sprintf("missing parent message-id %q, not a cycle?", msgid))
+			}
+			if !seen[pmsgid] {
+				seen[pmsgid] = true
+				msgid = pmsgid
+				continue
+			}
+
+			// Cycle detected. Make this message-id the thread root.
+			q := bstore.QueryTx[Message](tx)
+			q.FilterNonzero(Message{MessageID: msgid})
+			q.FilterEqual("ThreadID", int64(0))
+			q.FilterEqual("Expunged", false)
+			q.SortAsc("ID")
+			l, err := q.List()
+			if err == nil && len(l) == 0 {
+				err = errors.New("no messages")
+			}
+			if err != nil {
+				return fmt.Errorf("list message by message-id for cyclic thread root: %v", err)
+			}
+			for i, m := range l {
+				m.ThreadID = l[0].ID
+				m.ThreadMissingLink = true
+				if i == 0 {
+					m.ThreadParentIDs = nil
+					l[0] = m // For resolvePending below.
+				} else {
+					assignParent(&m, l[0], false)
+				}
+				if slices.Contains(m.ThreadParentIDs, m.ID) {
+					panic(fmt.Sprintf("message id %d/%q contains itself in parents %v", m.ID, m.MessageID, m.ThreadParentIDs))
+				}
+				if err := tx.Update(&m); err != nil {
+					return fmt.Errorf("assigning threadid to cyclic thread root: %v", err)
+				}
+			}
+
+			// Mark all children as done so we don't process these messages again.
+			walk := map[string]struct{}{msgid: {}}
+			for len(walk) > 0 {
+				for msgid := range walk {
+					delete(walk, msgid)
+					if done[msgid] {
+						continue
+					}
+					done[msgid] = true
+					for _, mi := range pending[msgid] {
+						if !done[mi.MessageID] {
+							walk[mi.MessageID] = struct{}{}
+						}
+					}
+				}
+			}
+
+			// Resolve all messages in this thread.
+			if err := resolvePending(l[0], true); err != nil {
+				return fmt.Errorf("resolving cyclic children of cyclic thread root: %v", err)
+			}
+
+			break
+		}
+	}
+
+	// Check that there are no more messages without threadid.
+	q := bstore.QueryTx[Message](tx)
+	q.FilterEqual("ThreadID", int64(0))
+	q.FilterEqual("Expunged", false)
+	l, err := q.List()
+	if err == nil && len(l) > 0 {
+		err = errors.New("found messages without threadid")
+	}
+	if err != nil {
+		return fmt.Errorf("listing messages without threadid: %v", err)
+	}
+
+	if txOpt == nil {
+		err := tx.Commit()
+		tx = nil
+		if err != nil {
+			return fmt.Errorf("commit resolving cyclic thread roots: %v", err)
+		}
+	}
+	return nil
+}
+
+// lookupThreadMessage tries to find the parent message with messageID that must
+// have a matching subjectBase.
+//
+// If the message isn't present (with a valid thread id), a nil message and nil
+// error is returned. The bool return value indicates if a message with the
+// message-id exists at all.
+func lookupThreadMessage(tx *bstore.Tx, mID int64, messageID, subjectBase string) (*Message, bool, error) {
+	q := bstore.QueryTx[Message](tx)
+	q.FilterNonzero(Message{MessageID: messageID})
+	q.FilterEqual("SubjectBase", subjectBase)
+	q.FilterEqual("Expunged", false)
+	q.FilterNotEqual("ID", mID)
+	q.SortAsc("ID")
+	l, err := q.List()
+	if err != nil {
+		return nil, false, fmt.Errorf("message-id %s: %w", messageID, err)
+	}
+	exists := len(l) > 0
+	for _, tm := range l {
+		if tm.ThreadID != 0 {
+			return &tm, true, nil
+		}
+	}
+	return nil, exists, nil
+}
+
+// lookupThreadMessageSubject looks up a parent/ancestor message for the message
+// thread based on a matching subject. The message must have been delivered to the same mailbox originally.
+//
+// If no message (with a threadid) is found a nil message and nil error is returned.
+func lookupThreadMessageSubject(tx *bstore.Tx, m Message, subjectBase string) (*Message, error) {
+	q := bstore.QueryTx[Message](tx)
+	q.FilterGreater("Received", m.Received.Add(-4*7*24*time.Hour))
+	q.FilterLess("Received", m.Received.Add(1*24*time.Hour))
+	q.FilterNonzero(Message{SubjectBase: subjectBase, MailboxOrigID: m.MailboxOrigID})
+	q.FilterEqual("Expunged", false)
+	q.FilterNotEqual("ID", m.ID)
+	q.FilterNotEqual("ThreadID", int64(0))
+	q.SortDesc("Received")
+	q.Limit(1)
+	tm, err := q.Get()
+	if err == bstore.ErrAbsent {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	return &tm, nil
+}
+
+func upgradeThreads(ctx context.Context, acc *Account, up *Upgrade) error {
+	log := xlog.Fields(mlog.Field("account", acc.Name))
+
+	if up.Threads == 0 {
+		// Step 1 in the threads upgrade is storing the canonicalized Message-ID for each
+		// message and the base subject for thread matching. This allows efficient thread
+		// lookup in the second step.
+
+		log.Info("upgrading account for threading, step 1/2: updating all messages with message-id and base subject")
+		t0 := time.Now()
+
+		const batchSize = 10000
+		total, err := acc.ResetThreading(ctx, xlog, batchSize, true)
+		if err != nil {
+			return fmt.Errorf("resetting message threading fields: %v", err)
+		}
+
+		up.Threads = 1
+		if err := acc.DB.Update(ctx, up); err != nil {
+			up.Threads = 0
+			return fmt.Errorf("saving upgrade process while upgrading account to threads storage, step 1/2: %w", err)
+		}
+		log.Info("upgrading account for threading, step 1/2: completed", mlog.Field("duration", time.Since(t0)), mlog.Field("messages", total))
+	}
+
+	if up.Threads == 1 {
+		// Step 2 of the upgrade is going through all messages and assigning threadid's.
+		// Lookup of messageid and base subject is now fast through indexed database
+		// access.
+
+		log.Info("upgrading account for threading, step 2/2: matching messages to threads")
+		t0 := time.Now()
+
+		const batchSize = 10000
+		if err := acc.AssignThreads(ctx, xlog, nil, 1, batchSize, io.Discard); err != nil {
+			return fmt.Errorf("upgrading to threads storage, step 2/2: %w", err)
+		}
+		up.Threads = 2
+		if err := acc.DB.Update(ctx, up); err != nil {
+			up.Threads = 1
+			return fmt.Errorf("saving upgrade process for thread storage, step 2/2: %w", err)
+		}
+		log.Info("upgrading account for threading, step 2/2: completed", mlog.Field("duration", time.Since(t0)))
+	}
+
+	// Note: Not bumping uidvalidity or setting modseq. Clients haven't been able to
+	// use threadid's before, so there is nothing to be out of date.
+
+	return nil
+}

store/threads_test.go

@@ -0,0 +1,208 @@
+package store
+
+import (
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+)
+
+func TestThreadingUpgrade(t *testing.T) {
+	os.RemoveAll("../testdata/store/data")
+	mox.ConfigStaticPath = "../testdata/store/mox.conf"
+	mox.MustLoadConfig(true, false)
+	acc, err := OpenAccount("mjl")
+	tcheck(t, err, "open account")
+	defer func() {
+		err = acc.Close()
+		tcheck(t, err, "closing account")
+	}()
+	defer Switchboard()()
+
+	log := mlog.New("store")
+
+	// New account already has threading. Add some messages, check the threading.
+	deliver := func(recv time.Time, s string, expThreadID int64) Message {
+		t.Helper()
+		f, err := CreateMessageTemp("account-test")
+		tcheck(t, err, "temp file")
+		defer f.Close()
+
+		s = strings.ReplaceAll(s, "\n", "\r\n")
+		m := Message{
+			Size:      int64(len(s)),
+			MsgPrefix: []byte(s),
+			Received:  recv,
+		}
+		err = acc.DeliverMailbox(log, "Inbox", &m, f, true)
+		tcheck(t, err, "deliver")
+		if expThreadID == 0 {
+			expThreadID = m.ID
+		}
+		if m.ThreadID != expThreadID {
+			t.Fatalf("got threadid %d, expected %d", m.ThreadID, expThreadID)
+		}
+		return m
+	}
+
+	now := time.Now()
+
+	m0 := deliver(now, "Message-ID: <m0@localhost>\nSubject: test1\n\ntest\n", 0)
+	m1 := deliver(now, "Message-ID: <m1@localhost>\nReferences: <m0@localhost>\nSubject: test1\n\ntest\n", m0.ID)  // References.
+	m2 := deliver(now, "Message-ID: <m2@localhost>\nReferences: <m0@localhost>\nSubject: other\n\ntest\n", 0)      // References, but different subject.
+	m3 := deliver(now, "Message-ID: <m3@localhost>\nIn-Reply-To: <m0@localhost>\nSubject: test1\n\ntest\n", m0.ID) // In-Reply-To.
+	m4 := deliver(now, "Message-ID: <m4@localhost>\nSubject: re: test1\n\ntest\n", m0.ID)                          // Subject.
+	m5 := deliver(now, "Message-ID: <m5@localhost>\nSubject: test1 (fwd)\n\ntest\n", m0.ID)                        // Subject.
+	m6 := deliver(now, "Message-ID: <m6@localhost>\nSubject: [fwd: test1]\n\ntest\n", m0.ID)                       // Subject.
+	m7 := deliver(now, "Message-ID: <m7@localhost>\nSubject: test1\n\ntest\n", 0)                                  // Only subject, but not a response.
+
+	// Thread with a cyclic head, a self-referencing message.
+	c1 := deliver(now, "Message-ID: <c1@localhost>\nReferences: <c2@localhost>\nSubject: cycle0\n\ntest\n", 0)     // Head cycle with m8.
+	c2 := deliver(now, "Message-ID: <c2@localhost>\nReferences: <c1@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Head cycle with c1.
+	c3 := deliver(now, "Message-ID: <c3@localhost>\nReferences: <c1@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Connected to one of the cycle elements.
+	c4 := deliver(now, "Message-ID: <c4@localhost>\nReferences: <c2@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Connected to other cycle element.
+	c5 := deliver(now, "Message-ID: <c5@localhost>\nReferences: <c4@localhost>\nSubject: cycle0\n\ntest\n", c1.ID)
+	c5b := deliver(now, "Message-ID: <c5@localhost>\nReferences: <c4@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Duplicate, e.g. Sent item, internal cycle during upgrade.
+	c6 := deliver(now, "Message-ID: <c6@localhost>\nReferences: <c5@localhost>\nSubject: cycle0\n\ntest\n", c1.ID)
+	c7 := deliver(now, "Message-ID: <c7@localhost>\nReferences: <c5@localhost> <c7@localhost>\nSubject: cycle0\n\ntest\n", c1.ID) // Self-referencing message that also points to actual parent.
+
+	// More than 2 messages to make a cycle.
+	d0 := deliver(now, "Message-ID: <d0@localhost>\nReferences: <d2@localhost>\nSubject: cycle1\n\ntest\n", 0)
+	d1 := deliver(now, "Message-ID: <d1@localhost>\nReferences: <d0@localhost>\nSubject: cycle1\n\ntest\n", d0.ID)
+	d2 := deliver(now, "Message-ID: <d2@localhost>\nReferences: <d1@localhost>\nSubject: cycle1\n\ntest\n", d0.ID)
+
+	// Cycle with messages delivered later. During import/upgrade, they will all be one thread.
+	e0 := deliver(now, "Message-ID: <e0@localhost>\nReferences: <e1@localhost>\nSubject: cycle2\n\ntest\n", 0)
+	e1 := deliver(now, "Message-ID: <e1@localhost>\nReferences: <e2@localhost>\nSubject: cycle2\n\ntest\n", 0)
+	e2 := deliver(now, "Message-ID: <e2@localhost>\nReferences: <e0@localhost>\nSubject: cycle2\n\ntest\n", e0.ID)
+
+	// Three messages in a cycle (f1, f2, f3), with one with an additional ancestor (f4) which is ignored due to the cycle. Has different threads during import.
+	f0 := deliver(now, "Message-ID: <f0@localhost>\nSubject: cycle3\n\ntest\n", 0)
+	f1 := deliver(now, "Message-ID: <f1@localhost>\nReferences: <f0@localhost> <f2@localhost>\nSubject: cycle3\n\ntest\n", f0.ID)
+	f2 := deliver(now, "Message-ID: <f2@localhost>\nReferences: <f3@localhost>\nSubject: cycle3\n\ntest\n", 0)
+	f3 := deliver(now, "Message-ID: <f3@localhost>\nReferences: <f1@localhost>\nSubject: cycle3\n\ntest\n", f0.ID)
+
+	// Duplicate single message (no larger thread).
+	g0 := deliver(now, "Message-ID: <g0@localhost>\nSubject: dup\n\ntest\n", 0)
+	g0b := deliver(now, "Message-ID: <g0@localhost>\nSubject: dup\n\ntest\n", g0.ID)
+
+	// Duplicate message with a child message.
+	h0 := deliver(now, "Message-ID: <h0@localhost>\nSubject: dup2\n\ntest\n", 0)
+	h0b := deliver(now, "Message-ID: <h0@localhost>\nSubject: dup2\n\ntest\n", h0.ID)
+	h1 := deliver(now, "Message-ID: <h1@localhost>\nReferences: <h0@localhost>\nSubject: dup2\n\ntest\n", h0.ID)
+
+	// Message has itself as reference.
+	s0 := deliver(now, "Message-ID: <s0@localhost>\nReferences: <s0@localhost>\nSubject: self-referencing message\n\ntest\n", 0)
+
+	// Message with \0 in subject, should get an empty base subject.
+	b0 := deliver(now, "Message-ID: <b0@localhost>\nSubject: bad\u0000subject\n\ntest\n", 0)
+	b1 := deliver(now, "Message-ID: <b1@localhost>\nSubject: bad\u0000subject\n\ntest\n", 0) // Not matched.
+
+	// Interleaved duplicate threaded messages. First child, then parent, then duplicate parent, then duplicat child again.
+	i0 := deliver(now, "Message-ID: <i0@localhost>\nReferences: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", 0)
+	i1 := deliver(now, "Message-ID: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", 0)
+	i2 := deliver(now, "Message-ID: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", i1.ID)
+	i3 := deliver(now, "Message-ID: <i0@localhost>\nReferences: <i1@localhost>\nSubject: interleaved duplicate\n\ntest\n", i0.ID)
+
+	j0 := deliver(now, "Message-ID: <j0@localhost>\nReferences: <>\nSubject: empty id in references\n\ntest\n", 0)
+
+	dbpath := acc.DBPath
+	err = acc.Close()
+	tcheck(t, err, "close account")
+
+	// Now clear the threading upgrade, and the threading fields and close the account.
+	// We open the database file directly, so we don't trigger the consistency checker.
+	db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
+	err = db.Write(ctxbg, func(tx *bstore.Tx) error {
+		up := Upgrade{ID: 1}
+		err := tx.Delete(&up)
+		tcheck(t, err, "delete upgrade")
+
+		q := bstore.QueryTx[Message](tx)
+		_, err = q.UpdateFields(map[string]any{
+			"MessageID":         "",
+			"SubjectBase":       "",
+			"ThreadID":          int64(0),
+			"ThreadParentIDs":   []int64(nil),
+			"ThreadMissingLink": false,
+		})
+		return err
+	})
+	tcheck(t, err, "reset threading fields")
+	err = db.Close()
+	tcheck(t, err, "closing db")
+
+	// Open the account again, that should get the account upgraded. Wait for upgrade to finish.
+	acc, err = OpenAccount("mjl")
+	tcheck(t, err, "open account")
+	err = acc.ThreadingWait(log)
+	tcheck(t, err, "wait for threading")
+
+	check := func(id int64, expThreadID int64, expParentIDs []int64, expMissingLink bool) {
+		t.Helper()
+
+		m := Message{ID: id}
+		err := acc.DB.Get(ctxbg, &m)
+		tcheck(t, err, "get message")
+		if m.ThreadID != expThreadID || !reflect.DeepEqual(m.ThreadParentIDs, expParentIDs) || m.ThreadMissingLink != expMissingLink {
+			t.Fatalf("got thread id %d, parent ids %v, missing link %v, expected %d %v %v", m.ThreadID, m.ThreadParentIDs, m.ThreadMissingLink, expThreadID, expParentIDs, expMissingLink)
+		}
+	}
+
+	parents0 := []int64{m0.ID}
+	check(m0.ID, m0.ID, nil, false)
+	check(m1.ID, m0.ID, parents0, false)
+	check(m2.ID, m2.ID, nil, true)
+	check(m3.ID, m0.ID, parents0, false)
+	check(m4.ID, m0.ID, parents0, true)
+	check(m5.ID, m0.ID, parents0, true)
+	check(m6.ID, m0.ID, parents0, true)
+	check(m7.ID, m7.ID, nil, false)
+
+	check(c1.ID, c1.ID, nil, true) // Head of cycle, hence missing link
+	check(c2.ID, c1.ID, []int64{c1.ID}, false)
+	check(c3.ID, c1.ID, []int64{c1.ID}, false)
+	check(c4.ID, c1.ID, []int64{c2.ID, c1.ID}, false)
+	check(c5.ID, c1.ID, []int64{c4.ID, c2.ID, c1.ID}, false)
+	check(c5b.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true)
+	check(c6.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, false)
+	check(c7.ID, c1.ID, []int64{c5.ID, c4.ID, c2.ID, c1.ID}, true)
+
+	check(d0.ID, d0.ID, nil, true)
+	check(d1.ID, d0.ID, []int64{d0.ID}, false)
+	check(d2.ID, d0.ID, []int64{d1.ID, d0.ID}, false)
+
+	check(e0.ID, e0.ID, nil, true)
+	check(e1.ID, e0.ID, []int64{e2.ID, e0.ID}, false)
+	check(e2.ID, e0.ID, []int64{e0.ID}, false)
+
+	check(f0.ID, f0.ID, nil, false)
+	check(f1.ID, f1.ID, nil, true)
+	check(f2.ID, f1.ID, []int64{f3.ID, f1.ID}, false)
+	check(f3.ID, f1.ID, []int64{f1.ID}, false)
+
+	check(g0.ID, g0.ID, nil, false)
+	check(g0b.ID, g0.ID, []int64{g0.ID}, true)
+
+	check(h0.ID, h0.ID, nil, false)
+	check(h0b.ID, h0.ID, []int64{h0.ID}, true)
+	check(h1.ID, h0.ID, []int64{h0.ID}, false)
+
+	check(s0.ID, s0.ID, nil, true)
+
+	check(b0.ID, b0.ID, nil, false)
+	check(b1.ID, b1.ID, nil, false)
+
+	check(i0.ID, i1.ID, []int64{i1.ID}, false)
+	check(i1.ID, i1.ID, nil, false)
+	check(i2.ID, i1.ID, []int64{i1.ID}, true)
+	check(i3.ID, i1.ID, []int64{i0.ID, i1.ID}, true)
+
+	check(j0.ID, j0.ID, nil, false)
+}

test-upgrade.sh

@@ -81,8 +81,8 @@ for tag in $tags; do
 	fi
 done
 echo "Testing final upgrade to current."
-time ../../mox verifydata -skip-size-check stepdata
+time ../../mox -cpuprof ../../upgrade-verifydata.cpu.pprof -memprof ../../upgrade-verifydata.mem.pprof verifydata -skip-size-check stepdata
-time ../../mox openaccounts stepdata test0 test1 test2
+time ../../mox -loglevel info -cpuprof ../../upgrade-openaccounts.cpu.pprof -memprof ../../upgrade-openaccounts.mem.pprof openaccounts stepdata test0 test1 test2
 rm -r stepdata
 rm */mox
 cd ../..

verifydata.go

@@ -11,6 +11,8 @@ import (
 	"strconv"
 	"strings"
 
+	"golang.org/x/exp/slices"
+
 	bolt "go.etcd.io/bbolt"
 
 	"github.com/mjl-/bstore"
@@ -241,6 +243,13 @@ possibly making them potentially no longer readable by the previous version.
 				checkf(err, dbpath, "missing nextuidvalidity")
 			}
 
+			up := store.Upgrade{ID: 1}
+			if err := db.Get(ctxbg, &up); err != nil {
+				log.Printf("warning: getting upgrade record (continuing, but not checking message threading): %v", err)
+			} else if up.Threads != 2 {
+				log.Printf("warning: no message threading in database, skipping checks for threading consistency")
+			}
+
 			mailboxes := map[int64]store.Mailbox{}
 			err := bstore.QueryDB[store.Mailbox](ctxbg, db).ForEach(func(mb store.Mailbox) error {
 				mailboxes[mb.ID] = mb
@@ -270,10 +279,37 @@ possibly making them potentially no longer readable by the previous version.
 				if m.Expunged {
 					return nil
 				}
+
 				mp := store.MessagePath(m.ID)
 				seen[mp] = struct{}{}
 				p := filepath.Join(accdir, "msg", mp)
 				checkFile(dbpath, p, len(m.MsgPrefix), m.Size)
+
+				if up.Threads != 2 {
+					return nil
+				}
+
+				if m.ThreadID <= 0 {
+					checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has bad threadid", m.ID, m.ThreadID, mb.Name, mb.ID)
+				}
+				if len(m.ThreadParentIDs) == 0 {
+					return nil
+				}
+				if slices.Contains(m.ThreadParentIDs, m.ID) {
+					checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message id %d, thread %d in mailbox %q (id %d) has itself as thread parent", m.ID, m.ThreadID, mb.Name, mb.ID)
+				}
+				for i, pid := range m.ThreadParentIDs {
+					am := store.Message{ID: pid}
+					if err := db.Get(ctxbg, &am); err == bstore.ErrAbsent {
+						continue
+					} else if err != nil {
+						return fmt.Errorf("get ancestor message: %v", err)
+					} else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) {
+						checkf(errors.New(`see "mox reassignthreads"`), dbpath, "message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)
+					} else {
+						break
+					}
+				}
 				return nil
 			})
 			checkf(err, dbpath, "reading messages in account database to check files")

webaccount/account.html

@@ -235,6 +235,11 @@ const index = async () => {
 				}
 				problems.appendChild(dom.div(box(yellow, data.Message)))
 			})
+			eventSource.addEventListener('step', (e) => {
+				const data = JSON.parse(e.data) // {Title: ...}
+				console.log('import step event', {e, data})
+				importProgress.appendChild(dom.div(dom.br(), box(blue, 'Step: '+data.Title)))
+			})
 			eventSource.addEventListener('done', (e) => {
 				console.log('import done event', {e})
 				importProgress.appendChild(dom.div(dom.br(), box(blue, 'Import finished')))
@@ -475,7 +480,7 @@ const index = async () => {
 				),
 				dom.div(
 					dom.button('Upload and import'),
-					dom.p(style({fontStyle: 'italic', marginTop: '.5ex'}), 'The file is uploaded first, then its messages are imported. Importing is done in a transaction, you can abort the entire import before it is finished.'),
+					dom.p(style({fontStyle: 'italic', marginTop: '.5ex'}), 'The file is uploaded first, then its messages are imported, finally messages are matched for threading. Importing is done in a transaction, you can abort the entire import before it is finished.'),
 				),
 			),
 		),

webaccount/account_test.go

@@ -131,6 +131,7 @@ func TestAccount(t *testing.T) {
 				count += x.Count
 			case importProblem:
 				t.Fatalf("unexpected problem: %q", x.Message)
+			case importStep:
 			case importDone:
 				break loop
 			case importAborted:

webaccount/import.go

@@ -193,6 +193,9 @@ type importProblem struct {
 }
 type importDone struct{}
 type importAborted struct{}
+type importStep struct {
+	Title string
+}
 
 // importStart prepare the import and launches the goroutine to actually import.
 // importStart is responsible for closing f.
@@ -284,12 +287,6 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 	// ID's of delivered messages. If we have to rollback, we have to remove this files.
 	var deliveredIDs []int64
 
-	ximportcheckf := func(err error, format string, args ...any) {
-		if err != nil {
-			panic(importError{fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), err)})
-		}
-	}
-
 	sendEvent := func(kind string, v any) {
 		buf, err := json.Marshal(v)
 		if err != nil {
@@ -300,11 +297,6 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		importers.Events <- importEvent{token, []byte(ssemsg), v, nil}
 	}
 
-	problemf := func(format string, args ...any) {
-		msg := fmt.Sprintf(format, args...)
-		sendEvent("problem", importProblem{Message: msg})
-	}
-
 	canceled := func() bool {
 		select {
 		case <-ctx.Done():
@@ -315,6 +307,11 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		}
 	}
 
+	problemf := func(format string, args ...any) {
+		msg := fmt.Sprintf(format, args...)
+		sendEvent("problem", importProblem{Message: msg})
+	}
+
 	defer func() {
 		err := f.Close()
 		log.Check(err, "closing uploaded messages file")
@@ -349,6 +346,15 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		}
 	}()
 
+	ximportcheckf := func(err error, format string, args ...any) {
+		if err != nil {
+			panic(importError{fmt.Errorf("%s: %s", fmt.Sprintf(format, args...), err)})
+		}
+	}
+
+	err := acc.ThreadingWait(log)
+	ximportcheckf(err, "waiting for account thread upgrade")
+
 	conf, _ := acc.Conf()
 
 	jf, _, err := acc.OpenJunkFilter(ctx, log)
@@ -515,6 +521,11 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		m.ParsedBuf, err = json.Marshal(p)
 		ximportcheckf(err, "marshal parsed message structure")
 
+		// Set fields needed for future threading. By doing it now, DeliverMessage won't
+		// have to parse the Part again.
+		p.SetReaderAt(store.FileMsgReader(m.MsgPrefix, f))
+		m.PrepareThreading(log, &p)
+
 		if m.Received.IsZero() {
 			if p.Envelope != nil && !p.Envelope.Date.IsZero() {
 				m.Received = p.Envelope.Date
@@ -534,7 +545,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		const consumeFile = true
 		const sync = false
 		const notrain = true
-		if err := acc.DeliverMessage(log, tx, m, f, consumeFile, sync, notrain); err != nil {
+		const nothreads = true
+		if err := acc.DeliverMessage(log, tx, m, f, consumeFile, sync, notrain, nothreads); err != nil {
 			problemf("delivering message %s: %s (continuing)", pos, err)
 			return
 		}
@@ -797,13 +809,20 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 	for _, count := range messages {
 		total += count
 	}
-	log.Debug("message imported", mlog.Field("total", total))
+	log.Debug("messages imported", mlog.Field("total", total))
 
 	// Send final update for count of last-imported mailbox.
 	if prevMailbox != "" {
 		sendEvent("count", importCount{prevMailbox, messages[prevMailbox]})
 	}
 
+	// Match threads.
+	if len(deliveredIDs) > 0 {
+		sendEvent("step", importStep{"matching messages with threads"})
+		err = acc.AssignThreads(ctx, log, tx, deliveredIDs[0], 0, io.Discard)
+		ximportcheckf(err, "assigning messages to threads")
+	}
+
 	// Update mailboxes with counts and keywords.
 	for mbID, mc := range destMailboxCounts {
 		mb := store.Mailbox{ID: mbID}

webmail/api.go

@@ -748,7 +748,7 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 			err = tx.Update(&sentmb)
 			xcheckf(ctx, err, "updating sent mailbox for counts")
 
-			err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, true, false)
+			err = acc.DeliverMessage(log, tx, &sentm, dataFile, true, true, false, false)
 			if err != nil {
 				metricSubmission.WithLabelValues("storesenterror").Inc()
 				metricked = true
@@ -1488,7 +1488,140 @@ func (Webmail) MailboxSetSpecialUse(ctx context.Context, mb store.Mailbox) {
 	})
 }
 
+// ThreadCollapse saves the ThreadCollapse field for the messages and its
+// children. The messageIDs are typically thread roots. But not all roots
+// (without parent) of a thread need to have the same collapsed state.
+func (Webmail) ThreadCollapse(ctx context.Context, messageIDs []int64, collapse bool) {
+	log := xlog.WithContext(ctx)
+	reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)
+	acc, err := store.OpenAccount(reqInfo.AccountName)
+	xcheckf(ctx, err, "open account")
+	defer func() {
+		err := acc.Close()
+		log.Check(err, "closing account")
+	}()
+
+	if len(messageIDs) == 0 {
+		xcheckuserf(ctx, errors.New("no messages"), "setting collapse")
+	}
+
+	acc.WithWLock(func() {
+		changes := make([]store.Change, 0, len(messageIDs))
+		xdbwrite(ctx, acc, func(tx *bstore.Tx) {
+			// Gather ThreadIDs to list all potential messages, for a way to get all potential
+			// (child) messages. Further refined in FilterFn.
+			threadIDs := map[int64]struct{}{}
+			msgIDs := map[int64]struct{}{}
+			for _, id := range messageIDs {
+				m := store.Message{ID: id}
+				err := tx.Get(&m)
+				if err == bstore.ErrAbsent {
+					xcheckuserf(ctx, err, "get message")
+				}
+				xcheckf(ctx, err, "get message")
+				threadIDs[m.ThreadID] = struct{}{}
+				msgIDs[id] = struct{}{}
+			}
+
+			var updated []store.Message
+			q := bstore.QueryTx[store.Message](tx)
+			q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)
+			q.FilterNotEqual("ThreadCollapsed", collapse)
+			q.FilterFn(func(tm store.Message) bool {
+				for _, id := range tm.ThreadParentIDs {
+					if _, ok := msgIDs[id]; ok {
+						return true
+					}
+				}
+				_, ok := msgIDs[tm.ID]
+				return ok
+			})
+			q.Gather(&updated)
+			q.SortAsc("ID") // Consistent order for testing.
+			_, err = q.UpdateFields(map[string]any{"ThreadCollapsed": collapse})
+			xcheckf(ctx, err, "updating collapse in database")
+
+			for _, m := range updated {
+				changes = append(changes, m.ChangeThread())
+			}
+		})
+		store.BroadcastChanges(acc, changes)
+	})
+}
+
+// ThreadMute saves the ThreadMute field for the messages and their children.
+// If messages are muted, they are also marked collapsed.
+func (Webmail) ThreadMute(ctx context.Context, messageIDs []int64, mute bool) {
+	log := xlog.WithContext(ctx)
+	reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo)
+	acc, err := store.OpenAccount(reqInfo.AccountName)
+	xcheckf(ctx, err, "open account")
+	defer func() {
+		err := acc.Close()
+		log.Check(err, "closing account")
+	}()
+
+	if len(messageIDs) == 0 {
+		xcheckuserf(ctx, errors.New("no messages"), "setting mute")
+	}
+
+	acc.WithWLock(func() {
+		changes := make([]store.Change, 0, len(messageIDs))
+		xdbwrite(ctx, acc, func(tx *bstore.Tx) {
+			threadIDs := map[int64]struct{}{}
+			msgIDs := map[int64]struct{}{}
+			for _, id := range messageIDs {
+				m := store.Message{ID: id}
+				err := tx.Get(&m)
+				if err == bstore.ErrAbsent {
+					xcheckuserf(ctx, err, "get message")
+				}
+				xcheckf(ctx, err, "get message")
+				threadIDs[m.ThreadID] = struct{}{}
+				msgIDs[id] = struct{}{}
+			}
+
+			var updated []store.Message
+
+			q := bstore.QueryTx[store.Message](tx)
+			q.FilterEqual("ThreadID", slicesAny(maps.Keys(threadIDs))...)
+			q.FilterFn(func(tm store.Message) bool {
+				if tm.ThreadMuted == mute && (!mute || tm.ThreadCollapsed) {
+					return false
+				}
+				for _, id := range tm.ThreadParentIDs {
+					if _, ok := msgIDs[id]; ok {
+						return true
+					}
+				}
+				_, ok := msgIDs[tm.ID]
+				return ok
+			})
+			q.Gather(&updated)
+			fields := map[string]any{"ThreadMuted": mute}
+			if mute {
+				fields["ThreadCollapsed"] = true
+			}
+			_, err = q.UpdateFields(fields)
+			xcheckf(ctx, err, "updating mute in database")
+
+			for _, m := range updated {
+				changes = append(changes, m.ChangeThread())
+			}
+		})
+		store.BroadcastChanges(acc, changes)
+	})
+}
+
+func slicesAny[T any](l []T) []any {
+	r := make([]any, len(l))
+	for i, v := range l {
+		r[i] = v
+	}
+	return r
+}
+
 // SSETypes exists to ensure the generated API contains the types, for use in SSE events.
-func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) {
+func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, msgThread ChangeMsgThread, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) {
 	return
 }

webmail/api.json

@@ -235,6 +235,46 @@
 			],
 			"Returns": []
 		},
+		{
+			"Name": "ThreadCollapse",
+			"Docs": "ThreadCollapse saves the ThreadCollapse field for the messages and its\nchildren. The messageIDs are typically thread roots. But not all roots\n(without parent) of a thread need to have the same collapsed state.",
+			"Params": [
+				{
+					"Name": "messageIDs",
+					"Typewords": [
+						"[]",
+						"int64"
+					]
+				},
+				{
+					"Name": "collapse",
+					"Typewords": [
+						"bool"
+					]
+				}
+			],
+			"Returns": []
+		},
+		{
+			"Name": "ThreadMute",
+			"Docs": "ThreadMute saves the ThreadMute field for the messages and their children.\nIf messages are muted, they are also marked collapsed.",
+			"Params": [
+				{
+					"Name": "messageIDs",
+					"Typewords": [
+						"[]",
+						"int64"
+					]
+				},
+				{
+					"Name": "mute",
+					"Typewords": [
+						"bool"
+					]
+				}
+			],
+			"Returns": []
+		},
 		{
 			"Name": "SSETypes",
 			"Docs": "SSETypes exists to ensure the generated API contains the types, for use in SSE events.",
@@ -288,6 +328,12 @@
 						"ChangeMsgFlags"
 					]
 				},
+				{
+					"Name": "msgThread",
+					"Typewords": [
+						"ChangeMsgThread"
+					]
+				},
 				{
 					"Name": "mailboxRemove",
 					"Typewords": [
@@ -394,6 +440,13 @@
 						"bool"
 					]
 				},
+				{
+					"Name": "Threading",
+					"Docs": "",
+					"Typewords": [
+						"ThreadMode"
+					]
+				},
 				{
 					"Name": "Filter",
 					"Docs": "",
@@ -783,7 +836,7 @@
 				},
 				{
 					"Name": "Subject",
-					"Docs": "",
+					"Docs": "Q/B-word-decoded.",
 					"Typewords": [
 						"string"
 					]
@@ -1319,8 +1372,9 @@
 				},
 				{
 					"Name": "MessageItems",
-					"Docs": "If empty, this was the last message for the request.",
+					"Docs": "If empty, this was the last message for the request. If non-empty, a list of thread messages. Each with the first message being the reason this thread is included and can be used as AnchorID in followup requests. If the threading mode is \"off\" in the query, there will always be only a single message. If a thread is sent, all messages in the thread are sent, including those that don't match the query (e.g. from another mailbox). Threads can be displayed based on the ThreadParentIDs field, with possibly slightly different display based on field ThreadMissingLink.",
 					"Typewords": [
+						"[]",
 						"[]",
 						"MessageItem"
 					]
@@ -1388,6 +1442,13 @@
 					"Typewords": [
 						"string"
 					]
+				},
+				{
+					"Name": "MatchQuery",
+					"Docs": "If message does not match query, it can still be included because of threading.",
+					"Typewords": [
+						"bool"
+					]
 				}
 			]
 		},
@@ -1630,19 +1691,62 @@
 				},
 				{
 					"Name": "MessageID",
-					"Docs": "Value of Message-Id header. Only set for messages that were delivered to the rejects mailbox. For ensuring such messages are delivered only once. Value includes \u003c\u003e.",
+					"Docs": "Canonicalized Message-Id, always lower-case and normalized quoting, without \u003c\u003e's. Empty if missing. Used for matching message threads, and to prevent duplicate reject delivery.",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "SubjectBase",
+					"Docs": "For matching threads in case there is no References/In-Reply-To header. It is lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.",
 					"Typewords": [
 						"string"
 					]
 				},
 				{
 					"Name": "MessageHash",
-					"Docs": "Hash of message. For rejects delivery, so optional like MessageID.",
+					"Docs": "Hash of message. For rejects delivery in case there is no Message-ID, only set when delivered as reject.",
 					"Typewords": [
 						"[]",
 						"uint8"
 					]
 				},
+				{
+					"Name": "ThreadID",
+					"Docs": "ID of message starting this thread.",
+					"Typewords": [
+						"int64"
+					]
+				},
+				{
+					"Name": "ThreadParentIDs",
+					"Docs": "IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.",
+					"Typewords": [
+						"[]",
+						"int64"
+					]
+				},
+				{
+					"Name": "ThreadMissingLink",
+					"Docs": "ThreadMissingLink is true if there is no match with a direct parent. E.g. first ID in ThreadParentIDs is not the direct ancestor (an intermediate message may have been deleted), or subject-based matching was done.",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "ThreadMuted",
+					"Docs": "If set, newly delivered child messages are automatically marked as read. This field is copied to new child messages. Changes are propagated to the webmail client.",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "ThreadCollapsed",
+					"Docs": "If set, this (sub)thread is collapsed in the webmail client, for threading mode \"on\" (mode \"unread\" ignores it). This field is copied to new child message. Changes are propagated to the webmail client.",
+					"Typewords": [
+						"bool"
+					]
+				},
 				{
 					"Name": "Seen",
 					"Docs": "",
@@ -1888,7 +1992,7 @@
 		},
 		{
 			"Name": "ChangeMsgAdd",
-			"Docs": "ChangeMsgAdd adds a new message to the view.",
+			"Docs": "ChangeMsgAdd adds a new message and possibly its thread to the view.",
 			"Fields": [
 				{
 					"Name": "MailboxID",
@@ -1927,9 +2031,10 @@
 					]
 				},
 				{
-					"Name": "MessageItem",
+					"Name": "MessageItems",
 					"Docs": "",
 					"Typewords": [
+						"[]",
 						"MessageItem"
 					]
 				}
@@ -2088,6 +2193,34 @@
 				}
 			]
 		},
+		{
+			"Name": "ChangeMsgThread",
+			"Docs": "ChangeMsgThread updates muted/collapsed fields for one message.",
+			"Fields": [
+				{
+					"Name": "MessageIDs",
+					"Docs": "",
+					"Typewords": [
+						"[]",
+						"int64"
+					]
+				},
+				{
+					"Name": "Muted",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "Collapsed",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				}
+			]
+		},
 		{
 			"Name": "ChangeMailboxRemove",
 			"Docs": "ChangeMailboxRemove indicates a mailbox was removed, including all its messages.",
@@ -2382,6 +2515,27 @@
 		}
 	],
 	"Strings": [
+		{
+			"Name": "ThreadMode",
+			"Docs": "",
+			"Values": [
+				{
+					"Name": "ThreadOff",
+					"Value": "off",
+					"Docs": ""
+				},
+				{
+					"Name": "ThreadOn",
+					"Value": "on",
+					"Docs": ""
+				},
+				{
+					"Name": "ThreadUnread",
+					"Value": "unread",
+					"Docs": ""
+				}
+			]
+		},
 		{
 			"Name": "AttachmentType",
 			"Docs": "AttachmentType is for filtering by attachment type.",

webmail/api.ts

@@ -16,6 +16,7 @@ export interface Request {
 // Query is a request for messages that match filters, in a given order.
 export interface Query {
 	OrderAsc: boolean  // Order by received ascending or desending.
+	Threading: ThreadMode
 	Filter: Filter
 	NotFilter: NotFilter
 }
@@ -91,7 +92,7 @@ export interface Part {
 // Envelope holds the basic/common message headers as used in IMAP4.
 export interface Envelope {
 	Date: Date
-	Subject: string
+	Subject: string  // Q/B-word-decoded.
 	From?: Address[] | null
 	Sender?: Address[] | null
 	ReplyTo?: Address[] | null
@@ -218,7 +219,7 @@ export interface EventViewReset {
 export interface EventViewMsgs {
 	ViewID: number
 	RequestID: number
-	MessageItems?: MessageItem[] | null  // If empty, this was the last message for the request.
+	MessageItems?: (MessageItem[] | null)[] | null  // If empty, this was the last message for the request. If non-empty, a list of thread messages. Each with the first message being the reason this thread is included and can be used as AnchorID in followup requests. If the threading mode is "off" in the query, there will always be only a single message. If a thread is sent, all messages in the thread are sent, including those that don't match the query (e.g. from another mailbox). Threads can be displayed based on the ThreadParentIDs field, with possibly slightly different display based on field ThreadMissingLink.
 	ParsedMessage?: ParsedMessage | null  // If set, will match the target page.DestMessageID from the request.
 	ViewEnd: boolean  // If set, there are no more messages in this view at this moment. Messages can be added, typically via Change messages, e.g. for new deliveries.
 }
@@ -233,6 +234,7 @@ export interface MessageItem {
 	IsSigned: boolean
 	IsEncrypted: boolean
 	FirstLine: string  // Of message body, for showing as preview.
+	MatchQuery: boolean  // If message does not match query, it can still be included because of threading.
 }
 
 // Message stored in database and per-message file on disk.
@@ -276,8 +278,14 @@ export interface Message {
 	DKIMDomains?: string[] | null  // Domains with verified DKIM signatures. Unicode string. For forwarded messages, a DKIM domain that matched a ruleset's verified domain is left out, but included in OrigDKIMDomains.
 	OrigEHLODomain: string  // For forwarded messages,
 	OrigDKIMDomains?: string[] | null
-	MessageID: string  // Value of Message-Id header. Only set for messages that were delivered to the rejects mailbox. For ensuring such messages are delivered only once. Value includes <>.
+	MessageID: string  // Canonicalized Message-Id, always lower-case and normalized quoting, without <>'s. Empty if missing. Used for matching message threads, and to prevent duplicate reject delivery.
-	MessageHash?: string | null  // Hash of message. For rejects delivery, so optional like MessageID.
+	SubjectBase: string  // For matching threads in case there is no References/In-Reply-To header. It is lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.
+	MessageHash?: string | null  // Hash of message. For rejects delivery in case there is no Message-ID, only set when delivered as reject.
+	ThreadID: number  // ID of message starting this thread.
+	ThreadParentIDs?: number[] | null  // IDs of parent messages, from closest parent to the root message. Parent messages may be in a different mailbox, or may no longer exist. ThreadParentIDs must never contain the message id itself (a cycle), and parent messages must reference the same ancestors.
+	ThreadMissingLink: boolean  // ThreadMissingLink is true if there is no match with a direct parent. E.g. first ID in ThreadParentIDs is not the direct ancestor (an intermediate message may have been deleted), or subject-based matching was done.
+	ThreadMuted: boolean  // If set, newly delivered child messages are automatically marked as read. This field is copied to new child messages. Changes are propagated to the webmail client.
+	ThreadCollapsed: boolean  // If set, this (sub)thread is collapsed in the webmail client, for threading mode "on" (mode "unread" ignores it). This field is copied to new child message. Changes are propagated to the webmail client.
 	Seen: boolean
 	Answered: boolean
 	Flagged: boolean
@@ -326,14 +334,14 @@ export interface EventViewChanges {
 	Changes?: (any[] | null)[] | null  // The first field of [2]any is a string, the second of the Change types below.
 }
 
-// ChangeMsgAdd adds a new message to the view.
+// ChangeMsgAdd adds a new message and possibly its thread to the view.
 export interface ChangeMsgAdd {
 	MailboxID: number
 	UID: UID
 	ModSeq: ModSeq
 	Flags: Flags  // System flags.
 	Keywords?: string[] | null  // Other flags.
-	MessageItem: MessageItem
+	MessageItems?: MessageItem[] | null
 }
 
 // Flags for a mail message.
@@ -367,6 +375,13 @@ export interface ChangeMsgFlags {
 	Keywords?: string[] | null  // Non-system/well-known flags/keywords/labels.
 }
 
+// ChangeMsgThread updates muted/collapsed fields for one message.
+export interface ChangeMsgThread {
+	MessageIDs?: number[] | null
+	Muted: boolean
+	Collapsed: boolean
+}
+
 // ChangeMailboxRemove indicates a mailbox was removed, including all its messages.
 export interface ChangeMailboxRemove {
 	MailboxID: number
@@ -446,6 +461,12 @@ export enum Validation {
 	ValidationNone = 10,  // E.g. No records.
 }
 
+export enum ThreadMode {
+	ThreadOff = "off",
+	ThreadOn = "on",
+	ThreadUnread = "unread",
+}
+
 // AttachmentType is for filtering by attachment type.
 export enum AttachmentType {
 	AttachmentIndifferent = "",
@@ -464,12 +485,12 @@ export enum AttachmentType {
 // An empty string can be a valid localpart.
 export type Localpart = string
 
-export const structTypes: {[typename: string]: boolean} = {"Address":true,"Attachment":true,"ChangeMailboxAdd":true,"ChangeMailboxCounts":true,"ChangeMailboxKeywords":true,"ChangeMailboxRemove":true,"ChangeMailboxRename":true,"ChangeMailboxSpecialUse":true,"ChangeMsgAdd":true,"ChangeMsgFlags":true,"ChangeMsgRemove":true,"Domain":true,"DomainAddressConfig":true,"Envelope":true,"EventStart":true,"EventViewChanges":true,"EventViewErr":true,"EventViewMsgs":true,"EventViewReset":true,"File":true,"Filter":true,"Flags":true,"ForwardAttachments":true,"Mailbox":true,"Message":true,"MessageAddress":true,"MessageEnvelope":true,"MessageItem":true,"NotFilter":true,"Page":true,"ParsedMessage":true,"Part":true,"Query":true,"Request":true,"SpecialUse":true,"SubmitMessage":true}
+export const structTypes: {[typename: string]: boolean} = {"Address":true,"Attachment":true,"ChangeMailboxAdd":true,"ChangeMailboxCounts":true,"ChangeMailboxKeywords":true,"ChangeMailboxRemove":true,"ChangeMailboxRename":true,"ChangeMailboxSpecialUse":true,"ChangeMsgAdd":true,"ChangeMsgFlags":true,"ChangeMsgRemove":true,"ChangeMsgThread":true,"Domain":true,"DomainAddressConfig":true,"Envelope":true,"EventStart":true,"EventViewChanges":true,"EventViewErr":true,"EventViewMsgs":true,"EventViewReset":true,"File":true,"Filter":true,"Flags":true,"ForwardAttachments":true,"Mailbox":true,"Message":true,"MessageAddress":true,"MessageEnvelope":true,"MessageItem":true,"NotFilter":true,"Page":true,"ParsedMessage":true,"Part":true,"Query":true,"Request":true,"SpecialUse":true,"SubmitMessage":true}
-export const stringsTypes: {[typename: string]: boolean} = {"AttachmentType":true,"Localpart":true}
+export const stringsTypes: {[typename: string]: boolean} = {"AttachmentType":true,"Localpart":true,"ThreadMode":true}
 export const intsTypes: {[typename: string]: boolean} = {"ModSeq":true,"UID":true,"Validation":true}
 export const types: TypenameMap = {
 	"Request": {"Name":"Request","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"SSEID","Docs":"","Typewords":["int64"]},{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Cancel","Docs":"","Typewords":["bool"]},{"Name":"Query","Docs":"","Typewords":["Query"]},{"Name":"Page","Docs":"","Typewords":["Page"]}]},
-	"Query": {"Name":"Query","Docs":"","Fields":[{"Name":"OrderAsc","Docs":"","Typewords":["bool"]},{"Name":"Filter","Docs":"","Typewords":["Filter"]},{"Name":"NotFilter","Docs":"","Typewords":["NotFilter"]}]},
+	"Query": {"Name":"Query","Docs":"","Fields":[{"Name":"OrderAsc","Docs":"","Typewords":["bool"]},{"Name":"Threading","Docs":"","Typewords":["ThreadMode"]},{"Name":"Filter","Docs":"","Typewords":["Filter"]},{"Name":"NotFilter","Docs":"","Typewords":["NotFilter"]}]},
 	"Filter": {"Name":"Filter","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"MailboxChildrenIncluded","Docs":"","Typewords":["bool"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"Words","Docs":"","Typewords":["[]","string"]},{"Name":"From","Docs":"","Typewords":["[]","string"]},{"Name":"To","Docs":"","Typewords":["[]","string"]},{"Name":"Oldest","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"Newest","Docs":"","Typewords":["nullable","timestamp"]},{"Name":"Subject","Docs":"","Typewords":["[]","string"]},{"Name":"Attachments","Docs":"","Typewords":["AttachmentType"]},{"Name":"Labels","Docs":"","Typewords":["[]","string"]},{"Name":"Headers","Docs":"","Typewords":["[]","[]","string"]},{"Name":"SizeMin","Docs":"","Typewords":["int64"]},{"Name":"SizeMax","Docs":"","Typewords":["int64"]}]},
 	"NotFilter": {"Name":"NotFilter","Docs":"","Fields":[{"Name":"Words","Docs":"","Typewords":["[]","string"]},{"Name":"From","Docs":"","Typewords":["[]","string"]},{"Name":"To","Docs":"","Typewords":["[]","string"]},{"Name":"Subject","Docs":"","Typewords":["[]","string"]},{"Name":"Attachments","Docs":"","Typewords":["AttachmentType"]},{"Name":"Labels","Docs":"","Typewords":["[]","string"]}]},
 	"Page": {"Name":"Page","Docs":"","Fields":[{"Name":"AnchorMessageID","Docs":"","Typewords":["int64"]},{"Name":"Count","Docs":"","Typewords":["int32"]},{"Name":"DestMessageID","Docs":"","Typewords":["int64"]}]},
@@ -487,16 +508,17 @@ export const types: TypenameMap = {
 	"DomainAddressConfig": {"Name":"DomainAddressConfig","Docs":"","Fields":[{"Name":"LocalpartCatchallSeparator","Docs":"","Typewords":["string"]},{"Name":"LocalpartCaseSensitive","Docs":"","Typewords":["bool"]}]},
 	"EventViewErr": {"Name":"EventViewErr","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"Err","Docs":"","Typewords":["string"]}]},
 	"EventViewReset": {"Name":"EventViewReset","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]}]},
-	"EventViewMsgs": {"Name":"EventViewMsgs","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]},{"Name":"ParsedMessage","Docs":"","Typewords":["nullable","ParsedMessage"]},{"Name":"ViewEnd","Docs":"","Typewords":["bool"]}]},
+	"EventViewMsgs": {"Name":"EventViewMsgs","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","[]","MessageItem"]},{"Name":"ParsedMessage","Docs":"","Typewords":["nullable","ParsedMessage"]},{"Name":"ViewEnd","Docs":"","Typewords":["bool"]}]},
-	"MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"FirstLine","Docs":"","Typewords":["string"]}]},
+	"MessageItem": {"Name":"MessageItem","Docs":"","Fields":[{"Name":"Message","Docs":"","Typewords":["Message"]},{"Name":"Envelope","Docs":"","Typewords":["MessageEnvelope"]},{"Name":"Attachments","Docs":"","Typewords":["[]","Attachment"]},{"Name":"IsSigned","Docs":"","Typewords":["bool"]},{"Name":"IsEncrypted","Docs":"","Typewords":["bool"]},{"Name":"FirstLine","Docs":"","Typewords":["string"]},{"Name":"MatchQuery","Docs":"","Typewords":["bool"]}]},
-	"Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]},
+	"Message": {"Name":"Message","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"CreateSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Expunged","Docs":"","Typewords":["bool"]},{"Name":"IsReject","Docs":"","Typewords":["bool"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"MailboxOrigID","Docs":"","Typewords":["int64"]},{"Name":"MailboxDestinedID","Docs":"","Typewords":["int64"]},{"Name":"Received","Docs":"","Typewords":["timestamp"]},{"Name":"RemoteIP","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked1","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked2","Docs":"","Typewords":["string"]},{"Name":"RemoteIPMasked3","Docs":"","Typewords":["string"]},{"Name":"EHLODomain","Docs":"","Typewords":["string"]},{"Name":"MailFrom","Docs":"","Typewords":["string"]},{"Name":"MailFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MailFromDomain","Docs":"","Typewords":["string"]},{"Name":"RcptToLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"RcptToDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromLocalpart","Docs":"","Typewords":["Localpart"]},{"Name":"MsgFromDomain","Docs":"","Typewords":["string"]},{"Name":"MsgFromOrgDomain","Docs":"","Typewords":["string"]},{"Name":"EHLOValidated","Docs":"","Typewords":["bool"]},{"Name":"MailFromValidated","Docs":"","Typewords":["bool"]},{"Name":"MsgFromValidated","Docs":"","Typewords":["bool"]},{"Name":"EHLOValidation","Docs":"","Typewords":["Validation"]},{"Name":"MailFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"MsgFromValidation","Docs":"","Typewords":["Validation"]},{"Name":"DKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"OrigEHLODomain","Docs":"","Typewords":["string"]},{"Name":"OrigDKIMDomains","Docs":"","Typewords":["[]","string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]},{"Name":"SubjectBase","Docs":"","Typewords":["string"]},{"Name":"MessageHash","Docs":"","Typewords":["nullable","string"]},{"Name":"ThreadID","Docs":"","Typewords":["int64"]},{"Name":"ThreadParentIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"ThreadMissingLink","Docs":"","Typewords":["bool"]},{"Name":"ThreadMuted","Docs":"","Typewords":["bool"]},{"Name":"ThreadCollapsed","Docs":"","Typewords":["bool"]},{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"Size","Docs":"","Typewords":["int64"]},{"Name":"TrainedJunk","Docs":"","Typewords":["nullable","bool"]},{"Name":"MsgPrefix","Docs":"","Typewords":["nullable","string"]},{"Name":"ParsedBuf","Docs":"","Typewords":["nullable","string"]}]},
 	"MessageEnvelope": {"Name":"MessageEnvelope","Docs":"","Fields":[{"Name":"Date","Docs":"","Typewords":["timestamp"]},{"Name":"Subject","Docs":"","Typewords":["string"]},{"Name":"From","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"Sender","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"ReplyTo","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"To","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"CC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"BCC","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"InReplyTo","Docs":"","Typewords":["string"]},{"Name":"MessageID","Docs":"","Typewords":["string"]}]},
 	"Attachment": {"Name":"Attachment","Docs":"","Fields":[{"Name":"Path","Docs":"","Typewords":["[]","int32"]},{"Name":"Filename","Docs":"","Typewords":["string"]},{"Name":"Part","Docs":"","Typewords":["Part"]}]},
 	"EventViewChanges": {"Name":"EventViewChanges","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"Changes","Docs":"","Typewords":["[]","[]","any"]}]},
-	"ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItem","Docs":"","Typewords":["MessageItem"]}]},
+	"ChangeMsgAdd": {"Name":"ChangeMsgAdd","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"MessageItems","Docs":"","Typewords":["[]","MessageItem"]}]},
 	"Flags": {"Name":"Flags","Docs":"","Fields":[{"Name":"Seen","Docs":"","Typewords":["bool"]},{"Name":"Answered","Docs":"","Typewords":["bool"]},{"Name":"Flagged","Docs":"","Typewords":["bool"]},{"Name":"Forwarded","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Notjunk","Docs":"","Typewords":["bool"]},{"Name":"Deleted","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Phishing","Docs":"","Typewords":["bool"]},{"Name":"MDNSent","Docs":"","Typewords":["bool"]}]},
 	"ChangeMsgRemove": {"Name":"ChangeMsgRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UIDs","Docs":"","Typewords":["[]","UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]}]},
 	"ChangeMsgFlags": {"Name":"ChangeMsgFlags","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"UID","Docs":"","Typewords":["UID"]},{"Name":"ModSeq","Docs":"","Typewords":["ModSeq"]},{"Name":"Mask","Docs":"","Typewords":["Flags"]},{"Name":"Flags","Docs":"","Typewords":["Flags"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]}]},
+	"ChangeMsgThread": {"Name":"ChangeMsgThread","Docs":"","Fields":[{"Name":"MessageIDs","Docs":"","Typewords":["[]","int64"]},{"Name":"Muted","Docs":"","Typewords":["bool"]},{"Name":"Collapsed","Docs":"","Typewords":["bool"]}]},
 	"ChangeMailboxRemove": {"Name":"ChangeMailboxRemove","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]}]},
 	"ChangeMailboxAdd": {"Name":"ChangeMailboxAdd","Docs":"","Fields":[{"Name":"Mailbox","Docs":"","Typewords":["Mailbox"]}]},
 	"ChangeMailboxRename": {"Name":"ChangeMailboxRename","Docs":"","Fields":[{"Name":"MailboxID","Docs":"","Typewords":["int64"]},{"Name":"OldName","Docs":"","Typewords":["string"]},{"Name":"NewName","Docs":"","Typewords":["string"]},{"Name":"Flags","Docs":"","Typewords":["[]","string"]}]},
@@ -507,6 +529,7 @@ export const types: TypenameMap = {
 	"UID": {"Name":"UID","Docs":"","Values":null},
 	"ModSeq": {"Name":"ModSeq","Docs":"","Values":null},
 	"Validation": {"Name":"Validation","Docs":"","Values":[{"Name":"ValidationUnknown","Value":0,"Docs":""},{"Name":"ValidationStrict","Value":1,"Docs":""},{"Name":"ValidationDMARC","Value":2,"Docs":""},{"Name":"ValidationRelaxed","Value":3,"Docs":""},{"Name":"ValidationPass","Value":4,"Docs":""},{"Name":"ValidationNeutral","Value":5,"Docs":""},{"Name":"ValidationTemperror","Value":6,"Docs":""},{"Name":"ValidationPermerror","Value":7,"Docs":""},{"Name":"ValidationFail","Value":8,"Docs":""},{"Name":"ValidationSoftfail","Value":9,"Docs":""},{"Name":"ValidationNone","Value":10,"Docs":""}]},
+	"ThreadMode": {"Name":"ThreadMode","Docs":"","Values":[{"Name":"ThreadOff","Value":"off","Docs":""},{"Name":"ThreadOn","Value":"on","Docs":""},{"Name":"ThreadUnread","Value":"unread","Docs":""}]},
 	"AttachmentType": {"Name":"AttachmentType","Docs":"","Values":[{"Name":"AttachmentIndifferent","Value":"","Docs":""},{"Name":"AttachmentNone","Value":"none","Docs":""},{"Name":"AttachmentAny","Value":"any","Docs":""},{"Name":"AttachmentImage","Value":"image","Docs":""},{"Name":"AttachmentPDF","Value":"pdf","Docs":""},{"Name":"AttachmentArchive","Value":"archive","Docs":""},{"Name":"AttachmentSpreadsheet","Value":"spreadsheet","Docs":""},{"Name":"AttachmentDocument","Value":"document","Docs":""},{"Name":"AttachmentPresentation","Value":"presentation","Docs":""}]},
 	"Localpart": {"Name":"Localpart","Docs":"","Values":null},
 }
@@ -541,6 +564,7 @@ export const parser = {
 	Flags: (v: any) => parse("Flags", v) as Flags,
 	ChangeMsgRemove: (v: any) => parse("ChangeMsgRemove", v) as ChangeMsgRemove,
 	ChangeMsgFlags: (v: any) => parse("ChangeMsgFlags", v) as ChangeMsgFlags,
+	ChangeMsgThread: (v: any) => parse("ChangeMsgThread", v) as ChangeMsgThread,
 	ChangeMailboxRemove: (v: any) => parse("ChangeMailboxRemove", v) as ChangeMailboxRemove,
 	ChangeMailboxAdd: (v: any) => parse("ChangeMailboxAdd", v) as ChangeMailboxAdd,
 	ChangeMailboxRename: (v: any) => parse("ChangeMailboxRename", v) as ChangeMailboxRename,
@@ -551,6 +575,7 @@ export const parser = {
 	UID: (v: any) => parse("UID", v) as UID,
 	ModSeq: (v: any) => parse("ModSeq", v) as ModSeq,
 	Validation: (v: any) => parse("Validation", v) as Validation,
+	ThreadMode: (v: any) => parse("ThreadMode", v) as ThreadMode,
 	AttachmentType: (v: any) => parse("AttachmentType", v) as AttachmentType,
 	Localpart: (v: any) => parse("Localpart", v) as Localpart,
 }
@@ -712,13 +737,34 @@ export class Client {
 		return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as void
 	}
 
+	// ThreadCollapse saves the ThreadCollapse field for the messages and its
+	// children. The messageIDs are typically thread roots. But not all roots
+	// (without parent) of a thread need to have the same collapsed state.
+	async ThreadCollapse(messageIDs: number[] | null, collapse: boolean): Promise<void> {
+		const fn: string = "ThreadCollapse"
+		const paramTypes: string[][] = [["[]","int64"],["bool"]]
+		const returnTypes: string[][] = []
+		const params: any[] = [messageIDs, collapse]
+		return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as void
+	}
+
+	// ThreadMute saves the ThreadMute field for the messages and their children.
+	// If messages are muted, they are also marked collapsed.
+	async ThreadMute(messageIDs: number[] | null, mute: boolean): Promise<void> {
+		const fn: string = "ThreadMute"
+		const paramTypes: string[][] = [["[]","int64"],["bool"]]
+		const returnTypes: string[][] = []
+		const params: any[] = [messageIDs, mute]
+		return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as void
+	}
+
 	// SSETypes exists to ensure the generated API contains the types, for use in SSE events.
-	async SSETypes(): Promise<[EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags]> {
+	async SSETypes(): Promise<[EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMsgThread, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags]> {
 		const fn: string = "SSETypes"
 		const paramTypes: string[][] = []
-		const returnTypes: string[][] = [["EventStart"],["EventViewErr"],["EventViewReset"],["EventViewMsgs"],["EventViewChanges"],["ChangeMsgAdd"],["ChangeMsgRemove"],["ChangeMsgFlags"],["ChangeMailboxRemove"],["ChangeMailboxAdd"],["ChangeMailboxRename"],["ChangeMailboxCounts"],["ChangeMailboxSpecialUse"],["ChangeMailboxKeywords"],["Flags"]]
+		const returnTypes: string[][] = [["EventStart"],["EventViewErr"],["EventViewReset"],["EventViewMsgs"],["EventViewChanges"],["ChangeMsgAdd"],["ChangeMsgRemove"],["ChangeMsgFlags"],["ChangeMsgThread"],["ChangeMailboxRemove"],["ChangeMailboxAdd"],["ChangeMailboxRename"],["ChangeMailboxCounts"],["ChangeMailboxSpecialUse"],["ChangeMailboxKeywords"],["Flags"]]
 		const params: any[] = []
-		return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as [EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags]
+		return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params) as [EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMsgThread, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags]
 	}
 }
 

webmail/message.go

@@ -26,7 +26,7 @@ func messageItem(log *mlog.Log, m store.Message, state *msgState) (MessageItem,
 	// Clear largish unused data.
 	m.MsgPrefix = nil
 	m.ParsedBuf = nil
-	return MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine}, nil
+	return MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine, true}, nil
 }
 
 // formatFirstLine returns a line the client can display next to the subject line

webmail/msg.js

@@ -17,6 +17,12 @@ var api;
 		Validation[Validation["ValidationSoftfail"] = 9] = "ValidationSoftfail";
 		Validation[Validation["ValidationNone"] = 10] = "ValidationNone";
 	})(Validation = api.Validation || (api.Validation = {}));
+	let ThreadMode;
+	(function (ThreadMode) {
+		ThreadMode["ThreadOff"] = "off";
+		ThreadMode["ThreadOn"] = "on";
+		ThreadMode["ThreadUnread"] = "unread";
+	})(ThreadMode = api.ThreadMode || (api.ThreadMode = {}));
 	// AttachmentType is for filtering by attachment type.
 	let AttachmentType;
 	(function (AttachmentType) {
@@ -30,12 +36,12 @@ var api;
 		AttachmentType["AttachmentDocument"] = "document";
 		AttachmentType["AttachmentPresentation"] = "presentation";
 	})(AttachmentType = api.AttachmentType || (api.AttachmentType = {}));
-	api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true };
+	api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true };
-	api.stringsTypes = { "AttachmentType": true, "Localpart": true };
+	api.stringsTypes = { "AttachmentType": true, "Localpart": true, "ThreadMode": true };
 	api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true };
 	api.types = {
 		"Request": { "Name": "Request", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Cancel", "Docs": "", "Typewords": ["bool"] }, { "Name": "Query", "Docs": "", "Typewords": ["Query"] }, { "Name": "Page", "Docs": "", "Typewords": ["Page"] }] },
-		"Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] },
+		"Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Threading", "Docs": "", "Typewords": ["ThreadMode"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] },
 		"Filter": { "Name": "Filter", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxChildrenIncluded", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Oldest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Newest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Headers", "Docs": "", "Typewords": ["[]", "[]", "string"] }, { "Name": "SizeMin", "Docs": "", "Typewords": ["int64"] }, { "Name": "SizeMax", "Docs": "", "Typewords": ["int64"] }] },
 		"NotFilter": { "Name": "NotFilter", "Docs": "", "Fields": [{ "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"Page": { "Name": "Page", "Docs": "", "Fields": [{ "Name": "AnchorMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Count", "Docs": "", "Typewords": ["int32"] }, { "Name": "DestMessageID", "Docs": "", "Typewords": ["int64"] }] },
@@ -53,16 +59,17 @@ var api;
 		"DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] },
 		"EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] },
 		"EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] },
-		"EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] },
+		"EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] },
-		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }] },
+		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }] },
-		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
+		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
 		"MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] },
 		"Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] },
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
-		"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItem", "Docs": "", "Typewords": ["MessageItem"] }] },
+		"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
 		"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
 		"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
+		"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }] },
 		"ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] },
 		"ChangeMailboxRename": { "Name": "ChangeMailboxRename", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "OldName", "Docs": "", "Typewords": ["string"] }, { "Name": "NewName", "Docs": "", "Typewords": ["string"] }, { "Name": "Flags", "Docs": "", "Typewords": ["[]", "string"] }] },
@@ -73,6 +80,7 @@ var api;
 		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
 		"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
+		"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
 		"AttachmentType": { "Name": "AttachmentType", "Docs": "", "Values": [{ "Name": "AttachmentIndifferent", "Value": "", "Docs": "" }, { "Name": "AttachmentNone", "Value": "none", "Docs": "" }, { "Name": "AttachmentAny", "Value": "any", "Docs": "" }, { "Name": "AttachmentImage", "Value": "image", "Docs": "" }, { "Name": "AttachmentPDF", "Value": "pdf", "Docs": "" }, { "Name": "AttachmentArchive", "Value": "archive", "Docs": "" }, { "Name": "AttachmentSpreadsheet", "Value": "spreadsheet", "Docs": "" }, { "Name": "AttachmentDocument", "Value": "document", "Docs": "" }, { "Name": "AttachmentPresentation", "Value": "presentation", "Docs": "" }] },
 		"Localpart": { "Name": "Localpart", "Docs": "", "Values": null },
 	};
@@ -106,6 +114,7 @@ var api;
 		Flags: (v) => api.parse("Flags", v),
 		ChangeMsgRemove: (v) => api.parse("ChangeMsgRemove", v),
 		ChangeMsgFlags: (v) => api.parse("ChangeMsgFlags", v),
+		ChangeMsgThread: (v) => api.parse("ChangeMsgThread", v),
 		ChangeMailboxRemove: (v) => api.parse("ChangeMailboxRemove", v),
 		ChangeMailboxAdd: (v) => api.parse("ChangeMailboxAdd", v),
 		ChangeMailboxRename: (v) => api.parse("ChangeMailboxRename", v),
@@ -116,6 +125,7 @@ var api;
 		UID: (v) => api.parse("UID", v),
 		ModSeq: (v) => api.parse("ModSeq", v),
 		Validation: (v) => api.parse("Validation", v),
+		ThreadMode: (v) => api.parse("ThreadMode", v),
 		AttachmentType: (v) => api.parse("AttachmentType", v),
 		Localpart: (v) => api.parse("Localpart", v),
 	};
@@ -261,11 +271,30 @@ var api;
 			const params = [mb];
 			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
 		}
+		// ThreadCollapse saves the ThreadCollapse field for the messages and its
+		// children. The messageIDs are typically thread roots. But not all roots
+		// (without parent) of a thread need to have the same collapsed state.
+		async ThreadCollapse(messageIDs, collapse) {
+			const fn = "ThreadCollapse";
+			const paramTypes = [["[]", "int64"], ["bool"]];
+			const returnTypes = [];
+			const params = [messageIDs, collapse];
+			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
+		}
+		// ThreadMute saves the ThreadMute field for the messages and their children.
+		// If messages are muted, they are also marked collapsed.
+		async ThreadMute(messageIDs, mute) {
+			const fn = "ThreadMute";
+			const paramTypes = [["[]", "int64"], ["bool"]];
+			const returnTypes = [];
+			const params = [messageIDs, mute];
+			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
+		}
 		// SSETypes exists to ensure the generated API contains the types, for use in SSE events.
 		async SSETypes() {
 			const fn = "SSETypes";
 			const paramTypes = [];
-			const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]];
+			const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMsgThread"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]];
 			const params = [];
 			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
 		}

webmail/text.js

@@ -17,6 +17,12 @@ var api;
 		Validation[Validation["ValidationSoftfail"] = 9] = "ValidationSoftfail";
 		Validation[Validation["ValidationNone"] = 10] = "ValidationNone";
 	})(Validation = api.Validation || (api.Validation = {}));
+	let ThreadMode;
+	(function (ThreadMode) {
+		ThreadMode["ThreadOff"] = "off";
+		ThreadMode["ThreadOn"] = "on";
+		ThreadMode["ThreadUnread"] = "unread";
+	})(ThreadMode = api.ThreadMode || (api.ThreadMode = {}));
 	// AttachmentType is for filtering by attachment type.
 	let AttachmentType;
 	(function (AttachmentType) {
@@ -30,12 +36,12 @@ var api;
 		AttachmentType["AttachmentDocument"] = "document";
 		AttachmentType["AttachmentPresentation"] = "presentation";
 	})(AttachmentType = api.AttachmentType || (api.AttachmentType = {}));
-	api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true };
+	api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "Request": true, "SpecialUse": true, "SubmitMessage": true };
-	api.stringsTypes = { "AttachmentType": true, "Localpart": true };
+	api.stringsTypes = { "AttachmentType": true, "Localpart": true, "ThreadMode": true };
 	api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true };
 	api.types = {
 		"Request": { "Name": "Request", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Cancel", "Docs": "", "Typewords": ["bool"] }, { "Name": "Query", "Docs": "", "Typewords": ["Query"] }, { "Name": "Page", "Docs": "", "Typewords": ["Page"] }] },
-		"Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] },
+		"Query": { "Name": "Query", "Docs": "", "Fields": [{ "Name": "OrderAsc", "Docs": "", "Typewords": ["bool"] }, { "Name": "Threading", "Docs": "", "Typewords": ["ThreadMode"] }, { "Name": "Filter", "Docs": "", "Typewords": ["Filter"] }, { "Name": "NotFilter", "Docs": "", "Typewords": ["NotFilter"] }] },
 		"Filter": { "Name": "Filter", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxChildrenIncluded", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Oldest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Newest", "Docs": "", "Typewords": ["nullable", "timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Headers", "Docs": "", "Typewords": ["[]", "[]", "string"] }, { "Name": "SizeMin", "Docs": "", "Typewords": ["int64"] }, { "Name": "SizeMax", "Docs": "", "Typewords": ["int64"] }] },
 		"NotFilter": { "Name": "NotFilter", "Docs": "", "Fields": [{ "Name": "Words", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Subject", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["AttachmentType"] }, { "Name": "Labels", "Docs": "", "Typewords": ["[]", "string"] }] },
 		"Page": { "Name": "Page", "Docs": "", "Fields": [{ "Name": "AnchorMessageID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Count", "Docs": "", "Typewords": ["int32"] }, { "Name": "DestMessageID", "Docs": "", "Typewords": ["int64"] }] },
@@ -53,16 +59,17 @@ var api;
 		"DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] },
 		"EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] },
 		"EventViewReset": { "Name": "EventViewReset", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }] },
-		"EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] },
+		"EventViewMsgs": { "Name": "EventViewMsgs", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "[]", "MessageItem"] }, { "Name": "ParsedMessage", "Docs": "", "Typewords": ["nullable", "ParsedMessage"] }, { "Name": "ViewEnd", "Docs": "", "Typewords": ["bool"] }] },
-		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }] },
+		"MessageItem": { "Name": "MessageItem", "Docs": "", "Fields": [{ "Name": "Message", "Docs": "", "Typewords": ["Message"] }, { "Name": "Envelope", "Docs": "", "Typewords": ["MessageEnvelope"] }, { "Name": "Attachments", "Docs": "", "Typewords": ["[]", "Attachment"] }, { "Name": "IsSigned", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsEncrypted", "Docs": "", "Typewords": ["bool"] }, { "Name": "FirstLine", "Docs": "", "Typewords": ["string"] }, { "Name": "MatchQuery", "Docs": "", "Typewords": ["bool"] }] },
-		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
+		"Message": { "Name": "Message", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "CreateSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Expunged", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsReject", "Docs": "", "Typewords": ["bool"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailboxOrigID", "Docs": "", "Typewords": ["int64"] }, { "Name": "MailboxDestinedID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Received", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "RemoteIP", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked1", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked2", "Docs": "", "Typewords": ["string"] }, { "Name": "RemoteIPMasked3", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFrom", "Docs": "", "Typewords": ["string"] }, { "Name": "MailFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MailFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "RcptToLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "RcptToDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromLocalpart", "Docs": "", "Typewords": ["Localpart"] }, { "Name": "MsgFromDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromOrgDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "EHLOValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MailFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "MsgFromValidated", "Docs": "", "Typewords": ["bool"] }, { "Name": "EHLOValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MailFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "MsgFromValidation", "Docs": "", "Typewords": ["Validation"] }, { "Name": "DKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "OrigEHLODomain", "Docs": "", "Typewords": ["string"] }, { "Name": "OrigDKIMDomains", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }, { "Name": "SubjectBase", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageHash", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ThreadID", "Docs": "", "Typewords": ["int64"] }, { "Name": "ThreadParentIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "ThreadMissingLink", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadMuted", "Docs": "", "Typewords": ["bool"] }, { "Name": "ThreadCollapsed", "Docs": "", "Typewords": ["bool"] }, { "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }, { "Name": "TrainedJunk", "Docs": "", "Typewords": ["nullable", "bool"] }, { "Name": "MsgPrefix", "Docs": "", "Typewords": ["nullable", "string"] }, { "Name": "ParsedBuf", "Docs": "", "Typewords": ["nullable", "string"] }] },
 		"MessageEnvelope": { "Name": "MessageEnvelope", "Docs": "", "Fields": [{ "Name": "Date", "Docs": "", "Typewords": ["timestamp"] }, { "Name": "Subject", "Docs": "", "Typewords": ["string"] }, { "Name": "From", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "Sender", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "ReplyTo", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "To", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "CC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "BCC", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "InReplyTo", "Docs": "", "Typewords": ["string"] }, { "Name": "MessageID", "Docs": "", "Typewords": ["string"] }] },
 		"Attachment": { "Name": "Attachment", "Docs": "", "Fields": [{ "Name": "Path", "Docs": "", "Typewords": ["[]", "int32"] }, { "Name": "Filename", "Docs": "", "Typewords": ["string"] }, { "Name": "Part", "Docs": "", "Typewords": ["Part"] }] },
 		"EventViewChanges": { "Name": "EventViewChanges", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Changes", "Docs": "", "Typewords": ["[]", "[]", "any"] }] },
-		"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItem", "Docs": "", "Typewords": ["MessageItem"] }] },
+		"ChangeMsgAdd": { "Name": "ChangeMsgAdd", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "MessageItems", "Docs": "", "Typewords": ["[]", "MessageItem"] }] },
 		"Flags": { "Name": "Flags", "Docs": "", "Fields": [{ "Name": "Seen", "Docs": "", "Typewords": ["bool"] }, { "Name": "Answered", "Docs": "", "Typewords": ["bool"] }, { "Name": "Flagged", "Docs": "", "Typewords": ["bool"] }, { "Name": "Forwarded", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Notjunk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Phishing", "Docs": "", "Typewords": ["bool"] }, { "Name": "MDNSent", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMsgRemove": { "Name": "ChangeMsgRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UIDs", "Docs": "", "Typewords": ["[]", "UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }] },
 		"ChangeMsgFlags": { "Name": "ChangeMsgFlags", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "UID", "Docs": "", "Typewords": ["UID"] }, { "Name": "ModSeq", "Docs": "", "Typewords": ["ModSeq"] }, { "Name": "Mask", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Flags", "Docs": "", "Typewords": ["Flags"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }] },
+		"ChangeMsgThread": { "Name": "ChangeMsgThread", "Docs": "", "Fields": [{ "Name": "MessageIDs", "Docs": "", "Typewords": ["[]", "int64"] }, { "Name": "Muted", "Docs": "", "Typewords": ["bool"] }, { "Name": "Collapsed", "Docs": "", "Typewords": ["bool"] }] },
 		"ChangeMailboxRemove": { "Name": "ChangeMailboxRemove", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }] },
 		"ChangeMailboxAdd": { "Name": "ChangeMailboxAdd", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["Mailbox"] }] },
 		"ChangeMailboxRename": { "Name": "ChangeMailboxRename", "Docs": "", "Fields": [{ "Name": "MailboxID", "Docs": "", "Typewords": ["int64"] }, { "Name": "OldName", "Docs": "", "Typewords": ["string"] }, { "Name": "NewName", "Docs": "", "Typewords": ["string"] }, { "Name": "Flags", "Docs": "", "Typewords": ["[]", "string"] }] },
@@ -73,6 +80,7 @@ var api;
 		"UID": { "Name": "UID", "Docs": "", "Values": null },
 		"ModSeq": { "Name": "ModSeq", "Docs": "", "Values": null },
 		"Validation": { "Name": "Validation", "Docs": "", "Values": [{ "Name": "ValidationUnknown", "Value": 0, "Docs": "" }, { "Name": "ValidationStrict", "Value": 1, "Docs": "" }, { "Name": "ValidationDMARC", "Value": 2, "Docs": "" }, { "Name": "ValidationRelaxed", "Value": 3, "Docs": "" }, { "Name": "ValidationPass", "Value": 4, "Docs": "" }, { "Name": "ValidationNeutral", "Value": 5, "Docs": "" }, { "Name": "ValidationTemperror", "Value": 6, "Docs": "" }, { "Name": "ValidationPermerror", "Value": 7, "Docs": "" }, { "Name": "ValidationFail", "Value": 8, "Docs": "" }, { "Name": "ValidationSoftfail", "Value": 9, "Docs": "" }, { "Name": "ValidationNone", "Value": 10, "Docs": "" }] },
+		"ThreadMode": { "Name": "ThreadMode", "Docs": "", "Values": [{ "Name": "ThreadOff", "Value": "off", "Docs": "" }, { "Name": "ThreadOn", "Value": "on", "Docs": "" }, { "Name": "ThreadUnread", "Value": "unread", "Docs": "" }] },
 		"AttachmentType": { "Name": "AttachmentType", "Docs": "", "Values": [{ "Name": "AttachmentIndifferent", "Value": "", "Docs": "" }, { "Name": "AttachmentNone", "Value": "none", "Docs": "" }, { "Name": "AttachmentAny", "Value": "any", "Docs": "" }, { "Name": "AttachmentImage", "Value": "image", "Docs": "" }, { "Name": "AttachmentPDF", "Value": "pdf", "Docs": "" }, { "Name": "AttachmentArchive", "Value": "archive", "Docs": "" }, { "Name": "AttachmentSpreadsheet", "Value": "spreadsheet", "Docs": "" }, { "Name": "AttachmentDocument", "Value": "document", "Docs": "" }, { "Name": "AttachmentPresentation", "Value": "presentation", "Docs": "" }] },
 		"Localpart": { "Name": "Localpart", "Docs": "", "Values": null },
 	};
@@ -106,6 +114,7 @@ var api;
 		Flags: (v) => api.parse("Flags", v),
 		ChangeMsgRemove: (v) => api.parse("ChangeMsgRemove", v),
 		ChangeMsgFlags: (v) => api.parse("ChangeMsgFlags", v),
+		ChangeMsgThread: (v) => api.parse("ChangeMsgThread", v),
 		ChangeMailboxRemove: (v) => api.parse("ChangeMailboxRemove", v),
 		ChangeMailboxAdd: (v) => api.parse("ChangeMailboxAdd", v),
 		ChangeMailboxRename: (v) => api.parse("ChangeMailboxRename", v),
@@ -116,6 +125,7 @@ var api;
 		UID: (v) => api.parse("UID", v),
 		ModSeq: (v) => api.parse("ModSeq", v),
 		Validation: (v) => api.parse("Validation", v),
+		ThreadMode: (v) => api.parse("ThreadMode", v),
 		AttachmentType: (v) => api.parse("AttachmentType", v),
 		Localpart: (v) => api.parse("Localpart", v),
 	};
@@ -261,11 +271,30 @@ var api;
 			const params = [mb];
 			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
 		}
+		// ThreadCollapse saves the ThreadCollapse field for the messages and its
+		// children. The messageIDs are typically thread roots. But not all roots
+		// (without parent) of a thread need to have the same collapsed state.
+		async ThreadCollapse(messageIDs, collapse) {
+			const fn = "ThreadCollapse";
+			const paramTypes = [["[]", "int64"], ["bool"]];
+			const returnTypes = [];
+			const params = [messageIDs, collapse];
+			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
+		}
+		// ThreadMute saves the ThreadMute field for the messages and their children.
+		// If messages are muted, they are also marked collapsed.
+		async ThreadMute(messageIDs, mute) {
+			const fn = "ThreadMute";
+			const paramTypes = [["[]", "int64"], ["bool"]];
+			const returnTypes = [];
+			const params = [messageIDs, mute];
+			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
+		}
 		// SSETypes exists to ensure the generated API contains the types, for use in SSE events.
 		async SSETypes() {
 			const fn = "SSETypes";
 			const paramTypes = [];
-			const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]];
+			const returnTypes = [["EventStart"], ["EventViewErr"], ["EventViewReset"], ["EventViewMsgs"], ["EventViewChanges"], ["ChangeMsgAdd"], ["ChangeMsgRemove"], ["ChangeMsgFlags"], ["ChangeMsgThread"], ["ChangeMailboxRemove"], ["ChangeMailboxAdd"], ["ChangeMailboxRename"], ["ChangeMailboxCounts"], ["ChangeMailboxSpecialUse"], ["ChangeMailboxKeywords"], ["Flags"]];
 			const params = [];
 			return await _sherpaCall(this.baseURL, { ...this.options }, paramTypes, returnTypes, fn, params);
 		}

webmail/view.go

@@ -52,9 +52,18 @@ type Request struct {
 	Page  Page
 }
 
+type ThreadMode string
+
+const (
+	ThreadOff    ThreadMode = "off"
+	ThreadOn     ThreadMode = "on"
+	ThreadUnread ThreadMode = "unread"
+)
+
 // Query is a request for messages that match filters, in a given order.
 type Query struct {
 	OrderAsc  bool // Order by received ascending or desending.
+	Threading ThreadMode
 	Filter    Filter
 	NotFilter NotFilter
 }
@@ -163,6 +172,7 @@ type MessageItem struct {
 	IsSigned    bool
 	IsEncrypted bool
 	FirstLine   string // Of message body, for showing as preview.
+	MatchQuery  bool   // If message does not match query, it can still be included because of threading.
 }
 
 // ParsedMessage has more parsed/derived information about a message, intended
@@ -219,8 +229,18 @@ type EventViewMsgs struct {
 	ViewID    int64
 	RequestID int64
 
-	MessageItems  []MessageItem  // If empty, this was the last message for the request.
+	// If empty, this was the last message for the request. If non-empty, a list of
-	ParsedMessage *ParsedMessage // If set, will match the target page.DestMessageID from the request.
+	// thread messages. Each with the first message being the reason this thread is
+	// included and can be used as AnchorID in followup requests. If the threading mode
+	// is "off" in the query, there will always be only a single message. If a thread
+	// is sent, all messages in the thread are sent, including those that don't match
+	// the query (e.g. from another mailbox). Threads can be displayed based on the
+	// ThreadParentIDs field, with possibly slightly different display based on field
+	// ThreadMissingLink.
+	MessageItems [][]MessageItem
+
+	// If set, will match the target page.DestMessageID from the request.
+	ParsedMessage *ParsedMessage
 
 	// If set, there are no more messages in this view at this moment. Messages can be
 	// added, typically via Change messages, e.g. for new deliveries.
@@ -253,10 +273,10 @@ type EventViewChanges struct {
 	Changes [][2]any // The first field of [2]any is a string, the second of the Change types below.
 }
 
-// ChangeMsgAdd adds a new message to the view.
+// ChangeMsgAdd adds a new message and possibly its thread to the view.
 type ChangeMsgAdd struct {
 	store.ChangeAddUID
-	MessageItem MessageItem
+	MessageItems []MessageItem
 }
 
 // ChangeMsgRemove removes one or more messages from the view.
@@ -269,6 +289,11 @@ type ChangeMsgFlags struct {
 	store.ChangeFlags
 }
 
+// ChangeMsgThread updates muted/collapsed fields for one message.
+type ChangeMsgThread struct {
+	store.ChangeThread
+}
+
 // ChangeMailboxRemove indicates a mailbox was removed, including all its messages.
 type ChangeMailboxRemove struct {
 	store.ChangeRemoveMailbox
@@ -308,9 +333,9 @@ type ChangeMailboxKeywords struct {
 type view struct {
 	Request Request
 
-	// Last message we sent to the client. We use it to decide if a newly delivered
+	// Received of last message we sent to the client. We use it to decide if a newly
-	// message is within the view and the client should get a notification.
+	// delivered message is within the view and the client should get a notification.
-	LastMessage store.Message
+	LastMessageReceived time.Time
 
 	// If set, the last message in the query view has been sent. There is no need to do
 	// another query, it will not return more data. Used to decide if an event for a
@@ -322,6 +347,12 @@ type view struct {
 	// Mailboxes to match, can be multiple, for matching children. If empty, there is
 	// no filter on mailboxes.
 	mailboxIDs map[int64]bool
+
+	// Threads sent to client. New messages for this thread are also sent, regardless
+	// of regular query matching, so also for other mailboxes. If the user (re)moved
+	// all messages of a thread, they may still receive events for the thread. Only
+	// filled when query with threading not off.
+	threadIDs map[int64]struct{}
 }
 
 // sses tracks all sse connections, and access to them.
@@ -513,6 +544,9 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 		http.Error(w, "400 - bad request - request cannot have Page.Count 0", http.StatusBadRequest)
 		return
 	}
+	if req.Query.Threading == "" {
+		req.Query.Threading = ThreadOff
+	}
 
 	var writer *eventWriter
 
@@ -699,7 +733,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 	// Start a view, it determines if we send a change to the client. And start an
 	// implicit query for messages, we'll send the messages to the client which can
 	// fill its ui with messages.
-	v := view{req, store.Message{}, false, matchMailboxes, mailboxIDs}
+	v := view{req, time.Time{}, false, matchMailboxes, mailboxIDs, map[int64]struct{}{}}
 	go viewRequestTx(reqctx, log, acc, qtx, v, viewMsgsc, viewErrc, viewResetc, donec)
 	qtx = nil // viewRequestTx closes qtx
 
@@ -764,7 +798,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 
 		// Return uids that are within range in view. Because the end has been reached, or
 		// because the UID is not after the last message.
-		xchangedUIDs := func(mailboxID int64, uids []store.UID) (changedUIDs []store.UID) {
+		xchangedUIDs := func(mailboxID int64, uids []store.UID, isRemove bool) (changedUIDs []store.UID) {
 			uidsAny := make([]any, len(uids))
 			for i, uid := range uids {
 				uidsAny[i] = uid
@@ -774,8 +808,10 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 			q := bstore.QueryTx[store.Message](xtx)
 			q.FilterNonzero(store.Message{MailboxID: mailboxID})
 			q.FilterEqual("UID", uidsAny...)
+			mbOK := v.matchesMailbox(mailboxID)
 			err = q.ForEach(func(m store.Message) error {
-				if v.inRange(m) {
+				_, thread := v.threadIDs[m.ThreadID]
+				if thread || mbOK && (v.inRange(m) || isRemove && m.Expunged) {
 					changedUIDs = append(changedUIDs, m.UID)
 				}
 				return nil
@@ -788,33 +824,40 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 		for _, change := range changes {
 			switch c := change.(type) {
 			case store.ChangeAddUID:
-				if ok, err := v.matches(log, acc, true, 0, c.MailboxID, c.UID, c.Flags, c.Keywords, getmsg); err != nil {
+				ok, err := v.matches(log, acc, true, 0, c.MailboxID, c.UID, c.Flags, c.Keywords, getmsg)
 				xcheckf(ctx, err, "matching new message against view")
-				} else if !ok {
-					continue
-				}
 				m, err := getmsg(0, c.MailboxID, c.UID)
 				xcheckf(ctx, err, "get message")
+				_, thread := v.threadIDs[m.ThreadID]
+				if !ok && !thread {
+					continue
+				}
 				state := msgState{acc: acc}
 				mi, err := messageItem(log, m, &state)
 				state.clear()
 				xcheckf(ctx, err, "make messageitem")
-				taggedChanges = append(taggedChanges, [2]any{"ChangeMsgAdd", ChangeMsgAdd{c, mi}})
+				mi.MatchQuery = ok
+
+				mil := []MessageItem{mi}
+				if !thread && req.Query.Threading != ThreadOff {
+					err := ensureTx()
+					xcheckf(ctx, err, "transaction")
+					more, _, err := gatherThread(log, xtx, acc, v, m, 0)
+					xcheckf(ctx, err, "gathering thread messages for id %d, thread %d", m.ID, m.ThreadID)
+					mil = append(mil, more...)
+					v.threadIDs[m.ThreadID] = struct{}{}
+				}
+
+				taggedChanges = append(taggedChanges, [2]any{"ChangeMsgAdd", ChangeMsgAdd{c, mil}})
 
 				// If message extends the view, store it as such.
-				if !v.Request.Query.OrderAsc && m.Received.Before(v.LastMessage.Received) || v.Request.Query.OrderAsc && m.Received.After(v.LastMessage.Received) {
+				if !v.Request.Query.OrderAsc && m.Received.Before(v.LastMessageReceived) || v.Request.Query.OrderAsc && m.Received.After(v.LastMessageReceived) {
-					v.LastMessage = m
+					v.LastMessageReceived = m.Received
 				}
 
 			case store.ChangeRemoveUIDs:
-				// We do a quick filter over changes, not sending UID updates for unselected
+				// We may send changes for uids the client doesn't know, that's fine.
-				// mailboxes or when the message is outside the range of the view. But we still may
+				changedUIDs := xchangedUIDs(c.MailboxID, c.UIDs, true)
-				// send messages that don't apply to the filter. If client doesn't recognize the
-				// messages, that's fine.
-				if !v.matchesMailbox(c.MailboxID) {
-					continue
-				}
-				changedUIDs := xchangedUIDs(c.MailboxID, c.UIDs)
 				if len(changedUIDs) == 0 {
 					continue
 				}
@@ -823,11 +866,8 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 				taggedChanges = append(taggedChanges, [2]any{"ChangeMsgRemove", ch})
 
 			case store.ChangeFlags:
-				// As with ChangeRemoveUIDs above, we send more changes than strictly needed.
+				// We may send changes for uids the client doesn't know, that's fine.
-				if !v.matchesMailbox(c.MailboxID) {
+				changedUIDs := xchangedUIDs(c.MailboxID, []store.UID{c.UID}, false)
-					continue
-				}
-				changedUIDs := xchangedUIDs(c.MailboxID, []store.UID{c.UID})
 				if len(changedUIDs) == 0 {
 					continue
 				}
@@ -835,6 +875,10 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 				ch.UID = changedUIDs[0]
 				taggedChanges = append(taggedChanges, [2]any{"ChangeMsgFlags", ch})
 
+			case store.ChangeThread:
+				// Change in muted/collaped state, just always ship it.
+				taggedChanges = append(taggedChanges, [2]any{"ChangeMsgThread", ChangeMsgThread{c}})
+
 			case store.ChangeRemoveMailbox:
 				taggedChanges = append(taggedChanges, [2]any{"ChangeMailboxRemove", ChangeMailboxRemove{c}})
 
@@ -910,7 +954,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 				v.End = true
 			}
 			if len(vm.MessageItems) > 0 {
-				v.LastMessage = vm.MessageItems[len(vm.MessageItems)-1].Message
+				v.LastMessageReceived = vm.MessageItems[len(vm.MessageItems)-1][0].Message.Received
 			}
 			writer.xsendEvent(ctx, log, "viewMsgs", vm)
 
@@ -948,7 +992,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 				cancelDrain()
 			}
 			if req.Cancel {
-				v = view{req, store.Message{}, false, false, nil}
+				v = view{req, time.Time{}, false, false, nil, nil}
 				continue
 			}
 
@@ -987,7 +1031,7 @@ func serveEvents(ctx context.Context, log *mlog.Log, w http.ResponseWriter, r *h
 					if req.Query.Filter.MailboxChildrenIncluded {
 						xgatherMailboxIDs(ctx, rtx, mailboxIDs, mailboxPrefixes)
 					}
-					v = view{req, store.Message{}, false, matchMailboxes, mailboxIDs}
+					v = view{req, time.Time{}, false, matchMailboxes, mailboxIDs, map[int64]struct{}{}}
 				} else {
 					v.Request = req
 				}
@@ -1067,7 +1111,7 @@ func (v view) matchesMailbox(mailboxID int64) bool {
 // inRange returns whether m is within the range for the view, whether a change for
 // this message should be sent to the client so it can update its state.
 func (v view) inRange(m store.Message) bool {
-	return v.End || !v.Request.Query.OrderAsc && !m.Received.Before(v.LastMessage.Received) || v.Request.Query.OrderAsc && !m.Received.After(v.LastMessage.Received)
+	return v.End || !v.Request.Query.OrderAsc && !m.Received.Before(v.LastMessageReceived) || v.Request.Query.OrderAsc && !m.Received.After(v.LastMessageReceived)
 }
 
 // matches checks if the message, identified by either messageID or mailboxID+UID,
@@ -1152,7 +1196,7 @@ type msgResp struct {
 	err     error          // If set, an error happened and fields below are not set.
 	reset   bool           // If set, the anchor message does not exist (anymore?) and we are sending messages from the start, fields below not set.
 	viewEnd bool           // If set, the last message for the view was seen, no more should be requested, fields below not set.
-	mi      MessageItem    // If none of the cases above apply, the message that was found matching the query.
+	mil     []MessageItem  // If none of the cases above apply, the messages that was found matching the query. First message was reason the thread is returned, for use as AnchorID in followup request.
 	pm      *ParsedMessage // If m was the target page.DestMessageID, or this is the first match, this is the parsed message of mi.
 }
 
@@ -1176,7 +1220,7 @@ func viewRequestTx(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b
 		}
 	}()
 
-	var msgitems []MessageItem // Gathering for 300ms, then flushing.
+	var msgitems [][]MessageItem // Gathering for 300ms, then flushing.
 	var parsedMessage *ParsedMessage
 	var viewEnd bool
 
@@ -1225,7 +1269,7 @@ func viewRequestTx(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b
 				return
 			}
 
-			msgitems = append(msgitems, mr.mi)
+			msgitems = append(msgitems, mr.mil)
 			if mr.pm != nil {
 				parsedMessage = mr.pm
 			}
@@ -1406,6 +1450,12 @@ func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b
 			end = false
 			return bstore.StopForEach
 		}
+
+		if _, ok := v.threadIDs[m.ThreadID]; ok {
+			// Message was already returned as part of a thread.
+			return nil
+		}
+
 		var pm *ParsedMessage
 		if m.ID == page.DestMessageID || page.DestMessageID == 0 && have == 0 && page.AnchorMessageID == 0 {
 			found = true
@@ -1415,12 +1465,60 @@ func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b
 			}
 			pm = &xpm
 		}
+
 		mi, err := messageItem(log, m, &state)
 		if err != nil {
 			return fmt.Errorf("making messageitem for message %d: %v", m.ID, err)
 		}
-		mrc <- msgResp{mi: mi, pm: pm}
+		mil := []MessageItem{mi}
+		if query.Threading != ThreadOff {
+			more, xpm, err := gatherThread(log, tx, acc, v, m, page.DestMessageID)
+			if err != nil {
+				return fmt.Errorf("gathering thread messages for id %d, thread %d: %v", m.ID, m.ThreadID, err)
+			}
+			if xpm != nil {
+				pm = xpm
+				found = true
+			}
+			mil = append(mil, more...)
+			v.threadIDs[m.ThreadID] = struct{}{}
+
+			// Calculate how many messages the frontend is going to show, and only count those as returned.
+			collapsed := map[int64]bool{}
+			for _, mi := range mil {
+				collapsed[mi.Message.ID] = mi.Message.ThreadCollapsed
+			}
+			unread := map[int64]bool{} // Propagated to thread root.
+			if query.Threading == ThreadUnread {
+				for _, mi := range mil {
+					m := mi.Message
+					if m.Seen {
+						continue
+					}
+					unread[m.ID] = true
+					for _, id := range m.ThreadParentIDs {
+						unread[id] = true
+					}
+				}
+			}
+			for _, mi := range mil {
+				m := mi.Message
+				threadRoot := true
+				rootID := m.ID
+				for _, id := range m.ThreadParentIDs {
+					if _, ok := collapsed[id]; ok {
+						threadRoot = false
+						rootID = id
+					}
+				}
+				if threadRoot || (query.Threading == ThreadOn && !collapsed[rootID] || query.Threading == ThreadUnread && unread[rootID]) {
 					have++
+				}
+			}
+		} else {
+			have++
+		}
+		mrc <- msgResp{mil: mil, pm: pm}
 		return nil
 	})
 	// Check for an error in one of the filters again. Check in ForEach would not
@@ -1437,6 +1535,57 @@ func queryMessages(ctx context.Context, log *mlog.Log, acc *store.Account, tx *b
 	}
 }
 
+func gatherThread(log *mlog.Log, tx *bstore.Tx, acc *store.Account, v view, m store.Message, destMessageID int64) ([]MessageItem, *ParsedMessage, error) {
+	if m.ThreadID == 0 {
+		// If we would continue, FilterNonzero would fail because there are no non-zero fields.
+		return nil, nil, fmt.Errorf("message has threadid 0, account is probably still being upgraded, try turning threading off until the upgrade is done")
+	}
+
+	// Fetch other messages for this thread.
+	qt := bstore.QueryTx[store.Message](tx)
+	qt.FilterNonzero(store.Message{ThreadID: m.ThreadID})
+	qt.FilterEqual("Expunged", false)
+	qt.FilterNotEqual("ID", m.ID)
+	tml, err := qt.List()
+	if err != nil {
+		return nil, nil, fmt.Errorf("listing other messages in thread for message %d, thread %d: %v", m.ID, m.ThreadID, err)
+	}
+
+	var mil []MessageItem
+	var pm *ParsedMessage
+	for _, tm := range tml {
+		err := func() error {
+			xstate := msgState{acc: acc}
+			defer xstate.clear()
+
+			mi, err := messageItem(log, tm, &xstate)
+			if err != nil {
+				return fmt.Errorf("making messageitem for message %d, for thread %d: %v", tm.ID, m.ThreadID, err)
+			}
+			mi.MatchQuery, err = v.matches(log, acc, false, tm.ID, tm.MailboxID, tm.UID, tm.Flags, tm.Keywords, func(int64, int64, store.UID) (store.Message, error) {
+				return tm, nil
+			})
+			if err != nil {
+				return fmt.Errorf("matching thread message %d against view query: %v", tm.ID, err)
+			}
+			mil = append(mil, mi)
+
+			if tm.ID == destMessageID {
+				xpm, err := parsedMessage(log, tm, &xstate, true, false)
+				if err != nil {
+					return fmt.Errorf("parsing thread message %d: %v", tm.ID, err)
+				}
+				pm = &xpm
+			}
+			return nil
+		}()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	return mil, pm, nil
+}
+
 // While checking the filters on a message, we may need to get more message
 // details as each filter passes. We check the filters that need the basic
 // information first, and load and cache more details for the next filters.

webmail/view_test.go

@@ -50,8 +50,10 @@ func TestView(t *testing.T) {
 		listsGoNutsMinimal = &testmsg{"Lists/Go/Nuts", store.Flags{}, nil, msgMinimal, zerom, 0}
 		trashMinimal       = &testmsg{"Trash", store.Flags{}, nil, msgMinimal, zerom, 0}
 		junkMinimal        = &testmsg{"Trash", store.Flags{}, nil, msgMinimal, zerom, 0}
+		trashAlt           = &testmsg{"Trash", store.Flags{}, nil, msgAlt, zerom, 0}
+		inboxAltReply      = &testmsg{"Inbox", store.Flags{}, nil, msgAltReply, zerom, 0}
 	)
-	var testmsgs = []*testmsg{inboxMinimal, inboxFlags, listsMinimal, listsGoNutsMinimal, trashMinimal, junkMinimal}
+	var testmsgs = []*testmsg{inboxMinimal, inboxFlags, listsMinimal, listsGoNutsMinimal, trashMinimal, junkMinimal, trashAlt, inboxAltReply}
 	for _, tm := range testmsgs {
 		tdeliver(t, acc, tm)
 	}
@@ -116,10 +118,10 @@ func TestView(t *testing.T) {
 	evr.Get("start", &start)
 	var viewMsgs EventViewMsgs
 	evr.Get("viewMsgs", &viewMsgs)
-	tcompare(t, len(viewMsgs.MessageItems), 2)
+	tcompare(t, len(viewMsgs.MessageItems), 3)
 	tcompare(t, viewMsgs.ViewEnd, true)
 
-	var inbox, archive, lists store.Mailbox
+	var inbox, archive, lists, trash store.Mailbox
 	for _, mb := range start.Mailboxes {
 		if mb.Archive {
 			archive = mb
@@ -127,6 +129,8 @@ func TestView(t *testing.T) {
 			inbox = mb
 		} else if mb.Name == "Lists" {
 			lists = mb
+		} else if mb.Name == "Trash" {
+			trash = mb
 		}
 	}
 
@@ -161,7 +165,7 @@ func TestView(t *testing.T) {
 	testConn(api.Token(ctx), "&waitMinMsec=1&waitMaxMsec=2", waitReq, func(start EventStart, evr eventReader) {
 		var vm EventViewMsgs
 		evr.Get("viewMsgs", &vm)
-		tcompare(t, len(vm.MessageItems), 2)
+		tcompare(t, len(vm.MessageItems), 3)
 	})
 
 	// Connection with DestMessageID.
@@ -174,7 +178,7 @@ func TestView(t *testing.T) {
 	testConn(tokens[len(tokens)-3], "", destMsgReq, func(start EventStart, evr eventReader) {
 		var vm EventViewMsgs
 		evr.Get("viewMsgs", &vm)
-		tcompare(t, len(vm.MessageItems), 2)
+		tcompare(t, len(vm.MessageItems), 3)
 		tcompare(t, vm.ParsedMessage.ID, destMsgReq.Page.DestMessageID)
 	})
 	// todo: destmessageid past count, needs large mailbox
@@ -189,7 +193,7 @@ func TestView(t *testing.T) {
 	testConn(api.Token(ctx), "", badDestMsgReq, func(start EventStart, evr eventReader) {
 		var vm EventViewMsgs
 		evr.Get("viewMsgs", &vm)
-		tcompare(t, len(vm.MessageItems), 2)
+		tcompare(t, len(vm.MessageItems), 3)
 	})
 
 	// Connection with missing unknown AnchorMessageID, resets view.
@@ -205,7 +209,7 @@ func TestView(t *testing.T) {
 
 		var vm EventViewMsgs
 		evr.Get("viewMsgs", &vm)
-		tcompare(t, len(vm.MessageItems), 2)
+		tcompare(t, len(vm.MessageItems), 3)
 	})
 
 	// Connection that starts with a filter, without mailbox.
@@ -219,12 +223,12 @@ func TestView(t *testing.T) {
 		var vm EventViewMsgs
 		evr.Get("viewMsgs", &vm)
 		tcompare(t, len(vm.MessageItems), 1)
-		tcompare(t, vm.MessageItems[0].Message.ID, inboxFlags.ID)
+		tcompare(t, vm.MessageItems[0][0].Message.ID, inboxFlags.ID)
 	})
 
 	// Paginate from previous last element. There is nothing new.
 	var viewID int64 = 1
-	api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}}, Page: Page{Count: 10, AnchorMessageID: viewMsgs.MessageItems[len(viewMsgs.MessageItems)-1].Message.ID}})
+	api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}}, Page: Page{Count: 10, AnchorMessageID: viewMsgs.MessageItems[len(viewMsgs.MessageItems)-1][0].Message.ID}})
 	evr.Get("viewMsgs", &viewMsgs)
 	tcompare(t, len(viewMsgs.MessageItems), 0)
 
@@ -235,6 +239,36 @@ func TestView(t *testing.T) {
 	tcompare(t, len(viewMsgs.MessageItems), 0)
 	tcompare(t, viewMsgs.ViewEnd, true)
 
+	threadlen := func(mil [][]MessageItem) int {
+		n := 0
+		for _, l := range mil {
+			n += len(l)
+		}
+		return n
+	}
+
+	// Request with threading, should also include parent message from Trash mailbox (trashAlt).
+	viewID++
+	api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}, Threading: "unread"}, Page: Page{Count: 10}})
+	evr.Get("viewMsgs", &viewMsgs)
+	tcompare(t, len(viewMsgs.MessageItems), 3)
+	tcompare(t, threadlen(viewMsgs.MessageItems), 3+1)
+	tcompare(t, viewMsgs.ViewEnd, true)
+	// And likewise when querying Trash, should also include child message in Inbox (inboxAltReply).
+	viewID++
+	api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: trash.ID}, Threading: "on"}, Page: Page{Count: 10}})
+	evr.Get("viewMsgs", &viewMsgs)
+	tcompare(t, len(viewMsgs.MessageItems), 3)
+	tcompare(t, threadlen(viewMsgs.MessageItems), 3+1)
+	tcompare(t, viewMsgs.ViewEnd, true)
+	// Without threading, the inbox has just 3 messages.
+	viewID++
+	api.Request(ctx, Request{ID: 1, SSEID: start.SSEID, ViewID: viewID, Query: Query{Filter: Filter{MailboxID: inbox.ID}, Threading: "off"}, Page: Page{Count: 10}})
+	evr.Get("viewMsgs", &viewMsgs)
+	tcompare(t, len(viewMsgs.MessageItems), 3)
+	tcompare(t, threadlen(viewMsgs.MessageItems), 3)
+	tcompare(t, viewMsgs.ViewEnd, true)
+
 	testFilter := func(orderAsc bool, f Filter, nf NotFilter, expIDs []int64) {
 		t.Helper()
 		viewID++
@@ -242,7 +276,7 @@ func TestView(t *testing.T) {
 		evr.Get("viewMsgs", &viewMsgs)
 		ids := make([]int64, len(viewMsgs.MessageItems))
 		for i, mi := range viewMsgs.MessageItems {
-			ids[i] = mi.Message.ID
+			ids[i] = mi[0].Message.ID
 		}
 		tcompare(t, ids, expIDs)
 		tcompare(t, viewMsgs.ViewEnd, true)
@@ -252,30 +286,30 @@ func TestView(t *testing.T) {
 	var znf NotFilter
 	testFilter(false, Filter{MailboxID: lists.ID, MailboxChildrenIncluded: true}, znf, []int64{listsGoNutsMinimal.ID, listsMinimal.ID})              // Mailbox and sub mailbox.
 	testFilter(true, Filter{MailboxID: lists.ID, MailboxChildrenIncluded: true}, znf, []int64{listsMinimal.ID, listsGoNutsMinimal.ID})               // Oldest first first.
-	testFilter(false, Filter{MailboxID: -1}, znf, []int64{listsGoNutsMinimal.ID, listsMinimal.ID, inboxFlags.ID, inboxMinimal.ID})      // All except trash/junk/rejects.
+	testFilter(false, Filter{MailboxID: -1}, znf, []int64{inboxAltReply.ID, listsGoNutsMinimal.ID, listsMinimal.ID, inboxFlags.ID, inboxMinimal.ID}) // All except trash/junk/rejects.
 	testFilter(false, Filter{Labels: []string{`\seen`}}, znf, []int64{inboxFlags.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`\seen`}}, []int64{inboxMinimal.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`\seen`}}, []int64{inboxAltReply.ID, inboxMinimal.ID})
 	testFilter(false, Filter{Labels: []string{`testlabel`}}, znf, []int64{inboxFlags.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`testlabel`}}, []int64{inboxMinimal.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Labels: []string{`testlabel`}}, []int64{inboxAltReply.ID, inboxMinimal.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID, Oldest: &inboxFlags.m.Received}, znf, []int64{inboxFlags.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID, Oldest: &inboxFlags.m.Received}, znf, []int64{inboxAltReply.ID, inboxFlags.ID})
 	testFilter(false, Filter{MailboxID: inbox.ID, Newest: &inboxMinimal.m.Received}, znf, []int64{inboxMinimal.ID})
 	testFilter(false, Filter{MailboxID: inbox.ID, SizeMin: inboxFlags.m.Size}, znf, []int64{inboxFlags.ID})
 	testFilter(false, Filter{MailboxID: inbox.ID, SizeMax: inboxMinimal.m.Size}, znf, []int64{inboxMinimal.ID})
 	testFilter(false, Filter{From: []string{"mjl+altrel@mox.example"}}, znf, []int64{inboxFlags.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{From: []string{"mjl+altrel@mox.example"}}, []int64{inboxMinimal.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{From: []string{"mjl+altrel@mox.example"}}, []int64{inboxAltReply.ID, inboxMinimal.ID})
 	testFilter(false, Filter{To: []string{"mox+altrel@other.example"}}, znf, []int64{inboxFlags.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{To: []string{"mox+altrel@other.example"}}, []int64{inboxMinimal.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{To: []string{"mox+altrel@other.example"}}, []int64{inboxAltReply.ID, inboxMinimal.ID})
 	testFilter(false, Filter{From: []string{"mjl+altrel@mox.example", "bogus"}}, znf, []int64{})
 	testFilter(false, Filter{To: []string{"mox+altrel@other.example", "bogus"}}, znf, []int64{})
 	testFilter(false, Filter{Subject: []string{"test", "alt", "rel"}}, znf, []int64{inboxFlags.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Subject: []string{"alt"}}, []int64{inboxMinimal.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Subject: []string{"alt"}}, []int64{inboxAltReply.ID, inboxMinimal.ID})
 	testFilter(false, Filter{MailboxID: inbox.ID, Words: []string{"the text body", "body", "the "}}, znf, []int64{inboxFlags.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Words: []string{"the text body"}}, []int64{inboxMinimal.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Words: []string{"the text body"}}, []int64{inboxAltReply.ID, inboxMinimal.ID})
 	testFilter(false, Filter{Headers: [][2]string{{"X-Special", ""}}}, znf, []int64{inboxFlags.ID})
 	testFilter(false, Filter{Headers: [][2]string{{"X-Special", "testing"}}}, znf, []int64{inboxFlags.ID})
 	testFilter(false, Filter{Headers: [][2]string{{"X-Special", "other"}}}, znf, []int64{})
 	testFilter(false, Filter{Attachments: AttachmentImage}, znf, []int64{inboxFlags.ID})
-	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Attachments: AttachmentImage}, []int64{inboxMinimal.ID})
+	testFilter(false, Filter{MailboxID: inbox.ID}, NotFilter{Attachments: AttachmentImage}, []int64{inboxAltReply.ID, inboxMinimal.ID})
 
 	// Test changes.
 	getChanges := func(changes ...any) {
@@ -341,13 +375,13 @@ func TestView(t *testing.T) {
 	var chmbcounts ChangeMailboxCounts
 	getChanges(&chmsgadd, &chmbcounts)
 	tcompare(t, chmsgadd.ChangeAddUID.MailboxID, inbox.ID)
-	tcompare(t, chmsgadd.MessageItem.Message.ID, inboxNew.ID)
+	tcompare(t, chmsgadd.MessageItems[0].Message.ID, inboxNew.ID)
 	chmbcounts.Size = 0
 	tcompare(t, chmbcounts, ChangeMailboxCounts{
 		ChangeMailboxCounts: store.ChangeMailboxCounts{
 			MailboxID:     inbox.ID,
 			MailboxName:   inbox.Name,
-			MailboxCounts: store.MailboxCounts{Total: 3, Unread: 2, Unseen: 2},
+			MailboxCounts: store.MailboxCounts{Total: 4, Unread: 3, Unseen: 3},
 		},
 	})
 
@@ -369,7 +403,7 @@ func TestView(t *testing.T) {
 		ChangeMailboxCounts: store.ChangeMailboxCounts{
 			MailboxID:     inbox.ID,
 			MailboxName:   inbox.Name,
-			MailboxCounts: store.MailboxCounts{Total: 3, Unread: 1, Unseen: 1},
+			MailboxCounts: store.MailboxCounts{Total: 4, Unread: 2, Unseen: 2},
 		},
 	})
 
@@ -384,10 +418,40 @@ func TestView(t *testing.T) {
 		ChangeMailboxCounts: store.ChangeMailboxCounts{
 			MailboxID:     inbox.ID,
 			MailboxName:   inbox.Name,
-			MailboxCounts: store.MailboxCounts{Total: 1},
+			MailboxCounts: store.MailboxCounts{Total: 2, Unread: 1, Unseen: 1},
 		},
 	})
 
+	// ChangeMsgThread
+	api.ThreadCollapse(ctx, []int64{inboxAltReply.ID}, true)
+	var chmsgthread ChangeMsgThread
+	getChanges(&chmsgthread)
+	tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: false, Collapsed: true})
+
+	// Now collapsing the thread root, the child is already collapsed so no change.
+	api.ThreadCollapse(ctx, []int64{trashAlt.ID}, true)
+	getChanges(&chmsgthread)
+	tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: false, Collapsed: true})
+
+	// Expand thread root, including change for child.
+	api.ThreadCollapse(ctx, []int64{trashAlt.ID}, false)
+	var chmsgthread2 ChangeMsgThread
+	getChanges(&chmsgthread, &chmsgthread2)
+	tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: false, Collapsed: false})
+	tcompare(t, chmsgthread2.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: false, Collapsed: false})
+
+	// Mute thread, including child, also collapses.
+	api.ThreadMute(ctx, []int64{trashAlt.ID}, true)
+	getChanges(&chmsgthread, &chmsgthread2)
+	tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: true, Collapsed: true})
+	tcompare(t, chmsgthread2.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: true, Collapsed: true})
+
+	// And unmute Mute thread, including child. Messages are not expanded.
+	api.ThreadMute(ctx, []int64{trashAlt.ID}, false)
+	getChanges(&chmsgthread, &chmsgthread2)
+	tcompare(t, chmsgthread.ChangeThread, store.ChangeThread{MessageIDs: []int64{trashAlt.ID}, Muted: false, Collapsed: true})
+	tcompare(t, chmsgthread2.ChangeThread, store.ChangeThread{MessageIDs: []int64{inboxAltReply.ID}, Muted: false, Collapsed: true})
+
 	// todo: check move operations and their changes, e.g. MailboxDelete, MailboxEmpty, MessageRemove.
 }
 

webmail/webmail.go

@@ -758,7 +758,7 @@ func handle(apiHandler http.Handler, w http.ResponseWriter, r *http.Request) {
 
 		m.MsgPrefix = nil
 		m.ParsedBuf = nil
-		mi := MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine}
+		mi := MessageItem{m, pm.envelope, pm.attachments, pm.isSigned, pm.isEncrypted, pm.firstLine, false}
 		mijson, err := json.Marshal(mi)
 		xcheckf(ctx, err, "marshal messageitem")
 

webmail/webmail.html

@@ -34,20 +34,24 @@ iframe { border: 0; }
 .msgitemcell { padding: 2px 4px; }
 /* note: we assign widths to .msgitemflags, .msgitemfrom, .msgitemsubject, .msgitemage, and offsets through a stylesheet created in js */
 .msgitemage { text-align: right; }
+.msgitemfrom { position: relative; }
 .msgitemfromtext { white-space: nowrap; overflow: hidden; }
+.msgitemfromthreadbar { position: absolute; border-right: 2px solid #666; right: 0; top: 0; bottom: 0; /* top or bottom set with inline style for first & last */ }
 .msgitemsubjecttext { white-space: nowrap; overflow: hidden; }
 .msgitemsubjectsnippet { font-weight: normal; color: #666; }
-.msgitemmailbox { background-color: #999; color: white; border: 1px solid #777; padding: 0 .15em; margin-left: .15em; border-radius: .15em; font-weight: normal; font-size: .9em; white-space: nowrap; }
+.msgitemmailbox { background: #999; color: white; border: 1px solid #777; padding: 0 .15em; margin-left: .15em; border-radius: .15em; font-weight: normal; font-size: .9em; white-space: nowrap; }
+.msgitemmailbox.msgitemmailboxcollapsed { background: #eee; color: #333; }
 .msgitemidentity { background-color: #999; color: white; border: 1px solid #777; padding: 0 .15em; margin-left: .15em; border-radius: .15em; font-weight: normal; font-size: .9em; white-space: nowrap; }
 
 .topbar, .mailboxesbar { background-color: #fdfdf1; }
 .msglist { background-color: #f5ffff; }
 table.search td { padding: .25em; }
 .keyword { background-color: gold; color: black; border: 1px solid #8c7600; padding: 0 .15em; border-radius: .15em; font-weight: normal; font-size: .9em; margin: 0 .15em; white-space: nowrap; }
+.keyword.keywordcollapsed { background-color: #ffeb7e; color: #333; }
 .mailbox { padding: .15em .25em; }
 .mailboxitem { cursor: pointer; border-radius: .15em; }
-.mailboxitem.dropping { background-color: gold !important; }
+.mailboxitem.dropping { background: gold !important; }
-.mailboxitem:hover { background-color: #eee; }
+.mailboxitem:hover { background: #eee; }
 .mailboxitem.active { background: linear-gradient(135deg, #ffc7ab 0%, #ffdeab 100%); }
 .mailboxhoveronly { visibility: hidden; }
 .mailboxitem:hover .mailboxhoveronly, .mailboxitem:focus .mailboxhoveronly { visibility: visible; }
@@ -59,6 +63,7 @@ table.search td { padding: .25em; }
 .msgitem.active { background: linear-gradient(135deg, #8bc8ff 0%, #8ee5ff 100%); }
 .msgitemsubject { position: relative; }
 .msgitemflag { margin-right: 1px; font-weight: normal; font-size: .9em; }
+.msgitemflag.msgitemflagcollapsed { color: #666; }
 .quoted1 { color: #03828f; }
 .quoted2 { color: #c7445c; }
 .quoted3 { color: #417c10; }

webmail/webmail_test.go

@@ -45,6 +45,7 @@ type Message struct {
 	From, To, Cc, Bcc, Subject, MessageID string
 	Headers                               [][2]string
 	Date                                  time.Time
+	References                            string
 	Part                                  Part
 }
 
@@ -84,6 +85,7 @@ func (m Message) Marshal(t *testing.T) []byte {
 	header("Subject", m.Subject)
 	header("Message-Id", m.MessageID)
 	header("Date", m.Date.Format(message.RFC5322Z))
+	header("References", m.References)
 	for _, t := range m.Headers {
 		header(t[0], t[1])
 	}
@@ -184,6 +186,7 @@ var (
 		From:      "mjl <mjl@mox.example>",
 		To:        "mox <mox@other.example>",
 		Subject:   "test",
+		MessageID: "<alt@localhost>",
 		Headers:   [][2]string{{"In-Reply-To", "<previous@host.example>"}},
 		Part: Part{
 			Type: "multipart/alternative",
@@ -193,6 +196,11 @@ var (
 			},
 		},
 	}
+	msgAltReply = Message{
+		Subject:    "Re: test",
+		References: "<alt@localhost>",
+		Part:       Part{Type: "text/plain", Content: "reply to alt"},
+	}
 	msgAltRel = Message{
 		From:    "mjl <mjl+altrel@mox.example>",
 		To:      "mox <mox+altrel@other.example>",