add condstore & qresync imap extensions

for conditional storing and quick resynchronisation (not sure if mail clients are actually using it that).

each message now has a "modseq". it is increased for each change. with
condstore, imap clients can request changes since a certain modseq. that
already allows quickly finding changes since a previous connection. condstore
also allows storing (e.g. setting new message flags) only when the modseq of a
message hasn't changed.

qresync should make it fast for clients to get a full list of changed messages
for a mailbox, including removals.

we now also keep basic metadata of messages that have been removed (expunged).
just enough (uid, modseq) to tell client that the messages have been removed.
this does mean we have to be careful when querying messages from the database.
we must now often filter the expunged messages out.

we also keep "createseq", the modseq when a message was created. this will be
useful for the jmap implementation.

README.md

@@ -108,7 +108,6 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili
 
 ## Roadmap
 
-- IMAP CONDSTORE and QRESYNC extensions
 - Webmail
 - IMAP THREAD extension
 - DANE and DNSSEC

backup.go

@@ -424,7 +424,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
 		tmMsgs := time.Now()
 		seen := map[string]struct{}{}
 		var nlinked, ncopied int
-		err = bstore.QueryDB[store.Message](ctx, db).ForEach(func(m store.Message) error {
+		err = bstore.QueryDB[store.Message](ctx, db).FilterEqual("Expunged", false).ForEach(func(m store.Message) error {
 			mp := store.MessagePath(m.ID)
 			seen[mp] = struct{}{}
 			amp := filepath.Join("accounts", acc.Name, "msg", mp)

ctl.go

@@ -649,6 +649,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
 			// Read through messages with junk or nonjunk flag set, and train them.
 			var total, trained int
 			q := bstore.QueryDB[store.Message](ctx, acc.DB)
+			q.FilterEqual("Expunged", false)
 			err = q.ForEach(func(m store.Message) error {
 				total++
 				ok, err := acc.TrainMessage(ctx, ctl.log, jf, m)

http/account_test.go

@@ -139,7 +139,7 @@ func TestAccount(t *testing.T) {
 
 	// Check there are messages, with the right flags.
 	acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {
-		_, err = bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get()
+		_, err = bstore.QueryTx[store.Message](tx).FilterEqual("Expunged", false).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get()
 		tcheck(t, err, `fetching message with keywords "other" and "test"`)
 
 		mb, err := acc.MailboxFind(tx, "importtest")
@@ -152,7 +152,7 @@ func TestAccount(t *testing.T) {
 			t.Fatalf(`expected mailbox keywords "other" and "test", got %v`, mb.Keywords)
 		}
 
-		n, err := bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "custom").Count()
+		n, err := bstore.QueryTx[store.Message](tx).FilterEqual("Expunged", false).FilterIn("Keywords", "custom").Count()
 		tcheck(t, err, `fetching message with keyword "custom"`)
 		if n != 2 {
 			t.Fatalf(`got %d messages with keyword "custom", expected 2`, n)

http/import.go

@@ -377,6 +377,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 	// finally at the end as a closing statement.
 	var prevMailbox string
 
+	var modseq store.ModSeq // Assigned on first message, used for all messages.
+
 	trainMessage := func(m *store.Message, p message.Part, pos string) {
 		words, err := jf.ParseMessage(p)
 		if err != nil {
@@ -478,6 +480,14 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 		m.MailboxID = mb.ID
 		m.MailboxOrigID = mb.ID
 
+		if modseq == 0 {
+			var err error
+			modseq, err = acc.NextModSeq(tx)
+			ximportcheckf(err, "assigning next modseq")
+		}
+		m.CreateSeq = modseq
+		m.ModSeq = modseq
+
 		if len(m.Keywords) > 0 {
 			if destMailboxKeywords[mb.ID] == nil {
 				destMailboxKeywords[mb.ID] = map[string]bool{}
@@ -519,7 +529,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 			return
 		}
 		deliveredIDs = append(deliveredIDs, m.ID)
-		changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords})
+		changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Flags: m.Flags, Keywords: m.Keywords})
 		messages[mb.Name]++
 		if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name {
 			prevMailbox = mb.Name
@@ -726,7 +736,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 					}
 					err = tx.Update(&m)
 					ximportcheckf(err, "updating message after flag update")
-					changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags, Keywords: m.Keywords})
+					changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Mask: flags, Flags: flags, Keywords: m.Keywords})
 				}
 				delete(mailboxMissingKeywordMessages, mailbox)
 			} else {

imapclient/parse.go

@@ -119,6 +119,7 @@ var knownCodes = stringMap(
 	"ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE", "UIDNOTSTICKY", "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED", "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE", "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT", "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "NOTSAVED", "HASCHILDREN", "CLOSED", "UNKNOWN-CTE",
 	// With parameters.
 	"BADCHARSET", "CAPABILITY", "PERMANENTFLAGS", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "APPENDUID", "COPYUID",
+	"HIGHESTMODSEQ", "MODIFIED",
 )
 
 func stringMap(l ...string) map[string]struct{} {
@@ -202,6 +203,13 @@ func (c *Conn) xrespCode() (string, CodeArg) {
 		c.xspace()
 		to := c.xuidset()
 		codeArg = CodeCopyUID{destUIDValidity, from, to}
+	case "HIGHESTMODSEQ":
+		c.xspace()
+		codeArg = CodeHighestModSeq(c.xint64())
+	case "MODIFIED":
+		c.xspace()
+		modified := c.xuidset()
+		codeArg = CodeModified(NumSet{Ranges: modified})
 	}
 	return W, codeArg
 }
@@ -248,7 +256,7 @@ func (c *Conn) xint32() int32 {
 
 func (c *Conn) xint64() int64 {
 	s := c.xdigits()
-	num, err := strconv.ParseInt(s, 10, 64)
+	num, err := strconv.ParseInt(s, 10, 63)
 	c.xcheckf(err, "parsing int64")
 	return num
 }
@@ -386,6 +394,8 @@ func (c *Conn) xuntagged() Untagged {
 				} else {
 					num = c.xint64()
 				}
+			case "HIGHESTMODSEQ":
+				num = c.xint64()
 			default:
 				c.xerrorf("status: unknown attribute %q", s)
 			}
@@ -415,6 +425,15 @@ func (c *Conn) xuntagged() Untagged {
 		c.xneedDisabled("untagged SEARCH response", CapIMAP4rev2)
 		var nums []uint32
 		for c.take(' ') {
+			// ../rfc/7162:2557
+			if c.take('(') {
+				c.xtake("MODSEQ")
+				c.xspace()
+				modseq := c.xint64()
+				c.xtake(")")
+				c.xcrlf()
+				return UntaggedSearchModSeq{nums, modseq}
+			}
 			nums = append(nums, c.xnzuint32())
 		}
 		r := UntaggedSearch(nums)
@@ -456,6 +475,20 @@ func (c *Conn) xuntagged() Untagged {
 		c.xcrlf()
 		return UntaggedID(params)
 
+	// ../rfc/7162:2623
+	case "VANISHED":
+		c.xspace()
+		var earlier bool
+		if c.take('(') {
+			c.xtake("EARLIER")
+			c.xtake(")")
+			c.xspace()
+			earlier = true
+		}
+		uids := c.xuidset()
+		c.xcrlf()
+		return UntaggedVanished{earlier, NumSet{Ranges: uids}}
+
 	default:
 		v, err := strconv.ParseUint(w, 10, 32)
 		if err == nil {
@@ -605,6 +638,14 @@ func (c *Conn) xmsgatt1() FetchAttr {
 	case "UID":
 		c.xspace()
 		return FetchUID(c.xuint32())
+
+	case "MODSEQ":
+		// ../rfc/7162:2488
+		c.xspace()
+		c.xtake("(")
+		modseq := c.xint64()
+		c.xtake(")")
+		return FetchModSeq(modseq)
 	}
 	c.xerrorf("unknown fetch attribute %q", f)
 	panic("not reached")
@@ -963,7 +1004,7 @@ func (c *Conn) xtaggedExtVal() TaggedExtVal {
 		return TaggedExtVal{SeqSet: &ss}
 	}
 	s := c.xdigits()
-	num, err := strconv.ParseInt(s, 10, 64)
+	num, err := strconv.ParseInt(s, 10, 63)
 	c.xcheckf(err, "parsing int")
 	if !c.peek(':') && !c.peek(',') {
 		// not a larger sequence-set
@@ -1146,6 +1187,11 @@ func (c *Conn) xesearchResponse() (r UntaggedEsearch) {
 			num := c.xuint32()
 			r.Count = &num
 
+		// ../rfc/7162:1211 ../rfc/4731:273
+		case "MODSEQ":
+			c.xspace()
+			r.ModSeq = c.xint64()
+
 		default:
 			// Validate ../rfc/9051:7090
 			for i, b := range []byte(w) {

imapclient/protocol.go

@@ -1,6 +1,7 @@
 package imapclient
 
 import (
+	"bufio"
 	"fmt"
 	"strings"
 )
@@ -134,6 +135,20 @@ func (c CodeCopyUID) CodeString() string {
 	return fmt.Sprintf("COPYUID %d %s %s", c.DestUIDValidity, str(c.From), str(c.To))
 }
 
+// For CONDSTORE.
+type CodeModified NumSet
+
+func (c CodeModified) CodeString() string {
+	return fmt.Sprintf("MODIFIED %s", NumSet(c).String())
+}
+
+// For CONDSTORE.
+type CodeHighestModSeq int64
+
+func (c CodeHighestModSeq) CodeString() string {
+	return fmt.Sprintf("HIGHESTMODSEQ %d", c)
+}
+
 // RespText represents a response line minus the leading tag.
 type RespText struct {
 	Code    string  // The first word between [] after the status.
@@ -201,6 +216,12 @@ type UntaggedFetch struct {
 	Attrs []FetchAttr
 }
 type UntaggedSearch []uint32
+
+// ../rfc/7162:1101
+type UntaggedSearchModSeq struct {
+	Nums   []uint32
+	ModSeq int64
+}
 type UntaggedStatus struct {
 	Mailbox string
 	Attrs   map[string]int64 // Upper case status attributes. ../rfc/9051:7059
@@ -224,9 +245,16 @@ type UntaggedEsearch struct {
 	Max        uint32
 	All        NumSet
 	Count      *uint32
+	ModSeq     int64
 	Exts       []EsearchDataExt
 }
 
+// UntaggedVanished is used in QRESYNC to send UIDs that have been removed.
+type UntaggedVanished struct {
+	Earlier bool
+	UIDs    NumSet
+}
+
 // ../rfc/2971:184
 
 type UntaggedID map[string]string
@@ -278,6 +306,13 @@ func (ns NumSet) String() string {
 	return r
 }
 
+func ParseNumSet(s string) (ns NumSet, rerr error) {
+	c := Conn{r: bufio.NewReader(strings.NewReader(s))}
+	defer c.recover(&rerr)
+	ns = c.xsequenceSet()
+	return
+}
+
 // NumRange is a single number or range.
 type NumRange struct {
 	First uint32  // 0 for "*".
@@ -450,3 +485,8 @@ func (f FetchBinarySize) Attr() string { return f.RespAttr }
 type FetchUID uint32
 
 func (f FetchUID) Attr() string { return "UID" }
+
+// "MODSEQ" fetch response.
+type FetchModSeq int64
+
+func (f FetchModSeq) Attr() string { return "MODSEQ" }

imapserver/condstore_test.go

@@ -0,0 +1,702 @@
+package imapserver
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/mjl-/bstore"
+
+	"github.com/mjl-/mox/imapclient"
+	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/store"
+)
+
+func TestCondstore(t *testing.T) {
+	testCondstoreQresync(t, false)
+}
+
+func TestQresync(t *testing.T) {
+	testCondstoreQresync(t, true)
+}
+
+func testCondstoreQresync(t *testing.T, qresync bool) {
+	defer mockUIDValidity()()
+	tc := start(t)
+	defer tc.close()
+
+	// todo: check whether marking \seen will cause modseq to be returned in case of qresync.
+
+	// Check basic requirements of CONDSTORE.
+
+	capability := "Condstore"
+	if qresync {
+		capability = "Qresync"
+	}
+
+	tc.client.Login("mjl@mox.example", "testtest")
+	tc.client.Enable(capability)
+	tc.transactf("ok", "Select inbox")
+	tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(1), More: "x"}})
+
+	// First some tests without any messages.
+
+	tc.transactf("ok", "Status inbox (Highestmodseq)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 1}})
+
+	// No messages, no matches.
+	tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 12345)")
+	tc.xuntagged()
+
+	// Also no messages with modseq 1, which we internally turn into modseq 0.
+	tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1)")
+	tc.xuntagged()
+
+	// Also try with modseq attribute.
+	tc.transactf("ok", "Uid Fetch 1:* (Flags Modseq) (Changedsince 1)")
+	tc.xuntagged()
+
+	// Search with modseq search criteria.
+	tc.transactf("ok", "Search Modseq 0") // Zero is valid, matches all.
+	tc.xsearch()
+
+	tc.transactf("ok", "Search Modseq 1") // Converted to zero internally.
+	tc.xsearch()
+
+	tc.transactf("ok", "Search Modseq 12345")
+	tc.xsearch()
+
+	tc.transactf("ok", `Search Modseq "/Flags/\\Draft" All 12345`)
+	tc.xsearch()
+
+	tc.transactf("ok", `Search Or Modseq 12345 Modseq 54321`)
+	tc.xsearch()
+
+	// esearch
+	tc.transactf("ok", "Search Return (All) Modseq 123")
+	tc.xesearch(imapclient.UntaggedEsearch{})
+
+	// Now we add, delete, expunge, modify some message flags and check if the
+	// responses are correct. We check in both a condstore-enabled and one without that
+	// we get the correct notifications.
+
+	// First we add 3 messages as if they were added before we implemented CONDSTORE.
+	// Later on, we'll update the second, and delete the third, leaving the first
+	// unmodified. Those messages have modseq 0 in the database. We use append for
+	// convenience, then adjust the records in the database.
+	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	_, err := bstore.QueryDB[store.Message](ctxbg, tc.account.DB).UpdateFields(map[string]any{
+		"ModSeq":    0,
+		"CreateSeq": 0,
+	})
+	tcheck(t, err, "clearing modseq from messages")
+	err = tc.account.DB.Update(ctxbg, &store.SyncState{ID: 1, LastModSeq: 1})
+	tcheck(t, err, "resetting modseq state")
+
+	tc.client.Create("otherbox")
+
+	// tc2 is a client without condstore, so no modseq responses.
+	tc2 := startNoSwitchboard(t)
+	defer tc2.close()
+	tc2.client.Login("mjl@mox.example", "testtest")
+	tc2.client.Select("inbox")
+
+	// tc2 is a client with condstore, so with modseq responses.
+	tc3 := startNoSwitchboard(t)
+	defer tc3.close()
+	tc3.client.Login("mjl@mox.example", "testtest")
+	tc3.client.Enable(capability)
+	tc3.client.Select("inbox")
+
+	var clientModseq int64 = 1 // We track the client-side modseq for inbox. Not a store.ModSeq.
+
+	// Add messages to: inbox, otherbox, inbox, inbox.
+	// We have these messages in order of modseq: 2+1 in inbox, 1 in otherbox, 2 in inbox.
+	// The original two in inbox appear to have modseq 1 (with 0 stored in the database).
+	// The ones we insert below will start with modseq 2. So we'll have modseq 1-5.
+	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.xuntagged(imapclient.UntaggedExists(4))
+	tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 4})
+
+	tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.xuntagged()
+	tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UID: 1})
+
+	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.xuntagged(imapclient.UntaggedExists(5))
+	tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 5})
+
+	tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
+	tc.xuntagged(imapclient.UntaggedExists(6))
+	tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 6})
+
+	tc2.transactf("ok", "Noop")
+	noflags := imapclient.FetchFlags(nil)
+	tc2.xuntagged(
+		imapclient.UntaggedExists(6),
+		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags}},
+		imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags}},
+		imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags}},
+	)
+
+	tc3.transactf("ok", "Noop")
+	tc3.xuntagged(
+		imapclient.UntaggedExists(6),
+		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(clientModseq + 1)}},
+		imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(clientModseq + 3)}},
+		imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq + 4)}},
+	)
+
+	moxvar.Pedantic = true
+	tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax.
+	moxvar.Pedantic = false
+	tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)")
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(clientModseq)}})
+
+	clientModseq += 4 // Four messages, over two mailboxes, modseq is per account.
+
+	// Check highestmodseq for mailboxes.
+	tc.transactf("ok", "Status inbox (highestmodseq)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": clientModseq}})
+
+	tc.transactf("ok", "Status otherbox (highestmodseq)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "otherbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 3}})
+
+	// Check highestmodseq when we select.
+	tc.transactf("ok", "Examine otherbox")
+	tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(3), More: "x"}})
+
+	tc.transactf("ok", "Select inbox")
+	tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}})
+
+	// Check fetch modseq response and changedsince.
+	tc.transactf("ok", `Fetch 1 (Modseq)`)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(1)}})
+
+	// Without modseq attribute, even with condseq enabled, there is no modseq response.
+	// For QRESYNC, we must always send MODSEQ for UID FETCH commands, but not for FETCH commands. ../rfc/7162:1427
+	tc.transactf("ok", `Uid Fetch 1 Flags`)
+	if qresync {
+		tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
+	} else {
+		tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
+	}
+	tc.transactf("ok", `Fetch 1 Flags`)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
+
+	// When CHANGEDSINCE is present, MODSEQ is automatically added to the response.
+	// ../rfc/7162:871
+	// ../rfc/7162:877
+	tc.transactf("ok", `Fetch 1 Flags (Changedsince 1)`)
+	tc.xuntagged()
+	tc.transactf("ok", `Fetch 1,4 Flags (Changedsince 1)`)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(2)}})
+	tc.transactf("ok", `Fetch 2 Flags (Changedsince 2)`)
+	tc.xuntagged()
+
+	// store and uid store.
+
+	// unchangedsince 0 never passes the check. ../rfc/7162:640
+	tc.transactf("ok", `Store 1 (Unchangedsince 0) +Flags ()`)
+	tc.xcodeArg(imapclient.CodeModified(xparseNumSet("1")))
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
+
+	// Modseq is 2 for first condstore-aware-appended message, so also no match.
+	tc.transactf("ok", `Uid Store 4 (Unchangedsince 1) +Flags ()`)
+	tc.xcodeArg(imapclient.CodeModified(xparseNumSet("4")))
+
+	// Modseq is 1 for original message.
+	tc.transactf("ok", `Store 1 (Unchangedsince 1) +Flags (label1)`)
+	tc.xcode("") // No MODIFIED.
+	clientModseq++
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}})
+	tc2.transactf("ok", "Noop")
+	tc2.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}},
+	)
+	tc3.transactf("ok", "Noop")
+	tc3.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}},
+	)
+
+	// Modify same message twice. Check that second application doesn't fail due to
+	// modseq change made in the first application. ../rfc/7162:823
+	tc.transactf("ok", `Uid Store 1,1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
+	clientModseq++
+	tc.xcode("") // No MODIFIED.
+	tc.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
+	)
+	// We do broadcast the changes twice. Not great, but doesn't hurt. This isn't common.
+	tc2.transactf("ok", "Noop")
+	tc2.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}},
+	)
+	tc3.transactf("ok", "Noop")
+	tc3.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
+	)
+
+	// Modify without actually changing flags, there will be no new modseq and no broadcast.
+	tc.transactf("ok", `Store 1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
+	tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}})
+	tc.xcode("") // No MODIFIED.
+	tc2.transactf("ok", "Noop")
+	tc2.xuntagged()
+	tc3.transactf("ok", "Noop")
+	tc3.xuntagged()
+
+	// search with modseq criteria and modseq in response
+	tc.transactf("ok", "Search Modseq %d", clientModseq)
+	tc.xsearchmodseq(clientModseq, 1)
+
+	tc.transactf("ok", "Uid Search Or Modseq %d Modseq %d", clientModseq, clientModseq)
+	tc.xsearchmodseq(clientModseq, 1)
+
+	// esearch
+	tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq)
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq})
+
+	uint32ptr := func(v uint32) *uint32 {
+		return &v
+	}
+	tc.transactf("ok", "Search Return (Count) 1:* Modseq 0")
+	tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq})
+
+	tc.transactf("ok", "Search Return (Min Max) 1:* Modseq 0")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 6, ModSeq: clientModseq})
+
+	tc.transactf("ok", "Search Return (Min) 1:* Modseq 0")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, ModSeq: clientModseq})
+
+	// expunge, we expunge the third and fourth messages. The third was originally with
+	// modseq 0, the fourth was added with condstore-aware append.
+	tc.transactf("ok", `Store 3:4 +Flags (\Deleted)`)
+	clientModseq++
+	tc2.transactf("ok", "Noop")
+	tc3.transactf("ok", "Noop")
+	tc.transactf("ok", "Expunge")
+	clientModseq++
+	if qresync {
+		tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
+	} else {
+		tc.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
+	}
+	tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
+	tc2.transactf("ok", "Noop")
+	tc2.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
+	tc3.transactf("ok", "Noop")
+	if qresync {
+		tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
+	} else {
+		tc3.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
+	}
+
+	// Again after expunge: status, select, conditional store/fetch/search
+	tc.transactf("ok", "Status inbox (Highestmodseq Messages Unseen Deleted)")
+	tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 4, "UNSEEN": 4, "DELETED": 0, "HIGHESTMODSEQ": clientModseq}})
+
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox")
+	tc.xuntaggedOpt(false,
+		imapclient.UntaggedExists(4),
+		imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
+	)
+
+	tc.transactf("ok", `Fetch 1:* (Modseq)`)
+	tc.xuntagged(
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(7)}},
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchModSeq(1)}},
+		imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), imapclient.FetchModSeq(4)}},
+		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchModSeq(5)}},
+	)
+	// Expunged messages, with higher modseq, should not show up.
+	tc.transactf("ok", "Uid Fetch 1:* (flags) (Changedsince 7)")
+	tc.xuntagged()
+
+	// search
+	tc.transactf("ok", "Search Modseq 7")
+	tc.xsearchmodseq(7, 1)
+	tc.transactf("ok", "Search Modseq 8")
+	tc.xsearch()
+
+	// esearch
+	tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 7")
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 7})
+	tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8")
+	tc.xuntagged(imapclient.UntaggedEsearch{Correlator: tc.client.LastTag})
+
+	// store, cannot modify expunged messages.
+	tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq)
+	tc.xuntagged()
+	tc.xcode("") // Not MODIFIED.
+	tc.transactf("ok", `Uid Store 3,4 +Flags (label2)`)
+	tc.xuntagged()
+	tc.xcode("") // Not MODIFIED.
+
+	// Check all condstore-enabling commands (and their syntax), ../rfc/7162:368
+
+	// We start a new connection, do the thing that should enable condstore, then
+	// change flags of a message in another connection, do a noop in the new connection
+	// which should result in an untagged fetch that includes modseq, the indicator
+	// that condstore was indeed enabled. It's a bit complicated, but i don't think
+	// there is a clearly specified mechanism to find out which capabilities are
+	// enabled at any point.
+	var tagcount int
+	checkCondstoreEnabled := func(fn func(xtc *testconn)) {
+		t.Helper()
+
+		xtc := startNoSwitchboard(t)
+		defer xtc.close()
+		xtc.client.Login("mjl@mox.example", "testtest")
+		fn(xtc)
+		tagcount++
+		label := fmt.Sprintf("l%d", tagcount)
+		tc.transactf("ok", "Store 4 Flags (%s)", label)
+		clientModseq++
+		xtc.transactf("ok", "Noop")
+		xtc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchFlags{label}, imapclient.FetchModSeq(clientModseq)}})
+	}
+	// SELECT/EXAMINE with CONDSTORE parameter, ../rfc/7162:373
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Select inbox (Condstore)")
+	})
+	// STATUS with HIGHESTMODSEQ attribute, ../rfc/7162:375
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Status otherbox (Highestmodseq)")
+		xtc.transactf("ok", "Select inbox")
+	})
+	// FETCH with MODSEQ ../rfc/7162:377
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Select inbox")
+		xtc.transactf("ok", "Fetch 4 (Modseq)")
+	})
+	// SEARCH with MODSEQ ../rfc/7162:377
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Select inbox")
+		xtc.transactf("ok", "Search 4 Modseq 1")
+	})
+	// FETCH with CHANGEDSINCE ../rfc/7162:380
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Select inbox")
+		xtc.transactf("ok", "Fetch 4 (Flags) (Changedsince %d)", clientModseq)
+	})
+	// STORE with UNCHANGEDSINCE ../rfc/7162:382
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Select inbox")
+		xtc.transactf("ok", "Store 4 (Unchangedsince 0) Flags ()")
+	})
+	// ENABLE CONDSTORE ../rfc/7162:384
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Enable Condstore")
+		xtc.transactf("ok", "Select inbox")
+	})
+	// ENABLE QRESYNC ../rfc/7162:1390
+	checkCondstoreEnabled(func(xtc *testconn) {
+		t.Helper()
+		xtc.transactf("ok", "Enable Qresync")
+		xtc.transactf("ok", "Select inbox")
+	})
+	tc.transactf("ok", "Store 4 Flags ()")
+	clientModseq++
+
+	if qresync {
+		testQresync(t, tc, clientModseq)
+	}
+
+	// Continue with some tests that further change the data.
+	// First we copy messages to a new mailbox, and check we get new modseq for those
+	// messages.
+	tc.transactf("ok", "Select otherbox")
+	tc2.transactf("ok", "Noop")
+	tc3.transactf("ok", "Noop")
+	tc.transactf("ok", "Copy 1 inbox")
+	clientModseq++
+	tc2.transactf("ok", "Noop")
+	tc3.transactf("ok", "Noop")
+	tc2.xuntagged(
+		imapclient.UntaggedExists(5),
+		imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags}},
+	)
+	tc3.xuntagged(
+		imapclient.UntaggedExists(5),
+		imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags, imapclient.FetchModSeq(clientModseq)}},
+	)
+
+	// Then we move some messages, and check if we get expunged/vanished in original
+	// and untagged fetch with modseq in destination mailbox.
+	// tc2o is a client without condstore, so no modseq responses.
+	tc2o := startNoSwitchboard(t)
+	defer tc2o.close()
+	tc2o.client.Login("mjl@mox.example", "testtest")
+	tc2o.client.Select("otherbox")
+
+	// tc3o is a client with condstore, so with modseq responses.
+	tc3o := startNoSwitchboard(t)
+	defer tc3o.close()
+	tc3o.client.Login("mjl@mox.example", "testtest")
+	tc3o.client.Enable(capability)
+	tc3o.client.Select("otherbox")
+
+	tc.transactf("ok", "Select inbox")
+	tc.transactf("ok", "Uid Move 2:4 otherbox") // Only UID 2, because UID 3 and 4 have already been expunged.
+	clientModseq++
+	if qresync {
+		tc.xuntaggedOpt(false, imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
+		tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
+	} else {
+		tc.xuntaggedOpt(false, imapclient.UntaggedExpunge(2))
+		tc.xcode("")
+	}
+	tc2.transactf("ok", "Noop")
+	tc2.xuntagged(imapclient.UntaggedExpunge(2))
+	tc3.transactf("ok", "Noop")
+	if qresync {
+		tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
+	} else {
+		tc3.xuntagged(imapclient.UntaggedExpunge(2))
+	}
+	tc2o.transactf("ok", "Noop")
+	tc2o.xuntagged(
+		imapclient.UntaggedExists(2),
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags}},
+	)
+	tc3o.transactf("ok", "Noop")
+	tc3o.xuntagged(
+		imapclient.UntaggedExists(2),
+		imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}},
+	)
+	tc2o.close()
+	tc2o = nil
+	tc3o.close()
+	tc3o = nil
+
+	// Then we rename inbox, which is special because it moves messages away instead of
+	// actually moving the mailbox. The mailbox stays and is cleared, so we check if we
+	// get expunged/vanished messages.
+	tc.transactf("ok", "Rename inbox oldbox")
+	// todo spec: server doesn't respond with untagged responses, find rfc reference that says this is ok.
+	tc2.transactf("ok", "Noop")
+	tc2.xuntagged(
+		imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
+		imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
+	)
+	tc3.transactf("ok", "Noop")
+	if qresync {
+		tc3.xuntagged(
+			imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
+			imapclient.UntaggedVanished{UIDs: xparseNumSet("1,5:7")},
+		)
+	} else {
+		tc3.xuntagged(
+			imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
+			imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
+		)
+	}
+
+	// Then we delete otherbox (we cannot delete inbox). We don't keep any history for removed mailboxes, so not actually a special case.
+	tc.transactf("ok", "Delete otherbox")
+}
+
+func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
+	// Vanished on non-uid fetch is not allowed. ../rfc/7162:1693
+	tc.transactf("bad", "fetch 1:* (Flags) (Changedsince 1 Vanished)")
+
+	// Vanished without changedsince is not allowed. ../rfc/7162:1701
+	tc.transactf("bad", "Uid Fetch 1:* (Flags) (Vanished)")
+
+	// Vanished not allowed without first enabling qresync. ../rfc/7162:1697
+	xtc := startNoSwitchboard(t)
+	xtc.client.Login("mjl@mox.example", "testtest")
+	xtc.transactf("ok", "Select inbox (Condstore)")
+	xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
+	xtc.close()
+	xtc = nil
+
+	// Check that we get proper vanished responses.
+	tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
+	noflags := imapclient.FetchFlags(nil)
+	tc.xuntagged(
+		imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+		imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
+		imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
+		imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+	)
+
+	// select/examine with qresync parameters, including the various optional fields.
+	tc.transactf("ok", "Close")
+
+	// Must enable qresync explicitly before using. ../rfc/7162:1446
+	xtc = startNoSwitchboard(t)
+	xtc.client.Login("mjl@mox.example", "testtest")
+	xtc.transactf("bad", "Select inbox (Qresync 1 0)")
+	xtc.close()
+	xtc = nil
+
+	tc.transactf("bad", "Select inbox (Qresync (0 1))")               // Both args must be > 0.
+	tc.transactf("bad", "Select inbox (Qresync (1 0))")               // Both args must be > 0.
+	tc.transactf("bad", "Select inbox (Qresync)")                     // Two args are minimum.
+	tc.transactf("bad", "Select inbox (Qresync (1))")                 // Two args are minimum.
+	tc.transactf("bad", "Select inbox (Qresync (1 1 1:*))")           // Known UIDs, * not allowed.
+	tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:6)))") // Known seqset cannot have *.
+	tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:6 1:*)))") // Known uidset cannot have *.
+	tc.transactf("bad", "Select inbox (Qresync (1 1) qresync (1 1))") // Duplicate qresync.
+
+	flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent label1 l1 l2 l3 l4 l5 l6 l7 l8`, " ")
+	permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ")
+	uflags := imapclient.UntaggedFlags(flags)
+	upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: permflags}, More: "x"}}
+
+	baseUntagged := []imapclient.Untagged{
+		uflags,
+		upermflags,
+		imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},
+		imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 7}, More: "x"}},
+		imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDVALIDITY", CodeArg: imapclient.CodeUint{Code: "UIDVALIDITY", Num: 1}, More: "x"}},
+		imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UNSEEN", CodeArg: imapclient.CodeUint{Code: "UNSEEN", Num: 1}, More: "x"}},
+		imapclient.UntaggedRecent(0),
+		imapclient.UntaggedExists(4),
+		imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
+	}
+
+	makeUntagged := func(l ...imapclient.Untagged) []imapclient.Untagged {
+		return append(append([]imapclient.Untagged{}, baseUntagged...), l...)
+	}
+
+	// uidvalidity 1, highest known modseq 1, sends full current state.
+	tc.transactf("ok", "Select inbox (Qresync (1 1))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+			imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
+			imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+
+	// Uidvalidity mismatch, server will not send any changes, so it's just a regular open.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (2 1))")
+	tc.xuntagged(baseUntagged...)
+
+	// We can tell which UIDs we know. First, send broader range then exist, should work.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 1 1:7))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+			imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
+			imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+
+	// Now send just the ones that exist. We won't get the vanished messages.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 1 1,2,5:6))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
+			imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+
+	// We'll only get updates for UIDs we specify.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 1 5))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
+		)...,
+	)
+
+	// We'll only get updates for UIDs we specify. ../rfc/7162:1523
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 1 3))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3")},
+		)...,
+	)
+
+	// If we specify the latest modseq, we'll get no changes.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 %d))", clientModseq)
+	tc.xuntagged(baseUntagged...)
+
+	// We can provide our own seqs & uids, and have server determine which uids we
+	// know. But the seqs & uids must be of equal length. First try with a few combinations
+	// that aren't valid. ../rfc/7162:1579
+	tc.transactf("ok", "Close")
+	tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1 1,2)))")   // Not same length.
+	tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1,2 1)))")   // Not same length.
+	tc.transactf("no", "Select inbox (Qresync (1 1 1:6 (1,2 1,1)))")  // Not ascending.
+	tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:4)))") // Star not allowed.
+
+	// With valid parameters, based on what a client would know at this stage.
+	tc.transactf("ok", "Select inbox (Qresync (1 1 1:6 (1,3,6 1,3,6)))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+			imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
+			imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+
+	// The 3rd parameter is optional, try without.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 5 (1,3,6 1,3,6)))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+			imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 8 (1,3,6 1,3,6)))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+
+	// Client will claim a highestmodseq but then include uids that have been removed
+	// since that time. Server detects this, sends full vanished history and continues
+	// working with modseq changed to 1 before the expunged uid.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 9 (1,3,6 1,3,6)))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}},
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+
+	// Client will claim a highestmodseq but then include uids that have been removed
+	// since that time. Server detects this, sends full vanished history and continues
+	// working with modseq changed to 1 before the expunged uid.
+	tc.transactf("ok", "Close")
+	tc.transactf("ok", "Select inbox (Qresync (1 18 (1,3,6 1,3,6)))")
+	tc.xuntagged(
+		makeUntagged(
+			imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}},
+			imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
+		)...,
+	)
+}

imapserver/fetch.go

@@ -11,24 +11,31 @@ import (
 	"sort"
 	"strings"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/mjl-/bstore"
 
 	"github.com/mjl-/mox/message"
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/moxio"
+	"github.com/mjl-/mox/moxvar"
 	"github.com/mjl-/mox/store"
 )
 
 // functions to handle fetch attribute requests are defined on fetchCmd.
 type fetchCmd struct {
-	conn          *conn
+	conn            *conn
-	mailboxID     int64
+	mailboxID       int64
-	uid           store.UID
+	uid             store.UID
-	tx            *bstore.Tx     // Writable tx, for storing message when first parsed as mime parts.
+	tx              *bstore.Tx     // Writable tx, for storing message when first parsed as mime parts.
-	changes       []store.Change // For updated Seen flag.
+	changes         []store.Change // For updated Seen flag.
-	markSeen      bool
+	markSeen        bool
-	needFlags     bool
+	needFlags       bool
-	expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
+	needModseq      bool         // Whether untagged responses needs modseq.
+	expungeIssued   bool         // Set if a message cannot be read. Can happen for expunged messages.
+	modseq          store.ModSeq // Initialized on first change, for marking messages as seen.
+	isUID           bool         // If this is a UID FETCH command.
+	hasChangedSince bool         // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
 
 	// Loaded when first needed, closed when message was processed.
 	m    *store.Message // Message currently being processed.
@@ -60,15 +67,61 @@ func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {
 //
 // State: Selected
 func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
-	// Command: ../rfc/9051:4330 ../rfc/3501:2992
+	// Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864
-	// Examples: ../rfc/9051:4463 ../rfc/9051:4520
+	// Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880
-	// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864
+	// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490
 
-	// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748
+	// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475
 	p.xspace()
 	nums := p.xnumSet()
 	p.xspace()
-	atts := p.xfetchAtts()
+	atts := p.xfetchAtts(isUID)
+	var changedSince int64
+	var haveChangedSince bool
+	var vanished bool
+	if p.space() {
+		// ../rfc/4466:542
+		// ../rfc/7162:2479
+		p.xtake("(")
+		seen := map[string]bool{}
+		for {
+			var w string
+			if isUID && p.conn.enabled[capQresync] {
+				// Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693
+				w = p.xtakelist("CHANGEDSINCE", "VANISHED")
+			} else {
+				w = p.xtakelist("CHANGEDSINCE")
+			}
+			if seen[w] {
+				xsyntaxErrorf("duplicate fetch modifier %s", w)
+			}
+			seen[w] = true
+			switch w {
+			case "CHANGEDSINCE":
+				p.xspace()
+				changedSince = p.xnumber64()
+				// workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.
+				if changedSince == 0 && moxvar.Pedantic {
+					// ../rfc/7162:2551
+					xsyntaxErrorf("changedsince modseq must be > 0")
+				}
+				// CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380
+				p.conn.xensureCondstore(nil)
+				haveChangedSince = true
+			case "VANISHED":
+				vanished = true
+			}
+			if p.take(")") {
+				break
+			}
+			p.xspace()
+		}
+
+		// ../rfc/7162:1701
+		if vanished && !haveChangedSince {
+			xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE")
+		}
+	}
 	p.xempty()
 
 	// We don't use c.account.WithRLock because we write to the client while reading messages.
@@ -81,21 +134,105 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 		runlock()
 	}()
 
-	cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID}
+	var vanishedUIDs []store.UID
+	cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID, isUID: isUID, hasChangedSince: haveChangedSince}
 	c.xdbwrite(func(tx *bstore.Tx) {
 		cmd.tx = tx
 
 		// Ensure the mailbox still exists.
 		c.xmailboxID(tx, c.mailboxID)
 
-		uids := c.xnumSetUIDs(isUID, nums)
+		var uids []store.UID
+
+		// With changedSince, the client is likely asking for a small set of changes. Use a
+		// database query to trim down the uids we need to look at.
+		// ../rfc/7162:871
+		if changedSince > 0 {
+			q := bstore.QueryTx[store.Message](tx)
+			q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
+			q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))
+			if !vanished {
+				q.FilterEqual("Expunged", false)
+			}
+			err := q.ForEach(func(m store.Message) error {
+				if m.Expunged {
+					vanishedUIDs = append(vanishedUIDs, m.UID)
+				} else if isUID {
+					if nums.containsUID(m.UID, c.uids, c.searchResult) {
+						uids = append(uids, m.UID)
+					}
+				} else {
+					seq := c.sequence(m.UID)
+					if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) {
+						uids = append(uids, m.UID)
+					}
+				}
+				return nil
+			})
+			xcheckf(err, "looking up messages with changedsince")
+		} else {
+			uids = c.xnumSetUIDs(isUID, nums)
+		}
+
+		// Send vanished for all missing requested UIDs. ../rfc/7162:1718
+		if vanished {
+			delModSeq, err := c.account.HighestDeletedModSeq(tx)
+			xcheckf(err, "looking up highest deleted modseq")
+			if changedSince < delModSeq.Client() {
+				// First sort the uids we already found, for fast lookup.
+				sort.Slice(vanishedUIDs, func(i, j int) bool {
+					return vanishedUIDs[i] < vanishedUIDs[j]
+				})
+
+				// We'll be gathering any more vanished uids in more.
+				more := map[store.UID]struct{}{}
+				checkVanished := func(uid store.UID) {
+					if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
+						more[uid] = struct{}{}
+					}
+				}
+				// Now look through the requested uids. We may have a searchResult, handle it
+				// separately from a numset with potential stars, over which we can more easily
+				// iterate.
+				if nums.searchResult {
+					for _, uid := range c.searchResult {
+						checkVanished(uid)
+					}
+				} else {
+					iter := nums.interpretStar(c.uids).newIter()
+					for {
+						num, ok := iter.Next()
+						if !ok {
+							break
+						}
+						checkVanished(store.UID(num))
+					}
+				}
+				vanishedUIDs = append(vanishedUIDs, maps.Keys(more)...)
+			}
+		}
 
 		// Release the account lock.
 		runlock()
 		runlock = func() {} // Prevent defer from unlocking again.
 
+		// First report all vanished UIDs. ../rfc/7162:1714
+		if len(vanishedUIDs) > 0 {
+			// Mention all vanished UIDs in compact numset form.
+			// ../rfc/7162:1985
+			sort.Slice(vanishedUIDs, func(i, j int) bool {
+				return vanishedUIDs[i] < vanishedUIDs[j]
+			})
+			// No hard limit on response sizes, but clients are recommended to not send more
+			// than 8k. We send a more conservative max 4k.
+			for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
+				c.bwritelinef("* VANISHED (EARLIER) %s", s)
+			}
+		}
+
 		for _, uid := range uids {
 			cmd.uid = uid
+			mlog.Field("processing uid", mlog.Field("uid", uid))
 			cmd.process(atts)
 		}
 	})
@@ -113,6 +250,15 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
 	}
 }
 
+func (cmd *fetchCmd) xmodseq() store.ModSeq {
+	if cmd.modseq == 0 {
+		var err error
+		cmd.modseq, err = cmd.conn.account.NextModSeq(cmd.tx)
+		cmd.xcheckf(err, "assigning next modseq")
+	}
+	return cmd.modseq
+}
+
 func (cmd *fetchCmd) xensureMessage() *store.Message {
 	if cmd.m != nil {
 		return cmd.m
@@ -120,6 +266,7 @@ func (cmd *fetchCmd) xensureMessage() *store.Message {
 
 	q := bstore.QueryTx[store.Message](cmd.tx)
 	q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
+	q.FilterEqual("Expunged", false)
 	m, err := q.Get()
 	cmd.xcheckf(err, "get message for uid %d", cmd.uid)
 	cmd.m = &m
@@ -178,6 +325,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
 
 	cmd.markSeen = false
 	cmd.needFlags = false
+	cmd.needModseq = false
 
 	for _, a := range atts {
 		data = append(data, cmd.xprocessAtt(a)...)
@@ -186,10 +334,11 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
 	if cmd.markSeen {
 		m := cmd.xensureMessage()
 		m.Seen = true
+		m.ModSeq = cmd.xmodseq()
 		err := cmd.tx.Update(m)
 		xcheckf(err, "marking message as seen")
 
-		cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
+		cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, ModSeq: m.ModSeq, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
 	}
 
 	if cmd.needFlags {
@@ -197,6 +346,26 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
 		data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))
 	}
 
+	// The wording around when to include the MODSEQ attribute is hard to follow and is
+	// specified and refined in several places.
+	//
+	// An additional rule applies to "QRESYNC servers" (we'll assume it only applies
+	// when QRESYNC is enabled on a connection): setting the \Seen flag also triggers
+	// sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421
+	//
+	// For example, ../rfc/7162:389 says the server must include modseq in "all
+	// subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID
+	// FETCH. That appears intentional, it is not a list of examples, it is the full
+	// list, and the "all subsequent untagged fetch responses" doesn't mean "all", just
+	// those covering the listed cases. That makes sense, because otherwise all the
+	// other mentioning of cases elsewhere in the RFC would be too superfluous.
+	//
+	// ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426
+	if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && (cmd.isUID || cmd.markSeen) {
+		m := cmd.xensureMessage()
+		data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})
+	}
+
 	// Write errors are turned into panics because we write through c.
 	fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
 	data.writeTo(cmd.conn, cmd.conn.bw)
@@ -301,6 +470,9 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
 	case "FLAGS":
 		cmd.needFlags = true
 
+	case "MODSEQ":
+		cmd.needModseq = true
+
 	default:
 		xserverErrorf("field %q not yet implemented", a.field)
 	}

imapserver/fuzz_test.go

@@ -1,7 +1,6 @@
 package imapserver
 
 import (
-	"context"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -58,7 +57,7 @@ func FuzzServer(f *testing.F) {
 		f.Add(tag + cmd)
 	}
 
-	mox.Context = context.Background()
+	mox.Context = ctxbg
 	mox.ConfigStaticPath = "../testdata/imapserverfuzz/mox.conf"
 	mox.MustLoadConfig(true, false)
 	dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)

imapserver/parse.go

@@ -246,13 +246,21 @@ func (p *parser) xnumber64() int64 {
 	if s == "" {
 		p.xerrorf("expected number64")
 	}
-	v, err := strconv.ParseInt(s, 10, 64)
+	v, err := strconv.ParseInt(s, 10, 63) // ../rfc/9051:6794 ../rfc/7162:297
 	if err != nil {
 		p.xerrorf("parsing number64 %q: %v", s, err)
 	}
 	return v
 }
 
+func (p *parser) xnznumber64() int64 {
+	v := p.xnumber64()
+	if v == 0 {
+		p.xerrorf("expected non-zero number64")
+	}
+	return v
+}
+
 // l should be a list of uppercase words, the first match is returned
 func (p *parser) takelist(l ...string) (string, bool) {
 	for _, w := range l {
@@ -423,36 +431,43 @@ func (p *parser) xmboxOrPat() ([]string, bool) {
 	return l, true
 }
 
-// ../rfc/9051:7056
+// ../rfc/9051:7056, RECENT ../rfc/3501:5047, APPENDLIMIT ../rfc/7889:252, HIGHESTMODSEQ ../rfc/7162:2452
-// RECENT only in ../rfc/3501:5047
-// APPENDLIMIT is from ../rfc/7889:252
 func (p *parser) xstatusAtt() string {
-	return p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT")
+	w := p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT", "HIGHESTMODSEQ")
+	if w == "HIGHESTMODSEQ" {
+		// HIGHESTMODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:375
+		p.conn.enabled[capCondstore] = true
+	}
+	return w
 }
 
 // ../rfc/9051:7133 ../rfc/9051:7034
-func (p *parser) xnumSet() (r numSet) {
+func (p *parser) xnumSet0(allowStar, allowSearch bool) (r numSet) {
 	defer p.context("numSet")()
-	if p.take("$") {
+	if allowSearch && p.take("$") {
 		return numSet{searchResult: true}
 	}
-	r.ranges = append(r.ranges, p.xnumRange())
+	r.ranges = append(r.ranges, p.xnumRange0(allowStar))
 	for p.take(",") {
-		r.ranges = append(r.ranges, p.xnumRange())
+		r.ranges = append(r.ranges, p.xnumRange0(allowStar))
 	}
 	return r
 }
 
+func (p *parser) xnumSet() (r numSet) {
+	return p.xnumSet0(true, true)
+}
+
 // parse numRange, which can be just a setNumber.
-func (p *parser) xnumRange() (r numRange) {
+func (p *parser) xnumRange0(allowStar bool) (r numRange) {
-	if p.take("*") {
+	if allowStar && p.take("*") {
 		r.first.star = true
 	} else {
 		r.first.number = p.xnznumber()
 	}
 	if p.take(":") {
 		r.last = &setNumber{}
-		if p.take("*") {
+		if allowStar && p.take("*") {
 			r.last.star = true
 		} else {
 			r.last.number = p.xnznumber()
@@ -550,14 +565,16 @@ func (p *parser) xsectionBinary() (r []uint32) {
 	return r
 }
 
-// ../rfc/9051:6557 ../rfc/3501:4751
+var fetchAttWords = []string{
-func (p *parser) xfetchAtt() (r fetchAtt) {
+	"ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY",
+	"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
+	"MODSEQ", // CONDSTORE extension.
+}
+
+// ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483
+func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) {
 	defer p.context("fetchAtt")()
-	words := []string{
+	f := p.xtakelist(fetchAttWords...)
-		"ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY",
-		"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
-	}
-	f := p.xtakelist(words...)
 	r.peek = strings.HasSuffix(f, ".PEEK")
 	r.field = strings.TrimSuffix(f, ".PEEK")
 
@@ -576,12 +593,19 @@ func (p *parser) xfetchAtt() (r fetchAtt) {
 		}
 	case "BINARY.SIZE":
 		r.sectionBinary = p.xsectionBinary()
+	case "MODSEQ":
+		// The RFC text mentions MODSEQ is only for FETCH, not UID FETCH, but the ABNF adds
+		// the attribute to the shared syntax, so UID FETCH also implements it.
+		// ../rfc/7162:905
+		// The wording about when to respond with a MODSEQ attribute could be more clear. ../rfc/7162:923 ../rfc/7162:388
+		// MODSEQ attribute is a CONDSTORE-enabling parameter. ../rfc/7162:377
+		p.conn.xensureCondstore(nil)
 	}
 	return
 }
 
 // ../rfc/9051:6553 ../rfc/3501:4748
-func (p *parser) xfetchAtts() []fetchAtt {
+func (p *parser) xfetchAtts(isUID bool) []fetchAtt {
 	defer p.context("fetchAtts")()
 
 	fields := func(l ...string) []fetchAtt {
@@ -605,13 +629,13 @@ func (p *parser) xfetchAtts() []fetchAtt {
 	}
 
 	if !p.hasPrefix("(") {
-		return []fetchAtt{p.xfetchAtt()}
+		return []fetchAtt{p.xfetchAtt(isUID)}
 	}
 
 	l := []fetchAtt{}
 	p.xtake("(")
 	for {
-		l = append(l, p.xfetchAtt())
+		l = append(l, p.xfetchAtt(isUID))
 		if !p.take(" ") {
 			break
 		}
@@ -748,9 +772,10 @@ var searchKeyWords = []string{
 	"SENTBEFORE", "SENTON",
 	"SENTSINCE", "SMALLER",
 	"UID", "UNDRAFT",
+	"MODSEQ", // CONDSTORE extension.
 }
 
-// ../rfc/9051:6923 ../rfc/3501:4957
+// ../rfc/9051:6923 ../rfc/3501:4957, MODSEQ ../rfc/7162:2492
 // differences: rfc 9051 removes NEW, OLD, RECENT and makes SMALLER and LARGER number64 instead of number.
 func (p *parser) xsearchKey() *searchKey {
 	if p.take("(") {
@@ -852,12 +877,50 @@ func (p *parser) xsearchKey() *searchKey {
 		p.xspace()
 		sk.uidSet = p.xnumSet()
 	case "UNDRAFT":
+	case "MODSEQ":
+		// ../rfc/7162:1045 ../rfc/7162:2499
+		p.xspace()
+		if p.take(`"`) {
+			// We don't do anything with this field, so parse and ignore.
+			p.xtake(`/FLAGS/`)
+			if p.take(`\`) {
+				p.xtake(`\`) // ../rfc/7162:1072
+			}
+			p.xatom()
+			p.xtake(`"`)
+			p.xspace()
+			p.xtakelist("PRIV", "SHARED", "ALL")
+			p.xspace()
+		}
+		v := p.xnumber64()
+		sk.clientModseq = &v
+		// MODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:377
+		p.conn.enabled[capCondstore] = true
 	default:
 		p.xerrorf("missing case for op %q", sk.op)
 	}
 	return sk
 }
 
+// hasModseq returns whether there is a modseq filter anywhere in the searchkey.
+func (sk searchKey) hasModseq() bool {
+	if sk.clientModseq != nil {
+		return true
+	}
+	for _, e := range sk.searchKeys {
+		if e.hasModseq() {
+			return true
+		}
+	}
+	if sk.searchKey != nil && sk.searchKey.hasModseq() {
+		return true
+	}
+	if sk.searchKey2 != nil && sk.searchKey2.hasModseq() {
+		return true
+	}
+	return false
+}
+
 // ../rfc/9051:6489 ../rfc/3501:4692
 func (p *parser) xdateDay() int {
 	d := p.xdigit()

imapserver/protocol.go

@@ -80,15 +80,31 @@ func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []sto
 	return false
 }
 
-func (ss numSet) String() string {
+// contains returns whether the numset contains the number.
-	if ss.searchResult {
+// only allowed on basic, strictly increasing numsets.
-		return "$"
+func (ss numSet) contains(v uint32) bool {
-	}
-	s := ""
 	for _, r := range ss.ranges {
-		if s != "" {
+		if r.first.number == v || r.last != nil && v > r.first.number && v <= r.last.number {
-			s += ","
+			return true
 		}
+	}
+	return false
+}
+
+func (ss numSet) empty() bool {
+	return !ss.searchResult && len(ss.ranges) == 0
+}
+
+// Strings returns the numset in zero or more strings of maxSize bytes. If
+// maxSize is <= 0, a single string is returned.
+func (ss numSet) Strings(maxSize int) []string {
+	if ss.searchResult {
+		return []string{"$"}
+	}
+	var l []string
+	var line string
+	for _, r := range ss.ranges {
+		s := ""
 		if r.first.star {
 			s += "*"
 		} else {
@@ -98,16 +114,41 @@ func (ss numSet) String() string {
 			if r.first.star {
 				panic("invalid numSet range first star without last")
 			}
+		} else {
+			s += ":"
+			if r.last.star {
+				s += "*"
+			} else {
+				s += fmt.Sprintf("%d", r.last.number)
+			}
+		}
+
+		nsize := len(line) + len(s)
+		if line != "" {
+			nsize++ // comma
+		}
+		if maxSize > 0 && nsize > maxSize {
+			l = append(l, line)
+			line = s
 			continue
 		}
-		s += ":"
+		if line != "" {
-		if r.last.star {
+			line += ","
-			s += "*"
-		} else {
-			s += fmt.Sprintf("%d", r.last.number)
 		}
+		line += s
 	}
-	return s
+	if line != "" {
+		l = append(l, line)
+	}
+	return l
+}
+
+func (ss numSet) String() string {
+	l := ss.Strings(0)
+	if len(l) == 0 {
+		return ""
+	}
+	return l[0]
 }
 
 type setNumber struct {
@@ -120,6 +161,127 @@ type numRange struct {
 	last  *setNumber // if nil, this numRange is just a setNumber in "first" and first.star will be false
 }
 
+// interpretStar returns a numset that interprets stars in a numset, returning a new
+// numset without stars with increasing first/last.
+func (s numSet) interpretStar(uids []store.UID) numSet {
+	var ns numSet
+	for _, r := range s.ranges {
+		first := r.first.number
+		if r.first.star {
+			if len(uids) == 0 {
+				continue
+			}
+			first = uint32(uids[0])
+		}
+		last := first
+		if r.last != nil {
+			last = r.last.number
+			if r.last.star {
+				if len(uids) == 0 {
+					continue
+				}
+				last = uint32(uids[len(uids)-1])
+				if first > last {
+					first = last
+				}
+			} else if r.first.star && last < first {
+				last = first
+			}
+		}
+		if first > last {
+			first, last = last, first
+		}
+		nr := numRange{first: setNumber{number: first}}
+		if first != last {
+			nr.last = &setNumber{number: last}
+		}
+		ns.ranges = append(ns.ranges, nr)
+	}
+	return ns
+}
+
+// whether numSet only has numbers (no star/search), and is strictly increasing.
+func (s *numSet) isBasicIncreasing() bool {
+	if s.searchResult {
+		return false
+	}
+	var last uint32
+	for _, r := range s.ranges {
+		if r.first.star || r.first.number <= last || r.last != nil && (r.last.star || r.last.number < r.first.number) {
+			return false
+		}
+		last = r.first.number
+		if r.last != nil {
+			last = r.last.number
+		}
+	}
+	return true
+}
+
+type numIter struct {
+	s numSet
+	i int
+	r *rangeIter
+}
+
+// newIter must only be called on a numSet that is basic (no star/search) and ascending.
+func (s numSet) newIter() *numIter {
+	return &numIter{s: s, i: 0, r: s.ranges[0].newIter()}
+}
+
+func (i *numIter) Next() (uint32, bool) {
+	if v, ok := i.r.Next(); ok {
+		return v, ok
+	}
+	i.i++
+	if i.i >= len(i.s.ranges) {
+		return 0, false
+	}
+	i.r = i.s.ranges[i.i].newIter()
+	return i.r.Next()
+}
+
+type rangeIter struct {
+	r numRange
+	o int
+}
+
+// newIter must only be called on a range in a numSet that is basic (no star/search) and ascending.
+func (r numRange) newIter() *rangeIter {
+	return &rangeIter{r: r, o: 0}
+}
+
+func (r *rangeIter) Next() (uint32, bool) {
+	if r.o == 0 {
+		r.o++
+		return r.r.first.number, true
+	}
+	if r.r.last == nil || r.r.first.number+uint32(r.o) > r.r.last.number {
+		return 0, false
+	}
+	v := r.r.first.number + uint32(r.o)
+	r.o++
+	return v, true
+}
+
+// append adds a new number to the set, extending a range, or starting a new one (possibly the first).
+// can only be used on basic numsets, without star/searchResult.
+func (s *numSet) append(v uint32) {
+	if len(s.ranges) == 0 {
+		s.ranges = []numRange{{first: setNumber{number: v}}}
+		return
+	}
+	ri := len(s.ranges) - 1
+	r := s.ranges[ri]
+	if v == r.first.number+1 && r.last == nil {
+		s.ranges[ri].last = &setNumber{number: v}
+	} else if r.last != nil && v == r.last.number+1 {
+		r.last.number++
+	} else {
+		s.ranges = append(s.ranges, numRange{first: setNumber{number: v}})
+	}
+}
+
 type partial struct {
 	offset uint32
 	count  uint32
@@ -156,17 +318,18 @@ type fetchAtt struct {
 
 type searchKey struct {
 	// Only one of searchKeys, seqSet and op can be non-nil/non-empty.
-	searchKeys  []searchKey // In case of nested/multiple keys. Also for the top-level command.
+	searchKeys   []searchKey // In case of nested/multiple keys. Also for the top-level command.
-	seqSet      *numSet     // In case of bare sequence set. For op UID, field uidSet contains the parameter.
+	seqSet       *numSet     // In case of bare sequence set. For op UID, field uidSet contains the parameter.
-	op          string      // Determines which of the fields below are set.
+	op           string      // Determines which of the fields below are set.
-	headerField string
+	headerField  string
-	astring     string
+	astring      string
-	date        time.Time
+	date         time.Time
-	atom        string
+	atom         string
-	number      int64
+	number       int64
-	searchKey   *searchKey
+	searchKey    *searchKey
-	searchKey2  *searchKey
+	searchKey2   *searchKey
-	uidSet      numSet
+	uidSet       numSet
+	clientModseq *int64
 }
 
 func compactUIDSet(l []store.UID) (r numSet) {

imapserver/protocol_test.go

@@ -62,3 +62,30 @@ func TestNumSetContains(t *testing.T) {
 	check(!ss3.containsUID(1, []store.UID{2, 3}, nil))
 	check(!ss3.containsUID(3, []store.UID{1, 2, 3}, nil))
 }
+
+func TestNumSetInterpret(t *testing.T) {
+	parseNumSet := func(s string) numSet {
+		p := parser{upper: s}
+		return p.xnumSet0(true, false)
+	}
+
+	checkEqual := func(uids []store.UID, a, s string) {
+		t.Helper()
+		n := parseNumSet(a).interpretStar(uids)
+		ns := n.String()
+		if ns != s {
+			t.Fatalf("%s != %s", ns, s)
+		}
+	}
+
+	checkEqual([]store.UID{}, "1:*", "")
+	checkEqual([]store.UID{1}, "1:*", "1")
+	checkEqual([]store.UID{1, 3}, "1:*", "1:3")
+	checkEqual([]store.UID{1, 3}, "4:*", "3")
+	checkEqual([]store.UID{2, 3}, "*:4", "2:4")
+	checkEqual([]store.UID{2, 3}, "*:1", "2")
+	checkEqual([]store.UID{1, 2, 3}, "1,2,3", "1,2,3")
+	checkEqual([]store.UID{}, "1,2,3", "1,2,3")
+	checkEqual([]store.UID{}, "1:3", "1:3")
+	checkEqual([]store.UID{}, "3:1", "1:3")
+}

imapserver/search.go

@@ -96,6 +96,7 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 	}
 
 	var expungeIssued bool
+	var maxModSeq store.ModSeq
 
 	var uids []store.UID
 	c.xdbread(func(tx *bstore.Tx) {
@@ -108,8 +109,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 		if eargs == nil || max == 0 || len(eargs) != 1 {
 			for i, uid := range c.uids {
 				lastIndex = i
-				if c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued) {
+				if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued); match {
 					uids = append(uids, uid)
+					if modseq > maxModSeq {
+						maxModSeq = modseq
+					}
 					if min == 1 && min+max == len(eargs) {
 						break
 					}
@@ -119,8 +123,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 		// And reverse search for MAX if we have only MAX or MAX combined with MIN.
 		if max == 1 && (len(eargs) == 1 || min+max == len(eargs)) {
 			for i := len(c.uids) - 1; i > lastIndex; i-- {
-				if c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued) {
+				if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued); match {
 					uids = append(uids, c.uids[i])
+					if modseq > maxModSeq {
+						maxModSeq = modseq
+					}
 					break
 				}
 			}
@@ -147,11 +154,24 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 				}
 				s += " " + fmt.Sprintf("%d", v)
 			}
+
+			// Since we don't have the max modseq for the possibly partial uid range we're
+			// writing here within hand reach, we conveniently interpret the ambiguous "for all
+			// messages being returned" in ../rfc/7162:1107 as meaning over all lines that we
+			// write. And that clients only commit this value after they have seen the tagged
+			// end of the command. Appears to be recommended behaviour, ../rfc/7162:2323.
+			// ../rfc/7162:1077 ../rfc/7162:1101
+			var modseq string
+			if sk.hasModseq() {
+				// ../rfc/7162:2557
+				modseq = fmt.Sprintf(" (MODSEQ %d)", maxModSeq.Client())
+			}
+
+			c.bwritelinef("* SEARCH%s%s", s, modseq)
 			uids = uids[n:]
-			c.bwritelinef("* SEARCH%s", s)
 		}
 	} else {
-		// New-style ESEARCH response. ../rfc/9051:6546 ../rfc/4466:522
+		// New-style ESEARCH response syntax: ../rfc/9051:6546 ../rfc/4466:522
 
 		if save {
 			// ../rfc/9051:3784 ../rfc/5182:13
@@ -195,6 +215,13 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 			if eargs["ALL"] && len(uids) > 0 {
 				resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String())
 			}
+
+			// Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273
+			// Summary: send the highest modseq of the returned messages.
+			if sk.hasModseq() && len(uids) > 0 {
+				resp += fmt.Sprintf(" MODSEQ %d", maxModSeq.Client())
+			}
+
 			c.bwritelinef("%s", resp)
 		}
 	}
@@ -215,10 +242,11 @@ type search struct {
 	m             store.Message
 	p             *message.Part
 	expungeIssued *bool
+	hasModseq     bool
 }
 
-func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) bool {
+func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) (bool, store.ModSeq) {
-	s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued}
+	s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued, hasModseq: sk.hasModseq()}
 	defer func() {
 		if s.mr != nil {
 			err := s.mr.Close()
@@ -229,12 +257,42 @@ func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKe
 	return s.match(sk)
 }
 
-func (s *search) match(sk searchKey) bool {
+func (s *search) match(sk searchKey) (match bool, modseq store.ModSeq) {
+	// Instead of littering all the cases in match0 with calls to get modseq, we do it once
+	// here in case of a match.
+	defer func() {
+		if match && s.hasModseq {
+			if s.m.ID == 0 {
+				match = s.xloadMessage()
+			}
+			modseq = s.m.ModSeq
+		}
+	}()
+
+	match = s.match0(sk)
+	return
+}
+
+func (s *search) xloadMessage() bool {
+	q := bstore.QueryTx[store.Message](s.tx)
+	q.FilterNonzero(store.Message{MailboxID: s.c.mailboxID, UID: s.uid})
+	m, err := q.Get()
+	if err == bstore.ErrAbsent || err == nil && m.Expunged {
+		// ../rfc/2180:607
+		*s.expungeIssued = true
+		return false
+	}
+	xcheckf(err, "get message")
+	s.m = m
+	return true
+}
+
+func (s *search) match0(sk searchKey) bool {
 	c := s.c
 
 	if sk.searchKeys != nil {
 		for _, ssk := range sk.searchKeys {
-			if !s.match(ssk) {
+			if !s.match0(ssk) {
 				return false
 			}
 		}
@@ -273,33 +331,26 @@ func (s *search) match(sk searchKey) bool {
 		// We do not implement the RECENT flag. All messages are not recent.
 		return false
 	case "NOT":
-		return !s.match(*sk.searchKey)
+		return !s.match0(*sk.searchKey)
 	case "OR":
-		return s.match(*sk.searchKey) || s.match(*sk.searchKey2)
+		return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2)
 	case "UID":
 		return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult)
 	}
 
 	// Parsed message.
 	if s.mr == nil {
-		q := bstore.QueryTx[store.Message](s.tx)
+		if !s.xloadMessage() {
-		q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: s.uid})
-		m, err := q.Get()
-		if err == bstore.ErrAbsent {
-			// ../rfc/2180:607
-			*s.expungeIssued = true
 			return false
 		}
-		xcheckf(err, "get message")
-		s.m = m
 
 		// Closed by searchMatch after all (recursive) search.match calls are finished.
-		s.mr = c.account.MessageReader(m)
+		s.mr = c.account.MessageReader(s.m)
 
-		if m.ParsedBuf == nil {
+		if s.m.ParsedBuf == nil {
 			c.log.Error("missing parsed message")
 		} else {
-			p, err := m.LoadPart(s.mr)
+			p, err := s.m.LoadPart(s.mr)
 			xcheckf(err, "load parsed message")
 			s.p = &p
 		}
@@ -385,6 +436,9 @@ func (s *search) match(sk searchKey) bool {
 		return s.m.Size > sk.number
 	case "SMALLER":
 		return s.m.Size < sk.number
+	case "MODSEQ":
+		// ../rfc/7162:1045
+		return s.m.ModSeq.Client() >= *sk.clientModseq
 	}
 
 	if s.p == nil {

imapserver/search_test.go

@@ -39,6 +39,16 @@ func (tc *testconn) xsearch(nums ...uint32) {
 	tc.xuntagged(imapclient.UntaggedSearch(nums))
 }
 
+func (tc *testconn) xsearchmodseq(modseq int64, nums ...uint32) {
+	tc.t.Helper()
+
+	if len(nums) == 0 {
+		tc.xnountagged()
+		return
+	}
+	tc.xuntagged(imapclient.UntaggedSearchModSeq{Nums: nums, ModSeq: modseq})
+}
+
 func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) {
 	tc.t.Helper()
 
@@ -220,43 +230,11 @@ func TestSearch(t *testing.T) {
 	tc.transactf("ok", `search charset utf-8 text "mox"`)
 	tc.xsearch(2, 3)
 
-	// esearchall makes an UntaggedEsearch response with All set, for comparisons.
-	esearchall0 := func(ss string) imapclient.NumSet {
-		seqset := imapclient.NumSet{}
-		for _, rs := range strings.Split(ss, ",") {
-			t := strings.Split(rs, ":")
-			if len(t) > 2 {
-				panic("bad seqset")
-			}
-			var first uint32
-			var last *uint32
-			if t[0] != "*" {
-				v, err := strconv.ParseUint(t[0], 10, 32)
-				if err != nil {
-					panic("parse first")
-				}
-				first = uint32(v)
-			}
-			if len(t) == 2 {
-				if t[1] != "*" {
-					v, err := strconv.ParseUint(t[1], 10, 32)
-					if err != nil {
-						panic("parse last")
-					}
-					u := uint32(v)
-					last = &u
-				}
-			}
-			seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last})
-		}
-		return seqset
-	}
-
 	esearchall := func(ss string) imapclient.UntaggedEsearch {
 		return imapclient.UntaggedEsearch{All: esearchall0(ss)}
 	}
 
-	uintptr := func(v uint32) *uint32 {
+	uint32ptr := func(v uint32) *uint32 {
 		return &v
 	}
 
@@ -265,10 +243,10 @@ func TestSearch(t *testing.T) {
 	tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit.
 
 	tc.transactf("ok", "search return (min max count all) all")
-	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(3), All: esearchall0("1:3")})
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(3), All: esearchall0("1:3")})
 
 	tc.transactf("ok", "UID search return (min max count all) all")
-	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(3), All: esearchall0("5:7")})
+	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")})
 
 	tc.transactf("ok", "search return (min) all")
 	tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
@@ -304,19 +282,19 @@ func TestSearch(t *testing.T) {
 	tc.xesearch(imapclient.UntaggedEsearch{})
 
 	tc.transactf("ok", "search return (min max all count) not all")
-	tc.xesearch(imapclient.UntaggedEsearch{Count: uintptr(0)})
+	tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(0)})
 
 	tc.transactf("ok", "search return (min max count all) 1,3")
-	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
 
 	tc.transactf("ok", "search return (min max count all) UID 5,7")
-	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
+	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
 
 	tc.transactf("ok", "uid search return (min max count all) 1,3")
-	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
+	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
 
 	tc.transactf("ok", "uid search return (min max count all) UID 5,7")
-	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
+	tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
 
 	tc.transactf("no", `search return () charset unknown text "mox"`)
 	tc.transactf("ok", `search return () charset us-ascii text "mox"`)
@@ -353,3 +331,35 @@ func TestSearch(t *testing.T) {
 	tc.transactf("ok", `search undraft`)
 	tc.xesearch(esearchall("1:2"))
 }
+
+// esearchall makes an UntaggedEsearch response with All set, for comparisons.
+func esearchall0(ss string) imapclient.NumSet {
+	seqset := imapclient.NumSet{}
+	for _, rs := range strings.Split(ss, ",") {
+		t := strings.Split(rs, ":")
+		if len(t) > 2 {
+			panic("bad seqset")
+		}
+		var first uint32
+		var last *uint32
+		if t[0] != "*" {
+			v, err := strconv.ParseUint(t[0], 10, 32)
+			if err != nil {
+				panic("parse first")
+			}
+			first = uint32(v)
+		}
+		if len(t) == 2 {
+			if t[1] != "*" {
+				v, err := strconv.ParseUint(t[1], 10, 32)
+				if err != nil {
+					panic("parse last")
+				}
+				u := uint32(v)
+				last = &u
+			}
+		}
+		seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last})
+	}
+	return seqset
+}

imapserver/server_test.go

@@ -21,6 +21,8 @@ import (
 	"github.com/mjl-/mox/store"
 )
 
+var ctxbg = context.Background()
+
 func init() {
 	sanityChecks = true
 
@@ -156,6 +158,7 @@ type testconn struct {
 	client     *imapclient.Conn
 	done       chan struct{}
 	serverConn net.Conn
+	account    *store.Account
 
 	// Result of last command.
 	lastUntagged []imapclient.Untagged
@@ -190,14 +193,15 @@ func (tc *testconn) xcodeArg(v any) {
 	}
 }
 
-func (tc *testconn) xuntagged(exps ...any) {
+func (tc *testconn) xuntagged(exps ...imapclient.Untagged) {
 	tc.t.Helper()
-	tc.xuntaggedCheck(true, exps...)
+	tc.xuntaggedOpt(true, exps...)
 }
 
-func (tc *testconn) xuntaggedCheck(all bool, exps ...any) {
+func (tc *testconn) xuntaggedOpt(all bool, exps ...imapclient.Untagged) {
 	tc.t.Helper()
 	last := append([]imapclient.Untagged{}, tc.lastUntagged...)
+	var mismatch any
 next:
 	for ei, exp := range exps {
 		for i, l := range last {
@@ -205,12 +209,16 @@ next:
 				continue
 			}
 			if !reflect.DeepEqual(l, exp) {
-				tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", l, l, exp, exp)
+				mismatch = l
+				continue
 			}
 			copy(last[i:], last[i+1:])
 			last = last[:len(last)-1]
 			continue next
 		}
+		if mismatch != nil {
+			tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", mismatch, mismatch, exp, exp)
+		}
 		var next string
 		if len(tc.lastUntagged) > 0 {
 			next = fmt.Sprintf(", next %#v", tc.lastUntagged[0])
@@ -293,11 +301,21 @@ func (tc *testconn) waitDone() {
 }
 
 func (tc *testconn) close() {
+	err := tc.account.Close()
+	tc.check(err, "close account")
 	tc.client.Close()
 	tc.serverConn.Close()
 	tc.waitDone()
 }
 
+func xparseNumSet(s string) imapclient.NumSet {
+	ns, err := imapclient.ParseNumSet(s)
+	if err != nil {
+		panic(fmt.Sprintf("parsing numset %s: %s", s, err))
+	}
+	return ns
+}
+
 var connCounter int64
 
 func start(t *testing.T) *testconn {
@@ -314,7 +332,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
 	if first {
 		os.RemoveAll("../testdata/imap/data")
 	}
-	mox.Context = context.Background()
+	mox.Context = ctxbg
 	mox.ConfigStaticPath = "../testdata/imap/mox.conf"
 	mox.MustLoadConfig(true, false)
 	acc, err := store.OpenAccount("mjl")
@@ -323,8 +341,6 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
 		err = acc.SetPassword("testtest")
 		tcheck(t, err, "set password")
 	}
-	err = acc.Close()
-	tcheck(t, err, "close account")
 	var switchDone chan struct{}
 	if first {
 		switchDone = store.Switchboard()
@@ -352,7 +368,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
 	}()
 	client, err := imapclient.New(clientConn, true)
 	tcheck(t, err, "new client")
-	return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn}
+	return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn, account: acc}
 }
 
 func fakeCert(t *testing.T) tls.Certificate {

imapserver/store_test.go

@@ -78,7 +78,7 @@ func TestStore(t *testing.T) {
 
 	// Flags are added to mailbox, not removed.
 	flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent new a b c d e different`, " ")
-	tc.xuntaggedCheck(false, imapclient.UntaggedFlags(flags))
+	tc.xuntaggedOpt(false, imapclient.UntaggedFlags(flags))
 
 	tc.transactf("no", `store 1 flags ()`) // No permission to set flags.
 }

import.go

@@ -270,6 +270,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 
 	var changes []store.Change
 
+	var modseq store.ModSeq // Assigned on first delivered messages, used for all messages.
+
 	xdeliver := func(m *store.Message, mf *os.File) {
 		// todo: possibly set dmarcdomain to the domain of the from address? at least for non-spams that have been seen. otherwise user would start without any reputations. the assumption would be that the user has accepted email and deemed it legit, coming from the indicated sender.
 
@@ -281,7 +283,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 		ctl.xcheck(err, "delivering message")
 		deliveredIDs = append(deliveredIDs, m.ID)
 		ctl.log.Debug("delivered message", mlog.Field("id", m.ID))
-		changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords})
+		changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Flags: m.Flags, Keywords: m.Keywords})
 	}
 
 	// todo: one goroutine for reading messages, one for parsing the message, one adding to database, one for junk filter training.
@@ -353,8 +355,16 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 				}
 			}
 
+			if modseq == 0 {
+				var err error
+				modseq, err = a.NextModSeq(tx)
+				ctl.xcheck(err, "assigning next modseq")
+			}
+
 			m.MailboxID = mb.ID
 			m.MailboxOrigID = mb.ID
+			m.CreateSeq = modseq
+			m.ModSeq = modseq
 			xdeliver(m, msgf)
 			err = msgf.Close()
 			ctl.log.Check(err, "closing message after delivery")

integration_test.go

@@ -184,5 +184,4 @@ a message.
 		tcheck(t, err, "sendmail")
 	})
 	xlog.Print("success", mlog.Field("duration", time.Since(t0)))
-
 }

main.go

@@ -2003,6 +2003,7 @@ func cmdEnsureParsed(c *cmd) {
 	n := 0
 	err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
 		q := bstore.QueryTx[store.Message](tx)
+		q.FilterEqual("Expunged", false)
 		q.FilterFn(func(m store.Message) bool {
 			return all || m.ParsedBuf == nil
 		})
@@ -2135,17 +2136,22 @@ open, or is not running.
 	err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
 		// Reassign UIDs, going per mailbox. We assign starting at 1, only changing the
 		// message if it isn't already at the intended UID. Doing it in this order ensures
-		// we don't get into trouble with duplicate UIDs for a mailbox.
+		// we don't get into trouble with duplicate UIDs for a mailbox. We assign a new
+		// modseq. Not strictly needed, for doesn't hurt.
+		modseq, err := a.NextModSeq(tx)
+		xcheckf(err, "assigning next modseq")
+
 		q := bstore.QueryTx[store.Message](tx)
 		if len(args) == 2 {
 			q.FilterNonzero(store.Message{MailboxID: mailboxID})
 		}
 		q.SortAsc("MailboxID", "UID")
-		err := q.ForEach(func(m store.Message) error {
+		err = q.ForEach(func(m store.Message) error {
 			uidlasts[m.MailboxID]++
 			uid := uidlasts[m.MailboxID]
 			if m.UID != uid {
 				m.UID = uid
+				m.ModSeq = modseq
 				if err := tx.Update(&m); err != nil {
 					return fmt.Errorf("updating uid for message: %v", err)
 				}

serve.go

@@ -301,7 +301,10 @@ requested, other TLS certificates are requested on demand.
 					log.Check(err, "closing temp changelog file")
 				}
 			}()
-			m := &store.Message{Received: time.Now(), Flags: store.Flags{Flagged: true}}
+			m := &store.Message{
+				Received: time.Now(),
+				Flags:    store.Flags{Flagged: true},
+			}
 			n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nRemember to make a backup with \"mox backup\" before upgrading.\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n"))
 			if err != nil {
 				log.Infox("writing temporary message file for changelog delivery", err)

smtpserver/rejects.go

@@ -54,6 +54,7 @@ func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m *
 
 			q := bstore.QueryTx[store.Message](tx)
 			q.FilterNonzero(store.Message{MailboxID: mb.ID})
+			q.FilterEqual("Expunged", false)
 			q.FilterFn(func(m store.Message) bool {
 				return msgID != "" && m.MessageID == msgID || len(hash) > 0 && bytes.Equal(m.MessageHash, hash)
 			})

smtpserver/reputation.go

@@ -124,6 +124,7 @@ func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rc
 	messageQuery := func(fm *store.Message, maxAge time.Duration, maxCount int) *bstore.Query[store.Message] {
 		q := bstore.QueryTx[store.Message](tx)
 		q.FilterEqual("MailboxOrigID", m.MailboxID)
+		q.FilterEqual("Expunged", false)
 		q.FilterFn(func(m store.Message) bool {
 			return m.Junk || m.Notjunk
 		})

smtpserver/server.go

@@ -2297,6 +2297,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 				q := bstore.QueryTx[store.Message](tx)
 				q.FilterNonzero(msg)
 				q.FilterGreater("Received", now.Add(-window))
+				q.FilterEqual("Expunged", false)
 				n, err := q.Count()
 				if err != nil {
 					retErr = err
@@ -2315,6 +2316,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 				q := bstore.QueryTx[store.Message](tx)
 				q.FilterNonzero(msg)
 				q.FilterGreater("Received", now.Add(-window))
+				q.FilterEqual("Expunged", false)
 				size := msgWriter.Size
 				err := q.ForEach(func(v store.Message) error {
 					size += v.Size

smtpserver/server_test.go

@@ -343,6 +343,7 @@ func tretrain(t *testing.T, acc *store.Account) {
 
 	// Fetch messags to retrain on.
 	q := bstore.QueryDB[store.Message](ctxbg, acc.DB)
+	q.FilterEqual("Expunged", false)
 	q.FilterFn(func(m store.Message) bool {
 		return m.Flags.Junk || m.Flags.Notjunk
 	})
@@ -412,6 +413,7 @@ func TestSpam(t *testing.T) {
 		tcheck(t, err, "get rejects mailbox")
 		qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
 		qm.FilterNonzero(store.Message{MailboxID: mb.ID})
+		qm.FilterEqual("Expunged", false)
 		n, err := qm.Count()
 		tcheck(t, err, "count messages in rejects mailbox")
 		if n != expect {
@@ -437,6 +439,7 @@ func TestSpam(t *testing.T) {
 
 	// Mark the messages as having good reputation.
 	q := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
+	q.FilterEqual("Expunged", false)
 	_, err := q.UpdateFields(map[string]any{"Junk": false, "Notjunk": true})
 	tcheck(t, err, "update junkiness")
 
@@ -456,6 +459,7 @@ func TestSpam(t *testing.T) {
 	// Undo dmarc pass, mark messages as junk, and train the filter.
 	resolver.TXT = nil
 	q = bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
+	q.FilterEqual("Expunged", false)
 	_, err = q.UpdateFields(map[string]any{"Junk": true, "Notjunk": false})
 	tcheck(t, err, "update junkiness")
 	tretrain(t, ts.acc)

store/account.go

@@ -150,6 +150,24 @@ type NextUIDValidity struct {
 	Next uint32
 }
 
+// SyncState track ModSeqs.
+type SyncState struct {
+	ID int // Just a single record with ID 1.
+
+	// Last used, next assigned will be one higher. The first value we hand out is 2.
+	// That's because 0 (the default value for old existing messages, from before the
+	// Message.ModSeq field) is special in IMAP, so we return it as 1.
+	LastModSeq ModSeq `bstore:"nonzero"`
+
+	// Highest ModSeq of expunged record that we deleted. When a clients synchronizes
+	// and requests changes based on a modseq before this one, we don't have the
+	// history to provide information about deletions. We normally keep these expunged
+	// records around, but we may periodically truly delete them to reclaim storage
+	// space. Initially set to -1 because we don't want to match with any ModSeq in the
+	// database, which can be zero values.
+	HighestDeletedModSeq ModSeq
+}
+
 // Mailbox is collection of messages, e.g. Inbox or Sent.
 type Mailbox struct {
 	ID int64
@@ -232,7 +250,23 @@ type Message struct {
 	ID int64
 
 	UID       UID   `bstore:"nonzero"` // UID, for IMAP. Set during deliver.
-	MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,ref Mailbox"`
+	MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`
+
+	// Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.
+	// ModSeq is the last modification. CreateSeq is the Seq the message was inserted,
+	// always <= ModSeq. If Expunged is set, the message has been removed and should not
+	// be returned to the user. In this case, ModSeq is the Seq where the message is
+	// removed, and will never be changed again.
+	// We have an index on both ModSeq (for JMAP that synchronizes per account) and
+	// MailboxID+ModSeq (for IMAP that synchronizes per mailbox).
+	// The index on CreateSeq helps efficiently finding created messages for JMAP.
+	// The value of ModSeq is special for IMAP. Messages that existed before ModSeq was
+	// added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If
+	// we get modseq 1 from a client, the IMAP server will translate it to 0. When we
+	// return modseq to clients, we turn 0 into 1.
+	ModSeq    ModSeq `bstore:"index"`
+	CreateSeq ModSeq `bstore:"index"`
+	Expunged  bool
 
 	// MailboxOrigID is the mailbox the message was originally delivered to. Typically
 	// Inbox or Rejects, but can also be Postmaster and TLS/DMARC reporting addresses.
@@ -290,7 +324,9 @@ type Message struct {
 	// delivered only once. Value includes <>.
 	MessageID string `bstore:"index"`
 
-	MessageHash []byte // Hash of message. For rejects delivery, so optional like MessageID.
+	// Hash of message. For rejects delivery, so optional like MessageID.
+	MessageHash []byte
+
 	Flags
 	Keywords    []string `bstore:"index"` // For keywords other than system flags or the basic well-known $-flags. Only in "atom" syntax, stored in lower case.
 	Size        int64
@@ -304,6 +340,41 @@ type Message struct {
 	ParsedBuf []byte
 }
 
+// ModSeq represents a modseq as stored in the database. ModSeq 0 in the
+// database is sent to the client as 1, because modseq 0 is special in IMAP.
+// ModSeq coming from the client are of type int64.
+type ModSeq int64
+
+func (ms ModSeq) Client() int64 {
+	if ms == 0 {
+		return 1
+	}
+	return int64(ms)
+}
+
+// ModSeqFromClient converts a modseq from a client to a modseq for internal
+// use, e.g. in a database query.
+// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).
+func ModSeqFromClient(modseq int64) ModSeq {
+	if modseq == 1 {
+		return 0
+	}
+	return ModSeq(modseq)
+}
+
+// PrepareExpunge clears fields that are no longer needed after an expunge, so
+// almost all fields. Does not change ModSeq, but does set Expunged.
+func (m *Message) PrepareExpunge() {
+	*m = Message{
+		ID:        m.ID,
+		UID:       m.UID,
+		MailboxID: m.MailboxID,
+		CreateSeq: m.CreateSeq,
+		ModSeq:    m.ModSeq,
+		Expunged:  true,
+	}
+}
+
 // LoadPart returns a message.Part by reading from m.ParsedBuf.
 func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {
 	if m.ParsedBuf == nil {
@@ -385,7 +456,7 @@ type Outgoing struct {
 }
 
 // Types stored in DB.
-var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}}
+var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}}
 
 // Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
 type Account struct {
@@ -564,6 +635,33 @@ func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {
 	return v, nil
 }
 
+// NextModSeq returns the next modification sequence, which is global per account,
+// over all types.
+func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {
+	v := SyncState{ID: 1}
+	if err := tx.Get(&v); err == bstore.ErrAbsent {
+		// We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so
+		// already used.
+		// HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value
+		// makes sense.
+		v = SyncState{1, 2, -1}
+		return v.LastModSeq, tx.Insert(&v)
+	} else if err != nil {
+		return 0, err
+	}
+	v.LastModSeq++
+	return v.LastModSeq, tx.Update(&v)
+}
+
+func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {
+	v := SyncState{ID: 1}
+	err := tx.Get(&v)
+	if err == bstore.ErrAbsent {
+		return 0, nil
+	}
+	return v.HighestDeletedModSeq, err
+}
+
 // WithWLock runs fn with account writelock held. Necessary for account/mailbox modification. For message delivery, a read lock is required.
 func (a *Account) WithWLock(fn func()) {
 	a.Lock()
@@ -593,10 +691,16 @@ func (a *Account) WithRLock(fn func()) {
 // If sync is true, the message file and its directory are synced. Should be true
 // for regular mail delivery, but can be false when importing many messages.
 //
+// If CreateSeq/ModSeq is not set, it is assigned automatically.
+//
 // Must be called with account rlock or wlock.
 //
 // Caller must broadcast new message.
 func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, consumeFile, isSent, sync, notrain bool) error {
+	if m.Expunged {
+		return fmt.Errorf("cannot deliver expunged message")
+	}
+
 	mb := Mailbox{ID: m.MailboxID}
 	if err := tx.Get(&mb); err != nil {
 		return fmt.Errorf("get mailbox: %w", err)
@@ -630,6 +734,14 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
 	if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {
 		m.MailboxDestinedID = 0
 	}
+	if m.CreateSeq == 0 || m.ModSeq == 0 {
+		modseq, err := a.NextModSeq(tx)
+		if err != nil {
+			return fmt.Errorf("assigning next modseq: %w", err)
+		}
+		m.CreateSeq = modseq
+		m.ModSeq = modseq
+	}
 
 	if err := tx.Insert(m); err != nil {
 		return fmt.Errorf("inserting message: %w", err)
@@ -1038,7 +1150,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF
 		return err
 	}
 
-	changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.Flags, m.Keywords})
+	changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords})
 	BroadcastChanges(a, changes)
 	return nil
 }
@@ -1126,27 +1238,33 @@ func (a *Account) removeMessages(ctx context.Context, log *mlog.Log, tx *bstore.
 		return nil, fmt.Errorf("deleting from message recipient: %w", err)
 	}
 
+	// Assign new modseq.
+	modseq, err := a.NextModSeq(tx)
+	if err != nil {
+		return nil, fmt.Errorf("assign next modseq: %w", err)
+	}
+
 	// Actually remove the messages.
-	qdm := bstore.QueryTx[Message](tx)
+	qx := bstore.QueryTx[Message](tx)
-	qdm.FilterIDs(ids)
+	qx.FilterIDs(ids)
-	var deleted []Message
+	var expunged []Message
-	qdm.Gather(&deleted)
+	qx.Gather(&expunged)
-	if _, err := qdm.Delete(); err != nil {
+	if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {
-		return nil, fmt.Errorf("deleting from messages: %w", err)
+		return nil, fmt.Errorf("expunging messages: %w", err)
 	}
 
 	// Mark as neutral and train so junk filter gets untrained with these (junk) messages.
-	for i := range deleted {
+	for i := range expunged {
-		deleted[i].Junk = false
+		expunged[i].Junk = false
-		deleted[i].Notjunk = false
+		expunged[i].Notjunk = false
 	}
-	if err := a.RetrainMessages(ctx, log, tx, deleted, true); err != nil {
+	if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {
-		return nil, fmt.Errorf("training deleted messages: %w", err)
+		return nil, fmt.Errorf("retraining expunged messages: %w", err)
 	}
 
 	changes := make([]Change, len(l))
 	for i, m := range l {
-		changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}}
+		changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}
 	}
 	return changes, nil
 }

store/state.go

@@ -28,6 +28,7 @@ type Change any
 type ChangeAddUID struct {
 	MailboxID int64
 	UID       UID
+	ModSeq    ModSeq
 	Flags     Flags    // System flags.
 	Keywords  []string // Other flags.
 }
@@ -36,12 +37,14 @@ type ChangeAddUID struct {
 type ChangeRemoveUIDs struct {
 	MailboxID int64
 	UIDs      []UID
+	ModSeq    ModSeq
 }
 
 // ChangeFlags is sent for an update to flags for a message, e.g. "Seen".
 type ChangeFlags struct {
 	MailboxID int64
 	UID       UID
+	ModSeq    ModSeq
 	Mask      Flags    // Which flags are actually modified.
 	Flags     Flags    // New flag values. All are set, not just mask.
 	Keywords  []string // Other flags.

test-upgrade.sh

@@ -9,9 +9,11 @@
 set -e
 # set -x
 
-# We'll allow max 256mb of memory during upgrades. We modify the softlimit when
+# We'll set a max memory limit during upgrades. We modify the softlimit when
 # importing the potentially large mbox file.
-ulimit -S -d 256000
+# Currently at 768MB, needed for upgrading with 500k messages from v0.0.5 to
+# v0.0.6 (two new indexes on store.Message).
+ulimit -S -d 768000
 
 (rm -r testdata/upgrade 2>/dev/null || exit 0)
 mkdir testdata/upgrade
@@ -59,7 +61,7 @@ for tag in $tags; do
 			echo 'Importing bulk data for upgrading.'
 			gunzip < ../upgradetest.mbox.gz | time ./$tag/mox ximport mbox ./stepdata/accounts/test0 upgradetest /dev/stdin
 			echo
-			ulimit -S -d 256000
+			ulimit -S -d 768000
 		fi
 
 		echo "Upgrade data to $tag."

verifydata.go

@@ -246,6 +246,10 @@ possibly making them potentially no longer readable by the previous version.
 				if uidnext := mailboxUIDNexts[m.MailboxID]; m.UID >= uidnext {
 					checkf(errors.New(`inconsistent uidnext for message/mailbox, see "mox fixuidmeta"`), dbpath, "message id %d in mailbox id %d has uid %d >= mailbox uidnext %d", m.ID, m.MailboxID, m.UID, uidnext)
 				}
+
+				if m.Expunged {
+					return nil
+				}
 				mp := store.MessagePath(m.ID)
 				seen[mp] = struct{}{}
 				p := filepath.Join(accdir, "msg", mp)