From 7f1b7198a879a360112c9e47eca4072b811c89e3 Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Mon, 24 Jul 2023 21:21:05 +0200 Subject: [PATCH] add condstore & qresync imap extensions for conditional storing and quick resynchronisation (not sure if mail clients are actually using it that). each message now has a "modseq". it is increased for each change. with condstore, imap clients can request changes since a certain modseq. that already allows quickly finding changes since a previous connection. condstore also allows storing (e.g. setting new message flags) only when the modseq of a message hasn't changed. qresync should make it fast for clients to get a full list of changed messages for a mailbox, including removals. we now also keep basic metadata of messages that have been removed (expunged). just enough (uid, modseq) to tell client that the messages have been removed. this does mean we have to be careful when querying messages from the database. we must now often filter the expunged messages out. we also keep "createseq", the modseq when a message was created. this will be useful for the jmap implementation. --- README.md | 1 - backup.go | 2 +- ctl.go | 1 + http/account_test.go | 4 +- http/import.go | 14 +- imapclient/parse.go | 50 ++- imapclient/protocol.go | 40 ++ imapserver/condstore_test.go | 702 +++++++++++++++++++++++++++++++++++ imapserver/fetch.go | 204 +++++++++- imapserver/fuzz_test.go | 3 +- imapserver/parse.go | 109 ++++-- imapserver/protocol.go | 211 +++++++++-- imapserver/protocol_test.go | 27 ++ imapserver/search.go | 96 +++-- imapserver/search_test.go | 90 +++-- imapserver/server.go | 613 +++++++++++++++++++++++++++--- imapserver/server_test.go | 32 +- imapserver/store_test.go | 2 +- import.go | 12 +- integration_test.go | 1 - main.go | 10 +- serve.go | 5 +- smtpserver/rejects.go | 1 + smtpserver/reputation.go | 1 + smtpserver/server.go | 2 + smtpserver/server_test.go | 4 + store/account.go | 150 +++++++- store/state.go | 3 + test-upgrade.sh | 8 +- verifydata.go | 4 + 30 files changed, 2181 insertions(+), 221 deletions(-) create mode 100644 imapserver/condstore_test.go diff --git a/README.md b/README.md index ed81b9e..799b9ab 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,6 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili ## Roadmap -- IMAP CONDSTORE and QRESYNC extensions - Webmail - IMAP THREAD extension - DANE and DNSSEC diff --git a/backup.go b/backup.go index 92200e7..7f1a3ed 100644 --- a/backup.go +++ b/backup.go @@ -424,7 +424,7 @@ func backupctl(ctx context.Context, ctl *ctl) { tmMsgs := time.Now() seen := map[string]struct{}{} var nlinked, ncopied int - err = bstore.QueryDB[store.Message](ctx, db).ForEach(func(m store.Message) error { + err = bstore.QueryDB[store.Message](ctx, db).FilterEqual("Expunged", false).ForEach(func(m store.Message) error { mp := store.MessagePath(m.ID) seen[mp] = struct{}{} amp := filepath.Join("accounts", acc.Name, "msg", mp) diff --git a/ctl.go b/ctl.go index f8d50c8..dea229a 100644 --- a/ctl.go +++ b/ctl.go @@ -649,6 +649,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) { // Read through messages with junk or nonjunk flag set, and train them. var total, trained int q := bstore.QueryDB[store.Message](ctx, acc.DB) + q.FilterEqual("Expunged", false) err = q.ForEach(func(m store.Message) error { total++ ok, err := acc.TrainMessage(ctx, ctl.log, jf, m) diff --git a/http/account_test.go b/http/account_test.go index 43efba2..e5ce685 100644 --- a/http/account_test.go +++ b/http/account_test.go @@ -139,7 +139,7 @@ func TestAccount(t *testing.T) { // Check there are messages, with the right flags. acc.DB.Read(ctxbg, func(tx *bstore.Tx) error { - _, err = bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get() + _, err = bstore.QueryTx[store.Message](tx).FilterEqual("Expunged", false).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get() tcheck(t, err, `fetching message with keywords "other" and "test"`) mb, err := acc.MailboxFind(tx, "importtest") @@ -152,7 +152,7 @@ func TestAccount(t *testing.T) { t.Fatalf(`expected mailbox keywords "other" and "test", got %v`, mb.Keywords) } - n, err := bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "custom").Count() + n, err := bstore.QueryTx[store.Message](tx).FilterEqual("Expunged", false).FilterIn("Keywords", "custom").Count() tcheck(t, err, `fetching message with keyword "custom"`) if n != 2 { t.Fatalf(`got %d messages with keyword "custom", expected 2`, n) diff --git a/http/import.go b/http/import.go index 1a91b33..a6d7880 100644 --- a/http/import.go +++ b/http/import.go @@ -377,6 +377,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store // finally at the end as a closing statement. var prevMailbox string + var modseq store.ModSeq // Assigned on first message, used for all messages. + trainMessage := func(m *store.Message, p message.Part, pos string) { words, err := jf.ParseMessage(p) if err != nil { @@ -478,6 +480,14 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store m.MailboxID = mb.ID m.MailboxOrigID = mb.ID + if modseq == 0 { + var err error + modseq, err = acc.NextModSeq(tx) + ximportcheckf(err, "assigning next modseq") + } + m.CreateSeq = modseq + m.ModSeq = modseq + if len(m.Keywords) > 0 { if destMailboxKeywords[mb.ID] == nil { destMailboxKeywords[mb.ID] = map[string]bool{} @@ -519,7 +529,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store return } deliveredIDs = append(deliveredIDs, m.ID) - changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords}) + changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Flags: m.Flags, Keywords: m.Keywords}) messages[mb.Name]++ if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name { prevMailbox = mb.Name @@ -726,7 +736,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store } err = tx.Update(&m) ximportcheckf(err, "updating message after flag update") - changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags, Keywords: m.Keywords}) + changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Mask: flags, Flags: flags, Keywords: m.Keywords}) } delete(mailboxMissingKeywordMessages, mailbox) } else { diff --git a/imapclient/parse.go b/imapclient/parse.go index 256f8d0..eb8fec1 100644 --- a/imapclient/parse.go +++ b/imapclient/parse.go @@ -119,6 +119,7 @@ var knownCodes = stringMap( "ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE", "UIDNOTSTICKY", "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED", "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE", "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT", "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "NOTSAVED", "HASCHILDREN", "CLOSED", "UNKNOWN-CTE", // With parameters. "BADCHARSET", "CAPABILITY", "PERMANENTFLAGS", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "APPENDUID", "COPYUID", + "HIGHESTMODSEQ", "MODIFIED", ) func stringMap(l ...string) map[string]struct{} { @@ -202,6 +203,13 @@ func (c *Conn) xrespCode() (string, CodeArg) { c.xspace() to := c.xuidset() codeArg = CodeCopyUID{destUIDValidity, from, to} + case "HIGHESTMODSEQ": + c.xspace() + codeArg = CodeHighestModSeq(c.xint64()) + case "MODIFIED": + c.xspace() + modified := c.xuidset() + codeArg = CodeModified(NumSet{Ranges: modified}) } return W, codeArg } @@ -248,7 +256,7 @@ func (c *Conn) xint32() int32 { func (c *Conn) xint64() int64 { s := c.xdigits() - num, err := strconv.ParseInt(s, 10, 64) + num, err := strconv.ParseInt(s, 10, 63) c.xcheckf(err, "parsing int64") return num } @@ -386,6 +394,8 @@ func (c *Conn) xuntagged() Untagged { } else { num = c.xint64() } + case "HIGHESTMODSEQ": + num = c.xint64() default: c.xerrorf("status: unknown attribute %q", s) } @@ -415,6 +425,15 @@ func (c *Conn) xuntagged() Untagged { c.xneedDisabled("untagged SEARCH response", CapIMAP4rev2) var nums []uint32 for c.take(' ') { + // ../rfc/7162:2557 + if c.take('(') { + c.xtake("MODSEQ") + c.xspace() + modseq := c.xint64() + c.xtake(")") + c.xcrlf() + return UntaggedSearchModSeq{nums, modseq} + } nums = append(nums, c.xnzuint32()) } r := UntaggedSearch(nums) @@ -456,6 +475,20 @@ func (c *Conn) xuntagged() Untagged { c.xcrlf() return UntaggedID(params) + // ../rfc/7162:2623 + case "VANISHED": + c.xspace() + var earlier bool + if c.take('(') { + c.xtake("EARLIER") + c.xtake(")") + c.xspace() + earlier = true + } + uids := c.xuidset() + c.xcrlf() + return UntaggedVanished{earlier, NumSet{Ranges: uids}} + default: v, err := strconv.ParseUint(w, 10, 32) if err == nil { @@ -605,6 +638,14 @@ func (c *Conn) xmsgatt1() FetchAttr { case "UID": c.xspace() return FetchUID(c.xuint32()) + + case "MODSEQ": + // ../rfc/7162:2488 + c.xspace() + c.xtake("(") + modseq := c.xint64() + c.xtake(")") + return FetchModSeq(modseq) } c.xerrorf("unknown fetch attribute %q", f) panic("not reached") @@ -963,7 +1004,7 @@ func (c *Conn) xtaggedExtVal() TaggedExtVal { return TaggedExtVal{SeqSet: &ss} } s := c.xdigits() - num, err := strconv.ParseInt(s, 10, 64) + num, err := strconv.ParseInt(s, 10, 63) c.xcheckf(err, "parsing int") if !c.peek(':') && !c.peek(',') { // not a larger sequence-set @@ -1146,6 +1187,11 @@ func (c *Conn) xesearchResponse() (r UntaggedEsearch) { num := c.xuint32() r.Count = &num + // ../rfc/7162:1211 ../rfc/4731:273 + case "MODSEQ": + c.xspace() + r.ModSeq = c.xint64() + default: // Validate ../rfc/9051:7090 for i, b := range []byte(w) { diff --git a/imapclient/protocol.go b/imapclient/protocol.go index d6fb992..b9679cc 100644 --- a/imapclient/protocol.go +++ b/imapclient/protocol.go @@ -1,6 +1,7 @@ package imapclient import ( + "bufio" "fmt" "strings" ) @@ -134,6 +135,20 @@ func (c CodeCopyUID) CodeString() string { return fmt.Sprintf("COPYUID %d %s %s", c.DestUIDValidity, str(c.From), str(c.To)) } +// For CONDSTORE. +type CodeModified NumSet + +func (c CodeModified) CodeString() string { + return fmt.Sprintf("MODIFIED %s", NumSet(c).String()) +} + +// For CONDSTORE. +type CodeHighestModSeq int64 + +func (c CodeHighestModSeq) CodeString() string { + return fmt.Sprintf("HIGHESTMODSEQ %d", c) +} + // RespText represents a response line minus the leading tag. type RespText struct { Code string // The first word between [] after the status. @@ -201,6 +216,12 @@ type UntaggedFetch struct { Attrs []FetchAttr } type UntaggedSearch []uint32 + +// ../rfc/7162:1101 +type UntaggedSearchModSeq struct { + Nums []uint32 + ModSeq int64 +} type UntaggedStatus struct { Mailbox string Attrs map[string]int64 // Upper case status attributes. ../rfc/9051:7059 @@ -224,9 +245,16 @@ type UntaggedEsearch struct { Max uint32 All NumSet Count *uint32 + ModSeq int64 Exts []EsearchDataExt } +// UntaggedVanished is used in QRESYNC to send UIDs that have been removed. +type UntaggedVanished struct { + Earlier bool + UIDs NumSet +} + // ../rfc/2971:184 type UntaggedID map[string]string @@ -278,6 +306,13 @@ func (ns NumSet) String() string { return r } +func ParseNumSet(s string) (ns NumSet, rerr error) { + c := Conn{r: bufio.NewReader(strings.NewReader(s))} + defer c.recover(&rerr) + ns = c.xsequenceSet() + return +} + // NumRange is a single number or range. type NumRange struct { First uint32 // 0 for "*". @@ -450,3 +485,8 @@ func (f FetchBinarySize) Attr() string { return f.RespAttr } type FetchUID uint32 func (f FetchUID) Attr() string { return "UID" } + +// "MODSEQ" fetch response. +type FetchModSeq int64 + +func (f FetchModSeq) Attr() string { return "MODSEQ" } diff --git a/imapserver/condstore_test.go b/imapserver/condstore_test.go new file mode 100644 index 0000000..cdb751f --- /dev/null +++ b/imapserver/condstore_test.go @@ -0,0 +1,702 @@ +package imapserver + +import ( + "fmt" + "strings" + "testing" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/imapclient" + "github.com/mjl-/mox/moxvar" + "github.com/mjl-/mox/store" +) + +func TestCondstore(t *testing.T) { + testCondstoreQresync(t, false) +} + +func TestQresync(t *testing.T) { + testCondstoreQresync(t, true) +} + +func testCondstoreQresync(t *testing.T, qresync bool) { + defer mockUIDValidity()() + tc := start(t) + defer tc.close() + + // todo: check whether marking \seen will cause modseq to be returned in case of qresync. + + // Check basic requirements of CONDSTORE. + + capability := "Condstore" + if qresync { + capability = "Qresync" + } + + tc.client.Login("mjl@mox.example", "testtest") + tc.client.Enable(capability) + tc.transactf("ok", "Select inbox") + tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(1), More: "x"}}) + + // First some tests without any messages. + + tc.transactf("ok", "Status inbox (Highestmodseq)") + tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 1}}) + + // No messages, no matches. + tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 12345)") + tc.xuntagged() + + // Also no messages with modseq 1, which we internally turn into modseq 0. + tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1)") + tc.xuntagged() + + // Also try with modseq attribute. + tc.transactf("ok", "Uid Fetch 1:* (Flags Modseq) (Changedsince 1)") + tc.xuntagged() + + // Search with modseq search criteria. + tc.transactf("ok", "Search Modseq 0") // Zero is valid, matches all. + tc.xsearch() + + tc.transactf("ok", "Search Modseq 1") // Converted to zero internally. + tc.xsearch() + + tc.transactf("ok", "Search Modseq 12345") + tc.xsearch() + + tc.transactf("ok", `Search Modseq "/Flags/\\Draft" All 12345`) + tc.xsearch() + + tc.transactf("ok", `Search Or Modseq 12345 Modseq 54321`) + tc.xsearch() + + // esearch + tc.transactf("ok", "Search Return (All) Modseq 123") + tc.xesearch(imapclient.UntaggedEsearch{}) + + // Now we add, delete, expunge, modify some message flags and check if the + // responses are correct. We check in both a condstore-enabled and one without that + // we get the correct notifications. + + // First we add 3 messages as if they were added before we implemented CONDSTORE. + // Later on, we'll update the second, and delete the third, leaving the first + // unmodified. Those messages have modseq 0 in the database. We use append for + // convenience, then adjust the records in the database. + tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + _, err := bstore.QueryDB[store.Message](ctxbg, tc.account.DB).UpdateFields(map[string]any{ + "ModSeq": 0, + "CreateSeq": 0, + }) + tcheck(t, err, "clearing modseq from messages") + err = tc.account.DB.Update(ctxbg, &store.SyncState{ID: 1, LastModSeq: 1}) + tcheck(t, err, "resetting modseq state") + + tc.client.Create("otherbox") + + // tc2 is a client without condstore, so no modseq responses. + tc2 := startNoSwitchboard(t) + defer tc2.close() + tc2.client.Login("mjl@mox.example", "testtest") + tc2.client.Select("inbox") + + // tc2 is a client with condstore, so with modseq responses. + tc3 := startNoSwitchboard(t) + defer tc3.close() + tc3.client.Login("mjl@mox.example", "testtest") + tc3.client.Enable(capability) + tc3.client.Select("inbox") + + var clientModseq int64 = 1 // We track the client-side modseq for inbox. Not a store.ModSeq. + + // Add messages to: inbox, otherbox, inbox, inbox. + // We have these messages in order of modseq: 2+1 in inbox, 1 in otherbox, 2 in inbox. + // The original two in inbox appear to have modseq 1 (with 0 stored in the database). + // The ones we insert below will start with modseq 2. So we'll have modseq 1-5. + tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc.xuntagged(imapclient.UntaggedExists(4)) + tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 4}) + + tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc.xuntagged() + tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UID: 1}) + + tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc.xuntagged(imapclient.UntaggedExists(5)) + tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 5}) + + tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc.xuntagged(imapclient.UntaggedExists(6)) + tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 6}) + + tc2.transactf("ok", "Noop") + noflags := imapclient.FetchFlags(nil) + tc2.xuntagged( + imapclient.UntaggedExists(6), + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags}}, + imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags}}, + imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags}}, + ) + + tc3.transactf("ok", "Noop") + tc3.xuntagged( + imapclient.UntaggedExists(6), + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(clientModseq + 1)}}, + imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(clientModseq + 3)}}, + imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq + 4)}}, + ) + + moxvar.Pedantic = true + tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax. + moxvar.Pedantic = false + tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)") + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(clientModseq)}}) + + clientModseq += 4 // Four messages, over two mailboxes, modseq is per account. + + // Check highestmodseq for mailboxes. + tc.transactf("ok", "Status inbox (highestmodseq)") + tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": clientModseq}}) + + tc.transactf("ok", "Status otherbox (highestmodseq)") + tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "otherbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 3}}) + + // Check highestmodseq when we select. + tc.transactf("ok", "Examine otherbox") + tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(3), More: "x"}}) + + tc.transactf("ok", "Select inbox") + tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}}) + + // Check fetch modseq response and changedsince. + tc.transactf("ok", `Fetch 1 (Modseq)`) + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(1)}}) + + // Without modseq attribute, even with condseq enabled, there is no modseq response. + // For QRESYNC, we must always send MODSEQ for UID FETCH commands, but not for FETCH commands. ../rfc/7162:1427 + tc.transactf("ok", `Uid Fetch 1 Flags`) + if qresync { + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}}) + } else { + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}}) + } + tc.transactf("ok", `Fetch 1 Flags`) + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}}) + + // When CHANGEDSINCE is present, MODSEQ is automatically added to the response. + // ../rfc/7162:871 + // ../rfc/7162:877 + tc.transactf("ok", `Fetch 1 Flags (Changedsince 1)`) + tc.xuntagged() + tc.transactf("ok", `Fetch 1,4 Flags (Changedsince 1)`) + tc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(2)}}) + tc.transactf("ok", `Fetch 2 Flags (Changedsince 2)`) + tc.xuntagged() + + // store and uid store. + + // unchangedsince 0 never passes the check. ../rfc/7162:640 + tc.transactf("ok", `Store 1 (Unchangedsince 0) +Flags ()`) + tc.xcodeArg(imapclient.CodeModified(xparseNumSet("1"))) + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}}) + + // Modseq is 2 for first condstore-aware-appended message, so also no match. + tc.transactf("ok", `Uid Store 4 (Unchangedsince 1) +Flags ()`) + tc.xcodeArg(imapclient.CodeModified(xparseNumSet("4"))) + + // Modseq is 1 for original message. + tc.transactf("ok", `Store 1 (Unchangedsince 1) +Flags (label1)`) + tc.xcode("") // No MODIFIED. + clientModseq++ + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}}) + tc2.transactf("ok", "Noop") + tc2.xuntagged( + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}}, + ) + tc3.transactf("ok", "Noop") + tc3.xuntagged( + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}}, + ) + + // Modify same message twice. Check that second application doesn't fail due to + // modseq change made in the first application. ../rfc/7162:823 + tc.transactf("ok", `Uid Store 1,1 (Unchangedsince %d) -Flags (label1)`, clientModseq) + clientModseq++ + tc.xcode("") // No MODIFIED. + tc.xuntagged( + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}}, + ) + // We do broadcast the changes twice. Not great, but doesn't hurt. This isn't common. + tc2.transactf("ok", "Noop") + tc2.xuntagged( + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}}, + ) + tc3.transactf("ok", "Noop") + tc3.xuntagged( + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}}, + ) + + // Modify without actually changing flags, there will be no new modseq and no broadcast. + tc.transactf("ok", `Store 1 (Unchangedsince %d) -Flags (label1)`, clientModseq) + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}}) + tc.xcode("") // No MODIFIED. + tc2.transactf("ok", "Noop") + tc2.xuntagged() + tc3.transactf("ok", "Noop") + tc3.xuntagged() + + // search with modseq criteria and modseq in response + tc.transactf("ok", "Search Modseq %d", clientModseq) + tc.xsearchmodseq(clientModseq, 1) + + tc.transactf("ok", "Uid Search Or Modseq %d Modseq %d", clientModseq, clientModseq) + tc.xsearchmodseq(clientModseq, 1) + + // esearch + tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq) + tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq}) + + uint32ptr := func(v uint32) *uint32 { + return &v + } + tc.transactf("ok", "Search Return (Count) 1:* Modseq 0") + tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq}) + + tc.transactf("ok", "Search Return (Min Max) 1:* Modseq 0") + tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 6, ModSeq: clientModseq}) + + tc.transactf("ok", "Search Return (Min) 1:* Modseq 0") + tc.xesearch(imapclient.UntaggedEsearch{Min: 1, ModSeq: clientModseq}) + + // expunge, we expunge the third and fourth messages. The third was originally with + // modseq 0, the fourth was added with condstore-aware append. + tc.transactf("ok", `Store 3:4 +Flags (\Deleted)`) + clientModseq++ + tc2.transactf("ok", "Noop") + tc3.transactf("ok", "Noop") + tc.transactf("ok", "Expunge") + clientModseq++ + if qresync { + tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")}) + } else { + tc.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3)) + } + tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq)) + tc2.transactf("ok", "Noop") + tc2.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3)) + tc3.transactf("ok", "Noop") + if qresync { + tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")}) + } else { + tc3.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3)) + } + + // Again after expunge: status, select, conditional store/fetch/search + tc.transactf("ok", "Status inbox (Highestmodseq Messages Unseen Deleted)") + tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 4, "UNSEEN": 4, "DELETED": 0, "HIGHESTMODSEQ": clientModseq}}) + + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox") + tc.xuntaggedOpt(false, + imapclient.UntaggedExists(4), + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}}, + ) + + tc.transactf("ok", `Fetch 1:* (Modseq)`) + tc.xuntagged( + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(7)}}, + imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchModSeq(1)}}, + imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), imapclient.FetchModSeq(4)}}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchModSeq(5)}}, + ) + // Expunged messages, with higher modseq, should not show up. + tc.transactf("ok", "Uid Fetch 1:* (flags) (Changedsince 7)") + tc.xuntagged() + + // search + tc.transactf("ok", "Search Modseq 7") + tc.xsearchmodseq(7, 1) + tc.transactf("ok", "Search Modseq 8") + tc.xsearch() + + // esearch + tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 7") + tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 7}) + tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8") + tc.xuntagged(imapclient.UntaggedEsearch{Correlator: tc.client.LastTag}) + + // store, cannot modify expunged messages. + tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq) + tc.xuntagged() + tc.xcode("") // Not MODIFIED. + tc.transactf("ok", `Uid Store 3,4 +Flags (label2)`) + tc.xuntagged() + tc.xcode("") // Not MODIFIED. + + // Check all condstore-enabling commands (and their syntax), ../rfc/7162:368 + + // We start a new connection, do the thing that should enable condstore, then + // change flags of a message in another connection, do a noop in the new connection + // which should result in an untagged fetch that includes modseq, the indicator + // that condstore was indeed enabled. It's a bit complicated, but i don't think + // there is a clearly specified mechanism to find out which capabilities are + // enabled at any point. + var tagcount int + checkCondstoreEnabled := func(fn func(xtc *testconn)) { + t.Helper() + + xtc := startNoSwitchboard(t) + defer xtc.close() + xtc.client.Login("mjl@mox.example", "testtest") + fn(xtc) + tagcount++ + label := fmt.Sprintf("l%d", tagcount) + tc.transactf("ok", "Store 4 Flags (%s)", label) + clientModseq++ + xtc.transactf("ok", "Noop") + xtc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchFlags{label}, imapclient.FetchModSeq(clientModseq)}}) + } + // SELECT/EXAMINE with CONDSTORE parameter, ../rfc/7162:373 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Select inbox (Condstore)") + }) + // STATUS with HIGHESTMODSEQ attribute, ../rfc/7162:375 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Status otherbox (Highestmodseq)") + xtc.transactf("ok", "Select inbox") + }) + // FETCH with MODSEQ ../rfc/7162:377 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Select inbox") + xtc.transactf("ok", "Fetch 4 (Modseq)") + }) + // SEARCH with MODSEQ ../rfc/7162:377 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Select inbox") + xtc.transactf("ok", "Search 4 Modseq 1") + }) + // FETCH with CHANGEDSINCE ../rfc/7162:380 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Select inbox") + xtc.transactf("ok", "Fetch 4 (Flags) (Changedsince %d)", clientModseq) + }) + // STORE with UNCHANGEDSINCE ../rfc/7162:382 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Select inbox") + xtc.transactf("ok", "Store 4 (Unchangedsince 0) Flags ()") + }) + // ENABLE CONDSTORE ../rfc/7162:384 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Enable Condstore") + xtc.transactf("ok", "Select inbox") + }) + // ENABLE QRESYNC ../rfc/7162:1390 + checkCondstoreEnabled(func(xtc *testconn) { + t.Helper() + xtc.transactf("ok", "Enable Qresync") + xtc.transactf("ok", "Select inbox") + }) + tc.transactf("ok", "Store 4 Flags ()") + clientModseq++ + + if qresync { + testQresync(t, tc, clientModseq) + } + + // Continue with some tests that further change the data. + // First we copy messages to a new mailbox, and check we get new modseq for those + // messages. + tc.transactf("ok", "Select otherbox") + tc2.transactf("ok", "Noop") + tc3.transactf("ok", "Noop") + tc.transactf("ok", "Copy 1 inbox") + clientModseq++ + tc2.transactf("ok", "Noop") + tc3.transactf("ok", "Noop") + tc2.xuntagged( + imapclient.UntaggedExists(5), + imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags}}, + ) + tc3.xuntagged( + imapclient.UntaggedExists(5), + imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags, imapclient.FetchModSeq(clientModseq)}}, + ) + + // Then we move some messages, and check if we get expunged/vanished in original + // and untagged fetch with modseq in destination mailbox. + // tc2o is a client without condstore, so no modseq responses. + tc2o := startNoSwitchboard(t) + defer tc2o.close() + tc2o.client.Login("mjl@mox.example", "testtest") + tc2o.client.Select("otherbox") + + // tc3o is a client with condstore, so with modseq responses. + tc3o := startNoSwitchboard(t) + defer tc3o.close() + tc3o.client.Login("mjl@mox.example", "testtest") + tc3o.client.Enable(capability) + tc3o.client.Select("otherbox") + + tc.transactf("ok", "Select inbox") + tc.transactf("ok", "Uid Move 2:4 otherbox") // Only UID 2, because UID 3 and 4 have already been expunged. + clientModseq++ + if qresync { + tc.xuntaggedOpt(false, imapclient.UntaggedVanished{UIDs: xparseNumSet("2")}) + tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq)) + } else { + tc.xuntaggedOpt(false, imapclient.UntaggedExpunge(2)) + tc.xcode("") + } + tc2.transactf("ok", "Noop") + tc2.xuntagged(imapclient.UntaggedExpunge(2)) + tc3.transactf("ok", "Noop") + if qresync { + tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")}) + } else { + tc3.xuntagged(imapclient.UntaggedExpunge(2)) + } + tc2o.transactf("ok", "Noop") + tc2o.xuntagged( + imapclient.UntaggedExists(2), + imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags}}, + ) + tc3o.transactf("ok", "Noop") + tc3o.xuntagged( + imapclient.UntaggedExists(2), + imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}}, + ) + tc2o.close() + tc2o = nil + tc3o.close() + tc3o = nil + + // Then we rename inbox, which is special because it moves messages away instead of + // actually moving the mailbox. The mailbox stays and is cleared, so we check if we + // get expunged/vanished messages. + tc.transactf("ok", "Rename inbox oldbox") + // todo spec: server doesn't respond with untagged responses, find rfc reference that says this is ok. + tc2.transactf("ok", "Noop") + tc2.xuntagged( + imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"}, + imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), + ) + tc3.transactf("ok", "Noop") + if qresync { + tc3.xuntagged( + imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"}, + imapclient.UntaggedVanished{UIDs: xparseNumSet("1,5:7")}, + ) + } else { + tc3.xuntagged( + imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"}, + imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), + ) + } + + // Then we delete otherbox (we cannot delete inbox). We don't keep any history for removed mailboxes, so not actually a special case. + tc.transactf("ok", "Delete otherbox") +} + +func testQresync(t *testing.T, tc *testconn, clientModseq int64) { + // Vanished on non-uid fetch is not allowed. ../rfc/7162:1693 + tc.transactf("bad", "fetch 1:* (Flags) (Changedsince 1 Vanished)") + + // Vanished without changedsince is not allowed. ../rfc/7162:1701 + tc.transactf("bad", "Uid Fetch 1:* (Flags) (Vanished)") + + // Vanished not allowed without first enabling qresync. ../rfc/7162:1697 + xtc := startNoSwitchboard(t) + xtc.client.Login("mjl@mox.example", "testtest") + xtc.transactf("ok", "Select inbox (Condstore)") + xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)") + xtc.close() + xtc = nil + + // Check that we get proper vanished responses. + tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)") + noflags := imapclient.FetchFlags(nil) + tc.xuntagged( + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}}, + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + ) + + // select/examine with qresync parameters, including the various optional fields. + tc.transactf("ok", "Close") + + // Must enable qresync explicitly before using. ../rfc/7162:1446 + xtc = startNoSwitchboard(t) + xtc.client.Login("mjl@mox.example", "testtest") + xtc.transactf("bad", "Select inbox (Qresync 1 0)") + xtc.close() + xtc = nil + + tc.transactf("bad", "Select inbox (Qresync (0 1))") // Both args must be > 0. + tc.transactf("bad", "Select inbox (Qresync (1 0))") // Both args must be > 0. + tc.transactf("bad", "Select inbox (Qresync)") // Two args are minimum. + tc.transactf("bad", "Select inbox (Qresync (1))") // Two args are minimum. + tc.transactf("bad", "Select inbox (Qresync (1 1 1:*))") // Known UIDs, * not allowed. + tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:6)))") // Known seqset cannot have *. + tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:6 1:*)))") // Known uidset cannot have *. + tc.transactf("bad", "Select inbox (Qresync (1 1) qresync (1 1))") // Duplicate qresync. + + flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent label1 l1 l2 l3 l4 l5 l6 l7 l8`, " ") + permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ") + uflags := imapclient.UntaggedFlags(flags) + upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: permflags}, More: "x"}} + + baseUntagged := []imapclient.Untagged{ + uflags, + upermflags, + imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"}, + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 7}, More: "x"}}, + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDVALIDITY", CodeArg: imapclient.CodeUint{Code: "UIDVALIDITY", Num: 1}, More: "x"}}, + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UNSEEN", CodeArg: imapclient.CodeUint{Code: "UNSEEN", Num: 1}, More: "x"}}, + imapclient.UntaggedRecent(0), + imapclient.UntaggedExists(4), + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}}, + } + + makeUntagged := func(l ...imapclient.Untagged) []imapclient.Untagged { + return append(append([]imapclient.Untagged{}, baseUntagged...), l...) + } + + // uidvalidity 1, highest known modseq 1, sends full current state. + tc.transactf("ok", "Select inbox (Qresync (1 1))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}}, + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) + + // Uidvalidity mismatch, server will not send any changes, so it's just a regular open. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (2 1))") + tc.xuntagged(baseUntagged...) + + // We can tell which UIDs we know. First, send broader range then exist, should work. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 1 1:7))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}}, + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) + + // Now send just the ones that exist. We won't get the vanished messages. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 1 1,2,5:6))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}}, + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) + + // We'll only get updates for UIDs we specify. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 1 5))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}}, + )..., + ) + + // We'll only get updates for UIDs we specify. ../rfc/7162:1523 + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 1 3))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3")}, + )..., + ) + + // If we specify the latest modseq, we'll get no changes. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 %d))", clientModseq) + tc.xuntagged(baseUntagged...) + + // We can provide our own seqs & uids, and have server determine which uids we + // know. But the seqs & uids must be of equal length. First try with a few combinations + // that aren't valid. ../rfc/7162:1579 + tc.transactf("ok", "Close") + tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1 1,2)))") // Not same length. + tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1,2 1)))") // Not same length. + tc.transactf("no", "Select inbox (Qresync (1 1 1:6 (1,2 1,1)))") // Not ascending. + tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:4)))") // Star not allowed. + + // With valid parameters, based on what a client would know at this stage. + tc.transactf("ok", "Select inbox (Qresync (1 1 1:6 (1,3,6 1,3,6)))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}}, + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) + + // The 3rd parameter is optional, try without. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 5 (1,3,6 1,3,6)))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) + + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 8 (1,3,6 1,3,6)))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) + + // Client will claim a highestmodseq but then include uids that have been removed + // since that time. Server detects this, sends full vanished history and continues + // working with modseq changed to 1 before the expunged uid. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 9 (1,3,6 1,3,6)))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}}, + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) + + // Client will claim a highestmodseq but then include uids that have been removed + // since that time. Server detects this, sends full vanished history and continues + // working with modseq changed to 1 before the expunged uid. + tc.transactf("ok", "Close") + tc.transactf("ok", "Select inbox (Qresync (1 18 (1,3,6 1,3,6)))") + tc.xuntagged( + makeUntagged( + imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}}, + imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")}, + imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}}, + )..., + ) +} diff --git a/imapserver/fetch.go b/imapserver/fetch.go index e14b6c9..87a6fb3 100644 --- a/imapserver/fetch.go +++ b/imapserver/fetch.go @@ -11,24 +11,31 @@ import ( "sort" "strings" + "golang.org/x/exp/maps" + "github.com/mjl-/bstore" "github.com/mjl-/mox/message" "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/moxio" + "github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/store" ) // functions to handle fetch attribute requests are defined on fetchCmd. type fetchCmd struct { - conn *conn - mailboxID int64 - uid store.UID - tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts. - changes []store.Change // For updated Seen flag. - markSeen bool - needFlags bool - expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages. + conn *conn + mailboxID int64 + uid store.UID + tx *bstore.Tx // Writable tx, for storing message when first parsed as mime parts. + changes []store.Change // For updated Seen flag. + markSeen bool + needFlags bool + needModseq bool // Whether untagged responses needs modseq. + expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages. + modseq store.ModSeq // Initialized on first change, for marking messages as seen. + isUID bool // If this is a UID FETCH command. + hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response. // Loaded when first needed, closed when message was processed. m *store.Message // Message currently being processed. @@ -60,15 +67,61 @@ func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) { // // State: Selected func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { - // Command: ../rfc/9051:4330 ../rfc/3501:2992 - // Examples: ../rfc/9051:4463 ../rfc/9051:4520 - // Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 + // Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864 + // Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880 + // Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490 - // Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 + // Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475 p.xspace() nums := p.xnumSet() p.xspace() - atts := p.xfetchAtts() + atts := p.xfetchAtts(isUID) + var changedSince int64 + var haveChangedSince bool + var vanished bool + if p.space() { + // ../rfc/4466:542 + // ../rfc/7162:2479 + p.xtake("(") + seen := map[string]bool{} + for { + var w string + if isUID && p.conn.enabled[capQresync] { + // Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693 + w = p.xtakelist("CHANGEDSINCE", "VANISHED") + } else { + w = p.xtakelist("CHANGEDSINCE") + } + if seen[w] { + xsyntaxErrorf("duplicate fetch modifier %s", w) + } + seen[w] = true + switch w { + case "CHANGEDSINCE": + p.xspace() + changedSince = p.xnumber64() + // workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled. + if changedSince == 0 && moxvar.Pedantic { + // ../rfc/7162:2551 + xsyntaxErrorf("changedsince modseq must be > 0") + } + // CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380 + p.conn.xensureCondstore(nil) + haveChangedSince = true + case "VANISHED": + vanished = true + } + if p.take(")") { + break + } + p.xspace() + } + + // ../rfc/7162:1701 + if vanished && !haveChangedSince { + xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE") + } + } p.xempty() // We don't use c.account.WithRLock because we write to the client while reading messages. @@ -81,21 +134,105 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { runlock() }() - cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID} + var vanishedUIDs []store.UID + cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID, isUID: isUID, hasChangedSince: haveChangedSince} c.xdbwrite(func(tx *bstore.Tx) { cmd.tx = tx // Ensure the mailbox still exists. c.xmailboxID(tx, c.mailboxID) - uids := c.xnumSetUIDs(isUID, nums) + var uids []store.UID + + // With changedSince, the client is likely asking for a small set of changes. Use a + // database query to trim down the uids we need to look at. + // ../rfc/7162:871 + if changedSince > 0 { + q := bstore.QueryTx[store.Message](tx) + q.FilterNonzero(store.Message{MailboxID: c.mailboxID}) + q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince)) + if !vanished { + q.FilterEqual("Expunged", false) + } + err := q.ForEach(func(m store.Message) error { + if m.Expunged { + vanishedUIDs = append(vanishedUIDs, m.UID) + } else if isUID { + if nums.containsUID(m.UID, c.uids, c.searchResult) { + uids = append(uids, m.UID) + } + } else { + seq := c.sequence(m.UID) + if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) { + uids = append(uids, m.UID) + } + } + return nil + }) + xcheckf(err, "looking up messages with changedsince") + } else { + uids = c.xnumSetUIDs(isUID, nums) + } + + // Send vanished for all missing requested UIDs. ../rfc/7162:1718 + if vanished { + delModSeq, err := c.account.HighestDeletedModSeq(tx) + xcheckf(err, "looking up highest deleted modseq") + if changedSince < delModSeq.Client() { + // First sort the uids we already found, for fast lookup. + sort.Slice(vanishedUIDs, func(i, j int) bool { + return vanishedUIDs[i] < vanishedUIDs[j] + }) + + // We'll be gathering any more vanished uids in more. + more := map[store.UID]struct{}{} + checkVanished := func(uid store.UID) { + if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 { + more[uid] = struct{}{} + } + } + // Now look through the requested uids. We may have a searchResult, handle it + // separately from a numset with potential stars, over which we can more easily + // iterate. + if nums.searchResult { + for _, uid := range c.searchResult { + checkVanished(uid) + } + } else { + iter := nums.interpretStar(c.uids).newIter() + for { + num, ok := iter.Next() + if !ok { + break + } + checkVanished(store.UID(num)) + } + } + vanishedUIDs = append(vanishedUIDs, maps.Keys(more)...) + } + } // Release the account lock. runlock() runlock = func() {} // Prevent defer from unlocking again. + // First report all vanished UIDs. ../rfc/7162:1714 + if len(vanishedUIDs) > 0 { + // Mention all vanished UIDs in compact numset form. + // ../rfc/7162:1985 + sort.Slice(vanishedUIDs, func(i, j int) bool { + return vanishedUIDs[i] < vanishedUIDs[j] + }) + // No hard limit on response sizes, but clients are recommended to not send more + // than 8k. We send a more conservative max 4k. + for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) { + c.bwritelinef("* VANISHED (EARLIER) %s", s) + } + } + for _, uid := range uids { cmd.uid = uid + mlog.Field("processing uid", mlog.Field("uid", uid)) cmd.process(atts) } }) @@ -113,6 +250,15 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) { } } +func (cmd *fetchCmd) xmodseq() store.ModSeq { + if cmd.modseq == 0 { + var err error + cmd.modseq, err = cmd.conn.account.NextModSeq(cmd.tx) + cmd.xcheckf(err, "assigning next modseq") + } + return cmd.modseq +} + func (cmd *fetchCmd) xensureMessage() *store.Message { if cmd.m != nil { return cmd.m @@ -120,6 +266,7 @@ func (cmd *fetchCmd) xensureMessage() *store.Message { q := bstore.QueryTx[store.Message](cmd.tx) q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid}) + q.FilterEqual("Expunged", false) m, err := q.Get() cmd.xcheckf(err, "get message for uid %d", cmd.uid) cmd.m = &m @@ -178,6 +325,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { cmd.markSeen = false cmd.needFlags = false + cmd.needModseq = false for _, a := range atts { data = append(data, cmd.xprocessAtt(a)...) @@ -186,10 +334,11 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { if cmd.markSeen { m := cmd.xensureMessage() m.Seen = true + m.ModSeq = cmd.xmodseq() err := cmd.tx.Update(m) xcheckf(err, "marking message as seen") - cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords}) + cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, ModSeq: m.ModSeq, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords}) } if cmd.needFlags { @@ -197,6 +346,26 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords)) } + // The wording around when to include the MODSEQ attribute is hard to follow and is + // specified and refined in several places. + // + // An additional rule applies to "QRESYNC servers" (we'll assume it only applies + // when QRESYNC is enabled on a connection): setting the \Seen flag also triggers + // sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421 + // + // For example, ../rfc/7162:389 says the server must include modseq in "all + // subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID + // FETCH. That appears intentional, it is not a list of examples, it is the full + // list, and the "all subsequent untagged fetch responses" doesn't mean "all", just + // those covering the listed cases. That makes sense, because otherwise all the + // other mentioning of cases elsewhere in the RFC would be too superfluous. + // + // ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426 + if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && (cmd.isUID || cmd.markSeen) { + m := cmd.xensureMessage() + data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))}) + } + // Write errors are turned into panics because we write through c. fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid)) data.writeTo(cmd.conn, cmd.conn.bw) @@ -301,6 +470,9 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token { case "FLAGS": cmd.needFlags = true + case "MODSEQ": + cmd.needModseq = true + default: xserverErrorf("field %q not yet implemented", a.field) } diff --git a/imapserver/fuzz_test.go b/imapserver/fuzz_test.go index 38c7964..515b3fc 100644 --- a/imapserver/fuzz_test.go +++ b/imapserver/fuzz_test.go @@ -1,7 +1,6 @@ package imapserver import ( - "context" "encoding/base64" "errors" "fmt" @@ -58,7 +57,7 @@ func FuzzServer(f *testing.F) { f.Add(tag + cmd) } - mox.Context = context.Background() + mox.Context = ctxbg mox.ConfigStaticPath = "../testdata/imapserverfuzz/mox.conf" mox.MustLoadConfig(true, false) dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir) diff --git a/imapserver/parse.go b/imapserver/parse.go index 15bff3b..0dc0a40 100644 --- a/imapserver/parse.go +++ b/imapserver/parse.go @@ -246,13 +246,21 @@ func (p *parser) xnumber64() int64 { if s == "" { p.xerrorf("expected number64") } - v, err := strconv.ParseInt(s, 10, 64) + v, err := strconv.ParseInt(s, 10, 63) // ../rfc/9051:6794 ../rfc/7162:297 if err != nil { p.xerrorf("parsing number64 %q: %v", s, err) } return v } +func (p *parser) xnznumber64() int64 { + v := p.xnumber64() + if v == 0 { + p.xerrorf("expected non-zero number64") + } + return v +} + // l should be a list of uppercase words, the first match is returned func (p *parser) takelist(l ...string) (string, bool) { for _, w := range l { @@ -423,36 +431,43 @@ func (p *parser) xmboxOrPat() ([]string, bool) { return l, true } -// ../rfc/9051:7056 -// RECENT only in ../rfc/3501:5047 -// APPENDLIMIT is from ../rfc/7889:252 +// ../rfc/9051:7056, RECENT ../rfc/3501:5047, APPENDLIMIT ../rfc/7889:252, HIGHESTMODSEQ ../rfc/7162:2452 func (p *parser) xstatusAtt() string { - return p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT") + w := p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT", "HIGHESTMODSEQ") + if w == "HIGHESTMODSEQ" { + // HIGHESTMODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:375 + p.conn.enabled[capCondstore] = true + } + return w } // ../rfc/9051:7133 ../rfc/9051:7034 -func (p *parser) xnumSet() (r numSet) { +func (p *parser) xnumSet0(allowStar, allowSearch bool) (r numSet) { defer p.context("numSet")() - if p.take("$") { + if allowSearch && p.take("$") { return numSet{searchResult: true} } - r.ranges = append(r.ranges, p.xnumRange()) + r.ranges = append(r.ranges, p.xnumRange0(allowStar)) for p.take(",") { - r.ranges = append(r.ranges, p.xnumRange()) + r.ranges = append(r.ranges, p.xnumRange0(allowStar)) } return r } +func (p *parser) xnumSet() (r numSet) { + return p.xnumSet0(true, true) +} + // parse numRange, which can be just a setNumber. -func (p *parser) xnumRange() (r numRange) { - if p.take("*") { +func (p *parser) xnumRange0(allowStar bool) (r numRange) { + if allowStar && p.take("*") { r.first.star = true } else { r.first.number = p.xnznumber() } if p.take(":") { r.last = &setNumber{} - if p.take("*") { + if allowStar && p.take("*") { r.last.star = true } else { r.last.number = p.xnznumber() @@ -550,14 +565,16 @@ func (p *parser) xsectionBinary() (r []uint32) { return r } -// ../rfc/9051:6557 ../rfc/3501:4751 -func (p *parser) xfetchAtt() (r fetchAtt) { +var fetchAttWords = []string{ + "ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY", + "RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP + "MODSEQ", // CONDSTORE extension. +} + +// ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483 +func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) { defer p.context("fetchAtt")() - words := []string{ - "ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY", - "RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP - } - f := p.xtakelist(words...) + f := p.xtakelist(fetchAttWords...) r.peek = strings.HasSuffix(f, ".PEEK") r.field = strings.TrimSuffix(f, ".PEEK") @@ -576,12 +593,19 @@ func (p *parser) xfetchAtt() (r fetchAtt) { } case "BINARY.SIZE": r.sectionBinary = p.xsectionBinary() + case "MODSEQ": + // The RFC text mentions MODSEQ is only for FETCH, not UID FETCH, but the ABNF adds + // the attribute to the shared syntax, so UID FETCH also implements it. + // ../rfc/7162:905 + // The wording about when to respond with a MODSEQ attribute could be more clear. ../rfc/7162:923 ../rfc/7162:388 + // MODSEQ attribute is a CONDSTORE-enabling parameter. ../rfc/7162:377 + p.conn.xensureCondstore(nil) } return } // ../rfc/9051:6553 ../rfc/3501:4748 -func (p *parser) xfetchAtts() []fetchAtt { +func (p *parser) xfetchAtts(isUID bool) []fetchAtt { defer p.context("fetchAtts")() fields := func(l ...string) []fetchAtt { @@ -605,13 +629,13 @@ func (p *parser) xfetchAtts() []fetchAtt { } if !p.hasPrefix("(") { - return []fetchAtt{p.xfetchAtt()} + return []fetchAtt{p.xfetchAtt(isUID)} } l := []fetchAtt{} p.xtake("(") for { - l = append(l, p.xfetchAtt()) + l = append(l, p.xfetchAtt(isUID)) if !p.take(" ") { break } @@ -748,9 +772,10 @@ var searchKeyWords = []string{ "SENTBEFORE", "SENTON", "SENTSINCE", "SMALLER", "UID", "UNDRAFT", + "MODSEQ", // CONDSTORE extension. } -// ../rfc/9051:6923 ../rfc/3501:4957 +// ../rfc/9051:6923 ../rfc/3501:4957, MODSEQ ../rfc/7162:2492 // differences: rfc 9051 removes NEW, OLD, RECENT and makes SMALLER and LARGER number64 instead of number. func (p *parser) xsearchKey() *searchKey { if p.take("(") { @@ -852,12 +877,50 @@ func (p *parser) xsearchKey() *searchKey { p.xspace() sk.uidSet = p.xnumSet() case "UNDRAFT": + case "MODSEQ": + // ../rfc/7162:1045 ../rfc/7162:2499 + p.xspace() + if p.take(`"`) { + // We don't do anything with this field, so parse and ignore. + p.xtake(`/FLAGS/`) + if p.take(`\`) { + p.xtake(`\`) // ../rfc/7162:1072 + } + p.xatom() + p.xtake(`"`) + p.xspace() + p.xtakelist("PRIV", "SHARED", "ALL") + p.xspace() + } + v := p.xnumber64() + sk.clientModseq = &v + // MODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:377 + p.conn.enabled[capCondstore] = true default: p.xerrorf("missing case for op %q", sk.op) } return sk } +// hasModseq returns whether there is a modseq filter anywhere in the searchkey. +func (sk searchKey) hasModseq() bool { + if sk.clientModseq != nil { + return true + } + for _, e := range sk.searchKeys { + if e.hasModseq() { + return true + } + } + if sk.searchKey != nil && sk.searchKey.hasModseq() { + return true + } + if sk.searchKey2 != nil && sk.searchKey2.hasModseq() { + return true + } + return false +} + // ../rfc/9051:6489 ../rfc/3501:4692 func (p *parser) xdateDay() int { d := p.xdigit() diff --git a/imapserver/protocol.go b/imapserver/protocol.go index 26e9a90..c8535a7 100644 --- a/imapserver/protocol.go +++ b/imapserver/protocol.go @@ -80,15 +80,31 @@ func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []sto return false } -func (ss numSet) String() string { - if ss.searchResult { - return "$" - } - s := "" +// contains returns whether the numset contains the number. +// only allowed on basic, strictly increasing numsets. +func (ss numSet) contains(v uint32) bool { for _, r := range ss.ranges { - if s != "" { - s += "," + if r.first.number == v || r.last != nil && v > r.first.number && v <= r.last.number { + return true } + } + return false +} + +func (ss numSet) empty() bool { + return !ss.searchResult && len(ss.ranges) == 0 +} + +// Strings returns the numset in zero or more strings of maxSize bytes. If +// maxSize is <= 0, a single string is returned. +func (ss numSet) Strings(maxSize int) []string { + if ss.searchResult { + return []string{"$"} + } + var l []string + var line string + for _, r := range ss.ranges { + s := "" if r.first.star { s += "*" } else { @@ -98,16 +114,41 @@ func (ss numSet) String() string { if r.first.star { panic("invalid numSet range first star without last") } + } else { + s += ":" + if r.last.star { + s += "*" + } else { + s += fmt.Sprintf("%d", r.last.number) + } + } + + nsize := len(line) + len(s) + if line != "" { + nsize++ // comma + } + if maxSize > 0 && nsize > maxSize { + l = append(l, line) + line = s continue } - s += ":" - if r.last.star { - s += "*" - } else { - s += fmt.Sprintf("%d", r.last.number) + if line != "" { + line += "," } + line += s } - return s + if line != "" { + l = append(l, line) + } + return l +} + +func (ss numSet) String() string { + l := ss.Strings(0) + if len(l) == 0 { + return "" + } + return l[0] } type setNumber struct { @@ -120,6 +161,127 @@ type numRange struct { last *setNumber // if nil, this numRange is just a setNumber in "first" and first.star will be false } +// interpretStar returns a numset that interprets stars in a numset, returning a new +// numset without stars with increasing first/last. +func (s numSet) interpretStar(uids []store.UID) numSet { + var ns numSet + for _, r := range s.ranges { + first := r.first.number + if r.first.star { + if len(uids) == 0 { + continue + } + first = uint32(uids[0]) + } + last := first + if r.last != nil { + last = r.last.number + if r.last.star { + if len(uids) == 0 { + continue + } + last = uint32(uids[len(uids)-1]) + if first > last { + first = last + } + } else if r.first.star && last < first { + last = first + } + } + if first > last { + first, last = last, first + } + nr := numRange{first: setNumber{number: first}} + if first != last { + nr.last = &setNumber{number: last} + } + ns.ranges = append(ns.ranges, nr) + } + return ns +} + +// whether numSet only has numbers (no star/search), and is strictly increasing. +func (s *numSet) isBasicIncreasing() bool { + if s.searchResult { + return false + } + var last uint32 + for _, r := range s.ranges { + if r.first.star || r.first.number <= last || r.last != nil && (r.last.star || r.last.number < r.first.number) { + return false + } + last = r.first.number + if r.last != nil { + last = r.last.number + } + } + return true +} + +type numIter struct { + s numSet + i int + r *rangeIter +} + +// newIter must only be called on a numSet that is basic (no star/search) and ascending. +func (s numSet) newIter() *numIter { + return &numIter{s: s, i: 0, r: s.ranges[0].newIter()} +} + +func (i *numIter) Next() (uint32, bool) { + if v, ok := i.r.Next(); ok { + return v, ok + } + i.i++ + if i.i >= len(i.s.ranges) { + return 0, false + } + i.r = i.s.ranges[i.i].newIter() + return i.r.Next() +} + +type rangeIter struct { + r numRange + o int +} + +// newIter must only be called on a range in a numSet that is basic (no star/search) and ascending. +func (r numRange) newIter() *rangeIter { + return &rangeIter{r: r, o: 0} +} + +func (r *rangeIter) Next() (uint32, bool) { + if r.o == 0 { + r.o++ + return r.r.first.number, true + } + if r.r.last == nil || r.r.first.number+uint32(r.o) > r.r.last.number { + return 0, false + } + v := r.r.first.number + uint32(r.o) + r.o++ + return v, true +} + +// append adds a new number to the set, extending a range, or starting a new one (possibly the first). +// can only be used on basic numsets, without star/searchResult. +func (s *numSet) append(v uint32) { + if len(s.ranges) == 0 { + s.ranges = []numRange{{first: setNumber{number: v}}} + return + } + ri := len(s.ranges) - 1 + r := s.ranges[ri] + if v == r.first.number+1 && r.last == nil { + s.ranges[ri].last = &setNumber{number: v} + } else if r.last != nil && v == r.last.number+1 { + r.last.number++ + } else { + s.ranges = append(s.ranges, numRange{first: setNumber{number: v}}) + } +} + type partial struct { offset uint32 count uint32 @@ -156,17 +318,18 @@ type fetchAtt struct { type searchKey struct { // Only one of searchKeys, seqSet and op can be non-nil/non-empty. - searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command. - seqSet *numSet // In case of bare sequence set. For op UID, field uidSet contains the parameter. - op string // Determines which of the fields below are set. - headerField string - astring string - date time.Time - atom string - number int64 - searchKey *searchKey - searchKey2 *searchKey - uidSet numSet + searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command. + seqSet *numSet // In case of bare sequence set. For op UID, field uidSet contains the parameter. + op string // Determines which of the fields below are set. + headerField string + astring string + date time.Time + atom string + number int64 + searchKey *searchKey + searchKey2 *searchKey + uidSet numSet + clientModseq *int64 } func compactUIDSet(l []store.UID) (r numSet) { diff --git a/imapserver/protocol_test.go b/imapserver/protocol_test.go index ff1caa8..c53defe 100644 --- a/imapserver/protocol_test.go +++ b/imapserver/protocol_test.go @@ -62,3 +62,30 @@ func TestNumSetContains(t *testing.T) { check(!ss3.containsUID(1, []store.UID{2, 3}, nil)) check(!ss3.containsUID(3, []store.UID{1, 2, 3}, nil)) } + +func TestNumSetInterpret(t *testing.T) { + parseNumSet := func(s string) numSet { + p := parser{upper: s} + return p.xnumSet0(true, false) + } + + checkEqual := func(uids []store.UID, a, s string) { + t.Helper() + n := parseNumSet(a).interpretStar(uids) + ns := n.String() + if ns != s { + t.Fatalf("%s != %s", ns, s) + } + } + + checkEqual([]store.UID{}, "1:*", "") + checkEqual([]store.UID{1}, "1:*", "1") + checkEqual([]store.UID{1, 3}, "1:*", "1:3") + checkEqual([]store.UID{1, 3}, "4:*", "3") + checkEqual([]store.UID{2, 3}, "*:4", "2:4") + checkEqual([]store.UID{2, 3}, "*:1", "2") + checkEqual([]store.UID{1, 2, 3}, "1,2,3", "1,2,3") + checkEqual([]store.UID{}, "1,2,3", "1,2,3") + checkEqual([]store.UID{}, "1:3", "1:3") + checkEqual([]store.UID{}, "3:1", "1:3") +} diff --git a/imapserver/search.go b/imapserver/search.go index c58aec9..11c0268 100644 --- a/imapserver/search.go +++ b/imapserver/search.go @@ -96,6 +96,7 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { } var expungeIssued bool + var maxModSeq store.ModSeq var uids []store.UID c.xdbread(func(tx *bstore.Tx) { @@ -108,8 +109,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { if eargs == nil || max == 0 || len(eargs) != 1 { for i, uid := range c.uids { lastIndex = i - if c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued) { + if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued); match { uids = append(uids, uid) + if modseq > maxModSeq { + maxModSeq = modseq + } if min == 1 && min+max == len(eargs) { break } @@ -119,8 +123,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { // And reverse search for MAX if we have only MAX or MAX combined with MIN. if max == 1 && (len(eargs) == 1 || min+max == len(eargs)) { for i := len(c.uids) - 1; i > lastIndex; i-- { - if c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued) { + if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued); match { uids = append(uids, c.uids[i]) + if modseq > maxModSeq { + maxModSeq = modseq + } break } } @@ -147,11 +154,24 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { } s += " " + fmt.Sprintf("%d", v) } + + // Since we don't have the max modseq for the possibly partial uid range we're + // writing here within hand reach, we conveniently interpret the ambiguous "for all + // messages being returned" in ../rfc/7162:1107 as meaning over all lines that we + // write. And that clients only commit this value after they have seen the tagged + // end of the command. Appears to be recommended behaviour, ../rfc/7162:2323. + // ../rfc/7162:1077 ../rfc/7162:1101 + var modseq string + if sk.hasModseq() { + // ../rfc/7162:2557 + modseq = fmt.Sprintf(" (MODSEQ %d)", maxModSeq.Client()) + } + + c.bwritelinef("* SEARCH%s%s", s, modseq) uids = uids[n:] - c.bwritelinef("* SEARCH%s", s) } } else { - // New-style ESEARCH response. ../rfc/9051:6546 ../rfc/4466:522 + // New-style ESEARCH response syntax: ../rfc/9051:6546 ../rfc/4466:522 if save { // ../rfc/9051:3784 ../rfc/5182:13 @@ -195,6 +215,13 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) { if eargs["ALL"] && len(uids) > 0 { resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String()) } + + // Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273 + // Summary: send the highest modseq of the returned messages. + if sk.hasModseq() && len(uids) > 0 { + resp += fmt.Sprintf(" MODSEQ %d", maxModSeq.Client()) + } + c.bwritelinef("%s", resp) } } @@ -215,10 +242,11 @@ type search struct { m store.Message p *message.Part expungeIssued *bool + hasModseq bool } -func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) bool { - s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued} +func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) (bool, store.ModSeq) { + s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued, hasModseq: sk.hasModseq()} defer func() { if s.mr != nil { err := s.mr.Close() @@ -229,12 +257,42 @@ func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKe return s.match(sk) } -func (s *search) match(sk searchKey) bool { +func (s *search) match(sk searchKey) (match bool, modseq store.ModSeq) { + // Instead of littering all the cases in match0 with calls to get modseq, we do it once + // here in case of a match. + defer func() { + if match && s.hasModseq { + if s.m.ID == 0 { + match = s.xloadMessage() + } + modseq = s.m.ModSeq + } + }() + + match = s.match0(sk) + return +} + +func (s *search) xloadMessage() bool { + q := bstore.QueryTx[store.Message](s.tx) + q.FilterNonzero(store.Message{MailboxID: s.c.mailboxID, UID: s.uid}) + m, err := q.Get() + if err == bstore.ErrAbsent || err == nil && m.Expunged { + // ../rfc/2180:607 + *s.expungeIssued = true + return false + } + xcheckf(err, "get message") + s.m = m + return true +} + +func (s *search) match0(sk searchKey) bool { c := s.c if sk.searchKeys != nil { for _, ssk := range sk.searchKeys { - if !s.match(ssk) { + if !s.match0(ssk) { return false } } @@ -273,33 +331,26 @@ func (s *search) match(sk searchKey) bool { // We do not implement the RECENT flag. All messages are not recent. return false case "NOT": - return !s.match(*sk.searchKey) + return !s.match0(*sk.searchKey) case "OR": - return s.match(*sk.searchKey) || s.match(*sk.searchKey2) + return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2) case "UID": return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult) } // Parsed message. if s.mr == nil { - q := bstore.QueryTx[store.Message](s.tx) - q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: s.uid}) - m, err := q.Get() - if err == bstore.ErrAbsent { - // ../rfc/2180:607 - *s.expungeIssued = true + if !s.xloadMessage() { return false } - xcheckf(err, "get message") - s.m = m // Closed by searchMatch after all (recursive) search.match calls are finished. - s.mr = c.account.MessageReader(m) + s.mr = c.account.MessageReader(s.m) - if m.ParsedBuf == nil { + if s.m.ParsedBuf == nil { c.log.Error("missing parsed message") } else { - p, err := m.LoadPart(s.mr) + p, err := s.m.LoadPart(s.mr) xcheckf(err, "load parsed message") s.p = &p } @@ -385,6 +436,9 @@ func (s *search) match(sk searchKey) bool { return s.m.Size > sk.number case "SMALLER": return s.m.Size < sk.number + case "MODSEQ": + // ../rfc/7162:1045 + return s.m.ModSeq.Client() >= *sk.clientModseq } if s.p == nil { diff --git a/imapserver/search_test.go b/imapserver/search_test.go index 47128f0..4b12ae2 100644 --- a/imapserver/search_test.go +++ b/imapserver/search_test.go @@ -39,6 +39,16 @@ func (tc *testconn) xsearch(nums ...uint32) { tc.xuntagged(imapclient.UntaggedSearch(nums)) } +func (tc *testconn) xsearchmodseq(modseq int64, nums ...uint32) { + tc.t.Helper() + + if len(nums) == 0 { + tc.xnountagged() + return + } + tc.xuntagged(imapclient.UntaggedSearchModSeq{Nums: nums, ModSeq: modseq}) +} + func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) { tc.t.Helper() @@ -220,43 +230,11 @@ func TestSearch(t *testing.T) { tc.transactf("ok", `search charset utf-8 text "mox"`) tc.xsearch(2, 3) - // esearchall makes an UntaggedEsearch response with All set, for comparisons. - esearchall0 := func(ss string) imapclient.NumSet { - seqset := imapclient.NumSet{} - for _, rs := range strings.Split(ss, ",") { - t := strings.Split(rs, ":") - if len(t) > 2 { - panic("bad seqset") - } - var first uint32 - var last *uint32 - if t[0] != "*" { - v, err := strconv.ParseUint(t[0], 10, 32) - if err != nil { - panic("parse first") - } - first = uint32(v) - } - if len(t) == 2 { - if t[1] != "*" { - v, err := strconv.ParseUint(t[1], 10, 32) - if err != nil { - panic("parse last") - } - u := uint32(v) - last = &u - } - } - seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last}) - } - return seqset - } - esearchall := func(ss string) imapclient.UntaggedEsearch { return imapclient.UntaggedEsearch{All: esearchall0(ss)} } - uintptr := func(v uint32) *uint32 { + uint32ptr := func(v uint32) *uint32 { return &v } @@ -265,10 +243,10 @@ func TestSearch(t *testing.T) { tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit. tc.transactf("ok", "search return (min max count all) all") - tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(3), All: esearchall0("1:3")}) + tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(3), All: esearchall0("1:3")}) tc.transactf("ok", "UID search return (min max count all) all") - tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(3), All: esearchall0("5:7")}) + tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")}) tc.transactf("ok", "search return (min) all") tc.xesearch(imapclient.UntaggedEsearch{Min: 1}) @@ -304,19 +282,19 @@ func TestSearch(t *testing.T) { tc.xesearch(imapclient.UntaggedEsearch{}) tc.transactf("ok", "search return (min max all count) not all") - tc.xesearch(imapclient.UntaggedEsearch{Count: uintptr(0)}) + tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(0)}) tc.transactf("ok", "search return (min max count all) 1,3") - tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")}) + tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")}) tc.transactf("ok", "search return (min max count all) UID 5,7") - tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")}) + tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")}) tc.transactf("ok", "uid search return (min max count all) 1,3") - tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")}) + tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")}) tc.transactf("ok", "uid search return (min max count all) UID 5,7") - tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")}) + tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")}) tc.transactf("no", `search return () charset unknown text "mox"`) tc.transactf("ok", `search return () charset us-ascii text "mox"`) @@ -353,3 +331,35 @@ func TestSearch(t *testing.T) { tc.transactf("ok", `search undraft`) tc.xesearch(esearchall("1:2")) } + +// esearchall makes an UntaggedEsearch response with All set, for comparisons. +func esearchall0(ss string) imapclient.NumSet { + seqset := imapclient.NumSet{} + for _, rs := range strings.Split(ss, ",") { + t := strings.Split(rs, ":") + if len(t) > 2 { + panic("bad seqset") + } + var first uint32 + var last *uint32 + if t[0] != "*" { + v, err := strconv.ParseUint(t[0], 10, 32) + if err != nil { + panic("parse first") + } + first = uint32(v) + } + if len(t) == 2 { + if t[1] != "*" { + v, err := strconv.ParseUint(t[1], 10, 32) + if err != nil { + panic("parse last") + } + u := uint32(v) + last = &u + } + } + seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last}) + } + return seqset +} diff --git a/imapserver/server.go b/imapserver/server.go index 47f9395..690a3cf 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -24,15 +24,14 @@ not in IMAP4rev2). ../rfc/3501:964 - When handling commands that modify the selected mailbox, always check that the mailbox is not opened readonly. And always revalidate the selected mailbox, another session may have deleted the mailbox. - After making changes to an account/mailbox/message, you must broadcast changes. You must do this with the account lock held. Otherwise, other later changes (e.g. message deliveries) may be made and broadcast before changes that were made earlier. Make sure to commit changes in the database first, because the commit may fail. - Mailbox hierarchies are slash separated, no leading slash. We keep the case, except INBOX is renamed to Inbox, also for submailboxes in INBOX. We don't allow existence of a child where its parent does not exist. We have no \NoInferiors or \NoSelect. Newly created mailboxes are automatically subscribed. +- For CONDSTORE and QRESYNC support, we set "modseq" for each change/expunge. Once expunged, a modseq doesn't change anymore. We don't yet remove old expunged records. The records aren't too big. Next step may be to let an admin reclaim space manually. */ /* - todo: do not return binary data for a fetch body. at least not for imap4rev1. we should be encoding it as base64? - todo: on expunge we currently remove the message even if other sessions still have a reference to the uid. if they try to query the uid, they'll get an error. we could be nicer and only actually remove the message when the last reference has gone. we could add a new flag to store.Message marking the message as expunged, not give new session access to such messages, and make store remove them at startup, and clean them when the last session referencing the session goes. however, it will get much more complicated. renaming messages would need special handling. and should we do the same for removed mailboxes? -- todo: CONDSTORE, QRESYNC. Add fields modseq on mailbox and each message. Keep (log of) deleted messages and their modseqs. - todo: try to recover from syntax errors when the last command line ends with a }, i.e. a literal. we currently abort the entire connection. we may want to read some amount of literal data and continue with a next command. - future: more extensions: STATUS=SIZE, OBJECTID, MULTISEARCH, REPLACE, NOTIFY, CATENATE, MULTIAPPEND, SORT, THREAD, CREATE-SPECIAL-USE. -- future: implement user-defined keyword flags? ../rfc/9051:566 */ import ( @@ -60,6 +59,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "golang.org/x/exp/maps" "golang.org/x/exp/slices" "golang.org/x/text/unicode/norm" @@ -155,7 +155,9 @@ var authFailDelay = time.Second // After authentication failure. // AUTH=SCRAM-SHA-1: ../rfc/5802 // AUTH=CRAM-MD5: ../rfc/2195 // APPENDLIMIT, we support the max possible size, 1<<63 - 1: ../rfc/7889:129 -const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ONLY LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807" +// CONDSTORE: ../rfc/7162:411 +// QRESYNC: ../rfc/7162:1323 +const serverCapabilities = "IMAP4rev2 IMAP4rev1 ENABLE LITERAL+ IDLE SASL-IR BINARY UNSELECT UIDPLUS ESEARCH SEARCHRES MOVE UTF8=ONLY LIST-EXTENDED SPECIAL-USE LIST-STATUS AUTH=SCRAM-SHA-256 AUTH=SCRAM-SHA-1 AUTH=CRAM-MD5 ID APPENDLIMIT=9223372036854775807 CONDSTORE QRESYNC" type conn struct { cid int64 @@ -207,6 +209,8 @@ type capability string const ( capIMAP4rev2 capability = "IMAP4REV2" capUTF8Accept capability = "UTF8=ACCEPT" + capCondstore capability = "CONDSTORE" + capQresync capability = "QRESYNC" ) type lineErr struct { @@ -444,6 +448,7 @@ func (c *conn) xtrace(level mlog.Level) func() { } // Cache of line buffers for reading commands. +// QRESYNC recommends 8k max line lengths. ../rfc/7162:2159 var bufpool = moxio.NewBufpool(8, 16*1024) // read line from connection, not going through line channel. @@ -524,11 +529,11 @@ func (c *conn) writeresultf(format string, args ...any) { c.xflush() } -// write buffered taggedcommand response, but first write pending changes. +// write buffered tagged command response, but first write pending changes. func (c *conn) bwriteresultf(format string, args ...any) { switch c.cmd { case "fetch", "store", "search": - // ../rfc/9051:5862 + // ../rfc/9051:5862 ../rfc/7162:2033 default: if c.comm != nil { c.applyChanges(c.comm.Get(), false) @@ -583,6 +588,19 @@ func (c *conn) xreadliteral(size int64, sync bool) string { return string(buf) } +func (c *conn) xhighestModSeq(tx *bstore.Tx, mailboxID int64) store.ModSeq { + qms := bstore.QueryTx[store.Message](tx) + qms.FilterNonzero(store.Message{MailboxID: mailboxID}) + qms.SortDesc("ModSeq") + qms.Limit(1) + m, err := qms.Get() + if err == bstore.ErrAbsent { + return store.ModSeq(0) + } + xcheckf(err, "looking up highest modseq for mailbox") + return m.ModSeq +} + var cleanClose struct{} // Sentinel value for panic/recover indicating clean close of connection. func serve(listenerName string, cid int64, tlsConfig *tls.Config, nc net.Conn, xtls, noRequireSTARTTLS bool) { @@ -1208,6 +1226,9 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { } changes = n + qresync := c.enabled[capQresync] + condstore := c.enabled[capCondstore] + i := 0 for i < len(changes) { // First process all new uids. So we only send a single EXISTS. @@ -1234,7 +1255,11 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { c.bwritelinef("* %d EXISTS", len(c.uids)) for _, add := range adds { seq := c.xsequence(add.UID) - c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", seq, add.UID, flaglist(add.Flags, add.Keywords).pack(c)) + var modseqStr string + if condstore { + modseqStr = fmt.Sprintf(" MODSEQ (%d)", add.ModSeq.Client()) + } + c.bwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, add.UID, flaglist(add.Flags, add.Keywords).pack(c), modseqStr) } continue } @@ -1244,6 +1269,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { switch ch := change.(type) { case store.ChangeRemoveUIDs: + var vanishedUIDs numSet for _, uid := range ch.UIDs { var seq msgseq if initial { @@ -1256,7 +1282,17 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { } c.sequenceRemove(seq, uid) if !initial { - c.bwritelinef("* %d EXPUNGE", seq) + if qresync { + vanishedUIDs.append(uint32(uid)) + } else { + c.bwritelinef("* %d EXPUNGE", seq) + } + } + } + if qresync { + // VANISHED without EARLIER. ../rfc/7162:2004 + for _, s := range vanishedUIDs.Strings(4*1024 - 32) { + c.bwritelinef("* VANISHED %s", s) } } case store.ChangeFlags: @@ -1266,7 +1302,11 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { continue } if !initial { - c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c)) + var modseqStr string + if condstore { + modseqStr = fmt.Sprintf(" MODSEQ (%d)", ch.ModSeq.Client()) + } + c.bwritelinef("* %d FETCH (UID %d FLAGS %s%s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c), modseqStr) } case store.ChangeRemoveMailbox: // Only announce \NonExistent to modern clients, otherwise they may ignore the @@ -1786,20 +1826,56 @@ func (c *conn) cmdEnable(tag, cmd string, p *parser) { // Clients should only send capabilities that need enabling. // We should only echo that we recognize as needing enabling. var enabled string + var qresync bool for _, s := range caps { cap := capability(strings.ToUpper(s)) switch cap { - case capIMAP4rev2, capUTF8Accept: + case capIMAP4rev2, + capUTF8Accept, + capCondstore: // ../rfc/7162:384 c.enabled[cap] = true enabled += " " + s + case capQresync: + c.enabled[cap] = true + enabled += " " + s + qresync = true } } + // QRESYNC enabled CONDSTORE too ../rfc/7162:1391 + if qresync && !c.enabled[capCondstore] { + c.xensureCondstore(nil) + enabled += " CONDSTORE" + } // Response syntax: ../rfc/9051:6520 ../rfc/5161:211 c.bwritelinef("* ENABLED%s", enabled) c.ok(tag, cmd) } +// The CONDSTORE extension can be enabled in many different ways. ../rfc/7162:368 +// If a mailbox is selected, an untagged OK with HIGHESTMODSEQ is written to the +// client. If tx is non-nil, it is used to read the HIGHESTMODSEQ from the +// database. Otherwise a new read-only transaction is created. +func (c *conn) xensureCondstore(tx *bstore.Tx) { + if !c.enabled[capCondstore] { + c.enabled[capCondstore] = true + // todo spec: can we send an untagged enabled response? + // ../rfc/7162:603 + if c.mailboxID <= 0 { + return + } + var modseq store.ModSeq + if tx != nil { + modseq = c.xhighestModSeq(tx, c.mailboxID) + } else { + c.xdbread(func(tx *bstore.Tx) { + modseq = c.xhighestModSeq(tx, c.mailboxID) + }) + } + c.bwritelinef("* OK [HIGHESTMODSEQ %d] after condstore-enabling command", modseq.Client()) + } +} + // State: Authenticated and selected. func (c *conn) cmdSelect(tag, cmd string, p *parser) { c.cmdSelectExamine(true, tag, cmd, p) @@ -1815,27 +1891,82 @@ func (c *conn) cmdExamine(tag, cmd string, p *parser) { // // State: Authenticated and selected. func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) { - // Select command: ../rfc/9051:1754 ../rfc/3501:1743 + // Select command: ../rfc/9051:1754 ../rfc/3501:1743 ../rfc/7162:1146 ../rfc/7162:1432 // Examine command: ../rfc/9051:1868 ../rfc/3501:1855 - // Select examples: ../rfc/9051:1831 ../rfc/3501:1826 + // Select examples: ../rfc/9051:1831 ../rfc/3501:1826 ../rfc/7162:1159 ../rfc/7162:1479 - // Select request syntax: ../rfc/9051:7005 ../rfc/3501:4996 + // Select request syntax: ../rfc/9051:7005 ../rfc/3501:4996 ../rfc/4466:652 ../rfc/7162:2559 ../rfc/7162:2598 // Examine request syntax: ../rfc/9051:6551 ../rfc/3501:4746 p.xspace() name := p.xmailbox() + + var qruidvalidity uint32 + var qrmodseq int64 // QRESYNC required parameters. + var qrknownUIDs, qrknownSeqSet, qrknownUIDSet *numSet // QRESYNC optional parameters. + if p.space() { + seen := map[string]bool{} + p.xtake("(") + for len(seen) == 0 || !p.take(")") { + w := p.xtakelist("CONDSTORE", "QRESYNC") + if seen[w] { + xsyntaxErrorf("duplicate select parameter %s", w) + } + seen[w] = true + + switch w { + case "CONDSTORE": + // ../rfc/7162:363 + c.xensureCondstore(nil) // ../rfc/7162:373 + case "QRESYNC": + // ../rfc/7162:2598 + // Note: unlike with CONDSTORE, there are no QRESYNC-related commands/parameters + // that enable capabilities. + if !c.enabled[capQresync] { + // ../rfc/7162:1446 + xsyntaxErrorf("QRESYNC must first be enabled") + } + p.xspace() + p.xtake("(") + qruidvalidity = p.xnznumber() // ../rfc/7162:2606 + p.xspace() + qrmodseq = p.xnznumber64() + if p.take(" ") { + seqMatchData := p.take("(") + if !seqMatchData { + ss := p.xnumSet0(false, false) // ../rfc/7162:2608 + qrknownUIDs = &ss + seqMatchData = p.take(" (") + } + if seqMatchData { + ss0 := p.xnumSet0(false, false) + qrknownSeqSet = &ss0 + p.xspace() + ss1 := p.xnumSet0(false, false) + qrknownUIDSet = &ss1 + p.xtake(")") + } + } + p.xtake(")") + default: + panic("missing case for select param " + w) + } + } + } p.xempty() // Deselect before attempting the new select. This means we will deselect when an // error occurs during select. // ../rfc/9051:1809 if c.state == stateSelected { - // ../rfc/9051:1812 + // ../rfc/9051:1812 ../rfc/7162:2111 c.bwritelinef("* OK [CLOSED] x") c.unselect() } name = xcheckmailboxname(name, true) + var highestModSeq store.ModSeq + var highDeletedModSeq store.ModSeq var firstUnseen msgseq = 0 var mb store.Mailbox c.account.WithRLock(func() { @@ -1844,6 +1975,7 @@ func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) { q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: mb.ID}) + q.FilterEqual("Expunged", false) q.SortAsc("UID") c.uids = []store.UID{} var seq msgseq = 1 @@ -1859,6 +1991,17 @@ func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) { checkUIDs(c.uids) } xcheckf(err, "fetching uids") + + // Condstore extension, find the highest modseq. + if c.enabled[capCondstore] { + highestModSeq = c.xhighestModSeq(tx, mb.ID) + } + // For QRESYNC, we need to know the highest modset of deleted expunged records to + // maintain synchronization. + if c.enabled[capQresync] { + highDeletedModSeq, err = c.account.HighestDeletedModSeq(tx) + xcheckf(err, "getting highest deleted modseq") + } }) }) c.applyChanges(c.comm.Get(), true) @@ -1880,6 +2023,151 @@ func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) { c.bwritelinef(`* OK [UIDVALIDITY %d] x`, mb.UIDValidity) c.bwritelinef(`* OK [UIDNEXT %d] x`, mb.UIDNext) c.bwritelinef(`* LIST () "/" %s`, astring(mb.Name).pack(c)) + if c.enabled[capCondstore] { + // ../rfc/7162:417 + // ../rfc/7162-eid5055 ../rfc/7162:484 ../rfc/7162:1167 + c.bwritelinef(`* OK [HIGHESTMODSEQ %d] x`, highestModSeq.Client()) + } + + // If QRESYNC uidvalidity matches, we send any changes. ../rfc/7162:1509 + if qruidvalidity == mb.UIDValidity { + // We send the vanished UIDs at the end, so we can easily combine the modseq + // changes and vanished UIDs that result from that, with the vanished UIDs from the + // case where we don't store enough history. + vanishedUIDs := map[store.UID]struct{}{} + + var preVanished store.UID + var oldClientUID store.UID + // If samples of known msgseq and uid pairs are given (they must be in order), we + // use them to determine the earliest UID for which we send VANISHED responses. + // ../rfc/7162:1579 + if qrknownSeqSet != nil { + if !qrknownSeqSet.isBasicIncreasing() { + xuserErrorf("QRESYNC known message sequence set must be numeric and strictly increasing") + } + if !qrknownUIDSet.isBasicIncreasing() { + xuserErrorf("QRESYNC known uid set must be numeric and strictly increasing") + } + seqiter := qrknownSeqSet.newIter() + uiditer := qrknownUIDSet.newIter() + for { + msgseq, ok0 := seqiter.Next() + uid, ok1 := uiditer.Next() + if !ok0 && !ok1 { + break + } else if !ok0 || !ok1 { + xsyntaxErrorf("invalid combination of known sequence set and uid set, must be of equal length") + } + i := int(msgseq - 1) + if i < 0 || i >= len(c.uids) || c.uids[i] != store.UID(uid) { + if uidSearch(c.uids, store.UID(uid)) <= 0 { + // We will check this old client UID for consistency below. + oldClientUID = store.UID(uid) + } + break + } + preVanished = store.UID(uid + 1) + } + } + + // We gather vanished UIDs and report them at the end. This seems OK because we + // already sent HIGHESTMODSEQ, and a client should know not to commit that value + // until after it has seen the tagged OK of this command. The RFC has a remark + // about ordering of some untagged responses, it's not immediately clear what it + // means, but given the examples appears to allude to servers that decide to not + // send expunge/vanished before the tagged OK. + // ../rfc/7162:1340 + + // We are reading without account lock. Similar to when we process FETCH/SEARCH + // requests. We don't have to reverify existence of the mailbox, so we don't + // rlock, even briefly. + c.xdbread(func(tx *bstore.Tx) { + if oldClientUID > 0 { + // The client sent a UID that is now removed. This is typically fine. But we check + // that it is consistent with the modseq the client sent. If the UID already didn't + // exist at that modseq, the client may be missing some information. + q := bstore.QueryTx[store.Message](tx) + q.FilterNonzero(store.Message{MailboxID: mb.ID, UID: oldClientUID}) + m, err := q.Get() + if err == nil { + // If client claims to be up to date up to and including qrmodseq, and the message + // was deleted at or before that time, we send changes from just before that + // modseq, and we send vanished for all UIDs. + if m.Expunged && qrmodseq >= m.ModSeq.Client() { + qrmodseq = m.ModSeq.Client() - 1 + preVanished = 0 + qrknownUIDs = nil + c.bwritelinef("* OK [ALERT] Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended.") + } + } else if err != bstore.ErrAbsent { + xcheckf(err, "checking old client uid") + } + } + + q := bstore.QueryTx[store.Message](tx) + q.FilterNonzero(store.Message{MailboxID: mb.ID}) + // Note: we don't filter by Expunged. + q.FilterGreater("ModSeq", store.ModSeqFromClient(qrmodseq)) + q.FilterLessEqual("ModSeq", highestModSeq) + q.SortAsc("ModSeq") + err := q.ForEach(func(m store.Message) error { + if m.Expunged && m.UID < preVanished { + return nil + } + // If known UIDs was specified, we only report about those UIDs. ../rfc/7162:1523 + if qrknownUIDs != nil && !qrknownUIDs.contains(uint32(m.UID)) { + return nil + } + if m.Expunged { + vanishedUIDs[m.UID] = struct{}{} + return nil + } + msgseq := c.sequence(m.UID) + if msgseq > 0 { + c.bwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", msgseq, m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client()) + } + return nil + }) + xcheckf(err, "listing changed messages") + }) + + // Add UIDs from client's known UID set to vanished list if we don't have enough history. + if qrmodseq < highDeletedModSeq.Client() { + // If no known uid set was in the request, we substitute 1:max or the empty set. + // ../rfc/7162:1524 + if qrknownUIDs == nil { + if len(c.uids) > 0 { + qrknownUIDs = &numSet{ranges: []numRange{{first: setNumber{number: 1}, last: &setNumber{number: uint32(c.uids[len(c.uids)-1])}}}} + } else { + qrknownUIDs = &numSet{} + } + } + + iter := qrknownUIDs.newIter() + for { + v, ok := iter.Next() + if !ok { + break + } + if c.sequence(store.UID(v)) <= 0 { + vanishedUIDs[store.UID(v)] = struct{}{} + } + } + } + + // Now that we have all vanished UIDs, send them over compactly. + if len(vanishedUIDs) > 0 { + l := maps.Keys(vanishedUIDs) + sort.Slice(l, func(i, j int) bool { + return l[i] < l[j] + }) + // ../rfc/7162:1985 + for _, s := range compactUIDSet(l).Strings(4*1024 - 32) { + c.bwritelinef("* VANISHED (EARLIER) %s", s) + } + } + } + if isselect { c.bwriteresultf("%s OK [READ-WRITE] x", tag) c.readonly = false @@ -2008,7 +2296,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) { _, err = qm.Delete() xcheckf(err, "removing messages") - // Mark messages as not needing training. Then retrain them, so that are untrained if they were. + // Mark messages as not needing training. Then retrain them, so they are untrained if they were. for i := range remove { remove[i].Junk = false remove[i].Notjunk = false @@ -2089,20 +2377,36 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) { err = tx.Insert(&dstMB) xcheckf(err, "create new destination mailbox") + modseq, err := c.account.NextModSeq(tx) + xcheckf(err, "assigning next modseq") + // Move existing messages, with their ID's and on-disk files intact, to the new - // mailbox. + // mailbox. We keep the expunged messages, the destination mailbox doesn't care + // about them. var oldUIDs []store.UID q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: srcMB.ID}) + q.FilterEqual("Expunged", false) q.SortAsc("UID") err = q.ForEach(func(m store.Message) error { - oldUIDs = append(oldUIDs, m.UID) + om := m + om.ID = 0 + om.ModSeq = modseq + om.PrepareExpunge() + oldUIDs = append(oldUIDs, om.UID) + m.MailboxID = dstMB.ID m.UID = dstMB.UIDNext dstMB.UIDNext++ + m.CreateSeq = modseq + m.ModSeq = modseq if err := tx.Update(&m); err != nil { return fmt.Errorf("updating message to move to new mailbox: %w", err) } + + if err := tx.Insert(&om); err != nil { + return fmt.Errorf("adding empty expunge message record to inbox: %w", err) + } return nil }) xcheckf(err, "moving messages from inbox to destination mailbox") @@ -2115,7 +2419,7 @@ func (c *conn) cmdRename(tag, cmd string, p *parser) { dstFlags = []string{`\Subscribed`} } changes = []store.Change{ - store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs}, + store.ChangeRemoveUIDs{MailboxID: srcMB.ID, UIDs: oldUIDs, ModSeq: modseq}, store.ChangeAddMailbox{Name: dstMB.Name, Flags: dstFlags}, // todo: in future, we could announce all messages. no one is listening now though. } @@ -2373,8 +2677,8 @@ func (c *conn) cmdNamespace(tag, cmd string, p *parser) { // // State: Authenticated and selected. func (c *conn) cmdStatus(tag, cmd string, p *parser) { - // Command: ../rfc/9051:3328 ../rfc/3501:2424 - // Examples: ../rfc/9051:3400 ../rfc/3501:2501 + // Command: ../rfc/9051:3328 ../rfc/3501:2424 ../rfc/7162:1127 + // Examples: ../rfc/9051:3400 ../rfc/3501:2501 ../rfc/7162:1139 // Request syntax: ../rfc/9051:7053 ../rfc/3501:5036 p.xspace() @@ -2409,8 +2713,11 @@ func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) stri var count, unseen, deleted int var size int64 + // todo optimize: should probably cache the values instead of reading through the database. must then be careful to keep it consistent... + q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: mb.ID}) + q.FilterEqual("Expunged", false) err := q.ForEach(func(m store.Message) error { count++ if !m.Seen { @@ -2445,6 +2752,9 @@ func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) stri case "APPENDLIMIT": // ../rfc/7889:255 status = append(status, A, "NIL") + case "HIGHESTMODSEQ": + // ../rfc/7162:366 + status = append(status, A, fmt.Sprintf("%d", c.xhighestModSeq(tx, mb.ID).Client())) default: xsyntaxErrorf("unknown attribute %q", a) } @@ -2623,7 +2933,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { } // Broadcast the change to other connections. - c.broadcast([]store.Change{store.ChangeAddUID{MailboxID: mb.ID, UID: msg.UID, Flags: msg.Flags, Keywords: msg.Keywords}}) + c.broadcast([]store.Change{store.ChangeAddUID{MailboxID: mb.ID, UID: msg.UID, ModSeq: msg.ModSeq, Flags: msg.Flags, Keywords: msg.Keywords}}) }) err = msgFile.Close() @@ -2633,6 +2943,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { if c.mailboxID == mb.ID { c.applyChanges(pendingChanges, false) c.uidAppend(msg.UID) + // todo spec: with condstore/qresync, is there a mechanism to the client know the modseq for the appended uid? in theory an untagged fetch with the modseq after the OK APPENDUID could make sense, but this probably isn't allowed. c.bwritelinef("* %d EXISTS", len(c.uids)) } @@ -2708,7 +3019,7 @@ func (c *conn) cmdCheck(tag, cmd string, p *parser) { // // State: Selected func (c *conn) cmdClose(tag, cmd string, p *parser) { - // Command: ../rfc/9051:3636 ../rfc/3501:2652 + // Command: ../rfc/9051:3636 ../rfc/3501:2652 ../rfc/7162:1836 // Request syntax: ../rfc/9051:6476 ../rfc/3501:4679 p.xempty() @@ -2719,7 +3030,7 @@ func (c *conn) cmdClose(tag, cmd string, p *parser) { return } - remove := c.xexpunge(nil, true) + remove, _ := c.xexpunge(nil, true) defer func() { for _, m := range remove { @@ -2735,9 +3046,15 @@ func (c *conn) cmdClose(tag, cmd string, p *parser) { // expunge messages marked for deletion in currently selected/active mailbox. // if uidSet is not nil, only messages matching the set are deleted. -// messages that have been deleted from the database returned, but the corresponding files still have to be removed. -func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) []store.Message { - var remove []store.Message +// +// messages that have been marked expunged from the database are returned, but the +// corresponding files still have to be removed. +// +// the highest modseq in the mailbox is returned, typically associated with the +// removal of the messages, but if no messages were expunged the current latest max +// modseq for the mailbox is returned. +func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) (remove []store.Message, highestModSeq store.ModSeq) { + var modseq store.ModSeq c.account.WithWLock(func() { c.xdbwrite(func(tx *bstore.Tx) { @@ -2753,6 +3070,7 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) []store.Message { qm := bstore.QueryTx[store.Message](tx) qm.FilterNonzero(store.Message{MailboxID: c.mailboxID}) qm.FilterEqual("Deleted", true) + qm.FilterEqual("Expunged", false) qm.FilterFn(func(m store.Message) bool { // Only remove if this session knows about the message and if present in optional uidSet. return uidSearch(c.uids, m.UID) > 0 && (uidSet == nil || uidSet.containsUID(m.UID, c.uids, c.searchResult)) @@ -2762,9 +3080,15 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) []store.Message { xcheckf(err, "listing messages to delete") if len(remove) == 0 { + highestModSeq = c.xhighestModSeq(tx, c.mailboxID) return } + // Assign new modseq. + modseq, err = c.account.NextModSeq(tx) + xcheckf(err, "assigning next modseq") + highestModSeq = modseq + removeIDs := make([]int64, len(remove)) anyIDs := make([]any, len(remove)) for i, m := range remove { @@ -2778,17 +3102,17 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) []store.Message { qm = bstore.QueryTx[store.Message](tx) qm.FilterIDs(removeIDs) - _, err = qm.Delete() - xcheckf(err, "removing messages marked for deletion") + _, err = qm.UpdateNonzero(store.Message{Expunged: true, ModSeq: modseq}) + xcheckf(err, "marking messages marked for deleted as expunged") - // Mark removed messages as not needing training, then retrain them, so if they + // Mark expunged messages as not needing training, then retrain them, so if they // were trained, they get untrained. for i := range remove { remove[i].Junk = false remove[i].Notjunk = false } err = c.account.RetrainMessages(context.TODO(), c.log, tx, remove, true) - xcheckf(err, "untraining deleted messages") + xcheckf(err, "untraining expunged messages") }) // Broadcast changes to other connections. We may not have actually removed any @@ -2798,11 +3122,11 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) []store.Message { for i, m := range remove { ouids[i] = m.UID } - changes := []store.Change{store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: ouids}} + changes := []store.Change{store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: ouids, ModSeq: modseq}} c.broadcast(changes) } }) - return remove + return remove, highestModSeq } // Unselect is similar to close in that it closes the currently active mailbox, but @@ -2825,7 +3149,7 @@ func (c *conn) cmdUnselect(tag, cmd string, p *parser) { // // State: Selected func (c *conn) cmdExpunge(tag, cmd string, p *parser) { - // Command: ../rfc/9051:3687 ../rfc/3501:2695 + // Command: ../rfc/9051:3687 ../rfc/3501:2695 ../rfc/7162:1770 // Request syntax: ../rfc/9051:6476 ../rfc/3501:4679 p.xempty() @@ -2842,7 +3166,7 @@ func (c *conn) cmdExpunge(tag, cmd string, p *parser) { // // State: Selected func (c *conn) cmdUIDExpunge(tag, cmd string, p *parser) { - // Command: ../rfc/9051:4775 ../rfc/4315:75 + // Command: ../rfc/9051:4775 ../rfc/4315:75 ../rfc/7162:1873 // Request syntax: ../rfc/9051:7125 ../rfc/9051:7129 ../rfc/4315:298 p.xspace() @@ -2862,7 +3186,7 @@ func (c *conn) cmdUIDExpunge(tag, cmd string, p *parser) { func (c *conn) cmdxExpunge(tag, cmd string, uidSet *numSet) { // Command: ../rfc/9051:3687 ../rfc/3501:2695 - remove := c.xexpunge(uidSet, false) + remove, highestModSeq := c.xexpunge(uidSet, false) defer func() { for _, m := range remove { @@ -2873,13 +3197,29 @@ func (c *conn) cmdxExpunge(tag, cmd string, uidSet *numSet) { }() // Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 + var vanishedUIDs numSet + qresync := c.enabled[capQresync] for _, m := range remove { seq := c.xsequence(m.UID) c.sequenceRemove(seq, m.UID) - c.bwritelinef("* %d EXPUNGE", seq) + if qresync { + vanishedUIDs.append(uint32(m.UID)) + } else { + c.bwritelinef("* %d EXPUNGE", seq) + } + } + if !vanishedUIDs.empty() { + // VANISHED without EARLIER. ../rfc/7162:2004 + for _, s := range vanishedUIDs.Strings(4*1024 - 32) { + c.bwritelinef("* VANISHED %s", s) + } } - c.ok(tag, cmd) + if c.enabled[capCondstore] { + c.writeresultf("%s OK [HIGHESTMODSEQ %d] expunged", tag, highestModSeq.Client()) + } else { + c.ok(tag, cmd) + } } // State: Selected @@ -2987,6 +3327,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { var origUIDs, newUIDs []store.UID var flags []store.Flags var keywords [][]string + var modseq store.ModSeq // For messages in new mailbox, assigned when first message is copied. c.account.WithWLock(func() { c.xdbwrite(func(tx *bstore.Tx) { @@ -3000,6 +3341,10 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { xuserErrorf("no matching messages to copy") } + var err error + modseq, err = c.account.NextModSeq(tx) + xcheckf(err, "assigning next modseq") + // Reserve the uids in the destination mailbox. uidFirst := mbDst.UIDNext mbDst.UIDNext += store.UID(len(uidargs)) @@ -3008,6 +3353,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: c.mailboxID}) q.FilterEqual("UID", uidargs...) + q.FilterEqual("Expunged", false) xmsgs, err := q.List() xcheckf(err, "fetching messages") @@ -3036,6 +3382,8 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { origMsgIDs = append(origMsgIDs, origID) m.ID = 0 m.UID = uidFirst + store.UID(i) + m.CreateSeq = modseq + m.ModSeq = modseq m.MailboxID = mbDst.ID if mbSrc.Name == conf.RejectsMailbox && m.MailboxDestinedID != 0 { // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message @@ -3107,7 +3455,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { if len(newUIDs) > 0 { changes := make([]store.Change, len(newUIDs)) for i, uid := range newUIDs { - changes[i] = store.ChangeAddUID{MailboxID: mbDst.ID, UID: uid, Flags: flags[i], Keywords: keywords[i]} + changes[i] = store.ChangeAddUID{MailboxID: mbDst.ID, UID: uid, ModSeq: modseq, Flags: flags[i], Keywords: keywords[i]} } c.broadcast(changes) } @@ -3124,9 +3472,9 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { // // State: Selected func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { - // Command: ../rfc/9051:4650 ../rfc/6851:119 + // Command: ../rfc/9051:4650 ../rfc/6851:119 ../rfc/6851:265 - // Request syntax: ../rfc/6851:320 ../rfc/9051:6744 + // Request syntax: ../rfc/6851:320 p.xspace() nums := p.xnumSet() p.xspace() @@ -3144,6 +3492,7 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { var mbDst store.Mailbox var changes []store.Change var newUIDs []store.UID + var modseq store.ModSeq c.account.WithWLock(func() { c.xdbwrite(func(tx *bstore.Tx) { @@ -3162,10 +3511,19 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { uidnext := uidFirst mbDst.UIDNext += store.UID(len(uids)) - // Update UID and MailboxID in database for messages. + // Assign a new modseq, for the new records and for the expunged records. + var err error + modseq, err = c.account.NextModSeq(tx) + xcheckf(err, "assigning next modseq") + + // Update existing record with new UID and MailboxID in database for messages. We + // add a new but expunged record again in the original/source mailbox, for qresync. + // Keeping the original ID for the live message means we don't have to move the + // on-disk message contents file. q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: c.mailboxID}) q.FilterEqual("UID", uidargs...) + q.FilterEqual("Expunged", false) q.SortAsc("UID") msgs, err := q.List() xcheckf(err, "listing messages to move") @@ -3182,6 +3540,13 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { if m.UID != uids[i] { xserverErrorf("internal error: got uid %d, expected %d, for index %d", m.UID, uids[i], i) } + + // Copy of message record that we'll insert when UID is freed up. + om := *m + om.PrepareExpunge() + om.ID = 0 // Assign new ID. + om.ModSeq = modseq + m.MailboxID = mbDst.ID if mbSrc.Name == conf.RejectsMailbox && m.MailboxDestinedID != 0 { // Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message @@ -3189,11 +3554,16 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { m.MailboxOrigID = m.MailboxDestinedID } m.UID = uidnext + m.ModSeq = modseq m.JunkFlagsForMailbox(mbDst.Name, conf) uidnext++ err := tx.Update(m) xcheckf(err, "updating moved message in database") + // Now that UID is unused, we can insert the old record again. + err = tx.Insert(&om) + xcheckf(err, "inserting record for expunge after moving message") + for _, kw := range m.Keywords { keywords[kw] = struct{}{} } @@ -3214,10 +3584,10 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { // Prepare broadcast changes to other connections. changes = make([]store.Change, 0, 1+len(msgs)) - changes = append(changes, store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: uids}) + changes = append(changes, store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: uids, ModSeq: modseq}) for _, m := range msgs { newUIDs = append(newUIDs, m.UID) - changes = append(changes, store.ChangeAddUID{MailboxID: mbDst.ID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords}) + changes = append(changes, store.ChangeAddUID{MailboxID: mbDst.ID, UID: m.UID, ModSeq: modseq, Flags: m.Flags, Keywords: m.Keywords}) } }) @@ -3227,13 +3597,30 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { // ../rfc/9051:4708 ../rfc/6851:254 // ../rfc/9051:4713 c.bwritelinef("* OK [COPYUID %d %s %s] moved", mbDst.UIDValidity, compactUIDSet(uids).String(), compactUIDSet(newUIDs).String()) + qresync := c.enabled[capQresync] + var vanishedUIDs numSet for i := 0; i < len(uids); i++ { seq := c.xsequence(uids[i]) c.sequenceRemove(seq, uids[i]) - c.bwritelinef("* %d EXPUNGE", seq) + if qresync { + vanishedUIDs.append(uint32(uids[i])) + } else { + c.bwritelinef("* %d EXPUNGE", seq) + } + } + if !vanishedUIDs.empty() { + // VANISHED without EARLIER. ../rfc/7162:2004 + for _, s := range vanishedUIDs.Strings(4*1024 - 32) { + c.bwritelinef("* VANISHED %s", s) + } } - c.ok(tag, cmd) + if c.enabled[capQresync] { + // ../rfc/9051:6744 ../rfc/7162:1334 + c.writeresultf("%s OK [HIGHESTMODSEQ %d] move", tag, modseq.Client()) + } else { + c.ok(tag, cmd) + } } // Store sets a full set of flags, or adds/removes specific flags. @@ -3242,10 +3629,22 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { // Command: ../rfc/9051:4543 ../rfc/3501:3214 - // Request syntax: ../rfc/9051:7076 ../rfc/3501:5052 + // Request syntax: ../rfc/9051:7076 ../rfc/3501:5052 ../rfc/4466:691 ../rfc/7162:2471 p.xspace() nums := p.xnumSet() p.xspace() + var unchangedSince *int64 + if p.take("(") { + // ../rfc/7162:2471 + p.xtake("UNCHANGEDSINCE") + p.xspace() + v := p.xnumber64() + unchangedSince = &v + p.xtake(")") + p.xspace() + // UNCHANGEDSINCE is a CONDSTORE-enabling parameter ../rfc/7162:382 + c.xensureCondstore(nil) + } var plus, minus bool if p.take("+") { plus = true @@ -3281,6 +3680,9 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { } var updated []store.Message + var changed []store.Message // ModSeq more recent than unchangedSince, will be in MODIFIED response code, and we will send untagged fetch responses so client is up to date. + var modseq store.ModSeq // Assigned when needed. + modified := map[int64]bool{} c.account.WithWLock(func() { c.xdbwrite(func(tx *bstore.Tx) { @@ -3305,8 +3707,16 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: c.mailboxID}) q.FilterEqual("UID", uidargs...) + q.FilterEqual("Expunged", false) err := q.ForEach(func(m store.Message) error { + // Client may specify a message multiple times, but we only process it once. ../rfc/7162:823 + if modified[m.ID] { + return nil + } + + origFlags := m.Flags m.Flags = m.Flags.Set(mask, flags) + oldKeywords := append([]string{}, m.Keywords...) if minus { m.Keywords = store.RemoveKeywords(m.Keywords, keywords) } else if plus { @@ -3314,6 +3724,49 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { } else { m.Keywords = keywords } + + keywordsChanged := func() bool { + sort.Strings(oldKeywords) + n := append([]string{}, m.Keywords...) + sort.Strings(n) + return !slices.Equal(oldKeywords, n) + } + + // If the message has a more recent modseq than the check requires, we won't modify + // it and report in the final command response. + // ../rfc/7162:555 + // + // unchangedSince 0 always fails the check, we don't turn it into 1 like with our + // internal modseqs. RFC implies that is not required for non-system flags, but we + // don't have per-flag modseq and this seems reasonable. ../rfc/7162:640 + if unchangedSince != nil && m.ModSeq.Client() > *unchangedSince { + changed = append(changed, m) + return nil + } + + // Note: we don't perform the optimization described in ../rfc/7162:1258 + // It requires that we keep track of the flags we think the client knows (but only + // on this connection). We don't track that. It also isn't clear why this is + // allowed because it is skipping the condstore conditional check, and the new + // combination of flags could be unintended. + + // We do not assign a new modseq if nothing actually changed. ../rfc/7162:1246 ../rfc/7162:312 + if origFlags == m.Flags && !keywordsChanged() { + // Note: since we didn't update the modseq, we are not adding m.ID to "modified", + // it would skip the modseq check above. We still add m to list of updated, so we + // send an untagged fetch response. But we don't broadcast it. + updated = append(updated, m) + return nil + } + + // Assign new modseq for first actual change. + if modseq == 0 { + var err error + modseq, err = c.account.NextModSeq(tx) + xcheckf(err, "next modseq") + } + m.ModSeq = modseq + modified[m.ID] = true updated = append(updated, m) return tx.Update(&m) }) @@ -3324,19 +3777,71 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { }) // Broadcast changes to other connections. - changes := make([]store.Change, len(updated)) - for i, m := range updated { - changes[i] = store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: mask, Flags: m.Flags, Keywords: m.Keywords} + changes := make([]store.Change, 0, len(updated)) + for _, m := range updated { + // We only notify about flags that actually changed. + if m.ModSeq == modseq { + ch := store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords} + changes = append(changes, ch) + } + } + if len(changes) > 0 { + c.broadcast(changes) } - c.broadcast(changes) }) - for _, m := range updated { - if !silent { - // ../rfc/9051:6749 ../rfc/3501:4869 - c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", c.xsequence(m.UID), m.UID, flaglist(m.Flags, m.Keywords).pack(c)) + // In the RFC, the section about STORE/UID STORE says we must return MODSEQ when + // UNCHANGEDSINCE was specified. It does not specify it in case UNCHANGEDSINCE + // isn't specified. For that case it does say MODSEQ is needed in unsolicited + // untagged fetch responses. Implying that solicited untagged fetch responses + // should not include MODSEQ (why else mention unsolicited explicitly?). But, in + // the introduction to CONDSTORE it does explicitly specify MODSEQ should be + // included in untagged fetch responses at all times with CONDSTORE-enabled + // connections. It would have been better if the command behaviour was specified in + // the command section, not the introduction to the extension. + // ../rfc/7162:388 ../rfc/7162:852 + // ../rfc/7162:549 + if !silent || c.enabled[capCondstore] { + for _, m := range updated { + var flags string + if !silent { + flags = fmt.Sprintf(" FLAGS %s", flaglist(m.Flags, m.Keywords).pack(c)) + } + var modseqStr string + if c.enabled[capCondstore] { + modseqStr = fmt.Sprintf(" MODSEQ (%d)", m.ModSeq.Client()) + } + // ../rfc/9051:6749 ../rfc/3501:4869 ../rfc/7162:2490 + c.bwritelinef("* %d FETCH (UID %d%s%s)", c.xsequence(m.UID), m.UID, flags, modseqStr) } } - c.ok(tag, cmd) + // We don't explicitly send flags for failed updated with silent set. The regular + // notification will get the flags to the client. + // ../rfc/7162:630 ../rfc/3501:3233 + + if len(changed) == 0 { + c.ok(tag, cmd) + return + } + + // Write unsolicited untagged fetch responses for messages that didn't pass the + // unchangedsince check. ../rfc/7162:679 + // Also gather UIDs or sequences for the MODIFIED response below. ../rfc/7162:571 + var mnums []store.UID + for _, m := range changed { + c.bwritelinef("* %d FETCH (UID %d FLAGS %s MODSEQ (%d))", c.xsequence(m.UID), m.UID, flaglist(m.Flags, m.Keywords).pack(c), m.ModSeq.Client()) + if isUID { + mnums = append(mnums, m.UID) + } else { + mnums = append(mnums, store.UID(c.xsequence(m.UID))) + } + } + + sort.Slice(mnums, func(i, j int) bool { + return mnums[i] < mnums[j] + }) + set := compactUIDSet(mnums) + // ../rfc/7162:2506 + c.writeresultf("%s OK [MODIFIED %s] conditional store did not modify all", tag, set.String()) } diff --git a/imapserver/server_test.go b/imapserver/server_test.go index 56964e6..408559e 100644 --- a/imapserver/server_test.go +++ b/imapserver/server_test.go @@ -21,6 +21,8 @@ import ( "github.com/mjl-/mox/store" ) +var ctxbg = context.Background() + func init() { sanityChecks = true @@ -156,6 +158,7 @@ type testconn struct { client *imapclient.Conn done chan struct{} serverConn net.Conn + account *store.Account // Result of last command. lastUntagged []imapclient.Untagged @@ -190,14 +193,15 @@ func (tc *testconn) xcodeArg(v any) { } } -func (tc *testconn) xuntagged(exps ...any) { +func (tc *testconn) xuntagged(exps ...imapclient.Untagged) { tc.t.Helper() - tc.xuntaggedCheck(true, exps...) + tc.xuntaggedOpt(true, exps...) } -func (tc *testconn) xuntaggedCheck(all bool, exps ...any) { +func (tc *testconn) xuntaggedOpt(all bool, exps ...imapclient.Untagged) { tc.t.Helper() last := append([]imapclient.Untagged{}, tc.lastUntagged...) + var mismatch any next: for ei, exp := range exps { for i, l := range last { @@ -205,12 +209,16 @@ next: continue } if !reflect.DeepEqual(l, exp) { - tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", l, l, exp, exp) + mismatch = l + continue } copy(last[i:], last[i+1:]) last = last[:len(last)-1] continue next } + if mismatch != nil { + tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", mismatch, mismatch, exp, exp) + } var next string if len(tc.lastUntagged) > 0 { next = fmt.Sprintf(", next %#v", tc.lastUntagged[0]) @@ -293,11 +301,21 @@ func (tc *testconn) waitDone() { } func (tc *testconn) close() { + err := tc.account.Close() + tc.check(err, "close account") tc.client.Close() tc.serverConn.Close() tc.waitDone() } +func xparseNumSet(s string) imapclient.NumSet { + ns, err := imapclient.ParseNumSet(s) + if err != nil { + panic(fmt.Sprintf("parsing numset %s: %s", s, err)) + } + return ns +} + var connCounter int64 func start(t *testing.T) *testconn { @@ -314,7 +332,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn if first { os.RemoveAll("../testdata/imap/data") } - mox.Context = context.Background() + mox.Context = ctxbg mox.ConfigStaticPath = "../testdata/imap/mox.conf" mox.MustLoadConfig(true, false) acc, err := store.OpenAccount("mjl") @@ -323,8 +341,6 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn err = acc.SetPassword("testtest") tcheck(t, err, "set password") } - err = acc.Close() - tcheck(t, err, "close account") var switchDone chan struct{} if first { switchDone = store.Switchboard() @@ -352,7 +368,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn }() client, err := imapclient.New(clientConn, true) tcheck(t, err, "new client") - return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn} + return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn, account: acc} } func fakeCert(t *testing.T) tls.Certificate { diff --git a/imapserver/store_test.go b/imapserver/store_test.go index 1600780..89484fa 100644 --- a/imapserver/store_test.go +++ b/imapserver/store_test.go @@ -78,7 +78,7 @@ func TestStore(t *testing.T) { // Flags are added to mailbox, not removed. flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent new a b c d e different`, " ") - tc.xuntaggedCheck(false, imapclient.UntaggedFlags(flags)) + tc.xuntaggedOpt(false, imapclient.UntaggedFlags(flags)) tc.transactf("no", `store 1 flags ()`) // No permission to set flags. } diff --git a/import.go b/import.go index 342b9d3..a0f4865 100644 --- a/import.go +++ b/import.go @@ -270,6 +270,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { var changes []store.Change + var modseq store.ModSeq // Assigned on first delivered messages, used for all messages. + xdeliver := func(m *store.Message, mf *os.File) { // todo: possibly set dmarcdomain to the domain of the from address? at least for non-spams that have been seen. otherwise user would start without any reputations. the assumption would be that the user has accepted email and deemed it legit, coming from the indicated sender. @@ -281,7 +283,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { ctl.xcheck(err, "delivering message") deliveredIDs = append(deliveredIDs, m.ID) ctl.log.Debug("delivered message", mlog.Field("id", m.ID)) - changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords}) + changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Flags: m.Flags, Keywords: m.Keywords}) } // todo: one goroutine for reading messages, one for parsing the message, one adding to database, one for junk filter training. @@ -353,8 +355,16 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { } } + if modseq == 0 { + var err error + modseq, err = a.NextModSeq(tx) + ctl.xcheck(err, "assigning next modseq") + } + m.MailboxID = mb.ID m.MailboxOrigID = mb.ID + m.CreateSeq = modseq + m.ModSeq = modseq xdeliver(m, msgf) err = msgf.Close() ctl.log.Check(err, "closing message after delivery") diff --git a/integration_test.go b/integration_test.go index a22e027..207eaff 100644 --- a/integration_test.go +++ b/integration_test.go @@ -184,5 +184,4 @@ a message. tcheck(t, err, "sendmail") }) xlog.Print("success", mlog.Field("duration", time.Since(t0))) - } diff --git a/main.go b/main.go index 29e7c83..93a4679 100644 --- a/main.go +++ b/main.go @@ -2003,6 +2003,7 @@ func cmdEnsureParsed(c *cmd) { n := 0 err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error { q := bstore.QueryTx[store.Message](tx) + q.FilterEqual("Expunged", false) q.FilterFn(func(m store.Message) bool { return all || m.ParsedBuf == nil }) @@ -2135,17 +2136,22 @@ open, or is not running. err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error { // Reassign UIDs, going per mailbox. We assign starting at 1, only changing the // message if it isn't already at the intended UID. Doing it in this order ensures - // we don't get into trouble with duplicate UIDs for a mailbox. + // we don't get into trouble with duplicate UIDs for a mailbox. We assign a new + // modseq. Not strictly needed, for doesn't hurt. + modseq, err := a.NextModSeq(tx) + xcheckf(err, "assigning next modseq") + q := bstore.QueryTx[store.Message](tx) if len(args) == 2 { q.FilterNonzero(store.Message{MailboxID: mailboxID}) } q.SortAsc("MailboxID", "UID") - err := q.ForEach(func(m store.Message) error { + err = q.ForEach(func(m store.Message) error { uidlasts[m.MailboxID]++ uid := uidlasts[m.MailboxID] if m.UID != uid { m.UID = uid + m.ModSeq = modseq if err := tx.Update(&m); err != nil { return fmt.Errorf("updating uid for message: %v", err) } diff --git a/serve.go b/serve.go index 989308b..148e5b0 100644 --- a/serve.go +++ b/serve.go @@ -301,7 +301,10 @@ requested, other TLS certificates are requested on demand. log.Check(err, "closing temp changelog file") } }() - m := &store.Message{Received: time.Now(), Flags: store.Flags{Flagged: true}} + m := &store.Message{ + Received: time.Now(), + Flags: store.Flags{Flagged: true}, + } n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nRemember to make a backup with \"mox backup\" before upgrading.\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n")) if err != nil { log.Infox("writing temporary message file for changelog delivery", err) diff --git a/smtpserver/rejects.go b/smtpserver/rejects.go index 1381263..45ef6b7 100644 --- a/smtpserver/rejects.go +++ b/smtpserver/rejects.go @@ -54,6 +54,7 @@ func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m * q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: mb.ID}) + q.FilterEqual("Expunged", false) q.FilterFn(func(m store.Message) bool { return msgID != "" && m.MessageID == msgID || len(hash) > 0 && bytes.Equal(m.MessageHash, hash) }) diff --git a/smtpserver/reputation.go b/smtpserver/reputation.go index f38b8eb..4f6ab1c 100644 --- a/smtpserver/reputation.go +++ b/smtpserver/reputation.go @@ -124,6 +124,7 @@ func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rc messageQuery := func(fm *store.Message, maxAge time.Duration, maxCount int) *bstore.Query[store.Message] { q := bstore.QueryTx[store.Message](tx) q.FilterEqual("MailboxOrigID", m.MailboxID) + q.FilterEqual("Expunged", false) q.FilterFn(func(m store.Message) bool { return m.Junk || m.Notjunk }) diff --git a/smtpserver/server.go b/smtpserver/server.go index c9b0151..68fdb84 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -2297,6 +2297,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(msg) q.FilterGreater("Received", now.Add(-window)) + q.FilterEqual("Expunged", false) n, err := q.Count() if err != nil { retErr = err @@ -2315,6 +2316,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(msg) q.FilterGreater("Received", now.Add(-window)) + q.FilterEqual("Expunged", false) size := msgWriter.Size err := q.ForEach(func(v store.Message) error { size += v.Size diff --git a/smtpserver/server_test.go b/smtpserver/server_test.go index 8a59df4..d8349f9 100644 --- a/smtpserver/server_test.go +++ b/smtpserver/server_test.go @@ -343,6 +343,7 @@ func tretrain(t *testing.T, acc *store.Account) { // Fetch messags to retrain on. q := bstore.QueryDB[store.Message](ctxbg, acc.DB) + q.FilterEqual("Expunged", false) q.FilterFn(func(m store.Message) bool { return m.Flags.Junk || m.Flags.Notjunk }) @@ -412,6 +413,7 @@ func TestSpam(t *testing.T) { tcheck(t, err, "get rejects mailbox") qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB) qm.FilterNonzero(store.Message{MailboxID: mb.ID}) + qm.FilterEqual("Expunged", false) n, err := qm.Count() tcheck(t, err, "count messages in rejects mailbox") if n != expect { @@ -437,6 +439,7 @@ func TestSpam(t *testing.T) { // Mark the messages as having good reputation. q := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB) + q.FilterEqual("Expunged", false) _, err := q.UpdateFields(map[string]any{"Junk": false, "Notjunk": true}) tcheck(t, err, "update junkiness") @@ -456,6 +459,7 @@ func TestSpam(t *testing.T) { // Undo dmarc pass, mark messages as junk, and train the filter. resolver.TXT = nil q = bstore.QueryDB[store.Message](ctxbg, ts.acc.DB) + q.FilterEqual("Expunged", false) _, err = q.UpdateFields(map[string]any{"Junk": true, "Notjunk": false}) tcheck(t, err, "update junkiness") tretrain(t, ts.acc) diff --git a/store/account.go b/store/account.go index 8fb821a..c73f112 100644 --- a/store/account.go +++ b/store/account.go @@ -150,6 +150,24 @@ type NextUIDValidity struct { Next uint32 } +// SyncState track ModSeqs. +type SyncState struct { + ID int // Just a single record with ID 1. + + // Last used, next assigned will be one higher. The first value we hand out is 2. + // That's because 0 (the default value for old existing messages, from before the + // Message.ModSeq field) is special in IMAP, so we return it as 1. + LastModSeq ModSeq `bstore:"nonzero"` + + // Highest ModSeq of expunged record that we deleted. When a clients synchronizes + // and requests changes based on a modseq before this one, we don't have the + // history to provide information about deletions. We normally keep these expunged + // records around, but we may periodically truly delete them to reclaim storage + // space. Initially set to -1 because we don't want to match with any ModSeq in the + // database, which can be zero values. + HighestDeletedModSeq ModSeq +} + // Mailbox is collection of messages, e.g. Inbox or Sent. type Mailbox struct { ID int64 @@ -232,7 +250,23 @@ type Message struct { ID int64 UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver. - MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,ref Mailbox"` + MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"` + + // Modification sequence, for faster syncing with IMAP QRESYNC and JMAP. + // ModSeq is the last modification. CreateSeq is the Seq the message was inserted, + // always <= ModSeq. If Expunged is set, the message has been removed and should not + // be returned to the user. In this case, ModSeq is the Seq where the message is + // removed, and will never be changed again. + // We have an index on both ModSeq (for JMAP that synchronizes per account) and + // MailboxID+ModSeq (for IMAP that synchronizes per mailbox). + // The index on CreateSeq helps efficiently finding created messages for JMAP. + // The value of ModSeq is special for IMAP. Messages that existed before ModSeq was + // added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If + // we get modseq 1 from a client, the IMAP server will translate it to 0. When we + // return modseq to clients, we turn 0 into 1. + ModSeq ModSeq `bstore:"index"` + CreateSeq ModSeq `bstore:"index"` + Expunged bool // MailboxOrigID is the mailbox the message was originally delivered to. Typically // Inbox or Rejects, but can also be Postmaster and TLS/DMARC reporting addresses. @@ -290,7 +324,9 @@ type Message struct { // delivered only once. Value includes <>. MessageID string `bstore:"index"` - MessageHash []byte // Hash of message. For rejects delivery, so optional like MessageID. + // Hash of message. For rejects delivery, so optional like MessageID. + MessageHash []byte + Flags Keywords []string `bstore:"index"` // For keywords other than system flags or the basic well-known $-flags. Only in "atom" syntax, stored in lower case. Size int64 @@ -304,6 +340,41 @@ type Message struct { ParsedBuf []byte } +// ModSeq represents a modseq as stored in the database. ModSeq 0 in the +// database is sent to the client as 1, because modseq 0 is special in IMAP. +// ModSeq coming from the client are of type int64. +type ModSeq int64 + +func (ms ModSeq) Client() int64 { + if ms == 0 { + return 1 + } + return int64(ms) +} + +// ModSeqFromClient converts a modseq from a client to a modseq for internal +// use, e.g. in a database query. +// ModSeq 1 is turned into 0 (the Go zero value for ModSeq). +func ModSeqFromClient(modseq int64) ModSeq { + if modseq == 1 { + return 0 + } + return ModSeq(modseq) +} + +// PrepareExpunge clears fields that are no longer needed after an expunge, so +// almost all fields. Does not change ModSeq, but does set Expunged. +func (m *Message) PrepareExpunge() { + *m = Message{ + ID: m.ID, + UID: m.UID, + MailboxID: m.MailboxID, + CreateSeq: m.CreateSeq, + ModSeq: m.ModSeq, + Expunged: true, + } +} + // LoadPart returns a message.Part by reading from m.ParsedBuf. func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) { if m.ParsedBuf == nil { @@ -385,7 +456,7 @@ type Outgoing struct { } // Types stored in DB. -var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}} +var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}} // Account holds the information about a user, includings mailboxes, messages, imap subscriptions. type Account struct { @@ -564,6 +635,33 @@ func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) { return v, nil } +// NextModSeq returns the next modification sequence, which is global per account, +// over all types. +func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) { + v := SyncState{ID: 1} + if err := tx.Get(&v); err == bstore.ErrAbsent { + // We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so + // already used. + // HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value + // makes sense. + v = SyncState{1, 2, -1} + return v.LastModSeq, tx.Insert(&v) + } else if err != nil { + return 0, err + } + v.LastModSeq++ + return v.LastModSeq, tx.Update(&v) +} + +func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) { + v := SyncState{ID: 1} + err := tx.Get(&v) + if err == bstore.ErrAbsent { + return 0, nil + } + return v.HighestDeletedModSeq, err +} + // WithWLock runs fn with account writelock held. Necessary for account/mailbox modification. For message delivery, a read lock is required. func (a *Account) WithWLock(fn func()) { a.Lock() @@ -593,10 +691,16 @@ func (a *Account) WithRLock(fn func()) { // If sync is true, the message file and its directory are synced. Should be true // for regular mail delivery, but can be false when importing many messages. // +// If CreateSeq/ModSeq is not set, it is assigned automatically. +// // Must be called with account rlock or wlock. // // Caller must broadcast new message. func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, consumeFile, isSent, sync, notrain bool) error { + if m.Expunged { + return fmt.Errorf("cannot deliver expunged message") + } + mb := Mailbox{ID: m.MailboxID} if err := tx.Get(&mb); err != nil { return fmt.Errorf("get mailbox: %w", err) @@ -630,6 +734,14 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID { m.MailboxDestinedID = 0 } + if m.CreateSeq == 0 || m.ModSeq == 0 { + modseq, err := a.NextModSeq(tx) + if err != nil { + return fmt.Errorf("assigning next modseq: %w", err) + } + m.CreateSeq = modseq + m.ModSeq = modseq + } if err := tx.Insert(m); err != nil { return fmt.Errorf("inserting message: %w", err) @@ -1038,7 +1150,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF return err } - changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.Flags, m.Keywords}) + changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}) BroadcastChanges(a, changes) return nil } @@ -1126,27 +1238,33 @@ func (a *Account) removeMessages(ctx context.Context, log *mlog.Log, tx *bstore. return nil, fmt.Errorf("deleting from message recipient: %w", err) } + // Assign new modseq. + modseq, err := a.NextModSeq(tx) + if err != nil { + return nil, fmt.Errorf("assign next modseq: %w", err) + } + // Actually remove the messages. - qdm := bstore.QueryTx[Message](tx) - qdm.FilterIDs(ids) - var deleted []Message - qdm.Gather(&deleted) - if _, err := qdm.Delete(); err != nil { - return nil, fmt.Errorf("deleting from messages: %w", err) + qx := bstore.QueryTx[Message](tx) + qx.FilterIDs(ids) + var expunged []Message + qx.Gather(&expunged) + if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil { + return nil, fmt.Errorf("expunging messages: %w", err) } // Mark as neutral and train so junk filter gets untrained with these (junk) messages. - for i := range deleted { - deleted[i].Junk = false - deleted[i].Notjunk = false + for i := range expunged { + expunged[i].Junk = false + expunged[i].Notjunk = false } - if err := a.RetrainMessages(ctx, log, tx, deleted, true); err != nil { - return nil, fmt.Errorf("training deleted messages: %w", err) + if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil { + return nil, fmt.Errorf("retraining expunged messages: %w", err) } changes := make([]Change, len(l)) for i, m := range l { - changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}} + changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq} } return changes, nil } diff --git a/store/state.go b/store/state.go index 6d0d4aa..eaa554e 100644 --- a/store/state.go +++ b/store/state.go @@ -28,6 +28,7 @@ type Change any type ChangeAddUID struct { MailboxID int64 UID UID + ModSeq ModSeq Flags Flags // System flags. Keywords []string // Other flags. } @@ -36,12 +37,14 @@ type ChangeAddUID struct { type ChangeRemoveUIDs struct { MailboxID int64 UIDs []UID + ModSeq ModSeq } // ChangeFlags is sent for an update to flags for a message, e.g. "Seen". type ChangeFlags struct { MailboxID int64 UID UID + ModSeq ModSeq Mask Flags // Which flags are actually modified. Flags Flags // New flag values. All are set, not just mask. Keywords []string // Other flags. diff --git a/test-upgrade.sh b/test-upgrade.sh index 025852a..a5b63b7 100755 --- a/test-upgrade.sh +++ b/test-upgrade.sh @@ -9,9 +9,11 @@ set -e # set -x -# We'll allow max 256mb of memory during upgrades. We modify the softlimit when +# We'll set a max memory limit during upgrades. We modify the softlimit when # importing the potentially large mbox file. -ulimit -S -d 256000 +# Currently at 768MB, needed for upgrading with 500k messages from v0.0.5 to +# v0.0.6 (two new indexes on store.Message). +ulimit -S -d 768000 (rm -r testdata/upgrade 2>/dev/null || exit 0) mkdir testdata/upgrade @@ -59,7 +61,7 @@ for tag in $tags; do echo 'Importing bulk data for upgrading.' gunzip < ../upgradetest.mbox.gz | time ./$tag/mox ximport mbox ./stepdata/accounts/test0 upgradetest /dev/stdin echo - ulimit -S -d 256000 + ulimit -S -d 768000 fi echo "Upgrade data to $tag." diff --git a/verifydata.go b/verifydata.go index 6f5c6ef..64e4201 100644 --- a/verifydata.go +++ b/verifydata.go @@ -246,6 +246,10 @@ possibly making them potentially no longer readable by the previous version. if uidnext := mailboxUIDNexts[m.MailboxID]; m.UID >= uidnext { checkf(errors.New(`inconsistent uidnext for message/mailbox, see "mox fixuidmeta"`), dbpath, "message id %d in mailbox id %d has uid %d >= mailbox uidnext %d", m.ID, m.MailboxID, m.UID, uidnext) } + + if m.Expunged { + return nil + } mp := store.MessagePath(m.ID) seen[mp] = struct{}{} p := filepath.Join(accdir, "msg", mp)