serv/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2024-12-26 16:33:47 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

add condstore & qresync imap extensions

Browse source 
for conditional storing and quick resynchronisation (not sure if mail clients are actually using it that).

each message now has a "modseq". it is increased for each change. with
condstore, imap clients can request changes since a certain modseq. that
already allows quickly finding changes since a previous connection. condstore
also allows storing (e.g. setting new message flags) only when the modseq of a
message hasn't changed.

qresync should make it fast for clients to get a full list of changed messages
for a mailbox, including removals.

we now also keep basic metadata of messages that have been removed (expunged).
just enough (uid, modseq) to tell client that the messages have been removed.
this does mean we have to be careful when querying messages from the database.
we must now often filter the expunged messages out.

we also keep "createseq", the modseq when a message was created. this will be
useful for the jmap implementation.
This commit is contained in:
Mechiel Lukkien 2023-07-24 21:21:05 +02:00
parent cc4ecf2927
commit 7f1b7198a8
No known key found for this signature in database
30 changed files with 2181 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -108,7 +108,6 @@ The code is heavily cross-referenced with the RFCs for readability/maintainabili
## Roadmap
- IMAP CONDSTORE and QRESYNC extensions
- Webmail
- IMAP THREAD extension
- DANE and DNSSEC

2
backup.go
View file

@ -424,7 +424,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
tmMsgs := time.Now()
seen := map[string]struct{}{}
var nlinked, ncopied int
err = bstore.QueryDB[store.Message](ctx, db).ForEach(func(m store.Message) error {
err = bstore.QueryDB[store.Message](ctx, db).FilterEqual("Expunged", false).ForEach(func(m store.Message) error {
mp := store.MessagePath(m.ID)
seen[mp] = struct{}{}
amp := filepath.Join("accounts", acc.Name, "msg", mp)

1
ctl.go
View file

@ -649,6 +649,7 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
// Read through messages with junk or nonjunk flag set, and train them.
var total, trained int
q := bstore.QueryDB[store.Message](ctx, acc.DB)
q.FilterEqual("Expunged", false)
err = q.ForEach(func(m store.Message) error {
total++
ok, err := acc.TrainMessage(ctx, ctl.log, jf, m)

4
http/account_test.go
View file

@ -139,7 +139,7 @@ func TestAccount(t *testing.T) {
// Check there are messages, with the right flags.
acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {
_, err = bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get()
_, err = bstore.QueryTx[store.Message](tx).FilterEqual("Expunged", false).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get()
tcheck(t, err, `fetching message with keywords "other" and "test"`)
mb, err := acc.MailboxFind(tx, "importtest")
@ -152,7 +152,7 @@ func TestAccount(t *testing.T) {
t.Fatalf(`expected mailbox keywords "other" and "test", got %v`, mb.Keywords)
}
n, err := bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "custom").Count()
n, err := bstore.QueryTx[store.Message](tx).FilterEqual("Expunged", false).FilterIn("Keywords", "custom").Count()
tcheck(t, err, `fetching message with keyword "custom"`)
if n != 2 {
t.Fatalf(`got %d messages with keyword "custom", expected 2`, n)

14
http/import.go
View file

@ -377,6 +377,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
// finally at the end as a closing statement.
var prevMailbox string
var modseq store.ModSeq // Assigned on first message, used for all messages.
trainMessage := func(m *store.Message, p message.Part, pos string) {
words, err := jf.ParseMessage(p)
if err != nil {
@ -478,6 +480,14 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
m.MailboxID = mb.ID
m.MailboxOrigID = mb.ID
if modseq == 0 {
var err error
modseq, err = acc.NextModSeq(tx)
ximportcheckf(err, "assigning next modseq")
}
m.CreateSeq = modseq
m.ModSeq = modseq
if len(m.Keywords) > 0 {
if destMailboxKeywords[mb.ID] == nil {
destMailboxKeywords[mb.ID] = map[string]bool{}
@ -519,7 +529,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
return
}
deliveredIDs = append(deliveredIDs, m.ID)
changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords})
changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Flags: m.Flags, Keywords: m.Keywords})
messages[mb.Name]++
if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name {
prevMailbox = mb.Name
@ -726,7 +736,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
}
err = tx.Update(&m)
ximportcheckf(err, "updating message after flag update")
changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags, Keywords: m.Keywords})
changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Mask: flags, Flags: flags, Keywords: m.Keywords})
}
delete(mailboxMissingKeywordMessages, mailbox)
} else {

50
imapclient/parse.go
View file

@ -119,6 +119,7 @@ var knownCodes = stringMap(
"ALERT", "PARSE", "READ-ONLY", "READ-WRITE", "TRYCREATE", "UIDNOTSTICKY", "UNAVAILABLE", "AUTHENTICATIONFAILED", "AUTHORIZATIONFAILED", "EXPIRED", "PRIVACYREQUIRED", "CONTACTADMIN", "NOPERM", "INUSE", "EXPUNGEISSUED", "CORRUPTION", "SERVERBUG", "CLIENTBUG", "CANNOT", "LIMIT", "OVERQUOTA", "ALREADYEXISTS", "NONEXISTENT", "NOTSAVED", "HASCHILDREN", "CLOSED", "UNKNOWN-CTE",
// With parameters.
"BADCHARSET", "CAPABILITY", "PERMANENTFLAGS", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "APPENDUID", "COPYUID",
"HIGHESTMODSEQ", "MODIFIED",
)
func stringMap(l ...string) map[string]struct{} {
@ -202,6 +203,13 @@ func (c *Conn) xrespCode() (string, CodeArg) {
c.xspace()
to := c.xuidset()
codeArg = CodeCopyUID{destUIDValidity, from, to}
case "HIGHESTMODSEQ":
c.xspace()
codeArg = CodeHighestModSeq(c.xint64())
case "MODIFIED":
c.xspace()
modified := c.xuidset()
codeArg = CodeModified(NumSet{Ranges: modified})
}
return W, codeArg
}
@ -248,7 +256,7 @@ func (c *Conn) xint32() int32 {
func (c *Conn) xint64() int64 {
s := c.xdigits()
num, err := strconv.ParseInt(s, 10, 64)
num, err := strconv.ParseInt(s, 10, 63)
c.xcheckf(err, "parsing int64")
return num
}
@ -386,6 +394,8 @@ func (c *Conn) xuntagged() Untagged {
} else {
num = c.xint64()
}
case "HIGHESTMODSEQ":
num = c.xint64()
default:
c.xerrorf("status: unknown attribute %q", s)
}
@ -415,6 +425,15 @@ func (c *Conn) xuntagged() Untagged {
c.xneedDisabled("untagged SEARCH response", CapIMAP4rev2)
var nums []uint32
for c.take(' ') {
// ../rfc/7162:2557
if c.take('(') {
c.xtake("MODSEQ")
c.xspace()
modseq := c.xint64()
c.xtake(")")
c.xcrlf()
return UntaggedSearchModSeq{nums, modseq}
}
nums = append(nums, c.xnzuint32())
}
r := UntaggedSearch(nums)
@ -456,6 +475,20 @@ func (c *Conn) xuntagged() Untagged {
c.xcrlf()
return UntaggedID(params)
// ../rfc/7162:2623
case "VANISHED":
c.xspace()
var earlier bool
if c.take('(') {
c.xtake("EARLIER")
c.xtake(")")
c.xspace()
earlier = true
}
uids := c.xuidset()
c.xcrlf()
return UntaggedVanished{earlier, NumSet{Ranges: uids}}
default:
v, err := strconv.ParseUint(w, 10, 32)
if err == nil {
@ -605,6 +638,14 @@ func (c *Conn) xmsgatt1() FetchAttr {
case "UID":
c.xspace()
return FetchUID(c.xuint32())
case "MODSEQ":
// ../rfc/7162:2488
c.xspace()
c.xtake("(")
modseq := c.xint64()
c.xtake(")")
return FetchModSeq(modseq)
}
c.xerrorf("unknown fetch attribute %q", f)
panic("not reached")
@ -963,7 +1004,7 @@ func (c *Conn) xtaggedExtVal() TaggedExtVal {
return TaggedExtVal{SeqSet: &ss}
}
s := c.xdigits()
num, err := strconv.ParseInt(s, 10, 64)
num, err := strconv.ParseInt(s, 10, 63)
c.xcheckf(err, "parsing int")
if !c.peek(':') && !c.peek(',') {
// not a larger sequence-set
@ -1146,6 +1187,11 @@ func (c *Conn) xesearchResponse() (r UntaggedEsearch) {
num := c.xuint32()
r.Count = &num
// ../rfc/7162:1211 ../rfc/4731:273
case "MODSEQ":
c.xspace()
r.ModSeq = c.xint64()
default:
// Validate ../rfc/9051:7090
for i, b := range []byte(w) {

40
imapclient/protocol.go
View file

@ -1,6 +1,7 @@
package imapclient
import (
"bufio"
"fmt"
"strings"
)
@ -134,6 +135,20 @@ func (c CodeCopyUID) CodeString() string {
return fmt.Sprintf("COPYUID %d %s %s", c.DestUIDValidity, str(c.From), str(c.To))
}
// For CONDSTORE.
type CodeModified NumSet
func (c CodeModified) CodeString() string {
return fmt.Sprintf("MODIFIED %s", NumSet(c).String())
}
// For CONDSTORE.
type CodeHighestModSeq int64
func (c CodeHighestModSeq) CodeString() string {
return fmt.Sprintf("HIGHESTMODSEQ %d", c)
}
// RespText represents a response line minus the leading tag.
type RespText struct {
Code string // The first word between [] after the status.
@ -201,6 +216,12 @@ type UntaggedFetch struct {
Attrs []FetchAttr
}
type UntaggedSearch []uint32
// ../rfc/7162:1101
type UntaggedSearchModSeq struct {
Nums []uint32
ModSeq int64
}
type UntaggedStatus struct {
Mailbox string
Attrs map[string]int64 // Upper case status attributes. ../rfc/9051:7059
@ -224,9 +245,16 @@ type UntaggedEsearch struct {
Max uint32
All NumSet
Count *uint32
ModSeq int64
Exts []EsearchDataExt
}
// UntaggedVanished is used in QRESYNC to send UIDs that have been removed.
type UntaggedVanished struct {
Earlier bool
UIDs NumSet
}
// ../rfc/2971:184
type UntaggedID map[string]string
@ -278,6 +306,13 @@ func (ns NumSet) String() string {
return r
}
func ParseNumSet(s string) (ns NumSet, rerr error) {
c := Conn{r: bufio.NewReader(strings.NewReader(s))}
defer c.recover(&rerr)
ns = c.xsequenceSet()
return
}
// NumRange is a single number or range.
type NumRange struct {
First uint32 // 0 for "*".
@ -450,3 +485,8 @@ func (f FetchBinarySize) Attr() string { return f.RespAttr }
type FetchUID uint32
func (f FetchUID) Attr() string { return "UID" }
// "MODSEQ" fetch response.
type FetchModSeq int64
func (f FetchModSeq) Attr() string { return "MODSEQ" }

702
imapserver/condstore_test.go Normal file
View file

@ -0,0 +1,702 @@
package imapserver
import (
"fmt"
"strings"
"testing"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store"
)
func TestCondstore(t *testing.T) {
testCondstoreQresync(t, false)
}
func TestQresync(t *testing.T) {
testCondstoreQresync(t, true)
}
func testCondstoreQresync(t *testing.T, qresync bool) {
defer mockUIDValidity()()
tc := start(t)
defer tc.close()
// todo: check whether marking \seen will cause modseq to be returned in case of qresync.
// Check basic requirements of CONDSTORE.
capability := "Condstore"
if qresync {
capability = "Qresync"
}
tc.client.Login("mjl@mox.example", "testtest")
tc.client.Enable(capability)
tc.transactf("ok", "Select inbox")
tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(1), More: "x"}})
// First some tests without any messages.
tc.transactf("ok", "Status inbox (Highestmodseq)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 1}})
// No messages, no matches.
tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 12345)")
tc.xuntagged()
// Also no messages with modseq 1, which we internally turn into modseq 0.
tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1)")
tc.xuntagged()
// Also try with modseq attribute.
tc.transactf("ok", "Uid Fetch 1:* (Flags Modseq) (Changedsince 1)")
tc.xuntagged()
// Search with modseq search criteria.
tc.transactf("ok", "Search Modseq 0") // Zero is valid, matches all.
tc.xsearch()
tc.transactf("ok", "Search Modseq 1") // Converted to zero internally.
tc.xsearch()
tc.transactf("ok", "Search Modseq 12345")
tc.xsearch()
tc.transactf("ok", `Search Modseq "/Flags/\\Draft" All 12345`)
tc.xsearch()
tc.transactf("ok", `Search Or Modseq 12345 Modseq 54321`)
tc.xsearch()
// esearch
tc.transactf("ok", "Search Return (All) Modseq 123")
tc.xesearch(imapclient.UntaggedEsearch{})
// Now we add, delete, expunge, modify some message flags and check if the
// responses are correct. We check in both a condstore-enabled and one without that
// we get the correct notifications.
// First we add 3 messages as if they were added before we implemented CONDSTORE.
// Later on, we'll update the second, and delete the third, leaving the first
// unmodified. Those messages have modseq 0 in the database. We use append for
// convenience, then adjust the records in the database.
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
_, err := bstore.QueryDB[store.Message](ctxbg, tc.account.DB).UpdateFields(map[string]any{
"ModSeq": 0,
"CreateSeq": 0,
})
tcheck(t, err, "clearing modseq from messages")
err = tc.account.DB.Update(ctxbg, &store.SyncState{ID: 1, LastModSeq: 1})
tcheck(t, err, "resetting modseq state")
tc.client.Create("otherbox")
// tc2 is a client without condstore, so no modseq responses.
tc2 := startNoSwitchboard(t)
defer tc2.close()
tc2.client.Login("mjl@mox.example", "testtest")
tc2.client.Select("inbox")
// tc2 is a client with condstore, so with modseq responses.
tc3 := startNoSwitchboard(t)
defer tc3.close()
tc3.client.Login("mjl@mox.example", "testtest")
tc3.client.Enable(capability)
tc3.client.Select("inbox")
var clientModseq int64 = 1 // We track the client-side modseq for inbox. Not a store.ModSeq.
// Add messages to: inbox, otherbox, inbox, inbox.
// We have these messages in order of modseq: 2+1 in inbox, 1 in otherbox, 2 in inbox.
// The original two in inbox appear to have modseq 1 (with 0 stored in the database).
// The ones we insert below will start with modseq 2. So we'll have modseq 1-5.
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged(imapclient.UntaggedExists(4))
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 4})
tc.transactf("ok", "Append otherbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged()
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 2, UID: 1})
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged(imapclient.UntaggedExists(5))
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 5})
tc.transactf("ok", "Append inbox () \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx")
tc.xuntagged(imapclient.UntaggedExists(6))
tc.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 6})
tc2.transactf("ok", "Noop")
noflags := imapclient.FetchFlags(nil)
tc2.xuntagged(
imapclient.UntaggedExists(6),
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags}},
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags}},
imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags}},
)
tc3.transactf("ok", "Noop")
tc3.xuntagged(
imapclient.UntaggedExists(6),
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(clientModseq + 1)}},
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(clientModseq + 3)}},
imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq + 4)}},
)
moxvar.Pedantic = true
tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax.
moxvar.Pedantic = false
tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)")
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(clientModseq)}})
clientModseq += 4 // Four messages, over two mailboxes, modseq is per account.
// Check highestmodseq for mailboxes.
tc.transactf("ok", "Status inbox (highestmodseq)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"HIGHESTMODSEQ": clientModseq}})
tc.transactf("ok", "Status otherbox (highestmodseq)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "otherbox", Attrs: map[string]int64{"HIGHESTMODSEQ": 3}})
// Check highestmodseq when we select.
tc.transactf("ok", "Examine otherbox")
tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(3), More: "x"}})
tc.transactf("ok", "Select inbox")
tc.xuntaggedOpt(false, imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}})
// Check fetch modseq response and changedsince.
tc.transactf("ok", `Fetch 1 (Modseq)`)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(1)}})
// Without modseq attribute, even with condseq enabled, there is no modseq response.
// For QRESYNC, we must always send MODSEQ for UID FETCH commands, but not for FETCH commands. ../rfc/7162:1427
tc.transactf("ok", `Uid Fetch 1 Flags`)
if qresync {
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
} else {
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
}
tc.transactf("ok", `Fetch 1 Flags`)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags}})
// When CHANGEDSINCE is present, MODSEQ is automatically added to the response.
// ../rfc/7162:871
// ../rfc/7162:877
tc.transactf("ok", `Fetch 1 Flags (Changedsince 1)`)
tc.xuntagged()
tc.transactf("ok", `Fetch 1,4 Flags (Changedsince 1)`)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(4), noflags, imapclient.FetchModSeq(2)}})
tc.transactf("ok", `Fetch 2 Flags (Changedsince 2)`)
tc.xuntagged()
// store and uid store.
// unchangedsince 0 never passes the check. ../rfc/7162:640
tc.transactf("ok", `Store 1 (Unchangedsince 0) +Flags ()`)
tc.xcodeArg(imapclient.CodeModified(xparseNumSet("1")))
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(1)}})
// Modseq is 2 for first condstore-aware-appended message, so also no match.
tc.transactf("ok", `Uid Store 4 (Unchangedsince 1) +Flags ()`)
tc.xcodeArg(imapclient.CodeModified(xparseNumSet("4")))
// Modseq is 1 for original message.
tc.transactf("ok", `Store 1 (Unchangedsince 1) +Flags (label1)`)
tc.xcode("") // No MODIFIED.
clientModseq++
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}})
tc2.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}}},
)
tc3.transactf("ok", "Noop")
tc3.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags{"label1"}, imapclient.FetchModSeq(clientModseq)}},
)
// Modify same message twice. Check that second application doesn't fail due to
// modseq change made in the first application. ../rfc/7162:823
tc.transactf("ok", `Uid Store 1,1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
clientModseq++
tc.xcode("") // No MODIFIED.
tc.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
)
// We do broadcast the changes twice. Not great, but doesn't hurt. This isn't common.
tc2.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil)}},
)
tc3.transactf("ok", "Noop")
tc3.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}},
)
// Modify without actually changing flags, there will be no new modseq and no broadcast.
tc.transactf("ok", `Store 1 (Unchangedsince %d) -Flags (label1)`, clientModseq)
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchFlags(nil), imapclient.FetchModSeq(clientModseq)}})
tc.xcode("") // No MODIFIED.
tc2.transactf("ok", "Noop")
tc2.xuntagged()
tc3.transactf("ok", "Noop")
tc3.xuntagged()
// search with modseq criteria and modseq in response
tc.transactf("ok", "Search Modseq %d", clientModseq)
tc.xsearchmodseq(clientModseq, 1)
tc.transactf("ok", "Uid Search Or Modseq %d Modseq %d", clientModseq, clientModseq)
tc.xsearchmodseq(clientModseq, 1)
// esearch
tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq)
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq})
uint32ptr := func(v uint32) *uint32 {
return &v
}
tc.transactf("ok", "Search Return (Count) 1:* Modseq 0")
tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq})
tc.transactf("ok", "Search Return (Min Max) 1:* Modseq 0")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 6, ModSeq: clientModseq})
tc.transactf("ok", "Search Return (Min) 1:* Modseq 0")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, ModSeq: clientModseq})
// expunge, we expunge the third and fourth messages. The third was originally with
// modseq 0, the fourth was added with condstore-aware append.
tc.transactf("ok", `Store 3:4 +Flags (\Deleted)`)
clientModseq++
tc2.transactf("ok", "Noop")
tc3.transactf("ok", "Noop")
tc.transactf("ok", "Expunge")
clientModseq++
if qresync {
tc.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
} else {
tc.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
}
tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
tc2.transactf("ok", "Noop")
tc2.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
tc3.transactf("ok", "Noop")
if qresync {
tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("3:4")})
} else {
tc3.xuntagged(imapclient.UntaggedExpunge(3), imapclient.UntaggedExpunge(3))
}
// Again after expunge: status, select, conditional store/fetch/search
tc.transactf("ok", "Status inbox (Highestmodseq Messages Unseen Deleted)")
tc.xuntagged(imapclient.UntaggedStatus{Mailbox: "Inbox", Attrs: map[string]int64{"MESSAGES": 4, "UNSEEN": 4, "DELETED": 0, "HIGHESTMODSEQ": clientModseq}})
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox")
tc.xuntaggedOpt(false,
imapclient.UntaggedExists(4),
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
)
tc.transactf("ok", `Fetch 1:* (Modseq)`)
tc.xuntagged(
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), imapclient.FetchModSeq(1)}},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchModSeq(5)}},
)
// Expunged messages, with higher modseq, should not show up.
tc.transactf("ok", "Uid Fetch 1:* (flags) (Changedsince 7)")
tc.xuntagged()
// search
tc.transactf("ok", "Search Modseq 7")
tc.xsearchmodseq(7, 1)
tc.transactf("ok", "Search Modseq 8")
tc.xsearch()
// esearch
tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 7")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 7})
tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8")
tc.xuntagged(imapclient.UntaggedEsearch{Correlator: tc.client.LastTag})
// store, cannot modify expunged messages.
tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq)
tc.xuntagged()
tc.xcode("") // Not MODIFIED.
tc.transactf("ok", `Uid Store 3,4 +Flags (label2)`)
tc.xuntagged()
tc.xcode("") // Not MODIFIED.
// Check all condstore-enabling commands (and their syntax), ../rfc/7162:368
// We start a new connection, do the thing that should enable condstore, then
// change flags of a message in another connection, do a noop in the new connection
// which should result in an untagged fetch that includes modseq, the indicator
// that condstore was indeed enabled. It's a bit complicated, but i don't think
// there is a clearly specified mechanism to find out which capabilities are
// enabled at any point.
var tagcount int
checkCondstoreEnabled := func(fn func(xtc *testconn)) {
t.Helper()
xtc := startNoSwitchboard(t)
defer xtc.close()
xtc.client.Login("mjl@mox.example", "testtest")
fn(xtc)
tagcount++
label := fmt.Sprintf("l%d", tagcount)
tc.transactf("ok", "Store 4 Flags (%s)", label)
clientModseq++
xtc.transactf("ok", "Noop")
xtc.xuntagged(imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), imapclient.FetchFlags{label}, imapclient.FetchModSeq(clientModseq)}})
}
// SELECT/EXAMINE with CONDSTORE parameter, ../rfc/7162:373
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox (Condstore)")
})
// STATUS with HIGHESTMODSEQ attribute, ../rfc/7162:375
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Status otherbox (Highestmodseq)")
xtc.transactf("ok", "Select inbox")
})
// FETCH with MODSEQ ../rfc/7162:377
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Fetch 4 (Modseq)")
})
// SEARCH with MODSEQ ../rfc/7162:377
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Search 4 Modseq 1")
})
// FETCH with CHANGEDSINCE ../rfc/7162:380
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Fetch 4 (Flags) (Changedsince %d)", clientModseq)
})
// STORE with UNCHANGEDSINCE ../rfc/7162:382
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Select inbox")
xtc.transactf("ok", "Store 4 (Unchangedsince 0) Flags ()")
})
// ENABLE CONDSTORE ../rfc/7162:384
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Enable Condstore")
xtc.transactf("ok", "Select inbox")
})
// ENABLE QRESYNC ../rfc/7162:1390
checkCondstoreEnabled(func(xtc *testconn) {
t.Helper()
xtc.transactf("ok", "Enable Qresync")
xtc.transactf("ok", "Select inbox")
})
tc.transactf("ok", "Store 4 Flags ()")
clientModseq++
if qresync {
testQresync(t, tc, clientModseq)
}
// Continue with some tests that further change the data.
// First we copy messages to a new mailbox, and check we get new modseq for those
// messages.
tc.transactf("ok", "Select otherbox")
tc2.transactf("ok", "Noop")
tc3.transactf("ok", "Noop")
tc.transactf("ok", "Copy 1 inbox")
clientModseq++
tc2.transactf("ok", "Noop")
tc3.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedExists(5),
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags}},
)
tc3.xuntagged(
imapclient.UntaggedExists(5),
imapclient.UntaggedFetch{Seq: 5, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(7), noflags, imapclient.FetchModSeq(clientModseq)}},
)
// Then we move some messages, and check if we get expunged/vanished in original
// and untagged fetch with modseq in destination mailbox.
// tc2o is a client without condstore, so no modseq responses.
tc2o := startNoSwitchboard(t)
defer tc2o.close()
tc2o.client.Login("mjl@mox.example", "testtest")
tc2o.client.Select("otherbox")
// tc3o is a client with condstore, so with modseq responses.
tc3o := startNoSwitchboard(t)
defer tc3o.close()
tc3o.client.Login("mjl@mox.example", "testtest")
tc3o.client.Enable(capability)
tc3o.client.Select("otherbox")
tc.transactf("ok", "Select inbox")
tc.transactf("ok", "Uid Move 2:4 otherbox") // Only UID 2, because UID 3 and 4 have already been expunged.
clientModseq++
if qresync {
tc.xuntaggedOpt(false, imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
tc.xcodeArg(imapclient.CodeHighestModSeq(clientModseq))
} else {
tc.xuntaggedOpt(false, imapclient.UntaggedExpunge(2))
tc.xcode("")
}
tc2.transactf("ok", "Noop")
tc2.xuntagged(imapclient.UntaggedExpunge(2))
tc3.transactf("ok", "Noop")
if qresync {
tc3.xuntagged(imapclient.UntaggedVanished{UIDs: xparseNumSet("2")})
} else {
tc3.xuntagged(imapclient.UntaggedExpunge(2))
}
tc2o.transactf("ok", "Noop")
tc2o.xuntagged(
imapclient.UntaggedExists(2),
imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags}},
)
tc3o.transactf("ok", "Noop")
tc3o.xuntagged(
imapclient.UntaggedExists(2),
imapclient.UntaggedFetch{Seq: 2, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(2), noflags, imapclient.FetchModSeq(clientModseq)}},
)
tc2o.close()
tc2o = nil
tc3o.close()
tc3o = nil
// Then we rename inbox, which is special because it moves messages away instead of
// actually moving the mailbox. The mailbox stays and is cleared, so we check if we
// get expunged/vanished messages.
tc.transactf("ok", "Rename inbox oldbox")
// todo spec: server doesn't respond with untagged responses, find rfc reference that says this is ok.
tc2.transactf("ok", "Noop")
tc2.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
)
tc3.transactf("ok", "Noop")
if qresync {
tc3.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
imapclient.UntaggedVanished{UIDs: xparseNumSet("1,5:7")},
)
} else {
tc3.xuntagged(
imapclient.UntaggedList{Separator: '/', Mailbox: "oldbox"},
imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1), imapclient.UntaggedExpunge(1),
)
}
// Then we delete otherbox (we cannot delete inbox). We don't keep any history for removed mailboxes, so not actually a special case.
tc.transactf("ok", "Delete otherbox")
}
func testQresync(t *testing.T, tc *testconn, clientModseq int64) {
// Vanished on non-uid fetch is not allowed. ../rfc/7162:1693
tc.transactf("bad", "fetch 1:* (Flags) (Changedsince 1 Vanished)")
// Vanished without changedsince is not allowed. ../rfc/7162:1701
tc.transactf("bad", "Uid Fetch 1:* (Flags) (Vanished)")
// Vanished not allowed without first enabling qresync. ../rfc/7162:1697
xtc := startNoSwitchboard(t)
xtc.client.Login("mjl@mox.example", "testtest")
xtc.transactf("ok", "Select inbox (Condstore)")
xtc.transactf("bad", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
xtc.close()
xtc = nil
// Check that we get proper vanished responses.
tc.transactf("ok", "Uid Fetch 1:* (Flags) (Changedsince 1 Vanished)")
noflags := imapclient.FetchFlags(nil)
tc.xuntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)
// select/examine with qresync parameters, including the various optional fields.
tc.transactf("ok", "Close")
// Must enable qresync explicitly before using. ../rfc/7162:1446
xtc = startNoSwitchboard(t)
xtc.client.Login("mjl@mox.example", "testtest")
xtc.transactf("bad", "Select inbox (Qresync 1 0)")
xtc.close()
xtc = nil
tc.transactf("bad", "Select inbox (Qresync (0 1))") // Both args must be > 0.
tc.transactf("bad", "Select inbox (Qresync (1 0))") // Both args must be > 0.
tc.transactf("bad", "Select inbox (Qresync)") // Two args are minimum.
tc.transactf("bad", "Select inbox (Qresync (1))") // Two args are minimum.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:*))") // Known UIDs, * not allowed.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:6)))") // Known seqset cannot have *.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:6 1:*)))") // Known uidset cannot have *.
tc.transactf("bad", "Select inbox (Qresync (1 1) qresync (1 1))") // Duplicate qresync.
flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent label1 l1 l2 l3 l4 l5 l6 l7 l8`, " ")
permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ")
uflags := imapclient.UntaggedFlags(flags)
upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: permflags}, More: "x"}}
baseUntagged := []imapclient.Untagged{
uflags,
upermflags,
imapclient.UntaggedList{Separator: '/', Mailbox: "Inbox"},
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDNEXT", CodeArg: imapclient.CodeUint{Code: "UIDNEXT", Num: 7}, More: "x"}},
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UIDVALIDITY", CodeArg: imapclient.CodeUint{Code: "UIDVALIDITY", Num: 1}, More: "x"}},
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "UNSEEN", CodeArg: imapclient.CodeUint{Code: "UNSEEN", Num: 1}, More: "x"}},
imapclient.UntaggedRecent(0),
imapclient.UntaggedExists(4),
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "HIGHESTMODSEQ", CodeArg: imapclient.CodeHighestModSeq(clientModseq), More: "x"}},
}
makeUntagged := func(l ...imapclient.Untagged) []imapclient.Untagged {
return append(append([]imapclient.Untagged{}, baseUntagged...), l...)
}
// uidvalidity 1, highest known modseq 1, sends full current state.
tc.transactf("ok", "Select inbox (Qresync (1 1))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Uidvalidity mismatch, server will not send any changes, so it's just a regular open.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (2 1))")
tc.xuntagged(baseUntagged...)
// We can tell which UIDs we know. First, send broader range then exist, should work.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 1:7))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Now send just the ones that exist. We won't get the vanished messages.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 1,2,5:6))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// We'll only get updates for UIDs we specify.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 5))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
)...,
)
// We'll only get updates for UIDs we specify. ../rfc/7162:1523
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 1 3))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3")},
)...,
)
// If we specify the latest modseq, we'll get no changes.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 %d))", clientModseq)
tc.xuntagged(baseUntagged...)
// We can provide our own seqs & uids, and have server determine which uids we
// know. But the seqs & uids must be of equal length. First try with a few combinations
// that aren't valid. ../rfc/7162:1579
tc.transactf("ok", "Close")
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1 1,2)))") // Not same length.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1,2 1)))") // Not same length.
tc.transactf("no", "Select inbox (Qresync (1 1 1:6 (1,2 1,1)))") // Not ascending.
tc.transactf("bad", "Select inbox (Qresync (1 1 1:6 (1:* 1:4)))") // Star not allowed.
// With valid parameters, based on what a client would know at this stage.
tc.transactf("ok", "Select inbox (Qresync (1 1 1:6 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 3, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(5), noflags, imapclient.FetchModSeq(4)}},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// The 3rd parameter is optional, try without.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 5 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(7)}},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 8 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Client will claim a highestmodseq but then include uids that have been removed
// since that time. Server detects this, sends full vanished history and continues
// working with modseq changed to 1 before the expunged uid.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 9 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}},
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
// Client will claim a highestmodseq but then include uids that have been removed
// since that time. Server detects this, sends full vanished history and continues
// working with modseq changed to 1 before the expunged uid.
tc.transactf("ok", "Close")
tc.transactf("ok", "Select inbox (Qresync (1 18 (1,3,6 1,3,6)))")
tc.xuntagged(
makeUntagged(
imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "ALERT", More: "Synchronization inconsistency in client detected. Client tried to sync with a UID that was removed at or after the MODSEQ it sent in the request. Sending all historic message removals for selected mailbox. Full syncronization recommended."}},
imapclient.UntaggedVanished{Earlier: true, UIDs: xparseNumSet("3:4")},
imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq)}},
)...,
)
}

188
imapserver/fetch.go
View file

@ -11,11 +11,14 @@ import (
"sort"
"strings"
"golang.org/x/exp/maps"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store"
)
@ -28,7 +31,11 @@ type fetchCmd struct {
changes []store.Change // For updated Seen flag.
markSeen bool
needFlags bool
needModseq bool // Whether untagged responses needs modseq.
expungeIssued bool // Set if a message cannot be read. Can happen for expunged messages.
modseq store.ModSeq // Initialized on first change, for marking messages as seen.
isUID bool // If this is a UID FETCH command.
hasChangedSince bool // Whether CHANGEDSINCE was set. Enables MODSEQ in response.
// Loaded when first needed, closed when message was processed.
m *store.Message // Message currently being processed.
@ -60,15 +67,61 @@ func (cmd *fetchCmd) xcheckf(err error, format string, args ...any) {
//
// State: Selected
func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
// Command: ../rfc/9051:4330 ../rfc/3501:2992
// Examples: ../rfc/9051:4463 ../rfc/9051:4520
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864
// Command: ../rfc/9051:4330 ../rfc/3501:2992 ../rfc/7162:864
// Examples: ../rfc/9051:4463 ../rfc/9051:4520 ../rfc/7162:880
// Response syntax: ../rfc/9051:6742 ../rfc/3501:4864 ../rfc/7162:2490
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748
// Request syntax: ../rfc/9051:6553 ../rfc/3501:4748 ../rfc/4466:535 ../rfc/7162:2475
p.xspace()
nums := p.xnumSet()
p.xspace()
atts := p.xfetchAtts()
atts := p.xfetchAtts(isUID)
var changedSince int64
var haveChangedSince bool
var vanished bool
if p.space() {
// ../rfc/4466:542
// ../rfc/7162:2479
p.xtake("(")
seen := map[string]bool{}
for {
var w string
if isUID && p.conn.enabled[capQresync] {
// Vanished only valid for uid fetch, and only for qresync. ../rfc/7162:1693
w = p.xtakelist("CHANGEDSINCE", "VANISHED")
} else {
w = p.xtakelist("CHANGEDSINCE")
}
if seen[w] {
xsyntaxErrorf("duplicate fetch modifier %s", w)
}
seen[w] = true
switch w {
case "CHANGEDSINCE":
p.xspace()
changedSince = p.xnumber64()
// workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.
if changedSince == 0 && moxvar.Pedantic {
// ../rfc/7162:2551
xsyntaxErrorf("changedsince modseq must be > 0")
}
// CHANGEDSINCE is a CONDSTORE-enabling parameter. ../rfc/7162:380
p.conn.xensureCondstore(nil)
haveChangedSince = true
case "VANISHED":
vanished = true
}
if p.take(")") {
break
}
p.xspace()
}
// ../rfc/7162:1701
if vanished && !haveChangedSince {
xsyntaxErrorf("VANISHED can only be used with CHANGEDSINCE")
}
}
p.xempty()
// We don't use c.account.WithRLock because we write to the client while reading messages.
@ -81,21 +134,105 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
runlock()
}()
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID}
var vanishedUIDs []store.UID
cmd := &fetchCmd{conn: c, mailboxID: c.mailboxID, isUID: isUID, hasChangedSince: haveChangedSince}
c.xdbwrite(func(tx *bstore.Tx) {
cmd.tx = tx
// Ensure the mailbox still exists.
c.xmailboxID(tx, c.mailboxID)
uids := c.xnumSetUIDs(isUID, nums)
var uids []store.UID
// With changedSince, the client is likely asking for a small set of changes. Use a
// database query to trim down the uids we need to look at.
// ../rfc/7162:871
if changedSince > 0 {
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MailboxID: c.mailboxID})
q.FilterGreater("ModSeq", store.ModSeqFromClient(changedSince))
if !vanished {
q.FilterEqual("Expunged", false)
}
err := q.ForEach(func(m store.Message) error {
if m.Expunged {
vanishedUIDs = append(vanishedUIDs, m.UID)
} else if isUID {
if nums.containsUID(m.UID, c.uids, c.searchResult) {
uids = append(uids, m.UID)
}
} else {
seq := c.sequence(m.UID)
if seq > 0 && nums.containsSeq(seq, c.uids, c.searchResult) {
uids = append(uids, m.UID)
}
}
return nil
})
xcheckf(err, "looking up messages with changedsince")
} else {
uids = c.xnumSetUIDs(isUID, nums)
}
// Send vanished for all missing requested UIDs. ../rfc/7162:1718
if vanished {
delModSeq, err := c.account.HighestDeletedModSeq(tx)
xcheckf(err, "looking up highest deleted modseq")
if changedSince < delModSeq.Client() {
// First sort the uids we already found, for fast lookup.
sort.Slice(vanishedUIDs, func(i, j int) bool {
return vanishedUIDs[i] < vanishedUIDs[j]
})
// We'll be gathering any more vanished uids in more.
more := map[store.UID]struct{}{}
checkVanished := func(uid store.UID) {
if uidSearch(c.uids, uid) <= 0 && uidSearch(vanishedUIDs, uid) <= 0 {
more[uid] = struct{}{}
}
}
// Now look through the requested uids. We may have a searchResult, handle it
// separately from a numset with potential stars, over which we can more easily
// iterate.
if nums.searchResult {
for _, uid := range c.searchResult {
checkVanished(uid)
}
} else {
iter := nums.interpretStar(c.uids).newIter()
for {
num, ok := iter.Next()
if !ok {
break
}
checkVanished(store.UID(num))
}
}
vanishedUIDs = append(vanishedUIDs, maps.Keys(more)...)
}
}
// Release the account lock.
runlock()
runlock = func() {} // Prevent defer from unlocking again.
// First report all vanished UIDs. ../rfc/7162:1714
if len(vanishedUIDs) > 0 {
// Mention all vanished UIDs in compact numset form.
// ../rfc/7162:1985
sort.Slice(vanishedUIDs, func(i, j int) bool {
return vanishedUIDs[i] < vanishedUIDs[j]
})
// No hard limit on response sizes, but clients are recommended to not send more
// than 8k. We send a more conservative max 4k.
for _, s := range compactUIDSet(vanishedUIDs).Strings(4*1024 - 32) {
c.bwritelinef("* VANISHED (EARLIER) %s", s)
}
}
for _, uid := range uids {
cmd.uid = uid
mlog.Field("processing uid", mlog.Field("uid", uid))
cmd.process(atts)
}
})
@ -113,6 +250,15 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
}
}
func (cmd *fetchCmd) xmodseq() store.ModSeq {
if cmd.modseq == 0 {
var err error
cmd.modseq, err = cmd.conn.account.NextModSeq(cmd.tx)
cmd.xcheckf(err, "assigning next modseq")
}
return cmd.modseq
}
func (cmd *fetchCmd) xensureMessage() *store.Message {
if cmd.m != nil {
return cmd.m
@ -120,6 +266,7 @@ func (cmd *fetchCmd) xensureMessage() *store.Message {
q := bstore.QueryTx[store.Message](cmd.tx)
q.FilterNonzero(store.Message{MailboxID: cmd.mailboxID, UID: cmd.uid})
q.FilterEqual("Expunged", false)
m, err := q.Get()
cmd.xcheckf(err, "get message for uid %d", cmd.uid)
cmd.m = &m
@ -178,6 +325,7 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
cmd.markSeen = false
cmd.needFlags = false
cmd.needModseq = false
for _, a := range atts {
data = append(data, cmd.xprocessAtt(a)...)
@ -186,10 +334,11 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
if cmd.markSeen {
m := cmd.xensureMessage()
m.Seen = true
m.ModSeq = cmd.xmodseq()
err := cmd.tx.Update(m)
xcheckf(err, "marking message as seen")
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, ModSeq: m.ModSeq, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords})
}
if cmd.needFlags {
@ -197,6 +346,26 @@ func (cmd *fetchCmd) process(atts []fetchAtt) {
data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords))
}
// The wording around when to include the MODSEQ attribute is hard to follow and is
// specified and refined in several places.
//
// An additional rule applies to "QRESYNC servers" (we'll assume it only applies
// when QRESYNC is enabled on a connection): setting the \Seen flag also triggers
// sending MODSEQ, and so does a UID FETCH command. ../rfc/7162:1421
//
// For example, ../rfc/7162:389 says the server must include modseq in "all
// subsequent untagged fetch responses", then lists cases, but leaves out FETCH/UID
// FETCH. That appears intentional, it is not a list of examples, it is the full
// list, and the "all subsequent untagged fetch responses" doesn't mean "all", just
// those covering the listed cases. That makes sense, because otherwise all the
// other mentioning of cases elsewhere in the RFC would be too superfluous.
//
// ../rfc/7162:877 ../rfc/7162:388 ../rfc/7162:909 ../rfc/7162:1426
if cmd.needModseq || cmd.hasChangedSince || cmd.conn.enabled[capQresync] && (cmd.isUID || cmd.markSeen) {
m := cmd.xensureMessage()
data = append(data, bare("MODSEQ"), listspace{bare(fmt.Sprintf("%d", m.ModSeq.Client()))})
}
// Write errors are turned into panics because we write through c.
fmt.Fprintf(cmd.conn.bw, "* %d FETCH ", cmd.conn.xsequence(cmd.uid))
data.writeTo(cmd.conn, cmd.conn.bw)
@ -301,6 +470,9 @@ func (cmd *fetchCmd) xprocessAtt(a fetchAtt) []token {
case "FLAGS":
cmd.needFlags = true
case "MODSEQ":
cmd.needModseq = true
default:
xserverErrorf("field %q not yet implemented", a.field)
}

3
imapserver/fuzz_test.go
View file

@ -1,7 +1,6 @@
package imapserver
import (
"context"
"encoding/base64"
"errors"
"fmt"
@ -58,7 +57,7 @@ func FuzzServer(f *testing.F) {
f.Add(tag + cmd)
}
mox.Context = context.Background()
mox.Context = ctxbg
mox.ConfigStaticPath = "../testdata/imapserverfuzz/mox.conf"
mox.MustLoadConfig(true, false)
dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)

107
imapserver/parse.go
View file

@ -246,13 +246,21 @@ func (p *parser) xnumber64() int64 {
if s == "" {
p.xerrorf("expected number64")
}
v, err := strconv.ParseInt(s, 10, 64)
v, err := strconv.ParseInt(s, 10, 63) // ../rfc/9051:6794 ../rfc/7162:297
if err != nil {
p.xerrorf("parsing number64 %q: %v", s, err)
}
return v
}
func (p *parser) xnznumber64() int64 {
v := p.xnumber64()
if v == 0 {
p.xerrorf("expected non-zero number64")
}
return v
}
// l should be a list of uppercase words, the first match is returned
func (p *parser) takelist(l ...string) (string, bool) {
for _, w := range l {
@ -423,36 +431,43 @@ func (p *parser) xmboxOrPat() ([]string, bool) {
return l, true
}
// ../rfc/9051:7056
// RECENT only in ../rfc/3501:5047
// APPENDLIMIT is from ../rfc/7889:252
// ../rfc/9051:7056, RECENT ../rfc/3501:5047, APPENDLIMIT ../rfc/7889:252, HIGHESTMODSEQ ../rfc/7162:2452
func (p *parser) xstatusAtt() string {
return p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT")
w := p.xtakelist("MESSAGES", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "DELETED", "SIZE", "RECENT", "APPENDLIMIT", "HIGHESTMODSEQ")
if w == "HIGHESTMODSEQ" {
// HIGHESTMODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:375
p.conn.enabled[capCondstore] = true
}
return w
}
// ../rfc/9051:7133 ../rfc/9051:7034
func (p *parser) xnumSet() (r numSet) {
func (p *parser) xnumSet0(allowStar, allowSearch bool) (r numSet) {
defer p.context("numSet")()
if p.take("$") {
if allowSearch && p.take("$") {
return numSet{searchResult: true}
}
r.ranges = append(r.ranges, p.xnumRange())
r.ranges = append(r.ranges, p.xnumRange0(allowStar))
for p.take(",") {
r.ranges = append(r.ranges, p.xnumRange())
r.ranges = append(r.ranges, p.xnumRange0(allowStar))
}
return r
}
func (p *parser) xnumSet() (r numSet) {
return p.xnumSet0(true, true)
}
// parse numRange, which can be just a setNumber.
func (p *parser) xnumRange() (r numRange) {
if p.take("*") {
func (p *parser) xnumRange0(allowStar bool) (r numRange) {
if allowStar && p.take("*") {
r.first.star = true
} else {
r.first.number = p.xnznumber()
}
if p.take(":") {
r.last = &setNumber{}
if p.take("*") {
if allowStar && p.take("*") {
r.last.star = true
} else {
r.last.number = p.xnznumber()
@ -550,14 +565,16 @@ func (p *parser) xsectionBinary() (r []uint32) {
return r
}
// ../rfc/9051:6557 ../rfc/3501:4751
func (p *parser) xfetchAtt() (r fetchAtt) {
defer p.context("fetchAtt")()
words := []string{
var fetchAttWords = []string{
"ENVELOPE", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODYSTRUCTURE", "UID", "BODY.PEEK", "BODY", "BINARY.PEEK", "BINARY.SIZE", "BINARY",
"RFC822.HEADER", "RFC822.TEXT", "RFC822", // older IMAP
}
f := p.xtakelist(words...)
"MODSEQ", // CONDSTORE extension.
}
// ../rfc/9051:6557 ../rfc/3501:4751 ../rfc/7162:2483
func (p *parser) xfetchAtt(isUID bool) (r fetchAtt) {
defer p.context("fetchAtt")()
f := p.xtakelist(fetchAttWords...)
r.peek = strings.HasSuffix(f, ".PEEK")
r.field = strings.TrimSuffix(f, ".PEEK")
@ -576,12 +593,19 @@ func (p *parser) xfetchAtt() (r fetchAtt) {
}
case "BINARY.SIZE":
r.sectionBinary = p.xsectionBinary()
case "MODSEQ":
// The RFC text mentions MODSEQ is only for FETCH, not UID FETCH, but the ABNF adds
// the attribute to the shared syntax, so UID FETCH also implements it.
// ../rfc/7162:905
// The wording about when to respond with a MODSEQ attribute could be more clear. ../rfc/7162:923 ../rfc/7162:388
// MODSEQ attribute is a CONDSTORE-enabling parameter. ../rfc/7162:377
p.conn.xensureCondstore(nil)
}
return
}
// ../rfc/9051:6553 ../rfc/3501:4748
func (p *parser) xfetchAtts() []fetchAtt {
func (p *parser) xfetchAtts(isUID bool) []fetchAtt {
defer p.context("fetchAtts")()
fields := func(l ...string) []fetchAtt {
@ -605,13 +629,13 @@ func (p *parser) xfetchAtts() []fetchAtt {
}
if !p.hasPrefix("(") {
return []fetchAtt{p.xfetchAtt()}
return []fetchAtt{p.xfetchAtt(isUID)}
}
l := []fetchAtt{}
p.xtake("(")
for {
l = append(l, p.xfetchAtt())
l = append(l, p.xfetchAtt(isUID))
if !p.take(" ") {
break
}
@ -748,9 +772,10 @@ var searchKeyWords = []string{
"SENTBEFORE", "SENTON",
"SENTSINCE", "SMALLER",
"UID", "UNDRAFT",
"MODSEQ", // CONDSTORE extension.
}
// ../rfc/9051:6923 ../rfc/3501:4957
// ../rfc/9051:6923 ../rfc/3501:4957, MODSEQ ../rfc/7162:2492
// differences: rfc 9051 removes NEW, OLD, RECENT and makes SMALLER and LARGER number64 instead of number.
func (p *parser) xsearchKey() *searchKey {
if p.take("(") {
@ -852,12 +877,50 @@ func (p *parser) xsearchKey() *searchKey {
p.xspace()
sk.uidSet = p.xnumSet()
case "UNDRAFT":
case "MODSEQ":
// ../rfc/7162:1045 ../rfc/7162:2499
p.xspace()
if p.take(`"`) {
// We don't do anything with this field, so parse and ignore.
p.xtake(`/FLAGS/`)
if p.take(`\`) {
p.xtake(`\`) // ../rfc/7162:1072
}
p.xatom()
p.xtake(`"`)
p.xspace()
p.xtakelist("PRIV", "SHARED", "ALL")
p.xspace()
}
v := p.xnumber64()
sk.clientModseq = &v
// MODSEQ is a CONDSTORE-enabling parameter. ../rfc/7162:377
p.conn.enabled[capCondstore] = true
default:
p.xerrorf("missing case for op %q", sk.op)
}
return sk
}
// hasModseq returns whether there is a modseq filter anywhere in the searchkey.
func (sk searchKey) hasModseq() bool {
if sk.clientModseq != nil {
return true
}
for _, e := range sk.searchKeys {
if e.hasModseq() {
return true
}
}
if sk.searchKey != nil && sk.searchKey.hasModseq() {
return true
}
if sk.searchKey2 != nil && sk.searchKey2.hasModseq() {
return true
}
return false
}
// ../rfc/9051:6489 ../rfc/3501:4692
func (p *parser) xdateDay() int {
d := p.xdigit()

183
imapserver/protocol.go
View file

@ -80,15 +80,31 @@ func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []sto
return false
}
func (ss numSet) String() string {
if ss.searchResult {
return "$"
}
s := ""
// contains returns whether the numset contains the number.
// only allowed on basic, strictly increasing numsets.
func (ss numSet) contains(v uint32) bool {
for _, r := range ss.ranges {
if s != "" {
s += ","
if r.first.number == v || r.last != nil && v > r.first.number && v <= r.last.number {
return true
}
}
return false
}
func (ss numSet) empty() bool {
return !ss.searchResult && len(ss.ranges) == 0
}
// Strings returns the numset in zero or more strings of maxSize bytes. If
// maxSize is <= 0, a single string is returned.
func (ss numSet) Strings(maxSize int) []string {
if ss.searchResult {
return []string{"$"}
}
var l []string
var line string
for _, r := range ss.ranges {
s := ""
if r.first.star {
s += "*"
} else {
@ -98,8 +114,7 @@ func (ss numSet) String() string {
if r.first.star {
panic("invalid numSet range first star without last")
}
continue
}
} else {
s += ":"
if r.last.star {
s += "*"
@ -107,7 +122,33 @@ func (ss numSet) String() string {
s += fmt.Sprintf("%d", r.last.number)
}
}
return s
nsize := len(line) + len(s)
if line != "" {
nsize++ // comma
}
if maxSize > 0 && nsize > maxSize {
l = append(l, line)
line = s
continue
}
if line != "" {
line += ","
}
line += s
}
if line != "" {
l = append(l, line)
}
return l
}
func (ss numSet) String() string {
l := ss.Strings(0)
if len(l) == 0 {
return ""
}
return l[0]
}
type setNumber struct {
@ -120,6 +161,127 @@ type numRange struct {
last *setNumber // if nil, this numRange is just a setNumber in "first" and first.star will be false
}
// interpretStar returns a numset that interprets stars in a numset, returning a new
// numset without stars with increasing first/last.
func (s numSet) interpretStar(uids []store.UID) numSet {
var ns numSet
for _, r := range s.ranges {
first := r.first.number
if r.first.star {
if len(uids) == 0 {
continue
}
first = uint32(uids[0])
}
last := first
if r.last != nil {
last = r.last.number
if r.last.star {
if len(uids) == 0 {
continue
}
last = uint32(uids[len(uids)-1])
if first > last {
first = last
}
} else if r.first.star && last < first {
last = first
}
}
if first > last {
first, last = last, first
}
nr := numRange{first: setNumber{number: first}}
if first != last {
nr.last = &setNumber{number: last}
}
ns.ranges = append(ns.ranges, nr)
}
return ns
}
// whether numSet only has numbers (no star/search), and is strictly increasing.
func (s *numSet) isBasicIncreasing() bool {
if s.searchResult {
return false
}
var last uint32
for _, r := range s.ranges {
if r.first.star || r.first.number <= last || r.last != nil && (r.last.star || r.last.number < r.first.number) {
return false
}
last = r.first.number
if r.last != nil {
last = r.last.number
}
}
return true
}
type numIter struct {
s numSet
i int
r *rangeIter
}
// newIter must only be called on a numSet that is basic (no star/search) and ascending.
func (s numSet) newIter() *numIter {
return &numIter{s: s, i: 0, r: s.ranges[0].newIter()}
}
func (i *numIter) Next() (uint32, bool) {
if v, ok := i.r.Next(); ok {
return v, ok
}
i.i++
if i.i >= len(i.s.ranges) {
return 0, false
}
i.r = i.s.ranges[i.i].newIter()
return i.r.Next()
}
type rangeIter struct {
r numRange
o int
}
// newIter must only be called on a range in a numSet that is basic (no star/search) and ascending.
func (r numRange) newIter() *rangeIter {
return &rangeIter{r: r, o: 0}
}
func (r *rangeIter) Next() (uint32, bool) {
if r.o == 0 {
r.o++
return r.r.first.number, true
}
if r.r.last == nil || r.r.first.number+uint32(r.o) > r.r.last.number {
return 0, false
}
v := r.r.first.number + uint32(r.o)
r.o++
return v, true
}
// append adds a new number to the set, extending a range, or starting a new one (possibly the first).
// can only be used on basic numsets, without star/searchResult.
func (s *numSet) append(v uint32) {
if len(s.ranges) == 0 {
s.ranges = []numRange{{first: setNumber{number: v}}}
return
}
ri := len(s.ranges) - 1
r := s.ranges[ri]
if v == r.first.number+1 && r.last == nil {
s.ranges[ri].last = &setNumber{number: v}
} else if r.last != nil && v == r.last.number+1 {
r.last.number++
} else {
s.ranges = append(s.ranges, numRange{first: setNumber{number: v}})
}
}
type partial struct {
offset uint32
count uint32
@ -167,6 +329,7 @@ type searchKey struct {
searchKey *searchKey
searchKey2 *searchKey
uidSet numSet
clientModseq *int64
}
func compactUIDSet(l []store.UID) (r numSet) {

27
imapserver/protocol_test.go
View file

@ -62,3 +62,30 @@ func TestNumSetContains(t *testing.T) {
check(!ss3.containsUID(1, []store.UID{2, 3}, nil))
check(!ss3.containsUID(3, []store.UID{1, 2, 3}, nil))
}
func TestNumSetInterpret(t *testing.T) {
parseNumSet := func(s string) numSet {
p := parser{upper: s}
return p.xnumSet0(true, false)
}
checkEqual := func(uids []store.UID, a, s string) {
t.Helper()
n := parseNumSet(a).interpretStar(uids)
ns := n.String()
if ns != s {
t.Fatalf("%s != %s", ns, s)
}
}
checkEqual([]store.UID{}, "1:*", "")
checkEqual([]store.UID{1}, "1:*", "1")
checkEqual([]store.UID{1, 3}, "1:*", "1:3")
checkEqual([]store.UID{1, 3}, "4:*", "3")
checkEqual([]store.UID{2, 3}, "*:4", "2:4")
checkEqual([]store.UID{2, 3}, "*:1", "2")
checkEqual([]store.UID{1, 2, 3}, "1,2,3", "1,2,3")
checkEqual([]store.UID{}, "1,2,3", "1,2,3")
checkEqual([]store.UID{}, "1:3", "1:3")
checkEqual([]store.UID{}, "3:1", "1:3")
}

96
imapserver/search.go
View file

@ -96,6 +96,7 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
}
var expungeIssued bool
var maxModSeq store.ModSeq
var uids []store.UID
c.xdbread(func(tx *bstore.Tx) {
@ -108,8 +109,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
if eargs == nil || max == 0 || len(eargs) != 1 {
for i, uid := range c.uids {
lastIndex = i
if c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued) {
if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, &expungeIssued); match {
uids = append(uids, uid)
if modseq > maxModSeq {
maxModSeq = modseq
}
if min == 1 && min+max == len(eargs) {
break
}
@ -119,8 +123,11 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
// And reverse search for MAX if we have only MAX or MAX combined with MIN.
if max == 1 && (len(eargs) == 1 || min+max == len(eargs)) {
for i := len(c.uids) - 1; i > lastIndex; i-- {
if c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued) {
if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, &expungeIssued); match {
uids = append(uids, c.uids[i])
if modseq > maxModSeq {
maxModSeq = modseq
}
break
}
}
@ -147,11 +154,24 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
}
s += " " + fmt.Sprintf("%d", v)
}
// Since we don't have the max modseq for the possibly partial uid range we're
// writing here within hand reach, we conveniently interpret the ambiguous "for all
// messages being returned" in ../rfc/7162:1107 as meaning over all lines that we
// write. And that clients only commit this value after they have seen the tagged
// end of the command. Appears to be recommended behaviour, ../rfc/7162:2323.
// ../rfc/7162:1077 ../rfc/7162:1101
var modseq string
if sk.hasModseq() {
// ../rfc/7162:2557
modseq = fmt.Sprintf(" (MODSEQ %d)", maxModSeq.Client())
}
c.bwritelinef("* SEARCH%s%s", s, modseq)
uids = uids[n:]
c.bwritelinef("* SEARCH%s", s)
}
} else {
// New-style ESEARCH response. ../rfc/9051:6546 ../rfc/4466:522
// New-style ESEARCH response syntax: ../rfc/9051:6546 ../rfc/4466:522
if save {
// ../rfc/9051:3784 ../rfc/5182:13
@ -195,6 +215,13 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
if eargs["ALL"] && len(uids) > 0 {
resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String())
}
// Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273
// Summary: send the highest modseq of the returned messages.
if sk.hasModseq() && len(uids) > 0 {
resp += fmt.Sprintf(" MODSEQ %d", maxModSeq.Client())
}
c.bwritelinef("%s", resp)
}
}
@ -215,10 +242,11 @@ type search struct {
m store.Message
p *message.Part
expungeIssued *bool
hasModseq bool
}
func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) bool {
s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued}
func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, expungeIssued *bool) (bool, store.ModSeq) {
s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued, hasModseq: sk.hasModseq()}
defer func() {
if s.mr != nil {
err := s.mr.Close()
@ -229,12 +257,42 @@ func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKe
return s.match(sk)
}
func (s *search) match(sk searchKey) bool {
func (s *search) match(sk searchKey) (match bool, modseq store.ModSeq) {
// Instead of littering all the cases in match0 with calls to get modseq, we do it once
// here in case of a match.
defer func() {
if match && s.hasModseq {
if s.m.ID == 0 {
match = s.xloadMessage()
}
modseq = s.m.ModSeq
}
}()
match = s.match0(sk)
return
}
func (s *search) xloadMessage() bool {
q := bstore.QueryTx[store.Message](s.tx)
q.FilterNonzero(store.Message{MailboxID: s.c.mailboxID, UID: s.uid})
m, err := q.Get()
if err == bstore.ErrAbsent || err == nil && m.Expunged {
// ../rfc/2180:607
*s.expungeIssued = true
return false
}
xcheckf(err, "get message")
s.m = m
return true
}
func (s *search) match0(sk searchKey) bool {
c := s.c
if sk.searchKeys != nil {
for _, ssk := range sk.searchKeys {
if !s.match(ssk) {
if !s.match0(ssk) {
return false
}
}
@ -273,33 +331,26 @@ func (s *search) match(sk searchKey) bool {
// We do not implement the RECENT flag. All messages are not recent.
return false
case "NOT":
return !s.match(*sk.searchKey)
return !s.match0(*sk.searchKey)
case "OR":
return s.match(*sk.searchKey) || s.match(*sk.searchKey2)
return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2)
case "UID":
return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult)
}
// Parsed message.
if s.mr == nil {
q := bstore.QueryTx[store.Message](s.tx)
q.FilterNonzero(store.Message{MailboxID: c.mailboxID, UID: s.uid})
m, err := q.Get()
if err == bstore.ErrAbsent {
// ../rfc/2180:607
*s.expungeIssued = true
if !s.xloadMessage() {
return false
}
xcheckf(err, "get message")
s.m = m
// Closed by searchMatch after all (recursive) search.match calls are finished.
s.mr = c.account.MessageReader(m)
s.mr = c.account.MessageReader(s.m)
if m.ParsedBuf == nil {
if s.m.ParsedBuf == nil {
c.log.Error("missing parsed message")
} else {
p, err := m.LoadPart(s.mr)
p, err := s.m.LoadPart(s.mr)
xcheckf(err, "load parsed message")
s.p = &p
}
@ -385,6 +436,9 @@ func (s *search) match(sk searchKey) bool {
return s.m.Size > sk.number
case "SMALLER":
return s.m.Size < sk.number
case "MODSEQ":
// ../rfc/7162:1045
return s.m.ModSeq.Client() >= *sk.clientModseq
}
if s.p == nil {

90
imapserver/search_test.go
View file

@ -39,6 +39,16 @@ func (tc *testconn) xsearch(nums ...uint32) {
tc.xuntagged(imapclient.UntaggedSearch(nums))
}
func (tc *testconn) xsearchmodseq(modseq int64, nums ...uint32) {
tc.t.Helper()
if len(nums) == 0 {
tc.xnountagged()
return
}
tc.xuntagged(imapclient.UntaggedSearchModSeq{Nums: nums, ModSeq: modseq})
}
func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) {
tc.t.Helper()
@ -220,43 +230,11 @@ func TestSearch(t *testing.T) {
tc.transactf("ok", `search charset utf-8 text "mox"`)
tc.xsearch(2, 3)
// esearchall makes an UntaggedEsearch response with All set, for comparisons.
esearchall0 := func(ss string) imapclient.NumSet {
seqset := imapclient.NumSet{}
for _, rs := range strings.Split(ss, ",") {
t := strings.Split(rs, ":")
if len(t) > 2 {
panic("bad seqset")
}
var first uint32
var last *uint32
if t[0] != "*" {
v, err := strconv.ParseUint(t[0], 10, 32)
if err != nil {
panic("parse first")
}
first = uint32(v)
}
if len(t) == 2 {
if t[1] != "*" {
v, err := strconv.ParseUint(t[1], 10, 32)
if err != nil {
panic("parse last")
}
u := uint32(v)
last = &u
}
}
seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last})
}
return seqset
}
esearchall := func(ss string) imapclient.UntaggedEsearch {
return imapclient.UntaggedEsearch{All: esearchall0(ss)}
}
uintptr := func(v uint32) *uint32 {
uint32ptr := func(v uint32) *uint32 {
return &v
}
@ -265,10 +243,10 @@ func TestSearch(t *testing.T) {
tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit.
tc.transactf("ok", "search return (min max count all) all")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(3), All: esearchall0("1:3")})
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(3), All: esearchall0("1:3")})
tc.transactf("ok", "UID search return (min max count all) all")
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(3), All: esearchall0("5:7")})
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")})
tc.transactf("ok", "search return (min) all")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1})
@ -304,19 +282,19 @@ func TestSearch(t *testing.T) {
tc.xesearch(imapclient.UntaggedEsearch{})
tc.transactf("ok", "search return (min max all count) not all")
tc.xesearch(imapclient.UntaggedEsearch{Count: uintptr(0)})
tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(0)})
tc.transactf("ok", "search return (min max count all) 1,3")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
tc.transactf("ok", "search return (min max count all) UID 5,7")
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uintptr(2), All: esearchall0("1,3")})
tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 3, Count: uint32ptr(2), All: esearchall0("1,3")})
tc.transactf("ok", "uid search return (min max count all) 1,3")
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
tc.transactf("ok", "uid search return (min max count all) UID 5,7")
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uintptr(2), All: esearchall0("5,7")})
tc.xesearch(imapclient.UntaggedEsearch{UID: true, Min: 5, Max: 7, Count: uint32ptr(2), All: esearchall0("5,7")})
tc.transactf("no", `search return () charset unknown text "mox"`)
tc.transactf("ok", `search return () charset us-ascii text "mox"`)
@ -353,3 +331,35 @@ func TestSearch(t *testing.T) {
tc.transactf("ok", `search undraft`)
tc.xesearch(esearchall("1:2"))
}
// esearchall makes an UntaggedEsearch response with All set, for comparisons.
func esearchall0(ss string) imapclient.NumSet {
seqset := imapclient.NumSet{}
for _, rs := range strings.Split(ss, ",") {
t := strings.Split(rs, ":")
if len(t) > 2 {
panic("bad seqset")
}
var first uint32
var last *uint32
if t[0] != "*" {
v, err := strconv.ParseUint(t[0], 10, 32)
if err != nil {
panic("parse first")
}
first = uint32(v)
}
if len(t) == 2 {
if t[1] != "*" {
v, err := strconv.ParseUint(t[1], 10, 32)
if err != nil {
panic("parse last")
}
u := uint32(v)
last = &u
}
}
seqset.Ranges = append(seqset.Ranges, imapclient.NumRange{First: first, Last: last})
}
return seqset
}

595
imapserver/server.go
View file

File diff suppressed because it is too large Load diff

32
imapserver/server_test.go
View file

@ -21,6 +21,8 @@ import (
"github.com/mjl-/mox/store"
)
var ctxbg = context.Background()
func init() {
sanityChecks = true
@ -156,6 +158,7 @@ type testconn struct {
client *imapclient.Conn
done chan struct{}
serverConn net.Conn
account *store.Account
// Result of last command.
lastUntagged []imapclient.Untagged
@ -190,14 +193,15 @@ func (tc *testconn) xcodeArg(v any) {
}
}
func (tc *testconn) xuntagged(exps ...any) {
func (tc *testconn) xuntagged(exps ...imapclient.Untagged) {
tc.t.Helper()
tc.xuntaggedCheck(true, exps...)
tc.xuntaggedOpt(true, exps...)
}
func (tc *testconn) xuntaggedCheck(all bool, exps ...any) {
func (tc *testconn) xuntaggedOpt(all bool, exps ...imapclient.Untagged) {
tc.t.Helper()
last := append([]imapclient.Untagged{}, tc.lastUntagged...)
var mismatch any
next:
for ei, exp := range exps {
for i, l := range last {
@ -205,12 +209,16 @@ next:
continue
}
if !reflect.DeepEqual(l, exp) {
tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", l, l, exp, exp)
mismatch = l
continue
}
copy(last[i:], last[i+1:])
last = last[:len(last)-1]
continue next
}
if mismatch != nil {
tc.t.Fatalf("untagged data mismatch, got:\n\t%T %#v\nexpected:\n\t%T %#v", mismatch, mismatch, exp, exp)
}
var next string
if len(tc.lastUntagged) > 0 {
next = fmt.Sprintf(", next %#v", tc.lastUntagged[0])
@ -293,11 +301,21 @@ func (tc *testconn) waitDone() {
}
func (tc *testconn) close() {
err := tc.account.Close()
tc.check(err, "close account")
tc.client.Close()
tc.serverConn.Close()
tc.waitDone()
}
func xparseNumSet(s string) imapclient.NumSet {
ns, err := imapclient.ParseNumSet(s)
if err != nil {
panic(fmt.Sprintf("parsing numset %s: %s", s, err))
}
return ns
}
var connCounter int64
func start(t *testing.T) *testconn {
@ -314,7 +332,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
if first {
os.RemoveAll("../testdata/imap/data")
}
mox.Context = context.Background()
mox.Context = ctxbg
mox.ConfigStaticPath = "../testdata/imap/mox.conf"
mox.MustLoadConfig(true, false)
acc, err := store.OpenAccount("mjl")
@ -323,8 +341,6 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
err = acc.SetPassword("testtest")
tcheck(t, err, "set password")
}
err = acc.Close()
tcheck(t, err, "close account")
var switchDone chan struct{}
if first {
switchDone = store.Switchboard()
@ -352,7 +368,7 @@ func startArgs(t *testing.T, first, isTLS, allowLoginWithoutTLS bool) *testconn
}()
client, err := imapclient.New(clientConn, true)
tcheck(t, err, "new client")
return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn}
return &testconn{t: t, conn: clientConn, client: client, done: done, serverConn: serverConn, account: acc}
}
func fakeCert(t *testing.T) tls.Certificate {

2
imapserver/store_test.go
View file

@ -78,7 +78,7 @@ func TestStore(t *testing.T) {
// Flags are added to mailbox, not removed.
flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent new a b c d e different`, " ")
tc.xuntaggedCheck(false, imapclient.UntaggedFlags(flags))
tc.xuntaggedOpt(false, imapclient.UntaggedFlags(flags))
tc.transactf("no", `store 1 flags ()`) // No permission to set flags.
}

12
import.go
View file

@ -270,6 +270,8 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
var changes []store.Change
var modseq store.ModSeq // Assigned on first delivered messages, used for all messages.
xdeliver := func(m *store.Message, mf *os.File) {
// todo: possibly set dmarcdomain to the domain of the from address? at least for non-spams that have been seen. otherwise user would start without any reputations. the assumption would be that the user has accepted email and deemed it legit, coming from the indicated sender.
@ -281,7 +283,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
ctl.xcheck(err, "delivering message")
deliveredIDs = append(deliveredIDs, m.ID)
ctl.log.Debug("delivered message", mlog.Field("id", m.ID))
changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords})
changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, ModSeq: modseq, Flags: m.Flags, Keywords: m.Keywords})
}
// todo: one goroutine for reading messages, one for parsing the message, one adding to database, one for junk filter training.
@ -353,8 +355,16 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) {
}
}
if modseq == 0 {
var err error
modseq, err = a.NextModSeq(tx)
ctl.xcheck(err, "assigning next modseq")
}
m.MailboxID = mb.ID
m.MailboxOrigID = mb.ID
m.CreateSeq = modseq
m.ModSeq = modseq
xdeliver(m, msgf)
err = msgf.Close()
ctl.log.Check(err, "closing message after delivery")

1
integration_test.go
View file

@ -184,5 +184,4 @@ a message.
tcheck(t, err, "sendmail")
})
xlog.Print("success", mlog.Field("duration", time.Since(t0)))
}

10
main.go
View file

@ -2003,6 +2003,7 @@ func cmdEnsureParsed(c *cmd) {
n := 0
err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
q := bstore.QueryTx[store.Message](tx)
q.FilterEqual("Expunged", false)
q.FilterFn(func(m store.Message) bool {
return all || m.ParsedBuf == nil
})
@ -2135,17 +2136,22 @@ open, or is not running.
err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
// Reassign UIDs, going per mailbox. We assign starting at 1, only changing the
// message if it isn't already at the intended UID. Doing it in this order ensures
// we don't get into trouble with duplicate UIDs for a mailbox.
// we don't get into trouble with duplicate UIDs for a mailbox. We assign a new
// modseq. Not strictly needed, for doesn't hurt.
modseq, err := a.NextModSeq(tx)
xcheckf(err, "assigning next modseq")
q := bstore.QueryTx[store.Message](tx)
if len(args) == 2 {
q.FilterNonzero(store.Message{MailboxID: mailboxID})
}
q.SortAsc("MailboxID", "UID")
err := q.ForEach(func(m store.Message) error {
err = q.ForEach(func(m store.Message) error {
uidlasts[m.MailboxID]++
uid := uidlasts[m.MailboxID]
if m.UID != uid {
m.UID = uid
m.ModSeq = modseq
if err := tx.Update(&m); err != nil {
return fmt.Errorf("updating uid for message: %v", err)
}

5
serve.go
View file

@ -301,7 +301,10 @@ requested, other TLS certificates are requested on demand.
log.Check(err, "closing temp changelog file")
}
}()
m := &store.Message{Received: time.Now(), Flags: store.Flags{Flagged: true}}
m := &store.Message{
Received: time.Now(),
Flags: store.Flags{Flagged: true},
}
n, err := fmt.Fprintf(f, "Date: %s\r\nSubject: mox %s available\r\n\r\nHi!\r\n\r\nVersion %s of mox is available, this install is at %s.\r\n\r\nChanges:\r\n\r\n%s\r\n\r\nRemember to make a backup with \"mox backup\" before upgrading.\r\nPlease report any issues at https://github.com/mjl-/mox, thanks!\r\n\r\nCheers,\r\nmox\r\n", time.Now().Format(message.RFC5322Z), latest, latest, current, strings.ReplaceAll(cl, "\n", "\r\n"))
if err != nil {
log.Infox("writing temporary message file for changelog delivery", err)

1
smtpserver/rejects.go
View file

@ -54,6 +54,7 @@ func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m *
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MailboxID: mb.ID})
q.FilterEqual("Expunged", false)
q.FilterFn(func(m store.Message) bool {
return msgID != "" && m.MessageID == msgID || len(hash) > 0 && bytes.Equal(m.MessageHash, hash)
})

1
smtpserver/reputation.go
View file

@ -124,6 +124,7 @@ func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rc
messageQuery := func(fm *store.Message, maxAge time.Duration, maxCount int) *bstore.Query[store.Message] {
q := bstore.QueryTx[store.Message](tx)
q.FilterEqual("MailboxOrigID", m.MailboxID)
q.FilterEqual("Expunged", false)
q.FilterFn(func(m store.Message) bool {
return m.Junk || m.Notjunk
})

2
smtpserver/server.go
View file

@ -2297,6 +2297,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(msg)
q.FilterGreater("Received", now.Add(-window))
q.FilterEqual("Expunged", false)
n, err := q.Count()
if err != nil {
retErr = err
@ -2315,6 +2316,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(msg)
q.FilterGreater("Received", now.Add(-window))
q.FilterEqual("Expunged", false)
size := msgWriter.Size
err := q.ForEach(func(v store.Message) error {
size += v.Size

4
smtpserver/server_test.go
View file

@ -343,6 +343,7 @@ func tretrain(t *testing.T, acc *store.Account) {
// Fetch messags to retrain on.
q := bstore.QueryDB[store.Message](ctxbg, acc.DB)
q.FilterEqual("Expunged", false)
q.FilterFn(func(m store.Message) bool {
return m.Flags.Junk || m.Flags.Notjunk
})
@ -412,6 +413,7 @@ func TestSpam(t *testing.T) {
tcheck(t, err, "get rejects mailbox")
qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
qm.FilterNonzero(store.Message{MailboxID: mb.ID})
qm.FilterEqual("Expunged", false)
n, err := qm.Count()
tcheck(t, err, "count messages in rejects mailbox")
if n != expect {
@ -437,6 +439,7 @@ func TestSpam(t *testing.T) {
// Mark the messages as having good reputation.
q := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
q.FilterEqual("Expunged", false)
_, err := q.UpdateFields(map[string]any{"Junk": false, "Notjunk": true})
tcheck(t, err, "update junkiness")
@ -456,6 +459,7 @@ func TestSpam(t *testing.T) {
// Undo dmarc pass, mark messages as junk, and train the filter.
resolver.TXT = nil
q = bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
q.FilterEqual("Expunged", false)
_, err = q.UpdateFields(map[string]any{"Junk": true, "Notjunk": false})
tcheck(t, err, "update junkiness")
tretrain(t, ts.acc)

150
store/account.go
View file

@ -150,6 +150,24 @@ type NextUIDValidity struct {
Next uint32
}
// SyncState track ModSeqs.
type SyncState struct {
ID int // Just a single record with ID 1.
// Last used, next assigned will be one higher. The first value we hand out is 2.
// That's because 0 (the default value for old existing messages, from before the
// Message.ModSeq field) is special in IMAP, so we return it as 1.
LastModSeq ModSeq `bstore:"nonzero"`
// Highest ModSeq of expunged record that we deleted. When a clients synchronizes
// and requests changes based on a modseq before this one, we don't have the
// history to provide information about deletions. We normally keep these expunged
// records around, but we may periodically truly delete them to reclaim storage
// space. Initially set to -1 because we don't want to match with any ModSeq in the
// database, which can be zero values.
HighestDeletedModSeq ModSeq
}
// Mailbox is collection of messages, e.g. Inbox or Sent.
type Mailbox struct {
ID int64
@ -232,7 +250,23 @@ type Message struct {
ID int64
UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.
MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,ref Mailbox"`
MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`
// Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.
// ModSeq is the last modification. CreateSeq is the Seq the message was inserted,
// always <= ModSeq. If Expunged is set, the message has been removed and should not
// be returned to the user. In this case, ModSeq is the Seq where the message is
// removed, and will never be changed again.
// We have an index on both ModSeq (for JMAP that synchronizes per account) and
// MailboxID+ModSeq (for IMAP that synchronizes per mailbox).
// The index on CreateSeq helps efficiently finding created messages for JMAP.
// The value of ModSeq is special for IMAP. Messages that existed before ModSeq was
// added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If
// we get modseq 1 from a client, the IMAP server will translate it to 0. When we
// return modseq to clients, we turn 0 into 1.
ModSeq ModSeq `bstore:"index"`
CreateSeq ModSeq `bstore:"index"`
Expunged bool
// MailboxOrigID is the mailbox the message was originally delivered to. Typically
// Inbox or Rejects, but can also be Postmaster and TLS/DMARC reporting addresses.
@ -290,7 +324,9 @@ type Message struct {
// delivered only once. Value includes <>.
MessageID string `bstore:"index"`
MessageHash []byte // Hash of message. For rejects delivery, so optional like MessageID.
// Hash of message. For rejects delivery, so optional like MessageID.
MessageHash []byte
Flags
Keywords []string `bstore:"index"` // For keywords other than system flags or the basic well-known $-flags. Only in "atom" syntax, stored in lower case.
Size int64
@ -304,6 +340,41 @@ type Message struct {
ParsedBuf []byte
}
// ModSeq represents a modseq as stored in the database. ModSeq 0 in the
// database is sent to the client as 1, because modseq 0 is special in IMAP.
// ModSeq coming from the client are of type int64.
type ModSeq int64
func (ms ModSeq) Client() int64 {
if ms == 0 {
return 1
}
return int64(ms)
}
// ModSeqFromClient converts a modseq from a client to a modseq for internal
// use, e.g. in a database query.
// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).
func ModSeqFromClient(modseq int64) ModSeq {
if modseq == 1 {
return 0
}
return ModSeq(modseq)
}
// PrepareExpunge clears fields that are no longer needed after an expunge, so
// almost all fields. Does not change ModSeq, but does set Expunged.
func (m *Message) PrepareExpunge() {
*m = Message{
ID: m.ID,
UID: m.UID,
MailboxID: m.MailboxID,
CreateSeq: m.CreateSeq,
ModSeq: m.ModSeq,
Expunged: true,
}
}
// LoadPart returns a message.Part by reading from m.ParsedBuf.
func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {
if m.ParsedBuf == nil {
@ -385,7 +456,7 @@ type Outgoing struct {
}
// Types stored in DB.
var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}}
var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}}
// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
type Account struct {
@ -564,6 +635,33 @@ func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {
return v, nil
}
// NextModSeq returns the next modification sequence, which is global per account,
// over all types.
func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {
v := SyncState{ID: 1}
if err := tx.Get(&v); err == bstore.ErrAbsent {
// We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so
// already used.
// HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value
// makes sense.
v = SyncState{1, 2, -1}
return v.LastModSeq, tx.Insert(&v)
} else if err != nil {
return 0, err
}
v.LastModSeq++
return v.LastModSeq, tx.Update(&v)
}
func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {
v := SyncState{ID: 1}
err := tx.Get(&v)
if err == bstore.ErrAbsent {
return 0, nil
}
return v.HighestDeletedModSeq, err
}
// WithWLock runs fn with account writelock held. Necessary for account/mailbox modification. For message delivery, a read lock is required.
func (a *Account) WithWLock(fn func()) {
a.Lock()
@ -593,10 +691,16 @@ func (a *Account) WithRLock(fn func()) {
// If sync is true, the message file and its directory are synced. Should be true
// for regular mail delivery, but can be false when importing many messages.
//
// If CreateSeq/ModSeq is not set, it is assigned automatically.
//
// Must be called with account rlock or wlock.
//
// Caller must broadcast new message.
func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, consumeFile, isSent, sync, notrain bool) error {
if m.Expunged {
return fmt.Errorf("cannot deliver expunged message")
}
mb := Mailbox{ID: m.MailboxID}
if err := tx.Get(&mb); err != nil {
return fmt.Errorf("get mailbox: %w", err)
@ -630,6 +734,14 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {
m.MailboxDestinedID = 0
}
if m.CreateSeq == 0 || m.ModSeq == 0 {
modseq, err := a.NextModSeq(tx)
if err != nil {
return fmt.Errorf("assigning next modseq: %w", err)
}
m.CreateSeq = modseq
m.ModSeq = modseq
}
if err := tx.Insert(m); err != nil {
return fmt.Errorf("inserting message: %w", err)
@ -1038,7 +1150,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF
return err
}
changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.Flags, m.Keywords})
changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords})
BroadcastChanges(a, changes)
return nil
}
@ -1126,27 +1238,33 @@ func (a *Account) removeMessages(ctx context.Context, log *mlog.Log, tx *bstore.
return nil, fmt.Errorf("deleting from message recipient: %w", err)
}
// Assign new modseq.
modseq, err := a.NextModSeq(tx)
if err != nil {
return nil, fmt.Errorf("assign next modseq: %w", err)
}
// Actually remove the messages.
qdm := bstore.QueryTx[Message](tx)
qdm.FilterIDs(ids)
var deleted []Message
qdm.Gather(&deleted)
if _, err := qdm.Delete(); err != nil {
return nil, fmt.Errorf("deleting from messages: %w", err)
qx := bstore.QueryTx[Message](tx)
qx.FilterIDs(ids)
var expunged []Message
qx.Gather(&expunged)
if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {
return nil, fmt.Errorf("expunging messages: %w", err)
}
// Mark as neutral and train so junk filter gets untrained with these (junk) messages.
for i := range deleted {
deleted[i].Junk = false
deleted[i].Notjunk = false
for i := range expunged {
expunged[i].Junk = false
expunged[i].Notjunk = false
}
if err := a.RetrainMessages(ctx, log, tx, deleted, true); err != nil {
return nil, fmt.Errorf("training deleted messages: %w", err)
if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {
return nil, fmt.Errorf("retraining expunged messages: %w", err)
}
changes := make([]Change, len(l))
for i, m := range l {
changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}}
changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}
}
return changes, nil
}

3
store/state.go
View file

@ -28,6 +28,7 @@ type Change any
type ChangeAddUID struct {
MailboxID int64
UID UID
ModSeq ModSeq
Flags Flags // System flags.
Keywords []string // Other flags.
}
@ -36,12 +37,14 @@ type ChangeAddUID struct {
type ChangeRemoveUIDs struct {
MailboxID int64
UIDs []UID
ModSeq ModSeq
}
// ChangeFlags is sent for an update to flags for a message, e.g. "Seen".
type ChangeFlags struct {
MailboxID int64
UID UID
ModSeq ModSeq
Mask Flags // Which flags are actually modified.
Flags Flags // New flag values. All are set, not just mask.
Keywords []string // Other flags.

8
test-upgrade.sh
View file

@ -9,9 +9,11 @@
set -e
# set -x
# We'll allow max 256mb of memory during upgrades. We modify the softlimit when
# We'll set a max memory limit during upgrades. We modify the softlimit when
# importing the potentially large mbox file.
ulimit -S -d 256000
# Currently at 768MB, needed for upgrading with 500k messages from v0.0.5 to
# v0.0.6 (two new indexes on store.Message).
ulimit -S -d 768000
(rm -r testdata/upgrade 2>/dev/null || exit 0)
mkdir testdata/upgrade
@ -59,7 +61,7 @@ for tag in $tags; do
echo 'Importing bulk data for upgrading.'
gunzip < ../upgradetest.mbox.gz | time ./$tag/mox ximport mbox ./stepdata/accounts/test0 upgradetest /dev/stdin
echo
ulimit -S -d 256000
ulimit -S -d 768000
fi
echo "Upgrade data to $tag."

4
verifydata.go
View file

@ -246,6 +246,10 @@ possibly making them potentially no longer readable by the previous version.
if uidnext := mailboxUIDNexts[m.MailboxID]; m.UID >= uidnext {
checkf(errors.New(`inconsistent uidnext for message/mailbox, see "mox fixuidmeta"`), dbpath, "message id %d in mailbox id %d has uid %d >= mailbox uidnext %d", m.ID, m.MailboxID, m.UID, uidnext)
}
if m.Expunged {
return nil
}
mp := store.MessagePath(m.ID)
seen[mp] = struct{}{}
p := filepath.Join(accdir, "msg", mp)