imapserver: implement the MULTISEARCH extension, with its ESEARCH command

README.md

@@ -152,9 +152,9 @@ https://nlnet.nl/project/Mox/.
 - External addresses in aliases/lists.
 - Autoresponder (out of office/vacation)
 - Mailing list manager
-- IMAP extensions for "online"/non-syncing/webmail clients (MULTISEARCH, SORT (including
+- IMAP extensions for "online"/non-syncing/webmail clients (SORT (including
   DISPLAYFROM, DISPLAYTO), THREAD, PARTIAL, CONTEXT=SEARCH CONTEXT=SORT ESORT,
-  FILTERS, PREVIEW)
+  FILTERS)
 - IMAP ACL support, for account sharing (interacts with many extensions and code)
 - Improve support for mobile clients with extensions: IMAP URLAUTH, SMTP
   CHUNKING and BINARYMIME, IMAP CATENATE

imapclient/parse.go

@@ -1414,15 +1414,54 @@ func (c *Conn) xneedDisabled(msg string, caps ...Capability) {
 // ../rfc/9051:6546
 // Already consumed: "ESEARCH"
 func (c *Conn) xesearchResponse() (r UntaggedEsearch) {
-
 	if !c.space() {
 		return
 	}
+
 	if c.take('(') {
-		// ../rfc/9051:6921
-		c.xtake("TAG")
-		c.xspace()
-		r.Correlator = c.xastring()
+		// ../rfc/9051:6921 ../rfc/7377:465
+		seen := map[string]bool{}
+		for {
+			var kind string
+			if c.peek('t') || c.peek('T') {
+				kind = "TAG"
+				c.xtake(kind)
+				c.xspace()
+				r.Tag = c.xastring()
+			} else if c.peek('m') || c.peek('M') {
+				kind = "MAILBOX"
+				c.xtake(kind)
+				c.xspace()
+				r.Mailbox = c.xastring()
+				if r.Mailbox == "" {
+					c.xerrorf("invalid empty mailbox in search correlator")
+				}
+			} else if c.peek('u') || c.peek('U') {
+				kind = "UIDVALIDITY"
+				c.xtake(kind)
+				c.xspace()
+				r.UIDValidity = c.xnzuint32()
+			} else {
+				c.xerrorf("expected tag/correlator, mailbox or uidvalidity")
+			}
+
+			if seen[kind] {
+				c.xerrorf("duplicate search correlator %q", kind)
+			}
+			seen[kind] = true
+
+			if !c.take(' ') {
+				break
+			}
+		}
+
+		if r.Tag == "" {
+			c.xerrorf("missing tag search correlator")
+		}
+		if (r.Mailbox != "") != (r.UIDValidity != 0) {
+			c.xerrorf("mailbox and uidvalidity correlators must both be absent or both be present")
+		}
+
 		c.xtake(")")
 	}
 	if !c.space() {

imapclient/protocol.go

@@ -41,6 +41,7 @@ const (
 	CapMultiAppend      Capability = "MULTIAPPEND"        // ../rfc/3502:33
 	CapReplace          Capability = "REPLACE"            // ../rfc/8508:155
 	CapPreview          Capability = "PREVIEW"            // ../rfc/8970:114
+	CapMultiSearch      Capability = "MULTISEARCH"        // ../rfc/7377:187
 )
 
 // Status is the tagged final result of a command.
@@ -314,15 +315,17 @@ type UntaggedLsub struct {
 
 // Fields are optional and zero if absent.
 type UntaggedEsearch struct {
-	// ../rfc/9051:6546
-	Correlator string
-	UID        bool
-	Min        uint32
-	Max        uint32
-	All        NumSet
-	Count      *uint32
-	ModSeq     int64
-	Exts       []EsearchDataExt
+	Tag         string // ../rfc/9051:6546
+	Mailbox     string // For MULTISEARCH. ../rfc/7377:437
+	UIDValidity uint32 // For MULTISEARCH, ../rfc/7377:438
+
+	UID    bool
+	Min    uint32
+	Max    uint32
+	All    NumSet
+	Count  *uint32
+	ModSeq int64
+	Exts   []EsearchDataExt
 }
 
 // UntaggedVanished is used in QRESYNC to send UIDs that have been removed.

imapserver/condstore_test.go

@@ -264,9 +264,6 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 	tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq %d", clientModseq)
 	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: clientModseq})
 
-	uint32ptr := func(v uint32) *uint32 {
-		return &v
-	}
 	tc.transactf("ok", "Search Return (Count) 1:* Modseq 0")
 	tc.xesearch(imapclient.UntaggedEsearch{Count: uint32ptr(6), ModSeq: clientModseq})
 
@@ -331,7 +328,7 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
 	tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 8")
 	tc.xesearch(imapclient.UntaggedEsearch{Min: 1, Max: 1, All: esearchall0("1"), ModSeq: 8})
 	tc.transactf("ok", "Search Return (Min Max All) 1:* Modseq 9")
-	tc.xuntagged(imapclient.UntaggedEsearch{Correlator: tc.client.LastTag})
+	tc.xuntagged(imapclient.UntaggedEsearch{Tag: tc.client.LastTag})
 
 	// store, cannot modify expunged messages.
 	tc.transactf("ok", `Uid Store 3,4 (Unchangedsince %d) +Flags (label2)`, clientModseq)

imapserver/parse.go

@@ -965,6 +965,23 @@ func (sk searchKey) hasModseq() bool {
 	return false
 }
 
+// Whether we need message sequence numbers to evaluate. If not, we cannot optimize
+// when only MAX is requested through a reverse query.
+func (sk searchKey) needSeq() bool {
+	for _, k := range sk.searchKeys {
+		if k.needSeq() {
+			return true
+		}
+	}
+	if sk.searchKey != nil && sk.searchKey.needSeq() {
+		return true
+	}
+	if sk.searchKey2 != nil && sk.searchKey2.needSeq() {
+		return true
+	}
+	return sk.seqSet != nil && !sk.seqSet.searchResult
+}
+
 // ../rfc/9051:6489 ../rfc/3501:4692
 func (p *parser) xdateDay() int {
 	d := p.xdigit()

imapserver/protocol.go

@@ -32,17 +32,25 @@ func (ss numSet) containsSeq(seq msgseq, uids []store.UID, searchResult []store.
 		uid := uids[int(seq)-1]
 		return uidSearch(searchResult, uid) > 0 && uidSearch(uids, uid) > 0
 	}
+	return ss.containsSeqCount(seq, uint32(len(uids)))
+}
+
+// containsSeqCount returns whether seq is contained in ss, which must not be a searchResult, assuming the message count.
+func (ss numSet) containsSeqCount(seq msgseq, msgCount uint32) bool {
+	if msgCount == 0 {
+		return false
+	}
 	for _, r := range ss.ranges {
 		first := r.first.number
-		if r.first.star || first > uint32(len(uids)) {
-			first = uint32(len(uids))
+		if r.first.star || first > msgCount {
+			first = msgCount
 		}
 
 		last := first
 		if r.last != nil {
 			last = r.last.number
-			if r.last.star || last > uint32(len(uids)) {
-				last = uint32(len(uids))
+			if r.last.star || last > msgCount {
+				last = msgCount
 			}
 		}
 		if first > last {
@@ -87,6 +95,56 @@ func (ss numSet) containsUID(uid store.UID, uids []store.UID, searchResult []sto
 	return false
 }
 
+// containsKnownUID returns whether uid, which is known to exist, matches the numSet.
+// highestUID must return the highest/last UID in the mailbox, or an error. A last UID must
+// exist, otherwise this method wouldn't have been called with a known uid.
+// highestUID is needed for interpreting UID sets like "<num>:*" where num is
+// higher than the uid to check.
+func (ss numSet) containsKnownUID(uid store.UID, searchResult []store.UID, highestUID func() (store.UID, error)) (bool, error) {
+	if ss.searchResult {
+		return uidSearch(searchResult, uid) > 0, nil
+	}
+
+	for _, r := range ss.ranges {
+		a := store.UID(r.first.number)
+		// Num in <num>:* can be larger than last, but it still matches the last...
+		// Similar for *:<num>. ../rfc/9051:4814
+		if r.first.star {
+			if r.last != nil && uid >= store.UID(r.last.number) {
+				return true, nil
+			}
+
+			var err error
+			a, err = highestUID()
+			if err != nil {
+				return false, err
+			}
+		}
+		b := a
+		if r.last != nil {
+			b = store.UID(r.last.number)
+			if r.last.star {
+				if uid >= a {
+					return true, nil
+				}
+
+				var err error
+				b, err = highestUID()
+				if err != nil {
+					return false, err
+				}
+			}
+		}
+		if a > b {
+			a, b = b, a
+		}
+		if uid >= a && uid <= b {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // contains returns whether the numset contains the number.
 // only allowed on basic, strictly increasing numsets.
 func (ss numSet) contains(v uint32) bool {
@@ -312,9 +370,10 @@ type fetchAtt struct {
 
 type searchKey struct {
 	// Only one of searchKeys, seqSet and op can be non-nil/non-empty.
-	searchKeys   []searchKey // In case of nested/multiple keys. Also for the top-level command.
-	seqSet       *numSet     // In case of bare sequence set. For op UID, field uidSet contains the parameter.
-	op           string      // Determines which of the fields below are set.
+	searchKeys []searchKey // In case of nested/multiple keys. Also for the top-level command.
+	seqSet     *numSet     // In case of bare sequence set. For op UID, field uidSet contains the parameter.
+	op         string      // Determines which of the fields below are set.
+
 	headerField  string
 	astring      string
 	date         time.Time

imapserver/search.go

@@ -1,9 +1,12 @@
 package imapserver
 
 import (
+	"cmp"
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/textproto"
+	"slices"
 	"strings"
 	"time"
 
@@ -11,29 +14,76 @@ import (
 
 	"github.com/mjl-/mox/message"
 	"github.com/mjl-/mox/store"
-	"slices"
 )
 
 // If last search output was this long ago, we write an untagged inprogress
 // response. Changed during tests. ../rfc/9585:109
 var inProgressPeriod = time.Duration(10 * time.Second)
 
+// ESEARCH allows searching multiple mailboxes, referenced through mailbox filters
+// borrowed from the NOTIFY extension. Unlike the regular extended SEARCH/UID
+// SEARCH command that always returns an ESEARCH response, the ESEARCH command only
+// returns ESEARCH responses when there were matches in a mailbox.
+//
+// ../rfc/7377:159
+func (c *conn) cmdEsearch(tag, cmd string, p *parser) {
+	c.cmdxSearch(true, true, tag, cmd, p)
+}
+
 // Search returns messages matching criteria specified in parameters.
 //
-// State: Selected
-func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
-	// Command: ../rfc/9051:3716 ../rfc/4731:31 ../rfc/4466:354 ../rfc/3501:2723
-	// Examples: ../rfc/9051:3986 ../rfc/4731:153 ../rfc/3501:2975
-	// Syntax: ../rfc/9051:6918 ../rfc/4466:611 ../rfc/3501:4954
+// State: Selected for SEARCH and UID SEARCH, Authenticated or selectd for ESEARCH.
+func (c *conn) cmdxSearch(isUID, isE bool, tag, cmd string, p *parser) {
+	// Command: ../rfc/9051:3716 ../rfc/7377:159 ../rfc/6237:142 ../rfc/4731:31 ../rfc/4466:354 ../rfc/3501:2723
+	// Examples: ../rfc/9051:3986 ../rfc/7377:385 ../rfc/6237:323 ../rfc/4731:153 ../rfc/3501:2975
+	// Syntax: ../rfc/9051:6918 ../rfc/7377:462 ../rfc/6237:403 ../rfc/4466:611 ../rfc/3501:4954
 
-	// We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2.
+	// We will respond with ESEARCH instead of SEARCH if "RETURN" is present or for IMAP4rev2 or for isE (ESEARCH command).
 	var eargs map[string]bool // Options except SAVE. Nil means old-style SEARCH response.
 	var save bool             // For SAVE option. Kept separately for easier handling of MIN/MAX later.
 
-	// IMAP4rev2 always returns ESEARCH, even with absent RETURN.
-	if c.enabled[capIMAP4rev2] {
+	if c.enabled[capIMAP4rev2] || isE {
 		eargs = map[string]bool{}
 	}
+
+	// The ESEARCH command has various ways to specify which mailboxes are to be
+	// searched. We parse and gather the request first, and evaluate them to mailboxes
+	// after parsing, when we start and have a DB transaction.
+	type mailboxSpec struct {
+		Kind string
+		Args []string
+	}
+	var mailboxSpecs []mailboxSpec
+
+	// ../rfc/7377:468
+	if isE && p.take(" IN (") {
+		for {
+			mbs := mailboxSpec{}
+			mbs.Kind = p.xtakelist("SELECTED", "INBOXES", "PERSONAL", "SUBSCRIBED", "SUBTREE-ONE", "SUBTREE", "MAILBOXES")
+			switch mbs.Kind {
+			case "SUBTREE", "SUBTREE-ONE", "MAILBOXES":
+				p.xtake(" ")
+				if p.take("(") {
+					for {
+						mbs.Args = append(mbs.Args, p.xmailbox())
+						if !p.take(" ") {
+							break
+						}
+					}
+					p.xtake(")")
+				} else {
+					mbs.Args = []string{p.xmailbox()}
+				}
+			}
+			mailboxSpecs = append(mailboxSpecs, mbs)
+
+			if !p.take(" ") {
+				break
+			}
+		}
+		p.xtake(")")
+		// We are not parsing the scope-options since there aren't any defined yet. ../rfc/7377:469
+	}
 	// ../rfc/9051:6967
 	if p.take(" RETURN (") {
 		eargs = map[string]bool{}
@@ -131,16 +181,22 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 
 	// If we only have a MIN and/or MAX, we can stop processing as soon as we
 	// have those matches.
-	var min, max int
+	var min1, max1 int
 	if eargs["MIN"] {
-		min = 1
+		min1 = 1
 	}
 	if eargs["MAX"] {
-		max = 1
+		max1 = 1
 	}
 
-	var expungeIssued bool
-	var maxModSeq store.ModSeq
+	// We'll have one Result per mailbox we are searching. For regular (UID) SEARCH
+	// commands, we'll have just one, for the selected mailbox.
+	type Result struct {
+		Mailbox   store.Mailbox
+		MaxModSeq store.ModSeq
+		UIDs      []store.UID
+	}
+	var results []Result
 
 	// We periodically send an untagged OK with INPROGRESS code while searching, to let
 	// clients doing slow searches know we're still working.
@@ -151,73 +207,308 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 		inProgressTag = dquote(tag).pack(c)
 	}
 
-	var uids []store.UID
 	c.xdbread(func(tx *bstore.Tx) {
-		c.xmailboxID(tx, c.mailboxID) // Validate.
+		// Gather mailboxes to operate on. Usually just the selected mailbox. But with the
+		// ESEARCH command, we may be searching multiple.
+		var mailboxes []store.Mailbox
+		if len(mailboxSpecs) > 0 {
+			// While gathering, we deduplicate mailboxes. ../rfc/7377:312
+			m := map[int64]store.Mailbox{}
+			for _, mbs := range mailboxSpecs {
+				switch mbs.Kind {
+				case "SELECTED":
+					// ../rfc/7377:306
+					if c.state != stateSelected {
+						xsyntaxErrorf("cannot use ESEARCH with selected when state is not selected")
+					}
+
+					mb := c.xmailboxID(tx, c.mailboxID) // Validate.
+					m[mb.ID] = mb
+
+				case "INBOXES":
+					// Inbox and everything below. And we look at destinations and rulesets. We all
+					// mailboxes from the destinations, and all from the rulesets except when
+					// ListAllowDomain is non-empty.
+					// ../rfc/5465:822
+					q := bstore.QueryTx[store.Mailbox](tx)
+					q.FilterEqual("Expunged", false)
+					q.FilterGreaterEqual("Name", "Inbox")
+					q.SortAsc("Name")
+					for mb, err := range q.All() {
+						xcheckf(err, "list mailboxes")
+						if mb.Name != "Inbox" && !strings.HasPrefix(mb.Name, "Inbox/") {
+							break
+						}
+						m[mb.ID] = mb
+					}
+
+					conf, _ := c.account.Conf()
+					for _, dest := range conf.Destinations {
+						if dest.Mailbox != "" && dest.Mailbox != "Inbox" {
+							mb, err := c.account.MailboxFind(tx, dest.Mailbox)
+							xcheckf(err, "find mailbox from destination")
+							if mb != nil {
+								m[mb.ID] = *mb
+							}
+						}
+
+						for _, rs := range dest.Rulesets {
+							if rs.ListAllowDomain != "" || rs.Mailbox == "" {
+								continue
+							}
+
+							mb, err := c.account.MailboxFind(tx, rs.Mailbox)
+							xcheckf(err, "find mailbox from ruleset")
+							if mb != nil {
+								m[mb.ID] = *mb
+							}
+						}
+					}
+
+				case "PERSONAL":
+					// All mailboxes in the personal namespace. Which is all mailboxes for us.
+					// ../rfc/5465:817
+					for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
+						xcheckf(err, "list mailboxes")
+						m[mb.ID] = mb
+					}
+
+				case "SUBSCRIBED":
+					// Mailboxes that are subscribed. Will typically be same as personal, since we
+					// subscribe to all mailboxes. But user can manage subscriptions differently.
+					// ../rfc/5465:831
+					for mb, err := range bstore.QueryTx[store.Mailbox](tx).FilterEqual("Expunged", false).All() {
+						xcheckf(err, "list mailboxes")
+						if err := tx.Get(&store.Subscription{Name: mb.Name}); err == nil {
+							m[mb.ID] = mb
+						} else if err != bstore.ErrAbsent {
+							xcheckf(err, "lookup subscription for mailbox")
+						}
+					}
+
+				case "SUBTREE", "SUBTREE-ONE":
+					// The mailbox name itself, and children. ../rfc/5465:847
+					// SUBTREE is arbitrarily deep, SUBTREE-ONE is one level deeper than requested
+					// mailbox. The mailbox itself is included too ../rfc/7377:274
+
+					// We don't have to worry about loops. Mailboxes are not in the file system.
+					// ../rfc/7377:291
+
+					for _, name := range mbs.Args {
+						name = xcheckmailboxname(name, true)
+
+						one := mbs.Kind == "SUBTREE-ONE"
+						var ntoken int
+						if one {
+							ntoken = len(strings.Split(name, "/"))
+						}
+
+						q := bstore.QueryTx[store.Mailbox](tx)
+						q.FilterEqual("Expunged", false)
+						q.FilterGreaterEqual("Name", name)
+						q.SortAsc("Name")
+						for mb, err := range q.All() {
+							xcheckf(err, "list mailboxes")
+							if mb.Name != name && !strings.HasPrefix(mb.Name, name+"/") {
+								break
+							}
+							if !one || mb.Name == name || len(strings.Split(mb.Name, "/")) == ntoken+1 {
+								m[mb.ID] = mb
+							}
+						}
+					}
+
+				case "MAILBOXES":
+					// Just the specified mailboxes. ../rfc/5465:853
+					for _, name := range mbs.Args {
+						name = xcheckmailboxname(name, true)
+
+						// If a mailbox doesn't exist, we don't treat it as an error. Seems reasonable
+						// giving we are searching. Messages may not exist. And likewise for the mailbox.
+						// Just results in no hits.
+						mb, err := c.account.MailboxFind(tx, name)
+						xcheckf(err, "looking up mailbox")
+						if mb != nil {
+							m[mb.ID] = *mb
+						}
+					}
+
+				default:
+					panic("missing case")
+				}
+			}
+			mailboxes = slices.Collect(maps.Values(m))
+			slices.SortFunc(mailboxes, func(a, b store.Mailbox) int {
+				return cmp.Compare(a.Name, b.Name)
+			})
+
+			// If no source mailboxes were specified (no mailboxSpecs), the selected mailbox is
+			// used below. ../rfc/7377:298
+		} else {
+			mb := c.xmailboxID(tx, c.mailboxID) // Validate.
+			mailboxes = []store.Mailbox{mb}
+		}
+
+		if save && !(len(mailboxes) == 1 && mailboxes[0].ID == c.mailboxID) {
+			// ../rfc/7377:319
+			xsyntaxErrorf("can only use SAVE on selected mailbox")
+		}
+
 		runlock()
 		runlock = func() {}
 
-		// Normal forward search when we don't have MAX only. We only send an "inprogress"
-		// goal if we know how many messages we have to check.
-		forward := eargs == nil || max == 0 || len(eargs) != 1
-		reverse := max == 1 && (len(eargs) == 1 || min+max == len(eargs))
+		// Determine if search has a sequence set without search results. If so, we need
+		// sequence numbers for matching, and we must always go through the messages in
+		// forward order. No reverse search for MAX only.
+		needSeq := (len(mailboxes) > 1 || len(mailboxes) == 1 && mailboxes[0].ID != c.mailboxID) && sk.needSeq()
+
+		forward := eargs == nil || max1 == 0 || len(eargs) != 1 || needSeq
+		reverse := max1 == 1 && (len(eargs) == 1 || min1+max1 == len(eargs)) && !needSeq
+
+		// We set a worst-case "goal" of having gone through all messages in all mailboxes.
+		// Sometimes, we can be faster, when we only do a MIN and/or MAX query and we can
+		// stop early. We'll account for that as we go. For the selected mailbox, we'll
+		// only look at those the session has already seen.
 		goal := "nil"
-		if len(c.uids) > 0 && forward != reverse {
-			goal = fmt.Sprintf("%d", len(c.uids))
+		var total uint32
+		for _, mb := range mailboxes {
+			if mb.ID == c.mailboxID {
+				total += uint32(len(c.uids))
+			} else {
+				total += uint32(mb.Total + mb.Deleted)
+			}
+		}
+		if total > 0 {
+			// Goal is always non-zero. ../rfc/9585:232
+			goal = fmt.Sprintf("%d", total)
 		}
 
-		var lastIndex = -1
-		if forward {
-			for i, uid := range c.uids {
-				lastIndex = i
-				if time.Since(inProgressLast) > inProgressPeriod {
-					c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, i, goal)
-					inProgressLast = time.Now()
+		var progress uint32
+		for _, mb := range mailboxes {
+			var lastUID store.UID
+
+			result := Result{Mailbox: mb}
+
+			msgCount := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
+			if mb.ID == c.mailboxID {
+				msgCount = uint32(len(c.uids))
+			}
+
+			// Used for interpreting UID sets with a star, like "1:*" and "10:*". Only called
+			// for UIDs that are higher than the number, since "10:*" evaluates to "10:5" if 5
+			// is the highest UID, and UID 5-10 would all match.
+			var cachedHighestUID store.UID
+			highestUID := func() (store.UID, error) {
+				if cachedHighestUID > 0 {
+					return cachedHighestUID, nil
 				}
-				if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, bodySearch, textSearch, &expungeIssued); match {
-					uids = append(uids, uid)
-					if modseq > maxModSeq {
-						maxModSeq = modseq
+
+				q := bstore.QueryTx[store.Message](tx)
+				q.FilterNonzero(store.Message{MailboxID: mb.ID})
+				q.FilterEqual("Expunged", false)
+				q.SortDesc("UID")
+				q.Limit(1)
+				m, err := q.Get()
+				cachedHighestUID = m.UID
+				return cachedHighestUID, err
+			}
+
+			progressOrig := progress
+
+			if forward {
+				// We track this for non-selected mailboxes. searchMatch will look the message
+				// sequence number for this session up if we are searching the selected mailbox.
+				var seq msgseq = 1
+
+				q := bstore.QueryTx[store.Message](tx)
+				q.FilterNonzero(store.Message{MailboxID: mb.ID})
+				q.FilterEqual("Expunged", false)
+				q.SortAsc("UID")
+				for m, err := range q.All() {
+					xcheckf(err, "list messages in mailbox")
+
+					// We track this for the "reverse" case, we'll stop before seeing lastUID.
+					lastUID = m.UID
+
+					if time.Since(inProgressLast) > inProgressPeriod {
+						c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
+						inProgressLast = time.Now()
 					}
-					if min == 1 && min+max == len(eargs) {
+					progress++
+
+					if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, highestUID) {
+						result.UIDs = append(result.UIDs, m.UID)
+						result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
+						if min1 == 1 && min1+max1 == len(eargs) {
+							if !needSeq {
+								break
+							}
+							// We only need a MIN and a MAX, but we also need sequence numbers so we are
+							// walking through and collecting all UIDs. Correct for that, keeping only the MIN
+							// (first)
+							// and MAX (second).
+							if len(result.UIDs) == 3 {
+								result.UIDs[1] = result.UIDs[2]
+								result.UIDs = result.UIDs[:2]
+							}
+						}
+					}
+					seq++
+				}
+			}
+			// And reverse search for MAX if we have only MAX or MAX combined with MIN, and
+			// don't need sequence numbers. We just need a single match, then we stop.
+			if reverse {
+				q := bstore.QueryTx[store.Message](tx)
+				q.FilterNonzero(store.Message{MailboxID: mb.ID})
+				q.FilterEqual("Expunged", false)
+				q.FilterGreater("UID", lastUID)
+				q.SortDesc("UID")
+				for m, err := range q.All() {
+					xcheckf(err, "list messages in mailbox")
+
+					if time.Since(inProgressLast) > inProgressPeriod {
+						c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, progress, goal)
+						inProgressLast = time.Now()
+					}
+					progress++
+
+					var seq msgseq // Filled in by searchMatch for messages in selected mailbox.
+					if c.searchMatch(tx, msgCount, seq, m, *sk, bodySearch, textSearch, highestUID) {
+						result.UIDs = append(result.UIDs, m.UID)
+						result.MaxModSeq = max(result.MaxModSeq, m.ModSeq)
 						break
 					}
 				}
 			}
-		}
-		// And reverse search for MAX if we have only MAX or MAX combined with MIN.
-		if reverse {
-			for i := len(c.uids) - 1; i > lastIndex; i-- {
-				if time.Since(inProgressLast) > inProgressPeriod {
-					c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, len(c.uids)-1-i, goal)
-					inProgressLast = time.Now()
-				}
-				if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, bodySearch, textSearch, &expungeIssued); match {
-					uids = append(uids, c.uids[i])
-					if modseq > maxModSeq {
-						maxModSeq = modseq
-					}
-					break
-				}
-			}
+
+			// We could have finished searching the mailbox with fewer
+			mailboxProcessed := progress - progressOrig
+			mailboxTotal := uint32(mb.MailboxCounts.Total + mb.MailboxCounts.Deleted)
+			progress += max(0, mailboxTotal-mailboxProcessed)
+
+			results = append(results, result)
 		}
 	})
 
 	if eargs == nil {
+		// We'll only have a result for the one selected mailbox.
+		result := results[0]
+
 		// In IMAP4rev1, an untagged SEARCH response is required. ../rfc/3501:2728
-		if len(uids) == 0 {
+		if len(result.UIDs) == 0 {
 			c.bwritelinef("* SEARCH")
 		}
 
 		// Old-style SEARCH response. We must spell out each number. So we may be splitting
 		// into multiple responses. ../rfc/9051:6809 ../rfc/3501:4833
-		for len(uids) > 0 {
-			n := len(uids)
+		for len(result.UIDs) > 0 {
+			n := len(result.UIDs)
 			if n > 100 {
 				n = 100
 			}
 			s := ""
-			for _, v := range uids[:n] {
+			for _, v := range result.UIDs[:n] {
 				if !isUID {
 					v = store.UID(c.xsequence(v))
 				}
@@ -233,18 +524,18 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 			var modseq string
 			if sk.hasModseq() {
 				// ../rfc/7162:2557
-				modseq = fmt.Sprintf(" (MODSEQ %d)", maxModSeq.Client())
+				modseq = fmt.Sprintf(" (MODSEQ %d)", result.MaxModSeq.Client())
 			}
 
 			c.bwritelinef("* SEARCH%s%s", s, modseq)
-			uids = uids[n:]
+			result.UIDs = result.UIDs[n:]
 		}
 	} else {
 		// New-style ESEARCH response syntax: ../rfc/9051:6546 ../rfc/4466:522
 
 		if save {
 			// ../rfc/9051:3784 ../rfc/5182:13
-			c.searchResult = uids
+			c.searchResult = results[0].UIDs
 			if sanityChecks {
 				checkUIDs(c.searchResult)
 			}
@@ -252,72 +543,88 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 
 		// No untagged ESEARCH response if nothing was requested. ../rfc/9051:4160
 		if len(eargs) > 0 {
-			// The tag was originally a string, became an astring in IMAP4rev2, better stick to
-			// string. ../rfc/4466:707 ../rfc/5259:1163 ../rfc/9051:7087
-			resp := fmt.Sprintf(`* ESEARCH (TAG "%s")`, tag)
-			if isUID {
-				resp += " UID"
-			}
-
-			// NOTE: we are converting UIDs to msgseq in the uids slice (if needed) while
-			// keeping the "uids" name!
-			if !isUID {
-				// If searchResult is hanging on to the slice, we need to work on a copy.
-				if save {
-					nuids := make([]store.UID, len(uids))
-					copy(nuids, uids)
-					uids = nuids
+			for _, result := range results {
+				// For the ESEARCH command, we must not return a response if there were no matching
+				// messages. This is unlike the later IMAP4rev2, where an ESEARCH response must be
+				// sent if there were no matches. ../rfc/7377:243 ../rfc/9051:3775
+				if isE && len(result.UIDs) == 0 {
+					continue
 				}
-				for i, uid := range uids {
-					uids[i] = store.UID(c.xsequence(uid))
+
+				// The tag was originally a string, became an astring in IMAP4rev2, better stick to
+				// string. ../rfc/4466:707 ../rfc/5259:1163 ../rfc/9051:7087
+				if isE {
+					fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s" MAILBOX %s UIDVALIDITY %d)`, tag, result.Mailbox.Name, result.Mailbox.UIDValidity)
+				} else {
+					fmt.Fprintf(c.xbw, `* ESEARCH (TAG "%s")`, tag)
+				}
+				if isUID {
+					fmt.Fprintf(c.xbw, " UID")
 				}
-			}
 
-			// If no matches, then no MIN/MAX response. ../rfc/4731:98 ../rfc/9051:3758
-			if eargs["MIN"] && len(uids) > 0 {
-				resp += fmt.Sprintf(" MIN %d", uids[0])
-			}
-			if eargs["MAX"] && len(uids) > 0 {
-				resp += fmt.Sprintf(" MAX %d", uids[len(uids)-1])
-			}
-			if eargs["COUNT"] {
-				resp += fmt.Sprintf(" COUNT %d", len(uids))
-			}
-			if eargs["ALL"] && len(uids) > 0 {
-				resp += fmt.Sprintf(" ALL %s", compactUIDSet(uids).String())
-			}
+				// NOTE: we are potentially converting UIDs to msgseq, but keep the store.UID type
+				// for convenience.
+				nums := result.UIDs
+				if !isUID {
+					// If searchResult is hanging on to the slice, we need to work on a copy.
+					if save {
+						nums = slices.Clone(nums)
+					}
+					for i, uid := range nums {
+						nums[i] = store.UID(c.xsequence(uid))
+					}
+				}
 
-			// Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273
-			// Summary: send the highest modseq of the returned messages.
-			if sk.hasModseq() && len(uids) > 0 {
-				resp += fmt.Sprintf(" MODSEQ %d", maxModSeq.Client())
-			}
+				// If no matches, then no MIN/MAX response. ../rfc/4731:98 ../rfc/9051:3758
+				if eargs["MIN"] && len(nums) > 0 {
+					fmt.Fprintf(c.xbw, " MIN %d", nums[0])
+				}
+				if eargs["MAX"] && len(result.UIDs) > 0 {
+					fmt.Fprintf(c.xbw, " MAX %d", nums[len(nums)-1])
+				}
+				if eargs["COUNT"] {
+					fmt.Fprintf(c.xbw, " COUNT %d", len(nums))
+				}
+				if eargs["ALL"] && len(nums) > 0 {
+					fmt.Fprintf(c.xbw, " ALL %s", compactUIDSet(nums).String())
+				}
 
-			c.bwritelinef("%s", resp)
+				// Interaction between ESEARCH and CONDSTORE: ../rfc/7162:1211 ../rfc/4731:273
+				// Summary: send the highest modseq of the returned messages.
+				if sk.hasModseq() && len(nums) > 0 {
+					fmt.Fprintf(c.xbw, " MODSEQ %d", result.MaxModSeq.Client())
+				}
+
+				c.bwritelinef("")
+			}
 		}
 	}
-	if expungeIssued {
-		// ../rfc/9051:5102
-		c.writeresultf("%s OK [EXPUNGEISSUED] done", tag)
-	} else {
-		c.ok(tag, cmd)
-	}
+
+	c.ok(tag, cmd)
 }
 
 type search struct {
-	c             *conn
-	tx            *bstore.Tx
-	seq           msgseq
-	uid           store.UID
-	mr            *store.MsgReader
-	m             store.Message
-	p             *message.Part
-	expungeIssued *bool
-	hasModseq     bool
+	c          *conn
+	tx         *bstore.Tx
+	msgCount   uint32 // Number of messages in mailbox (or session when selected).
+	seq        msgseq // Can be 0, for other mailboxes than selected in case of MAX.
+	m          store.Message
+	mr         *store.MsgReader
+	p          *message.Part
+	highestUID func() (store.UID, error)
 }
 
-func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKey, bodySearch, textSearch *store.WordSearch, expungeIssued *bool) (bool, store.ModSeq) {
-	s := search{c: c, tx: tx, seq: seq, uid: uid, expungeIssued: expungeIssued, hasModseq: sk.hasModseq()}
+func (c *conn) searchMatch(tx *bstore.Tx, msgCount uint32, seq msgseq, m store.Message, sk searchKey, bodySearch, textSearch *store.WordSearch, highestUID func() (store.UID, error)) bool {
+	if m.MailboxID == c.mailboxID {
+		seq = c.sequence(m.UID)
+		if seq == 0 {
+			// Session has not yet seen this message, and is not expecting to get a result that
+			// includes it.
+			return false
+		}
+	}
+
+	s := search{c: c, tx: tx, msgCount: msgCount, seq: seq, m: m, highestUID: highestUID}
 	defer func() {
 		if s.mr != nil {
 			err := s.mr.Close()
@@ -328,18 +635,7 @@ func (c *conn) searchMatch(tx *bstore.Tx, seq msgseq, uid store.UID, sk searchKe
 	return s.match(sk, bodySearch, textSearch)
 }
 
-func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool, modseq store.ModSeq) {
-	// Instead of littering all the cases in match0 with calls to get modseq, we do it once
-	// here in case of a match.
-	defer func() {
-		if match && s.hasModseq {
-			if s.m.ID == 0 {
-				match = s.xensureMessage()
-			}
-			modseq = s.m.ModSeq
-		}
-	}()
-
+func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (match bool) {
 	match = s.match0(sk)
 	if match && bodySearch != nil {
 		if !s.xensurePart() {
@@ -362,24 +658,6 @@ func (s *search) match(sk searchKey, bodySearch, textSearch *store.WordSearch) (
 	return
 }
 
-func (s *search) xensureMessage() bool {
-	if s.m.ID > 0 {
-		return true
-	}
-
-	q := bstore.QueryTx[store.Message](s.tx)
-	q.FilterNonzero(store.Message{MailboxID: s.c.mailboxID, UID: s.uid})
-	m, err := q.Get()
-	if err == bstore.ErrAbsent || err == nil && m.Expunged {
-		// ../rfc/2180:607
-		*s.expungeIssued = true
-		return false
-	}
-	xcheckf(err, "get message")
-	s.m = m
-	return true
-}
-
 // ensure message, reader and part are loaded. returns whether that was
 // successful.
 func (s *search) xensurePart() bool {
@@ -387,10 +665,6 @@ func (s *search) xensurePart() bool {
 		return s.p != nil
 	}
 
-	if !s.xensureMessage() {
-		return false
-	}
-
 	// Closed by searchMatch after all (recursive) search.match calls are finished.
 	s.mr = s.c.account.MessageReader(s.m)
 
@@ -417,14 +691,23 @@ func (s *search) match0(sk searchKey) bool {
 		}
 		return true
 	} else if sk.seqSet != nil {
-		return sk.seqSet.containsSeq(s.seq, c.uids, c.searchResult)
+		if sk.seqSet.searchResult {
+			// Interpreting search results on a mailbox that isn't selected during multisearch
+			// is likely a mistake. No mention about it in the RFC. ../rfc/7377:257
+			if s.m.MailboxID != c.mailboxID {
+				xuserErrorf("can only use search result with the selected mailbox")
+			}
+			return uidSearch(c.searchResult, s.m.UID) > 0
+		}
+		// For multisearch, we have arranged to have a seq for non-selected mailboxes too.
+		return sk.seqSet.containsSeqCount(s.seq, s.msgCount)
 	}
 
 	filterHeader := func(field, value string) bool {
 		lower := strings.ToLower(value)
 		h, err := s.p.Header()
 		if err != nil {
-			c.log.Debugx("parsing message header", err, slog.Any("uid", s.uid))
+			c.log.Debugx("parsing message header", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
 			return false
 		}
 		for _, v := range h.Values(field) {
@@ -454,7 +737,14 @@ func (s *search) match0(sk searchKey) bool {
 	case "OR":
 		return s.match0(*sk.searchKey) || s.match0(*sk.searchKey2)
 	case "UID":
-		return sk.uidSet.containsUID(s.uid, c.uids, c.searchResult)
+		if sk.uidSet.searchResult && s.m.MailboxID != c.mailboxID {
+			// Interpreting search results on a mailbox that isn't selected during multisearch
+			// is likely a mistake. No mention about it in the RFC. ../rfc/7377:257
+			xuserErrorf("cannot use search result from another mailbox")
+		}
+		match, err := sk.uidSet.containsKnownUID(s.m.UID, c.searchResult, s.highestUID)
+		xcheckf(err, "checking for presence in uid set")
+		return match
 	}
 
 	// Parsed part.
@@ -570,7 +860,7 @@ func (s *search) match0(sk searchKey) bool {
 	}
 
 	if s.p == nil {
-		c.log.Info("missing parsed message, not matching", slog.Any("uid", s.uid))
+		c.log.Info("missing parsed message, not matching", slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
 		return false
 	}
 
@@ -599,7 +889,7 @@ func (s *search) match0(sk searchKey) bool {
 		lower := strings.ToLower(sk.astring)
 		h, err := s.p.Header()
 		if err != nil {
-			c.log.Errorx("parsing header for search", err, slog.Any("uid", s.uid))
+			c.log.Errorx("parsing header for search", err, slog.Any("uid", s.m.UID), slog.Int64("msgid", s.m.ID))
 			return false
 		}
 		k := textproto.CanonicalMIMEHeaderKey(sk.headerField)

imapserver/search_test.go

@@ -34,6 +34,10 @@ this is html.
 --x--
 `, "\n", "\r\n")
 
+func uint32ptr(v uint32) *uint32 {
+	return &v
+}
+
 func (tc *testconn) xsearch(nums ...uint32) {
 	tc.t.Helper()
 
@@ -53,7 +57,7 @@ func (tc *testconn) xsearchmodseq(modseq int64, nums ...uint32) {
 func (tc *testconn) xesearch(exp imapclient.UntaggedEsearch) {
 	tc.t.Helper()
 
-	exp.Correlator = tc.client.LastTag
+	exp.Tag = tc.client.LastTag
 	tc.xuntagged(exp)
 }
 
@@ -94,6 +98,11 @@ func TestSearch(t *testing.T) {
 
 	// We now have sequence numbers 1,2,3 and UIDs 5,6,7.
 
+	// We need to be selected. Not the case for ESEARCH command.
+	tc.client.Unselect()
+	tc.transactf("no", "search all")
+	tc.client.Select("inbox")
+
 	tc.transactf("ok", "search all")
 	tc.xsearch(1, 2, 3)
 
@@ -289,10 +298,6 @@ func TestSearch(t *testing.T) {
 		return imapclient.UntaggedEsearch{All: esearchall0(ss)}
 	}
 
-	uint32ptr := func(v uint32) *uint32 {
-		return &v
-	}
-
 	// Do new-style ESEARCH requests with RETURN. We should get an ESEARCH response.
 	tc.transactf("ok", "search return () all")
 	tc.xesearch(esearchall("1:3")) // Without any options, "ALL" is implicit.
@@ -464,3 +469,361 @@ func esearchall0(ss string) imapclient.NumSet {
 	}
 	return seqset
 }
+
+// Test the MULTISEARCH extension. Where we don't need to have a mailbox selected,
+// operating without messag sequence numbers, and return untagged esearch responses
+// that include the mailbox and uidvalidity.
+func TestSearchMulti(t *testing.T) {
+	testSearchMulti(t, false)
+	testSearchMulti(t, true)
+}
+
+// Run multisearch tests with or without a mailbox selected.
+func testSearchMulti(t *testing.T, selected bool) {
+	defer mockUIDValidity()()
+
+	tc := start(t)
+	defer tc.close()
+	tc.client.Login("mjl@mox.example", password0)
+	tc.client.Select("inbox")
+
+	// Add 5 messages to Inbox and delete first 4 messages. So UIDs start at 5.
+	received := time.Date(2020, time.January, 1, 10, 0, 0, 0, time.UTC)
+	for range 6 {
+		tc.client.Append("inbox", makeAppendTime(exampleMsg, received))
+	}
+	tc.client.StoreFlagsSet("1:4", true, `\Deleted`)
+	tc.client.Expunge()
+
+	// Unselecting mailbox, esearch works in authenticated state.
+	if !selected {
+		tc.client.Unselect()
+	}
+
+	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
+	tc.client.Append("inbox", makeAppendTime(searchMsg, received))
+
+	received = time.Date(2022, time.January, 1, 9, 0, 0, 0, time.UTC)
+	mostFlags := []string{
+		`\Deleted`,
+		`\Seen`,
+		`\Answered`,
+		`\Flagged`,
+		`\Draft`,
+		`$Forwarded`,
+		`$Junk`,
+		`$Notjunk`,
+		`$Phishing`,
+		`$MDNSent`,
+		`custom1`,
+		`Custom2`,
+	}
+	tc.client.Append("Archive", imapclient.Append{Flags: mostFlags, Received: &received, Size: int64(len(searchMsg)), Data: strings.NewReader(searchMsg)})
+
+	// We now have sequence numbers 1,2,3 and UIDs 5,6,7 in Inbox, and UID 1 in Archive.
+
+	// Basic esearch with mailboxes.
+	tc.cmdf("Tag1", `Esearch In (Personal) Return () All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, All: esearchall0("1")},
+	)
+
+	// Again, but with progress information.
+	orig := inProgressPeriod
+	inProgressPeriod = 0
+	inprogress := func(cur, goal uint32) imapclient.UntaggedResult {
+		return imapclient.UntaggedResult{
+			Status: "OK",
+			RespText: imapclient.RespText{
+				Code:    "INPROGRESS",
+				CodeArg: imapclient.CodeInProgress{Tag: "Tag1", Current: &cur, Goal: &goal},
+				More:    "still searching",
+			},
+		}
+	}
+	tc.cmdf("Tag1", `Esearch In (Personal) Return () All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, All: esearchall0("1")},
+		inprogress(0, 4),
+		inprogress(1, 4),
+		inprogress(2, 4),
+		inprogress(3, 4),
+	)
+	inProgressPeriod = orig
+
+	// Explicit mailboxes listed, including non-existent one that is ignored,
+	// duplicates are ignored as well.
+	tc.cmdf("Tag1", `Esearch In (Mailboxes (INBOX Archive Archive)) Return (Min Max Count All) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7, Count: uint32ptr(3), All: esearchall0("5:7")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1, Max: 1, Count: uint32ptr(1), All: esearchall0("1")},
+	)
+
+	// No response if none of the mailboxes exist.
+	tc.cmdf("Tag1", `Esearch In (Mailboxes bogus Mailboxes (nonexistent)) Return (Min Max Count All) All`)
+	tc.response("ok")
+	tc.xuntagged()
+
+	// Inboxes evaluates to just inbox on new account. We'll add more mailboxes
+	// matching "inboxes" later on.
+	tc.cmdf("Tag1", `Esearch In (Inboxes) Return () All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
+	)
+
+	// Subscribed is set for created mailboxes by default.
+	tc.cmdf("Tag1", `Esearch In (Subscribed) Return (Max) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 7},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Max: 1},
+	)
+
+	// Asking for max does a reverse search.
+	tc.cmdf("Tag1", `Esearch In (Personal) Return (Max) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 7},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Max: 1},
+	)
+
+	// Min stops early.
+	tc.cmdf("Tag1", `Esearch In (Personal) Return (Min) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1},
+	)
+
+	// Min and max do forward and reverse search, stopping early.
+	tc.cmdf("Tag1", `Esearch In (Personal) Return (Min Max) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Min: 1, Max: 1},
+	)
+
+	if selected {
+		// With only 1 inbox, we can use SAVE with Inboxes. Can't anymore when we have multiple.
+		tc.transactf("ok", `Esearch In (Inboxes) Return (Save) All`)
+		tc.xuntagged()
+
+		// Using search result ($) works with selected mailbox.
+		tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
+		tc.response("ok")
+		tc.xuntagged(
+			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
+		)
+	} else {
+		// Cannot use "selected" if we are not in selected state.
+		tc.transactf("bad", `Esearch In (Selected) Return () All`)
+	}
+
+	// Add more "inboxes", and other mailboxes for testing "subtree" and "subtree-one".
+	more := []string{
+		"Inbox/Sub1",
+		"Inbox/Sub2",
+		"Inbox/Sub2/SubA",
+		"Inbox/Sub2/SubB",
+		"Other",
+		"Other/Sub1", // sub1@mox.example in config.
+		"Other/Sub2",
+		"Other/Sub2/SubA", // ruleset for sub2@mox.example in config.
+		"Other/Sub2/SubB",
+		"List", // ruleset for a mailing list
+	}
+	for _, name := range more {
+		tc.client.Create(name, nil)
+		tc.client.Append(name, makeAppendTime(exampleMsg, received))
+	}
+
+	// Cannot use SAVE with multiple mailboxes that match.
+	tc.transactf("bad", `Esearch In (Inboxes) Return (Save) All`)
+
+	// "inboxes" includes everything below Inbox, and also anything that we might
+	// deliver to based on account addresses and rulesets, but not mailing lists.
+	tc.cmdf("Tag1", `Esearch In (Inboxes) Return () All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:7")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub1", UIDValidity: 3, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2", UIDValidity: 4, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2/SubA", UIDValidity: 5, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox/Sub2/SubB", UIDValidity: 6, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubA", UIDValidity: 10, UID: true, All: esearchall0("1")},
+	)
+
+	// subtree
+	tc.cmdf("Tag1", `Esearch In (Subtree Other) Return () All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other", UIDValidity: 7, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2", UIDValidity: 9, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubA", UIDValidity: 10, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2/SubB", UIDValidity: 11, UID: true, All: esearchall0("1")},
+	)
+
+	// subtree-one
+	tc.cmdf("Tag1", `Esearch In (Subtree-One Other) Return () All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other", UIDValidity: 7, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub1", UIDValidity: 8, UID: true, All: esearchall0("1")},
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Other/Sub2", UIDValidity: 9, UID: true, All: esearchall0("1")},
+	)
+
+	// Search with sequence set also for non-selected mailboxes(!). The min/max would
+	// get the first and last message, but the message sequence set forces a scan.
+	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) 1:*`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7},
+	)
+
+	// Search with uid set with "$highnum:*" forces getting highest uid.
+	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid *:100`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 7, Max: 7},
+	)
+	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid 100:*`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 7, Max: 7},
+	)
+	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return (Min Max) Uid 1:*`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 7},
+	)
+
+	// We use another session to add a new message to Inbox and to Archive. Searching
+	// with Inbox selected will not return the new message since it isn't available in
+	// the session yet. The message in Archive is returned, since there is no session
+	// limitation.
+	tc2 := startNoSwitchboard(t)
+	defer tc2.closeNoWait()
+	tc2.client.Login("mjl@mox.example", password0)
+	tc2.client.Append("inbox", makeAppendTime(searchMsg, received))
+	tc2.client.Append("Archive", makeAppendTime(searchMsg, received))
+
+	tc.cmdf("Tag1", `Esearch In (Mailboxes (Inbox Archive)) Return (Count) All`)
+	tc.response("ok")
+	if selected {
+		tc.xuntagged(
+			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Count: uint32ptr(3)},
+			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Count: uint32ptr(2)},
+			imapclient.UntaggedExists(4),
+			imapclient.UntaggedFetch{Seq: 4, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(8), imapclient.FetchFlags(nil)}},
+		)
+	} else {
+		tc.xuntagged(
+			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Count: uint32ptr(4)},
+			imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Archive", UIDValidity: 1, UID: true, Count: uint32ptr(2)},
+		)
+	}
+
+	if selected {
+		// Saving a search result, and then using it with another mailbox results in error.
+		tc.transactf("ok", `Esearch In (Mailboxes Inbox) Return (Save) All`)
+		tc.transactf("no", `Esearch In (Mailboxes Archive) Return () $`)
+	} else {
+		tc.transactf("bad", `Esearch In (Inboxes) Return (Save) All`) // Need a selected mailbox with SAVE.
+		tc.transactf("no", `Esearch In (Inboxes) Return () $`)        // Cannot use saved result with non-selected mailbox.
+	}
+
+	tc.transactf("bad", `Esearch In () Return () All`)                    // Missing values for "IN"-list.
+	tc.transactf("bad", `Esearch In (Bogus) Return () All`)               // Bogus word for "IN".
+	tc.transactf("bad", `Esearch In ("Selected") Return () All`)          // IN-words can't be quoted.
+	tc.transactf("bad", `Esearch In (Selected-Delayed) Return () All`)    // From NOTIFY, not in ESEARCH.
+	tc.transactf("bad", `Esearch In (Subtree-One) Return () All`)         // After subtree-one we need a list.
+	tc.transactf("bad", `Esearch In (Subtree-One ) Return () All`)        // After subtree-one we need a list.
+	tc.transactf("bad", `Esearch In (Subtree-One (Test) ) Return () All`) // Bogus space.
+
+	if !selected {
+		return
+	}
+	// From now on, we are in selected state.
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return () All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
+	)
+
+	// Testing combinations of SAVE with MIN/MAX/others ../rfc/9051:4100
+	tc.transactf("ok", `Esearch In (Selected) Return (Save) All`)
+	tc.xuntagged()
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
+	)
+
+	// Inbox happens to be the selected mailbox, so OK.
+	tc.cmdf("Tag1", `Esearch In (Mailboxes Inbox) Return () $`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
+	)
+
+	// Non-selected mailboxes aren't allowed to use the saved result.
+	tc.transactf("no", `Esearch In (Mailboxes Archive) Return () $`)
+	tc.transactf("no", `Esearch In (Mailboxes Archive) Return () uid $`)
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min Max) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 8},
+	)
+	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5,8")},
+	)
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5},
+	)
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5")},
+	)
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Max) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Max: 8},
+	)
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("8")},
+	)
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return (Save Min Max Count) All`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, Min: 5, Max: 8, Count: uint32ptr(4)},
+	)
+
+	tc.cmdf("Tag1", `Esearch In (Selected) Return () $`)
+	tc.response("ok")
+	tc.xuntagged(
+		imapclient.UntaggedEsearch{Tag: "Tag1", Mailbox: "Inbox", UIDValidity: 1, UID: true, All: esearchall0("5:8")},
+	)
+}

imapserver/server.go

@@ -182,6 +182,7 @@ var serverCapabilities = strings.Join([]string{
 	"REPLACE",                         // ../rfc/8508
 	"PREVIEW",                         // ../rfc/8970:114
 	"INPROGRESS",                      // ../rfc/9585:101
+	"MULTISEARCH",                     // ../rfc/7377:187
 	// "COMPRESS=DEFLATE", // ../rfc/4978, disabled for interoperability issues: The flate reader (inflate) still blocks on partial flushes, preventing progress.
 }, " ")
 
@@ -281,8 +282,8 @@ func stateCommands(cmds ...string) map[string]struct{} {
 var (
 	commandsStateAny              = stateCommands("capability", "noop", "logout", "id")
 	commandsStateNotAuthenticated = stateCommands("starttls", "authenticate", "login")
-	commandsStateAuthenticated    = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress")
-	commandsStateSelected         = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move", "replace", "uid replace")
+	commandsStateAuthenticated    = stateCommands("enable", "select", "examine", "create", "delete", "rename", "subscribe", "unsubscribe", "list", "namespace", "status", "append", "idle", "lsub", "getquotaroot", "getquota", "getmetadata", "setmetadata", "compress", "esearch")
+	commandsStateSelected         = stateCommands("close", "unselect", "expunge", "search", "fetch", "store", "copy", "move", "uid expunge", "uid search", "uid fetch", "uid store", "uid copy", "uid move", "replace", "uid replace", "esearch")
 )
 
 var commands = map[string]func(c *conn, tag, cmd string, p *parser){
@@ -317,6 +318,7 @@ var commands = map[string]func(c *conn, tag, cmd string, p *parser){
 	"getmetadata":  (*conn).cmdGetmetadata,
 	"setmetadata":  (*conn).cmdSetmetadata,
 	"compress":     (*conn).cmdCompress,
+	"esearch":      (*conn).cmdEsearch,
 
 	// Selected.
 	"check":       (*conn).cmdCheck,
@@ -3847,12 +3849,12 @@ func (c *conn) cmdxExpunge(tag, cmd string, uidSet *numSet) {
 
 // State: Selected
 func (c *conn) cmdSearch(tag, cmd string, p *parser) {
-	c.cmdxSearch(false, tag, cmd, p)
+	c.cmdxSearch(false, false, tag, cmd, p)
 }
 
 // State: Selected
 func (c *conn) cmdUIDSearch(tag, cmd string, p *parser) {
-	c.cmdxSearch(true, tag, cmd, p)
+	c.cmdxSearch(true, false, tag, cmd, p)
 }
 
 // State: Selected

rfc/index.txt

@@ -218,13 +218,13 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
 5957	Roadmap	-	Display-Based Address Sorting for the IMAP4 SORT Extension
 6154	Yes	-	IMAP LIST Extension for Special-Use Mailboxes
 6203	No	-	IMAP4 Extension for Fuzzy Search
-6237	-Roadmap	Obs	(RFC 7377) IMAP4 Multimailbox SEARCH Extension
+6237	-Yes	Obs	(RFC 7377) IMAP4 Multimailbox SEARCH Extension
 6851	Yes	-	Internet Message Access Protocol (IMAP) - MOVE Extension
 6855	Yes	-	IMAP Support for UTF-8
 6858	No	-	Simplified POP and IMAP Downgrading for Internationalized Email
 7162	Yes	-	IMAP Extensions: Quick Flag Changes Resynchronization (CONDSTORE) and Quick Mailbox Resynchronization (QRESYNC)
 7162-eid5055	-	-	errata: space after untagged OK
-7377	Roadmap	-	IMAP4 Multimailbox SEARCH Extension
+7377	Yes	-	IMAP4 Multimailbox SEARCH Extension
 7888	Yes	-	IMAP4 Non-synchronizing Literals
 7889	Yes	-	The IMAP APPENDLIMIT Extension
 8437	No	-	IMAP UNAUTHENTICATE Extension for Connection Reuse

testdata/imap/domains.conf

@@ -6,6 +6,17 @@ Accounts:
 		Domain: mox.example
 		Destinations:
 			mjl@mox.example: nil
+			sub1@mox.example:
+				Mailbox: Other/Sub1
+			sub2@mox.example:
+				Rulesets:
+					-
+						VerifiedDomain: test.example
+						Mailbox: Other/Sub2/SubA
+					-
+						VerifiedDomain: list.example
+						ListAllowDomain: list.example
+						Mailbox: List
 			""@mox.example: nil
 			móx@mox.example: nil
 		JunkFilter: