imapserver: implement "inprogress" response code (RFC 9585) for keepalive during long search

For long searches in big mailboxes, without any matches, we would previously
keep working and not say anything. Clients could interpret this silence as a
broken connection at some point. We now send a "we're still searching" untagged
OK responses with code INPROGRESS every 10 seconds while we're still searching,
to prevent the client from closing the connection. We also send how many
messages we've processed, and usually also how many we need to process in grand
total. Clients can use this to show a progress bar.

imapclient/parse.go

@@ -131,6 +131,7 @@ var knownCodes = stringMap(
 	// With parameters.
 	"BADCHARSET", "CAPABILITY", "PERMANENTFLAGS", "UIDNEXT", "UIDVALIDITY", "UNSEEN", "APPENDUID", "COPYUID",
 	"HIGHESTMODSEQ", "MODIFIED",
+	"INPROGRESS", // ../rfc/9585:104
 )
 
 func stringMap(l ...string) map[string]struct{} {
@@ -222,6 +223,30 @@ func (c *Conn) xrespCode() (string, CodeArg) {
 		c.xspace()
 		modified := c.xuidset()
 		codeArg = CodeModified(NumSet{Ranges: modified})
+	case "INPROGRESS":
+		// ../rfc/9585:238
+		var tag string
+		var current, goal *uint32
+		if c.space() {
+			c.xtake("(")
+			tag = c.xquoted()
+			c.xspace()
+			if c.peek('n') || c.peek('N') {
+				c.xtake("nil")
+			} else {
+				v := c.xuint32()
+				current = &v
+			}
+			c.xspace()
+			if c.peek('n') || c.peek('N') {
+				c.xtake("nil")
+			} else {
+				v := c.xnzuint32()
+				goal = &v
+			}
+			c.xtake(")")
+		}
+		codeArg = CodeInProgress{tag, current, goal}
 	}
 	return W, codeArg
 }

imapclient/protocol.go

@@ -159,6 +159,32 @@ func (c CodeHighestModSeq) CodeString() string {
 	return fmt.Sprintf("HIGHESTMODSEQ %d", c)
 }
 
+// "INPROGRESS" response code.
+type CodeInProgress struct {
+	Tag     string // Nil is empty string.
+	Current *uint32
+	Goal    *uint32
+}
+
+func (c CodeInProgress) CodeString() string {
+	// ABNF allows inprogress-tag/state with all nil values. Doesn't seem useful enough
+	// to keep track of.
+	if c.Tag == "" && c.Current == nil && c.Goal == nil {
+		return "INPROGRESS"
+	}
+
+	// todo: quote tag properly
+	current := "nil"
+	goal := "nil"
+	if c.Current != nil {
+		current = fmt.Sprintf("%d", *c.Current)
+	}
+	if c.Goal != nil {
+		goal = fmt.Sprintf("%d", *c.Goal)
+	}
+	return fmt.Sprintf("INPROGRESS (%q %s %s)", c.Tag, current, goal)
+}
+
 // RespText represents a response line minus the leading tag.
 type RespText struct {
 	Code    string  // The first word between [] after the status.

imapserver/search.go

@@ -14,6 +14,10 @@ import (
 	"slices"
 )
 
+// If last search output was this long ago, we write an untagged inprogress
+// response. Changed during tests. ../rfc/9585:109
+var inProgressPeriod = time.Duration(10 * time.Second)
+
 // Search returns messages matching criteria specified in parameters.
 //
 // State: Selected
@@ -138,17 +142,38 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 	var expungeIssued bool
 	var maxModSeq store.ModSeq
 
+	// We periodically send an untagged OK with INPROGRESS code while searching, to let
+	// clients doing slow searches know we're still working.
+	inProgressLast := time.Now()
+	// Only respond with tag if it can't be confused as end of response code. ../rfc/9585:122
+	inProgressTag := "nil"
+	if !strings.Contains(tag, "]") {
+		inProgressTag = dquote(tag).pack(c)
+	}
+
 	var uids []store.UID
 	c.xdbread(func(tx *bstore.Tx) {
 		c.xmailboxID(tx, c.mailboxID) // Validate.
 		runlock()
 		runlock = func() {}
 
-		// Normal forward search when we don't have MAX only.
+		// Normal forward search when we don't have MAX only. We only send an "inprogress"
+		// goal if we know how many messages we have to check.
+		forward := eargs == nil || max == 0 || len(eargs) != 1
+		reverse := max == 1 && (len(eargs) == 1 || min+max == len(eargs))
+		goal := "nil"
+		if len(c.uids) > 0 && forward != reverse {
+			goal = fmt.Sprintf("%d", len(c.uids))
+		}
+
 		var lastIndex = -1
-		if eargs == nil || max == 0 || len(eargs) != 1 {
+		if forward {
 			for i, uid := range c.uids {
 				lastIndex = i
+				if time.Since(inProgressLast) > inProgressPeriod {
+					c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, i, goal)
+					inProgressLast = time.Now()
+				}
 				if match, modseq := c.searchMatch(tx, msgseq(i+1), uid, *sk, bodySearch, textSearch, &expungeIssued); match {
 					uids = append(uids, uid)
 					if modseq > maxModSeq {
@@ -161,8 +186,12 @@ func (c *conn) cmdxSearch(isUID bool, tag, cmd string, p *parser) {
 			}
 		}
 		// And reverse search for MAX if we have only MAX or MAX combined with MIN.
-		if max == 1 && (len(eargs) == 1 || min+max == len(eargs)) {
+		if reverse {
 			for i := len(c.uids) - 1; i > lastIndex; i-- {
+				if time.Since(inProgressLast) > inProgressPeriod {
+					c.writelinef("* OK [INPROGRESS (%s %d %s)] still searching", inProgressTag, len(c.uids)-1-i, goal)
+					inProgressLast = time.Now()
+				}
 				if match, modseq := c.searchMatch(tx, msgseq(i+1), c.uids[i], *sk, bodySearch, textSearch, &expungeIssued); match {
 					uids = append(uids, c.uids[i])
 					if modseq > maxModSeq {

imapserver/search_test.go

@@ -260,6 +260,31 @@ func TestSearch(t *testing.T) {
 	tc.transactf("ok", `search charset utf-8 text "mox"`)
 	tc.xsearch(2, 3)
 
+	// Check for properly formed INPROGRESS response code.
+	orig := inProgressPeriod
+	inProgressPeriod = 0
+	tc.cmdf("tag1", "search undraft")
+	tc.response("ok")
+
+	inprogress := func(cur, goal uint32) imapclient.UntaggedResult {
+		return imapclient.UntaggedResult{
+			Status: "OK",
+			RespText: imapclient.RespText{
+				Code:    "INPROGRESS",
+				CodeArg: imapclient.CodeInProgress{Tag: "tag1", Current: &cur, Goal: &goal},
+				More:    "still searching",
+			},
+		}
+	}
+	tc.xuntagged(
+		imapclient.UntaggedSearch([]uint32{1, 2}),
+		// Due to inProgressPeriod 0, we get an inprogress response for each message in the mailbox.
+		inprogress(0, 3),
+		inprogress(1, 3),
+		inprogress(2, 3),
+	)
+	inProgressPeriod = orig
+
 	esearchall := func(ss string) imapclient.UntaggedEsearch {
 		return imapclient.UntaggedEsearch{All: esearchall0(ss)}
 	}

imapserver/server.go

@@ -181,6 +181,7 @@ var serverCapabilities = strings.Join([]string{
 	"MULTIAPPEND",                     // ../rfc/3502
 	"REPLACE",                         // ../rfc/8508
 	"PREVIEW",                         // ../rfc/8970:114
+	"INPROGRESS",                      // ../rfc/9585:101
 	// "COMPRESS=DEFLATE", // ../rfc/4978, disabled for interoperability issues: The flate reader (inflate) still blocks on partial flushes, preventing progress.
 }, " ")
 

rfc/index.txt

@@ -237,10 +237,12 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml
 8970	Yes	-	IMAP4 Extension: Message Preview Generation
 9208	Partial	-	IMAP QUOTA Extension
 9394	Roadmap	-	IMAP PARTIAL Extension for Paged SEARCH and FETCH
-9585	?	-	IMAP Response Code for Command Progress Notifications
+9585	Yes	-	IMAP Response Code for Command Progress Notifications
 9586	Roadmap	-	IMAP Extension for Using and Returning Unique Identifiers (UIDs) Only
 9590	Yes	-	IMAP Extension for Returning Mailbox METADATA in Extended LIST
 9698	?	-	The JMAPACCESS Extension for IMAP
+9738	No	-	IMAP MESSAGELIMIT Extension
+9755	Roadmap	-	IMAP Support for UTF-8
 
 5198	-?	-	Unicode Format for Network Interchange