diff --git a/go.mod b/go.mod index 422f078..99ac6db 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/prometheus/client_golang v1.14.0 go.etcd.io/bbolt v1.3.7 golang.org/x/crypto v0.8.0 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/net v0.9.0 golang.org/x/text v0.9.0 ) diff --git a/go.sum b/go.sum index 6e553a0..8c9993a 100644 --- a/go.sum +++ b/go.sum @@ -236,6 +236,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/http/account_test.go b/http/account_test.go index b13c2a1..43efba2 100644 --- a/http/account_test.go +++ b/http/account_test.go @@ -14,14 +14,19 @@ import ( "os" "path" "path/filepath" + "sort" "strings" "testing" + "github.com/mjl-/bstore" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/store" ) +var ctxbg = context.Background() + func tcheck(t *testing.T, err error, msg string) { t.Helper() if err != nil { @@ -50,7 +55,7 @@ func TestAccount(t *testing.T) { if authHdr != "" { r.Header.Add("Authorization", authHdr) } - ok := checkAccountAuth(context.Background(), log, w, r) + ok := checkAccountAuth(ctxbg, log, w, r) if ok != expect { t.Fatalf("got %v, expected %v", ok, expect) } @@ -59,7 +64,7 @@ func TestAccount(t *testing.T) { const authOK = "Basic bWpsQG1veC5leGFtcGxlOnRlc3QxMjM0" // mjl@mox.example:test1234 const authBad = "Basic bWpsQG1veC5leGFtcGxlOmJhZHBhc3N3b3Jk" // mjl@mox.example:badpassword - authCtx := context.WithValue(context.Background(), authCtxKey, "mjl") + authCtx := context.WithValue(ctxbg, authCtxKey, "mjl") test(authOK, "") // No password set yet. Account{}.SetPassword(authCtx, "test1234") @@ -132,6 +137,39 @@ func TestAccount(t *testing.T) { testImport("../testdata/importtest.mbox.zip", 2) testImport("../testdata/importtest.maildir.tgz", 2) + // Check there are messages, with the right flags. + acc.DB.Read(ctxbg, func(tx *bstore.Tx) error { + _, err = bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "other").FilterIn("Keywords", "test").Get() + tcheck(t, err, `fetching message with keywords "other" and "test"`) + + mb, err := acc.MailboxFind(tx, "importtest") + tcheck(t, err, "looking up mailbox importtest") + if mb == nil { + t.Fatalf("missing mailbox importtest") + } + sort.Strings(mb.Keywords) + if strings.Join(mb.Keywords, " ") != "other test" { + t.Fatalf(`expected mailbox keywords "other" and "test", got %v`, mb.Keywords) + } + + n, err := bstore.QueryTx[store.Message](tx).FilterIn("Keywords", "custom").Count() + tcheck(t, err, `fetching message with keyword "custom"`) + if n != 2 { + t.Fatalf(`got %d messages with keyword "custom", expected 2`, n) + } + + mb, err = acc.MailboxFind(tx, "maildir") + tcheck(t, err, "looking up mailbox maildir") + if mb == nil { + t.Fatalf("missing mailbox maildir") + } + if strings.Join(mb.Keywords, " ") != "custom" { + t.Fatalf(`expected mailbox keywords "custom", got %v`, mb.Keywords) + } + + return nil + }) + testExport := func(httppath string, iszip bool, expectFiles int) { t.Helper() diff --git a/http/admin_test.go b/http/admin_test.go index b06862f..c32a7a4 100644 --- a/http/admin_test.go +++ b/http/admin_test.go @@ -1,7 +1,6 @@ package http import ( - "context" "crypto/ed25519" "net" "net/http/httptest" @@ -29,7 +28,7 @@ func TestAdminAuth(t *testing.T) { if authHdr != "" { r.Header.Add("Authorization", authHdr) } - ok := checkAdminAuth(context.Background(), passwordfile, w, r) + ok := checkAdminAuth(ctxbg, passwordfile, w, r) if ok != expect { t.Fatalf("got %v, expected %v", ok, expect) } @@ -125,9 +124,9 @@ func TestCheckDomain(t *testing.T) { close(done) dialer := &net.Dialer{Deadline: time.Now().Add(-time.Second), Cancel: done} - checkDomain(context.Background(), resolver, dialer, "mox.example") + checkDomain(ctxbg, resolver, dialer, "mox.example") // todo: check returned data - Admin{}.Domains(context.Background()) // todo: check results - dnsblsStatus(context.Background(), resolver) // todo: check results + Admin{}.Domains(ctxbg) // todo: check results + dnsblsStatus(ctxbg, resolver) // todo: check results } diff --git a/http/import.go b/http/import.go index ec0c8d0..55a0369 100644 --- a/http/import.go +++ b/http/import.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "golang.org/x/exp/maps" "golang.org/x/text/unicode/norm" "github.com/mjl-/bstore" @@ -361,10 +362,16 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store mailboxes := map[string]store.Mailbox{} messages := map[string]int{} - // For maildirs, we are likely to get a possible dovecot-keywords file after having imported the messages. Once we see the keywords, we use them. But before that time we remember which messages miss a keywords. Once the keywords become available, we'll fix up the flags for the unknown messages + // For maildirs, we are likely to get a possible dovecot-keywords file after having + // imported the messages. Once we see the keywords, we use them. But before that + // time we remember which messages miss a keywords. Once the keywords become + // available, we'll fix up the flags for the unknown messages mailboxKeywords := map[string]map[rune]string{} // Mailbox to 'a'-'z' to flag name. mailboxMissingKeywordMessages := map[string]map[int64]string{} // Mailbox to message id to string consisting of the unrecognized flags. + // We keep the mailboxes we deliver to up to date with their keywords (non-system flags). + destMailboxKeywords := map[int64]map[string]bool{} + // Previous mailbox an event was sent for. We send an event for new mailboxes, when // another 100 messages were added, when adding a message to another mailbox, and // finally at the end as a closing statement. @@ -471,6 +478,15 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store m.MailboxID = mb.ID m.MailboxOrigID = mb.ID + if len(m.Keywords) > 0 { + if destMailboxKeywords[mb.ID] == nil { + destMailboxKeywords[mb.ID] = map[string]bool{} + } + for _, k := range m.Keywords { + destMailboxKeywords[mb.ID][k] = true + } + } + // Parse message and store parsed information for later fast retrieval. p, err := message.EnsurePart(f, m.Size) if err != nil { @@ -503,7 +519,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store return } deliveredIDs = append(deliveredIDs, m.ID) - changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags}) + changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords}) messages[mb.Name]++ if messages[mb.Name]%100 == 0 || prevMailbox != mb.Name { prevMailbox = mb.Name @@ -583,7 +599,8 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store // Parse flags. See https://cr.yp.to/proto/maildir.html. var keepFlags string - flags := store.Flags{} + var flags store.Flags + keywords := map[string]bool{} t = strings.SplitN(path.Base(filename), ":2,", 2) if len(t) == 2 { for _, c := range t[1] { @@ -602,12 +619,12 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store flags.Flagged = true default: if c >= 'a' && c <= 'z' { - keywords, ok := mailboxKeywords[mailbox] + dovecotKeywords, ok := mailboxKeywords[mailbox] if !ok { // No keywords file seen yet, we'll try later if it comes in. keepFlags += string(c) - } else if kw, ok := keywords[c]; ok { - flagSet(&flags, strings.ToLower(kw)) + } else if kw, ok := dovecotKeywords[c]; ok { + flagSet(&flags, keywords, strings.ToLower(kw)) } } } @@ -617,6 +634,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store m := store.Message{ Received: received, Flags: flags, + Keywords: maps.Keys(keywords), Size: size, } xdeliver(mb, &m, f, filename) @@ -663,38 +681,52 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store default: if path.Base(name) == "dovecot-keywords" { mailbox := path.Dir(name) - keywords := map[rune]string{} + dovecotKeywords := map[rune]string{} words, err := store.ParseDovecotKeywords(r, log) log.Check(err, "parsing dovecot keywords for mailbox", mlog.Field("mailbox", mailbox)) for i, kw := range words { - keywords['a'+rune(i)] = kw + dovecotKeywords['a'+rune(i)] = kw } - mailboxKeywords[mailbox] = keywords + mailboxKeywords[mailbox] = dovecotKeywords for id, chars := range mailboxMissingKeywordMessages[mailbox] { var flags, zeroflags store.Flags + keywords := map[string]bool{} for _, c := range chars { - kw, ok := keywords[c] + kw, ok := dovecotKeywords[c] if !ok { - problemf("unspecified message flag %c for message id %d (continuing)", c, id) + problemf("unspecified dovecot message flag %c for message id %d (continuing)", c, id) continue } - flagSet(&flags, strings.ToLower(kw)) + flagSet(&flags, keywords, strings.ToLower(kw)) } - if flags == zeroflags { + if flags == zeroflags && len(keywords) == 0 { continue } + m := store.Message{ID: id} err := tx.Get(&m) ximportcheckf(err, "get imported message for flag update") + m.Flags = m.Flags.Set(flags, flags) + m.Keywords = maps.Keys(keywords) + + if len(m.Keywords) > 0 { + if destMailboxKeywords[m.MailboxID] == nil { + destMailboxKeywords[m.MailboxID] = map[string]bool{} + } + for _, k := range m.Keywords { + destMailboxKeywords[m.MailboxID][k] = true + } + } + // We train before updating, training may set m.TrainedJunk. if jf != nil && m.NeedsTraining() { openTrainMessage(&m) } err = tx.Update(&m) ximportcheckf(err, "updating message after flag update") - changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags}) + changes = append(changes, store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: flags, Flags: flags, Keywords: m.Keywords}) } delete(mailboxMissingKeywordMessages, mailbox) } else { @@ -744,6 +776,19 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store sendEvent("count", importCount{prevMailbox, messages[prevMailbox]}) } + // Update mailboxes with keywords. + for mbID, keywords := range destMailboxKeywords { + mb := store.Mailbox{ID: mbID} + err := tx.Get(&mb) + ximportcheckf(err, "loading mailbox for updating keywords") + var changed bool + mb.Keywords, changed = store.MergeKeywords(mb.Keywords, maps.Keys(keywords)) + if changed { + err = tx.Update(&mb) + ximportcheckf(err, "updating mailbox with keywords") + } + } + err = tx.Commit() tx = nil ximportcheckf(err, "commit") @@ -768,9 +813,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store sendEvent("done", importDone{}) } -func flagSet(flags *store.Flags, word string) { - // todo: custom labels, e.g. $label1, JunkRecorded? - +func flagSet(flags *store.Flags, keywords map[string]bool, word string) { switch word { case "forwarded", "$forwarded": flags.Forwarded = true @@ -782,5 +825,9 @@ func flagSet(flags *store.Flags, word string) { flags.Phishing = true case "mdnsent", "$mdnsent": flags.MDNSent = true + default: + if store.ValidLowercaseKeyword(word) { + keywords[word] = true + } } } diff --git a/imapclient/parse.go b/imapclient/parse.go index 7bdb30c..b0588db 100644 --- a/imapclient/parse.go +++ b/imapclient/parse.go @@ -178,9 +178,9 @@ func (c *Conn) xrespCode() (string, CodeArg) { l := []string{} // Must be non-nil. if c.take(' ') { c.xtake("(") - l = []string{c.xflag()} + l = []string{c.xflagPerm()} for c.take(' ') { - l = append(l, c.xflag()) + l = append(l, c.xflagPerm()) } c.xtake(")") } @@ -694,10 +694,13 @@ func (c *Conn) xliteral() []byte { // ../rfc/9051:6565 // todo: stricter -func (c *Conn) xflag() string { +func (c *Conn) xflag0(allowPerm bool) string { s := "" if c.take('\\') { - s = "\\" + s = `\` + if allowPerm && c.take('*') { + return `\*` + } } else if c.take('$') { s = "$" } @@ -705,6 +708,14 @@ func (c *Conn) xflag() string { return s } +func (c *Conn) xflag() string { + return c.xflag0(false) +} + +func (c *Conn) xflagPerm() string { + return c.xflag0(true) +} + func (c *Conn) xsection() string { c.xtake("[") s := c.xtakeuntil(']') diff --git a/imapserver/append_test.go b/imapserver/append_test.go index 16445a1..e9c51e9 100644 --- a/imapserver/append_test.go +++ b/imapserver/append_test.go @@ -44,14 +44,14 @@ func TestAppend(t *testing.T) { tc2.transactf("no", "append nobox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1}") tc2.xcode("TRYCREATE") - tc2.transactf("ok", "append inbox (\\Seen) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") + tc2.transactf("ok", "append inbox (\\Seen Label1 $label2) \" 1-Jan-2022 10:10:00 +0100\" {1+}\r\nx") tc2.xuntagged(imapclient.UntaggedExists(1)) tc2.xcodeArg(imapclient.CodeAppendUID{UIDValidity: 1, UID: 1}) tc.transactf("ok", "noop") uid1 := imapclient.FetchUID(1) - flagsSeen := imapclient.FetchFlags{`\Seen`} - tc.xuntagged(imapclient.UntaggedExists(1), imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, flagsSeen}}) + flags := imapclient.FetchFlags{`\Seen`, "label1", "$label2"} + tc.xuntagged(imapclient.UntaggedExists(1), imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, flags}}) tc3.transactf("ok", "noop") tc3.xuntagged() // Inbox is not selected, nothing to report. diff --git a/imapserver/fetch.go b/imapserver/fetch.go index 39d4359..e14b6c9 100644 --- a/imapserver/fetch.go +++ b/imapserver/fetch.go @@ -189,12 +189,12 @@ func (cmd *fetchCmd) process(atts []fetchAtt) { err := cmd.tx.Update(m) xcheckf(err, "marking message as seen") - cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags}) + cmd.changes = append(cmd.changes, store.ChangeFlags{MailboxID: cmd.mailboxID, UID: cmd.uid, Mask: store.Flags{Seen: true}, Flags: m.Flags, Keywords: m.Keywords}) } if cmd.needFlags { m := cmd.xensureMessage() - data = append(data, bare("FLAGS"), flaglist(m.Flags)) + data = append(data, bare("FLAGS"), flaglist(m.Flags, m.Keywords)) } // Write errors are turned into panics because we write through c. diff --git a/imapserver/parse.go b/imapserver/parse.go index d6236f6..74f5f09 100644 --- a/imapserver/parse.go +++ b/imapserver/parse.go @@ -376,8 +376,18 @@ func (p *parser) remainder() string { return p.orig[p.o:] } +// ../rfc/9051:6565 func (p *parser) xflag() string { - return p.xtakelist(`\`, "$") + p.xatom() + w, _ := p.takelist(`\`, "$") + s := w + p.xatom() + if s[0] == '\\' { + switch strings.ToLower(s) { + case `\answered`, `\flagged`, `\deleted`, `\seen`, `\draft`: + default: + p.xerrorf("unknown system flag %s", s) + } + } + return s } func (p *parser) xflagList() (l []string) { diff --git a/imapserver/search.go b/imapserver/search.go index 2f623cd..46daa4e 100644 --- a/imapserver/search.go +++ b/imapserver/search.go @@ -309,19 +309,24 @@ func (s *search) match(sk searchKey) bool { case "FLAGGED": return s.m.Flagged case "KEYWORD": - switch sk.atom { - case "$Forwarded": + kw := strings.ToLower(sk.atom) + switch kw { + case "$forwarded": return s.m.Forwarded - case "$Junk": + case "$junk": return s.m.Junk - case "$NotJunk": + case "$notjunk": return s.m.Notjunk - case "$Phishing": + case "$phishing": return s.m.Phishing - case "$MDNSent": + case "$mdnsent": return s.m.MDNSent default: - c.log.Info("search with unknown keyword", mlog.Field("keyword", sk.atom)) + for _, k := range s.m.Keywords { + if k == kw { + return true + } + } return false } case "SEEN": @@ -333,20 +338,25 @@ func (s *search) match(sk searchKey) bool { case "UNFLAGGED": return !s.m.Flagged case "UNKEYWORD": - switch sk.atom { - case "$Forwarded": + kw := strings.ToLower(sk.atom) + switch kw { + case "$forwarded": return !s.m.Forwarded - case "$Junk": + case "$junk": return !s.m.Junk - case "$NotJunk": + case "$notjunk": return !s.m.Notjunk - case "$Phishing": + case "$phishing": return !s.m.Phishing - case "$MDNSent": + case "$mdnsent": return !s.m.MDNSent default: - c.log.Info("search with unknown keyword", mlog.Field("keyword", sk.atom)) - return false + for _, k := range s.m.Keywords { + if k == kw { + return false + } + } + return true } case "UNSEEN": return !s.m.Seen diff --git a/imapserver/search_test.go b/imapserver/search_test.go index 2495cb0..621bc6e 100644 --- a/imapserver/search_test.go +++ b/imapserver/search_test.go @@ -79,6 +79,8 @@ func TestSearch(t *testing.T) { `$Notjunk`, `$Phishing`, `$MDNSent`, + `custom1`, + `Custom2`, } tc.client.Append("inbox", mostFlags, &received, []byte(searchMsg)) @@ -123,6 +125,12 @@ func TestSearch(t *testing.T) { tc.transactf("ok", `search keyword $Forwarded`) tc.xsearch(3) + tc.transactf("ok", `search keyword Custom1`) + tc.xsearch(3) + + tc.transactf("ok", `search keyword custom2`) + tc.xsearch(3) + tc.transactf("ok", `search new`) tc.xsearch() // New requires a message to be recent. We pretend all messages are not recent. @@ -162,6 +170,9 @@ func TestSearch(t *testing.T) { tc.transactf("ok", `search unkeyword $Junk`) tc.xsearch(1, 2) + tc.transactf("ok", `search unkeyword custom1`) + tc.xsearch(1, 2) + tc.transactf("ok", `search unseen`) tc.xsearch(1, 2) diff --git a/imapserver/selectexamine_test.go b/imapserver/selectexamine_test.go index dea4b1d..44c733f 100644 --- a/imapserver/selectexamine_test.go +++ b/imapserver/selectexamine_test.go @@ -32,8 +32,9 @@ func testSelectExamine(t *testing.T, examine bool) { uclosed := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "CLOSED", More: "x"}} flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent`, " ") + permflags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*`, " ") uflags := imapclient.UntaggedFlags(flags) - upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: flags}, More: "x"}} + upermflags := imapclient.UntaggedResult{Status: imapclient.OK, RespText: imapclient.RespText{Code: "PERMANENTFLAGS", CodeArg: imapclient.CodeList{Code: "PERMANENTFLAGS", Args: permflags}, More: "x"}} urecent := imapclient.UntaggedRecent(0) uexists0 := imapclient.UntaggedExists(0) uexists1 := imapclient.UntaggedExists(1) diff --git a/imapserver/server.go b/imapserver/server.go index d43c057..5519d71 100644 --- a/imapserver/server.go +++ b/imapserver/server.go @@ -1233,7 +1233,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { c.bwritelinef("* %d EXISTS", len(c.uids)) for _, add := range adds { seq := c.xsequence(add.UID) - c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", seq, add.UID, flaglist(add.Flags).pack(c)) + c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", seq, add.UID, flaglist(add.Flags, add.Keywords).pack(c)) } continue } @@ -1265,7 +1265,7 @@ func (c *conn) applyChanges(changes []store.Change, initial bool) { continue } if !initial { - c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", seq, ch.UID, flaglist(ch.Flags).pack(c)) + c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", seq, ch.UID, flaglist(ch.Flags, ch.Keywords).pack(c)) } case store.ChangeRemoveMailbox: // Only announce \NonExistent to modern clients, otherwise they may ignore the @@ -1862,8 +1862,12 @@ func (c *conn) cmdSelectExamine(isselect bool, tag, cmd string, p *parser) { }) c.applyChanges(c.comm.Get(), true) - c.bwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent)`) - c.bwritelinef(`* OK [PERMANENTFLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent)] x`) + var flags string + if len(mb.Keywords) > 0 { + flags = " " + strings.Join(mb.Keywords, " ") + } + c.bwritelinef(`* FLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent%s)`, flags) + c.bwritelinef(`* OK [PERMANENTFLAGS (\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent \*)] x`) if !c.enabled[capIMAP4rev2] { c.bwritelinef(`* 0 RECENT`) } @@ -2436,7 +2440,7 @@ func (c *conn) xstatusLine(tx *bstore.Tx, mb store.Mailbox, attrs []string) stri return fmt.Sprintf("* STATUS %s (%s)", astring(mb.Name).pack(c), strings.Join(status, " ")) } -func xparseStoreFlags(l []string, syntax bool) (flags store.Flags) { +func xparseStoreFlags(l []string, syntax bool) (flags store.Flags, keywords []string) { fields := map[string]*bool{ `\answered`: &flags.Answered, `\flagged`: &flags.Flagged, @@ -2449,20 +2453,24 @@ func xparseStoreFlags(l []string, syntax bool) (flags store.Flags) { `$phishing`: &flags.Phishing, `$mdnsent`: &flags.MDNSent, } + seen := map[string]bool{} for _, f := range l { - if field, ok := fields[strings.ToLower(f)]; !ok { - if syntax { - xsyntaxErrorf("unknown flag %q", f) - } - xuserErrorf("unknown flag %q", f) - } else { + f = strings.ToLower(f) + if field, ok := fields[f]; ok { *field = true + } else if seen[f] { + if moxvar.Pedantic { + xuserErrorf("duplicate keyword %s", f) + } + } else { + keywords = append(keywords, f) + seen[f] = true } } return } -func flaglist(fl store.Flags) listspace { +func flaglist(fl store.Flags, keywords []string) listspace { l := listspace{} flag := func(v bool, s string) { if v { @@ -2479,6 +2487,9 @@ func flaglist(fl store.Flags) listspace { flag(fl.Notjunk, `$NotJunk`) flag(fl.Phishing, `$Phishing`) flag(fl.MDNSent, `$MDNSent`) + for _, k := range keywords { + l = append(l, bare(k)) + } return l } @@ -2494,9 +2505,10 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { name := p.xmailbox() p.xspace() var storeFlags store.Flags + var keywords []string if p.hasPrefix("(") { // Error must be a syntax error, to properly abort the connection due to literal. - storeFlags = xparseStoreFlags(p.xflagList(), true) + storeFlags, keywords = xparseStoreFlags(p.xflagList(), true) p.xspace() } var tm time.Time @@ -2570,11 +2582,21 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { c.account.WithWLock(func() { c.xdbwrite(func(tx *bstore.Tx) { mb = c.xmailbox(tx, name, "TRYCREATE") + + // Ensure keywords are stored in mailbox. + var changed bool + mb.Keywords, changed = store.MergeKeywords(mb.Keywords, keywords) + if changed { + err := tx.Update(&mb) + xcheckf(err, "updating keywords in mailbox") + } + msg = store.Message{ MailboxID: mb.ID, MailboxOrigID: mb.ID, Received: tm, Flags: storeFlags, + Keywords: keywords, Size: size, MsgPrefix: msgPrefix, } @@ -2589,7 +2611,7 @@ func (c *conn) cmdAppend(tag, cmd string, p *parser) { } // Broadcast the change to other connections. - c.broadcast([]store.Change{store.ChangeAddUID{MailboxID: mb.ID, UID: msg.UID, Flags: msg.Flags}}) + c.broadcast([]store.Change{store.ChangeAddUID{MailboxID: mb.ID, UID: msg.UID, Flags: msg.Flags, Keywords: msg.Keywords}}) }) err = msgFile.Close() @@ -2952,6 +2974,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { var mbDst store.Mailbox var origUIDs, newUIDs []store.UID var flags []store.Flags + var keywords [][]string c.account.WithWLock(func() { c.xdbwrite(func(tx *bstore.Tx) { @@ -3017,6 +3040,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { newUIDs = append(newUIDs, m.UID) newMsgIDs = append(newMsgIDs, m.ID) flags = append(flags, m.Flags) + keywords = append(keywords, m.Keywords) qmr := bstore.QueryTx[store.Recipient](tx) qmr.FilterNonzero(store.Recipient{MessageID: origID}) @@ -3048,7 +3072,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) { if len(newUIDs) > 0 { changes := make([]store.Change, len(newUIDs)) for i, uid := range newUIDs { - changes[i] = store.ChangeAddUID{MailboxID: mbDst.ID, UID: uid, Flags: flags[i]} + changes[i] = store.ChangeAddUID{MailboxID: mbDst.ID, UID: uid, Flags: flags[i], Keywords: keywords[i]} } c.broadcast(changes) } @@ -3187,7 +3211,7 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) { changes = append(changes, store.ChangeRemoveUIDs{MailboxID: c.mailboxID, UIDs: uids}) for _, m := range msgs { newUIDs = append(newUIDs, m.UID) - changes = append(changes, store.ChangeAddUID{MailboxID: mbDst.ID, UID: m.UID, Flags: m.Flags}) + changes = append(changes, store.ChangeAddUID{MailboxID: mbDst.ID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords}) } }) @@ -3240,25 +3264,21 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { xuserErrorf("mailbox open in read-only mode") } - var mask, flags store.Flags + flags, keywords := xparseStoreFlags(flagstrs, false) + var mask store.Flags if plus { - mask = xparseStoreFlags(flagstrs, false) - flags = store.FlagsAll + mask, flags = flags, store.FlagsAll } else if minus { - mask = xparseStoreFlags(flagstrs, false) - flags = store.Flags{} + mask, flags = flags, store.Flags{} } else { mask = store.FlagsAll - flags = xparseStoreFlags(flagstrs, false) } - updates := store.FlagsQuerySet(mask, flags) - var updated []store.Message c.account.WithWLock(func() { c.xdbwrite(func(tx *bstore.Tx) { - c.xmailboxID(tx, c.mailboxID) // Validate. + mb := c.xmailboxID(tx, c.mailboxID) // Validate. uidargs := c.xnumSetCondition(isUID, nums) @@ -3266,27 +3286,41 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { return } + // Ensure keywords are in mailbox. + if !minus { + var changed bool + mb.Keywords, changed = store.MergeKeywords(mb.Keywords, keywords) + if changed { + err := tx.Update(&mb) + xcheckf(err, "updating mailbox with keywords") + } + } + q := bstore.QueryTx[store.Message](tx) q.FilterNonzero(store.Message{MailboxID: c.mailboxID}) q.FilterEqual("UID", uidargs...) - if len(updates) == 0 { - var err error - updated, err = q.List() - xcheckf(err, "listing for flags") - } else { - q.Gather(&updated) - _, err := q.UpdateFields(updates) - xcheckf(err, "updating flags") - } + err := q.ForEach(func(m store.Message) error { + m.Flags = m.Flags.Set(mask, flags) + if minus { + m.Keywords = store.RemoveKeywords(m.Keywords, keywords) + } else if plus { + m.Keywords, _ = store.MergeKeywords(m.Keywords, keywords) + } else { + m.Keywords = keywords + } + updated = append(updated, m) + return tx.Update(&m) + }) + xcheckf(err, "storing flags in messages") - err := c.account.RetrainMessages(context.TODO(), c.log, tx, updated, false) + err = c.account.RetrainMessages(context.TODO(), c.log, tx, updated, false) xcheckf(err, "training messages") }) // Broadcast changes to other connections. changes := make([]store.Change, len(updated)) for i, m := range updated { - changes[i] = store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: mask, Flags: m.Flags} + changes[i] = store.ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, Mask: mask, Flags: m.Flags, Keywords: m.Keywords} } c.broadcast(changes) }) @@ -3294,7 +3328,7 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) { for _, m := range updated { if !silent { // ../rfc/9051:6749 ../rfc/3501:4869 - c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", c.xsequence(m.UID), m.UID, flaglist(m.Flags).pack(c)) + c.bwritelinef("* %d FETCH (UID %d FLAGS %s)", c.xsequence(m.UID), m.UID, flaglist(m.Flags, m.Keywords).pack(c)) } } diff --git a/imapserver/server_test.go b/imapserver/server_test.go index 564aa82..fa4a718 100644 --- a/imapserver/server_test.go +++ b/imapserver/server_test.go @@ -191,6 +191,10 @@ func (tc *testconn) xcodeArg(v any) { } func (tc *testconn) xuntagged(exps ...any) { + tc.xuntaggedCheck(true, exps...) +} + +func (tc *testconn) xuntaggedCheck(all bool, exps ...any) { tc.t.Helper() last := append([]imapclient.Untagged{}, tc.lastUntagged...) next: @@ -212,7 +216,7 @@ next: } tc.t.Fatalf("did not find untagged response %#v %T (%d) in %v%s", exp, exp, ei, tc.lastUntagged, next) } - if len(last) > 0 { + if len(last) > 0 && all { tc.t.Fatalf("leftover untagged responses %v", last) } } @@ -525,7 +529,7 @@ func TestScenario(t *testing.T) { tc.transactf("ok", `store 1 flags.silent (\seen \answered)`) tc.transactf("ok", `store 1 -flags.silent (\answered)`) tc.transactf("ok", `store 1 +flags.silent (\answered)`) - tc.transactf("no", `store 1 flags (\badflag)`) + tc.transactf("bad", `store 1 flags (\badflag)`) tc.transactf("ok", "noop") tc.transactf("ok", "copy 1 Trash") diff --git a/imapserver/store_test.go b/imapserver/store_test.go index 9c73815..1600780 100644 --- a/imapserver/store_test.go +++ b/imapserver/store_test.go @@ -1,6 +1,7 @@ package imapserver import ( + "strings" "testing" "github.com/mjl-/mox/imapclient" @@ -54,15 +55,30 @@ func TestStore(t *testing.T) { tc.transactf("ok", "uid store 1 flags ()") tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, noflags}}) - tc.transactf("bad", "store") // Need numset, flags and args. - tc.transactf("bad", "store 1") // Need flags. - tc.transactf("bad", "store 1 +") // Need flags. - tc.transactf("bad", "store 1 -") // Need flags. - tc.transactf("bad", "store 1 flags ") // Need flags. - tc.transactf("bad", "store 1 flags ") // Need flags. - tc.transactf("bad", "store 1 flags (bogus)") // Unknown flag. + tc.transactf("ok", "store 1 flags (new)") // New flag. + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"new"}}}) + tc.transactf("ok", "store 1 flags (new new a b c)") // Duplicates are ignored. + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"new", "a", "b", "c"}}}) + tc.transactf("ok", "store 1 +flags (new new c d e)") + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"new", "a", "b", "c", "d", "e"}}}) + tc.transactf("ok", "store 1 -flags (new new e a c)") + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"b", "d"}}}) + tc.transactf("ok", "store 1 flags ($Forwarded Different)") + tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{uid1, imapclient.FetchFlags{"$Forwarded", "different"}}}) + + tc.transactf("bad", "store") // Need numset, flags and args. + tc.transactf("bad", "store 1") // Need flags. + tc.transactf("bad", "store 1 +") // Need flags. + tc.transactf("bad", "store 1 -") // Need flags. + tc.transactf("bad", "store 1 flags ") // Need flags. + tc.transactf("bad", "store 1 flags ") // Need flags. tc.client.Unselect() - tc.client.Examine("inbox") // Open read-only. + tc.transactf("ok", "examine inbox") // Open read-only. + + // Flags are added to mailbox, not removed. + flags := strings.Split(`\Seen \Answered \Flagged \Deleted \Draft $Forwarded $Junk $NotJunk $Phishing $MDNSent new a b c d e different`, " ") + tc.xuntaggedCheck(false, imapclient.UntaggedFlags(flags)) + tc.transactf("no", `store 1 flags ()`) // No permission to set flags. } diff --git a/import.go b/import.go index c0aaf4d..df2f978 100644 --- a/import.go +++ b/import.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "golang.org/x/exp/maps" + "github.com/mjl-/mox/message" "github.com/mjl-/mox/metrics" "github.com/mjl-/mox/mlog" @@ -229,7 +231,7 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { ctl.xcheck(err, "delivering message") deliveredIDs = append(deliveredIDs, m.ID) ctl.log.Debug("delivered message", mlog.Field("id", m.ID)) - changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags}) + changes = append(changes, store.ChangeAddUID{MailboxID: m.MailboxID, UID: m.UID, Flags: m.Flags, Keywords: m.Keywords}) } // todo: one goroutine for reading messages, one for parsing the message, one adding to database, one for junk filter training. @@ -240,6 +242,9 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { mb, changes, err = a.MailboxEnsure(tx, mailbox, true) ctl.xcheck(err, "ensuring mailbox exists") + // We ensure keywords in messages make it to the mailbox as well. + mailboxKeywords := map[string]bool{} + jf, _, err := a.OpenJunkFilter(ctx, ctl.log) if err != nil && !errors.Is(err, store.ErrNoJunkFilter) { ctl.xcheck(err, "open junk filter") @@ -264,6 +269,10 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { ctl.log.Check(err, "closing temporary message after failing to import") }() + for _, kw := range m.Keywords { + mailboxKeywords[kw] = true + } + // Parse message and store parsed information for later fast retrieval. p, err := message.EnsurePart(msgf, m.Size) if err != nil { @@ -317,6 +326,14 @@ func importctl(ctx context.Context, ctl *ctl, mbox bool) { process(m, msgf, origPath) } + // If there are any new keywords, update the mailbox. + var changed bool + mb.Keywords, changed = store.MergeKeywords(mb.Keywords, maps.Keys(mailboxKeywords)) + if changed { + err := tx.Update(&mb) + ctl.xcheck(err, "updating keywords in mailbox") + } + err = tx.Commit() ctl.xcheck(err, "commit") tx = nil diff --git a/store/account.go b/store/account.go index 7d02cdb..5b484e9 100644 --- a/store/account.go +++ b/store/account.go @@ -39,6 +39,7 @@ import ( "time" "golang.org/x/crypto/bcrypt" + "golang.org/x/exp/slices" "golang.org/x/text/unicode/norm" "github.com/mjl-/bstore" @@ -172,6 +173,11 @@ type Mailbox struct { Junk bool Sent bool Trash bool + + // Keywords as used in messages. Storing a non-system keyword for a message + // automatically adds it to this list. Used in the IMAP FLAGS response. Only + // "atoms", stored in lower case. + Keywords []string } // Subscriptions are separate from existence of mailboxes. @@ -286,6 +292,7 @@ type Message struct { MessageHash []byte // Hash of message. For rejects delivery, so optional like MessageID. Flags + Keywords []string `bstore:"index"` // Non-system or well-known $-flags. Only in "atom" syntax, stored in lower case. Size int64 TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk. MsgPrefix []byte // Typically holds received headers and/or header separator. @@ -1054,7 +1061,7 @@ func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgF return err } - changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.Flags}) + changes = append(changes, ChangeAddUID{m.MailboxID, m.UID, m.Flags, m.Keywords}) comm := RegisterComm(a) defer comm.Unregister() comm.Broadcast(changes) @@ -1348,24 +1355,44 @@ func (f Flags) Set(mask, flags Flags) Flags { return r } -// FlagsQuerySet returns a map with the flags that are true in mask, with -// values from flags. -func FlagsQuerySet(mask, flags Flags) map[string]any { - r := map[string]any{} - set := func(f string, m, v bool) { - if m { - r[f] = v +// RemoveKeywords removes keywords from l, modifying and returning it. Should only +// be used with lower-case keywords, not with system flags like \Seen. +func RemoveKeywords(l, remove []string) []string { + for _, k := range remove { + if i := slices.Index(l, k); i >= 0 { + copy(l[i:], l[i+1:]) + l = l[:len(l)-1] } } - set("Seen", mask.Seen, flags.Seen) - set("Answered", mask.Answered, flags.Answered) - set("Flagged", mask.Flagged, flags.Flagged) - set("Forwarded", mask.Forwarded, flags.Forwarded) - set("Junk", mask.Junk, flags.Junk) - set("Notjunk", mask.Notjunk, flags.Notjunk) - set("Deleted", mask.Deleted, flags.Deleted) - set("Draft", mask.Draft, flags.Draft) - set("Phishing", mask.Phishing, flags.Phishing) - set("MDNSent", mask.MDNSent, flags.MDNSent) - return r + return l +} + +// MergeKeywords adds keywords from add into l, updating and returning it along +// with whether it added any keyword. Keywords are only added if they aren't +// already present. Should only be used with lower-case keywords, not with system +// flags like \Seen. +func MergeKeywords(l, add []string) ([]string, bool) { + var changed bool + for _, k := range add { + if slices.Index(l, k) < 0 { + l = append(l, k) + changed = true + } + } + return l, changed +} + +// ValidLowercaseKeyword returns whether s is a valid, lower-case, keyword. +func ValidLowercaseKeyword(s string) bool { + for _, c := range s { + if c >= 'a' && c <= 'z' { + continue + } + // ../rfc/9051:6334 + const atomspecials = `(){%*"\]` + if c <= ' ' || c > 0x7e || strings.ContainsRune(atomspecials, c) { + return false + } + } + return len(s) > 0 } diff --git a/store/import.go b/store/import.go index 3fbb0e1..3802d0c 100644 --- a/store/import.go +++ b/store/import.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "golang.org/x/exp/maps" + "github.com/mjl-/mox/mlog" ) @@ -92,6 +94,7 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) { fromLine := mr.fromLine bf := bufio.NewWriter(f) var flags Flags + keywords := map[string]bool{} var size int64 for { line, err := mr.r.ReadBytes('\n') @@ -132,7 +135,23 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) { } else if bytes.HasPrefix(line, []byte("X-Keywords:")) { s := strings.TrimSpace(strings.SplitN(string(line), ":", 2)[1]) for _, t := range strings.Split(s, ",") { - flagSet(&flags, strings.ToLower(strings.TrimSpace(t))) + word := strings.ToLower(strings.TrimSpace(t)) + switch word { + case "forwarded", "$forwarded": + flags.Forwarded = true + case "junk", "$junk": + flags.Junk = true + case "notjunk", "$notjunk", "nonjunk", "$nonjunk": + flags.Notjunk = true + case "phishing", "$phishing": + flags.Phishing = true + case "mdnsent", "$mdnsent": + flags.MDNSent = true + default: + if ValidLowercaseKeyword(word) { + keywords[word] = true + } + } } } } @@ -165,7 +184,7 @@ func (mr *MboxReader) Next() (*Message, *os.File, string, error) { return nil, nil, mr.Position(), fmt.Errorf("flush: %v", err) } - m := &Message{Flags: flags, Size: size} + m := &Message{Flags: flags, Keywords: maps.Keys(keywords), Size: size} if t := strings.SplitN(fromLine, " ", 3); len(t) == 3 { layouts := []string{time.ANSIC, time.UnixDate, time.RubyDate} @@ -297,6 +316,7 @@ func (mr *MaildirReader) Next() (*Message, *os.File, string, error) { // Parse flags. See https://cr.yp.to/proto/maildir.html. flags := Flags{} + keywords := map[string]bool{} t = strings.SplitN(filepath.Base(sf.Name()), ":2,", 2) if len(t) == 2 { for _, c := range t[1] { @@ -319,26 +339,29 @@ func (mr *MaildirReader) Next() (*Message, *os.File, string, error) { if index >= len(mr.dovecotKeywords) { continue } - kw := mr.dovecotKeywords[index] + kw := strings.ToLower(mr.dovecotKeywords[index]) switch kw { - case "$Forwarded", "Forwarded": + case "$forwarded", "forwarded": flags.Forwarded = true - case "$Junk", "Junk": + case "$junk", "junk": flags.Junk = true - case "$NotJunk", "NotJunk", "NonJunk": + case "$notjunk", "notjunk", "nonjunk": flags.Notjunk = true - case "$MDNSent": + case "$mdnsent", "mdnsent": flags.MDNSent = true - case "$Phishing", "Phishing": + case "$phishing", "phishing": flags.Phishing = true + default: + if ValidLowercaseKeyword(kw) { + keywords[kw] = true + } } - // todo: custom labels, e.g. $label1, JunkRecorded? } } } } - m := &Message{Received: received, Flags: flags, Size: size} + m := &Message{Received: received, Flags: flags, Keywords: maps.Keys(keywords), Size: size} // Prevent cleanup by defer. mf := f @@ -397,18 +420,3 @@ func ParseDovecotKeywords(r io.Reader, log *mlog.Log) ([]string, error) { } return keywords[:end], err } - -func flagSet(flags *Flags, word string) { - switch word { - case "forwarded", "$forwarded": - flags.Forwarded = true - case "junk", "$junk": - flags.Junk = true - case "notjunk", "$notjunk", "nonjunk", "$nonjunk": - flags.Notjunk = true - case "phishing", "$phishing": - flags.Phishing = true - case "mdnsent", "$mdnsent": - flags.MDNSent = true - } -} diff --git a/store/state.go b/store/state.go index e59a6ce..8a30c7b 100644 --- a/store/state.go +++ b/store/state.go @@ -26,7 +26,8 @@ type Change any type ChangeAddUID struct { MailboxID int64 UID UID - Flags Flags + Flags Flags // System flags. + Keywords []string // Other flags. } // ChangeRemoveUIDs is sent for removal of one or more messages from a mailbox. @@ -39,8 +40,9 @@ type ChangeRemoveUIDs struct { type ChangeFlags struct { MailboxID int64 UID UID - Mask Flags // Which flags are actually modified. - Flags Flags // New flag values. All are set, not just mask. + Mask Flags // Which flags are actually modified. + Flags Flags // New flag values. All are set, not just mask. + Keywords []string // Other flags. } // ChangeRemoveMailbox is sent for a removed mailbox. diff --git a/testdata/importtest.maildir.tgz b/testdata/importtest.maildir.tgz index 1e6b199..63ac2d9 100644 Binary files a/testdata/importtest.maildir.tgz and b/testdata/importtest.maildir.tgz differ diff --git a/testdata/importtest.mbox.zip b/testdata/importtest.mbox.zip index bd5cfec..47a4a0b 100644 Binary files a/testdata/importtest.mbox.zip and b/testdata/importtest.mbox.zip differ diff --git a/vendor/golang.org/x/exp/LICENSE b/vendor/golang.org/x/exp/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/golang.org/x/exp/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/exp/PATENTS b/vendor/golang.org/x/exp/PATENTS new file mode 100644 index 0000000..7330990 --- /dev/null +++ b/vendor/golang.org/x/exp/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/exp/constraints/constraints.go b/vendor/golang.org/x/exp/constraints/constraints.go new file mode 100644 index 0000000..2c033df --- /dev/null +++ b/vendor/golang.org/x/exp/constraints/constraints.go @@ -0,0 +1,50 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package constraints defines a set of useful constraints to be used +// with type parameters. +package constraints + +// Signed is a constraint that permits any signed integer type. +// If future releases of Go add new predeclared signed integer types, +// this constraint will be modified to include them. +type Signed interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 +} + +// Unsigned is a constraint that permits any unsigned integer type. +// If future releases of Go add new predeclared unsigned integer types, +// this constraint will be modified to include them. +type Unsigned interface { + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +// Integer is a constraint that permits any integer type. +// If future releases of Go add new predeclared integer types, +// this constraint will be modified to include them. +type Integer interface { + Signed | Unsigned +} + +// Float is a constraint that permits any floating-point type. +// If future releases of Go add new predeclared floating-point types, +// this constraint will be modified to include them. +type Float interface { + ~float32 | ~float64 +} + +// Complex is a constraint that permits any complex numeric type. +// If future releases of Go add new predeclared complex numeric types, +// this constraint will be modified to include them. +type Complex interface { + ~complex64 | ~complex128 +} + +// Ordered is a constraint that permits any ordered type: any type +// that supports the operators < <= >= >. +// If future releases of Go add new ordered types, +// this constraint will be modified to include them. +type Ordered interface { + Integer | Float | ~string +} diff --git a/vendor/golang.org/x/exp/maps/maps.go b/vendor/golang.org/x/exp/maps/maps.go new file mode 100644 index 0000000..ecc0dab --- /dev/null +++ b/vendor/golang.org/x/exp/maps/maps.go @@ -0,0 +1,94 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package maps defines various functions useful with maps of any type. +package maps + +// Keys returns the keys of the map m. +// The keys will be in an indeterminate order. +func Keys[M ~map[K]V, K comparable, V any](m M) []K { + r := make([]K, 0, len(m)) + for k := range m { + r = append(r, k) + } + return r +} + +// Values returns the values of the map m. +// The values will be in an indeterminate order. +func Values[M ~map[K]V, K comparable, V any](m M) []V { + r := make([]V, 0, len(m)) + for _, v := range m { + r = append(r, v) + } + return r +} + +// Equal reports whether two maps contain the same key/value pairs. +// Values are compared using ==. +func Equal[M1, M2 ~map[K]V, K, V comparable](m1 M1, m2 M2) bool { + if len(m1) != len(m2) { + return false + } + for k, v1 := range m1 { + if v2, ok := m2[k]; !ok || v1 != v2 { + return false + } + } + return true +} + +// EqualFunc is like Equal, but compares values using eq. +// Keys are still compared with ==. +func EqualFunc[M1 ~map[K]V1, M2 ~map[K]V2, K comparable, V1, V2 any](m1 M1, m2 M2, eq func(V1, V2) bool) bool { + if len(m1) != len(m2) { + return false + } + for k, v1 := range m1 { + if v2, ok := m2[k]; !ok || !eq(v1, v2) { + return false + } + } + return true +} + +// Clear removes all entries from m, leaving it empty. +func Clear[M ~map[K]V, K comparable, V any](m M) { + for k := range m { + delete(m, k) + } +} + +// Clone returns a copy of m. This is a shallow clone: +// the new keys and values are set using ordinary assignment. +func Clone[M ~map[K]V, K comparable, V any](m M) M { + // Preserve nil in case it matters. + if m == nil { + return nil + } + r := make(M, len(m)) + for k, v := range m { + r[k] = v + } + return r +} + +// Copy copies all key/value pairs in src adding them to dst. +// When a key in src is already present in dst, +// the value in dst will be overwritten by the value associated +// with the key in src. +func Copy[M1 ~map[K]V, M2 ~map[K]V, K comparable, V any](dst M1, src M2) { + for k, v := range src { + dst[k] = v + } +} + +// DeleteFunc deletes any key/value pairs from m for which del returns true. +func DeleteFunc[M ~map[K]V, K comparable, V any](m M, del func(K, V) bool) { + for k, v := range m { + if del(k, v) { + delete(m, k) + } + } +} diff --git a/vendor/golang.org/x/exp/slices/slices.go b/vendor/golang.org/x/exp/slices/slices.go new file mode 100644 index 0000000..2540bd6 --- /dev/null +++ b/vendor/golang.org/x/exp/slices/slices.go @@ -0,0 +1,258 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package slices defines various functions useful with slices of any type. +// Unless otherwise specified, these functions all apply to the elements +// of a slice at index 0 <= i < len(s). +// +// Note that the less function in IsSortedFunc, SortFunc, SortStableFunc requires a +// strict weak ordering (https://en.wikipedia.org/wiki/Weak_ordering#Strict_weak_orderings), +// or the sorting may fail to sort correctly. A common case is when sorting slices of +// floating-point numbers containing NaN values. +package slices + +import "golang.org/x/exp/constraints" + +// Equal reports whether two slices are equal: the same length and all +// elements equal. If the lengths are different, Equal returns false. +// Otherwise, the elements are compared in increasing index order, and the +// comparison stops at the first unequal pair. +// Floating point NaNs are not considered equal. +func Equal[E comparable](s1, s2 []E) bool { + if len(s1) != len(s2) { + return false + } + for i := range s1 { + if s1[i] != s2[i] { + return false + } + } + return true +} + +// EqualFunc reports whether two slices are equal using a comparison +// function on each pair of elements. If the lengths are different, +// EqualFunc returns false. Otherwise, the elements are compared in +// increasing index order, and the comparison stops at the first index +// for which eq returns false. +func EqualFunc[E1, E2 any](s1 []E1, s2 []E2, eq func(E1, E2) bool) bool { + if len(s1) != len(s2) { + return false + } + for i, v1 := range s1 { + v2 := s2[i] + if !eq(v1, v2) { + return false + } + } + return true +} + +// Compare compares the elements of s1 and s2. +// The elements are compared sequentially, starting at index 0, +// until one element is not equal to the other. +// The result of comparing the first non-matching elements is returned. +// If both slices are equal until one of them ends, the shorter slice is +// considered less than the longer one. +// The result is 0 if s1 == s2, -1 if s1 < s2, and +1 if s1 > s2. +// Comparisons involving floating point NaNs are ignored. +func Compare[E constraints.Ordered](s1, s2 []E) int { + s2len := len(s2) + for i, v1 := range s1 { + if i >= s2len { + return +1 + } + v2 := s2[i] + switch { + case v1 < v2: + return -1 + case v1 > v2: + return +1 + } + } + if len(s1) < s2len { + return -1 + } + return 0 +} + +// CompareFunc is like Compare but uses a comparison function +// on each pair of elements. The elements are compared in increasing +// index order, and the comparisons stop after the first time cmp +// returns non-zero. +// The result is the first non-zero result of cmp; if cmp always +// returns 0 the result is 0 if len(s1) == len(s2), -1 if len(s1) < len(s2), +// and +1 if len(s1) > len(s2). +func CompareFunc[E1, E2 any](s1 []E1, s2 []E2, cmp func(E1, E2) int) int { + s2len := len(s2) + for i, v1 := range s1 { + if i >= s2len { + return +1 + } + v2 := s2[i] + if c := cmp(v1, v2); c != 0 { + return c + } + } + if len(s1) < s2len { + return -1 + } + return 0 +} + +// Index returns the index of the first occurrence of v in s, +// or -1 if not present. +func Index[E comparable](s []E, v E) int { + for i := range s { + if v == s[i] { + return i + } + } + return -1 +} + +// IndexFunc returns the first index i satisfying f(s[i]), +// or -1 if none do. +func IndexFunc[E any](s []E, f func(E) bool) int { + for i := range s { + if f(s[i]) { + return i + } + } + return -1 +} + +// Contains reports whether v is present in s. +func Contains[E comparable](s []E, v E) bool { + return Index(s, v) >= 0 +} + +// ContainsFunc reports whether at least one +// element e of s satisfies f(e). +func ContainsFunc[E any](s []E, f func(E) bool) bool { + return IndexFunc(s, f) >= 0 +} + +// Insert inserts the values v... into s at index i, +// returning the modified slice. +// In the returned slice r, r[i] == v[0]. +// Insert panics if i is out of range. +// This function is O(len(s) + len(v)). +func Insert[S ~[]E, E any](s S, i int, v ...E) S { + tot := len(s) + len(v) + if tot <= cap(s) { + s2 := s[:tot] + copy(s2[i+len(v):], s[i:]) + copy(s2[i:], v) + return s2 + } + s2 := make(S, tot) + copy(s2, s[:i]) + copy(s2[i:], v) + copy(s2[i+len(v):], s[i:]) + return s2 +} + +// Delete removes the elements s[i:j] from s, returning the modified slice. +// Delete panics if s[i:j] is not a valid slice of s. +// Delete modifies the contents of the slice s; it does not create a new slice. +// Delete is O(len(s)-j), so if many items must be deleted, it is better to +// make a single call deleting them all together than to delete one at a time. +// Delete might not modify the elements s[len(s)-(j-i):len(s)]. If those +// elements contain pointers you might consider zeroing those elements so that +// objects they reference can be garbage collected. +func Delete[S ~[]E, E any](s S, i, j int) S { + _ = s[i:j] // bounds check + + return append(s[:i], s[j:]...) +} + +// Replace replaces the elements s[i:j] by the given v, and returns the +// modified slice. Replace panics if s[i:j] is not a valid slice of s. +func Replace[S ~[]E, E any](s S, i, j int, v ...E) S { + _ = s[i:j] // verify that i:j is a valid subslice + tot := len(s[:i]) + len(v) + len(s[j:]) + if tot <= cap(s) { + s2 := s[:tot] + copy(s2[i+len(v):], s[j:]) + copy(s2[i:], v) + return s2 + } + s2 := make(S, tot) + copy(s2, s[:i]) + copy(s2[i:], v) + copy(s2[i+len(v):], s[j:]) + return s2 +} + +// Clone returns a copy of the slice. +// The elements are copied using assignment, so this is a shallow clone. +func Clone[S ~[]E, E any](s S) S { + // Preserve nil in case it matters. + if s == nil { + return nil + } + return append(S([]E{}), s...) +} + +// Compact replaces consecutive runs of equal elements with a single copy. +// This is like the uniq command found on Unix. +// Compact modifies the contents of the slice s; it does not create a new slice. +// When Compact discards m elements in total, it might not modify the elements +// s[len(s)-m:len(s)]. If those elements contain pointers you might consider +// zeroing those elements so that objects they reference can be garbage collected. +func Compact[S ~[]E, E comparable](s S) S { + if len(s) < 2 { + return s + } + i := 1 + for k := 1; k < len(s); k++ { + if s[k] != s[k-1] { + if i != k { + s[i] = s[k] + } + i++ + } + } + return s[:i] +} + +// CompactFunc is like Compact but uses a comparison function. +func CompactFunc[S ~[]E, E any](s S, eq func(E, E) bool) S { + if len(s) < 2 { + return s + } + i := 1 + for k := 1; k < len(s); k++ { + if !eq(s[k], s[k-1]) { + if i != k { + s[i] = s[k] + } + i++ + } + } + return s[:i] +} + +// Grow increases the slice's capacity, if necessary, to guarantee space for +// another n elements. After Grow(n), at least n elements can be appended +// to the slice without another allocation. If n is negative or too large to +// allocate the memory, Grow panics. +func Grow[S ~[]E, E any](s S, n int) S { + if n < 0 { + panic("cannot be negative") + } + if n -= cap(s) - len(s); n > 0 { + // TODO(https://go.dev/issue/53888): Make using []E instead of S + // to workaround a compiler bug where the runtime.growslice optimization + // does not take effect. Revert when the compiler is fixed. + s = append([]E(s)[:cap(s)], make([]E, n)...)[:len(s)] + } + return s +} + +// Clip removes unused capacity from the slice, returning s[:len(s):len(s)]. +func Clip[S ~[]E, E any](s S) S { + return s[:len(s):len(s)] +} diff --git a/vendor/golang.org/x/exp/slices/sort.go b/vendor/golang.org/x/exp/slices/sort.go new file mode 100644 index 0000000..231b644 --- /dev/null +++ b/vendor/golang.org/x/exp/slices/sort.go @@ -0,0 +1,128 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slices + +import ( + "math/bits" + + "golang.org/x/exp/constraints" +) + +// Sort sorts a slice of any ordered type in ascending order. +// Sort may fail to sort correctly when sorting slices of floating-point +// numbers containing Not-a-number (NaN) values. +// Use slices.SortFunc(x, func(a, b float64) bool {return a < b || (math.IsNaN(a) && !math.IsNaN(b))}) +// instead if the input may contain NaNs. +func Sort[E constraints.Ordered](x []E) { + n := len(x) + pdqsortOrdered(x, 0, n, bits.Len(uint(n))) +} + +// SortFunc sorts the slice x in ascending order as determined by the less function. +// This sort is not guaranteed to be stable. +// +// SortFunc requires that less is a strict weak ordering. +// See https://en.wikipedia.org/wiki/Weak_ordering#Strict_weak_orderings. +func SortFunc[E any](x []E, less func(a, b E) bool) { + n := len(x) + pdqsortLessFunc(x, 0, n, bits.Len(uint(n)), less) +} + +// SortStableFunc sorts the slice x while keeping the original order of equal +// elements, using less to compare elements. +func SortStableFunc[E any](x []E, less func(a, b E) bool) { + stableLessFunc(x, len(x), less) +} + +// IsSorted reports whether x is sorted in ascending order. +func IsSorted[E constraints.Ordered](x []E) bool { + for i := len(x) - 1; i > 0; i-- { + if x[i] < x[i-1] { + return false + } + } + return true +} + +// IsSortedFunc reports whether x is sorted in ascending order, with less as the +// comparison function. +func IsSortedFunc[E any](x []E, less func(a, b E) bool) bool { + for i := len(x) - 1; i > 0; i-- { + if less(x[i], x[i-1]) { + return false + } + } + return true +} + +// BinarySearch searches for target in a sorted slice and returns the position +// where target is found, or the position where target would appear in the +// sort order; it also returns a bool saying whether the target is really found +// in the slice. The slice must be sorted in increasing order. +func BinarySearch[E constraints.Ordered](x []E, target E) (int, bool) { + // Inlining is faster than calling BinarySearchFunc with a lambda. + n := len(x) + // Define x[-1] < target and x[n] >= target. + // Invariant: x[i-1] < target, x[j] >= target. + i, j := 0, n + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + if x[h] < target { + i = h + 1 // preserves x[i-1] < target + } else { + j = h // preserves x[j] >= target + } + } + // i == j, x[i-1] < target, and x[j] (= x[i]) >= target => answer is i. + return i, i < n && x[i] == target +} + +// BinarySearchFunc works like BinarySearch, but uses a custom comparison +// function. The slice must be sorted in increasing order, where "increasing" +// is defined by cmp. cmp should return 0 if the slice element matches +// the target, a negative number if the slice element precedes the target, +// or a positive number if the slice element follows the target. +// cmp must implement the same ordering as the slice, such that if +// cmp(a, t) < 0 and cmp(b, t) >= 0, then a must precede b in the slice. +func BinarySearchFunc[E, T any](x []E, target T, cmp func(E, T) int) (int, bool) { + n := len(x) + // Define cmp(x[-1], target) < 0 and cmp(x[n], target) >= 0 . + // Invariant: cmp(x[i - 1], target) < 0, cmp(x[j], target) >= 0. + i, j := 0, n + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + if cmp(x[h], target) < 0 { + i = h + 1 // preserves cmp(x[i - 1], target) < 0 + } else { + j = h // preserves cmp(x[j], target) >= 0 + } + } + // i == j, cmp(x[i-1], target) < 0, and cmp(x[j], target) (= cmp(x[i], target)) >= 0 => answer is i. + return i, i < n && cmp(x[i], target) == 0 +} + +type sortedHint int // hint for pdqsort when choosing the pivot + +const ( + unknownHint sortedHint = iota + increasingHint + decreasingHint +) + +// xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf +type xorshift uint64 + +func (r *xorshift) Next() uint64 { + *r ^= *r << 13 + *r ^= *r >> 17 + *r ^= *r << 5 + return uint64(*r) +} + +func nextPowerOfTwo(length int) uint { + return 1 << bits.Len(uint(length)) +} diff --git a/vendor/golang.org/x/exp/slices/zsortfunc.go b/vendor/golang.org/x/exp/slices/zsortfunc.go new file mode 100644 index 0000000..2a63247 --- /dev/null +++ b/vendor/golang.org/x/exp/slices/zsortfunc.go @@ -0,0 +1,479 @@ +// Code generated by gen_sort_variants.go; DO NOT EDIT. + +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slices + +// insertionSortLessFunc sorts data[a:b] using insertion sort. +func insertionSortLessFunc[E any](data []E, a, b int, less func(a, b E) bool) { + for i := a + 1; i < b; i++ { + for j := i; j > a && less(data[j], data[j-1]); j-- { + data[j], data[j-1] = data[j-1], data[j] + } + } +} + +// siftDownLessFunc implements the heap property on data[lo:hi]. +// first is an offset into the array where the root of the heap lies. +func siftDownLessFunc[E any](data []E, lo, hi, first int, less func(a, b E) bool) { + root := lo + for { + child := 2*root + 1 + if child >= hi { + break + } + if child+1 < hi && less(data[first+child], data[first+child+1]) { + child++ + } + if !less(data[first+root], data[first+child]) { + return + } + data[first+root], data[first+child] = data[first+child], data[first+root] + root = child + } +} + +func heapSortLessFunc[E any](data []E, a, b int, less func(a, b E) bool) { + first := a + lo := 0 + hi := b - a + + // Build heap with greatest element at top. + for i := (hi - 1) / 2; i >= 0; i-- { + siftDownLessFunc(data, i, hi, first, less) + } + + // Pop elements, largest first, into end of data. + for i := hi - 1; i >= 0; i-- { + data[first], data[first+i] = data[first+i], data[first] + siftDownLessFunc(data, lo, i, first, less) + } +} + +// pdqsortLessFunc sorts data[a:b]. +// The algorithm based on pattern-defeating quicksort(pdqsort), but without the optimizations from BlockQuicksort. +// pdqsort paper: https://arxiv.org/pdf/2106.05123.pdf +// C++ implementation: https://github.com/orlp/pdqsort +// Rust implementation: https://docs.rs/pdqsort/latest/pdqsort/ +// limit is the number of allowed bad (very unbalanced) pivots before falling back to heapsort. +func pdqsortLessFunc[E any](data []E, a, b, limit int, less func(a, b E) bool) { + const maxInsertion = 12 + + var ( + wasBalanced = true // whether the last partitioning was reasonably balanced + wasPartitioned = true // whether the slice was already partitioned + ) + + for { + length := b - a + + if length <= maxInsertion { + insertionSortLessFunc(data, a, b, less) + return + } + + // Fall back to heapsort if too many bad choices were made. + if limit == 0 { + heapSortLessFunc(data, a, b, less) + return + } + + // If the last partitioning was imbalanced, we need to breaking patterns. + if !wasBalanced { + breakPatternsLessFunc(data, a, b, less) + limit-- + } + + pivot, hint := choosePivotLessFunc(data, a, b, less) + if hint == decreasingHint { + reverseRangeLessFunc(data, a, b, less) + // The chosen pivot was pivot-a elements after the start of the array. + // After reversing it is pivot-a elements before the end of the array. + // The idea came from Rust's implementation. + pivot = (b - 1) - (pivot - a) + hint = increasingHint + } + + // The slice is likely already sorted. + if wasBalanced && wasPartitioned && hint == increasingHint { + if partialInsertionSortLessFunc(data, a, b, less) { + return + } + } + + // Probably the slice contains many duplicate elements, partition the slice into + // elements equal to and elements greater than the pivot. + if a > 0 && !less(data[a-1], data[pivot]) { + mid := partitionEqualLessFunc(data, a, b, pivot, less) + a = mid + continue + } + + mid, alreadyPartitioned := partitionLessFunc(data, a, b, pivot, less) + wasPartitioned = alreadyPartitioned + + leftLen, rightLen := mid-a, b-mid + balanceThreshold := length / 8 + if leftLen < rightLen { + wasBalanced = leftLen >= balanceThreshold + pdqsortLessFunc(data, a, mid, limit, less) + a = mid + 1 + } else { + wasBalanced = rightLen >= balanceThreshold + pdqsortLessFunc(data, mid+1, b, limit, less) + b = mid + } + } +} + +// partitionLessFunc does one quicksort partition. +// Let p = data[pivot] +// Moves elements in data[a:b] around, so that data[i]
=p for i =p for i