diff --git a/config/config.go b/config/config.go index 2033e70..c8861a1 100644 --- a/config/config.go +++ b/config/config.go @@ -444,9 +444,10 @@ func (d Destination) Equal(o Destination) bool { type Ruleset struct { SMTPMailFromRegexp string `sconf:"optional" sconf-doc:"Matches if this regular expression matches (a substring of) the SMTP MAIL FROM address (not the message From-header). E.g. '^user@example\\.org$'."` + MsgFromRegexp string `sconf:"optional" sconf-doc:"Matches if this regular expression matches (a substring of) the single address in the message From header."` VerifiedDomain string `sconf:"optional" sconf-doc:"Matches if this domain matches an SPF- and/or DKIM-verified (sub)domain."` HeadersRegexp map[string]string `sconf:"optional" sconf-doc:"Matches if these header field/value regular expressions all match (substrings of) the message headers. Header fields and valuees are converted to lower case before matching. Whitespace is trimmed from the value before matching. A header field can occur multiple times in a message, only one instance has to match. For mailing lists, you could match on ^list-id$ with the value typically the mailing list address in angled brackets with @ replaced with a dot, e.g. ."` - // todo: add a SMTPRcptTo check, and MessageFrom that works on a properly parsed From header. + // todo: add a SMTPRcptTo check // todo: once we implement ARC, we can use dkim domains that we cannot verify but that the arc-verified forwarding mail server was able to verify. IsForward bool `sconf:"optional" sconf-doc:"Influences spam filtering only, this option does not change whether a message matches this ruleset. Can only be used together with SMTPMailFromRegexp and VerifiedDomain. SMTPMailFromRegexp must be set to the address used to deliver the forwarded message, e.g. '^user(|\\+.*)@forward\\.example$'. Changes to junk analysis: 1. Messages are not rejected for failing a DMARC policy, because a legitimate forwarded message without valid/intact/aligned DKIM signature would be rejected because any verified SPF domain will be 'unaligned', of the forwarding mail server. 2. The sending mail server IP address, and sending EHLO and MAIL FROM domains and matching DKIM domain aren't used in future reputation-based spam classifications (but other verified DKIM domains are) because the forwarding server is not a useful spam signal for future messages."` @@ -454,8 +455,10 @@ type Ruleset struct { AcceptRejectsToMailbox string `sconf:"optional" sconf-doc:"Influences spam filtering only, this option does not change whether a message matches this ruleset. If a message is classified as spam, it isn't rejected during the SMTP transaction (the normal behaviour), but accepted during the SMTP transaction and delivered to the specified mailbox. The specified mailbox is not automatically cleaned up like the account global Rejects mailbox, unless set to that Rejects mailbox."` Mailbox string `sconf-doc:"Mailbox to deliver to if this ruleset matches."` + Comment string `sconf:"optional" sconf-doc:"Free-form comments."` SMTPMailFromRegexpCompiled *regexp.Regexp `sconf:"-" json:"-"` + MsgFromRegexpCompiled *regexp.Regexp `sconf:"-" json:"-"` VerifiedDNSDomain dns.Domain `sconf:"-"` HeadersRegexpCompiled [][2]*regexp.Regexp `sconf:"-" json:"-"` ListAllowDNSDomain dns.Domain `sconf:"-"` @@ -463,7 +466,7 @@ type Ruleset struct { // Equal returns whether r and o are equal, only looking at their user-changeable fields. func (r Ruleset) Equal(o Ruleset) bool { - if r.SMTPMailFromRegexp != o.SMTPMailFromRegexp || r.VerifiedDomain != o.VerifiedDomain || r.IsForward != o.IsForward || r.ListAllowDomain != o.ListAllowDomain || r.AcceptRejectsToMailbox != o.AcceptRejectsToMailbox || r.Mailbox != o.Mailbox { + if r.SMTPMailFromRegexp != o.SMTPMailFromRegexp || r.MsgFromRegexp != o.MsgFromRegexp || r.VerifiedDomain != o.VerifiedDomain || r.IsForward != o.IsForward || r.ListAllowDomain != o.ListAllowDomain || r.AcceptRejectsToMailbox != o.AcceptRejectsToMailbox || r.Mailbox != o.Mailbox || r.Comment != o.Comment { return false } if !reflect.DeepEqual(r.HeadersRegexp, o.HeadersRegexp) { diff --git a/config/doc.go b/config/doc.go index a72d499..964ee4e 100644 --- a/config/doc.go +++ b/config/doc.go @@ -998,6 +998,10 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details. # address (not the message From-header). E.g. '^user@example\.org$'. (optional) SMTPMailFromRegexp: + # Matches if this regular expression matches (a substring of) the single address + # in the message From header. (optional) + MsgFromRegexp: + # Matches if this domain matches an SPF- and/or DKIM-verified (sub)domain. # (optional) VerifiedDomain: @@ -1048,6 +1052,9 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details. # Mailbox to deliver to if this ruleset matches. Mailbox: + # Free-form comments. (optional) + Comment: + # Full name to use in message From header when composing messages coming from this # address with webmail. (optional) FullName: diff --git a/mox-/config.go b/mox-/config.go index 7770202..dd05484 100644 --- a/mox-/config.go +++ b/mox-/config.go @@ -1335,6 +1335,14 @@ func prepareDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string, } c.Accounts[accName].Destinations[addrName].Rulesets[i].SMTPMailFromRegexpCompiled = r } + if rs.MsgFromRegexp != "" { + n++ + r, err := regexp.Compile(rs.MsgFromRegexp) + if err != nil { + addErrorf("invalid MsgFrom regular expression: %v", err) + } + c.Accounts[accName].Destinations[addrName].Rulesets[i].MsgFromRegexpCompiled = r + } if rs.VerifiedDomain != "" { n++ d, err := dns.ParseDomain(rs.VerifiedDomain) diff --git a/store/account.go b/store/account.go index 4de7e71..78df8b4 100644 --- a/store/account.go +++ b/store/account.go @@ -757,6 +757,37 @@ type FromAddressSettings struct { ViewMode ViewMode } +// RulesetNoListID records a user "no" response to the question of +// creating/removing a ruleset after moving a message with list-id header from/to +// the inbox. +type RulesetNoListID struct { + ID int64 + RcptToAddress string `bstore:"nonzero"` + ListID string `bstore:"nonzero"` + ToInbox bool // Otherwise from Inbox to other mailbox. +} + +// RulesetNoMsgFrom records a user "no" response to the question of +// creating/moveing a ruleset after moving a mesage with message "from" address +// from/to the inbox. +type RulesetNoMsgFrom struct { + ID int64 + RcptToAddress string `bstore:"nonzero"` + MsgFromAddress string `bstore:"nonzero"` // Unicode. + ToInbox bool // Otherwise from Inbox to other mailbox. +} + +// RulesetNoMailbox represents a "never from/to this mailbox" response to the +// question of adding/removing a ruleset after moving a message. +type RulesetNoMailbox struct { + ID int64 + + // The mailbox from/to which the move has happened. + // Not a references, if mailbox is deleted, an entry becomes ineffective. + MailboxID int64 `bstore:"nonzero"` + ToMailbox bool // Whether MailboxID is the destination of the move (instead of source). +} + // Types stored in DB. var DBTypes = []any{ NextUIDValidity{}, @@ -774,6 +805,9 @@ var DBTypes = []any{ LoginSession{}, Settings{}, FromAddressSettings{}, + RulesetNoListID{}, + RulesetNoMsgFrom{}, + RulesetNoMailbox{}, } // Account holds the information about a user, includings mailboxes, messages, imap subscriptions. @@ -1758,7 +1792,7 @@ func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, erro return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil } -// MessageRuleset returns the first ruleset (if any) that message the message +// MessageRuleset returns the first ruleset (if any) that matches the message // represented by msgPrefix and msgFile, with smtp and validation fields from m. func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset { if len(dest.Rulesets) == 0 { @@ -1786,6 +1820,11 @@ ruleset: continue ruleset } } + if rs.MsgFromRegexpCompiled != nil { + if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) { + continue ruleset + } + } if !rs.VerifiedDNSDomain.IsZero() { d := rs.VerifiedDNSDomain.Name() diff --git a/testdata/webmail/domains.conf b/testdata/webmail/domains.conf index 342d6ad..e4aad0f 100644 --- a/testdata/webmail/domains.conf +++ b/testdata/webmail/domains.conf @@ -6,24 +6,26 @@ Domains: PrivateKeyFile: testsel.rsakey.pkcs8.pem Sign: - testsel + other.example: nil Accounts: + mjl: + Domain: mox.example + Destinations: + mjl@mox.example: nil + mox@other.example: nil + móx@mox.example: nil + møx@mox.example: nil + RejectsMailbox: Rejects + JunkFilter: + Threshold: 0.950000 + Params: + Twograms: true + MaxPower: 0.100000 + TopWords: 10 + IgnoreWords: 0.100000 + MaxOutgoingMessagesPerDay: 30 + MaxFirstTimeRecipientsPerDay: 10 other: Domain: mox.example Destinations: other@mox.example: nil - mjl: - MaxOutgoingMessagesPerDay: 30 - MaxFirstTimeRecipientsPerDay: 10 - Domain: mox.example - Destinations: - mjl@mox.example: nil - møx@mox.example: nil - móx@mox.example: nil - RejectsMailbox: Rejects - JunkFilter: - Threshold: 0.95 - Params: - Twograms: true - MaxPower: 0.1 - TopWords: 10 - IgnoreWords: 0.1 diff --git a/webaccount/account.js b/webaccount/account.js index 30bd6a9..e6ebc78 100644 --- a/webaccount/account.js +++ b/webaccount/account.js @@ -263,7 +263,7 @@ var api; "OutgoingWebhook": { "Name": "OutgoingWebhook", "Docs": "", "Fields": [{ "Name": "URL", "Docs": "", "Typewords": ["string"] }, { "Name": "Authorization", "Docs": "", "Typewords": ["string"] }, { "Name": "Events", "Docs": "", "Typewords": ["[]", "string"] }] }, "IncomingWebhook": { "Name": "IncomingWebhook", "Docs": "", "Fields": [{ "Name": "URL", "Docs": "", "Typewords": ["string"] }, { "Name": "Authorization", "Docs": "", "Typewords": ["string"] }] }, "Destination": { "Name": "Destination", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Rulesets", "Docs": "", "Typewords": ["[]", "Ruleset"] }, { "Name": "FullName", "Docs": "", "Typewords": ["string"] }] }, - "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, + "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, "Domain": { "Name": "Domain", "Docs": "", "Fields": [{ "Name": "ASCII", "Docs": "", "Typewords": ["string"] }, { "Name": "Unicode", "Docs": "", "Typewords": ["string"] }] }, "SubjectPass": { "Name": "SubjectPass", "Docs": "", "Fields": [{ "Name": "Period", "Docs": "", "Typewords": ["int64"] }] }, "AutomaticJunkFlags": { "Name": "AutomaticJunkFlags", "Docs": "", "Fields": [{ "Name": "Enabled", "Docs": "", "Typewords": ["bool"] }, { "Name": "JunkMailboxRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "NeutralMailboxRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "NotJunkMailboxRegexp", "Docs": "", "Typewords": ["string"] }] }, @@ -1601,24 +1601,28 @@ const destination = async (name) => { headersCell.appendChild(root); }; let smtpMailFromRegexp; + let msgFromRegexp; let verifiedDomain; let isForward; // Checkbox let listAllowDomain; let acceptRejectsToMailbox; let mailbox; - const root = dom.tr(dom.td(smtpMailFromRegexp = dom.input(attr.value(rs.SMTPMailFromRegexp || ''))), dom.td(verifiedDomain = dom.input(attr.value(rs.VerifiedDomain || ''))), headersCell, dom.td(dom.label(isForward = dom.input(attr.type('checkbox'), rs.IsForward ? attr.checked('') : []))), dom.td(listAllowDomain = dom.input(attr.value(rs.ListAllowDomain || ''))), dom.td(acceptRejectsToMailbox = dom.input(attr.value(rs.AcceptRejectsToMailbox || ''))), dom.td(mailbox = dom.input(attr.value(rs.Mailbox || ''))), dom.td(dom.clickbutton('Remove ruleset', function click() { + let comment; + const root = dom.tr(dom.td(smtpMailFromRegexp = dom.input(attr.value(rs.SMTPMailFromRegexp || ''))), dom.td(msgFromRegexp = dom.input(attr.value(rs.MsgFromRegexp || ''))), dom.td(verifiedDomain = dom.input(attr.value(rs.VerifiedDomain || ''))), headersCell, dom.td(dom.label(isForward = dom.input(attr.type('checkbox'), rs.IsForward ? attr.checked('') : []))), dom.td(listAllowDomain = dom.input(attr.value(rs.ListAllowDomain || ''))), dom.td(acceptRejectsToMailbox = dom.input(attr.value(rs.AcceptRejectsToMailbox || ''))), dom.td(mailbox = dom.input(attr.value(rs.Mailbox || ''))), dom.td(comment = dom.input(attr.value(rs.Comment || ''))), dom.td(dom.clickbutton('Remove ruleset', function click() { row.root.remove(); rulesetsRows = rulesetsRows.filter(e => e !== row); }))); row = { root: root, smtpMailFromRegexp: smtpMailFromRegexp, + msgFromRegexp: msgFromRegexp, verifiedDomain: verifiedDomain, headers: [], isForward: isForward, listAllowDomain: listAllowDomain, acceptRejectsToMailbox: acceptRejectsToMailbox, mailbox: mailbox, + comment: comment, }; rulesetsRows.push(row); Object.entries(rs.HeadersRegexp || {}).sort().forEach(t => addHeader(t[0], t[1])); @@ -1638,15 +1642,17 @@ const destination = async (name) => { let fullName; let saveButton; const addresses = [name, ...Object.keys(acc.Destinations || {}).filter(a => !a.startsWith('@') && a !== name)]; - dom._kids(page, crumbs(crumblink('Mox Account', '#'), 'Destination ' + name), dom.div(dom.span('Default mailbox', attr.title('Default mailbox where email for this recipient is delivered to if it does not match any ruleset. Default is Inbox.')), dom.br(), defaultMailbox = dom.input(attr.value(dest.Mailbox), attr.placeholder('Inbox'))), dom.br(), dom.div(dom.span('Full name', attr.title('Name to use in From header when composing messages. If not set, the account default full name is used.')), dom.br(), fullName = dom.input(attr.value(dest.FullName))), dom.br(), dom.h2('Rulesets'), dom.p('Incoming messages are checked against the rulesets. If a ruleset matches, the message is delivered to the mailbox configured for the ruleset instead of to the default mailbox.'), dom.p('"Is Forward" does not affect matching, but changes prevents the sending mail server from being included in future junk classifications by clearing fields related to the forwarding email server (IP address, EHLO domain, MAIL FROM domain and a matching DKIM domain), and prevents DMARC rejects for forwarded messages.'), dom.p('"List allow domain" does not affect matching, but skips the regular spam checks if one of the verified domains is a (sub)domain of the domain mentioned here.'), dom.p('"Accept rejects to mailbox" does not affect matching, but causes messages classified as junk to be accepted and delivered to this mailbox, instead of being rejected during the SMTP transaction. Useful for incoming forwarded messages where rejecting incoming messages may cause the forwarding server to stop forwarding.'), dom.table(dom.thead(dom.tr(dom.th('SMTP "MAIL FROM" regexp', attr.title('Matches if this regular expression matches (a substring of) the SMTP MAIL FROM address (not the message From-header). E.g. user@example.org.')), dom.th('Verified domain', attr.title('Matches if this domain matches an SPF- and/or DKIM-verified (sub)domain.')), dom.th('Headers regexp', attr.title('Matches if these header field/value regular expressions all match (substrings of) the message headers. Header fields and valuees are converted to lower case before matching. Whitespace is trimmed from the value before matching. A header field can occur multiple times in a message, only one instance has to match. For mailing lists, you could match on ^list-id$ with the value typically the mailing list address in angled brackets with @ replaced with a dot, e.g. .')), dom.th('Is Forward', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. Can only be used together with SMTPMailFromRegexp and VerifiedDomain. SMTPMailFromRegexp must be set to the address used to deliver the forwarded message, e.g. '^user(|\\+.*)@forward\\.example$'. Changes to junk analysis: 1. Messages are not rejected for failing a DMARC policy, because a legitimate forwarded message without valid/intact/aligned DKIM signature would be rejected because any verified SPF domain will be 'unaligned', of the forwarding mail server. 2. The sending mail server IP address, and sending EHLO and MAIL FROM domains and matching DKIM domain aren't used in future reputation-based spam classifications (but other verified DKIM domains are) because the forwarding server is not a useful spam signal for future messages.")), dom.th('List allow domain', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. If this domain matches an SPF- and/or DKIM-verified (sub)domain, the message is accepted without further spam checks, such as a junk filter or DMARC reject evaluation. DMARC rejects should not apply for mailing lists that are not configured to rewrite the From-header of messages that don't have a passing DKIM signature of the From-domain. Otherwise, by rejecting messages, you may be automatically unsubscribed from the mailing list. The assumption is that mailing lists do their own spam filtering/moderation.")), dom.th('Allow rejects to mailbox', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. If a message is classified as spam, it isn't rejected during the SMTP transaction (the normal behaviour), but accepted during the SMTP transaction and delivered to the specified mailbox. The specified mailbox is not automatically cleaned up like the account global Rejects mailbox, unless set to that Rejects mailbox.")), dom.th('Mailbox', attr.title('Mailbox to deliver to if this ruleset matches.')), dom.th('Action'))), rulesetsTbody, dom.tfoot(dom.tr(dom.td(attr.colspan('7')), dom.td(dom.clickbutton('Add ruleset', function click() { + dom._kids(page, crumbs(crumblink('Mox Account', '#'), 'Destination ' + name), dom.div(dom.span('Default mailbox', attr.title('Default mailbox where email for this recipient is delivered to if it does not match any ruleset. Default is Inbox.')), dom.br(), defaultMailbox = dom.input(attr.value(dest.Mailbox), attr.placeholder('Inbox'))), dom.br(), dom.div(dom.span('Full name', attr.title('Name to use in From header when composing messages. If not set, the account default full name is used.')), dom.br(), fullName = dom.input(attr.value(dest.FullName))), dom.br(), dom.h2('Rulesets'), dom.p('Incoming messages are checked against the rulesets. If a ruleset matches, the message is delivered to the mailbox configured for the ruleset instead of to the default mailbox.'), dom.p('"Is Forward" does not affect matching, but changes prevents the sending mail server from being included in future junk classifications by clearing fields related to the forwarding email server (IP address, EHLO domain, MAIL FROM domain and a matching DKIM domain), and prevents DMARC rejects for forwarded messages.'), dom.p('"List allow domain" does not affect matching, but skips the regular spam checks if one of the verified domains is a (sub)domain of the domain mentioned here.'), dom.p('"Accept rejects to mailbox" does not affect matching, but causes messages classified as junk to be accepted and delivered to this mailbox, instead of being rejected during the SMTP transaction. Useful for incoming forwarded messages where rejecting incoming messages may cause the forwarding server to stop forwarding.'), dom.table(dom.thead(dom.tr(dom.th('SMTP "MAIL FROM" regexp', attr.title('Matches if this regular expression matches (a substring of) the SMTP MAIL FROM address (not the message From-header). E.g. user@example.org.')), dom.th('Message "From" address regexp', attr.title('Matches if this regular expression matches (a substring of) the single address in the message From header.')), dom.th('Verified domain', attr.title('Matches if this domain matches an SPF- and/or DKIM-verified (sub)domain.')), dom.th('Headers regexp', attr.title('Matches if these header field/value regular expressions all match (substrings of) the message headers. Header fields and valuees are converted to lower case before matching. Whitespace is trimmed from the value before matching. A header field can occur multiple times in a message, only one instance has to match. For mailing lists, you could match on ^list-id$ with the value typically the mailing list address in angled brackets with @ replaced with a dot, e.g. .')), dom.th('Is Forward', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. Can only be used together with SMTPMailFromRegexp and VerifiedDomain. SMTPMailFromRegexp must be set to the address used to deliver the forwarded message, e.g. '^user(|\\+.*)@forward\\.example$'. Changes to junk analysis: 1. Messages are not rejected for failing a DMARC policy, because a legitimate forwarded message without valid/intact/aligned DKIM signature would be rejected because any verified SPF domain will be 'unaligned', of the forwarding mail server. 2. The sending mail server IP address, and sending EHLO and MAIL FROM domains and matching DKIM domain aren't used in future reputation-based spam classifications (but other verified DKIM domains are) because the forwarding server is not a useful spam signal for future messages.")), dom.th('List allow domain', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. If this domain matches an SPF- and/or DKIM-verified (sub)domain, the message is accepted without further spam checks, such as a junk filter or DMARC reject evaluation. DMARC rejects should not apply for mailing lists that are not configured to rewrite the From-header of messages that don't have a passing DKIM signature of the From-domain. Otherwise, by rejecting messages, you may be automatically unsubscribed from the mailing list. The assumption is that mailing lists do their own spam filtering/moderation.")), dom.th('Allow rejects to mailbox', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. If a message is classified as spam, it isn't rejected during the SMTP transaction (the normal behaviour), but accepted during the SMTP transaction and delivered to the specified mailbox. The specified mailbox is not automatically cleaned up like the account global Rejects mailbox, unless set to that Rejects mailbox.")), dom.th('Mailbox', attr.title('Mailbox to deliver to if this ruleset matches.')), dom.th('Comment', attr.title('Free-form comments.')), dom.th('Action'))), rulesetsTbody, dom.tfoot(dom.tr(dom.td(attr.colspan('9')), dom.td(dom.clickbutton('Add ruleset', function click() { addRulesetsRow({ SMTPMailFromRegexp: '', + MsgFromRegexp: '', VerifiedDomain: '', HeadersRegexp: {}, IsForward: false, ListAllowDomain: '', AcceptRejectsToMailbox: '', Mailbox: '', + Comment: '', VerifiedDNSDomain: { ASCII: '', Unicode: '' }, ListAllowDNSDomain: { ASCII: '', Unicode: '' }, }); @@ -1657,12 +1663,14 @@ const destination = async (name) => { Rulesets: rulesetsRows.map(row => { return { SMTPMailFromRegexp: row.smtpMailFromRegexp.value, + MsgFromRegexp: row.msgFromRegexp.value, VerifiedDomain: row.verifiedDomain.value, HeadersRegexp: Object.fromEntries(row.headers.map(h => [h.key.value, h.value.value])), IsForward: row.isForward.checked, ListAllowDomain: row.listAllowDomain.value, AcceptRejectsToMailbox: row.acceptRejectsToMailbox.value, Mailbox: row.mailbox.value, + Comment: row.comment.value, VerifiedDNSDomain: { ASCII: '', Unicode: '' }, ListAllowDNSDomain: { ASCII: '', Unicode: '' }, }; diff --git a/webaccount/account.ts b/webaccount/account.ts index 52cf0c8..7f279f1 100644 --- a/webaccount/account.ts +++ b/webaccount/account.ts @@ -1323,12 +1323,14 @@ const destination = async (name: string) => { root: HTMLElement smtpMailFromRegexp: HTMLInputElement + msgFromRegexp: HTMLInputElement verifiedDomain: HTMLInputElement headers: Header[] isForward: HTMLInputElement // Checkbox listAllowDomain: HTMLInputElement acceptRejectsToMailbox: HTMLInputElement mailbox: HTMLInputElement + comment: HTMLInputElement } let rulesetsTbody = dom.tbody() @@ -1370,20 +1372,24 @@ const destination = async (name: string) => { } let smtpMailFromRegexp: HTMLInputElement + let msgFromRegexp: HTMLInputElement let verifiedDomain: HTMLInputElement let isForward: HTMLInputElement // Checkbox let listAllowDomain: HTMLInputElement let acceptRejectsToMailbox: HTMLInputElement let mailbox: HTMLInputElement + let comment: HTMLInputElement const root = dom.tr( dom.td(smtpMailFromRegexp=dom.input(attr.value(rs.SMTPMailFromRegexp || ''))), + dom.td(msgFromRegexp=dom.input(attr.value(rs.MsgFromRegexp || ''))), dom.td(verifiedDomain=dom.input(attr.value(rs.VerifiedDomain || ''))), headersCell, dom.td(dom.label(isForward=dom.input(attr.type('checkbox'), rs.IsForward ? attr.checked('') : [] ))), dom.td(listAllowDomain=dom.input(attr.value(rs.ListAllowDomain || ''))), dom.td(acceptRejectsToMailbox=dom.input(attr.value(rs.AcceptRejectsToMailbox || ''))), dom.td(mailbox=dom.input(attr.value(rs.Mailbox || ''))), + dom.td(comment=dom.input(attr.value(rs.Comment || ''))), dom.td( dom.clickbutton('Remove ruleset', function click() { row.root.remove() @@ -1394,12 +1400,14 @@ const destination = async (name: string) => { row = { root: root, smtpMailFromRegexp: smtpMailFromRegexp, + msgFromRegexp: msgFromRegexp, verifiedDomain: verifiedDomain, headers: [], isForward: isForward, listAllowDomain: listAllowDomain, acceptRejectsToMailbox: acceptRejectsToMailbox, mailbox: mailbox, + comment: comment, } rulesetsRows.push(row) @@ -1454,29 +1462,33 @@ const destination = async (name: string) => { dom.thead( dom.tr( dom.th('SMTP "MAIL FROM" regexp', attr.title('Matches if this regular expression matches (a substring of) the SMTP MAIL FROM address (not the message From-header). E.g. user@example.org.')), + dom.th('Message "From" address regexp', attr.title('Matches if this regular expression matches (a substring of) the single address in the message From header.')), dom.th('Verified domain', attr.title('Matches if this domain matches an SPF- and/or DKIM-verified (sub)domain.')), dom.th('Headers regexp', attr.title('Matches if these header field/value regular expressions all match (substrings of) the message headers. Header fields and valuees are converted to lower case before matching. Whitespace is trimmed from the value before matching. A header field can occur multiple times in a message, only one instance has to match. For mailing lists, you could match on ^list-id$ with the value typically the mailing list address in angled brackets with @ replaced with a dot, e.g. .')), dom.th('Is Forward', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. Can only be used together with SMTPMailFromRegexp and VerifiedDomain. SMTPMailFromRegexp must be set to the address used to deliver the forwarded message, e.g. '^user(|\\+.*)@forward\\.example$'. Changes to junk analysis: 1. Messages are not rejected for failing a DMARC policy, because a legitimate forwarded message without valid/intact/aligned DKIM signature would be rejected because any verified SPF domain will be 'unaligned', of the forwarding mail server. 2. The sending mail server IP address, and sending EHLO and MAIL FROM domains and matching DKIM domain aren't used in future reputation-based spam classifications (but other verified DKIM domains are) because the forwarding server is not a useful spam signal for future messages.")), dom.th('List allow domain', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. If this domain matches an SPF- and/or DKIM-verified (sub)domain, the message is accepted without further spam checks, such as a junk filter or DMARC reject evaluation. DMARC rejects should not apply for mailing lists that are not configured to rewrite the From-header of messages that don't have a passing DKIM signature of the From-domain. Otherwise, by rejecting messages, you may be automatically unsubscribed from the mailing list. The assumption is that mailing lists do their own spam filtering/moderation.")), dom.th('Allow rejects to mailbox', attr.title("Influences spam filtering only, this option does not change whether a message matches this ruleset. If a message is classified as spam, it isn't rejected during the SMTP transaction (the normal behaviour), but accepted during the SMTP transaction and delivered to the specified mailbox. The specified mailbox is not automatically cleaned up like the account global Rejects mailbox, unless set to that Rejects mailbox.")), dom.th('Mailbox', attr.title('Mailbox to deliver to if this ruleset matches.')), + dom.th('Comment', attr.title('Free-form comments.')), dom.th('Action'), ) ), rulesetsTbody, dom.tfoot( dom.tr( - dom.td(attr.colspan('7')), + dom.td(attr.colspan('9')), dom.td( dom.clickbutton('Add ruleset', function click() { addRulesetsRow({ SMTPMailFromRegexp: '', + MsgFromRegexp: '', VerifiedDomain: '', HeadersRegexp: {}, IsForward: false, ListAllowDomain: '', AcceptRejectsToMailbox: '', Mailbox: '', + Comment: '', VerifiedDNSDomain: {ASCII: '', Unicode: ''}, ListAllowDNSDomain: {ASCII: '', Unicode: ''}, }) @@ -1493,12 +1505,14 @@ const destination = async (name: string) => { Rulesets: rulesetsRows.map(row => { return { SMTPMailFromRegexp: row.smtpMailFromRegexp.value, + MsgFromRegexp: row.msgFromRegexp.value, VerifiedDomain: row.verifiedDomain.value, HeadersRegexp: Object.fromEntries(row.headers.map(h => [h.key.value, h.value.value])), IsForward: row.isForward.checked, ListAllowDomain: row.listAllowDomain.value, AcceptRejectsToMailbox: row.acceptRejectsToMailbox.value, Mailbox: row.mailbox.value, + Comment: row.comment.value, VerifiedDNSDomain: {ASCII: '', Unicode: ''}, ListAllowDNSDomain: {ASCII: '', Unicode: ''}, } diff --git a/webaccount/api.json b/webaccount/api.json index 8df6ae4..bff2998 100644 --- a/webaccount/api.json +++ b/webaccount/api.json @@ -679,6 +679,13 @@ "string" ] }, + { + "Name": "MsgFromRegexp", + "Docs": "", + "Typewords": [ + "string" + ] + }, { "Name": "VerifiedDomain", "Docs": "", @@ -722,6 +729,13 @@ "string" ] }, + { + "Name": "Comment", + "Docs": "", + "Typewords": [ + "string" + ] + }, { "Name": "VerifiedDNSDomain", "Docs": "", diff --git a/webaccount/api.ts b/webaccount/api.ts index 25f8068..5aa396a 100644 --- a/webaccount/api.ts +++ b/webaccount/api.ts @@ -44,12 +44,14 @@ export interface Destination { export interface Ruleset { SMTPMailFromRegexp: string + MsgFromRegexp: string VerifiedDomain: string HeadersRegexp?: { [key: string]: string } IsForward: boolean // todo: once we implement ARC, we can use dkim domains that we cannot verify but that the arc-verified forwarding mail server was able to verify. ListAllowDomain: string AcceptRejectsToMailbox: string Mailbox: string + Comment: string VerifiedDNSDomain: Domain ListAllowDNSDomain: Domain } @@ -209,7 +211,7 @@ export const types: TypenameMap = { "OutgoingWebhook": {"Name":"OutgoingWebhook","Docs":"","Fields":[{"Name":"URL","Docs":"","Typewords":["string"]},{"Name":"Authorization","Docs":"","Typewords":["string"]},{"Name":"Events","Docs":"","Typewords":["[]","string"]}]}, "IncomingWebhook": {"Name":"IncomingWebhook","Docs":"","Fields":[{"Name":"URL","Docs":"","Typewords":["string"]},{"Name":"Authorization","Docs":"","Typewords":["string"]}]}, "Destination": {"Name":"Destination","Docs":"","Fields":[{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Rulesets","Docs":"","Typewords":["[]","Ruleset"]},{"Name":"FullName","Docs":"","Typewords":["string"]}]}, - "Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]}, + "Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"MsgFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Comment","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]}, "Domain": {"Name":"Domain","Docs":"","Fields":[{"Name":"ASCII","Docs":"","Typewords":["string"]},{"Name":"Unicode","Docs":"","Typewords":["string"]}]}, "SubjectPass": {"Name":"SubjectPass","Docs":"","Fields":[{"Name":"Period","Docs":"","Typewords":["int64"]}]}, "AutomaticJunkFlags": {"Name":"AutomaticJunkFlags","Docs":"","Fields":[{"Name":"Enabled","Docs":"","Typewords":["bool"]},{"Name":"JunkMailboxRegexp","Docs":"","Typewords":["string"]},{"Name":"NeutralMailboxRegexp","Docs":"","Typewords":["string"]},{"Name":"NotJunkMailboxRegexp","Docs":"","Typewords":["string"]}]}, diff --git a/webadmin/admin.js b/webadmin/admin.js index 2b28322..9a37b90 100644 --- a/webadmin/admin.js +++ b/webadmin/admin.js @@ -384,7 +384,7 @@ var api; "OutgoingWebhook": { "Name": "OutgoingWebhook", "Docs": "", "Fields": [{ "Name": "URL", "Docs": "", "Typewords": ["string"] }, { "Name": "Authorization", "Docs": "", "Typewords": ["string"] }, { "Name": "Events", "Docs": "", "Typewords": ["[]", "string"] }] }, "IncomingWebhook": { "Name": "IncomingWebhook", "Docs": "", "Fields": [{ "Name": "URL", "Docs": "", "Typewords": ["string"] }, { "Name": "Authorization", "Docs": "", "Typewords": ["string"] }] }, "Destination": { "Name": "Destination", "Docs": "", "Fields": [{ "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Rulesets", "Docs": "", "Typewords": ["[]", "Ruleset"] }, { "Name": "FullName", "Docs": "", "Typewords": ["string"] }] }, - "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, + "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, "SubjectPass": { "Name": "SubjectPass", "Docs": "", "Fields": [{ "Name": "Period", "Docs": "", "Typewords": ["int64"] }] }, "AutomaticJunkFlags": { "Name": "AutomaticJunkFlags", "Docs": "", "Fields": [{ "Name": "Enabled", "Docs": "", "Typewords": ["bool"] }, { "Name": "JunkMailboxRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "NeutralMailboxRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "NotJunkMailboxRegexp", "Docs": "", "Typewords": ["string"] }] }, "JunkFilter": { "Name": "JunkFilter", "Docs": "", "Fields": [{ "Name": "Threshold", "Docs": "", "Typewords": ["float64"] }, { "Name": "Onegrams", "Docs": "", "Typewords": ["bool"] }, { "Name": "Twograms", "Docs": "", "Typewords": ["bool"] }, { "Name": "Threegrams", "Docs": "", "Typewords": ["bool"] }, { "Name": "MaxPower", "Docs": "", "Typewords": ["float64"] }, { "Name": "TopWords", "Docs": "", "Typewords": ["int32"] }, { "Name": "IgnoreWords", "Docs": "", "Typewords": ["float64"] }, { "Name": "RareWords", "Docs": "", "Typewords": ["int32"] }] }, diff --git a/webadmin/api.json b/webadmin/api.json index 6a2a4d5..dd2c78d 100644 --- a/webadmin/api.json +++ b/webadmin/api.json @@ -3772,6 +3772,13 @@ "string" ] }, + { + "Name": "MsgFromRegexp", + "Docs": "", + "Typewords": [ + "string" + ] + }, { "Name": "VerifiedDomain", "Docs": "", @@ -3815,6 +3822,13 @@ "string" ] }, + { + "Name": "Comment", + "Docs": "", + "Typewords": [ + "string" + ] + }, { "Name": "VerifiedDNSDomain", "Docs": "", diff --git a/webadmin/api.ts b/webadmin/api.ts index 399bad7..69303ba 100644 --- a/webadmin/api.ts +++ b/webadmin/api.ts @@ -375,12 +375,14 @@ export interface Destination { export interface Ruleset { SMTPMailFromRegexp: string + MsgFromRegexp: string VerifiedDomain: string HeadersRegexp?: { [key: string]: string } IsForward: boolean // todo: once we implement ARC, we can use dkim domains that we cannot verify but that the arc-verified forwarding mail server was able to verify. ListAllowDomain: string AcceptRejectsToMailbox: string Mailbox: string + Comment: string VerifiedDNSDomain: Domain ListAllowDNSDomain: Domain } @@ -1191,7 +1193,7 @@ export const types: TypenameMap = { "OutgoingWebhook": {"Name":"OutgoingWebhook","Docs":"","Fields":[{"Name":"URL","Docs":"","Typewords":["string"]},{"Name":"Authorization","Docs":"","Typewords":["string"]},{"Name":"Events","Docs":"","Typewords":["[]","string"]}]}, "IncomingWebhook": {"Name":"IncomingWebhook","Docs":"","Fields":[{"Name":"URL","Docs":"","Typewords":["string"]},{"Name":"Authorization","Docs":"","Typewords":["string"]}]}, "Destination": {"Name":"Destination","Docs":"","Fields":[{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Rulesets","Docs":"","Typewords":["[]","Ruleset"]},{"Name":"FullName","Docs":"","Typewords":["string"]}]}, - "Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]}, + "Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"MsgFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Comment","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]}, "SubjectPass": {"Name":"SubjectPass","Docs":"","Fields":[{"Name":"Period","Docs":"","Typewords":["int64"]}]}, "AutomaticJunkFlags": {"Name":"AutomaticJunkFlags","Docs":"","Fields":[{"Name":"Enabled","Docs":"","Typewords":["bool"]},{"Name":"JunkMailboxRegexp","Docs":"","Typewords":["string"]},{"Name":"NeutralMailboxRegexp","Docs":"","Typewords":["string"]},{"Name":"NotJunkMailboxRegexp","Docs":"","Typewords":["string"]}]}, "JunkFilter": {"Name":"JunkFilter","Docs":"","Fields":[{"Name":"Threshold","Docs":"","Typewords":["float64"]},{"Name":"Onegrams","Docs":"","Typewords":["bool"]},{"Name":"Twograms","Docs":"","Typewords":["bool"]},{"Name":"Threegrams","Docs":"","Typewords":["bool"]},{"Name":"MaxPower","Docs":"","Typewords":["float64"]},{"Name":"TopWords","Docs":"","Typewords":["int32"]},{"Name":"IgnoreWords","Docs":"","Typewords":["float64"]},{"Name":"RareWords","Docs":"","Typewords":["int32"]}]}, diff --git a/webmail/api.go b/webmail/api.go index 62f50ca..2715c4b 100644 --- a/webmail/api.go +++ b/webmail/api.go @@ -16,8 +16,10 @@ import ( "net/mail" "net/textproto" "os" + "regexp" "runtime/debug" "slices" + "sort" "strings" "sync" "time" @@ -31,10 +33,12 @@ import ( "github.com/mjl-/sherpadoc" "github.com/mjl-/sherpaprom" + "github.com/mjl-/mox/config" "github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/message" "github.com/mjl-/mox/metrics" + "github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mox-" "github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxvar" @@ -1904,6 +1908,254 @@ func (Webmail) SettingsSave(ctx context.Context, settings store.Settings) { xcheckf(ctx, err, "save settings") } +func (Webmail) RulesetSuggestMove(ctx context.Context, msgID, mbSrcID, mbDstID int64) (listID string, msgFrom string, isRemove bool, rcptTo string, ruleset *config.Ruleset) { + withAccount(ctx, func(log mlog.Log, acc *store.Account) { + xdbread(ctx, acc, func(tx *bstore.Tx) { + m := xmessageID(ctx, tx, msgID) + mbSrc := xmailboxID(ctx, tx, mbSrcID) + mbDst := xmailboxID(ctx, tx, mbDstID) + + if m.RcptToLocalpart == "" && m.RcptToDomain == "" { + return + } + rcptTo = m.RcptToLocalpart.String() + "@" + m.RcptToDomain + + conf, _ := acc.Conf() + dest := conf.Destinations[rcptTo] // May not be present. + defaultMailbox := "Inbox" + if dest.Mailbox != "" { + defaultMailbox = dest.Mailbox + } + + // Only suggest rules for messages moved into/out of the default mailbox (Inbox). + if mbSrc.Name != defaultMailbox && mbDst.Name != defaultMailbox { + return + } + + // Check if we have a previous answer "No" answer for moving from/to mailbox. + exists, err := bstore.QueryTx[store.RulesetNoMailbox](tx).FilterNonzero(store.RulesetNoMailbox{MailboxID: mbSrcID}).FilterEqual("ToMailbox", false).Exists() + xcheckf(ctx, err, "looking up previous response for source mailbox") + if exists { + return + } + exists, err = bstore.QueryTx[store.RulesetNoMailbox](tx).FilterNonzero(store.RulesetNoMailbox{MailboxID: mbDstID}).FilterEqual("ToMailbox", true).Exists() + xcheckf(ctx, err, "looking up previous response for destination mailbox") + if exists { + return + } + + // Parse message for List-Id header. + state := msgState{acc: acc} + defer state.clear() + pm, err := parsedMessage(log, m, &state, true, false) + xcheckf(ctx, err, "parsing message") + + // The suggested ruleset. Once all is checked, we'll return it. + var nrs *config.Ruleset + + // If List-Id header is present, we'll treat it as a (mailing) list message. + if l, ok := pm.Headers["List-Id"]; ok { + if len(l) != 1 { + log.Debug("not exactly one list-id header", slog.Any("listid", l)) + return + } + var listIDDom dns.Domain + listID, listIDDom = parseListID(l[0]) + if listID == "" { + log.Debug("invalid list-id header", slog.String("listid", l[0])) + return + } + + // Check if we have a previous "No" answer for this list-id. + no := store.RulesetNoListID{ + RcptToAddress: rcptTo, + ListID: listID, + ToInbox: mbDst.Name == "Inbox", + } + exists, err = bstore.QueryTx[store.RulesetNoListID](tx).FilterNonzero(no).Exists() + xcheckf(ctx, err, "looking up previous response for list-id") + if exists { + return + } + + // Find the "ListAllowDomain" to use. We only match and move messages with verified + // SPF/DKIM. Otherwise spammers could add a list-id headers for mailing lists you + // are subscribed to, and take advantage of any reduced junk filtering. + listIDDomStr := listIDDom.Name() + + doms := m.DKIMDomains + if m.MailFromValidated { + doms = append(doms, m.MailFromDomain) + } + // Sort, we prefer the shortest name, e.g. DKIM signature on whole domain instead + // of SPF verification of one host. + sort.Slice(doms, func(i, j int) bool { + return len(doms[i]) < len(doms[j]) + }) + var listAllowDom string + for _, dom := range doms { + if dom == listIDDomStr || strings.HasSuffix(listIDDomStr, "."+dom) { + listAllowDom = dom + break + } + } + if listAllowDom == "" { + return + } + + listIDRegExp := regexp.QuoteMeta(fmt.Sprintf("<%s>", listID)) + "$" + nrs = &config.Ruleset{ + HeadersRegexp: map[string]string{"^list-id$": listIDRegExp}, + ListAllowDomain: listAllowDom, + Mailbox: mbDst.Name, + } + } else { + // Otherwise, try to make a rule based on message "From" address. + if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" { + return + } + msgFrom = m.MsgFromLocalpart.String() + "@" + m.MsgFromDomain + + no := store.RulesetNoMsgFrom{ + RcptToAddress: rcptTo, + MsgFromAddress: msgFrom, + ToInbox: mbDst.Name == "Inbox", + } + exists, err = bstore.QueryTx[store.RulesetNoMsgFrom](tx).FilterNonzero(no).Exists() + xcheckf(ctx, err, "looking up previous response for message from address") + if exists { + return + } + + nrs = &config.Ruleset{ + MsgFromRegexp: "^" + regexp.QuoteMeta(msgFrom) + "$", + Mailbox: mbDst.Name, + } + } + + // Only suggest adding/removing rule if it isn't/is present. + var have bool + for _, rs := range dest.Rulesets { + xrs := config.Ruleset{ + MsgFromRegexp: rs.MsgFromRegexp, + HeadersRegexp: rs.HeadersRegexp, + ListAllowDomain: rs.ListAllowDomain, + Mailbox: nrs.Mailbox, + } + if xrs.Equal(*nrs) { + have = true + break + } + } + isRemove = mbDst.Name == defaultMailbox + if isRemove { + nrs.Mailbox = mbSrc.Name + } + if isRemove && !have || !isRemove && have { + return + } + + // We'll be returning a suggested ruleset. + nrs.Comment = "by webmail on " + time.Now().Format("2006-01-02") + ruleset = nrs + }) + }) + return +} + +// Parse the list-id value (the value between <>) from a list-id header. +// Returns an empty string if it couldn't be parsed. +func parseListID(s string) (listID string, dom dns.Domain) { + // ../rfc/2919:198 + s = strings.TrimRight(s, " \t") + if !strings.HasSuffix(s, ">") { + return "", dns.Domain{} + } + s = s[:len(s)-1] + t := strings.Split(s, "<") + if len(t) == 1 { + return "", dns.Domain{} + } + s = t[len(t)-1] + dom, err := dns.ParseDomain(s) + if err != nil { + return "", dom + } + return s, dom +} + +func (Webmail) RulesetAdd(ctx context.Context, rcptTo string, ruleset config.Ruleset) { + reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) + + err := mox.AccountSave(ctx, reqInfo.AccountName, func(acc *config.Account) { + dest, ok := acc.Destinations[rcptTo] + if !ok { + // todo: we could find the catchall address and add the rule, or add the address explicitly. + xcheckuserf(ctx, errors.New("destination address not found in account (hint: if this is a catchall address, configure the address explicitly to configure rulesets)"), "looking up address") + } + + nd := map[string]config.Destination{} + for addr, d := range acc.Destinations { + nd[addr] = d + } + dest.Rulesets = append(slices.Clone(dest.Rulesets), ruleset) + nd[rcptTo] = dest + acc.Destinations = nd + }) + xcheckf(ctx, err, "saving account with new ruleset") +} + +func (Webmail) RulesetRemove(ctx context.Context, rcptTo string, ruleset config.Ruleset) { + reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) + + err := mox.AccountSave(ctx, reqInfo.AccountName, func(acc *config.Account) { + dest, ok := acc.Destinations[rcptTo] + if !ok { + xcheckuserf(ctx, errors.New("destination address not found in account"), "looking up address") + } + + nd := map[string]config.Destination{} + for addr, d := range acc.Destinations { + nd[addr] = d + } + var l []config.Ruleset + skipped := 0 + for _, rs := range dest.Rulesets { + if rs.Equal(ruleset) { + skipped++ + } else { + l = append(l, rs) + } + } + if skipped != 1 { + xcheckuserf(ctx, fmt.Errorf("affected %d configured rulesets, expected 1", skipped), "changing rulesets") + } + dest.Rulesets = l + nd[rcptTo] = dest + acc.Destinations = nd + }) + xcheckf(ctx, err, "saving account with new ruleset") +} + +func (Webmail) RulesetMessageNever(ctx context.Context, rcptTo, listID, msgFrom string, toInbox bool) { + withAccount(ctx, func(log mlog.Log, acc *store.Account) { + var err error + if listID != "" { + err = acc.DB.Insert(ctx, &store.RulesetNoListID{RcptToAddress: rcptTo, ListID: listID, ToInbox: toInbox}) + } else { + err = acc.DB.Insert(ctx, &store.RulesetNoMsgFrom{RcptToAddress: rcptTo, MsgFromAddress: msgFrom, ToInbox: toInbox}) + } + xcheckf(ctx, err, "storing user response") + }) +} + +func (Webmail) RulesetMailboxNever(ctx context.Context, mailboxID int64, toMailbox bool) { + withAccount(ctx, func(log mlog.Log, acc *store.Account) { + err := acc.DB.Insert(ctx, &store.RulesetNoMailbox{MailboxID: mailboxID, ToMailbox: toMailbox}) + xcheckf(ctx, err, "storing user response") + }) +} + func slicesAny[T any](l []T) []any { r := make([]any, len(l)) for i, v := range l { @@ -1912,6 +2164,18 @@ func slicesAny[T any](l []T) []any { return r } +func withAccount(ctx context.Context, fn func(log mlog.Log, acc *store.Account)) { + log := pkglog.WithContext(ctx) + reqInfo := ctx.Value(requestInfoCtxKey).(requestInfo) + acc, err := store.OpenAccount(log, reqInfo.AccountName) + xcheckf(ctx, err, "open account") + defer func() { + err := acc.Close() + log.Check(err, "closing account") + }() + fn(log, acc) +} + // SSETypes exists to ensure the generated API contains the types, for use in SSE events. func (Webmail) SSETypes() (start EventStart, viewErr EventViewErr, viewReset EventViewReset, viewMsgs EventViewMsgs, viewChanges EventViewChanges, msgAdd ChangeMsgAdd, msgRemove ChangeMsgRemove, msgFlags ChangeMsgFlags, msgThread ChangeMsgThread, mailboxRemove ChangeMailboxRemove, mailboxAdd ChangeMailboxAdd, mailboxRename ChangeMailboxRename, mailboxCounts ChangeMailboxCounts, mailboxSpecialUse ChangeMailboxSpecialUse, mailboxKeywords ChangeMailboxKeywords, flags store.Flags) { return diff --git a/webmail/api.json b/webmail/api.json index 68ec1ba..da298b8 100644 --- a/webmail/api.json +++ b/webmail/api.json @@ -438,6 +438,151 @@ ], "Returns": [] }, + { + "Name": "RulesetSuggestMove", + "Docs": "", + "Params": [ + { + "Name": "msgID", + "Typewords": [ + "int64" + ] + }, + { + "Name": "mbSrcID", + "Typewords": [ + "int64" + ] + }, + { + "Name": "mbDstID", + "Typewords": [ + "int64" + ] + } + ], + "Returns": [ + { + "Name": "listID", + "Typewords": [ + "string" + ] + }, + { + "Name": "msgFrom", + "Typewords": [ + "string" + ] + }, + { + "Name": "isRemove", + "Typewords": [ + "bool" + ] + }, + { + "Name": "rcptTo", + "Typewords": [ + "string" + ] + }, + { + "Name": "ruleset", + "Typewords": [ + "nullable", + "Ruleset" + ] + } + ] + }, + { + "Name": "RulesetAdd", + "Docs": "", + "Params": [ + { + "Name": "rcptTo", + "Typewords": [ + "string" + ] + }, + { + "Name": "ruleset", + "Typewords": [ + "Ruleset" + ] + } + ], + "Returns": [] + }, + { + "Name": "RulesetRemove", + "Docs": "", + "Params": [ + { + "Name": "rcptTo", + "Typewords": [ + "string" + ] + }, + { + "Name": "ruleset", + "Typewords": [ + "Ruleset" + ] + } + ], + "Returns": [] + }, + { + "Name": "RulesetMessageNever", + "Docs": "", + "Params": [ + { + "Name": "rcptTo", + "Typewords": [ + "string" + ] + }, + { + "Name": "listID", + "Typewords": [ + "string" + ] + }, + { + "Name": "msgFrom", + "Typewords": [ + "string" + ] + }, + { + "Name": "toInbox", + "Typewords": [ + "bool" + ] + } + ], + "Returns": [] + }, + { + "Name": "RulesetMailboxNever", + "Docs": "", + "Params": [ + { + "Name": "mailboxID", + "Typewords": [ + "int64" + ] + }, + { + "Name": "toMailbox", + "Typewords": [ + "bool" + ] + } + ], + "Returns": [] + }, { "Name": "SSETypes", "Docs": "SSETypes exists to ensure the generated API contains the types, for use in SSE events.", @@ -1601,6 +1746,90 @@ } ] }, + { + "Name": "Ruleset", + "Docs": "", + "Fields": [ + { + "Name": "SMTPMailFromRegexp", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "MsgFromRegexp", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "VerifiedDomain", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "HeadersRegexp", + "Docs": "", + "Typewords": [ + "{}", + "string" + ] + }, + { + "Name": "IsForward", + "Docs": "todo: once we implement ARC, we can use dkim domains that we cannot verify but that the arc-verified forwarding mail server was able to verify.", + "Typewords": [ + "bool" + ] + }, + { + "Name": "ListAllowDomain", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "AcceptRejectsToMailbox", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "Mailbox", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "Comment", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "VerifiedDNSDomain", + "Docs": "", + "Typewords": [ + "Domain" + ] + }, + { + "Name": "ListAllowDNSDomain", + "Docs": "", + "Typewords": [ + "Domain" + ] + } + ] + }, { "Name": "EventStart", "Docs": "EventStart is the first message sent on an SSE connection, giving the client\nbasic data to populate its UI. After this event, messages will follow quickly in\nan EventViewMsgs event.", diff --git a/webmail/api.ts b/webmail/api.ts index 9d7fb9b..6481d93 100644 --- a/webmail/api.ts +++ b/webmail/api.ts @@ -220,6 +220,20 @@ export interface Settings { ShowAddressSecurity: boolean // Whether to show the bars underneath the address input fields indicating starttls/dnssec/dane/mtasts/requiretls support by address. } +export interface Ruleset { + SMTPMailFromRegexp: string + MsgFromRegexp: string + VerifiedDomain: string + HeadersRegexp?: { [key: string]: string } + IsForward: boolean // todo: once we implement ARC, we can use dkim domains that we cannot verify but that the arc-verified forwarding mail server was able to verify. + ListAllowDomain: string + AcceptRejectsToMailbox: string + Mailbox: string + Comment: string + VerifiedDNSDomain: Domain + ListAllowDNSDomain: Domain +} + // EventStart is the first message sent on an SSE connection, giving the client // basic data to populate its UI. After this event, messages will follow quickly in // an EventViewMsgs event. @@ -567,7 +581,7 @@ export enum Quoting { // Localparts are in Unicode NFC. export type Localpart = string -export const structTypes: {[typename: string]: boolean} = {"Address":true,"Attachment":true,"ChangeMailboxAdd":true,"ChangeMailboxCounts":true,"ChangeMailboxKeywords":true,"ChangeMailboxRemove":true,"ChangeMailboxRename":true,"ChangeMailboxSpecialUse":true,"ChangeMsgAdd":true,"ChangeMsgFlags":true,"ChangeMsgRemove":true,"ChangeMsgThread":true,"ComposeMessage":true,"Domain":true,"DomainAddressConfig":true,"Envelope":true,"EventStart":true,"EventViewChanges":true,"EventViewErr":true,"EventViewMsgs":true,"EventViewReset":true,"File":true,"Filter":true,"Flags":true,"ForwardAttachments":true,"FromAddressSettings":true,"Mailbox":true,"Message":true,"MessageAddress":true,"MessageEnvelope":true,"MessageItem":true,"NotFilter":true,"Page":true,"ParsedMessage":true,"Part":true,"Query":true,"RecipientSecurity":true,"Request":true,"Settings":true,"SpecialUse":true,"SubmitMessage":true} +export const structTypes: {[typename: string]: boolean} = {"Address":true,"Attachment":true,"ChangeMailboxAdd":true,"ChangeMailboxCounts":true,"ChangeMailboxKeywords":true,"ChangeMailboxRemove":true,"ChangeMailboxRename":true,"ChangeMailboxSpecialUse":true,"ChangeMsgAdd":true,"ChangeMsgFlags":true,"ChangeMsgRemove":true,"ChangeMsgThread":true,"ComposeMessage":true,"Domain":true,"DomainAddressConfig":true,"Envelope":true,"EventStart":true,"EventViewChanges":true,"EventViewErr":true,"EventViewMsgs":true,"EventViewReset":true,"File":true,"Filter":true,"Flags":true,"ForwardAttachments":true,"FromAddressSettings":true,"Mailbox":true,"Message":true,"MessageAddress":true,"MessageEnvelope":true,"MessageItem":true,"NotFilter":true,"Page":true,"ParsedMessage":true,"Part":true,"Query":true,"RecipientSecurity":true,"Request":true,"Ruleset":true,"Settings":true,"SpecialUse":true,"SubmitMessage":true} export const stringsTypes: {[typename: string]: boolean} = {"AttachmentType":true,"CSRFToken":true,"Localpart":true,"Quoting":true,"SecurityResult":true,"ThreadMode":true,"ViewMode":true} export const intsTypes: {[typename: string]: boolean} = {"ModSeq":true,"UID":true,"Validation":true} export const types: TypenameMap = { @@ -590,6 +604,7 @@ export const types: TypenameMap = { "Mailbox": {"Name":"Mailbox","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["int64"]},{"Name":"Name","Docs":"","Typewords":["string"]},{"Name":"UIDValidity","Docs":"","Typewords":["uint32"]},{"Name":"UIDNext","Docs":"","Typewords":["UID"]},{"Name":"Archive","Docs":"","Typewords":["bool"]},{"Name":"Draft","Docs":"","Typewords":["bool"]},{"Name":"Junk","Docs":"","Typewords":["bool"]},{"Name":"Sent","Docs":"","Typewords":["bool"]},{"Name":"Trash","Docs":"","Typewords":["bool"]},{"Name":"Keywords","Docs":"","Typewords":["[]","string"]},{"Name":"HaveCounts","Docs":"","Typewords":["bool"]},{"Name":"Total","Docs":"","Typewords":["int64"]},{"Name":"Deleted","Docs":"","Typewords":["int64"]},{"Name":"Unread","Docs":"","Typewords":["int64"]},{"Name":"Unseen","Docs":"","Typewords":["int64"]},{"Name":"Size","Docs":"","Typewords":["int64"]}]}, "RecipientSecurity": {"Name":"RecipientSecurity","Docs":"","Fields":[{"Name":"STARTTLS","Docs":"","Typewords":["SecurityResult"]},{"Name":"MTASTS","Docs":"","Typewords":["SecurityResult"]},{"Name":"DNSSEC","Docs":"","Typewords":["SecurityResult"]},{"Name":"DANE","Docs":"","Typewords":["SecurityResult"]},{"Name":"RequireTLS","Docs":"","Typewords":["SecurityResult"]}]}, "Settings": {"Name":"Settings","Docs":"","Fields":[{"Name":"ID","Docs":"","Typewords":["uint8"]},{"Name":"Signature","Docs":"","Typewords":["string"]},{"Name":"Quoting","Docs":"","Typewords":["Quoting"]},{"Name":"ShowAddressSecurity","Docs":"","Typewords":["bool"]}]}, + "Ruleset": {"Name":"Ruleset","Docs":"","Fields":[{"Name":"SMTPMailFromRegexp","Docs":"","Typewords":["string"]},{"Name":"MsgFromRegexp","Docs":"","Typewords":["string"]},{"Name":"VerifiedDomain","Docs":"","Typewords":["string"]},{"Name":"HeadersRegexp","Docs":"","Typewords":["{}","string"]},{"Name":"IsForward","Docs":"","Typewords":["bool"]},{"Name":"ListAllowDomain","Docs":"","Typewords":["string"]},{"Name":"AcceptRejectsToMailbox","Docs":"","Typewords":["string"]},{"Name":"Mailbox","Docs":"","Typewords":["string"]},{"Name":"Comment","Docs":"","Typewords":["string"]},{"Name":"VerifiedDNSDomain","Docs":"","Typewords":["Domain"]},{"Name":"ListAllowDNSDomain","Docs":"","Typewords":["Domain"]}]}, "EventStart": {"Name":"EventStart","Docs":"","Fields":[{"Name":"SSEID","Docs":"","Typewords":["int64"]},{"Name":"LoginAddress","Docs":"","Typewords":["MessageAddress"]},{"Name":"Addresses","Docs":"","Typewords":["[]","MessageAddress"]},{"Name":"DomainAddressConfigs","Docs":"","Typewords":["{}","DomainAddressConfig"]},{"Name":"MailboxName","Docs":"","Typewords":["string"]},{"Name":"Mailboxes","Docs":"","Typewords":["[]","Mailbox"]},{"Name":"RejectsMailbox","Docs":"","Typewords":["string"]},{"Name":"Settings","Docs":"","Typewords":["Settings"]},{"Name":"AccountPath","Docs":"","Typewords":["string"]},{"Name":"Version","Docs":"","Typewords":["string"]}]}, "DomainAddressConfig": {"Name":"DomainAddressConfig","Docs":"","Fields":[{"Name":"LocalpartCatchallSeparator","Docs":"","Typewords":["string"]},{"Name":"LocalpartCaseSensitive","Docs":"","Typewords":["bool"]}]}, "EventViewErr": {"Name":"EventViewErr","Docs":"","Fields":[{"Name":"ViewID","Docs":"","Typewords":["int64"]},{"Name":"RequestID","Docs":"","Typewords":["int64"]},{"Name":"Err","Docs":"","Typewords":["string"]}]}, @@ -644,6 +659,7 @@ export const parser = { Mailbox: (v: any) => parse("Mailbox", v) as Mailbox, RecipientSecurity: (v: any) => parse("RecipientSecurity", v) as RecipientSecurity, Settings: (v: any) => parse("Settings", v) as Settings, + Ruleset: (v: any) => parse("Ruleset", v) as Ruleset, EventStart: (v: any) => parse("EventStart", v) as EventStart, DomainAddressConfig: (v: any) => parse("DomainAddressConfig", v) as DomainAddressConfig, EventViewErr: (v: any) => parse("EventViewErr", v) as EventViewErr, @@ -959,6 +975,46 @@ export class Client { return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void } + async RulesetSuggestMove(msgID: number, mbSrcID: number, mbDstID: number): Promise<[string, string, boolean, string, Ruleset | null]> { + const fn: string = "RulesetSuggestMove" + const paramTypes: string[][] = [["int64"],["int64"],["int64"]] + const returnTypes: string[][] = [["string"],["string"],["bool"],["string"],["nullable","Ruleset"]] + const params: any[] = [msgID, mbSrcID, mbDstID] + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as [string, string, boolean, string, Ruleset | null] + } + + async RulesetAdd(rcptTo: string, ruleset: Ruleset): Promise { + const fn: string = "RulesetAdd" + const paramTypes: string[][] = [["string"],["Ruleset"]] + const returnTypes: string[][] = [] + const params: any[] = [rcptTo, ruleset] + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void + } + + async RulesetRemove(rcptTo: string, ruleset: Ruleset): Promise { + const fn: string = "RulesetRemove" + const paramTypes: string[][] = [["string"],["Ruleset"]] + const returnTypes: string[][] = [] + const params: any[] = [rcptTo, ruleset] + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void + } + + async RulesetMessageNever(rcptTo: string, listID: string, msgFrom: string, toInbox: boolean): Promise { + const fn: string = "RulesetMessageNever" + const paramTypes: string[][] = [["string"],["string"],["string"],["bool"]] + const returnTypes: string[][] = [] + const params: any[] = [rcptTo, listID, msgFrom, toInbox] + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void + } + + async RulesetMailboxNever(mailboxID: number, toMailbox: boolean): Promise { + const fn: string = "RulesetMailboxNever" + const paramTypes: string[][] = [["int64"],["bool"]] + const returnTypes: string[][] = [] + const params: any[] = [mailboxID, toMailbox] + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params) as void + } + // SSETypes exists to ensure the generated API contains the types, for use in SSE events. async SSETypes(): Promise<[EventStart, EventViewErr, EventViewReset, EventViewMsgs, EventViewChanges, ChangeMsgAdd, ChangeMsgRemove, ChangeMsgFlags, ChangeMsgThread, ChangeMailboxRemove, ChangeMailboxAdd, ChangeMailboxRename, ChangeMailboxCounts, ChangeMailboxSpecialUse, ChangeMailboxKeywords, Flags]> { const fn: string = "SSETypes" diff --git a/webmail/api_test.go b/webmail/api_test.go index e1ced10..0f998f3 100644 --- a/webmail/api_test.go +++ b/webmail/api_test.go @@ -53,6 +53,7 @@ func TestAPI(t *testing.T) { os.RemoveAll("../testdata/webmail/data") mox.Context = ctxbg mox.ConfigStaticPath = filepath.FromSlash("../testdata/webmail/mox.conf") + mox.ConfigDynamicPath = filepath.FromSlash("../testdata/webmail/domains.conf") mox.MustLoadConfig(true, false) defer store.Switchboard()() @@ -469,4 +470,65 @@ func TestAPI(t *testing.T) { rs, err = recipientSecurity(ctx, resolver, "mjl@a.mox.example") tcompare(t, err, nil) tcompare(t, rs, RecipientSecurity{SecurityResultYes, SecurityResultNo, SecurityResultNo, SecurityResultNo, SecurityResultNo}) + + // Suggesting/adding/removing rulesets. + + testSuggest := func(msgID int64, expListID string, expMsgFrom string) { + listID, msgFrom, isRemove, rcptTo, ruleset := api.RulesetSuggestMove(ctx, msgID, inbox.ID, testbox1.ID) + tcompare(t, listID, expListID) + tcompare(t, msgFrom, expMsgFrom) + tcompare(t, isRemove, false) + tcompare(t, rcptTo, "mox@other.example") + tcompare(t, ruleset == nil, false) + + // Moving in opposite direction doesn't get a suggestion without the rule present. + _, _, _, _, rs0 := api.RulesetSuggestMove(ctx, msgID, testbox1.ID, inbox.ID) + tcompare(t, rs0 == nil, true) + + api.RulesetAdd(ctx, rcptTo, *ruleset) + + // Ruleset that exists won't get a suggestion again. + _, _, _, _, ruleset = api.RulesetSuggestMove(ctx, msgID, inbox.ID, testbox1.ID) + tcompare(t, ruleset == nil, true) + + // Moving in oppositive direction, with rule present, gets the suggestion to remove. + _, _, _, _, ruleset = api.RulesetSuggestMove(ctx, msgID, testbox1.ID, inbox.ID) + tcompare(t, ruleset == nil, false) + + api.RulesetRemove(ctx, rcptTo, *ruleset) + + // If ListID/MsgFrom is marked as never, we won't get a suggestion. + api.RulesetMessageNever(ctx, rcptTo, expListID, expMsgFrom, false) + _, _, _, _, ruleset = api.RulesetSuggestMove(ctx, msgID, inbox.ID, testbox1.ID) + tcompare(t, ruleset == nil, true) + + var n int + if expListID != "" { + n, err = bstore.QueryDB[store.RulesetNoListID](ctx, acc.DB).Delete() + } else { + n, err = bstore.QueryDB[store.RulesetNoMsgFrom](ctx, acc.DB).Delete() + } + tcheck(t, err, "remove never-answer for listid/msgfrom") + tcompare(t, n, 1) + _, _, _, _, ruleset = api.RulesetSuggestMove(ctx, msgID, inbox.ID, testbox1.ID) + tcompare(t, ruleset == nil, false) + + // If Mailbox is marked as never, we won't get a suggestion. + api.RulesetMailboxNever(ctx, testbox1.ID, true) + _, _, _, _, ruleset = api.RulesetSuggestMove(ctx, msgID, inbox.ID, testbox1.ID) + tcompare(t, ruleset == nil, true) + + n, err = bstore.QueryDB[store.RulesetNoMailbox](ctx, acc.DB).Delete() + tcheck(t, err, "remove never-answer for mailbox") + tcompare(t, n, 1) + + } + + // For MsgFrom. + tdeliver(t, acc, inboxText) + testSuggest(inboxText.ID, "", "mjl@mox.example") + + // For List-Id. + tdeliver(t, acc, inboxHTML) + testSuggest(inboxHTML.ID, "list.mox.example", "") } diff --git a/webmail/msg.js b/webmail/msg.js index 8ee2d0b..e3f542f 100644 --- a/webmail/msg.js +++ b/webmail/msg.js @@ -290,7 +290,7 @@ var api; Quoting["Bottom"] = "bottom"; Quoting["Top"] = "top"; })(Quoting = api.Quoting || (api.Quoting = {})); - api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "ComposeMessage": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "FromAddressSettings": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "RecipientSecurity": true, "Request": true, "Settings": true, "SpecialUse": true, "SubmitMessage": true }; + api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "ComposeMessage": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "FromAddressSettings": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "RecipientSecurity": true, "Request": true, "Ruleset": true, "Settings": true, "SpecialUse": true, "SubmitMessage": true }; api.stringsTypes = { "AttachmentType": true, "CSRFToken": true, "Localpart": true, "Quoting": true, "SecurityResult": true, "ThreadMode": true, "ViewMode": true }; api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true }; api.types = { @@ -313,6 +313,7 @@ var api; "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, "RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] }, "Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }] }, + "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, "EventStart": { "Name": "EventStart", "Docs": "", "Fields": [{ "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "LoginAddress", "Docs": "", "Typewords": ["MessageAddress"] }, { "Name": "Addresses", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "DomainAddressConfigs", "Docs": "", "Typewords": ["{}", "DomainAddressConfig"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailboxes", "Docs": "", "Typewords": ["[]", "Mailbox"] }, { "Name": "RejectsMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Settings", "Docs": "", "Typewords": ["Settings"] }, { "Name": "AccountPath", "Docs": "", "Typewords": ["string"] }, { "Name": "Version", "Docs": "", "Typewords": ["string"] }] }, "DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] }, "EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] }, @@ -366,6 +367,7 @@ var api; Mailbox: (v) => api.parse("Mailbox", v), RecipientSecurity: (v) => api.parse("RecipientSecurity", v), Settings: (v) => api.parse("Settings", v), + Ruleset: (v) => api.parse("Ruleset", v), EventStart: (v) => api.parse("EventStart", v), DomainAddressConfig: (v) => api.parse("DomainAddressConfig", v), EventViewErr: (v) => api.parse("EventViewErr", v), @@ -650,6 +652,41 @@ var api; const params = [settings]; return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); } + async RulesetSuggestMove(msgID, mbSrcID, mbDstID) { + const fn = "RulesetSuggestMove"; + const paramTypes = [["int64"], ["int64"], ["int64"]]; + const returnTypes = [["string"], ["string"], ["bool"], ["string"], ["nullable", "Ruleset"]]; + const params = [msgID, mbSrcID, mbDstID]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetAdd(rcptTo, ruleset) { + const fn = "RulesetAdd"; + const paramTypes = [["string"], ["Ruleset"]]; + const returnTypes = []; + const params = [rcptTo, ruleset]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetRemove(rcptTo, ruleset) { + const fn = "RulesetRemove"; + const paramTypes = [["string"], ["Ruleset"]]; + const returnTypes = []; + const params = [rcptTo, ruleset]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetMessageNever(rcptTo, listID, msgFrom, toInbox) { + const fn = "RulesetMessageNever"; + const paramTypes = [["string"], ["string"], ["string"], ["bool"]]; + const returnTypes = []; + const params = [rcptTo, listID, msgFrom, toInbox]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetMailboxNever(mailboxID, toMailbox) { + const fn = "RulesetMailboxNever"; + const paramTypes = [["int64"], ["bool"]]; + const returnTypes = []; + const params = [mailboxID, toMailbox]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } // SSETypes exists to ensure the generated API contains the types, for use in SSE events. async SSETypes() { const fn = "SSETypes"; diff --git a/webmail/text.js b/webmail/text.js index 6814784..a2219a0 100644 --- a/webmail/text.js +++ b/webmail/text.js @@ -290,7 +290,7 @@ var api; Quoting["Bottom"] = "bottom"; Quoting["Top"] = "top"; })(Quoting = api.Quoting || (api.Quoting = {})); - api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "ComposeMessage": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "FromAddressSettings": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "RecipientSecurity": true, "Request": true, "Settings": true, "SpecialUse": true, "SubmitMessage": true }; + api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "ComposeMessage": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "FromAddressSettings": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "RecipientSecurity": true, "Request": true, "Ruleset": true, "Settings": true, "SpecialUse": true, "SubmitMessage": true }; api.stringsTypes = { "AttachmentType": true, "CSRFToken": true, "Localpart": true, "Quoting": true, "SecurityResult": true, "ThreadMode": true, "ViewMode": true }; api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true }; api.types = { @@ -313,6 +313,7 @@ var api; "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, "RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] }, "Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }] }, + "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, "EventStart": { "Name": "EventStart", "Docs": "", "Fields": [{ "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "LoginAddress", "Docs": "", "Typewords": ["MessageAddress"] }, { "Name": "Addresses", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "DomainAddressConfigs", "Docs": "", "Typewords": ["{}", "DomainAddressConfig"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailboxes", "Docs": "", "Typewords": ["[]", "Mailbox"] }, { "Name": "RejectsMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Settings", "Docs": "", "Typewords": ["Settings"] }, { "Name": "AccountPath", "Docs": "", "Typewords": ["string"] }, { "Name": "Version", "Docs": "", "Typewords": ["string"] }] }, "DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] }, "EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] }, @@ -366,6 +367,7 @@ var api; Mailbox: (v) => api.parse("Mailbox", v), RecipientSecurity: (v) => api.parse("RecipientSecurity", v), Settings: (v) => api.parse("Settings", v), + Ruleset: (v) => api.parse("Ruleset", v), EventStart: (v) => api.parse("EventStart", v), DomainAddressConfig: (v) => api.parse("DomainAddressConfig", v), EventViewErr: (v) => api.parse("EventViewErr", v), @@ -650,6 +652,41 @@ var api; const params = [settings]; return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); } + async RulesetSuggestMove(msgID, mbSrcID, mbDstID) { + const fn = "RulesetSuggestMove"; + const paramTypes = [["int64"], ["int64"], ["int64"]]; + const returnTypes = [["string"], ["string"], ["bool"], ["string"], ["nullable", "Ruleset"]]; + const params = [msgID, mbSrcID, mbDstID]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetAdd(rcptTo, ruleset) { + const fn = "RulesetAdd"; + const paramTypes = [["string"], ["Ruleset"]]; + const returnTypes = []; + const params = [rcptTo, ruleset]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetRemove(rcptTo, ruleset) { + const fn = "RulesetRemove"; + const paramTypes = [["string"], ["Ruleset"]]; + const returnTypes = []; + const params = [rcptTo, ruleset]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetMessageNever(rcptTo, listID, msgFrom, toInbox) { + const fn = "RulesetMessageNever"; + const paramTypes = [["string"], ["string"], ["string"], ["bool"]]; + const returnTypes = []; + const params = [rcptTo, listID, msgFrom, toInbox]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetMailboxNever(mailboxID, toMailbox) { + const fn = "RulesetMailboxNever"; + const paramTypes = [["int64"], ["bool"]]; + const returnTypes = []; + const params = [mailboxID, toMailbox]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } // SSETypes exists to ensure the generated API contains the types, for use in SSE events. async SSETypes() { const fn = "SSETypes"; diff --git a/webmail/webmail.js b/webmail/webmail.js index 84fc976..8046b9f 100644 --- a/webmail/webmail.js +++ b/webmail/webmail.js @@ -290,7 +290,7 @@ var api; Quoting["Bottom"] = "bottom"; Quoting["Top"] = "top"; })(Quoting = api.Quoting || (api.Quoting = {})); - api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "ComposeMessage": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "FromAddressSettings": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "RecipientSecurity": true, "Request": true, "Settings": true, "SpecialUse": true, "SubmitMessage": true }; + api.structTypes = { "Address": true, "Attachment": true, "ChangeMailboxAdd": true, "ChangeMailboxCounts": true, "ChangeMailboxKeywords": true, "ChangeMailboxRemove": true, "ChangeMailboxRename": true, "ChangeMailboxSpecialUse": true, "ChangeMsgAdd": true, "ChangeMsgFlags": true, "ChangeMsgRemove": true, "ChangeMsgThread": true, "ComposeMessage": true, "Domain": true, "DomainAddressConfig": true, "Envelope": true, "EventStart": true, "EventViewChanges": true, "EventViewErr": true, "EventViewMsgs": true, "EventViewReset": true, "File": true, "Filter": true, "Flags": true, "ForwardAttachments": true, "FromAddressSettings": true, "Mailbox": true, "Message": true, "MessageAddress": true, "MessageEnvelope": true, "MessageItem": true, "NotFilter": true, "Page": true, "ParsedMessage": true, "Part": true, "Query": true, "RecipientSecurity": true, "Request": true, "Ruleset": true, "Settings": true, "SpecialUse": true, "SubmitMessage": true }; api.stringsTypes = { "AttachmentType": true, "CSRFToken": true, "Localpart": true, "Quoting": true, "SecurityResult": true, "ThreadMode": true, "ViewMode": true }; api.intsTypes = { "ModSeq": true, "UID": true, "Validation": true }; api.types = { @@ -313,6 +313,7 @@ var api; "Mailbox": { "Name": "Mailbox", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Name", "Docs": "", "Typewords": ["string"] }, { "Name": "UIDValidity", "Docs": "", "Typewords": ["uint32"] }, { "Name": "UIDNext", "Docs": "", "Typewords": ["UID"] }, { "Name": "Archive", "Docs": "", "Typewords": ["bool"] }, { "Name": "Draft", "Docs": "", "Typewords": ["bool"] }, { "Name": "Junk", "Docs": "", "Typewords": ["bool"] }, { "Name": "Sent", "Docs": "", "Typewords": ["bool"] }, { "Name": "Trash", "Docs": "", "Typewords": ["bool"] }, { "Name": "Keywords", "Docs": "", "Typewords": ["[]", "string"] }, { "Name": "HaveCounts", "Docs": "", "Typewords": ["bool"] }, { "Name": "Total", "Docs": "", "Typewords": ["int64"] }, { "Name": "Deleted", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unread", "Docs": "", "Typewords": ["int64"] }, { "Name": "Unseen", "Docs": "", "Typewords": ["int64"] }, { "Name": "Size", "Docs": "", "Typewords": ["int64"] }] }, "RecipientSecurity": { "Name": "RecipientSecurity", "Docs": "", "Fields": [{ "Name": "STARTTLS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "MTASTS", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DNSSEC", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "DANE", "Docs": "", "Typewords": ["SecurityResult"] }, { "Name": "RequireTLS", "Docs": "", "Typewords": ["SecurityResult"] }] }, "Settings": { "Name": "Settings", "Docs": "", "Fields": [{ "Name": "ID", "Docs": "", "Typewords": ["uint8"] }, { "Name": "Signature", "Docs": "", "Typewords": ["string"] }, { "Name": "Quoting", "Docs": "", "Typewords": ["Quoting"] }, { "Name": "ShowAddressSecurity", "Docs": "", "Typewords": ["bool"] }] }, + "Ruleset": { "Name": "Ruleset", "Docs": "", "Fields": [{ "Name": "SMTPMailFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "MsgFromRegexp", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "HeadersRegexp", "Docs": "", "Typewords": ["{}", "string"] }, { "Name": "IsForward", "Docs": "", "Typewords": ["bool"] }, { "Name": "ListAllowDomain", "Docs": "", "Typewords": ["string"] }, { "Name": "AcceptRejectsToMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Comment", "Docs": "", "Typewords": ["string"] }, { "Name": "VerifiedDNSDomain", "Docs": "", "Typewords": ["Domain"] }, { "Name": "ListAllowDNSDomain", "Docs": "", "Typewords": ["Domain"] }] }, "EventStart": { "Name": "EventStart", "Docs": "", "Fields": [{ "Name": "SSEID", "Docs": "", "Typewords": ["int64"] }, { "Name": "LoginAddress", "Docs": "", "Typewords": ["MessageAddress"] }, { "Name": "Addresses", "Docs": "", "Typewords": ["[]", "MessageAddress"] }, { "Name": "DomainAddressConfigs", "Docs": "", "Typewords": ["{}", "DomainAddressConfig"] }, { "Name": "MailboxName", "Docs": "", "Typewords": ["string"] }, { "Name": "Mailboxes", "Docs": "", "Typewords": ["[]", "Mailbox"] }, { "Name": "RejectsMailbox", "Docs": "", "Typewords": ["string"] }, { "Name": "Settings", "Docs": "", "Typewords": ["Settings"] }, { "Name": "AccountPath", "Docs": "", "Typewords": ["string"] }, { "Name": "Version", "Docs": "", "Typewords": ["string"] }] }, "DomainAddressConfig": { "Name": "DomainAddressConfig", "Docs": "", "Fields": [{ "Name": "LocalpartCatchallSeparator", "Docs": "", "Typewords": ["string"] }, { "Name": "LocalpartCaseSensitive", "Docs": "", "Typewords": ["bool"] }] }, "EventViewErr": { "Name": "EventViewErr", "Docs": "", "Fields": [{ "Name": "ViewID", "Docs": "", "Typewords": ["int64"] }, { "Name": "RequestID", "Docs": "", "Typewords": ["int64"] }, { "Name": "Err", "Docs": "", "Typewords": ["string"] }] }, @@ -366,6 +367,7 @@ var api; Mailbox: (v) => api.parse("Mailbox", v), RecipientSecurity: (v) => api.parse("RecipientSecurity", v), Settings: (v) => api.parse("Settings", v), + Ruleset: (v) => api.parse("Ruleset", v), EventStart: (v) => api.parse("EventStart", v), DomainAddressConfig: (v) => api.parse("DomainAddressConfig", v), EventViewErr: (v) => api.parse("EventViewErr", v), @@ -650,6 +652,41 @@ var api; const params = [settings]; return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); } + async RulesetSuggestMove(msgID, mbSrcID, mbDstID) { + const fn = "RulesetSuggestMove"; + const paramTypes = [["int64"], ["int64"], ["int64"]]; + const returnTypes = [["string"], ["string"], ["bool"], ["string"], ["nullable", "Ruleset"]]; + const params = [msgID, mbSrcID, mbDstID]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetAdd(rcptTo, ruleset) { + const fn = "RulesetAdd"; + const paramTypes = [["string"], ["Ruleset"]]; + const returnTypes = []; + const params = [rcptTo, ruleset]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetRemove(rcptTo, ruleset) { + const fn = "RulesetRemove"; + const paramTypes = [["string"], ["Ruleset"]]; + const returnTypes = []; + const params = [rcptTo, ruleset]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetMessageNever(rcptTo, listID, msgFrom, toInbox) { + const fn = "RulesetMessageNever"; + const paramTypes = [["string"], ["string"], ["string"], ["bool"]]; + const returnTypes = []; + const params = [rcptTo, listID, msgFrom, toInbox]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } + async RulesetMailboxNever(mailboxID, toMailbox) { + const fn = "RulesetMailboxNever"; + const paramTypes = [["int64"], ["bool"]]; + const returnTypes = []; + const params = [mailboxID, toMailbox]; + return await _sherpaCall(this.baseURL, this.authState, { ...this.options }, paramTypes, returnTypes, fn, params); + } // SSETypes exists to ensure the generated API contains the types, for use in SSE events. async SSETypes() { const fn = "SSETypes"; @@ -1297,7 +1334,7 @@ Enable consistency checking in UI updates: - todo: buttons/mechanism to operate on all messages in a mailbox/search query, without having to list and select all messages. e.g. clearing flags/labels. - todo: can we detect if browser supports proper CSP? if not, refuse to load html messages? - todo: more search criteria? Date header field (instead of time received), text vs html (only, either or both), attachment filenames and sizes -- todo: integrate more of the account page into webmail? importing/exporting messages, configuring delivery rules (possibly with sieve). for messages moved out of inbox to non-special-use mailbox, show button that helps make an automatic rule to move such messages again (e.g. based on message From address, message From domain or List-ID header). +- todo: integrate more of the account page into webmail? importing/exporting messages, configuring delivery rules (possibly with sieve). - todo: configurable keyboard shortcuts? we use strings like "ctrl p" which we already generate and match on, add a mapping from command name to cmd* functions, and have a map of keys to command names. the commands for up/down with shift/ctrl modifiers may need special attention. - todo: nicer address input fields like other mail clients do. with tab to autocomplete and turn input into a box and delete removing of the entire address. - todo: consider composing messages with bcc headers that are kept as message Bcc headers, optionally with checkbox. @@ -2937,11 +2974,61 @@ const movePopover = (e, mailboxes, msgs) => { } let msgsMailboxID = (msgs[0].MailboxID && msgs.filter(m => m.MailboxID === msgs[0].MailboxID).length === msgs.length) ? msgs[0].MailboxID : 0; const remove = popover(e.target, {}, dom.div(style({ display: 'flex', flexDirection: 'column', gap: '.25em' }), mailboxes.map(mb => dom.div(dom.clickbutton(mb.Name, mb.ID === msgsMailboxID ? attr.disabled('') : [], async function click() { - const msgIDs = msgs.filter(m => m.MailboxID !== mb.ID).map(m => m.ID); + const moveMsgs = msgs.filter(m => m.MailboxID !== mb.ID); + const msgIDs = moveMsgs.map(m => m.ID); await withStatus('Moving to mailbox', client.MessageMove(msgIDs, mb.ID)); + if (moveMsgs.length === 1) { + await moveAskRuleset(moveMsgs[0].ID, moveMsgs[0].MailboxID, mb, mailboxes); + } remove(); }))))); }; +// We've moved a single message. If the source or destination mailbox is not a +// "special-use" mailbox (other than inbox), and there isn't a rule yet or there is +// one we may want to delete, and we haven't asked about adding/removing this +// ruleset before, ask the user to add/remove a ruleset for moving. If the message +// has a list-id header, we ask to create a ruleset treating it as a mailing list +// message matching on future list-id header and spf/dkim verified domain, +// otherwise we make a rule based on message "from" address. +const moveAskRuleset = async (msgID, mbSrcID, mbDst, mailboxes) => { + const mbSrc = mailboxes.find(mb => mb.ID === mbSrcID); + if (!mbSrc || isSpecialUse(mbDst) || isSpecialUse(mbSrc)) { + return; + } + const [listID, msgFrom, isRemove, rcptTo, ruleset] = await withStatus('Checking rulesets', client.RulesetSuggestMove(msgID, mbSrc.ID, mbDst.ID)); + if (!ruleset) { + return; + } + const what = listID ? ['list with id "', listID, '"'] : ['address "', msgFrom, '"']; + if (isRemove) { + const remove = popup(dom.h1('Remove rule?'), dom.p(style({ maxWidth: '30em' }), 'Would you like to remove the server-side rule that automatically delivers messages from ', what, ' to mailbox "', mbDst.Name, '"?'), dom.br(), dom.div(dom.clickbutton('Yes, remove rule', async function click() { + await withStatus('Remove ruleset', client.RulesetRemove(rcptTo, ruleset)); + remove(); + }), ' ', dom.clickbutton('Not now', async function click() { + remove(); + })), dom.br(), dom.div(style({ marginBottom: '1ex' }), dom.clickbutton("No, and don't ask again for ", what, async function click() { + await withStatus('Store ruleset response', client.RulesetMessageNever(rcptTo, listID, msgFrom, true)); + remove(); + })), dom.div(dom.clickbutton("No, and don't ask again when moving messages out of \"", mbSrc.Name, '"', async function click() { + await withStatus('Store ruleset response', client.RulesetMailboxNever(mbSrc.ID, false)); + remove(); + }))); + return; + } + const remove = popup(dom.h1('Add rule?'), dom.p(style({ maxWidth: '30em' }), 'Would you like to create a server-side ruleset that automatically delivers future messages from ', what, ' to mailbox "', mbDst.Name, '"?'), dom.br(), dom.div(dom.clickbutton('Yes, add rule', async function click() { + await withStatus('Add ruleset', client.RulesetAdd(rcptTo, ruleset)); + remove(); + }), ' ', dom.clickbutton('Not now', async function click() { + remove(); + })), dom.br(), dom.div(style({ marginBottom: '1ex' }), dom.clickbutton("No, and don't ask again for ", what, async function click() { + await withStatus('Store ruleset response', client.RulesetMessageNever(rcptTo, listID, msgFrom, false)); + remove(); + })), dom.div(dom.clickbutton("No, and don't ask again when moving messages to \"", mbDst.Name, '"', async function click() { + await withStatus('Store ruleset response', client.RulesetMailboxNever(mbDst.ID, true)); + remove(); + }))); +}; +const isSpecialUse = (mb) => mb.Archive || mb.Draft || mb.Junk || mb.Sent || mb.Trash; // Make new MsgitemView, to be added to the list. const newMsgitemView = (mi, msglistView, otherMailbox, listMailboxes, receivedTime, initialCollapsed) => { // note: mi may be replaced. @@ -5237,6 +5324,11 @@ const newMailboxView = (xmb, mailboxlistView, otherMailbox) => { .filter(mbMsgID => mailboxMsgIDs.length === 1 || !sentMailboxID || mbMsgID[0] !== sentMailboxID || !otherMailbox(sentMailboxID)) .map(mbMsgID => mbMsgID[1]); await withStatus('Moving to ' + xmb.Name, client.MessageMove(msgIDs, xmb.ID)); + if (msgIDs.length === 1) { + const msgID = msgIDs[0]; + const mbSrcID = mailboxMsgIDs.find(mbMsgID => mbMsgID[1] === msgID)[0]; + await moveAskRuleset(msgID, mbSrcID, xmb, mailboxlistView.mailboxes()); + } }, dom.div(dom._class('mailbox'), style({ display: 'flex', justifyContent: 'space-between' }), name = dom.div(style({ whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis' })), dom.div(style({ whiteSpace: 'nowrap' }), actionBtn = dom.clickbutton(dom._class('mailboxhoveronly'), '...', attr.tabindex('-1'), // Without, tab breaks because this disappears when mailbox loses focus. attr.arialabel('Mailbox actions'), attr.title('Actions on mailbox, like deleting, emptying, renaming.'), function click(e) { e.stopPropagation(); diff --git a/webmail/webmail.ts b/webmail/webmail.ts index 70fa910..6148084 100644 --- a/webmail/webmail.ts +++ b/webmail/webmail.ts @@ -81,7 +81,7 @@ Enable consistency checking in UI updates: - todo: buttons/mechanism to operate on all messages in a mailbox/search query, without having to list and select all messages. e.g. clearing flags/labels. - todo: can we detect if browser supports proper CSP? if not, refuse to load html messages? - todo: more search criteria? Date header field (instead of time received), text vs html (only, either or both), attachment filenames and sizes -- todo: integrate more of the account page into webmail? importing/exporting messages, configuring delivery rules (possibly with sieve). for messages moved out of inbox to non-special-use mailbox, show button that helps make an automatic rule to move such messages again (e.g. based on message From address, message From domain or List-ID header). +- todo: integrate more of the account page into webmail? importing/exporting messages, configuring delivery rules (possibly with sieve). - todo: configurable keyboard shortcuts? we use strings like "ctrl p" which we already generate and match on, add a mapping from command name to cmd* functions, and have a map of keys to command names. the commands for up/down with shift/ctrl modifiers may need special attention. - todo: nicer address input fields like other mail clients do. with tab to autocomplete and turn input into a box and delete removing of the entire address. - todo: consider composing messages with bcc headers that are kept as message Bcc headers, optionally with checkbox. @@ -2208,8 +2208,12 @@ const movePopover = (e: MouseEvent, mailboxes: api.Mailbox[], msgs: api.Message[ mb.Name, mb.ID === msgsMailboxID ? attr.disabled('') : [], async function click() { - const msgIDs = msgs.filter(m => m.MailboxID !== mb.ID).map(m => m.ID) + const moveMsgs = msgs.filter(m => m.MailboxID !== mb.ID) + const msgIDs = moveMsgs.map(m => m.ID) await withStatus('Moving to mailbox', client.MessageMove(msgIDs, mb.ID)) + if (moveMsgs.length === 1) { + await moveAskRuleset(moveMsgs[0].ID, moveMsgs[0].MailboxID, mb, mailboxes) + } remove() } ), @@ -2219,6 +2223,95 @@ const movePopover = (e: MouseEvent, mailboxes: api.Mailbox[], msgs: api.Message[ ) } +// We've moved a single message. If the source or destination mailbox is not a +// "special-use" mailbox (other than inbox), and there isn't a rule yet or there is +// one we may want to delete, and we haven't asked about adding/removing this +// ruleset before, ask the user to add/remove a ruleset for moving. If the message +// has a list-id header, we ask to create a ruleset treating it as a mailing list +// message matching on future list-id header and spf/dkim verified domain, +// otherwise we make a rule based on message "from" address. +const moveAskRuleset = async (msgID: number, mbSrcID: number, mbDst: api.Mailbox, mailboxes: api.Mailbox[]) => { + const mbSrc = mailboxes.find(mb => mb.ID === mbSrcID) + if (!mbSrc || isSpecialUse(mbDst) || isSpecialUse(mbSrc)) { + return + } + + const [listID, msgFrom, isRemove, rcptTo, ruleset] = await withStatus('Checking rulesets', client.RulesetSuggestMove(msgID, mbSrc.ID, mbDst.ID)) + if (!ruleset) { + return + } + + const what = listID ? ['list with id "', listID, '"'] : ['address "', msgFrom, '"'] + + if (isRemove) { + const remove = popup( + dom.h1('Remove rule?'), + dom.p( + style({maxWidth: '30em'}), + 'Would you like to remove the server-side rule that automatically delivers messages from ', what, ' to mailbox "', mbDst.Name, '"?', + ), + dom.br(), + dom.div( + dom.clickbutton('Yes, remove rule', async function click() { + await withStatus('Remove ruleset', client.RulesetRemove(rcptTo, ruleset)) + remove() + }), ' ', + dom.clickbutton('Not now', async function click() { + remove() + }), + ), + dom.br(), + dom.div( + style({marginBottom: '1ex'}), + dom.clickbutton("No, and don't ask again for ", what, async function click() { + await withStatus('Store ruleset response', client.RulesetMessageNever(rcptTo, listID, msgFrom, true)) + remove() + }), + ), + dom.div( + dom.clickbutton("No, and don't ask again when moving messages out of \"", mbSrc.Name, '"', async function click() { + await withStatus('Store ruleset response', client.RulesetMailboxNever(mbSrc.ID, false)) + remove() + }), + ), + ) + return + } + const remove = popup( + dom.h1('Add rule?'), + dom.p( + style({maxWidth: '30em'}), + 'Would you like to create a server-side ruleset that automatically delivers future messages from ', what, ' to mailbox "', mbDst.Name, '"?', + ), + dom.br(), + dom.div( + dom.clickbutton('Yes, add rule', async function click() { + await withStatus('Add ruleset', client.RulesetAdd(rcptTo, ruleset)) + remove() + }), ' ', + dom.clickbutton('Not now', async function click() { + remove() + }), + ), + dom.br(), + dom.div( + style({marginBottom: '1ex'}), + dom.clickbutton("No, and don't ask again for ", what, async function click() { + await withStatus('Store ruleset response', client.RulesetMessageNever(rcptTo, listID, msgFrom, false)) + remove() + }), + ), + dom.div( + dom.clickbutton("No, and don't ask again when moving messages to \"", mbDst.Name, '"', async function click() { + await withStatus('Store ruleset response', client.RulesetMailboxNever(mbDst.ID, true)) + remove() + }), + ), + ) +} + +const isSpecialUse = (mb: api.Mailbox) => mb.Archive || mb.Draft || mb.Junk || mb.Sent || mb.Trash + // MsgitemView is a message-line in the list of messages. Selecting it loads and displays the message, a MsgView. interface MsgitemView { root: HTMLElement // MsglistView toggles active/focus classes on the root element. @@ -5047,6 +5140,11 @@ const newMailboxView = (xmb: api.Mailbox, mailboxlistView: MailboxlistView, othe .filter(mbMsgID => mailboxMsgIDs.length === 1 || !sentMailboxID || mbMsgID[0] !== sentMailboxID || !otherMailbox(sentMailboxID)) .map(mbMsgID => mbMsgID[1]) await withStatus('Moving to '+xmb.Name, client.MessageMove(msgIDs, xmb.ID)) + if (msgIDs.length === 1) { + const msgID = msgIDs[0] + const mbSrcID = mailboxMsgIDs.find(mbMsgID => mbMsgID[1] === msgID)![0] + await moveAskRuleset(msgID, mbSrcID, xmb, mailboxlistView.mailboxes()) + } }, dom.div(dom._class('mailbox'), style({display: 'flex', justifyContent: 'space-between'}), diff --git a/webmail/webmail_test.go b/webmail/webmail_test.go index c92358f..b16fbfa 100644 --- a/webmail/webmail_test.go +++ b/webmail/webmail_test.go @@ -188,6 +188,7 @@ var ( From: "mjl ", To: "mox ", Subject: "html message", + Headers: [][2]string{{"List-Id", "test "}}, Part: Part{Type: "text/html", Content: `the body `}, } msgAlt = Message{ @@ -265,7 +266,16 @@ func tdeliver(t *testing.T, acc *store.Account, tm *testmsg) { defer msgFile.Close() size, err := msgFile.Write(tm.msg.Marshal(t)) tcheck(t, err, "write message temp") - m := store.Message{Flags: tm.Flags, Keywords: tm.Keywords, Size: int64(size)} + m := store.Message{ + Flags: tm.Flags, + RcptToLocalpart: "mox", + RcptToDomain: "other.example", + MsgFromLocalpart: "mjl", + MsgFromDomain: "mox.example", + DKIMDomains: []string{"mox.example"}, + Keywords: tm.Keywords, + Size: int64(size), + } err = acc.DeliverMailbox(pkglog, tm.Mailbox, &m, msgFile) tcheck(t, err, "deliver test message") err = msgFile.Close()