mox/smtpserver/reputation.go

package smtpserver

import (
	"errors"
	"fmt"
	"log/slog"
	"strings"
	"time"

	"github.com/mjl-/bstore"

	"github.com/mjl-/mox/dns"
	"github.com/mjl-/mox/mlog"
	"github.com/mjl-/mox/smtp"
	"github.com/mjl-/mox/store"
)

type reputationMethod string

const (
	methodMsgfromFull      reputationMethod = "msgfromfull"
	methodMsgtoFull        reputationMethod = "msgtofull"
	methodMsgfromDomain    reputationMethod = "msgfromdomain"
	methodMsgfromOrgDomain reputationMethod = "msgfromorgdomain"
	methodMsgtoDomain      reputationMethod = "msgtodomain"
	methodMsgtoOrgDomain   reputationMethod = "msgtoorgdomain"
	methodDKIMSPF          reputationMethod = "dkimspf"
	methodIP1              reputationMethod = "ip1"
	methodIP2              reputationMethod = "ip2"
	methodIP3              reputationMethod = "ip3"
	methodNone             reputationMethod = "none"
)

// Reputation returns whether message m is likely junk.
//
// This function is called after checking for a manually configured spf mailfrom
// allow (e.g. for mailing lists), and after checking for a dmarc reject policy.
//
// The decision is made based on historic messages delivered to the same
// destination mailbox, MailboxOrigID. Because each mailbox may have a different
// accept policy. We only use messages that have been marked as either junk or
// non-junk. We help users by automatically marking them as non-junk when moving to
// certain folders in the default config (e.g. the archive folder). We expect users
// to mark junk messages as such when they read it. And to keep it in their inbox,
// regular trash or archive if it is not.
//
// The basic idea is to keep accepting messages that were accepted in the past, and
// keep rejecting those that were rejected. This is relatively easy to check if
// mail passes SPF and/or DKIM with Message-From alignment. Regular email from
// known people will be let in. But spammers are trickier. They will use new IPs,
// (sub)domains, no or newly created SPF and/or DKIM identifiers, new localparts,
// etc. This function likely ends up returning "inconclusive" for such emails. The
// junkfilter will have to take care of a final decision.
//
// In case of doubt, it doesn't hurt much to accept another mail that a user has
// communicated successfully with in the past. If the most recent message is marked
// as junk that could have happened accidentally. If another message is let in, and
// it is again junk, future messages will be rejected.
//
// Actual spammers will probably try to use identifiers, i.e. (sub)domain, dkim/spf
// identifiers and ip addresses for which we have no history. We may only have
// ip-based reputation, perhaps only an ip range, perhaps nothing.
//
// Some profiles of first-time senders:
//
//   - Individuals. They can typically get past the junkfilter if needed.
//   - Transactional emails. They should get past the junkfilter. If they use one of
//     the larger email service providers, their reputation could help. If the
//     junkfilter rejects the message, users can recover the message from the Rejects
//     mailbox. The first message is typically initiated by a user, e.g. by registering.
//   - Desired commercial email will have to get past the junkfilter based on its
//     content. There will typically be earlier communication with the (organizational)
//     domain that would let the message through.
//   - Mailing list. May get past the junkfilter. If delivery is to a separate
//     mailbox, the junkfilter will let it in because of little history. Long enough to
//     build reputation based on DKIM/SPF signals. Users are best off to
//     configure accept rules for messages from mailing lists.
//
// The decision-making process looks at historic messages. The following properties
// are checked until matching messages are found. If they are found, a decision is
// returned, which may be inconclusive. The next property on the list is only
// checked if a step did not match any messages.
//
//   - Messages matching full "message from" address, either with strict/relaxed
//     dkim/spf-verification, or without.
//   - Messages the user sent to the "message from" address.
//   - Messages matching only the domain of the "message from" address (different
//     localpart), again with verification or without.
//   - Messages sent to an address in the domain of the "message from" address.
//   - The previous two checks again, but now checking against the organizational
//     domain instead of the exact domain.
//   - Matching DKIM domains and a matching SPF mailfrom, or mailfrom domain, or ehlo
//     domain.
//   - "Exact" IP, or nearby IPs.
//
// References:
// ../rfc/5863
// ../rfc/7960
// ../rfc/6376:1915
// ../rfc/6376:3716
// ../rfc/7208:2167
func reputation(tx *bstore.Tx, log mlog.Log, m *store.Message, smtputf8 bool) (rjunk *bool, rconclusive bool, rmethod reputationMethod, reasonText string, rerr error) {
	boolptr := func(v bool) *bool {
		return &v
	}
	xfalse := boolptr(false)
	xtrue := boolptr(true)

	type queryError string

	defer func() {
		x := recover()
		if x == nil {
			return
		}
		if xerr, ok := x.(queryError); ok {
			rerr = errors.New(string(xerr))
			return
		}
		panic(x)
	}()

	now := time.Now()

	// messageQuery returns a base query for historic seen messages to the same
	// mailbox, at most maxAge old, and at most maxCount messages.
	messageQuery := func(fm *store.Message, maxAge time.Duration, maxCount int) *bstore.Query[store.Message] {
		q := bstore.QueryTx[store.Message](tx)
		q.FilterEqual("MailboxOrigID", m.MailboxID)
		q.FilterEqual("Expunged", false)
		q.FilterFn(func(m store.Message) bool {
			return m.Junk || m.Notjunk
		})
		if fm != nil {
			q.FilterNonzero(*fm)
		}
		q.FilterGreaterEqual("Received", now.Add(-maxAge))
		q.Limit(maxCount)
		q.SortDesc("Received")
		return q
	}

	// Execute the query, returning messages or returning error through panic.
	xmessageList := func(q *bstore.Query[store.Message], descr string) []store.Message {
		t0 := time.Now()
		l, err := q.List()
		log.Debugx("querying messages for reputation", err,
			slog.Int("msgs", len(l)),
			slog.String("descr", descr),
			slog.Duration("queryduration", time.Since(t0)))
		if err != nil {
			panic(queryError(fmt.Sprintf("listing messages: %v", err)))
		}
		return l
	}

	xrecipientExists := func(q *bstore.Query[store.Recipient]) bool {
		exists, err := q.Exists()
		if err != nil {
			panic(queryError(fmt.Sprintf("checking for recipient: %v", err)))
		}
		return exists
	}

	const year = 365 * 24 * time.Hour

	// Look for historic messages with same "message from" address. We'll
	// treat any validation (strict/dmarc/relaxed) the same, but "none"
	// separately.
	//
	// We only need 1 message, and sometimes look at a second message. If
	// the last message or the message before was an accept, we accept. If
	// the single last or last two were a reject, we reject.
	//
	// If there was no validation, any signal is inconclusive.
	if m.MsgFromDomain != "" {
		q := messageQuery(&store.Message{MsgFromLocalpart: m.MsgFromLocalpart, MsgFromDomain: m.MsgFromDomain}, 3*year, 2)
		q.FilterEqual("MsgFromValidated", m.MsgFromValidated)
		msgs := xmessageList(q, "mgsfromfull")
		if len(msgs) > 0 {
			// todo: we may want to look at dkim/spf in this case.
			spam := msgs[0].Junk && (len(msgs) == 1 || msgs[1].Junk)
			conclusive := m.MsgFromValidated
			return &spam, conclusive, methodMsgfromFull, "reputation of exact message-from address", nil
		}
		if !m.MsgFromValidated {
			// Look for historic messages that were validated. If present, this is likely spam.
			// Only return as conclusively spam if history also says this From-address sent
			// spam.
			q := messageQuery(&store.Message{MsgFromLocalpart: m.MsgFromLocalpart, MsgFromDomain: m.MsgFromDomain, MsgFromValidated: true}, 3*year, 2)
			msgs = xmessageList(q, "msgfromfull-validated")
			if len(msgs) > 0 {
				spam := msgs[0].Junk && (len(msgs) == 1 || msgs[1].Junk)
				return xtrue, spam, methodMsgfromFull, "unvalidated message with validated historic messages with exact message-from address", nil
			}
		}

		// Look if we ever sent to this address. If so, we accept,
		qr := bstore.QueryTx[store.Recipient](tx)
		qr.FilterEqual("Localpart", m.MsgFromLocalpart)
		qr.FilterEqual("Domain", m.MsgFromDomain)
		qr.FilterGreaterEqual("Sent", now.Add(-3*year))
		if xrecipientExists(qr) {
			return xfalse, true, methodMsgtoFull, "exact message-from address was earlier message recipient", nil
		}

		// Look for domain match, then for organizational domain match.
		for _, orgdomain := range []bool{false, true} {
			qm := store.Message{}
			var method reputationMethod
			var source, descr string
			if orgdomain {
				qm.MsgFromOrgDomain = m.MsgFromOrgDomain
				method = methodMsgfromOrgDomain
				source = "organizational domain of message-from address"
				descr = "msgfromorgdomain"
			} else {
				qm.MsgFromDomain = m.MsgFromDomain
				method = methodMsgfromDomain
				source = "exact domain of message-from address"
				descr = "msgfromdomain"
			}

			q := messageQuery(&qm, 2*year, 20)
			q.FilterEqual("MsgFromValidated", m.MsgFromValidated)
			msgs := xmessageList(q, descr)
			if len(msgs) > 0 {
				nonjunk := 0
				for _, m := range msgs {
					if !m.Junk {
						nonjunk++
					}
				}
				if 100*nonjunk/len(msgs) > 80 {
					reasonText = fmt.Sprintf("positive reputation with %s based on %d messages", source, len(msgs))
					return xfalse, true, method, reasonText, nil
				}
				if nonjunk == 0 {
					// Only conclusive with at least 3 different localparts.
					localparts := map[smtp.Localpart]struct{}{}
					for _, m := range msgs {
						localparts[m.MsgFromLocalpart] = struct{}{}
						if len(localparts) == 3 {
							reasonText = fmt.Sprintf("negative reputation of at least 3 addresses with %s based on %d messages", source, len(msgs))
							return xtrue, true, method, reasonText, nil
						}
					}
					reasonText = fmt.Sprintf("negative reputation with %s based on %d messages", source, len(msgs))
					return xtrue, false, method, reasonText, nil
				}
				// Mixed signals from domain. We don't want to block a new sender.
				reasonText = fmt.Sprintf("mixed signals with %s based on %d messages", source, len(msgs))
				return nil, false, method, reasonText, nil
			}
			if !m.MsgFromValidated {
				// Look for historic messages that were validated. If present, this is likely spam.
				// Only return as conclusively spam if history also says this From-address sent
				// spam.
				q := messageQuery(&qm, 2*year, 2)
				q.FilterEqual("MsgFromValidated", true)
				msgs = xmessageList(q, descr+"-validated")
				if len(msgs) > 0 {
					spam := msgs[0].Junk && (len(msgs) == 1 || msgs[1].Junk)
					reasonText = fmt.Sprintf("unvalidated message with %s while we have validated messages from that source", source)
					return xtrue, spam, method, reasonText, nil
				}
			}

			// Look if we ever sent to this address. If so, we accept,
			qr := bstore.QueryTx[store.Recipient](tx)
			if orgdomain {
				qr.FilterEqual("OrgDomain", m.MsgFromOrgDomain)
				method = methodMsgtoOrgDomain
				source = "organizational domain of message-from address"
			} else {
				qr.FilterEqual("Domain", m.MsgFromDomain)
				method = methodMsgtoDomain
				source = "exact domain of message-from address"
			}
			qr.FilterGreaterEqual("Sent", now.Add(-2*year))
			if xrecipientExists(qr) {
				reasonText = fmt.Sprintf("%s was recipient address", source)
				return xfalse, true, method, reasonText, nil
			}
		}
	}

	// DKIM and SPF.
	// We only use identities that passed validation. Failed identities are ignored. ../rfc/6376:2447
	// todo future: we could do something with the DKIM identity (i=) field if it is more specific than just the domain (d=).
	dkimspfsignals := []float64{}
	dkimspfreasondoms := []string{}
	dkimspfmsgs := 0
	for _, dom := range m.DKIMDomains {
		q := messageQuery(nil, year/2, 50)
		q.FilterIn("DKIMDomains", dom)
		msgs := xmessageList(q, "dkimdomain")
		if len(msgs) > 0 {
			nspam := 0
			for _, m := range msgs {
				if m.Junk {
					nspam++
				}
			}
			pspam := float64(nspam) / float64(len(msgs))
			dkimspfsignals = append(dkimspfsignals, pspam)
			dkimspfreasondoms = append(dkimspfreasondoms, dom)
			dkimspfmsgs = len(msgs)
		}
	}
	if m.MailFromValidated || m.EHLOValidated {
		var dom string
		var msgs []store.Message
		if m.MailFromValidated && m.MailFromDomain != "" {
			dom = m.MailFromDomain
			q := messageQuery(&store.Message{MailFromLocalpart: m.MailFromLocalpart, MailFromDomain: m.MailFromDomain}, year/2, 50)
			msgs = xmessageList(q, "mailfrom")
			if len(msgs) == 0 {
				q := messageQuery(&store.Message{MailFromDomain: m.MailFromDomain}, year/2, 50)
				msgs = xmessageList(q, "mailfromdomain")
			}
		}
		if len(msgs) == 0 && m.EHLOValidated && m.EHLODomain != "" {
			dom = m.EHLODomain
			q := messageQuery(&store.Message{EHLODomain: m.EHLODomain}, year/2, 50)
			msgs = xmessageList(q, "ehlodomain")
		}
		if len(msgs) > 0 {
			nspam := 0
			for _, m := range msgs {
				if m.Junk {
					nspam++
				}
			}
			pspam := float64(nspam) / float64(len(msgs))
			dkimspfsignals = append(dkimspfsignals, pspam)
			dkimspfreasondoms = append(dkimspfreasondoms, dom)
			if len(msgs) > dkimspfmsgs {
				dkimspfmsgs = len(msgs)
			}
		}
	}
	if len(dkimspfsignals) > 0 {
		var nham, nspam int
		var hamdoms, spamdoms []string
		for i, p := range dkimspfsignals {
			d, _ := dns.ParseDomain(dkimspfreasondoms[i])
			if p < .1 {
				nham++
				hamdoms = append(hamdoms, d.XName(smtputf8))
			} else if p > .9 {
				nspam++
				spamdoms = append(spamdoms, d.XName(smtputf8))
			}
		}
		if nham > 0 && nspam == 0 {
			reasonText = fmt.Sprintf("positive dkim/spf reputation for domain(s) %s", strings.Join(hamdoms, ","))
			return xfalse, true, methodDKIMSPF, reasonText, nil
		}
		if nspam > 0 && nham == 0 {
			reasonText = fmt.Sprintf("negative dkim/spf reputation for domain(s) %s", strings.Join(hamdoms, ","))
			return xtrue, dkimspfmsgs > 1, methodDKIMSPF, reasonText, nil
		}
		reasonText = fmt.Sprintf("mixed dkim/spf reputation, positive for %s, negative for %s", strings.Join(hamdoms, ","), strings.Join(spamdoms, ","))
		return nil, false, methodDKIMSPF, reasonText, nil
	}

	// IP-based. A wider mask needs more messages to be conclusive.
	// We require the resulting signal to be strong, i.e. likely ham or likely spam.
	var msgs []store.Message
	var need int
	var method reputationMethod
	var ip string
	if m.RemoteIPMasked1 != "" {
		q := messageQuery(&store.Message{RemoteIPMasked1: m.RemoteIPMasked1}, year/4, 50)
		msgs = xmessageList(q, "ip1")
		need = 2
		method = methodIP1
		ip = m.RemoteIPMasked1
	}
	if len(msgs) == 0 && m.RemoteIPMasked2 != "" {
		q := messageQuery(&store.Message{RemoteIPMasked2: m.RemoteIPMasked2}, year/4, 50)
		msgs = xmessageList(q, "ip2")
		need = 5
		method = methodIP2
		ip = m.RemoteIPMasked2
	}
	if len(msgs) == 0 && m.RemoteIPMasked3 != "" {
		q := messageQuery(&store.Message{RemoteIPMasked3: m.RemoteIPMasked3}, year/4, 50)
		msgs = xmessageList(q, "ip3")
		need = 10
		method = methodIP3
		ip = m.RemoteIPMasked3
	}
	if len(msgs) > 0 {
		nspam := 0
		for _, m := range msgs {
			if m.Junk {
				nspam++
			}
		}
		pspam := float64(nspam) / float64(len(msgs))
		var spam *bool
		if pspam < .25 {
			spam = xfalse
		} else if pspam > .75 {
			spam = xtrue
		}
		conclusive := len(msgs) >= need && (pspam <= 0.1 || pspam >= 0.9)
		v6 := strings.Contains(m.RemoteIP, ":")
		reasonText = fmt.Sprintf("reputation for ip %s%s, spam score %.2f", ip, maskclasses[classmask{v6, method}], pspam)
		return spam, conclusive, method, reasonText, nil
	}

	return nil, false, methodNone, "no address/spf/dkim/ip reputation", nil
}

type classmask struct {
	v6     bool
	method reputationMethod
}

var maskclasses = map[classmask]string{
	{false, methodIP1}: "/32",
	{false, methodIP2}: "/26",
	{false, methodIP3}: "/21",
	{true, methodIP1}:  "/64",
	{true, methodIP2}:  "/48",
	{true, methodIP3}:  "/32",
}