2023-01-30 16:27:06 +03:00
package smtpserver
import (
"context"
2023-04-20 15:16:56 +03:00
"fmt"
2024-02-08 16:49:01 +03:00
"log/slog"
2023-01-30 16:27:06 +03:00
"net"
"os"
2023-11-10 21:34:00 +03:00
"strings"
2023-01-30 16:27:06 +03:00
"time"
"github.com/mjl-/bstore"
2024-04-24 20:15:30 +03:00
"github.com/mjl-/mox/config"
2023-01-30 16:27:06 +03:00
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dmarc"
"github.com/mjl-/mox/dmarcrpt"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dnsbl"
"github.com/mjl-/mox/iprev"
2023-11-27 12:34:01 +03:00
"github.com/mjl-/mox/message"
2023-01-30 16:27:06 +03:00
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
2023-11-10 21:34:00 +03:00
"github.com/mjl-/mox/publicsuffix"
2023-01-30 16:27:06 +03:00
"github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/store"
"github.com/mjl-/mox/subjectpass"
"github.com/mjl-/mox/tlsrpt"
)
type delivery struct {
2024-04-24 20:15:30 +03:00
tls bool
m * store . Message
dataFile * os . File
smtpRcptTo smtp . Path // As used in SMTP, possibly address of alias.
deliverTo smtp . Path // To deliver to, either smtpRcptTo or an alias member address.
destination config . Destination
canonicalAddress string
acc * store . Account
msgTo [ ] message . Address
msgCc [ ] message . Address
msgFrom smtp . Address
dnsBLs [ ] dns . Domain
dmarcUse bool
dmarcResult dmarc . Result
dkimResults [ ] dkim . Result
iprevStatus iprev . Status
2023-01-30 16:27:06 +03:00
}
type analysis struct {
2024-04-24 20:15:30 +03:00
d delivery
2023-11-01 19:55:40 +03:00
accept bool
mailbox string
code int
secode string
userError bool
errmsg string
err error // For our own logging, not sent to remote.
dmarcReport * dmarcrpt . Feedback // Validated DMARC aggregate report, not yet stored.
tlsReport * tlsrpt . Report // Validated TLS report, not yet stored.
reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.
dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride
2023-11-10 21:34:00 +03:00
// Additional headers to add during delivery. Used for reasons a message to a
// dmarc/tls reporting address isn't processed.
headers string
2023-01-30 16:27:06 +03:00
}
const (
reasonListAllow = "list-allow"
reasonDMARCPolicy = "dmarc-policy"
reasonReputationError = "reputation-error"
reasonReporting = "reporting"
reasonSPFPolicy = "spf-policy"
reasonJunkClassifyError = "junk-classify-error"
reasonJunkFilterError = "junk-filter-error"
reasonGiveSubjectpass = "give-subjectpass"
reasonNoBadSignals = "no-bad-signals"
reasonJunkContent = "junk-content"
reasonJunkContentStrict = "junk-content-strict"
reasonDNSBlocklisted = "dns-blocklisted"
reasonSubjectpass = "subjectpass"
reasonSubjectpassError = "subjectpass-error"
2024-04-24 20:15:30 +03:00
reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev.
reasonHighRate = "high-rate" // Too many messages, not added to rejects.
2023-01-30 16:27:06 +03:00
)
2023-11-02 22:03:47 +03:00
func isListDomain ( d delivery , ld dns . Domain ) bool {
if d . m . MailFromValidated && ld . Name ( ) == d . m . MailFromDomain {
return true
}
for _ , r := range d . dkimResults {
if r . Status == dkim . StatusPass && r . Sig . Domain == ld {
return true
}
}
return false
}
2023-12-05 15:35:58 +03:00
func analyze ( ctx context . Context , log mlog . Log , resolver dns . Resolver , d delivery ) analysis {
2023-11-10 21:34:00 +03:00
var headers string
2024-04-24 20:15:30 +03:00
// We don't want to let a single IP or network deliver too many messages to an
// account. They may fill up the mailbox, either with messages that have to be
// purged, or by filling the disk. We check both cases for IP's and networks.
var rateError bool // Whether returned error represents a rate error.
err := d . acc . DB . Read ( ctx , func ( tx * bstore . Tx ) ( retErr error ) {
now := time . Now ( )
defer func ( ) {
log . Debugx ( "checking message and size delivery rates" , retErr , slog . Duration ( "duration" , time . Since ( now ) ) )
} ( )
checkCount := func ( msg store . Message , window time . Duration , limit int ) {
if retErr != nil {
return
}
q := bstore . QueryTx [ store . Message ] ( tx )
q . FilterNonzero ( msg )
q . FilterGreater ( "Received" , now . Add ( - window ) )
q . FilterEqual ( "Expunged" , false )
n , err := q . Count ( )
if err != nil {
retErr = err
return
}
if n >= limit {
rateError = true
retErr = fmt . Errorf ( "more than %d messages in past %s from your ip/network" , limit , window )
}
}
checkSize := func ( msg store . Message , window time . Duration , limit int64 ) {
if retErr != nil {
return
}
q := bstore . QueryTx [ store . Message ] ( tx )
q . FilterNonzero ( msg )
q . FilterGreater ( "Received" , now . Add ( - window ) )
q . FilterEqual ( "Expunged" , false )
size := d . m . Size
err := q . ForEach ( func ( v store . Message ) error {
size += v . Size
return nil
} )
if err != nil {
retErr = err
return
}
if size > limit {
rateError = true
retErr = fmt . Errorf ( "more than %d bytes in past %s from your ip/network" , limit , window )
}
}
// todo future: make these configurable
// todo: should we have a limit for forwarded messages? they are stored with empty RemoteIPMasked*
const day = 24 * time . Hour
checkCount ( store . Message { RemoteIPMasked1 : d . m . RemoteIPMasked1 } , time . Minute , limitIPMasked1MessagesPerMinute )
checkCount ( store . Message { RemoteIPMasked1 : d . m . RemoteIPMasked1 } , day , 20 * 500 )
checkCount ( store . Message { RemoteIPMasked2 : d . m . RemoteIPMasked2 } , time . Minute , 1500 )
checkCount ( store . Message { RemoteIPMasked2 : d . m . RemoteIPMasked2 } , day , 20 * 1500 )
checkCount ( store . Message { RemoteIPMasked3 : d . m . RemoteIPMasked3 } , time . Minute , 4500 )
checkCount ( store . Message { RemoteIPMasked3 : d . m . RemoteIPMasked3 } , day , 20 * 4500 )
const MB = 1024 * 1024
checkSize ( store . Message { RemoteIPMasked1 : d . m . RemoteIPMasked1 } , time . Minute , limitIPMasked1SizePerMinute )
checkSize ( store . Message { RemoteIPMasked1 : d . m . RemoteIPMasked1 } , day , 3 * 1000 * MB )
checkSize ( store . Message { RemoteIPMasked2 : d . m . RemoteIPMasked2 } , time . Minute , 3000 * MB )
checkSize ( store . Message { RemoteIPMasked2 : d . m . RemoteIPMasked2 } , day , 3 * 3000 * MB )
checkSize ( store . Message { RemoteIPMasked3 : d . m . RemoteIPMasked3 } , time . Minute , 9000 * MB )
checkSize ( store . Message { RemoteIPMasked3 : d . m . RemoteIPMasked3 } , day , 3 * 9000 * MB )
return retErr
} )
if err != nil && ! rateError {
log . Errorx ( "checking delivery rates" , err )
metricDelivery . WithLabelValues ( "checkrates" , "" ) . Inc ( )
return analysis { d , false , "" , smtp . C451LocalErr , smtp . SeSys3Other0 , false , "error processing" , err , nil , nil , reasonReputationError , "" , headers }
} else if err != nil {
log . Debugx ( "refusing due to high delivery rate" , err )
metricDelivery . WithLabelValues ( "highrate" , "" ) . Inc ( )
return analysis { d , false , "" , smtp . C452StorageFull , smtp . SeMailbox2Full2 , true , err . Error ( ) , err , nil , nil , reasonHighRate , "" , headers }
}
mailbox := d . destination . Mailbox
2023-08-09 19:03:29 +03:00
if mailbox == "" {
mailbox = "Inbox"
2023-01-30 16:27:06 +03:00
}
// If destination mailbox has a mailing list domain (for SPF/DKIM) configured,
// check it for a pass.
2024-04-24 20:15:30 +03:00
rs := store . MessageRuleset ( log , d . destination , d . m , d . m . MsgPrefix , d . dataFile )
2023-08-09 19:03:29 +03:00
if rs != nil {
mailbox = rs . Mailbox
}
2023-01-30 16:27:06 +03:00
if rs != nil && ! rs . ListAllowDNSDomain . IsZero ( ) {
// todo: on temporary failures, reject temporarily?
2023-11-02 22:03:47 +03:00
if isListDomain ( d , rs . ListAllowDNSDomain ) {
d . m . IsMailingList = true
2024-04-24 20:15:30 +03:00
return analysis { d : d , accept : true , mailbox : mailbox , reason : reasonListAllow , dmarcOverrideReason : string ( dmarcrpt . PolicyOverrideMailingList ) , headers : headers }
2023-01-30 16:27:06 +03:00
}
}
2023-11-01 19:55:40 +03:00
var dmarcOverrideReason string
2023-08-09 23:31:37 +03:00
// For forwarded messages, we have different junk analysis. We don't reject for
// failing DMARC, and we clear fields that could implicate the forwarding mail
// server during future classifications on incoming messages (the forwarding mail
// server isn't responsible for the message).
if rs != nil && rs . IsForward {
d . dmarcUse = false
d . m . IsForward = true
d . m . RemoteIPMasked1 = ""
d . m . RemoteIPMasked2 = ""
d . m . RemoteIPMasked3 = ""
d . m . OrigEHLODomain = d . m . EHLODomain
d . m . EHLODomain = ""
d . m . MailFromDomain = "" // Still available in MailFrom.
d . m . OrigDKIMDomains = d . m . DKIMDomains
dkimdoms := [ ] string { }
for _ , dom := range d . m . DKIMDomains {
if dom != rs . VerifiedDNSDomain . Name ( ) {
dkimdoms = append ( dkimdoms , dom )
}
}
d . m . DKIMDomains = dkimdoms
2023-11-01 19:55:40 +03:00
dmarcOverrideReason = string ( dmarcrpt . PolicyOverrideForwarded )
2023-08-09 23:31:37 +03:00
log . Info ( "forwarded message, clearing identifying signals of forwarding mail server" )
}
2023-09-22 16:43:25 +03:00
assignMailbox := func ( tx * bstore . Tx ) error {
// Set message MailboxID to which mail will be delivered. Reputation is
// per-mailbox. If referenced mailbox is not found (e.g. does not yet exist), we
// can still determine a reputation because we also base it on outgoing
// messages and those are account-global.
mb , err := d . acc . MailboxFind ( tx , mailbox )
if err != nil {
return fmt . Errorf ( "finding destination mailbox: %w" , err )
}
if mb != nil {
// We want to deliver to mb.ID, but this message may be rejected and sent to the
// Rejects mailbox instead, with MailboxID overwritten. Record the ID in
// MailboxDestinedID too. If the message is later moved out of the Rejects mailbox,
// we'll adjust the MailboxOrigID so it gets taken into account during reputation
// calculating in future deliveries. If we end up delivering to the intended
// mailbox (i.e. not rejecting), MailboxDestinedID is cleared during delivery so we
// don't store it unnecessarily.
d . m . MailboxID = mb . ID
d . m . MailboxDestinedID = mb . ID
} else {
2023-12-05 15:35:58 +03:00
log . Debug ( "mailbox not found in database" , slog . String ( "mailbox" , mailbox ) )
2023-09-22 16:43:25 +03:00
}
return nil
}
2023-08-09 19:03:29 +03:00
reject := func ( code int , secode string , errmsg string , err error , reason string ) analysis {
2023-09-22 16:43:25 +03:00
// We may have set MailboxDestinedID below already while we had a transaction. If
// not, do it now. This makes it possible to use the per-mailbox reputation when a
// user moves the message out of the Rejects mailbox to the intended mailbox
// (typically Inbox).
if d . m . MailboxDestinedID == 0 {
var mberr error
d . acc . WithRLock ( func ( ) {
mberr = d . acc . DB . Read ( ctx , func ( tx * bstore . Tx ) error {
return assignMailbox ( tx )
} )
} )
if mberr != nil {
2024-04-24 20:15:30 +03:00
return analysis { d , false , mailbox , smtp . C451LocalErr , smtp . SeSys3Other0 , false , "error processing" , err , nil , nil , reasonReputationError , dmarcOverrideReason , headers }
2023-09-22 16:43:25 +03:00
}
d . m . MailboxID = 0 // We plan to reject, no need to set intended MailboxID.
}
2023-08-09 19:03:29 +03:00
accept := false
if rs != nil && rs . AcceptRejectsToMailbox != "" {
accept = true
mailbox = rs . AcceptRejectsToMailbox
d . m . IsReject = true
// Don't draw attention, but don't go so far as to mark as junk.
d . m . Seen = true
log . Info ( "accepting reject to configured mailbox due to ruleset" )
}
2024-04-24 20:15:30 +03:00
return analysis { d , accept , mailbox , code , secode , err == nil , errmsg , err , nil , nil , reason , dmarcOverrideReason , headers }
2023-08-09 19:03:29 +03:00
}
2023-01-30 16:27:06 +03:00
if d . dmarcUse && d . dmarcResult . Reject {
return reject ( smtp . C550MailboxUnavail , smtp . SePol7MultiAuthFails26 , "rejecting per dmarc policy" , nil , reasonDMARCPolicy )
}
// todo: should we also reject messages that have a dmarc pass but an spf record "v=spf1 -all"? suggested by m3aawg best practices.
// If destination is the DMARC reporting mailbox, do additional checks and keep
// track of the report. We'll check reputation, defaulting to accept.
var dmarcReport * dmarcrpt . Feedback
2024-04-24 20:15:30 +03:00
if d . destination . DMARCReports {
2023-11-01 19:55:40 +03:00
// Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866
2023-01-30 16:27:06 +03:00
if d . dmarcResult . Status != dmarc . StatusPass {
2023-11-01 19:55:40 +03:00
log . Info ( "received dmarc aggregate report without dmarc pass, not processing as dmarc report" )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-DMARCReport-Error: no DMARC pass\r\n"
2023-12-05 15:35:58 +03:00
} else if report , err := dmarcrpt . ParseMessageReport ( log . Logger , store . FileMsgReader ( d . m . MsgPrefix , d . dataFile ) ) ; err != nil {
2023-11-01 19:55:40 +03:00
log . Infox ( "parsing dmarc aggregate report" , err )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-DMARCReport-Error: could not parse report\r\n"
2023-01-30 16:27:06 +03:00
} else if d , err := dns . ParseDomain ( report . PolicyPublished . Domain ) ; err != nil {
2023-11-01 19:55:40 +03:00
log . Infox ( "parsing domain in dmarc aggregate report" , err )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-DMARCReport-Error: could not parse domain in published policy\r\n"
2023-01-30 16:27:06 +03:00
} else if _ , ok := mox . Conf . Domain ( d ) ; ! ok {
2023-12-05 15:35:58 +03:00
log . Info ( "dmarc aggregate report for domain not configured, ignoring" , slog . Any ( "domain" , d ) )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-DMARCReport-Error: published policy domain unrecognized\r\n"
2023-01-30 16:27:06 +03:00
} else if report . ReportMetadata . DateRange . End > time . Now ( ) . Unix ( ) + 60 {
2023-12-05 15:35:58 +03:00
log . Info ( "dmarc aggregate report with end date in the future, ignoring" , slog . Any ( "domain" , d ) , slog . Time ( "end" , time . Unix ( report . ReportMetadata . DateRange . End , 0 ) ) )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-DMARCReport-Error: report has end date in the future\r\n"
2023-01-30 16:27:06 +03:00
} else {
dmarcReport = report
}
}
// Similar to DMARC reporting, we check for the required DKIM. We'll check
// reputation, defaulting to accept.
var tlsReport * tlsrpt . Report
2024-04-24 20:15:30 +03:00
if d . destination . HostTLSReports || d . destination . DomainTLSReports {
2023-11-10 21:34:00 +03:00
matchesDomain := func ( sigDomain dns . Domain ) bool {
// RFC seems to require exact DKIM domain match with submitt and message From, we
// also allow msgFrom to be subdomain. ../rfc/8460:322
2023-12-05 15:35:58 +03:00
return sigDomain == d . msgFrom . Domain || strings . HasSuffix ( d . msgFrom . Domain . ASCII , "." + sigDomain . ASCII ) && publicsuffix . Lookup ( ctx , log . Logger , d . msgFrom . Domain ) == publicsuffix . Lookup ( ctx , log . Logger , sigDomain )
2023-11-10 21:34:00 +03:00
}
2023-01-30 16:27:06 +03:00
// Valid DKIM signature for domain must be present. We take "valid" to assume
// "passing", not "syntactically valid". We also check for "tlsrpt" as service.
// This check is optional, but if anyone goes through the trouble to explicitly
// list allowed services, they would be surprised to see them ignored.
// ../rfc/8460:320
ok := false
for _ , r := range d . dkimResults {
implement outgoing tls reports
we were already accepting, processing and displaying incoming tls reports. now
we start tracking TLS connection and security-policy-related errors for
outgoing message deliveries as well. we send reports once a day, to the
reporting addresses specified in TLSRPT records (rua) of a policy domain. these
reports are about MTA-STS policies and/or DANE policies, and about
STARTTLS-related failures.
sending reports is enabled by default, but can be disabled through setting
NoOutgoingTLSReports in mox.conf.
only at the end of the implementation process came the realization that the
TLSRPT policy domain for DANE (MX) hosts are separate from the TLSRPT policy
for the recipient domain, and that MTA-STS and DANE TLS/policy results are
typically delivered in separate reports. so MX hosts need their own TLSRPT
policies.
config for the per-host TLSRPT policy should be added to mox.conf for existing
installs, in field HostTLSRPT. it is automatically configured by quickstart for
new installs. with a HostTLSRPT config, the "dns records" and "dns check" admin
pages now suggest the per-host TLSRPT record. by creating that record, you're
requesting TLS reports about your MX host.
gathering all the TLS/policy results is somewhat tricky. the tentacles go
throughout the code. the positive result is that the TLS/policy-related code
had to be cleaned up a bit. for example, the smtpclient TLS modes now reflect
reality better, with independent settings about whether PKIX and/or DANE
verification has to be done, and/or whether verification errors have to be
ignored (e.g. for tls-required: no header). also, cached mtasts policies of
mode "none" are now cleaned up once the MTA-STS DNS record goes away.
2023-11-09 19:40:46 +03:00
// The record should have an allowed service "tlsrpt". The RFC mentions it as if
// the service must be specified explicitly, but the default allowed services for a
2023-11-10 21:34:00 +03:00
// DKIM record are "*", which includes "tlsrpt". Unless a DKIM record explicitly
// specifies services (e.g. s=email), a record will work for TLS reports. The DKIM
// records seen used for TLS reporting in the wild don't explicitly set "s" for
// services.
implement outgoing tls reports
we were already accepting, processing and displaying incoming tls reports. now
we start tracking TLS connection and security-policy-related errors for
outgoing message deliveries as well. we send reports once a day, to the
reporting addresses specified in TLSRPT records (rua) of a policy domain. these
reports are about MTA-STS policies and/or DANE policies, and about
STARTTLS-related failures.
sending reports is enabled by default, but can be disabled through setting
NoOutgoingTLSReports in mox.conf.
only at the end of the implementation process came the realization that the
TLSRPT policy domain for DANE (MX) hosts are separate from the TLSRPT policy
for the recipient domain, and that MTA-STS and DANE TLS/policy results are
typically delivered in separate reports. so MX hosts need their own TLSRPT
policies.
config for the per-host TLSRPT policy should be added to mox.conf for existing
installs, in field HostTLSRPT. it is automatically configured by quickstart for
new installs. with a HostTLSRPT config, the "dns records" and "dns check" admin
pages now suggest the per-host TLSRPT record. by creating that record, you're
requesting TLS reports about your MX host.
gathering all the TLS/policy results is somewhat tricky. the tentacles go
throughout the code. the positive result is that the TLS/policy-related code
had to be cleaned up a bit. for example, the smtpclient TLS modes now reflect
reality better, with independent settings about whether PKIX and/or DANE
verification has to be done, and/or whether verification errors have to be
ignored (e.g. for tls-required: no header). also, cached mtasts policies of
mode "none" are now cleaned up once the MTA-STS DNS record goes away.
2023-11-09 19:40:46 +03:00
// ../rfc/8460:326
2023-11-10 21:34:00 +03:00
if r . Status == dkim . StatusPass && matchesDomain ( r . Sig . Domain ) && r . Sig . Length < 0 && r . Record . ServiceAllowed ( "tlsrpt" ) {
2023-01-30 16:27:06 +03:00
ok = true
break
}
}
if ! ok {
2023-02-12 01:54:22 +03:00
log . Info ( "received mail to tlsrpt without acceptable DKIM signature, not processing as tls report" )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-TLSReport-Error: no acceptable DKIM signature\r\n"
2023-12-31 13:55:22 +03:00
} else if reportJSON , err := tlsrpt . ParseMessage ( log . Logger , store . FileMsgReader ( d . m . MsgPrefix , d . dataFile ) ) ; err != nil {
2023-02-12 01:54:22 +03:00
log . Infox ( "parsing tls report" , err )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-TLSReport-Error: could not parse TLS report\r\n"
2023-01-30 16:27:06 +03:00
} else {
var known bool
2023-12-31 13:55:22 +03:00
for _ , p := range reportJSON . Policies {
2023-12-05 15:35:58 +03:00
log . Info ( "tlsrpt policy domain" , slog . String ( "domain" , p . Policy . Domain ) )
2023-01-30 16:27:06 +03:00
if d , err := dns . ParseDomain ( p . Policy . Domain ) ; err != nil {
2023-02-12 01:54:22 +03:00
log . Infox ( "parsing domain in tls report" , err )
2023-11-13 10:36:13 +03:00
} else if _ , ok := mox . Conf . Domain ( d ) ; ok || d == mox . Conf . Static . HostnameDomain {
2023-01-30 16:27:06 +03:00
known = true
break
}
}
if ! known {
2023-02-12 01:54:22 +03:00
log . Info ( "tls report without one of configured domains, ignoring" )
2023-11-10 21:34:00 +03:00
headers += "X-Mox-TLSReport-Error: report for unknown domain\r\n"
2023-01-30 16:27:06 +03:00
} else {
2023-12-31 13:55:22 +03:00
report := reportJSON . Convert ( )
tlsReport = & report
2023-01-30 16:27:06 +03:00
}
}
}
// Determine if message is acceptable based on DMARC domain, DKIM identities, or
// host-based reputation.
var isjunk * bool
var conclusive bool
var method reputationMethod
var reason string
d . acc . WithRLock ( func ( ) {
2023-05-22 15:40:36 +03:00
err = d . acc . DB . Read ( ctx , func ( tx * bstore . Tx ) error {
2023-09-22 16:43:25 +03:00
if err := assignMailbox ( tx ) ; err != nil {
return err
2023-01-30 16:27:06 +03:00
}
isjunk , conclusive , method , err = reputation ( tx , log , d . m )
reason = string ( method )
return err
} )
} )
if err != nil {
2023-12-05 15:35:58 +03:00
log . Infox ( "determining reputation" , err , slog . Any ( "message" , d . m ) )
2023-01-30 16:27:06 +03:00
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , err , reasonReputationError )
}
2023-12-05 18:06:50 +03:00
log . Info ( "reputation analyzed" ,
slog . Bool ( "conclusive" , conclusive ) ,
slog . Any ( "isjunk" , isjunk ) ,
slog . String ( "method" , string ( method ) ) )
2023-01-30 16:27:06 +03:00
if conclusive {
if ! * isjunk {
2024-04-24 20:15:30 +03:00
return analysis { d : d , accept : true , mailbox : mailbox , dmarcReport : dmarcReport , tlsReport : tlsReport , reason : reason , dmarcOverrideReason : dmarcOverrideReason , headers : headers }
2023-01-30 16:27:06 +03:00
}
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , err , string ( method ) )
} else if dmarcReport != nil || tlsReport != nil {
2023-11-01 19:55:40 +03:00
log . Info ( "accepting message with dmarc aggregate report or tls report without reputation" )
2024-04-24 20:15:30 +03:00
return analysis { d : d , accept : true , mailbox : mailbox , dmarcReport : dmarcReport , tlsReport : tlsReport , reason : reasonReporting , dmarcOverrideReason : dmarcOverrideReason , headers : headers }
2023-01-30 16:27:06 +03:00
}
// If there was no previous message from sender or its domain, and we have an SPF
// (soft)fail, reject the message.
switch method {
case methodDKIMSPF , methodIP1 , methodIP2 , methodIP3 , methodNone :
switch d . m . MailFromValidation {
case store . ValidationFail , store . ValidationSoftfail :
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , nil , reasonSPFPolicy )
}
}
// Senders without reputation and without iprev pass, are likely spam.
var suspiciousIPrevFail bool
switch method {
case methodDKIMSPF , methodIP1 , methodIP2 , methodIP3 , methodNone :
suspiciousIPrevFail = d . iprevStatus != iprev . StatusPass
}
// With already a mild junk signal, an iprev fail on top is enough to reject.
if suspiciousIPrevFail && isjunk != nil && * isjunk {
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , nil , reasonIPrev )
}
var subjectpassKey string
conf , _ := d . acc . Conf ( )
if conf . SubjectPass . Period > 0 {
2024-04-24 20:15:30 +03:00
subjectpassKey , err = d . acc . Subjectpass ( d . canonicalAddress )
2023-01-30 16:27:06 +03:00
if err != nil {
log . Errorx ( "get key for verifying subject token" , err )
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , err , reasonSubjectpassError )
}
2023-12-05 15:35:58 +03:00
err = subjectpass . Verify ( log . Logger , d . dataFile , [ ] byte ( subjectpassKey ) , conf . SubjectPass . Period )
2023-01-30 16:27:06 +03:00
pass := err == nil
2023-12-05 15:35:58 +03:00
log . Infox ( "pass by subject token" , err , slog . Bool ( "pass" , pass ) )
2023-01-30 16:27:06 +03:00
if pass {
2024-04-24 20:15:30 +03:00
return analysis { d : d , accept : true , mailbox : mailbox , reason : reasonSubjectpass , dmarcOverrideReason : dmarcOverrideReason , headers : headers }
2023-01-30 16:27:06 +03:00
}
}
reason = reasonNoBadSignals
accept := true
var junkSubjectpass bool
2023-05-22 15:40:36 +03:00
f , jf , err := d . acc . OpenJunkFilter ( ctx , log )
2023-01-30 16:27:06 +03:00
if err == nil {
defer func ( ) {
2023-02-16 15:22:00 +03:00
err := f . Close ( )
log . Check ( err , "closing junkfilter" )
2023-01-30 16:27:06 +03:00
} ( )
2023-05-22 15:40:36 +03:00
contentProb , _ , _ , _ , err := f . ClassifyMessageReader ( ctx , store . FileMsgReader ( d . m . MsgPrefix , d . dataFile ) , d . m . Size )
2023-01-30 16:27:06 +03:00
if err != nil {
log . Errorx ( "testing for spam" , err )
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , err , reasonJunkClassifyError )
}
// todo: if isjunk is not nil (i.e. there was inconclusive reputation), use it in the probability calculation. give reputation a score of 0.25 or .75 perhaps?
// todo: if there aren't enough historic messages, we should just let messages in.
// todo: we could require nham and nspam to be above a certain number when there were plenty of words in the message, and in the database. can indicate a spammer is misspelling words. however, it can also mean a message in a different language/script...
// If we don't accept, we may still respond with a "subjectpass" hint below.
// We add some jitter to the threshold we use. So we don't act as too easy an
// oracle for words that are a strong indicator of haminess.
// todo: we should rate-limit uses of the junkfilter.
jitter := ( jitterRand . Float64 ( ) - 0.5 ) / 10
threshold := jf . Threshold + jitter
2023-11-27 12:34:01 +03:00
rcptToMatch := func ( l [ ] message . Address ) bool {
// todo: we use Go's net/mail to parse message header addresses. it does not allow empty quoted strings (contrary to spec), leaving To empty. so we don't verify To address for that unusual case for now. ../rfc/5322:961 ../rfc/5322:743
2024-04-24 20:15:30 +03:00
if d . smtpRcptTo . Localpart == "" {
2023-11-27 12:34:01 +03:00
return true
}
for _ , a := range l {
dom , err := dns . ParseDomain ( a . Host )
if err != nil {
continue
}
2024-03-08 23:08:40 +03:00
lp , err := smtp . ParseLocalpart ( a . User )
2024-04-24 20:15:30 +03:00
if err == nil && dom == d . smtpRcptTo . IPDomain . Domain && lp == d . smtpRcptTo . Localpart {
2023-11-27 12:34:01 +03:00
return true
}
}
return false
}
// todo: some of these checks should also apply for reputation-based analysis with a weak signal, e.g. verified dkim/spf signal from new domain.
// With an iprev fail, non-TLS connection or our address not in To/Cc header, we set a higher bar for content.
2023-01-30 16:27:06 +03:00
reason = reasonJunkContent
if suspiciousIPrevFail && threshold > 0.25 {
threshold = 0.25
2023-12-05 15:35:58 +03:00
log . Info ( "setting junk threshold due to iprev fail" , slog . Float64 ( "threshold" , threshold ) )
2023-11-27 12:34:01 +03:00
reason = reasonJunkContentStrict
} else if ! d . tls && threshold > 0.25 {
threshold = 0.25
2023-12-05 15:35:58 +03:00
log . Info ( "setting junk threshold due to plaintext smtp" , slog . Float64 ( "threshold" , threshold ) )
2023-11-27 12:34:01 +03:00
reason = reasonJunkContentStrict
} else if ( rs == nil || ! rs . IsForward ) && threshold > 0.25 && ! rcptToMatch ( d . msgTo ) && ! rcptToMatch ( d . msgCc ) {
// A common theme in junk messages is your recipient address not being in the To/Cc
// headers. We may be in Bcc, but that's unusual for first-time senders. Some
// providers (e.g. gmail) does not DKIM-sign Bcc headers, so junk messages can be
// sent with matching Bcc headers. We don't get here for known senders.
threshold = 0.25
2023-12-05 15:35:58 +03:00
log . Info ( "setting junk threshold due to smtp rcpt to and message to/cc address mismatch" , slog . Float64 ( "threshold" , threshold ) )
2023-01-30 16:27:06 +03:00
reason = reasonJunkContentStrict
}
accept = contentProb <= threshold
junkSubjectpass = contentProb < threshold - 0.2
2023-12-05 18:06:50 +03:00
log . Info ( "content analyzed" ,
slog . Bool ( "accept" , accept ) ,
slog . Float64 ( "contentprob" , contentProb ) ,
slog . Bool ( "subjectpass" , junkSubjectpass ) )
2023-01-30 16:27:06 +03:00
} else if err != store . ErrNoJunkFilter {
log . Errorx ( "open junkfilter" , err )
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , err , reasonJunkFilterError )
}
// If content looks good, we'll still look at DNS block lists for a reason to
// reject. We normally won't get here if we've communicated with this sender
// before.
var dnsblocklisted bool
if accept {
blocked := func ( zone dns . Domain ) bool {
dnsblctx , dnsblcancel := context . WithTimeout ( ctx , 30 * time . Second )
defer dnsblcancel ( )
2023-12-05 15:35:58 +03:00
if ! checkDNSBLHealth ( dnsblctx , log , resolver , zone ) {
log . Info ( "dnsbl not healthy, skipping" , slog . Any ( "zone" , zone ) )
2023-01-30 16:27:06 +03:00
return false
}
2023-12-05 15:35:58 +03:00
status , expl , err := dnsbl . Lookup ( dnsblctx , log . Logger , resolver , zone , net . ParseIP ( d . m . RemoteIP ) )
2023-01-30 16:27:06 +03:00
dnsblcancel ( )
if status == dnsbl . StatusFail {
2023-12-05 15:35:58 +03:00
log . Info ( "rejecting due to listing in dnsbl" , slog . Any ( "zone" , zone ) , slog . String ( "explanation" , expl ) )
2023-01-30 16:27:06 +03:00
return true
} else if err != nil {
2023-12-05 15:35:58 +03:00
log . Infox ( "dnsbl lookup" , err , slog . Any ( "zone" , zone ) , slog . Any ( "status" , status ) )
2023-01-30 16:27:06 +03:00
}
return false
}
// Note: We don't check in parallel, we are in no hurry to accept possible spam.
for _ , zone := range d . dnsBLs {
if blocked ( zone ) {
accept = false
dnsblocklisted = true
reason = reasonDNSBlocklisted
break
}
}
}
if accept {
2024-04-24 20:15:30 +03:00
return analysis { d : d , accept : true , mailbox : mailbox , reason : reasonNoBadSignals , dmarcOverrideReason : dmarcOverrideReason , headers : headers }
2023-01-30 16:27:06 +03:00
}
if subjectpassKey != "" && d . dmarcResult . Status == dmarc . StatusPass && method == methodNone && ( dnsblocklisted || junkSubjectpass ) {
log . Info ( "permanent reject with subjectpass hint of moderately spammy email without reputation" )
2023-12-05 15:35:58 +03:00
pass := subjectpass . Generate ( log . Logger , d . msgFrom , [ ] byte ( subjectpassKey ) , time . Now ( ) )
2023-01-30 16:27:06 +03:00
return reject ( smtp . C550MailboxUnavail , smtp . SePol7DeliveryUnauth1 , subjectpass . Explanation + pass , nil , reasonGiveSubjectpass )
}
return reject ( smtp . C451LocalErr , smtp . SeSys3Other0 , "error processing" , nil , reason )
}