diff --git a/.gitignore b/.gitignore index dcbe24a..00d5a2f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /testdata/check/ /testdata/ctl/data/ /testdata/ctl/dkim/ +/testdata/dmarcdb/data/ /testdata/empty/ /testdata/exportmaildir/ /testdata/exportmbox/ diff --git a/README.md b/README.md index 78218f0..bef2f39 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ See Quickstart below to get started. - SMTP (with extensions) for receiving, submitting and delivering email. - IMAP4 (with extensions) for giving email clients access to email. - Webmail for reading/sending email from the browser. -- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC reports. +- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC aggregate + reports. - Reputation tracking, learning (per user) host-, domain- and sender address-based reputation from (Non-)Junk email classification. - Bayesian spam filtering that learns (per user) from (Non-)Junk email. @@ -113,7 +114,7 @@ https://nlnet.nl/project/Mox/. ## Roadmap -- Sending DMARC and TLS reports (currently only receiving) +- Sending TLS reports (currently only receiving) - Authentication other than HTTP-basic for webmail/webadmin/webaccount - Per-domain webmail and IMAP/SMTP host name (and TLS cert) and client settings - Make mox Go packages more easily reusable, each pulling in fewer (internal) diff --git a/backup.go b/backup.go index c2bab73..a4dd60b 100644 --- a/backup.go +++ b/backup.go @@ -279,7 +279,8 @@ func backupctl(ctx context.Context, ctl *ctl) { if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil { xerrx("writing moxversion", err) } - backupDB(dmarcdb.DB, "dmarcrpt.db") + backupDB(dmarcdb.ReportsDB, "dmarcrpt.db") + backupDB(dmarcdb.EvalDB, "dmarceval.db") backupDB(mtastsdb.DB, "mtasts.db") backupDB(tlsrptdb.DB, "tlsrpt.db") backupFile("receivedid.key") @@ -529,7 +530,7 @@ func backupctl(ctx context.Context, ctl *ctl) { } switch p { - case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl": + case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl": // Already handled. return nil case "lastknownversion": // Optional file, not yet handled. diff --git a/config/config.go b/config/config.go index 2161389..625eae3 100644 --- a/config/config.go +++ b/config/config.go @@ -57,9 +57,10 @@ type Static struct { Account string Mailbox string `sconf-doc:"E.g. Postmaster or Inbox."` } `sconf-doc:"Destination for emails delivered to postmaster addresses: a plain 'postmaster' without domain, 'postmaster@' (also for each listener with SMTP enabled), and as fallback for each domain without explicitly configured postmaster destination."` - InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."` - DefaultMailboxes []string `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."` - Transports map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."` + InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."` + DefaultMailboxes []string `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."` + Transports map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."` + NoOutgoingDMARCReports bool `sconf:"optional" sconf-doc:"Do not send DMARC reports (aggregate only). By default, aggregate reports on DMARC evaluations are sent to domains if their DMARC policy requests them. Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24 hours, rounded up so a whole number of intervals cover 24 hours, aligned at whole days in UTC."` // All IPs that were explicitly listen on for external SMTP. Only set when there // are no unspecified external SMTP listeners and there is at most one for IPv4 and diff --git a/config/doc.go b/config/doc.go index e1cfbf0..80241a3 100644 --- a/config/doc.go +++ b/config/doc.go @@ -549,6 +549,13 @@ describe-static" and "mox config describe-domains": # typically the hostname of the host in the Address field. RemoteHostname: + # Do not send DMARC reports (aggregate only). By default, aggregate reports on + # DMARC evaluations are sent to domains if their DMARC policy requests them. + # Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24 + # hours, rounded up so a whole number of intervals cover 24 hours, aligned at + # whole days in UTC. (optional) + NoOutgoingDMARCReports: false + # domains.conf # NOTE: This config file is in 'sconf' format. Indent with tabs. Comments must be diff --git a/dkim/dkim.go b/dkim/dkim.go index 8f044b9..83a34e5 100644 --- a/dkim/dkim.go +++ b/dkim/dkim.go @@ -382,14 +382,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig) if err != nil { - results = append(results, Result{StatusPermerror, nil, nil, false, err}) + results = append(results, Result{StatusPermerror, sig, nil, false, err}) continue } // ../rfc/6376:2560 if err := policy(sig); err != nil { err := fmt.Errorf("%w: %s", ErrPolicy, err) - results = append(results, Result{StatusPolicy, nil, nil, false, err}) + results = append(results, Result{StatusPolicy, sig, nil, false, err}) continue } diff --git a/dmarc/dmarc.go b/dmarc/dmarc.go index 0af56cd..50d5dbb 100644 --- a/dmarc/dmarc.go +++ b/dmarc/dmarc.go @@ -75,7 +75,9 @@ type Result struct { Reject bool // Result of DMARC validation. A message can fail validation, but still // not be rejected, e.g. if the policy is "none". - Status Status + Status Status + AlignedSPFPass bool + AlignedDKIMPass bool // Domain with the DMARC DNS record. May be the organizational domain instead of // the domain in the From-header. Domain dns.Domain @@ -142,7 +144,7 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain) return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err) } if record != nil { - // ../ ../rfc/7489:1388 + // ../rfc/7489:1388 return StatusNone, nil, "", result.Authentic, ErrMultipleRecords } text = txt @@ -152,14 +154,15 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain) return StatusNone, record, text, result.Authentic, rerr } -func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, *Record, string, bool, error) { +func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, []*Record, []string, bool, error) { + // ../rfc/7489:1566 name := dmarcDomain.ASCII + "._report._dmarc." + extDestDomain.ASCII + "." txts, result, err := dns.WithPackage(resolver, "dmarc").LookupTXT(ctx, name) if err != nil && !dns.IsNotFound(err) { - return StatusTemperror, nil, "", result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err) + return StatusTemperror, nil, nil, result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err) } - var record *Record - var text string + var records []*Record + var texts []string var rerr error = ErrNoRecord for _, txt := range txts { r, isdmarc, err := ParseRecordNoRequired(txt) @@ -171,44 +174,44 @@ func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain r, isdmarc, err = &xr, true, nil } if !isdmarc { - // ../rfc/7489:1374 + // ../rfc/7489:1586 continue - } else if err != nil { - return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err) } - if record != nil { - // ../ ../rfc/7489:1388 - return StatusNone, nil, "", result.Authentic, ErrMultipleRecords + texts = append(texts, txt) + records = append(records, r) + if err != nil { + return StatusPermerror, records, texts, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err) } - text = txt - record = r + // Multiple records are allowed for the _report record, unlike for policies. ../rfc/7489:1593 rerr = nil } - return StatusNone, record, text, result.Authentic, rerr + return StatusNone, records, texts, result.Authentic, rerr } // LookupExternalReportsAccepted returns whether the extDestDomain has opted in // to receiving dmarc reports for dmarcDomain (where the dmarc record was found), // through a "._report._dmarc." DNS TXT DMARC record. // -// Callers should look at status for interpretation, not err, because err will -// be set to ErrNoRecord when the DNS TXT record isn't present, which means the -// extDestDomain does not opt in (not a failure condition). +// accepts is true if the external domain has opted in. +// If a temporary error occurred, the returned status is StatusTemperror, and a +// later retry may give an authoritative result. +// The returned error is ErrNoRecord if no opt-in DNS record exists, which is +// not a failure condition. // // The normally invalid "v=DMARC1" record is accepted since it is used as // example in RFC 7489. // // authentic indicates if the DNS results were DNSSEC-verified. -func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, record *Record, txt string, authentic bool, rerr error) { +func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) { log := xlog.WithContext(ctx) start := time.Now() defer func() { - log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("record", record), mlog.Field("duration", time.Since(start))) + log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("records", records), mlog.Field("duration", time.Since(start))) }() - status, record, txt, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain) + status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain) accepts = rerr == nil - return accepts, status, record, txt, authentic, rerr + return accepts, status, records, txts, authentic, rerr } // Verify evaluates the DMARC policy for the domain in the From-header of a @@ -241,7 +244,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes status, recordDomain, record, _, authentic, err := Lookup(ctx, resolver, from) if record == nil { - return false, Result{false, status, recordDomain, record, authentic, err} + return false, Result{false, status, false, false, recordDomain, record, authentic, err} } result.Domain = recordDomain result.Record = record @@ -251,8 +254,8 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes // See ../rfc/7489:1432 useResult = !applyRandomPercentage || record.Percentage == 100 || mathrand.Intn(100) < record.Percentage - // We reject treat "quarantine" and "reject" the same. Thus, we also don't - // "downgrade" from reject to quarantine if this message was sampled out. + // We treat "quarantine" and "reject" the same. Thus, we also don't "downgrade" + // from reject to quarantine if this message was sampled out. // ../rfc/7489:1446 ../rfc/7489:1024 if recordDomain != from && record.SubdomainPolicy != PolicyEmpty { result.Reject = record.SubdomainPolicy != PolicyNone @@ -282,9 +285,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes // ../rfc/7489:1319 // ../rfc/7489:544 if spfResult == spf.StatusPass && spfIdentity != nil && (*spfIdentity == from || result.Record.ASPF == "r" && pubsuffix(from) == pubsuffix(*spfIdentity)) { - result.Reject = false - result.Status = StatusPass - return + result.AlignedSPFPass = true } for _, dkimResult := range dkimResults { @@ -296,10 +297,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes // ../rfc/7489:511 if dkimResult.Status == dkim.StatusPass && dkimResult.Sig != nil && (dkimResult.Sig.Domain == from || result.Record.ADKIM == "r" && pubsuffix(from) == pubsuffix(dkimResult.Sig.Domain)) { // ../rfc/7489:535 - result.Reject = false - result.Status = StatusPass - return + result.AlignedDKIMPass = true + break } } + + if result.AlignedSPFPass || result.AlignedDKIMPass { + result.Reject = false + result.Status = StatusPass + } return } diff --git a/dmarc/dmarc_test.go b/dmarc/dmarc_test.go index 019ee94..4e8c617 100644 --- a/dmarc/dmarc_test.go +++ b/dmarc/dmarc_test.go @@ -84,7 +84,7 @@ func TestLookupExternalReportsAccepted(t *testing.T) { test("example.com", "simple2.example", StatusNone, true, nil) test("example.com", "one.example", StatusNone, true, nil) test("example.com", "absent.example", StatusNone, false, ErrNoRecord) - test("example.com", "multiple.example", StatusNone, false, ErrMultipleRecords) + test("example.com", "multiple.example", StatusNone, true, nil) test("example.com", "malformed.example", StatusPermerror, false, ErrSyntax) test("example.com", "temperror.example", StatusTemperror, false, ErrDNS) } @@ -137,7 +137,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusNone, nil, - true, Result{true, StatusFail, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, + true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, ) // Accept with spf pass. @@ -145,7 +145,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusPass, &dns.Domain{ASCII: "sub.reject.example"}, - true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, + true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, ) // Accept with dkim pass. @@ -161,7 +161,7 @@ func TestVerify(t *testing.T) { }, spf.StatusFail, &dns.Domain{ASCII: "reject.example"}, - true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, + true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, ) // Reject due to spf and dkim "strict". @@ -181,7 +181,7 @@ func TestVerify(t *testing.T) { }, spf.StatusPass, &dns.Domain{ASCII: "sub.strict.example"}, - true, Result{true, StatusFail, dns.Domain{ASCII: "strict.example"}, &strict, false, nil}, + true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "strict.example"}, &strict, false, nil}, ) // No dmarc policy, nothing to say. @@ -189,7 +189,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusNone, nil, - false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord}, + false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord}, ) // No dmarc policy, spf pass does nothing. @@ -197,7 +197,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusPass, &dns.Domain{ASCII: "absent.example"}, - false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord}, + false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord}, ) none := DefaultRecord @@ -207,7 +207,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusPass, &dns.Domain{ASCII: "none.example"}, - true, Result{false, StatusPass, dns.Domain{ASCII: "none.example"}, &none, false, nil}, + true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "none.example"}, &none, false, nil}, ) // No actual reject due to pct=0. @@ -218,7 +218,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusNone, nil, - false, Result{true, StatusFail, dns.Domain{ASCII: "test.example"}, &testr, false, nil}, + false, Result{true, StatusFail, false, false, dns.Domain{ASCII: "test.example"}, &testr, false, nil}, ) // No reject if subdomain has "none" policy. @@ -229,7 +229,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusFail, &dns.Domain{ASCII: "sub.subnone.example"}, - true, Result{false, StatusFail, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil}, + true, Result{false, StatusFail, false, false, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil}, ) // No reject if spf temperror and no other pass. @@ -237,7 +237,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusTemperror, &dns.Domain{ASCII: "mail.reject.example"}, - true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, + true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, ) // No reject if dkim temperror and no other pass. @@ -253,7 +253,7 @@ func TestVerify(t *testing.T) { }, spf.StatusNone, nil, - true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, + true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, ) // No reject if spf temperror but still dkim pass. @@ -269,7 +269,7 @@ func TestVerify(t *testing.T) { }, spf.StatusTemperror, &dns.Domain{ASCII: "mail.reject.example"}, - true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, + true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, ) // No reject if dkim temperror but still spf pass. @@ -285,7 +285,7 @@ func TestVerify(t *testing.T) { }, spf.StatusPass, &dns.Domain{ASCII: "mail.reject.example"}, - true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, + true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil}, ) // Bad DMARC record results in permerror without reject. @@ -293,7 +293,7 @@ func TestVerify(t *testing.T) { []dkim.Result{}, spf.StatusNone, nil, - false, Result{false, StatusPermerror, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax}, + false, Result{false, StatusPermerror, false, false, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax}, ) // DKIM domain that is higher-level than organizational can not result in a pass. ../rfc/7489:525 @@ -309,6 +309,6 @@ func TestVerify(t *testing.T) { }, spf.StatusNone, nil, - true, Result{true, StatusFail, dns.Domain{ASCII: "example.com"}, &reject, false, nil}, + true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "example.com"}, &reject, false, nil}, ) } diff --git a/dmarcdb/dmarcdb.go b/dmarcdb/dmarcdb.go new file mode 100644 index 0000000..76b957d --- /dev/null +++ b/dmarcdb/dmarcdb.go @@ -0,0 +1,29 @@ +// Package dmarcdb stores incoming DMARC aggrate reports and evaluations for outgoing aggregate reports. +// +// With DMARC, a domain can request reports with DMARC evaluation results to be +// sent to a specified address. Mox parses such reports, stores them in its +// database and makes them available through its admin web interface. Mox also +// keeps track of the evaluations it does for incoming messages and sends reports +// to mail servers that request reports. +// +// Only aggregate reports are stored and sent. Failure reports about individual +// messages are not implemented. +package dmarcdb + +import ( + "github.com/mjl-/mox/mox-" +) + +// Init opens the databases. +// +// The incoming reports and evaluations for outgoing reports are in separate +// databases for simpler file-based handling of the databases. +func Init() error { + if _, err := reportsDB(mox.Shutdown); err != nil { + return err + } + if _, err := evalDB(mox.Shutdown); err != nil { + return err + } + return nil +} diff --git a/dmarcdb/eval.go b/dmarcdb/eval.go new file mode 100644 index 0000000..1bd87f6 --- /dev/null +++ b/dmarcdb/eval.go @@ -0,0 +1,1112 @@ +package dmarcdb + +import ( + "bufio" + "compress/gzip" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net/mail" + "net/textproto" + "net/url" + "os" + "path/filepath" + "runtime/debug" + "sort" + "strings" + "sync" + "time" + + "golang.org/x/exp/maps" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/mjl-/bstore" + + "github.com/mjl-/mox/dkim" + "github.com/mjl-/mox/dmarc" + "github.com/mjl-/mox/dmarcrpt" + "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/message" + "github.com/mjl-/mox/metrics" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/moxio" + "github.com/mjl-/mox/moxvar" + "github.com/mjl-/mox/publicsuffix" + "github.com/mjl-/mox/queue" + "github.com/mjl-/mox/smtp" + "github.com/mjl-/mox/store" +) + +var ( + metricReport = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "mox_dmarcdb_report_queued_total", + Help: "Total messages with DMARC aggregate/error reports queued.", + }, + ) + metricReportError = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "mox_dmarcdb_report_error_total", + Help: "Total errors while composing or queueing DMARC aggregate/error reports.", + }, + ) +) + +var ( + EvalDBTypes = []any{Evaluation{}} // Types stored in DB. + // Exported for backups. For incoming deliveries the SMTP server adds evaluations + // to the database. Every hour, a goroutine wakes up that gathers evaluations from + // the last hour(s), sends a report, and removes the evaluations from the database. + EvalDB *bstore.DB + evalMutex sync.Mutex +) + +// Evaluation is the result of an evaluation of a DMARC policy, to be included +// in a DMARC report. +type Evaluation struct { + ID int64 + + // Domain where DMARC policy was found, could be the organizational domain while + // evaluation was for a subdomain. Unicode. Same as domain found in + // PolicyPublished. A separate field for its index. + PolicyDomain string `bstore:"index"` + + // Time of evaluation, determines which report (covering whole hours) this + // evaluation will be included in. + Evaluated time.Time `bstore:"default now"` + + // If optional, this evaluation is not a reason to send a DMARC report, but it will + // be included when a report is sent due to other non-optional evaluations. Set for + // evaluations of incoming DMARC reports. We don't want such deliveries causing us to + // send a report, or we would keep exchanging reporting messages forever. Also set + // for when evaluation is a DMARC reject for domains we haven't positively + // interacted with, to prevent being used to flood an unsuspecting domain with + // reports. + Optional bool + + // Effective aggregate reporting interval in hours. Between 1 and 24, rounded up + // from seconds from policy to first number that can divide 24. + IntervalHours int + + // "rua" in DMARC record, we only store evaluations for records with aggregate reporting addresses, so always non-empty. + Addresses []string + + // Policy used for evaluation. We don't store the "fo" field for failure reporting + // options, since we don't send failure reports for individual messages. + PolicyPublished dmarcrpt.PolicyPublished + + // For "row" in a report record. + SourceIP string + Disposition dmarcrpt.Disposition + AlignedDKIMPass bool + AlignedSPFPass bool + OverrideReasons []dmarcrpt.PolicyOverrideReason + + // For "identifiers" in a report record. + EnvelopeTo string + EnvelopeFrom string + HeaderFrom string + + // For "auth_results" in a report record. + DKIMResults []dmarcrpt.DKIMAuthResult + SPFResults []dmarcrpt.SPFAuthResult +} + +var dmarcResults = map[bool]dmarcrpt.DMARCResult{ + false: dmarcrpt.DMARCFail, + true: dmarcrpt.DMARCPass, +} + +// ReportRecord turns an evaluation into a record that can be included in a +// report. +func (e Evaluation) ReportRecord(count int) dmarcrpt.ReportRecord { + return dmarcrpt.ReportRecord{ + Row: dmarcrpt.Row{ + SourceIP: e.SourceIP, + Count: count, + PolicyEvaluated: dmarcrpt.PolicyEvaluated{ + Disposition: e.Disposition, + DKIM: dmarcResults[e.AlignedDKIMPass], + SPF: dmarcResults[e.AlignedSPFPass], + Reasons: e.OverrideReasons, + }, + }, + Identifiers: dmarcrpt.Identifiers{ + EnvelopeTo: e.EnvelopeTo, + EnvelopeFrom: e.EnvelopeFrom, + HeaderFrom: e.HeaderFrom, + }, + AuthResults: dmarcrpt.AuthResults{ + DKIM: e.DKIMResults, + SPF: e.SPFResults, + }, + } +} + +func evalDB(ctx context.Context) (rdb *bstore.DB, rerr error) { + evalMutex.Lock() + defer evalMutex.Unlock() + if EvalDB == nil { + p := mox.DataDirPath("dmarceval.db") + os.MkdirAll(filepath.Dir(p), 0770) + db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, EvalDBTypes...) + if err != nil { + return nil, err + } + EvalDB = db + } + return EvalDB, nil +} + +var intervalOpts = []int{24, 12, 8, 6, 4, 3, 2} + +func intervalHours(seconds int) int { + hours := (seconds + 3600 - 1) / 3600 + for _, opt := range intervalOpts { + if hours >= opt { + return opt + } + } + return 1 +} + +// AddEvaluation adds the result of a DMARC evaluation for an incoming message +// to the database. +// +// AddEvaluation sets Evaluation.IntervalHours based on +// aggregateReportingIntervalSeconds. +func AddEvaluation(ctx context.Context, aggregateReportingIntervalSeconds int, e *Evaluation) error { + e.IntervalHours = intervalHours(aggregateReportingIntervalSeconds) + + db, err := evalDB(ctx) + if err != nil { + return err + } + + e.ID = 0 + return db.Insert(ctx, e) +} + +// Evaluations returns all evaluations in the database. +func Evaluations(ctx context.Context) ([]Evaluation, error) { + db, err := evalDB(ctx) + if err != nil { + return nil, err + } + + q := bstore.QueryDB[Evaluation](ctx, db) + q.SortAsc("Evaluated") + return q.List() +} + +// EvaluationStat summarizes stored evaluations, for inclusion in an upcoming +// aggregate report, for a domain. +type EvaluationStat struct { + Count int + SendReport bool + Domain dns.Domain +} + +// EvaluationStats returns evaluation counts and report-sending status per domain. +func EvaluationStats(ctx context.Context) (map[string]EvaluationStat, error) { + db, err := evalDB(ctx) + if err != nil { + return nil, err + } + + r := map[string]EvaluationStat{} + + err = bstore.QueryDB[Evaluation](ctx, db).ForEach(func(e Evaluation) error { + if stat, ok := r[e.PolicyDomain]; ok { + stat.Count++ + stat.SendReport = stat.SendReport || !e.Optional + r[e.PolicyDomain] = stat + } else { + dom, err := dns.ParseDomain(e.PolicyDomain) + if err != nil { + return fmt.Errorf("parsing domain %q: %v", e.PolicyDomain, err) + } + r[e.PolicyDomain] = EvaluationStat{ + Count: 1, + SendReport: !e.Optional, + Domain: dom, + } + } + return nil + }) + return r, err +} + +// EvaluationsDomain returns all evaluations for a domain. +func EvaluationsDomain(ctx context.Context, domain dns.Domain) ([]Evaluation, error) { + db, err := evalDB(ctx) + if err != nil { + return nil, err + } + + q := bstore.QueryDB[Evaluation](ctx, db) + q.FilterNonzero(Evaluation{PolicyDomain: domain.Name()}) + q.SortAsc("Evaluated") + return q.List() +} + +// RemoveEvaluationsDomain removes evaluations for domain so they won't be sent in +// an aggregate report. +func RemoveEvaluationsDomain(ctx context.Context, domain dns.Domain) error { + db, err := evalDB(ctx) + if err != nil { + return err + } + + q := bstore.QueryDB[Evaluation](ctx, db) + q.FilterNonzero(Evaluation{PolicyDomain: domain.Name()}) + _, err = q.Delete() + return err +} + +var jitterRand = mox.NewRand() + +// time to sleep until next whole hour t, replaced by tests. +// Jitter so we don't cause load at exactly whole hours, other processes may +// already be doing that. +var jitteredTimeUntil = func(t time.Time) time.Duration { + return time.Until(t.Add(time.Duration(30+jitterRand.Intn(60)) * time.Second)) +} + +// Start launches a goroutine that wakes up at each whole hour (plus jitter) and +// sends DMARC reports to domains that requested them. +func Start(resolver dns.Resolver) { + go func() { + log := mlog.New("dmarcdb") + + defer func() { + // In case of panic don't take the whole program down. + x := recover() + if x != nil { + log.Error("recover from panic", mlog.Field("panic", x)) + debug.PrintStack() + metrics.PanicInc(metrics.Dmarcdb) + } + }() + + timer := time.NewTimer(time.Hour) + defer timer.Stop() + + ctx := mox.Shutdown + + db, err := evalDB(ctx) + if err != nil { + log.Errorx("opening dmarc evaluations database for sending dmarc aggregate reports, not sending reports", err) + return + } + + for { + now := time.Now() + nextEnd := nextWholeHour(now) + timer.Reset(jitteredTimeUntil(nextEnd)) + + select { + case <-ctx.Done(): + log.Info("dmarc aggregate report sender shutting down") + return + case <-timer.C: + } + + // Gather report intervals we want to process now. Multiples of hours that can + // divide 24, starting from UTC. + // ../rfc/7489:1750 + utchour := nextEnd.UTC().Hour() + if utchour == 0 { + utchour = 24 + } + intervals := []int{} + for _, ival := range intervalOpts { + if ival*(utchour/ival) == utchour { + intervals = append(intervals, ival) + } + } + intervals = append(intervals, 1) + + // Remove evaluations older than 48 hours (2 reports with the default and maximum + // 24 hour interval). They should have been processed by now. We may have kept them + // during temporary errors, but persistent temporary errors shouldn't fill up our + // database. This also cleans up evaluations that were all optional for a domain. + _, err := bstore.QueryDB[Evaluation](ctx, db).FilterLess("Evaluated", nextEnd.Add(-48*time.Hour)).Delete() + log.Check(err, "removing stale dmarc evaluations from database") + + if err := sendReports(ctx, log.WithCid(mox.Cid()), resolver, db, nextEnd, intervals); err != nil { + log.Errorx("sending dmarc aggregate reports", err) + metricReportError.Inc() + } + } + }() +} + +func nextWholeHour(now time.Time) time.Time { + t := now + t = t.Add(time.Hour) + return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location()) +} + +// We don't send reports at full speed. In the future, we could try to stretch out +// reports a bit smarter. E.g. over 5 minutes with some minimum interval, and +// perhaps faster and in parallel when there are lots of reports. Perhaps also +// depending on reporting interval (faster for 1h, slower for 24h). +// Replaced by tests. +var sleepBetween = func() { + time.Sleep(3 * time.Second) +} + +// sendReports gathers all policy domains that have evaluations that should +// receive a DMARC report and sends a report to each. +func sendReports(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, intervals []int) error { + log.Info("sending dmarc aggregate reports", mlog.Field("end", endTime.UTC()), mlog.Field("intervals", intervals)) + + ivals := make([]any, len(intervals)) + for i, v := range intervals { + ivals[i] = v + } + + destDomains := map[string]bool{} + + // Gather all domains that we plan to send to. + q := bstore.QueryDB[Evaluation](ctx, db) + q.FilterLess("Evaluated", endTime) + q.FilterEqual("IntervalHours", ivals...) + err := q.ForEach(func(e Evaluation) error { + destDomains[e.PolicyPublished.Domain] = destDomains[e.PolicyPublished.Domain] || !e.Optional + return nil + }) + if err != nil { + return fmt.Errorf("looking for domains to send reports to: %v", err) + } + + // Attempt to send report to each domain. + for d, send := range destDomains { + // Cleanup evaluations for domain with only optionals. + if !send { + removeEvaluations(ctx, log, db, endTime, d) + continue + } + + rlog := log.WithCid(mox.Cid()).Fields(mlog.Field("domain", d)) + if _, err := sendReportDomain(ctx, rlog, resolver, db, endTime, d); err != nil { + rlog.Errorx("sending dmarc aggregate report to domain", err) + metricReportError.Inc() + } + + sleepBetween() + } + + return nil +} + +type recipient struct { + address smtp.Address + maxSize uint64 +} + +func parseRecipient(log *mlog.Log, uri dmarc.URI) (r recipient, ok bool) { + log = log.Fields(mlog.Field("uri", uri.Address)) + + u, err := url.Parse(uri.Address) + if err != nil { + log.Debugx("parsing uri in dmarc record rua value", err) + return r, false + } + if !strings.EqualFold(u.Scheme, "mailto") { + log.Debug("skipping unrecognized scheme in dmarc record rua value") + return r, false + } + addr, err := smtp.ParseAddress(u.Opaque) + if err != nil { + log.Debugx("parsing mailto uri in dmarc record rua value", err) + return r, false + } + + r = recipient{addr, uri.MaxSize} + // ../rfc/7489:1197 + switch uri.Unit { + case "k", "K": + r.maxSize *= 1024 + case "m", "M": + r.maxSize *= 1024 * 1024 + case "g", "G": + r.maxSize *= 1024 * 1024 * 1024 + case "t", "T": + // Oh yeah, terabyte-sized reports! + r.maxSize *= 1024 * 1024 * 1024 * 1024 + case "": + default: + log.Debug("unrecognized max size unit in dmarc record rua value", mlog.Field("unit", uri.Unit)) + return r, false + } + + return r, true +} + +func removeEvaluations(ctx context.Context, log *mlog.Log, db *bstore.DB, endTime time.Time, domain string) { + q := bstore.QueryDB[Evaluation](ctx, db) + q.FilterLess("Evaluated", endTime) + q.FilterNonzero(Evaluation{PolicyDomain: domain}) + _, err := q.Delete() + log.Check(err, "removing evaluations after processing for dmarc aggregate report") +} + +// replaceable for testing. +var queueAdd = queue.Add + +func sendReportDomain(ctx context.Context, log *mlog.Log, resolver dns.Resolver, db *bstore.DB, endTime time.Time, domain string) (cleanup bool, rerr error) { + dom, err := dns.ParseDomain(domain) + if err != nil { + return false, fmt.Errorf("parsing domain for sending reports: %v", err) + } + + // We'll cleanup records by default. + cleanup = true + // If we encounter a temporary error we cancel cleanup of evaluations on error. + tempError := false + + defer func() { + if !cleanup || tempError { + log.Debug("not cleaning up evaluations after attempting to send dmarc aggregate report") + } else { + removeEvaluations(ctx, log, db, endTime, domain) + } + }() + + // We're going to build up this report. + report := dmarcrpt.Feedback{ + Version: "1.0", + ReportMetadata: dmarcrpt.ReportMetadata{ + OrgName: mox.Conf.Static.HostnameDomain.ASCII, + Email: "postmaster@" + mox.Conf.Static.HostnameDomain.ASCII, + // ReportID and DateRange are set after we've seen evaluations. + // Errors is filled below when we encounter problems. + }, + // We'll fill the records below. + Records: []dmarcrpt.ReportRecord{}, + } + + var errors []string // For report.ReportMetaData.Errors + + // Check if we should be sending a report at all: if there are rua URIs in the + // current DMARC record. The interval may have changed too, but we'll flush out our + // evaluations regardless. We always use the latest DMARC record when sending, but + // we'll lump all policies of the last interval into one report. + // ../rfc/7489:1714 + status, _, record, _, _, err := dmarc.Lookup(ctx, resolver, dom) + if err != nil { + // todo future: we could perhaps still send this report, assuming the values we know. in case of temporary error, we could also schedule again regardless of next interval hour (we would now only retry a 24h-interval report after 24h passed). + // Remove records unless it was a temporary error. We'll try again next round. + cleanup = status != dmarc.StatusTemperror + return cleanup, fmt.Errorf("looking up current dmarc record for reporting address") + } + + var recipients []recipient + + // Gather all aggregate reporting addresses to try to send to. We'll start with + // those in the initial DMARC record, but will follow external reporting addresses + // and possibly update the list. + for _, uri := range record.AggregateReportAddresses { + r, ok := parseRecipient(log, uri) + if !ok { + continue + } + + // Check if domain of rua recipient has the same organizational domain as for the + // evaluations. If not, we need to verify we are allowed to send. + ruaOrgDom := publicsuffix.Lookup(ctx, r.address.Domain) + evalOrgDom := publicsuffix.Lookup(ctx, dom) + + if ruaOrgDom == evalOrgDom { + recipients = append(recipients, r) + continue + } + + // Verify and follow addresses in other organizational domain through + // ._report._dmarc. lookup. + // ../rfc/7489:1556 + accepts, status, records, _, _, err := dmarc.LookupExternalReportsAccepted(ctx, resolver, evalOrgDom, r.address.Domain) + log.Debugx("checking if rua address with different organization domain has opted into receiving dmarc reports", err, + mlog.Field("policydomain", evalOrgDom), + mlog.Field("destinationdomain", r.address.Domain), + mlog.Field("accepts", accepts), + mlog.Field("status", status)) + if status == dmarc.StatusTemperror { + // With a temporary error, we'll try to get the report the delivered anyway, + // perhaps there are multiple recipients. + // ../rfc/7489:1578 + tempError = true + errors = append(errors, "temporary error checking authorization for report delegation to external address") + } + if !accepts { + errors = append(errors, fmt.Sprintf("rua %s is external domain that does not opt-in to receiving dmarc records through _report dmarc record", r.address)) + continue + } + + // We can follow a _report DMARC DNS record once. In that record, a domain may + // specify alternative addresses that we should send reports to instead. Such + // alternative address(es) must have the same host. If not, we ignore the new + // value. Behaviour for multiple records and/or multiple new addresses is + // underspecified. We'll replace an address with one or more new addresses, and + // keep the original if there was no candidate (which covers the case of invalid + // alternative addresses and no new address specified). + // ../rfc/7489:1600 + foundReplacement := false + rlog := log.Fields(mlog.Field("followedaddress", uri.Address)) + for _, record := range records { + for _, exturi := range record.AggregateReportAddresses { + extr, ok := parseRecipient(rlog, exturi) + if !ok { + continue + } + if extr.address.Domain != r.address.Domain { + rlog.Debug("rua address in external _report dmarc record has different host than initial dmarc record, ignoring new name", mlog.Field("externaladdress", extr.address)) + errors = append(errors, fmt.Sprintf("rua %s is external domain with a replacement address %s with different host", r.address, extr.address)) + } else { + rlog.Debug("using replacement rua address from external _report dmarc record", mlog.Field("externaladdress", extr.address)) + foundReplacement = true + recipients = append(recipients, extr) + } + } + } + if !foundReplacement { + recipients = append(recipients, r) + } + } + + if len(recipients) == 0 { + // No reports requested, perfectly fine, no work to do for us. + log.Debug("no aggregate reporting addresses configured") + return true, nil + } + + // We count idential records. Can be common with a domain sending quite some email. + // Though less if the sending domain has many IPs. In the future, we may want to + // remove some details from records so we can aggregate them into fewer rows. + type recordCount struct { + dmarcrpt.ReportRecord + count int + } + counts := map[string]recordCount{} + + var first, last Evaluation // For daterange. + var sendReport bool + + q := bstore.QueryDB[Evaluation](ctx, db) + q.FilterLess("Evaluated", endTime) + q.FilterNonzero(Evaluation{PolicyDomain: domain}) + q.SortAsc("Evaluated") + err = q.ForEach(func(e Evaluation) error { + if first.ID == 0 { + first = e + } + last = e + + record := e.ReportRecord(0) + + // todo future: if we see many unique records from a single ip (exact ipv4 or ipv6 subnet), we may want to coalesce them into a single record, leaving out the fields that make them: a single ip could cause a report to contain many records with many unique domains, selectors, etc. it may compress relatively well, but the reports could still be huge. + + // Simple but inefficient way to aggregate identical records. We may want to turn + // records into smaller representation in the future. + recbuf, err := xml.Marshal(record) + if err != nil { + return fmt.Errorf("xml marshal of report record: %v", err) + } + recstr := string(recbuf) + counts[recstr] = recordCount{record, counts[recstr].count + 1} + if !e.Optional { + sendReport = true + } + return nil + }) + if err != nil { + return false, fmt.Errorf("gathering evaluations for report: %v", err) + } + + if !sendReport { + log.Debug("no non-optional evaluations for domain, not sending dmarc aggregate report") + return true, nil + } + + // Set begin and end date range. We try to set it to whole intervals as requested + // by the domain owner. The typical, default and maximum interval is 24 hours. But + // we allow any whole number of hours that can divide 24 hours. If we have an + // evaluation that is older, we may have had a failure to send earlier. We include + // those earlier intervals in this report as well. + // + // Although "end" could be interpreted as exclusive, to be on the safe side + // regarding client behaviour, and (related) to mimic large existing DMARC report + // senders, we set it to the last second of the period this report covers. + report.ReportMetadata.DateRange.End = endTime.Add(-time.Second).Unix() + interval := time.Duration(first.IntervalHours) * time.Hour + beginTime := endTime.Add(-interval) + for first.Evaluated.Before(beginTime) { + beginTime = beginTime.Add(-interval) + } + report.ReportMetadata.DateRange.Begin = beginTime.Unix() + + // yyyymmddHH, we only send one report per hour, so should be unique per policy + // domain. We also add a truly unique id based on first evaluation id used without + // revealing the number of evaluations we have. Reuse of ReceivedID is not great, + // but shouldn't hurt. + report.ReportMetadata.ReportID = endTime.UTC().Format("20060102.15") + "." + mox.ReceivedID(first.ID) + + // We may include errors we encountered when composing the report. We + // don't currently include errors about dmarc evaluations, e.g. DNS + // lookup errors during incoming deliveries. + report.ReportMetadata.Errors = errors + + // We'll fill this with the last-used record, not the one we fetch fresh from DSN. + // They will almost always be the same, but if not, the fresh record was never + // actually used for evaluations, so no point in reporting it. + report.PolicyPublished = last.PolicyPublished + + // Process records in-order for testable results. + recstrs := maps.Keys(counts) + sort.Strings(recstrs) + for _, recstr := range recstrs { + rc := counts[recstr] + rc.ReportRecord.Row.Count = rc.count + report.Records = append(report.Records, rc.ReportRecord) + } + + reportFile, err := store.CreateMessageTemp("dmarcreportout") + if err != nil { + return false, fmt.Errorf("creating temporary file for outgoing dmarc aggregate report: %v", err) + } + defer store.CloseRemoveTempFile(log, reportFile, "generated dmarc aggregate report") + + gzw := gzip.NewWriter(reportFile) + _, err = fmt.Fprint(gzw, xml.Header) + enc := xml.NewEncoder(gzw) + enc.Indent("", "\t") // Keep up pretention that xml is human-readable. + if err == nil { + err = enc.Encode(report) + } + if err == nil { + err = enc.Close() + } + if err == nil { + err = gzw.Close() + } + if err != nil { + return true, fmt.Errorf("writing dmarc aggregate report as xml with gzip: %v", err) + } + + msgf, err := store.CreateMessageTemp("dmarcreportmsgout") + if err != nil { + return false, fmt.Errorf("creating temporary message file with outgoing dmarc aggregate report: %v", err) + } + defer store.CloseRemoveTempFile(log, msgf, "message with generated dmarc aggregate report") + + // We are sending reports from our host's postmaster address. In a + // typical setup the host is a subdomain of a configured domain with + // DKIM keys, so we can DKIM-sign our reports. SPF should pass anyway. + // A single report can contain deliveries from a single policy domain + // to multiple of our configured domains. + from := smtp.Address{Localpart: "postmaster", Domain: mox.Conf.Static.HostnameDomain} + + // Subject follows the form in RFC. ../rfc/7489:1871 + subject := fmt.Sprintf("Report Domain: %s Submitter: %s Report-ID: %s", dom.ASCII, mox.Conf.Static.HostnameDomain.ASCII, report.ReportMetadata.ReportID) + + // Human-readable part for convenience. ../rfc/7489:1803 + text := fmt.Sprintf(`Attached is an aggregate DMARC report with results of evaluations of the DMARC +policy of your domain for messages received by us that have your domain in the +message From header. You are receiving this message because your address is +specified in the "rua" field of the DMARC record for your domain. + +Report domain: %s +Submitter: %s +Report-ID: %s +Period: %s - %s in UTC +`, dom, mox.Conf.Static.HostnameDomain, report.ReportMetadata.ReportID, beginTime.Format(time.DateTime), endTime.Format(time.DateTime)) + + // The attached file follows the naming convention from the RFC. ../rfc/7489:1812 + reportFilename := fmt.Sprintf("%s!%s!%d!%d!%s.xml.gz", mox.Conf.Static.HostnameDomain.ASCII, dom.ASCII, beginTime.Unix(), endTime.Add(-time.Second).Unix(), report.ReportMetadata.ReportID) + + var addrs []smtp.Address + for _, rcpt := range recipients { + addrs = append(addrs, rcpt.address) + } + + // Compose the message. + msgPrefix, has8bit, smtputf8, messageID, err := composeAggregateReport(ctx, log, msgf, from, addrs, subject, text, reportFilename, reportFile) + if err != nil { + return false, fmt.Errorf("composing message with outgoing dmarc aggregate report: %v", err) + } + + // Get size of message after all compression and encodings (base64 makes it big + // again), and go through potentials recipients (rua). If they are willing to + // accept the report, queue it. + msgInfo, err := msgf.Stat() + if err != nil { + return false, fmt.Errorf("stat message with outgoing dmarc aggregate report: %v", err) + } + msgSize := int64(len(msgPrefix)) + msgInfo.Size() + var queued bool + for _, rcpt := range recipients { + // Only send to addresses where we don't exceed their size limit. The RFC mentions + // the size of the report, but then continues about the size after compression and + // transport encodings (i.e. gzip and the mime base64 attachment, so the intention + // is probably to compare against the size of the message that contains the report. + // ../rfc/7489:1773 + if rcpt.maxSize > 0 && msgSize > int64(rcpt.maxSize) { + continue + } + + qm := queue.MakeMsg(mox.Conf.Static.Postmaster.Account, from.Path(), rcpt.address.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil) + // Don't try as long as regular deliveries, and stop before we would send the + // delayed DSN. Though we also won't send that due to IsDMARCReport. + qm.MaxAttempts = 5 + qm.IsDMARCReport = true + + err := queueAdd(ctx, log, &qm, msgf) + if err != nil { + tempError = true + log.Errorx("queueing message with dmarc aggregate report", err) + metricReportError.Inc() + } else { + log.Debug("dmarc aggregate report queued", mlog.Field("recipient", rcpt.address)) + queued = true + metricReport.Inc() + } + } + + if !queued { + if err := sendErrorReport(ctx, log, from, addrs, dom, report.ReportMetadata.ReportID, msgSize); err != nil { + log.Errorx("sending dmarc error reports", err) + metricReportError.Inc() + } + } + + // Regardless of whether we queued a report, we are not going to keep the + // evaluations around. Though this can be overridden if tempError is set. + // ../rfc/7489:1785 + + return true, nil +} + +// todo future: factor out common code for composing messages, between webmail/api.go and the two compose functions below + +// xcomposer is a helper to compose a message. Operations on xcomposer panic +// with "err", which xcomposer.recover recovers. +type xcomposer struct { + w *bufio.Writer + err error + has8bit bool + smtputf8 bool +} + +// Write implements io.Writer, but calls panic (that is handled higher up) on +// i/o errors. +func (xc *xcomposer) Write(buf []byte) (int, error) { + n, err := xc.w.Write(buf) + xc.checkf(err, "write") + return n, nil +} + +// recover sentical error, storing it into rerr. +func (xc *xcomposer) recover(rerr *error) { + x := recover() + if x == nil { + return + } + if err, ok := x.(error); ok && errors.Is(err, xc.err) { + *rerr = err + } else { + panic(x) + } +} + +// check error, panicing with sentinal error value. +func (xc *xcomposer) checkf(err error, format string, args ...any) { + if err != nil { + panic(fmt.Errorf("%w: %s: %v", xc.err, err, fmt.Sprintf(format, args...))) + } +} + +// write a message header. +func (xc *xcomposer) header(k, v string) { + fmt.Fprintf(xc, "%s: %s\r\n", k, v) +} + +// write a message header with addresses. +func (xc *xcomposer) headerAddrs(k string, l []smtp.Address) { + if len(l) == 0 { + return + } + v := "" + linelen := len(k) + len(": ") + for _, a := range l { + if v != "" { + v += "," + linelen++ + } + addr := mail.Address{Address: a.Pack(xc.smtputf8)} + s := addr.String() + if v != "" && linelen+1+len(s) > 77 { + v += "\r\n\t" + linelen = 1 + } else if v != "" { + v += " " + linelen++ + } + v += s + linelen += len(s) + } + fmt.Fprintf(xc, "%s: %s\r\n", k, v) +} + +// write a subject message header. +func (xc *xcomposer) subject(subject string) { + var subjectValue string + subjectLineLen := len("Subject: ") + subjectWord := false + for i, word := range strings.Split(subject, " ") { + if !xc.smtputf8 && !isASCII(word) { + word = mime.QEncoding.Encode("utf-8", word) + } + if i > 0 { + subjectValue += " " + subjectLineLen++ + } + if subjectWord && subjectLineLen+len(word) > 77 { + subjectValue += "\r\n\t" + subjectLineLen = 1 + } + subjectValue += word + subjectLineLen += len(word) + subjectWord = true + } + xc.header("Subject", subjectValue) +} + +// write an empty line. +func (xc *xcomposer) line() { + _, _ = xc.Write([]byte("\r\n")) +} + +func (xc *xcomposer) textPart(text string) (textBody []byte, ct, cte string) { + if !strings.HasSuffix(text, "\n") { + text += "\n" + } + text = strings.ReplaceAll(text, "\n", "\r\n") + charset := "us-ascii" + if !isASCII(text) { + charset = "utf-8" + } + if message.NeedsQuotedPrintable(text) { + var sb strings.Builder + _, err := io.Copy(quotedprintable.NewWriter(&sb), strings.NewReader(text)) + xc.checkf(err, "converting text to quoted printable") + text = sb.String() + cte = "quoted-printable" + } else if xc.has8bit || charset == "utf-8" { + cte = "8bit" + } else { + cte = "7bit" + } + + ct = mime.FormatMediaType("text/plain", map[string]string{"charset": charset}) + return []byte(text), ct, cte +} + +func isASCII(s string) bool { + for _, c := range s { + if c >= 0x80 { + return false + } + } + return true +} + +func composeAggregateReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []smtp.Address, subject, text, filename string, reportXMLGzipFile *os.File) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { + xc := &xcomposer{bufio.NewWriter(mf), errors.New("compose"), false, false} + defer xc.recover(&rerr) + + // We only use smtputf8 if we have to, with a utf-8 localpart. For IDNA, we use ASCII domains. + for _, a := range recipients { + if a.Localpart.IsInternational() { + xc.smtputf8 = true + break + } + } + + xc.headerAddrs("From", []smtp.Address{fromAddr}) + xc.headerAddrs("To", recipients) + xc.subject(subject) + messageID = fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8)) + xc.header("Message-Id", messageID) + xc.header("Date", time.Now().Format(message.RFC5322Z)) + xc.header("User-Agent", "mox/"+moxvar.Version) + xc.header("MIME-Version", "1.0") + + // Multipart message, with a text/plain and the report attached. + mp := multipart.NewWriter(xc) + xc.header("Content-Type", fmt.Sprintf(`multipart/mixed; boundary="%s"`, mp.Boundary())) + xc.line() + + // Textual part, just mentioning this is a DMARC report. + textBody, ct, cte := xc.textPart(text) + textHdr := textproto.MIMEHeader{} + textHdr.Set("Content-Type", ct) + textHdr.Set("Content-Transfer-Encoding", cte) + textp, err := mp.CreatePart(textHdr) + xc.checkf(err, "adding text part to message") + _, err = textp.Write(textBody) + xc.checkf(err, "writing text part") + + // DMARC report as attachment. + ahdr := textproto.MIMEHeader{} + ahdr.Set("Content-Type", "application/gzip") + ahdr.Set("Content-Transfer-Encoding", "base64") + cd := mime.FormatMediaType("attachment", map[string]string{"filename": filename}) + ahdr.Set("Content-Disposition", cd) + ap, err := mp.CreatePart(ahdr) + xc.checkf(err, "adding dmarc aggregate report to message") + wc := moxio.Base64Writer(ap) + _, err = io.Copy(wc, &moxio.AtReader{R: reportXMLGzipFile}) + xc.checkf(err, "adding attachment") + err = wc.Close() + xc.checkf(err, "flushing attachment") + + err = mp.Close() + xc.checkf(err, "closing multipart") + + xc.w.Flush() + + msgPrefix = dkimSign(ctx, log, fromAddr, smtputf8, mf) + + return msgPrefix, xc.has8bit, xc.smtputf8, messageID, nil +} + +// Though this functionality is quite underspecified, we'll do our best to send our +// an error report in case our report is too large for all recipients. +// ../rfc/7489:1918 +func sendErrorReport(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, recipients []smtp.Address, reportDomain dns.Domain, reportID string, reportMsgSize int64) error { + log.Debug("no reporting addresses willing to accept report given size, queuing short error message") + + msgf, err := store.CreateMessageTemp("dmarcreportmsg-out") + if err != nil { + return fmt.Errorf("creating temporary message file for outgoing dmarc error report: %v", err) + } + defer store.CloseRemoveTempFile(log, msgf, "outgoing dmarc error report message") + + var recipientStrs []string + for _, rcpt := range recipients { + recipientStrs = append(recipientStrs, rcpt.String()) + } + + subject := fmt.Sprintf("DMARC aggregate reporting error report for %s", reportDomain.ASCII) + // ../rfc/7489:1926 + text := fmt.Sprintf(`Report-Date: %s +Report-Domain: %s +Report-ID: %s +Report-Size: %d +Submitter: %s +Submitting-URI: %s +`, time.Now().Format(message.RFC5322Z), reportDomain.ASCII, reportID, reportMsgSize, mox.Conf.Static.HostnameDomain.ASCII, strings.Join(recipientStrs, ",")) + text = strings.ReplaceAll(text, "\n", "\r\n") + + msgPrefix, has8bit, smtputf8, messageID, err := composeErrorReport(ctx, log, msgf, fromAddr, recipients, subject, text) + if err != nil { + return err + } + + msgInfo, err := msgf.Stat() + if err != nil { + return fmt.Errorf("stat message with outgoing dmarc error report: %v", err) + } + msgSize := int64(len(msgPrefix)) + msgInfo.Size() + + for _, rcpt := range recipients { + qm := queue.MakeMsg(mox.Conf.Static.Postmaster.Account, fromAddr.Path(), rcpt.Path(), has8bit, smtputf8, msgSize, messageID, []byte(msgPrefix), nil) + // Don't try as long as regular deliveries, and stop before we would send the + // delayed DSN. Though we also won't send that due to IsDMARCReport. + qm.MaxAttempts = 5 + qm.IsDMARCReport = true + + if err := queueAdd(ctx, log, &qm, msgf); err != nil { + log.Errorx("queueing message with dmarc error report", err) + metricReportError.Inc() + } else { + log.Debug("dmarc error report queued", mlog.Field("recipient", rcpt)) + metricReport.Inc() + } + } + return nil +} + +func composeErrorReport(ctx context.Context, log *mlog.Log, mf *os.File, fromAddr smtp.Address, recipients []smtp.Address, subject, text string) (msgPrefix string, has8bit, smtputf8 bool, messageID string, rerr error) { + xc := &xcomposer{bufio.NewWriter(mf), errors.New("compose"), false, false} + defer xc.recover(&rerr) + + // We only use smtputf8 if we have to, with a utf-8 localpart. For IDNA, we use ASCII domains. + for _, a := range recipients { + if a.Localpart.IsInternational() { + xc.smtputf8 = true + break + } + } + + xc.headerAddrs("From", []smtp.Address{fromAddr}) + xc.headerAddrs("To", recipients) + xc.header("Subject", subject) + messageID = fmt.Sprintf("<%s>", mox.MessageIDGen(smtputf8)) + xc.header("Message-Id", messageID) + xc.header("Date", time.Now().Format(message.RFC5322Z)) + xc.header("User-Agent", "mox/"+moxvar.Version) + xc.header("MIME-Version", "1.0") + + textBody, ct, cte := xc.textPart(text) + xc.header("Content-Type", ct) + xc.header("Content-Transfer-Encoding", cte) + xc.line() + _, err := xc.Write(textBody) + xc.checkf(err, "writing text") + + xc.w.Flush() + + msgPrefix = dkimSign(ctx, log, fromAddr, smtputf8, mf) + + return msgPrefix, xc.has8bit, xc.smtputf8, messageID, nil +} + +func dkimSign(ctx context.Context, log *mlog.Log, fromAddr smtp.Address, smtputf8 bool, mf *os.File) string { + // Add DKIM-Signature headers if we have a key for (a higher) domain than the from + // address, which is a host name. A signature will only be useful with higher-level + // domains if they have a relaxed dkim check (which is the default). If the dkim + // check is strict, there is no harm, there will simply not be a dkim pass. + fd := fromAddr.Domain + var zerodom dns.Domain + for fd != zerodom { + confDom, ok := mox.Conf.Domain(fd) + if len(confDom.DKIM.Sign) > 0 { + dkimHeaders, err := dkim.Sign(ctx, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf) + if err != nil { + log.Errorx("dkim-signing dmarc report, continuing without signature", err) + metricReportError.Inc() + return "" + } + return dkimHeaders + } else if ok { + return "" + } + + var nfd dns.Domain + _, nfd.ASCII, _ = strings.Cut(fd.ASCII, ".") + _, nfd.Unicode, _ = strings.Cut(fd.Unicode, ".") + fd = nfd + } + return "" +} diff --git a/dmarcdb/eval_test.go b/dmarcdb/eval_test.go new file mode 100644 index 0000000..147e593 --- /dev/null +++ b/dmarcdb/eval_test.go @@ -0,0 +1,384 @@ +package dmarcdb + +import ( + "context" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/mjl-/mox/dmarcrpt" + "github.com/mjl-/mox/dns" + "github.com/mjl-/mox/mlog" + "github.com/mjl-/mox/mox-" + "github.com/mjl-/mox/moxio" + "github.com/mjl-/mox/queue" +) + +func tcheckf(t *testing.T, err error, format string, args ...any) { + t.Helper() + if err != nil { + t.Fatalf("%s: %s", fmt.Sprintf(format, args...), err) + } +} + +func tcompare(t *testing.T, got, expect any) { + t.Helper() + if !reflect.DeepEqual(got, expect) { + t.Fatalf("got:\n%v\nexpected:\n%v", got, expect) + } +} + +func TestEvaluations(t *testing.T) { + os.RemoveAll("../testdata/dmarcdb/data") + mox.Context = ctxbg + mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf") + mox.MustLoadConfig(true, false) + EvalDB = nil + + _, err := evalDB(ctxbg) + tcheckf(t, err, "database") + defer func() { + EvalDB.Close() + EvalDB = nil + }() + + parseJSON := func(s string) (e Evaluation) { + t.Helper() + err := json.Unmarshal([]byte(s), &e) + tcheckf(t, err, "unmarshal") + return + } + packJSON := func(e Evaluation) string { + t.Helper() + buf, err := json.Marshal(e) + tcheckf(t, err, "marshal") + return string(buf) + } + + e0 := Evaluation{ + PolicyDomain: "sender1.example", + Evaluated: time.Now().Round(0), + IntervalHours: 1, + PolicyPublished: dmarcrpt.PolicyPublished{ + Domain: "sender1.example", + ADKIM: dmarcrpt.AlignmentRelaxed, + ASPF: dmarcrpt.AlignmentRelaxed, + Policy: dmarcrpt.DispositionReject, + SubdomainPolicy: dmarcrpt.DispositionReject, + Percentage: 100, + }, + SourceIP: "10.1.2.3", + Disposition: dmarcrpt.DispositionNone, + AlignedDKIMPass: true, + AlignedSPFPass: true, + EnvelopeTo: "mox.example", + EnvelopeFrom: "sender1.example", + HeaderFrom: "sender1.example", + DKIMResults: []dmarcrpt.DKIMAuthResult{ + { + Domain: "sender1.example", + Selector: "test", + Result: dmarcrpt.DKIMPass, + }, + }, + SPFResults: []dmarcrpt.SPFAuthResult{ + { + Domain: "sender1.example", + Scope: dmarcrpt.SPFDomainScopeMailFrom, + Result: dmarcrpt.SPFPass, + }, + }, + } + e1 := e0 + e2 := parseJSON(strings.ReplaceAll(packJSON(e0), "sender1.example", "sender2.example")) + e3 := parseJSON(strings.ReplaceAll(packJSON(e0), "10.1.2.3", "10.3.2.1")) + e3.Optional = true + + for i, e := range []*Evaluation{&e0, &e1, &e2, &e3} { + e.Evaluated = e.Evaluated.Add(time.Duration(i) * time.Second) + err = AddEvaluation(ctxbg, 3600, e) + tcheckf(t, err, "add evaluation") + } + + expStats := map[string]EvaluationStat{ + "sender1.example": { + Count: 3, + SendReport: true, + Domain: dns.Domain{ASCII: "sender1.example"}, + }, + "sender2.example": { + Count: 1, + SendReport: true, + Domain: dns.Domain{ASCII: "sender2.example"}, + }, + } + stats, err := EvaluationStats(ctxbg) + tcheckf(t, err, "evaluation stats") + tcompare(t, stats, expStats) + + // EvaluationsDomain + evals, err := EvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender1.example"}) + tcheckf(t, err, "get evaluations for domain") + tcompare(t, evals, []Evaluation{e0, e1, e3}) + + evals, err = EvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender2.example"}) + tcheckf(t, err, "get evaluations for domain") + tcompare(t, evals, []Evaluation{e2}) + + evals, err = EvaluationsDomain(ctxbg, dns.Domain{ASCII: "bogus.example"}) + tcheckf(t, err, "get evaluations for domain") + tcompare(t, evals, []Evaluation{}) + + // RemoveEvaluationsDomain + err = RemoveEvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender1.example"}) + tcheckf(t, err, "remove evaluations") + + expStats = map[string]EvaluationStat{ + "sender2.example": { + Count: 1, + SendReport: true, + Domain: dns.Domain{ASCII: "sender2.example"}, + }, + } + stats, err = EvaluationStats(ctxbg) + tcheckf(t, err, "evaluation stats") + tcompare(t, stats, expStats) +} + +func TestSendReports(t *testing.T) { + mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug}) + + os.RemoveAll("../testdata/dmarcdb/data") + mox.Context = ctxbg + mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf") + mox.MustLoadConfig(true, false) + EvalDB = nil + + db, err := evalDB(ctxbg) + tcheckf(t, err, "database") + defer func() { + EvalDB.Close() + EvalDB = nil + }() + + resolver := dns.MockResolver{ + TXT: map[string][]string{ + "_dmarc.sender.example.": { + "v=DMARC1; rua=mailto:dmarcrpt@sender.example; ri=3600", + }, + }, + } + + end := nextWholeHour(time.Now()) + + eval := Evaluation{ + PolicyDomain: "sender.example", + Evaluated: end.Add(-time.Hour / 2), + IntervalHours: 1, + PolicyPublished: dmarcrpt.PolicyPublished{ + Domain: "sender.example", + ADKIM: dmarcrpt.AlignmentRelaxed, + ASPF: dmarcrpt.AlignmentRelaxed, + Policy: dmarcrpt.DispositionReject, + SubdomainPolicy: dmarcrpt.DispositionReject, + Percentage: 100, + }, + SourceIP: "10.1.2.3", + Disposition: dmarcrpt.DispositionNone, + AlignedDKIMPass: true, + AlignedSPFPass: true, + EnvelopeTo: "mox.example", + EnvelopeFrom: "sender.example", + HeaderFrom: "sender.example", + DKIMResults: []dmarcrpt.DKIMAuthResult{ + { + Domain: "sender.example", + Selector: "test", + Result: dmarcrpt.DKIMPass, + }, + }, + SPFResults: []dmarcrpt.SPFAuthResult{ + { + Domain: "sender.example", + Scope: dmarcrpt.SPFDomainScopeMailFrom, + Result: dmarcrpt.SPFPass, + }, + }, + } + + expFeedback := &dmarcrpt.Feedback{ + XMLName: xml.Name{Local: "feedback"}, + Version: "1.0", + ReportMetadata: dmarcrpt.ReportMetadata{ + OrgName: "mail.mox.example", + Email: "postmaster@mail.mox.example", + DateRange: dmarcrpt.DateRange{ + Begin: end.Add(-1 * time.Hour).Unix(), + End: end.Add(-time.Second).Unix(), + }, + }, + PolicyPublished: dmarcrpt.PolicyPublished{ + Domain: "sender.example", + ADKIM: dmarcrpt.AlignmentRelaxed, + ASPF: dmarcrpt.AlignmentRelaxed, + Policy: dmarcrpt.DispositionReject, + SubdomainPolicy: dmarcrpt.DispositionReject, + Percentage: 100, + }, + Records: []dmarcrpt.ReportRecord{ + { + Row: dmarcrpt.Row{ + SourceIP: "10.1.2.3", + Count: 1, + PolicyEvaluated: dmarcrpt.PolicyEvaluated{ + Disposition: dmarcrpt.DispositionNone, + DKIM: dmarcrpt.DMARCPass, + SPF: dmarcrpt.DMARCPass, + }, + }, + Identifiers: dmarcrpt.Identifiers{ + EnvelopeTo: "mox.example", + EnvelopeFrom: "sender.example", + HeaderFrom: "sender.example", + }, + AuthResults: dmarcrpt.AuthResults{ + DKIM: []dmarcrpt.DKIMAuthResult{ + { + Domain: "sender.example", + Selector: "test", + Result: dmarcrpt.DKIMPass, + }, + }, + SPF: []dmarcrpt.SPFAuthResult{ + { + Domain: "sender.example", + Scope: dmarcrpt.SPFDomainScopeMailFrom, + Result: dmarcrpt.SPFPass, + }, + }, + }, + }, + }, + } + + // Set a timeUntil that we steplock and that causes the actual sleep to return immediately when we want to. + wait := make(chan struct{}) + step := make(chan time.Duration) + jitteredTimeUntil = func(_ time.Time) time.Duration { + wait <- struct{}{} + return <-step + } + + sleepBetween = func() {} + + test := func(evals []Evaluation, expAggrAddrs map[string]struct{}, expErrorAddrs map[string]struct{}, optExpReport *dmarcrpt.Feedback) { + t.Helper() + + mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg) + + for _, e := range evals { + err := db.Insert(ctxbg, &e) + tcheckf(t, err, "inserting evaluation") + } + + aggrAddrs := map[string]struct{}{} + errorAddrs := map[string]struct{}{} + + queueAdd = func(ctx context.Context, log *mlog.Log, qm *queue.Msg, msgFile *os.File) error { + // Read message file. Also write copy to disk for inspection. + buf, err := io.ReadAll(&moxio.AtReader{R: msgFile}) + tcheckf(t, err, "read report message") + err = os.WriteFile("../testdata/dmarcdb/data/report.eml", append(append([]byte{}, qm.MsgPrefix...), buf...), 0600) + tcheckf(t, err, "write report message") + + var feedback *dmarcrpt.Feedback + addr := qm.Recipient().String() + isErrorReport := strings.Contains(string(buf), "DMARC aggregate reporting error report") + if isErrorReport { + errorAddrs[addr] = struct{}{} + } else { + aggrAddrs[addr] = struct{}{} + + feedback, err = dmarcrpt.ParseMessageReport(log, msgFile) + tcheckf(t, err, "parsing generated report message") + } + + if optExpReport != nil { + // Parse report in message and compare with expected. + expFeedback.ReportMetadata.ReportID = feedback.ReportMetadata.ReportID + tcompare(t, feedback, expFeedback) + } + + return nil + } + + Start(resolver) + // Run first loop. + <-wait + step <- 0 + <-wait + tcompare(t, aggrAddrs, expAggrAddrs) + tcompare(t, errorAddrs, expErrorAddrs) + + // Second loop. Evaluations cleaned, should not result in report messages. + aggrAddrs = map[string]struct{}{} + errorAddrs = map[string]struct{}{} + step <- 0 + <-wait + tcompare(t, aggrAddrs, map[string]struct{}{}) + tcompare(t, errorAddrs, map[string]struct{}{}) + + // Caus Start to stop. + mox.ShutdownCancel() + step <- time.Minute + } + + // Typical case, with a single address that receives an aggregate report. + test([]Evaluation{eval}, map[string]struct{}{"dmarcrpt@sender.example": {}}, map[string]struct{}{}, expFeedback) + + // Only optional evaluations, no report at all. + evalOpt := eval + evalOpt.Optional = true + test([]Evaluation{evalOpt}, map[string]struct{}{}, map[string]struct{}{}, nil) + + // Two RUA's, one with a size limit that doesn't pass, and one that does pass. + resolver.TXT["_dmarc.sender.example."] = []string{"v=DMARC1; rua=mailto:dmarcrpt1@sender.example!1,mailto:dmarcrpt2@sender.example!10t; ri=3600"} + test([]Evaluation{eval}, map[string]struct{}{"dmarcrpt2@sender.example": {}}, map[string]struct{}{}, nil) + + // Redirect to external domain, without permission, no report sent. + resolver.TXT["_dmarc.sender.example."] = []string{"v=DMARC1; rua=mailto:unauthorized@other.example"} + test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{}, nil) + + // Redirect to external domain, with basic permission. + resolver.TXT = map[string][]string{ + "_dmarc.sender.example.": {"v=DMARC1; rua=mailto:authorized@other.example"}, + "sender.example._report._dmarc.other.example.": {"v=DMARC1"}, + } + test([]Evaluation{eval}, map[string]struct{}{"authorized@other.example": {}}, map[string]struct{}{}, nil) + + // Redirect to authorized external domain, with 2 allowed replacements and 1 invalid and 1 refusing due to size. + resolver.TXT = map[string][]string{ + "_dmarc.sender.example.": {"v=DMARC1; rua=mailto:authorized@other.example"}, + "sender.example._report._dmarc.other.example.": {"v=DMARC1; rua=mailto:good1@other.example,mailto:bad1@yetanother.example,mailto:good2@other.example,mailto:badsize@other.example!1"}, + } + test([]Evaluation{eval}, map[string]struct{}{"good1@other.example": {}, "good2@other.example": {}}, map[string]struct{}{}, nil) + + // Without RUA, we send no message. + resolver.TXT = map[string][]string{ + "_dmarc.sender.example.": {"v=DMARC1;"}, + } + test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{}, nil) + + // If message size limit is reached, an error repor is sent. + resolver.TXT = map[string][]string{ + "_dmarc.sender.example.": {"v=DMARC1; rua=mailto:dmarcrpt@sender.example!1"}, + } + test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{"dmarcrpt@sender.example": {}}, nil) +} diff --git a/dmarcdb/db.go b/dmarcdb/reports.go similarity index 82% rename from dmarcdb/db.go rename to dmarcdb/reports.go index 7845869..f7281ba 100644 --- a/dmarcdb/db.go +++ b/dmarcdb/reports.go @@ -1,9 +1,3 @@ -// Package dmarcdb stores incoming DMARC reports. -// -// With DMARC, a domain can request emails with DMARC verification results by -// remote mail servers to be sent to a specified address. Mox parses such -// reports, stores them in its database and makes them available through its -// admin web interface. package dmarcdb import ( @@ -25,9 +19,9 @@ import ( ) var ( - DBTypes = []any{DomainFeedback{}} // Types stored in DB. - DB *bstore.DB // Exported for backups. - mutex sync.Mutex + ReportsDBTypes = []any{DomainFeedback{}} // Types stored in DB. + ReportsDB *bstore.DB // Exported for backups. + reportsMutex sync.Mutex ) var ( @@ -65,25 +59,19 @@ type DomainFeedback struct { dmarcrpt.Feedback } -func database(ctx context.Context) (rdb *bstore.DB, rerr error) { - mutex.Lock() - defer mutex.Unlock() - if DB == nil { +func reportsDB(ctx context.Context) (rdb *bstore.DB, rerr error) { + reportsMutex.Lock() + defer reportsMutex.Unlock() + if ReportsDB == nil { p := mox.DataDirPath("dmarcrpt.db") os.MkdirAll(filepath.Dir(p), 0770) - db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...) + db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, ReportsDBTypes...) if err != nil { return nil, err } - DB = db + ReportsDB = db } - return DB, nil -} - -// Init opens the database. -func Init() error { - _, err := database(mox.Shutdown) - return err + return ReportsDB, nil } // AddReport adds a DMARC aggregate feedback report from an email to the database, @@ -91,7 +79,7 @@ func Init() error { // // fromDomain is the domain in the report message From header. func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) error { - db, err := database(ctx) + db, err := reportsDB(ctx) if err != nil { return err } @@ -141,7 +129,7 @@ func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) // Records returns all reports in the database. func Records(ctx context.Context) ([]DomainFeedback, error) { - db, err := database(ctx) + db, err := reportsDB(ctx) if err != nil { return nil, err } @@ -151,7 +139,7 @@ func Records(ctx context.Context) ([]DomainFeedback, error) { // RecordID returns the report for the ID. func RecordID(ctx context.Context, id int64) (DomainFeedback, error) { - db, err := database(ctx) + db, err := reportsDB(ctx) if err != nil { return DomainFeedback{}, err } @@ -164,7 +152,7 @@ func RecordID(ctx context.Context, id int64) (DomainFeedback, error) { // RecordsPeriodDomain returns the reports overlapping start and end, for the given // domain. If domain is empty, all records match for domain. func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]DomainFeedback, error) { - db, err := database(ctx) + db, err := reportsDB(ctx) if err != nil { return nil, err } diff --git a/dmarcdb/db_test.go b/dmarcdb/reports_test.go similarity index 96% rename from dmarcdb/db_test.go rename to dmarcdb/reports_test.go index 54f970f..1e86a39 100644 --- a/dmarcdb/db_test.go +++ b/dmarcdb/reports_test.go @@ -17,8 +17,8 @@ var ctxbg = context.Background() func TestDMARCDB(t *testing.T) { mox.Shutdown = ctxbg - mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/fake.conf") - mox.Conf.Static.DataDir = "." + mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf") + mox.MustLoadConfig(true, false) dbpath := mox.DataDirPath("dmarcrpt.db") os.MkdirAll(filepath.Dir(dbpath), 0770) @@ -27,7 +27,10 @@ func TestDMARCDB(t *testing.T) { t.Fatalf("init database: %s", err) } defer os.Remove(dbpath) - defer DB.Close() + defer func() { + ReportsDB.Close() + ReportsDB = nil + }() feedback := &dmarcrpt.Feedback{ ReportMetadata: dmarcrpt.ReportMetadata{ diff --git a/dmarcrpt/feedback.go b/dmarcrpt/feedback.go index dc9358e..38ea507 100644 --- a/dmarcrpt/feedback.go +++ b/dmarcrpt/feedback.go @@ -1,9 +1,14 @@ package dmarcrpt +import ( + "encoding/xml" +) + // Initially generated by xsdgen, then modified. // Feedback is the top-level XML field returned. type Feedback struct { + XMLName xml.Name `xml:"feedback" json:"-"` // todo: removing the json tag triggers bug in sherpadoc, should fix. Version string `xml:"version"` ReportMetadata ReportMetadata `xml:"report_metadata"` PolicyPublished PolicyPublished `xml:"policy_published"` @@ -26,6 +31,9 @@ type DateRange struct { // PolicyPublished is the policy as found in DNS for the domain. type PolicyPublished struct { + // Domain is where DMARC record was found, not necessarily message From. Reports we + // generate use unicode names, incoming reports may have either ASCII-only or + // Unicode domains. Domain string `xml:"domain"` ADKIM Alignment `xml:"adkim,omitempty"` ASPF Alignment `xml:"aspf,omitempty"` diff --git a/dmarcrpt/parse.go b/dmarcrpt/parse.go index a727bb3..435f838 100644 --- a/dmarcrpt/parse.go +++ b/dmarcrpt/parse.go @@ -17,7 +17,7 @@ import ( "github.com/mjl-/mox/moxio" ) -var ErrNoReport = errors.New("no dmarc report found in message") +var ErrNoReport = errors.New("no dmarc aggregate report found in message") // ParseReport parses an XML aggregate feedback report. // The maximum report size is 20MB. diff --git a/dmarcrpt/parse_test.go b/dmarcrpt/parse_test.go index d8a6107..ade0134 100644 --- a/dmarcrpt/parse_test.go +++ b/dmarcrpt/parse_test.go @@ -1,6 +1,7 @@ package dmarcrpt import ( + "encoding/xml" "os" "path/filepath" "reflect" @@ -62,6 +63,7 @@ const reportExample = ` func TestParseReport(t *testing.T) { var expect = &Feedback{ + XMLName: xml.Name{Local: "feedback"}, ReportMetadata: ReportMetadata{ OrgName: "google.com", Email: "noreply-dmarc-support@google.com", @@ -126,7 +128,7 @@ func TestParseMessageReport(t *testing.T) { dir := filepath.FromSlash("../testdata/dmarc-reports") files, err := os.ReadDir(dir) if err != nil { - t.Fatalf("listing dmarc report emails: %s", err) + t.Fatalf("listing dmarc aggregate report emails: %s", err) } for _, file := range files { diff --git a/gentestdata.go b/gentestdata.go index d3c5496..7d20e3c 100644 --- a/gentestdata.go +++ b/gentestdata.go @@ -192,9 +192,9 @@ Accounts: err = dmarcdb.Init() xcheckf(err, "dmarcdb init") report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport)) - xcheckf(err, "parsing dmarc report") + xcheckf(err, "parsing dmarc aggregate report") err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"}) - xcheckf(err, "adding dmarc report") + xcheckf(err, "adding dmarc aggregate report") // Populate mtasts.db. err = mtastsdb.Init(false) @@ -233,7 +233,8 @@ Accounts: const qmsg = "From: \r\nTo: \r\nSubject: test\r\n\r\nthe message...\r\n" _, err = fmt.Fprint(mf, qmsg) xcheckf(err, "writing message") - _, err = queue.Add(ctxbg, log, "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "", prefix, mf, nil, nil) + qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "", prefix, nil) + err = queue.Add(ctxbg, log, &qm, mf) xcheckf(err, "enqueue message") // Create three accounts. diff --git a/localserve.go b/localserve.go index 581aee9..65d052b 100644 --- a/localserve.go +++ b/localserve.go @@ -124,8 +124,9 @@ during those commands instead of during "data". queue.Localserve = true const mtastsdbRefresher = false + const sendDMARCReports = false const skipForkExec = true - if err := start(mtastsdbRefresher, skipForkExec); err != nil { + if err := start(mtastsdbRefresher, sendDMARCReports, skipForkExec); err != nil { log.Fatalx("starting mox", err) } golog.Printf("mox, version %s", moxvar.Version) diff --git a/main.go b/main.go index b18c38c..39316e9 100644 --- a/main.go +++ b/main.go @@ -2436,13 +2436,13 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at return } - accepts, status, _, txt, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom) + accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom) var txtstr string txtaddr := fmt.Sprintf("%s._report._dmarc.%s", domain.ASCII, destdom.ASCII) - if txt == "" { - txtstr = fmt.Sprintf(" (no txt record %s)", txtaddr) + if len(txts) == 0 { + txtstr = fmt.Sprintf(" (no txt records %s)", txtaddr) } else { - txtstr = fmt.Sprintf(" (txt record %s: %q)", txtaddr, txt) + txtstr = fmt.Sprintf(" (txt record %s: %q)", txtaddr, txts) } if status != dmarc.StatusNone { printResult("fail: %s%s", err, txtstr) diff --git a/metrics/panic.go b/metrics/panic.go index 106bebc..f9a1443 100644 --- a/metrics/panic.go +++ b/metrics/panic.go @@ -22,6 +22,7 @@ const ( Import Panic = "import" Serve Panic = "serve" Imapserver Panic = "imapserver" + Dmarcdb Panic = "dmarcdb" Mtastsdb Panic = "mtastsdb" Queue Panic = "queue" Smtpclient Panic = "smtpclient" diff --git a/queue/direct.go b/queue/direct.go index 7d3d775..a752654 100644 --- a/queue/direct.go +++ b/queue/direct.go @@ -92,7 +92,7 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT // todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls? // todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379 - if permanent || m.Attempts >= 8 { + if permanent || m.MaxAttempts == 0 && m.Attempts >= 8 || m.MaxAttempts > 0 && m.Attempts >= m.MaxAttempts { qlog.Errorx("permanent failure delivering from queue", errors.New(errmsg)) deliverDSNFailure(qlog, m, remoteMTA, secodeOpt, errmsg) @@ -230,12 +230,13 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce permanent, daneRequired, badTLS, secodeOpt, remoteIP, errmsg, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode) - // If we had a TLS-related failure when doing TLS, and we don't have a requirement for MTA-STS/DANE, - // we try again without TLS. This could be an old - // server that only does ancient TLS versions, or has a misconfiguration. Note that + // If we had a TLS-related failure when doing TLS, and we don't have a requirement + // for MTA-STS/DANE, we try again without TLS. This could be an old server that + // only does ancient TLS versions, or has a misconfiguration. Note that // opportunistic TLS does not do regular certificate verification, so that can't be // the problem. - if !ok && badTLS && (!enforceMTASTS && tlsMode == smtpclient.TLSOpportunistic && !daneRequired || m.RequireTLS != nil && !*m.RequireTLS) { + // We don't fall back to plain text for DMARC reports. ../rfc/7489:1768 ../rfc/7489:2683 + if !ok && badTLS && (!enforceMTASTS && tlsMode == smtpclient.TLSOpportunistic && !daneRequired && !m.IsDMARCReport || m.RequireTLS != nil && !*m.RequireTLS) { metricPlaintextFallback.Inc() if m.RequireTLS != nil && !*m.RequireTLS { metricTLSRequiredNoIgnored.WithLabelValues("badtls").Inc() diff --git a/queue/dsn.go b/queue/dsn.go index 9f4bd9d..06c1e83 100644 --- a/queue/dsn.go +++ b/queue/dsn.go @@ -6,6 +6,9 @@ import ( "os" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/mjl-/mox/dns" "github.com/mjl-/mox/dsn" "github.com/mjl-/mox/message" @@ -15,6 +18,15 @@ import ( "github.com/mjl-/mox/store" ) +var ( + metricDMARCReportFailure = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "mox_queue_dmarcreport_failure_total", + Help: "Permanent failures to deliver a DMARC report.", + }, + ) +) + func deliverDSNFailure(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { const subject = "mail delivery failed" message := fmt.Sprintf(` @@ -33,6 +45,12 @@ Error during the last delivery attempt: } func deliverDSNDelay(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) { + // Should not happen, but doesn't hurt to prevent sending delayed delivery + // notifications for DMARC reports. We don't want to waste postmaster attention. + if m.IsDMARCReport { + return + } + const subject = "mail delivery delayed" message := fmt.Sprintf(` Delivery has been delayed of your email to: @@ -133,7 +151,12 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st msgData = append(msgData, []byte("Return-Path: <"+dsnMsg.From.XString(m.SMTPUTF8)+">\r\n")...) mailbox := "Inbox" - acc, err := store.OpenAccount(m.SenderAccount) + senderAccount := m.SenderAccount + if m.IsDMARCReport { + // senderAccount should already by postmaster, but doesn't hurt to ensure it. + senderAccount = mox.Conf.Static.Postmaster.Account + } + acc, err := store.OpenAccount(senderAccount) if err != nil { acc, err = store.OpenAccount(mox.Conf.Static.Postmaster.Account) if err != nil { @@ -171,6 +194,17 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st Size: msgWriter.Size, MsgPrefix: []byte{}, } + + // If this is a DMARC report, deliver it as seen message to a submailbox of the + // postmaster mailbox. We mark it as seen so it doesn't waste postmaster attention, + // but we deliver them so they can be checked in case of problems. + if m.IsDMARCReport { + mailbox = fmt.Sprintf("%s/dmarc", mox.Conf.Static.Postmaster.Mailbox) + msg.Seen = true + metricDMARCReportFailure.Inc() + log.Info("delivering dsn for failure to deliver outgoing dmarc report") + } + acc.WithWLock(func() { if err := acc.DeliverMailbox(log, mailbox, msg, msgFile); err != nil { qlog("delivering dsn to mailbox", err) diff --git a/queue/queue.go b/queue/queue.go index 70efce3..c63721c 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -70,6 +70,9 @@ var DB *bstore.DB // Exported for making backups. var Localserve bool // Msg is a message in the queue. +// +// Use MakeMsg to make a message with fields that Add needs. Add will further set +// queueing related fields. type Msg struct { ID int64 Queued time.Time `bstore:"default now"` @@ -80,15 +83,19 @@ type Msg struct { RecipientDomain dns.IPDomain RecipientDomainStr string // For filtering. Attempts int // Next attempt is based on last attempt and exponential back off based on attempts. + MaxAttempts int // Max number of attempts before giving up. If 0, then the default of 8 attempts is used instead. DialedIPs map[string][]net.IP // For each host, the IPs that were dialed. Used for IP selection for later attempts. NextAttempt time.Time // For scheduling. LastAttempt *time.Time LastError string - Has8bit bool // Whether message contains bytes with high bit set, determines whether 8BITMIME SMTP extension is needed. - SMTPUTF8 bool // Whether message requires use of SMTPUTF8. - Size int64 // Full size of message, combined MsgPrefix with contents of message file. - MessageID string // Used when composing a DSN, in its References header. - MsgPrefix []byte + + Has8bit bool // Whether message contains bytes with high bit set, determines whether 8BITMIME SMTP extension is needed. + SMTPUTF8 bool // Whether message requires use of SMTPUTF8. + IsDMARCReport bool // Delivery failures for DMARC reports are handled differently. + IsTLSReport bool // Delivery failures for TLS reports are handled differently. + Size int64 // Full size of message, combined MsgPrefix with contents of message file. + MessageID string // Used when composing a DSN, in its References header. + MsgPrefix []byte // If set, this message is a DSN and this is a version using utf-8, for the case // the remote MTA supports smtputf8. In this case, Size and MsgPrefix are not @@ -188,44 +195,71 @@ func Count(ctx context.Context) (int, error) { return bstore.QueryDB[Msg](ctx, DB).Count() } +// MakeMsg is a convenience function that sets the commonly used fields for a Msg. +func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, prefix []byte, requireTLS *bool) Msg { + return Msg{ + SenderAccount: mox.Conf.Static.Postmaster.Account, + SenderLocalpart: sender.Localpart, + SenderDomain: sender.IPDomain, + RecipientLocalpart: recipient.Localpart, + RecipientDomain: recipient.IPDomain, + Has8bit: has8bit, + SMTPUTF8: smtputf8, + Size: size, + MessageID: messageID, + MsgPrefix: prefix, + RequireTLS: requireTLS, + } +} + // Add a new message to the queue. The queue is kicked immediately to start a // first delivery attempt. // -// dnsutf8Opt is a utf8-version of the message, to be used only for DNSs. If set, -// this data is used as the message when delivering the DSN and the remote SMTP -// server supports SMTPUTF8. If the remote SMTP server does not support SMTPUTF8, -// the regular non-utf8 message is delivered. -func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, requireTLS *bool) (int64, error) { +// ID must be 0 and will be set after inserting in the queue. +// +// Add sets derived fields like RecipientDomainStr, and fields related to queueing, +// such as Queued, NextAttempt, LastAttempt, LastError. +func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error { // todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759 + if qm.ID != 0 { + return fmt.Errorf("id of queued message must be 0") + } + qm.Queued = time.Now() + qm.DialedIPs = nil + qm.NextAttempt = qm.Queued + qm.LastAttempt = nil + qm.LastError = "" + qm.RecipientDomainStr = formatIPDomain(qm.RecipientDomain) + if Localserve { - if senderAccount == "" { - return 0, fmt.Errorf("cannot queue with localserve without local account") + if qm.SenderAccount == "" { + return fmt.Errorf("cannot queue with localserve without local account") } - acc, err := store.OpenAccount(senderAccount) + acc, err := store.OpenAccount(qm.SenderAccount) if err != nil { - return 0, fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err) + return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err) } defer func() { err := acc.Close() log.Check(err, "closing account") }() - m := store.Message{Size: size, MsgPrefix: msgPrefix} + m := store.Message{Size: qm.Size, MsgPrefix: qm.MsgPrefix} conf, _ := acc.Conf() - dest := conf.Destinations[mailFrom.String()] + dest := conf.Destinations[qm.Sender().String()] acc.WithWLock(func() { err = acc.DeliverDestination(log, dest, &m, msgFile) }) if err != nil { - return 0, fmt.Errorf("delivering message: %v", err) + return fmt.Errorf("delivering message: %v", err) } log.Debug("immediately delivered from queue to sender") - return 0, nil + return nil } tx, err := DB.Begin(ctx, true) if err != nil { - return 0, fmt.Errorf("begin transaction: %w", err) + return fmt.Errorf("begin transaction: %w", err) } defer func() { if tx != nil { @@ -235,11 +269,8 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp } }() - now := time.Now() - qm := Msg{0, now, senderAccount, mailFrom.Localpart, mailFrom.IPDomain, rcptTo.Localpart, rcptTo.IPDomain, formatIPDomain(rcptTo.IPDomain), 0, nil, now, nil, "", has8bit, smtputf8, size, messageID, msgPrefix, dsnutf8Opt, "", requireTLS} - - if err := tx.Insert(&qm); err != nil { - return 0, err + if err := tx.Insert(qm); err != nil { + return err } dst := qm.MessagePath() @@ -252,19 +283,19 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp dstDir := filepath.Dir(dst) os.MkdirAll(dstDir, 0770) if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil { - return 0, fmt.Errorf("linking/copying message to new file: %s", err) + return fmt.Errorf("linking/copying message to new file: %s", err) } else if err := moxio.SyncDir(dstDir); err != nil { - return 0, fmt.Errorf("sync directory: %v", err) + return fmt.Errorf("sync directory: %v", err) } if err := tx.Commit(); err != nil { - return 0, fmt.Errorf("commit transaction: %s", err) + return fmt.Errorf("commit transaction: %s", err) } tx = nil dst = "" queuekick() - return qm.ID, nil + return nil } func formatIPDomain(d dns.IPDomain) string { diff --git a/queue/queue_test.go b/queue/queue_test.go index c7e3f2f..02906cb 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -110,10 +110,14 @@ func TestQueue(t *testing.T) { defer os.Remove(mf.Name()) defer mf.Close() - _, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + var qm Msg + + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - _, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") msgs, err = List(ctxbg) @@ -440,7 +444,8 @@ func TestQueue(t *testing.T) { // Add a message to be delivered with submit because of its route. topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}} - _, err = Add(ctxbg, xlog, "mjl", path, topath, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") wasNetDialer = testDeliver(fakeSubmitServer) if !wasNetDialer { @@ -448,10 +453,11 @@ func TestQueue(t *testing.T) { } // Add a message to be delivered with submit because of explicitly configured transport, that uses TLS. - msgID, err := Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") transportSubmitTLS := "submittls" - n, err = Kick(ctxbg, msgID, "", "", &transportSubmitTLS) + n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -472,10 +478,11 @@ func TestQueue(t *testing.T) { } // Add a message to be delivered with socks. - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") transportSocks := "socks" - n, err = Kick(ctxbg, msgID, "", "", &transportSocks) + n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -486,9 +493,10 @@ func TestQueue(t *testing.T) { } // Add message to be delivered with opportunistic TLS verification. - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -496,9 +504,10 @@ func TestQueue(t *testing.T) { testDeliver(fakeSMTPSTARTTLSServer) // Test fallback to plain text with TLS handshake fails. - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -512,9 +521,10 @@ func TestQueue(t *testing.T) { {Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: moxCert.Leaf.RawSubjectPublicKeyInfo}, }, } - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -530,9 +540,10 @@ func TestQueue(t *testing.T) { // Add message to be delivered with verified TLS and REQUIRETLS. yes := true - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, &yes) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &yes) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -545,9 +556,10 @@ func TestQueue(t *testing.T) { {}, }, } - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -562,9 +574,10 @@ func TestQueue(t *testing.T) { {Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: make([]byte, sha256.Size)}, }, } - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -581,9 +594,10 @@ func TestQueue(t *testing.T) { // Check that message is delivered with TLS-Required: No and non-matching DANE record. no := false - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, &no) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &no) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -591,9 +605,10 @@ func TestQueue(t *testing.T) { testDeliver(fakeSMTPSTARTTLSServer) // Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text. - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, &no) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &no) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -601,9 +616,10 @@ func TestQueue(t *testing.T) { testDeliver(makeBadFakeSMTPSTARTTLSServer(true)) // Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers. - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, &yes) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &yes) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -615,9 +631,10 @@ func TestQueue(t *testing.T) { resolver.TLSA = nil // Add message with requiretls that fails immediately due to no verification policy for recipient domain. - msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, &yes) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, &yes) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") - n, err = Kick(ctxbg, msgID, "", "", nil) + n, err = Kick(ctxbg, qm.ID, "", "", nil) tcheck(t, err, "kick queue") if n != 1 { t.Fatalf("kick changed %d messages, expected 1", n) @@ -629,7 +646,8 @@ func TestQueue(t *testing.T) { }) // Add another message that we'll fail to deliver entirely. - _, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") msgs, err = List(ctxbg) @@ -788,7 +806,8 @@ func TestQueueStart(t *testing.T) { mf := prepareFile(t) defer os.Remove(mf.Name()) defer mf.Close() - _, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "", nil, mf, nil, nil) + qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "", nil, nil) + err = Add(ctxbg, xlog, &qm, mf) tcheck(t, err, "add message to queue for delivery") checkDialed(true) diff --git a/serve.go b/serve.go index 7f8cb55..511ed3c 100644 --- a/serve.go +++ b/serve.go @@ -52,7 +52,7 @@ func shutdown(log *mlog.Log) { // start initializes all packages, starts all listeners and the switchboard // goroutine, then returns. -func start(mtastsdbRefresher, skipForkExec bool) error { +func start(mtastsdbRefresher, sendDMARCReports, skipForkExec bool) error { smtpserver.Listen() imapserver.Listen() http.Listen() @@ -69,10 +69,6 @@ func start(mtastsdbRefresher, skipForkExec bool) error { } } - if err := dmarcdb.Init(); err != nil { - return fmt.Errorf("dmarc init: %s", err) - } - if err := mtastsdb.Init(mtastsdbRefresher); err != nil { return fmt.Errorf("mtasts init: %s", err) } @@ -86,6 +82,14 @@ func start(mtastsdbRefresher, skipForkExec bool) error { return fmt.Errorf("queue start: %s", err) } + // dmarcdb starts after queue because it may start sending reports through the queue. + if err := dmarcdb.Init(); err != nil { + return fmt.Errorf("dmarc init: %s", err) + } + if sendDMARCReports { + dmarcdb.Start(dns.StrictResolver{Pkg: "dmarcdb"}) + } + store.StartAuthCache() smtpserver.Serve() imapserver.Serve() diff --git a/serve_unix.go b/serve_unix.go index afaff93..79414cd 100644 --- a/serve_unix.go +++ b/serve_unix.go @@ -224,7 +224,7 @@ Only implemented on unix systems, not Windows. // taken. const mtastsdbRefresher = true const skipForkExec = false - if err := start(mtastsdbRefresher, skipForkExec); err != nil { + if err := start(mtastsdbRefresher, !mox.Conf.Static.NoOutgoingDMARCReports, skipForkExec); err != nil { log.Fatalx("start", err) } log.Print("ready to serve") diff --git a/smtp/address.go b/smtp/address.go index 68de0bc..0d3a245 100644 --- a/smtp/address.go +++ b/smtp/address.go @@ -107,6 +107,10 @@ func NewAddress(localpart Localpart, domain dns.Domain) Address { return Address{localpart, domain} } +func (a Address) Path() Path { + return Path{Localpart: a.Localpart, IPDomain: dns.IPDomain{Domain: a.Domain}} +} + func (a Address) IsZero() bool { return a == Address{} } diff --git a/smtpserver/analyze.go b/smtpserver/analyze.go index 94988a7..fe621fe 100644 --- a/smtpserver/analyze.go +++ b/smtpserver/analyze.go @@ -37,16 +37,17 @@ type delivery struct { } type analysis struct { - accept bool - mailbox string - code int - secode string - userError bool - errmsg string - err error // For our own logging, not sent to remote. - dmarcReport *dmarcrpt.Feedback // Validated dmarc aggregate report, not yet stored. - tlsReport *tlsrpt.Report // Validated TLS report, not yet stored. - reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens. + accept bool + mailbox string + code int + secode string + userError bool + errmsg string + err error // For our own logging, not sent to remote. + dmarcReport *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored. + tlsReport *tlsrpt.Report // Validated TLS report, not yet stored. + reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens. + dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride } const ( @@ -64,7 +65,7 @@ const ( reasonDNSBlocklisted = "dns-blocklisted" reasonSubjectpass = "subjectpass" reasonSubjectpassError = "subjectpass-error" - reasonIPrev = "iprev" // No or mil junk reputation signals, and bad iprev. + reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev. ) func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis { @@ -83,15 +84,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive ld := rs.ListAllowDNSDomain // todo: on temporary failures, reject temporarily? if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain { - return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow} + return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList)} } for _, r := range d.dkimResults { if r.Status == dkim.StatusPass && r.Sig.Domain == ld { - return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow} + return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList)} } } } + var dmarcOverrideReason string + // For forwarded messages, we have different junk analysis. We don't reject for // failing DMARC, and we clear fields that could implicate the forwarding mail // server during future classifications on incoming messages (the forwarding mail @@ -113,6 +116,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive } } d.m.DKIMDomains = dkimdoms + dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded) log.Info("forwarded message, clearing identifying signals of forwarding mail server") } @@ -154,7 +158,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive }) }) if mberr != nil { - return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError} + return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, dmarcOverrideReason} } d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID. } @@ -168,7 +172,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive d.m.Seen = true log.Info("accepting reject to configured mailbox due to ruleset") } - return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason} + return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason, dmarcOverrideReason} } if d.dmarcUse && d.dmarcResult.Reject { @@ -180,17 +184,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive // track of the report. We'll check reputation, defaulting to accept. var dmarcReport *dmarcrpt.Feedback if d.rcptAcc.destination.DMARCReports { - // Messages with DMARC aggregate reports must have a dmarc pass. ../rfc/7489:1866 + // Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866 if d.dmarcResult.Status != dmarc.StatusPass { - log.Info("received dmarc report without dmarc pass, not processing as dmarc report") + log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report") } else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil { - log.Infox("parsing dmarc report", err) + log.Infox("parsing dmarc aggregate report", err) } else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil { - log.Infox("parsing domain in dmarc report", err) + log.Infox("parsing domain in dmarc aggregate report", err) } else if _, ok := mox.Conf.Domain(d); !ok { - log.Info("dmarc report for domain not configured, ignoring", mlog.Field("domain", d)) + log.Info("dmarc aggregate report for domain not configured, ignoring", mlog.Field("domain", d)) } else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 { - log.Info("dmarc report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0))) + log.Info("dmarc aggregate report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0))) } else { dmarcReport = report } @@ -261,12 +265,12 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method))) if conclusive { if !*isjunk { - return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason} + return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason} } return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method)) } else if dmarcReport != nil || tlsReport != nil { - log.Info("accepting dmarc reporting or tlsrpt message without reputation") - return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting} + log.Info("accepting message with dmarc aggregate report or tls report without reputation") + return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting, dmarcOverrideReason: dmarcOverrideReason} } // If there was no previous message from sender or its domain, and we have an SPF // (soft)fail, reject the message. @@ -302,7 +306,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive pass := err == nil log.Infox("pass by subject token", err, mlog.Field("pass", pass)) if pass { - return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass} + return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason} } } @@ -382,7 +386,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive } if accept { - return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals} + return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals, dmarcOverrideReason: dmarcOverrideReason} } if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) { diff --git a/smtpserver/dsn.go b/smtpserver/dsn.go index d5acffd..2a5d6e7 100644 --- a/smtpserver/dsn.go +++ b/smtpserver/dsn.go @@ -50,7 +50,9 @@ func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message, req if requireTLS { reqTLS = &requireTLS } - if _, err := queue.Add(ctx, c.log, "", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, f, bufUTF8, reqTLS); err != nil { + qm := queue.MakeMsg("", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, reqTLS) + qm.DSNUTF8 = bufUTF8 + if err := queue.Add(ctx, c.log, &qm, f); err != nil { return err } return nil diff --git a/smtpserver/server.go b/smtpserver/server.go index d97f786..7a24257 100644 --- a/smtpserver/server.go +++ b/smtpserver/server.go @@ -37,6 +37,7 @@ import ( "github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dmarc" "github.com/mjl-/mox/dmarcdb" + "github.com/mjl-/mox/dmarcrpt" "github.com/mjl-/mox/dns" "github.com/mjl-/mox/dsn" "github.com/mjl-/mox/iprev" @@ -1835,7 +1836,8 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...) msgSize := int64(len(xmsgPrefix)) + msgWriter.Size - if _, err := queue.Add(ctx, c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, dataFile, nil, c.requireTLS); err != nil { + qm := queue.MakeMsg(c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, c.requireTLS) + if err := queue.Add(ctx, c.log, &qm, dataFile); err != nil { // Aborting the transaction is not great. But continuing and generating DSNs will // probably result in errors as well... metricSubmission.WithLabelValues("queueerror").Inc() @@ -2065,7 +2067,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW var comment string var props []message.AuthProp if r.Sig != nil { - // todo future: also specify whether dns record was dnssec-signed. if r.Record != nil && r.Record.PublicKey != nil { if pubkey, ok := r.Record.PublicKey.(*rsa.PublicKey); ok { comment = fmt.Sprintf("%d bit rsa, ", pubkey.N.BitLen()) @@ -2167,6 +2168,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW var dmarcUse bool var dmarcResult dmarc.Result const applyRandomPercentage = true + // dmarcMethod is added to authResults when delivering to recipients: accounts can + // have different policy override rules. var dmarcMethod message.AuthMethod var msgFromValidation = store.ValidationNone if msgFrom.IsZero() { @@ -2178,6 +2181,15 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW } else { msgFromValidation = alignment(ctx, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity) + // We are doing the DMARC evaluation now. But we only store it for inclusion in an + // aggregate report when we actually use it. We use an evaluation for each + // recipient, with each a potentially different result due to mailing + // list/forwarding configuration. If we reject a message due to being spam, we + // don't want to spend any resources for the sender domain, and we don't want to + // give the sender any more information about us, so we won't record the + // evaluation. + // todo future: also not send for first-time senders? they could be spammers getting through our filter, don't want to give them insights either. though we currently would have no reasonable way to decide if they are still reputationless at the time we are composing/sending aggregate reports. + dmarcctx, dmarccancel := context.WithTimeout(ctx, time.Minute) defer dmarccancel() dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage) @@ -2202,9 +2214,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW msgFromValidation = store.ValidationDMARC } - // todo future: consider enforcing an spf fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507 + // todo future: consider enforcing an spf (soft)fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507 } - authResults.Methods = append(authResults.Methods, dmarcMethod) c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain)) // Prepare for analyzing content, calculating reputation. @@ -2366,16 +2377,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW continue } - // ../rfc/5321:3204 - // Received-SPF header goes before Received. ../rfc/7208:2038 - msgPrefix := []byte( - "Delivered-To: " + rcptAcc.rcptTo.XString(c.smtputf8) + "\r\n" + // ../rfc/9228:274 - "Return-Path: <" + c.mailFrom.String() + ">\r\n" + // ../rfc/5321:3300 - authResults.Header() + - receivedSPF.Header() + - recvHdrFor(rcptAcc.rcptTo.String()), - ) - m := &store.Message{ Received: time.Now(), RemoteIP: c.remoteIP.String(), @@ -2398,16 +2399,187 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW MailFromValidation: mailFromValidation, MsgFromValidation: msgFromValidation, DKIMDomains: verifiedDKIMDomains, - Size: int64(len(msgPrefix)) + msgWriter.Size, - MsgPrefix: msgPrefix, + Size: msgWriter.Size, } d := delivery{m, dataFile, rcptAcc, acc, msgFrom, c.dnsBLs, dmarcUse, dmarcResult, dkimResults, iprevStatus} a := analyze(ctx, log, c.resolver, d) - if a.reason != "" { - xmoxreason := "X-Mox-Reason: " + a.reason + "\r\n" - m.MsgPrefix = append([]byte(xmoxreason), m.MsgPrefix...) - m.Size += int64(len(xmoxreason)) + + // Any DMARC result override is stored in the evaluation for outgoing DMARC + // aggregate reports, and added to the Authentication-Results message header. + var dmarcOverride string + if dmarcResult.Record != nil { + if !dmarcUse { + dmarcOverride = string(dmarcrpt.PolicyOverrideSampledOut) + } else if a.dmarcOverrideReason != "" && (a.accept && !m.IsReject) == dmarcResult.Reject { + dmarcOverride = a.dmarcOverrideReason + } } + + // Add per-recipient DMARC method to Authentication-Results. Each account can have + // their own override rules, e.g. based on configured mailing lists/forwards. + // ../rfc/7489:1486 + rcptDMARCMethod := dmarcMethod + if dmarcOverride != "" { + if rcptDMARCMethod.Comment != "" { + rcptDMARCMethod.Comment += ", " + } + rcptDMARCMethod.Comment += "override " + dmarcOverride + } + rcptAuthResults := authResults + rcptAuthResults.Methods = append([]message.AuthMethod{}, authResults.Methods...) + rcptAuthResults.Methods = append(rcptAuthResults.Methods, rcptDMARCMethod) + + // Prepend reason as message header, for easy display in mail clients. + var xmoxreason string + if a.reason != "" { + xmoxreason = "X-Mox-Reason: " + a.reason + "\r\n" + } + + // ../rfc/5321:3204 + // Received-SPF header goes before Received. ../rfc/7208:2038 + m.MsgPrefix = []byte( + xmoxreason + + "Delivered-To: " + rcptAcc.rcptTo.XString(c.smtputf8) + "\r\n" + // ../rfc/9228:274 + "Return-Path: <" + c.mailFrom.String() + ">\r\n" + // ../rfc/5321:3300 + rcptAuthResults.Header() + + receivedSPF.Header() + + recvHdrFor(rcptAcc.rcptTo.String()), + ) + m.Size += int64(len(m.MsgPrefix)) + + // Store DMARC evaluation for inclusion in an aggregate report. Only if there is at + // least one reporting address: We don't want to needlessly store a row in a + // database for each delivery attempt. If we reject a message for being junk, we + // are also not going to send it a DMARC report. The DMARC check is done early in + // the analysis, we will report on rejects because of DMARC, because it could be + // valuable feedback about forwarded or mailing list messages. + // ../rfc/7489:1492 + if !mox.Conf.Static.NoOutgoingDMARCReports && dmarcResult.Record != nil && len(dmarcResult.Record.AggregateReportAddresses) > 0 && (a.accept && !m.IsReject || a.reason == reasonDMARCPolicy) { + // Disposition holds our decision on whether to accept the message. Not what the + // DMARC evaluation resulted in. We can override, e.g. because of mailing lists, + // forwarding, or local policy. + // We treat quarantine as reject, so never claim to quarantine. + // ../rfc/7489:1691 + disposition := dmarcrpt.DispositionNone + if !a.accept { + disposition = dmarcrpt.DispositionReject + } + + // unknownDomain returns whether the sender is domain with which this account has + // not had positive interaction. + unknownDomain := func() (unknown bool) { + err := acc.DB.Read(ctx, func(tx *bstore.Tx) (err error) { + // See if we received a non-junk message from this organizational domain. + q := bstore.QueryTx[store.Message](tx) + q.FilterNonzero(store.Message{MsgFromOrgDomain: m.MsgFromOrgDomain}) + q.FilterEqual("Notjunk", false) + exists, err := q.Exists() + if err != nil { + return fmt.Errorf("querying for non-junk message from organizational domain: %v", err) + } + if exists { + return nil + } + + // See if we sent a message to this organizational domain. + qr := bstore.QueryTx[store.Recipient](tx) + qr.FilterNonzero(store.Recipient{OrgDomain: m.MsgFromOrgDomain}) + exists, err = qr.Exists() + if err != nil { + return fmt.Errorf("querying for message sent to organizational domain: %v", err) + } + if !exists { + unknown = true + } + return nil + }) + if err != nil { + log.Errorx("checking if sender is unknown domain, for dmarc aggregate report evaluation", err) + } + return + } + + r := dmarcResult.Record + addresses := make([]string, len(r.AggregateReportAddresses)) + for i, a := range r.AggregateReportAddresses { + addresses[i] = a.String() + } + sp := dmarcrpt.Disposition(r.SubdomainPolicy) + if r.SubdomainPolicy == dmarc.PolicyEmpty { + sp = dmarcrpt.Disposition(r.Policy) + } + eval := dmarcdb.Evaluation{ + // Evaluated and IntervalHours set by AddEvaluation. + PolicyDomain: dmarcResult.Domain.Name(), + + // Optional evaluations don't cause a report to be sent, but will be included. + // Useful for automated inter-mailer messages, we don't want to get in a reporting + // loop. We also don't want to be used for sending reports to unsuspecting domains + // we have no relation with. + // todo: would it make sense to also mark some percentage of mailing-list-policy-overrides optional? to lower the load on mail servers of folks sending to large mailing lists. + Optional: rcptAcc.destination.DMARCReports || rcptAcc.destination.TLSReports || a.reason == reasonDMARCPolicy && unknownDomain(), + + Addresses: addresses, + + PolicyPublished: dmarcrpt.PolicyPublished{ + Domain: dmarcResult.Domain.Name(), + ADKIM: dmarcrpt.Alignment(r.ADKIM), + ASPF: dmarcrpt.Alignment(r.ASPF), + Policy: dmarcrpt.Disposition(r.Policy), + SubdomainPolicy: sp, + Percentage: r.Percentage, + // We don't save ReportingOptions, we don't do per-message failure reporting. + }, + SourceIP: c.remoteIP.String(), + Disposition: disposition, + AlignedDKIMPass: dmarcResult.AlignedDKIMPass, + AlignedSPFPass: dmarcResult.AlignedSPFPass, + EnvelopeTo: rcptAcc.rcptTo.IPDomain.String(), + EnvelopeFrom: c.mailFrom.IPDomain.String(), + HeaderFrom: msgFrom.Domain.Name(), + } + + if dmarcOverride != "" { + eval.OverrideReasons = []dmarcrpt.PolicyOverrideReason{ + {Type: dmarcrpt.PolicyOverride(dmarcOverride)}, + } + } + + // We'll include all signatures for the organizational domain, even if they weren't + // relevant due to strict alignment requirement. + for _, dkimResult := range dkimResults { + if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, msgFrom.Domain) != publicsuffix.Lookup(ctx, dkimResult.Sig.Domain) { + continue + } + r := dmarcrpt.DKIMAuthResult{ + Domain: dkimResult.Sig.Domain.Name(), + Selector: dkimResult.Sig.Selector.ASCII, + Result: dmarcrpt.DKIMResult(dkimResult.Status), + } + eval.DKIMResults = append(eval.DKIMResults, r) + } + + switch receivedSPF.Identity { + case spf.ReceivedHELO: + spfAuthResult := dmarcrpt.SPFAuthResult{ + Domain: spfArgs.HelloDomain.String(), // Can be unicode and also IP. + Scope: dmarcrpt.SPFDomainScopeHelo, + Result: dmarcrpt.SPFResult(receivedSPF.Result), + } + eval.SPFResults = []dmarcrpt.SPFAuthResult{spfAuthResult} + case spf.ReceivedMailFrom: + spfAuthResult := dmarcrpt.SPFAuthResult{ + Domain: spfArgs.MailFromDomain.Name(), // Can be unicode. + Scope: dmarcrpt.SPFDomainScopeMailFrom, + Result: dmarcrpt.SPFResult(receivedSPF.Result), + } + eval.SPFResults = []dmarcrpt.SPFAuthResult{spfAuthResult} + } + + err := dmarcdb.AddEvaluation(ctx, dmarcResult.Record.AggregateReportingInterval, &eval) + log.Check(err, "adding dmarc evaluation to database for aggregate report") + } + if !a.accept { conf, _ := acc.Conf() if conf.RejectsMailbox != "" { @@ -2455,9 +2627,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW if a.dmarcReport != nil { // todo future: add rate limiting to prevent DoS attacks. ../rfc/7489:2570 if err := dmarcdb.AddReport(ctx, a.dmarcReport, msgFrom.Domain); err != nil { - log.Errorx("saving dmarc report in database", err) + log.Errorx("saving dmarc aggregate report in database", err) } else { - log.Info("dmarc report processed") + log.Info("dmarc aggregate report processed") m.Flags.Seen = true delayFirstTime = false } diff --git a/smtpserver/server_test.go b/smtpserver/server_test.go index 69e2baf..22963c7 100644 --- a/smtpserver/server_test.go +++ b/smtpserver/server_test.go @@ -100,6 +100,11 @@ func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *test ts := testserver{t: t, cid: 1, resolver: resolver, tlsmode: smtpclient.TLSOpportunistic} + if dmarcdb.EvalDB != nil { + dmarcdb.EvalDB.Close() + dmarcdb.EvalDB = nil + } + mox.Context = ctxbg mox.ConfigStaticPath = configPath mox.MustLoadConfig(true, false) @@ -192,6 +197,15 @@ func fakeCert(t *testing.T) tls.Certificate { return c } +// check expected dmarc evaluations for outgoing aggregate reports. +func checkEvaluationCount(t *testing.T, n int) []dmarcdb.Evaluation { + t.Helper() + l, err := dmarcdb.Evaluations(ctxbg) + tcheck(t, err, "get dmarc evaluations") + tcompare(t, len(l), n) + return l +} + // Test submission from authenticated user. func TestSubmission(t *testing.T) { ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/mox.conf"), dns.MockResolver{}) @@ -229,6 +243,7 @@ func TestSubmission(t *testing.T) { if expErr == nil && err != nil || expErr != nil && (err == nil || !errors.As(err, &cerr) || cerr.Secode != expErr.Secode) { t.Fatalf("got err %#v (%q), expected %#v", err, err, expErr) } + checkEvaluationCount(t, 0) }) } @@ -329,6 +344,8 @@ func TestDelivery(t *testing.T) { t.Fatalf("no delivery in 1s") } }) + + checkEvaluationCount(t, 0) } func tinsertmsg(t *testing.T, acc *store.Account, mailbox string, m *store.Message, msg string) { @@ -392,7 +409,7 @@ func TestSpam(t *testing.T) { }, TXT: map[string][]string{ "example.org.": {"v=spf1 ip4:127.0.0.10 -all"}, - "_dmarc.example.org.": {"v=DMARC1;p=reject"}, + "_dmarc.example.org.": {"v=DMARC1;p=reject; rua=mailto:dmarcrpt@example.org"}, }, } ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver) @@ -451,6 +468,7 @@ func TestSpam(t *testing.T) { } checkCount("Rejects", 1) + checkEvaluationCount(t, 0) // No positive interactions yet. }) // Delivery from sender with bad reputation matching AcceptRejectsToMailbox should @@ -463,8 +481,9 @@ func TestSpam(t *testing.T) { } tcheck(t, err, "deliver") - checkCount("mjl2junk", 1) // In ruleset rejects mailbox. - checkCount("Rejects", 1) // Same as before. + checkCount("mjl2junk", 1) // In ruleset rejects mailbox. + checkCount("Rejects", 1) // Same as before. + checkEvaluationCount(t, 0) // This is not an actual accept. }) // Mark the messages as having good reputation. @@ -485,6 +504,7 @@ func TestSpam(t *testing.T) { // Message should now be removed from Rejects mailboxes. checkCount("Rejects", 0) checkCount("mjl2junk", 1) + checkEvaluationCount(t, 1) }) // Undo dmarc pass, mark messages as junk, and train the filter. @@ -506,6 +526,7 @@ func TestSpam(t *testing.T) { if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr { t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr) } + checkEvaluationCount(t, 1) // No new evaluation, this isn't a DMARC reject. }) } @@ -525,9 +546,9 @@ func TestForward(t *testing.T) { "bad.example.": {"v=spf1 ip4:127.0.0.1 -all"}, "good.example.": {"v=spf1 ip4:127.0.0.1 -all"}, "forward.example.": {"v=spf1 ip4:127.0.0.10 -all"}, - "_dmarc.bad.example.": {"v=DMARC1;p=reject"}, - "_dmarc.good.example.": {"v=DMARC1;p=reject"}, - "_dmarc.forward.example.": {"v=DMARC1;p=reject"}, + "_dmarc.bad.example.": {"v=DMARC1;p=reject; rua=mailto:dmarc@bad.example"}, + "_dmarc.good.example.": {"v=DMARC1;p=reject; rua=mailto:dmarc@good.example"}, + "_dmarc.forward.example.": {"v=DMARC1;p=reject; rua=mailto:dmarc@forward.example"}, }, PTR: map[string][]string{ "127.0.0.10": {"forward.example."}, // For iprev check. @@ -544,6 +565,8 @@ func TestForward(t *testing.T) { ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver) defer ts.close() + totalEvaluations := 0 + var msgBad = strings.ReplaceAll(`From: To: Subject: test @@ -580,6 +603,7 @@ happens to come from forwarding mail server. err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgBad)), strings.NewReader(msgBad), false, false, false) tcheck(t, err, "deliver message") } + totalEvaluations += 10 n, err := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB).UpdateFields(map[string]any{"Junk": true, "MsgFromValidated": true}) tcheck(t, err, "marking messages as junk") @@ -591,6 +615,8 @@ happens to come from forwarding mail server. if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr { t.Fatalf("delivery by bad sender, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr) } + + checkEvaluationCount(t, totalEvaluations) }) // Delivery from different "message From" without reputation, but from same @@ -607,12 +633,14 @@ happens to come from forwarding mail server. err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgOK)), strings.NewReader(msgOK), false, false, false) if forward { tcheck(t, err, "deliver") + totalEvaluations += 1 } else { var cerr smtpclient.Error if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr { t.Fatalf("delivery by bad ip, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr) } } + checkEvaluationCount(t, totalEvaluations) }) // Delivery from forwarding server that isn't a forward should get same treatment. @@ -624,12 +652,14 @@ happens to come from forwarding mail server. err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgOK2)), strings.NewReader(msgOK2), false, false, false) if forward { tcheck(t, err, "deliver") + totalEvaluations += 1 } else { var cerr smtpclient.Error if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr { t.Fatalf("delivery by bad ip, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr) } } + checkEvaluationCount(t, totalEvaluations) }) } @@ -644,13 +674,31 @@ func TestDMARCSent(t *testing.T) { "example.org.": {"127.0.0.1"}, // For mx check. }, TXT: map[string][]string{ - "example.org.": {"v=spf1 ip4:127.0.0.10 -all"}, - "_dmarc.example.org.": {"v=DMARC1;p=reject"}, + "example.org.": {"v=spf1 ip4:127.0.0.1 -all"}, + "_dmarc.example.org.": {"v=DMARC1;p=reject;rua=mailto:dmarcrpt@example.org"}, }, } ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver) defer ts.close() + // First check that DMARC policy rejects message and results in optional evaluation. + ts.run(func(err error, client *smtpclient.Client) { + mailFrom := "remote@example.org" + rcptTo := "mjl@mox.example" + if err == nil { + err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false) + } + var cerr smtpclient.Error + if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail { + t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C550MailboxUnavail) + } + l := checkEvaluationCount(t, 1) + tcompare(t, l[0].Optional, true) + }) + + // Update DNS for an SPF pass, and DMARC pass. + resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.10 -all"} + // Insert spammy messages not related to the test message. m := store.Message{ MailFrom: "remote@test.example", @@ -676,6 +724,7 @@ func TestDMARCSent(t *testing.T) { if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr { t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr) } + checkEvaluationCount(t, 1) // No new evaluation. }) // Insert a message that we sent to the address that is about to send to us. @@ -684,7 +733,26 @@ func TestDMARCSent(t *testing.T) { err := ts.acc.DB.Insert(ctxbg, &store.Recipient{MessageID: sentMsg.ID, Localpart: "remote", Domain: "example.org", OrgDomain: "example.org", Sent: time.Now()}) tcheck(t, err, "inserting message recipient") + // Reject a message due to DMARC again. Since we sent a message to the domain, it + // is no longer unknown and we should see a non-optional evaluation that will + // result in a DMARC report. + resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.1 -all"} + ts.run(func(err error, client *smtpclient.Client) { + mailFrom := "remote@example.org" + rcptTo := "mjl@mox.example" + if err == nil { + err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false) + } + var cerr smtpclient.Error + if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail { + t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C550MailboxUnavail) + } + l := checkEvaluationCount(t, 2) // New evaluation. + tcompare(t, l[1].Optional, false) + }) + // We should now be accepting the message because we recently sent a message. + resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.10 -all"} ts.run(func(err error, client *smtpclient.Client) { mailFrom := "remote@example.org" rcptTo := "mjl@mox.example" @@ -692,6 +760,8 @@ func TestDMARCSent(t *testing.T) { err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false) } tcheck(t, err, "deliver") + l := checkEvaluationCount(t, 3) // New evaluation. + tcompare(t, l[2].Optional, false) }) } @@ -773,7 +843,7 @@ func TestDMARCReport(t *testing.T) { }, TXT: map[string][]string{ "example.org.": {"v=spf1 ip4:127.0.0.10 -all"}, - "_dmarc.example.org.": {"v=DMARC1;p=reject"}, + "_dmarc.example.org.": {"v=DMARC1;p=reject; rua=mailto:dmarcrpt@example.org"}, }, PTR: map[string][]string{ "127.0.0.10": {"example.org."}, // For iprev check. @@ -815,6 +885,11 @@ func TestDMARCReport(t *testing.T) { run(dmarcReport, 0) run(strings.ReplaceAll(dmarcReport, "xmox.nl", "mox.example"), 1) + + // We always store as an evaluation, but as optional for reports. + evals := checkEvaluationCount(t, 2) + tcompare(t, evals[0].Optional, true) + tcompare(t, evals[1].Optional, true) } const dmarcReport = ` @@ -896,7 +971,7 @@ func TestTLSReport(t *testing.T) { TXT: map[string][]string{ "testsel._domainkey.example.org.": {dkimTxt}, "example.org.": {"v=spf1 ip4:127.0.0.10 -all"}, - "_dmarc.example.org.": {"v=DMARC1;p=reject"}, + "_dmarc.example.org.": {"v=DMARC1;p=reject;rua=mailto:dmarcrpt@example.org"}, }, PTR: map[string][]string{ "127.0.0.10": {"example.org."}, // For iprev check. @@ -939,6 +1014,11 @@ func TestTLSReport(t *testing.T) { run(tlsrpt, 0) run(strings.ReplaceAll(tlsrpt, "xmox.nl", "mox.example"), 1) + + // We always store as an evaluation, but as optional for reports. + evals := checkEvaluationCount(t, 2) + tcompare(t, evals[0].Optional, true) + tcompare(t, evals[1].Optional, true) } func TestRatelimitConnectionrate(t *testing.T) { diff --git a/store/cleanuptemp.go b/store/cleanuptemp.go new file mode 100644 index 0000000..0bfc757 --- /dev/null +++ b/store/cleanuptemp.go @@ -0,0 +1,17 @@ +package store + +import ( + "os" + + "github.com/mjl-/mox/mlog" +) + +// CloseRemoveTempFile closes and removes f, a file described by descr. Often +// used in a defer after creating a temporary file. +func CloseRemoveTempFile(log *mlog.Log, f *os.File, descr string) { + name := f.Name() + err := f.Close() + log.Check(err, "closing temporary file", mlog.Field("kind", descr)) + err = os.Remove(name) + log.Check(err, "removing temporary file", mlog.Field("kind", descr)) +} diff --git a/testdata/dmarcdb/domains.conf b/testdata/dmarcdb/domains.conf new file mode 100644 index 0000000..339e2c2 --- /dev/null +++ b/testdata/dmarcdb/domains.conf @@ -0,0 +1,28 @@ +Domains: + mox.example: + DKIM: + Selectors: + testsel: + PrivateKeyFile: testsel.rsakey.pkcs8.pem + Sign: + - testsel +Accounts: + other: + Domain: mox.example + Destinations: + other@mox.example: nil + mjl: + MaxOutgoingMessagesPerDay: 30 + MaxFirstTimeRecipientsPerDay: 10 + Domain: mox.example + Destinations: + mjl@mox.example: nil + møx@mox.example: nil + RejectsMailbox: Rejects + JunkFilter: + Threshold: 0.95 + Params: + Twograms: true + MaxPower: 0.1 + TopWords: 10 + IgnoreWords: 0.1 diff --git a/testdata/dmarcdb/mox.conf b/testdata/dmarcdb/mox.conf new file mode 100644 index 0000000..3f6fdcf --- /dev/null +++ b/testdata/dmarcdb/mox.conf @@ -0,0 +1,11 @@ +DataDir: data +User: 1000 +LogLevel: trace +Hostname: mail.mox.example +Listeners: + local: + IPs: + - 0.0.0.0 +Postmaster: + Account: mjl + Mailbox: postmaster diff --git a/testdata/dmarcdb/testsel.rsakey.pkcs8.pem b/testdata/dmarcdb/testsel.rsakey.pkcs8.pem new file mode 100644 index 0000000..73d742c --- /dev/null +++ b/testdata/dmarcdb/testsel.rsakey.pkcs8.pem @@ -0,0 +1,30 @@ +-----BEGIN PRIVATE KEY----- +Note: RSA private key for use with DKIM, generated by mox + +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDdkh3fKzvRUWym +n9UwVrEw6s2Mc0+DTg04TWJKGKHXpvcTHuEcE6ALVS9MZKasyVsIHU7FNeS9/qNb +pLihhGdlhU3KAfrMpTBhiFpJoYiDXED98Of4iBxNHIuheLMxSBSClMbLGE2vAgha +/6LuONuzdMqk/c1TijBD+vGjCZI2qD58cgXWWKRK9e+WNhKNoVdedZ9iJtbtN0MI +UWk3iwHmjXf5qzS7i8vDoy86Ln0HW0vKl7UtwemLVv09/E23OdNN163eQvSlrEhx +a0odPQsM9SizxhiaI9rmcZtSqULt37hhPaNA+/AbELCzWijZPDqePVRqKGd5gYDK +8STLj0UHAgMBAAECggEBAKVkJJgplYUx2oCmXmSu0aVKIBTvHjNNV+DnIq9co7Ju +F5BWRILIw3ayJ5RGrYPc6e6ssdfT2uNX6GjIFGm8g9HsJ5zazXNk+zBSr9K2mUg0 +3O6xnPaP41BMNo5ZoqjuvSCcHagMhDBWvBXxLJXWK2lRjNKMAXCSfmTANQ8WXeYd +XG2nYTPtBu6UgY8W6sKAx1xetxBrzk8q6JTxb5eVG22BSiUniWYif+XVmAj1u6TH +0m6X0Kb6zsMYYgKPC2hmDsxD3uZ7qBNxxJzzLjpK6eP9aeFKzNyfnaoO4s+9K6Di +31oxTBpqLI4dcrvg4xWl+YkEknXXaomMqM8hyDzfcAECgYEA9/zmjRpoTAoY3fu9 +mn16wxReFXZZZhqV0+c+gyYtao2Kf2pUNAdhD62HQv7KtAPPHKvLfL8PH0u7bzK0 +vVNzBUukwxGI7gsoTMdc3L5x4v9Yb6jUx7RrDZn93sDod/1f/sb56ARCFQoqbUck +dSjnVUyF/l5oeh6CgKhvtghJ/AcCgYEA5Lq4kL82qWjIuNUT/C3lzjPfQVU+WvQ9 +wa+x4B4mxm5r4na3AU1T8H+peh4YstAJUgscGfYnLzxuMGuP1ReIuWYy29eDptKl +WTzVZDcZrAPciP1FOL6jm03PT2UAEuoPRr4OHLg8DxoOqG8pxqk1izDSHG2Tof6l +0ToafeIALwECgYEA8wvLTgnOpI/U1WNP7aUDd0Rz/WbzsW1m4Lsn+lOleWPllIE6 +q4974mi5Q8ECG7IL/9aj5cw/XvXTauVwXIn4Ff2QKpr58AvBYJaX/cUtS0PlgfIf +MOczcK43MWUxscADoGmVLn9V4NcIw/dQ1P7U0zXfsXEHxoA2eTAb5HV1RWsCgYBd +TcXoVfgIV1Q6AcGrR1XNLd/OmOVc2PEwR2l6ERKkM3sS4HZ6s36gRpNt20Ub/D0x +GJMYDA+j9zTDz7zWokkFyCjLATkVHiyRIH2z6b4xK0oVH6vTIAFBYxZEPuEu1gfx +RaogEQ9+4ZRFJUOXZIMRCpNLQW/Nz0D4/oi7/SsyAQKBgHEA27Js8ivt+EFCBjwB +UbkW+LonDAXuUbw91lh5jICCigqUg73HNmV5xpoYI9JNPc6fy6wLyInVUC2w9tpO +eH2Rl8n79vQMLbzsFClGEC/Q1kAbK5bwUjlfvKBZjvE0RknWX9e1ZY04DSsunSrM +prS2eHVZ24hecd7j9XfAbHLC +-----END PRIVATE KEY----- diff --git a/verifydata.go b/verifydata.go index 11d7442..dc3bec1 100644 --- a/verifydata.go +++ b/verifydata.go @@ -97,9 +97,12 @@ possibly making them potentially no longer readable by the previous version. // Check a database file by opening it with BoltDB and bstore and lightly checking // its contents. - checkDB := func(path string, types []any) { + checkDB := func(required bool, path string, types []any) { _, err := os.Stat(path) - checkf(err, path, "checking if file exists") + if !required && err != nil && errors.Is(err, fs.ErrNotExist) { + return + } + checkf(err, path, "checking if database file exists") if err != nil { return } @@ -156,7 +159,7 @@ possibly making them potentially no longer readable by the previous version. checkQueue := func() { dbpath := filepath.Join(dataDir, "queue/index.db") - checkDB(dbpath, queue.DBTypes) + checkDB(true, dbpath, queue.DBTypes) // Check that all messages present in the database also exist on disk. seen := map[string]struct{}{} @@ -222,12 +225,12 @@ possibly making them potentially no longer readable by the previous version. // Check an account, with its database file and messages. checkAccount := func(name string) { accdir := filepath.Join(dataDir, "accounts", name) - checkDB(filepath.Join(accdir, "index.db"), store.DBTypes) + checkDB(true, filepath.Join(accdir, "index.db"), store.DBTypes) jfdbpath := filepath.Join(accdir, "junkfilter.db") jfbloompath := filepath.Join(accdir, "junkfilter.bloom") if exists(jfdbpath) || exists(jfbloompath) { - checkDB(jfdbpath, junk.DBTypes) + checkDB(true, jfdbpath, junk.DBTypes) } // todo: add some kind of check for the bloom filter? @@ -399,7 +402,7 @@ possibly making them potentially no longer readable by the previous version. p = p[len(dataDir)+1:] } switch p { - case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion": + case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion": return nil case "acme", "queue", "accounts", "tmp", "moved": return fs.SkipDir @@ -417,9 +420,10 @@ possibly making them potentially no longer readable by the previous version. checkf(err, dataDir, "walking data directory") } - checkDB(filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.DBTypes) - checkDB(filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes) - checkDB(filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes) + checkDB(true, filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.ReportsDBTypes) + checkDB(false, filepath.Join(dataDir, "dmarceval.db"), dmarcdb.EvalDBTypes) // After v0.0.7. + checkDB(true, filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes) + checkDB(true, filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes) checkQueue() checkAccounts() checkOther() diff --git a/webadmin/admin.go b/webadmin/admin.go index 7ea92ab..ab2e929 100644 --- a/webadmin/admin.go +++ b/webadmin/admin.go @@ -1546,7 +1546,7 @@ func (Admin) TLSRPTSummaries(ctx context.Context, start, end time.Time, domain s // end (most recent first), then by domain. func (Admin) DMARCReports(ctx context.Context, start, end time.Time, domain string) (reports []dmarcdb.DomainFeedback) { reports, err := dmarcdb.RecordsPeriodDomain(ctx, start, end, domain) - xcheckf(ctx, err, "fetching dmarc reports from database") + xcheckf(ctx, err, "fetching dmarc aggregate reports from database") sort.Slice(reports, func(i, j int) bool { iend := reports[i].ReportMetadata.DateRange.End jend := reports[j].ReportMetadata.DateRange.End @@ -1565,9 +1565,9 @@ func (Admin) DMARCReportID(ctx context.Context, domain string, reportID int64) ( err = bstore.ErrAbsent } if err == bstore.ErrAbsent { - xcheckuserf(ctx, err, "fetching dmarc report from database") + xcheckuserf(ctx, err, "fetching dmarc aggregate report from database") } - xcheckf(ctx, err, "fetching dmarc report from database") + xcheckf(ctx, err, "fetching dmarc aggregate report from database") return report } @@ -1589,7 +1589,7 @@ type DMARCSummary struct { // The returned summaries are ordered by domain name. func (Admin) DMARCSummaries(ctx context.Context, start, end time.Time, domain string) (domainSummaries []DMARCSummary) { reports, err := dmarcdb.RecordsPeriodDomain(ctx, start, end, domain) - xcheckf(ctx, err, "fetching dmarc reports from database") + xcheckf(ctx, err, "fetching dmarc aggregate reports from database") summaries := map[string]DMARCSummary{} for _, r := range reports { sum := summaries[r.Domain] @@ -1932,3 +1932,31 @@ func (Admin) WebserverConfigSave(ctx context.Context, oldConf, newConf Webserver func (Admin) Transports(ctx context.Context) map[string]config.Transport { return mox.Conf.Static.Transports } + +// DMARCEvaluationStats returns a map of all domains with evaluations to a count of +// the evaluations and whether those evaluations will cause a report to be sent. +func (Admin) DMARCEvaluationStats(ctx context.Context) map[string]dmarcdb.EvaluationStat { + stats, err := dmarcdb.EvaluationStats(ctx) + xcheckf(ctx, err, "get evaluation stats") + return stats +} + +// DMARCEvaluationsDomain returns all evaluations for aggregate reports for the +// domain, sorted from oldest to most recent. +func (Admin) DMARCEvaluationsDomain(ctx context.Context, domain string) (dns.Domain, []dmarcdb.Evaluation) { + dom, err := dns.ParseDomain(domain) + xcheckf(ctx, err, "parsing domain") + + evals, err := dmarcdb.EvaluationsDomain(ctx, dom) + xcheckf(ctx, err, "get evaluations for domain") + return dom, evals +} + +// DMARCRemoveEvaluations removes evaluations for a domain. +func (Admin) DMARCRemoveEvaluations(ctx context.Context, domain string) { + dom, err := dns.ParseDomain(domain) + xcheckf(ctx, err, "parsing domain") + + err = dmarcdb.RemoveEvaluationsDomain(ctx, dom) + xcheckf(ctx, err, "removing evaluations for domain") +} diff --git a/webadmin/admin.html b/webadmin/admin.html index 66ddbc3..d19c774 100644 --- a/webadmin/admin.html +++ b/webadmin/admin.html @@ -260,11 +260,13 @@ const index = async () => { ), ), dom.br(), - dom.h2('Reporting'), - dom.div(dom.a('DMARC', attr({href: '#dmarc'}))), + dom.h2('Reports'), + dom.div(dom.a('DMARC', attr({href: '#dmarc/reports'}))), dom.div(dom.a('TLS', attr({href: '#tlsrpt'}))), + dom.br(), + dom.h2('Operations'), dom.div(dom.a('MTA-STS policies', attr({href: '#mtasts'}))), - // todo: outgoing DMARC findings + dom.div(dom.a('DMARC evaluations', attr({href: '#dmarc/evaluations'}))), // todo: outgoing TLSRPT findings // todo: routing, globally, per domain and per account dom.br(), @@ -418,6 +420,16 @@ const box = (color, ...l) => [ ), dom.br(), ] +const inlineBox = (color, ...l) => + dom.span( + style({ + display: 'inline-block', + padding: color ? '0.05em 0.2em' : '', + backgroundColor: color, + borderRadius: '3px', + }), + l, + ) const accounts = async () => { const accounts = await api.Accounts() @@ -1004,7 +1016,25 @@ const domainDNSCheck = async (d) => { ) } -const dmarc = async () => { +const dmarcIndex = async () => { + const page = document.getElementById('page') + dom._kids(page, + crumbs( + crumblink('Mox Admin', '#'), + 'DMARC reports and evaluations', + ), + dom.ul( + dom.li( + dom.a(attr({href: '#dmarc/reports'}), 'Reports'), ', incoming DMARC aggregate reports.', + ), + dom.li( + dom.a(attr({href: '#dmarc/evaluations'}), 'Evaluations'), ', for outgoing DMARC aggregate reports.', + ), + ), + ) +} + +const dmarcReports = async () => { const end = new Date().toISOString() const start = new Date(new Date().getTime() - 30*24*3600*1000).toISOString() const summaries = await api.DMARCSummaries(start, end, "") @@ -1013,7 +1043,8 @@ const dmarc = async () => { dom._kids(page, crumbs( crumblink('Mox Admin', '#'), - 'DMARC aggregate reporting summary', + crumblink('DMARC', '#dmarc'), + 'Aggregate reporting summary', ), dom.p('DMARC reports are periodically sent by other mail servers that received an email message with a "From" header with our domain. Domains can have a DMARC DNS record that asks other mail servers to send these aggregate reports for analysis.'), renderDMARCSummaries(summaries), @@ -1051,6 +1082,167 @@ const renderDMARCSummaries = (summaries) => { ] } +const dmarcEvaluations = async () => { + const evalStats = await api.DMARCEvaluationStats() + + const isEmpty = (o) => { + for (const e in o) { + return false + } + return true + } + + const page = document.getElementById('page') + dom._kids(page, + crumbs( + crumblink('Mox Admin', '#'), + crumblink('DMARC', '#dmarc'), + 'Evaluations', + ), + dom.p('Incoming messages are checked against the DMARC policy of the domain in the message From header. If the policy requests reporting on the resulting evaluations, they are stored in the database. Each interval of 1 to 24 hours, the evaluations may be sent to a reporting address specified in the domain\'s DMARC policy. Not all evaluations are a reason to send a report, but if a report is sent all evaluations are included.'), + dom.table( + dom.thead( + dom.tr( + dom.th('Domain', attr({title: 'Domain in the message From header. Keep in mind these can be forged, so this does not necessarily mean someone from this domain authentically tried delivering email.'})), + dom.th('Evaluations', attr({title: 'Total number of message delivery attempts, including retries.'})), + dom.th('Send report', attr({title: 'Whether the current evaluations will cause a report to be sent.'})), + ), + ), + dom.tbody( + Object.entries(evalStats).sort((a, b) => a[0] < b[0] ? -1 : 1).map(t => + dom.tr( + dom.td(dom.a(attr({href: '#dmarc/evaluations/'+domainName(t[1].Domain)}), domainString(t[1].Domain))), + dom.td(style({textAlign: 'right'}), ''+t[1].Count), + dom.td(style({textAlign: 'right'}), t[1].SendReport ? '✓' : ''), + ), + ), + isEmpty(evalStats) ? dom.tr(dom.td(attr({colspan: '3'}), 'No evaluations.')) : [], + ), + ), + ) +} + +const dmarcEvaluationsDomain = async (domain) => { + const [d, evaluations] = await api.DMARCEvaluationsDomain(domain) + + let lastInterval = '' + let lastAddresses = '' + + const formatPolicy = (e) => { + const p = e.PolicyPublished + let s = '' + const add = (k, v) => { + if (v) { + s += k+'='+v+'; ' + } + } + add('p', p.Policy) + add('sp', p.SubdomainPolicy) + add('adkim', p.ADKIM) + add('aspf', p.ASPF) + add('pct', ''+p.Percentage) + add('fo', ''+p.ReportingOptions) + return s + } + let lastPolicy = '' + + const authStatus = (v) => inlineBox(v ? '' : yellow, v ? 'pass' : 'fail') + const formatDKIMResults = (results) => results.map(r => dom.div('selector '+r.Selector+(r.Domain !== domain ? ', domain '+r.Domain : '') + ': ', inlineBox(r.Result === "pass" ? '' : yellow, r.Result))) + const formatSPFResults = (results) => results.map(r => dom.div(''+r.Scope+(r.Domain !== domain ? ', domain '+r.Domain : '') + ': ', inlineBox(r.Result === "pass" ? '' : yellow, r.Result))) + + const sourceIP = (ip) => { + const r = dom.span(ip, attr({title: 'Click to do a reverse lookup of the IP.'}), style({cursor: 'pointer'}), async function click(e) { + e.preventDefault() + try { + const rev = await api.LookupIP(ip) + r.innerText = ip + '\n' + rev.Hostnames.join('\n') + } catch (err) { + r.innerText = ip + '\nerror: ' +err.message + } + }) + return r + } + + const page = document.getElementById('page') + dom._kids(page, + crumbs( + crumblink('Mox Admin', '#'), + crumblink('DMARC', '#dmarc'), + crumblink('Evaluations', '#dmarc/evaluations'), + 'Domain '+domainString(d), + ), + dom.div( + dom.button('Remove evaluations', async function click(e) { + e.target.disabled = true + try { + await api.DMARCRemoveEvaluations(domain) + window.location.reload() // todo: only clear the table? + } catch (err) { + console.log({err}) + window.alert('Error: ' + err.message) + } finally { + e.target.disabled = false + } + }), + ), + dom.br(), + dom.p('The evaluations below will be sent in a DMARC aggregate report to the addresses found in the published DMARC DNS record, which is fetched again before sending the report. The fields Interval hours, Addresses and Policy are only filled for the first row and whenever a new value in the published DMARC record is encountered.'), + dom.table( + dom.thead( + dom.tr( + dom.th('ID'), + dom.th('Evaluated'), + dom.th('Optional', attr({title: 'Some evaluations will not cause a DMARC aggregate report to be sent. But if a report is sent, optional records are included.'})), + dom.th('Interval hours', attr({title: 'DMARC policies published by a domain can specify how often they would like to receive reports. The default is 24 hours, but can be as often as each hour. To keep reports comparable between different mail servers that send reports, reports are sent at rounded up intervals of whole hours that can divide a 24 hour day, and are aligned with the start of a day at UTC.'})), + dom.th('Addresses', attr({title: 'Addresses that will receive the report. An address can have a maximum report size configured. If there is no address, no report will be sent.'})), + dom.th('Policy', attr({title: 'Summary of the policy as encountered in the DMARC DNS record of the domain, and used for evaluation.'})), + dom.th('IP', attr({title: 'IP address of delivery attempt that was evaluated, relevant for SPF.'})), + dom.th('Disposition', attr({title: 'Our decision to accept/reject this message. It may be different than requested by the published policy. For example, when overriding due to delivery from a mailing list or forwarded address.'})), + dom.th('DKIM/SPF', attr({title: 'Whether DKIM and SPF had an aligned pass, where strict/relaxed alignment means whether the domain of an SPF pass and DKIM pass matches the exact domain (strict) or optionally a subdomain (relaxed). A DMARC pass requires at least one pass.'})), + dom.th('Envelope to', attr({title: 'Domain used in SMTP RCPT TO during delivery.'})), + dom.th('Envelope from', attr({title: 'Domain used in SMTP MAIL FROM during delivery.'})), + dom.th('Message from', attr({title: 'Domain in "From" message header.'})), + dom.th('DKIM details', attr({title: 'Results of verifying DKIM-Signature headers in message. Only signatures with matching organizational domain are included, regardless of strict/relaxed DKIM alignment in DMARC policy.'})), + dom.th('SPF details', attr({title: 'Results of SPF check used in DMARC evaluation. "mfrom" indicates the "SMTP MAIL FROM" domain was used, "helo" indicates the SMTP EHLO domain was used.'})), + ), + ), + dom.tbody( + evaluations.map(e => { + const ival = e.IntervalHours + 'h' + const interval = ival === lastInterval ? '' : ival + lastInterval = ival + + const a = (e.Addresses || []).join('\n') + const addresses = a === lastAddresses ? '' : a + lastAddresses = a + + const p = formatPolicy(e) + const policy = p === lastPolicy ? '' : p + lastPolicy = p + + return dom.tr( + dom.td(''+e.ID), + dom.td(new Date(e.Evaluated).toUTCString()), + dom.td(e.Optional ? 'Yes' : ''), + dom.td(interval), + dom.td(addresses), + dom.td(policy), + dom.td(sourceIP(e.SourceIP)), + dom.td(inlineBox(e.Disposition === 'none' ? '' : 'red', e.Disposition), (e.OverrideReasons || []).length > 0 ? ' ('+e.OverrideReasons.map(r => r.Type).join(', ')+')' : ''), + dom.td(authStatus(e.AlignedDKIMPass), '/', authStatus(e.AlignedSPFPass)), + dom.td(e.EnvelopeTo), + dom.td(e.EnvelopeFrom), + dom.td(e.HeaderFrom), + dom.td(formatDKIMResults(e.DKIMResults || [])), + dom.td(formatSPFResults(e.SPFResults || [])), + ) + }), + evaluations.length === 0 ? dom.tr(dom.td(attr({colspan: '14'}), 'No evaluations.')) : [], + ), + ), + ) +} + const utcDate = (dt) => new Date(Date.UTC(dt.getUTCFullYear(), dt.getUTCMonth(), dt.getUTCDate(), dt.getUTCHours(), dt.getUTCMinutes(), dt.getUTCSeconds())) const utcDateStr = (dt) => [dt.getUTCFullYear(), 1+dt.getUTCMonth(), dt.getUTCDate()].join('-') const isDayChange = (dt) => utcDateStr(new Date(dt.getTime() - 2*60*1000)) !== utcDateStr(new Date(dt.getTime() + 2*60*1000)) @@ -2241,7 +2433,13 @@ const init = async () => { } else if (h === 'tlsrpt') { await tlsrpt() } else if (h === 'dmarc') { - await dmarc() + await dmarcIndex() + } else if (h === 'dmarc/reports') { + await dmarcReports() + } else if (h === 'dmarc/evaluations') { + await dmarcEvaluations() + } else if (t[0] == 'dmarc' && t[1] == 'evaluations' && t.length === 3) { + await dmarcEvaluationsDomain(t[2]) } else if (h === 'mtasts') { await mtasts() } else if (h === 'dnsbl') { diff --git a/webadmin/adminapi.json b/webadmin/adminapi.json index 09549b9..f0c2028 100644 --- a/webadmin/adminapi.json +++ b/webadmin/adminapi.json @@ -753,6 +753,60 @@ ] } ] + }, + { + "Name": "DMARCEvaluationStats", + "Docs": "DMARCEvaluationStats returns a map of all domains with evaluations to a count of\nthe evaluations and whether those evaluations will cause a report to be sent.", + "Params": [], + "Returns": [ + { + "Name": "r0", + "Typewords": [ + "{}", + "EvaluationStat" + ] + } + ] + }, + { + "Name": "DMARCEvaluationsDomain", + "Docs": "DMARCEvaluationsDomain returns all evaluations for aggregate reports for the\ndomain, sorted from oldest to most recent.", + "Params": [ + { + "Name": "domain", + "Typewords": [ + "string" + ] + } + ], + "Returns": [ + { + "Name": "r0", + "Typewords": [ + "Domain" + ] + }, + { + "Name": "r1", + "Typewords": [ + "[]", + "Evaluation" + ] + } + ] + }, + { + "Name": "DMARCRemoveEvaluations", + "Docs": "DMARCRemoveEvaluations removes evaluations for a domain.", + "Params": [ + { + "Name": "domain", + "Typewords": [ + "string" + ] + } + ], + "Returns": [] } ], "Sections": [], @@ -2512,7 +2566,7 @@ "Fields": [ { "Name": "Domain", - "Docs": "", + "Docs": "Domain is where DMARC record was found, not necessarily message From. Reports we generate use unicode names, incoming reports may have either ASCII-only or Unicode domains.", "Typewords": [ "string" ] @@ -2914,7 +2968,7 @@ }, { "Name": "Msg", - "Docs": "Msg is a message in the queue.", + "Docs": "Msg is a message in the queue.\n\nUse MakeMsg to make a message with fields that Add needs. Add will further set\nqueueing related fields.", "Fields": [ { "Name": "ID", @@ -2979,6 +3033,13 @@ "int32" ] }, + { + "Name": "MaxAttempts", + "Docs": "Max number of attempts before giving up. If 0, then the default of 8 attempts is used instead.", + "Typewords": [ + "int32" + ] + }, { "Name": "DialedIPs", "Docs": "For each host, the IPs that were dialed. Used for IP selection for later attempts.", @@ -3024,6 +3085,20 @@ "bool" ] }, + { + "Name": "IsDMARCReport", + "Docs": "Delivery failures for DMARC reports are handled differently.", + "Typewords": [ + "bool" + ] + }, + { + "Name": "IsTLSReport", + "Docs": "Delivery failures for TLS reports are handled differently.", + "Typewords": [ + "bool" + ] + }, { "Name": "Size", "Docs": "Full size of message, combined MsgPrefix with contents of message file.", @@ -3441,6 +3516,162 @@ ] } ] + }, + { + "Name": "EvaluationStat", + "Docs": "EvaluationStat summarizes stored evaluations, for inclusion in an upcoming\naggregate report, for a domain.", + "Fields": [ + { + "Name": "Count", + "Docs": "", + "Typewords": [ + "int32" + ] + }, + { + "Name": "SendReport", + "Docs": "", + "Typewords": [ + "bool" + ] + }, + { + "Name": "Domain", + "Docs": "", + "Typewords": [ + "Domain" + ] + } + ] + }, + { + "Name": "Evaluation", + "Docs": "Evaluation is the result of an evaluation of a DMARC policy, to be included\nin a DMARC report.", + "Fields": [ + { + "Name": "ID", + "Docs": "", + "Typewords": [ + "int64" + ] + }, + { + "Name": "PolicyDomain", + "Docs": "Domain where DMARC policy was found, could be the organizational domain while evaluation was for a subdomain. Unicode. Same as domain found in PolicyPublished. A separate field for its index.", + "Typewords": [ + "string" + ] + }, + { + "Name": "Evaluated", + "Docs": "Time of evaluation, determines which report (covering whole hours) this evaluation will be included in.", + "Typewords": [ + "timestamp" + ] + }, + { + "Name": "Optional", + "Docs": "If optional, this evaluation is not a reason to send a DMARC report, but it will be included when a report is sent due to other non-optional evaluations. Set for evaluations of incoming DMARC reports. We don't want such deliveries causing us to send a report, or we would keep exchanging reporting messages forever. Also set for when evaluation is a DMARC reject for domains we haven't positively interacted with, to prevent being used to flood an unsuspecting domain with reports.", + "Typewords": [ + "bool" + ] + }, + { + "Name": "IntervalHours", + "Docs": "Effective aggregate reporting interval in hours. Between 1 and 24, rounded up from seconds from policy to first number that can divide 24.", + "Typewords": [ + "int32" + ] + }, + { + "Name": "Addresses", + "Docs": "\"rua\" in DMARC record, we only store evaluations for records with aggregate reporting addresses, so always non-empty.", + "Typewords": [ + "[]", + "string" + ] + }, + { + "Name": "PolicyPublished", + "Docs": "Policy used for evaluation. We don't store the \"fo\" field for failure reporting options, since we don't send failure reports for individual messages.", + "Typewords": [ + "PolicyPublished" + ] + }, + { + "Name": "SourceIP", + "Docs": "For \"row\" in a report record.", + "Typewords": [ + "string" + ] + }, + { + "Name": "Disposition", + "Docs": "", + "Typewords": [ + "Disposition" + ] + }, + { + "Name": "AlignedDKIMPass", + "Docs": "", + "Typewords": [ + "bool" + ] + }, + { + "Name": "AlignedSPFPass", + "Docs": "", + "Typewords": [ + "bool" + ] + }, + { + "Name": "OverrideReasons", + "Docs": "", + "Typewords": [ + "[]", + "PolicyOverrideReason" + ] + }, + { + "Name": "EnvelopeTo", + "Docs": "For \"identifiers\" in a report record.", + "Typewords": [ + "string" + ] + }, + { + "Name": "EnvelopeFrom", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "HeaderFrom", + "Docs": "", + "Typewords": [ + "string" + ] + }, + { + "Name": "DKIMResults", + "Docs": "For \"auth_results\" in a report record.", + "Typewords": [ + "[]", + "DKIMAuthResult" + ] + }, + { + "Name": "SPFResults", + "Docs": "", + "Typewords": [ + "[]", + "SPFAuthResult" + ] + } + ] } ], "Ints": [], diff --git a/webmail/api.go b/webmail/api.go index 9e94ab7..c828ba7 100644 --- a/webmail/api.go +++ b/webmail/api.go @@ -691,7 +691,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) { Localpart: rcpt.Localpart, IPDomain: dns.IPDomain{Domain: rcpt.Domain}, } - _, err := queue.Add(ctx, log, reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), dataFile, nil, m.RequireTLS) + qm := queue.MakeMsg(reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS) + err := queue.Add(ctx, log, &qm, dataFile) if err != nil { metricSubmission.WithLabelValues("queueerror").Inc() }