serv/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2024-12-26 08:23:48 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

implement outgoing dmarc aggregate reporting

Browse source 
in smtpserver, we store dmarc evaluations (under the right conditions).
in dmarcdb, we periodically (hourly) send dmarc reports if there are
evaluations. for failed deliveries, we deliver the dsn quietly to a submailbox
of the postmaster mailbox.

this is on by default, but can be disabled in mox.conf.
This commit is contained in:
Mechiel Lukkien 2023-11-01 17:55:40 +01:00
parent d1e93020d8
commit e7699708ef
No known key found for this signature in database
40 changed files with 2689 additions and 245 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -5,6 +5,7 @@
/testdata/check/
/testdata/ctl/data/
/testdata/ctl/dkim/
/testdata/dmarcdb/data/
/testdata/empty/
/testdata/exportmaildir/
/testdata/exportmbox/

5
README.md
View file

@ -8,7 +8,8 @@ See Quickstart below to get started.
- SMTP (with extensions) for receiving, submitting and delivering email.
- IMAP4 (with extensions) for giving email clients access to email.
- Webmail for reading/sending email from the browser.
- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC reports.
- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC aggregate
reports.
- Reputation tracking, learning (per user) host-, domain- and
sender address-based reputation from (Non-)Junk email classification.
- Bayesian spam filtering that learns (per user) from (Non-)Junk email.
@ -113,7 +114,7 @@ https://nlnet.nl/project/Mox/.
## Roadmap
- Sending DMARC and TLS reports (currently only receiving)
- Sending TLS reports (currently only receiving)
- Authentication other than HTTP-basic for webmail/webadmin/webaccount
- Per-domain webmail and IMAP/SMTP host name (and TLS cert) and client settings
- Make mox Go packages more easily reusable, each pulling in fewer (internal)

5
backup.go
View file

@ -279,7 +279,8 @@ func backupctl(ctx context.Context, ctl *ctl) {
if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil {
xerrx("writing moxversion", err)
}
backupDB(dmarcdb.DB, "dmarcrpt.db")
backupDB(dmarcdb.ReportsDB, "dmarcrpt.db")
backupDB(dmarcdb.EvalDB, "dmarceval.db")
backupDB(mtastsdb.DB, "mtasts.db")
backupDB(tlsrptdb.DB, "tlsrpt.db")
backupFile("receivedid.key")
@ -529,7 +530,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
}
switch p {
case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
// Already handled.
return nil
case "lastknownversion": // Optional file, not yet handled.

7
config/config.go
View file

@ -57,9 +57,10 @@ type Static struct {
Account string
Mailbox string `sconf-doc:"E.g. Postmaster or Inbox."`
} `sconf-doc:"Destination for emails delivered to postmaster addresses: a plain 'postmaster' without domain, 'postmaster@<hostname>' (also for each listener with SMTP enabled), and as fallback for each domain without explicitly configured postmaster destination."`
InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
DefaultMailboxes []string `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
Transports map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
InitialMailboxes InitialMailboxes `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
DefaultMailboxes []string `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
Transports map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
NoOutgoingDMARCReports bool `sconf:"optional" sconf-doc:"Do not send DMARC reports (aggregate only). By default, aggregate reports on DMARC evaluations are sent to domains if their DMARC policy requests them. Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24 hours, rounded up so a whole number of intervals cover 24 hours, aligned at whole days in UTC."`
// All IPs that were explicitly listen on for external SMTP. Only set when there
// are no unspecified external SMTP listeners and there is at most one for IPv4 and

7
config/doc.go
View file

@ -549,6 +549,13 @@ describe-static" and "mox config describe-domains":
# typically the hostname of the host in the Address field.
RemoteHostname:
# Do not send DMARC reports (aggregate only). By default, aggregate reports on
# DMARC evaluations are sent to domains if their DMARC policy requests them.
# Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24
# hours, rounded up so a whole number of intervals cover 24 hours, aligned at
# whole days in UTC. (optional)
NoOutgoingDMARCReports: false
# domains.conf
# NOTE: This config file is in 'sconf' format. Indent with tabs. Comments must be

4
dkim/dkim.go
View file

@ -382,14 +382,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig)
if err != nil {
results = append(results, Result{StatusPermerror, nil, nil, false, err})
results = append(results, Result{StatusPermerror, sig, nil, false, err})
continue
}
// ../rfc/6376:2560
if err := policy(sig); err != nil {
err := fmt.Errorf("%w: %s", ErrPolicy, err)
results = append(results, Result{StatusPolicy, nil, nil, false, err})
results = append(results, Result{StatusPolicy, sig, nil, false, err})
continue
}

67
dmarc/dmarc.go
View file

@ -75,7 +75,9 @@ type Result struct {
Reject bool
// Result of DMARC validation. A message can fail validation, but still
// not be rejected, e.g. if the policy is "none".
Status Status
Status Status
AlignedSPFPass bool
AlignedDKIMPass bool
// Domain with the DMARC DNS record. May be the organizational domain instead of
// the domain in the From-header.
Domain dns.Domain
@ -142,7 +144,7 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain)
return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
}
if record != nil {
// ../ ../rfc/7489:1388
// ../rfc/7489:1388
return StatusNone, nil, "", result.Authentic, ErrMultipleRecords
}
text = txt
@ -152,14 +154,15 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain)
return StatusNone, record, text, result.Authentic, rerr
}
func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, *Record, string, bool, error) {
func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, []*Record, []string, bool, error) {
// ../rfc/7489:1566
name := dmarcDomain.ASCII + "._report._dmarc." + extDestDomain.ASCII + "."
txts, result, err := dns.WithPackage(resolver, "dmarc").LookupTXT(ctx, name)
if err != nil && !dns.IsNotFound(err) {
return StatusTemperror, nil, "", result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err)
return StatusTemperror, nil, nil, result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err)
}
var record *Record
var text string
var records []*Record
var texts []string
var rerr error = ErrNoRecord
for _, txt := range txts {
r, isdmarc, err := ParseRecordNoRequired(txt)
@ -171,44 +174,44 @@ func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain
r, isdmarc, err = &xr, true, nil
}
if !isdmarc {
// ../rfc/7489:1374
// ../rfc/7489:1586
continue
} else if err != nil {
return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
}
if record != nil {
// ../ ../rfc/7489:1388
return StatusNone, nil, "", result.Authentic, ErrMultipleRecords
texts = append(texts, txt)
records = append(records, r)
if err != nil {
return StatusPermerror, records, texts, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
}
text = txt
record = r
// Multiple records are allowed for the _report record, unlike for policies. ../rfc/7489:1593
rerr = nil
}
return StatusNone, record, text, result.Authentic, rerr
return StatusNone, records, texts, result.Authentic, rerr
}
// LookupExternalReportsAccepted returns whether the extDestDomain has opted in
// to receiving dmarc reports for dmarcDomain (where the dmarc record was found),
// through a "._report._dmarc." DNS TXT DMARC record.
//
// Callers should look at status for interpretation, not err, because err will
// be set to ErrNoRecord when the DNS TXT record isn't present, which means the
// extDestDomain does not opt in (not a failure condition).
// accepts is true if the external domain has opted in.
// If a temporary error occurred, the returned status is StatusTemperror, and a
// later retry may give an authoritative result.
// The returned error is ErrNoRecord if no opt-in DNS record exists, which is
// not a failure condition.
//
// The normally invalid "v=DMARC1" record is accepted since it is used as
// example in RFC 7489.
//
// authentic indicates if the DNS results were DNSSEC-verified.
func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, record *Record, txt string, authentic bool, rerr error) {
func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) {
log := xlog.WithContext(ctx)
start := time.Now()
defer func() {
log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("record", record), mlog.Field("duration", time.Since(start)))
log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("records", records), mlog.Field("duration", time.Since(start)))
}()
status, record, txt, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
accepts = rerr == nil
return accepts, status, record, txt, authentic, rerr
return accepts, status, records, txts, authentic, rerr
}
// Verify evaluates the DMARC policy for the domain in the From-header of a
@ -241,7 +244,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
status, recordDomain, record, _, authentic, err := Lookup(ctx, resolver, from)
if record == nil {
return false, Result{false, status, recordDomain, record, authentic, err}
return false, Result{false, status, false, false, recordDomain, record, authentic, err}
}
result.Domain = recordDomain
result.Record = record
@ -251,8 +254,8 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
// See ../rfc/7489:1432
useResult = !applyRandomPercentage || record.Percentage == 100 || mathrand.Intn(100) < record.Percentage
// We reject treat "quarantine" and "reject" the same. Thus, we also don't
// "downgrade" from reject to quarantine if this message was sampled out.
// We treat "quarantine" and "reject" the same. Thus, we also don't "downgrade"
// from reject to quarantine if this message was sampled out.
// ../rfc/7489:1446 ../rfc/7489:1024
if recordDomain != from && record.SubdomainPolicy != PolicyEmpty {
result.Reject = record.SubdomainPolicy != PolicyNone
@ -282,9 +285,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
// ../rfc/7489:1319
// ../rfc/7489:544
if spfResult == spf.StatusPass && spfIdentity != nil && (*spfIdentity == from || result.Record.ASPF == "r" && pubsuffix(from) == pubsuffix(*spfIdentity)) {
result.Reject = false
result.Status = StatusPass
return
result.AlignedSPFPass = true
}
for _, dkimResult := range dkimResults {
@ -296,10 +297,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
// ../rfc/7489:511
if dkimResult.Status == dkim.StatusPass && dkimResult.Sig != nil && (dkimResult.Sig.Domain == from || result.Record.ADKIM == "r" && pubsuffix(from) == pubsuffix(dkimResult.Sig.Domain)) {
// ../rfc/7489:535
result.Reject = false
result.Status = StatusPass
return
result.AlignedDKIMPass = true
break
}
}
if result.AlignedSPFPass || result.AlignedDKIMPass {
result.Reject = false
result.Status = StatusPass
}
return
}

32
dmarc/dmarc_test.go
View file

@ -84,7 +84,7 @@ func TestLookupExternalReportsAccepted(t *testing.T) {
test("example.com", "simple2.example", StatusNone, true, nil)
test("example.com", "one.example", StatusNone, true, nil)
test("example.com", "absent.example", StatusNone, false, ErrNoRecord)
test("example.com", "multiple.example", StatusNone, false, ErrMultipleRecords)
test("example.com", "multiple.example", StatusNone, true, nil)
test("example.com", "malformed.example", StatusPermerror, false, ErrSyntax)
test("example.com", "temperror.example", StatusTemperror, false, ErrDNS)
}
@ -137,7 +137,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusNone,
nil,
true, Result{true, StatusFail, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
)
// Accept with spf pass.
@ -145,7 +145,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusPass,
&dns.Domain{ASCII: "sub.reject.example"},
true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
)
// Accept with dkim pass.
@ -161,7 +161,7 @@ func TestVerify(t *testing.T) {
},
spf.StatusFail,
&dns.Domain{ASCII: "reject.example"},
true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
)
// Reject due to spf and dkim "strict".
@ -181,7 +181,7 @@ func TestVerify(t *testing.T) {
},
spf.StatusPass,
&dns.Domain{ASCII: "sub.strict.example"},
true, Result{true, StatusFail, dns.Domain{ASCII: "strict.example"}, &strict, false, nil},
true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "strict.example"}, &strict, false, nil},
)
// No dmarc policy, nothing to say.
@ -189,7 +189,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusNone,
nil,
false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
)
// No dmarc policy, spf pass does nothing.
@ -197,7 +197,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusPass,
&dns.Domain{ASCII: "absent.example"},
false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
)
none := DefaultRecord
@ -207,7 +207,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusPass,
&dns.Domain{ASCII: "none.example"},
true, Result{false, StatusPass, dns.Domain{ASCII: "none.example"}, &none, false, nil},
true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "none.example"}, &none, false, nil},
)
// No actual reject due to pct=0.
@ -218,7 +218,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusNone,
nil,
false, Result{true, StatusFail, dns.Domain{ASCII: "test.example"}, &testr, false, nil},
false, Result{true, StatusFail, false, false, dns.Domain{ASCII: "test.example"}, &testr, false, nil},
)
// No reject if subdomain has "none" policy.
@ -229,7 +229,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusFail,
&dns.Domain{ASCII: "sub.subnone.example"},
true, Result{false, StatusFail, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil},
true, Result{false, StatusFail, false, false, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil},
)
// No reject if spf temperror and no other pass.
@ -237,7 +237,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusTemperror,
&dns.Domain{ASCII: "mail.reject.example"},
true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
)
// No reject if dkim temperror and no other pass.
@ -253,7 +253,7 @@ func TestVerify(t *testing.T) {
},
spf.StatusNone,
nil,
true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
)
// No reject if spf temperror but still dkim pass.
@ -269,7 +269,7 @@ func TestVerify(t *testing.T) {
},
spf.StatusTemperror,
&dns.Domain{ASCII: "mail.reject.example"},
true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
)
// No reject if dkim temperror but still spf pass.
@ -285,7 +285,7 @@ func TestVerify(t *testing.T) {
},
spf.StatusPass,
&dns.Domain{ASCII: "mail.reject.example"},
true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
)
// Bad DMARC record results in permerror without reject.
@ -293,7 +293,7 @@ func TestVerify(t *testing.T) {
[]dkim.Result{},
spf.StatusNone,
nil,
false, Result{false, StatusPermerror, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax},
false, Result{false, StatusPermerror, false, false, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax},
)
// DKIM domain that is higher-level than organizational can not result in a pass. ../rfc/7489:525
@ -309,6 +309,6 @@ func TestVerify(t *testing.T) {
},
spf.StatusNone,
nil,
true, Result{true, StatusFail, dns.Domain{ASCII: "example.com"}, &reject, false, nil},
true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "example.com"}, &reject, false, nil},
)
}

29
dmarcdb/dmarcdb.go Normal file
View file

@ -0,0 +1,29 @@
// Package dmarcdb stores incoming DMARC aggrate reports and evaluations for outgoing aggregate reports.
//
// With DMARC, a domain can request reports with DMARC evaluation results to be
// sent to a specified address. Mox parses such reports, stores them in its
// database and makes them available through its admin web interface. Mox also
// keeps track of the evaluations it does for incoming messages and sends reports
// to mail servers that request reports.
//
// Only aggregate reports are stored and sent. Failure reports about individual
// messages are not implemented.
package dmarcdb
import (
"github.com/mjl-/mox/mox-"
)
// Init opens the databases.
//
// The incoming reports and evaluations for outgoing reports are in separate
// databases for simpler file-based handling of the databases.
func Init() error {
if _, err := reportsDB(mox.Shutdown); err != nil {
return err
}
if _, err := evalDB(mox.Shutdown); err != nil {
return err
}
return nil
}

1112
dmarcdb/eval.go Normal file
View file

File diff suppressed because it is too large Load diff

384
dmarcdb/eval_test.go Normal file
View file

@ -0,0 +1,384 @@
package dmarcdb
import (
"context"
"encoding/json"
"encoding/xml"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"github.com/mjl-/mox/dmarcrpt"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/queue"
)
func tcheckf(t *testing.T, err error, format string, args ...any) {
t.Helper()
if err != nil {
t.Fatalf("%s: %s", fmt.Sprintf(format, args...), err)
}
}
func tcompare(t *testing.T, got, expect any) {
t.Helper()
if !reflect.DeepEqual(got, expect) {
t.Fatalf("got:\n%v\nexpected:\n%v", got, expect)
}
}
func TestEvaluations(t *testing.T) {
os.RemoveAll("../testdata/dmarcdb/data")
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
mox.MustLoadConfig(true, false)
EvalDB = nil
_, err := evalDB(ctxbg)
tcheckf(t, err, "database")
defer func() {
EvalDB.Close()
EvalDB = nil
}()
parseJSON := func(s string) (e Evaluation) {
t.Helper()
err := json.Unmarshal([]byte(s), &e)
tcheckf(t, err, "unmarshal")
return
}
packJSON := func(e Evaluation) string {
t.Helper()
buf, err := json.Marshal(e)
tcheckf(t, err, "marshal")
return string(buf)
}
e0 := Evaluation{
PolicyDomain: "sender1.example",
Evaluated: time.Now().Round(0),
IntervalHours: 1,
PolicyPublished: dmarcrpt.PolicyPublished{
Domain: "sender1.example",
ADKIM: dmarcrpt.AlignmentRelaxed,
ASPF: dmarcrpt.AlignmentRelaxed,
Policy: dmarcrpt.DispositionReject,
SubdomainPolicy: dmarcrpt.DispositionReject,
Percentage: 100,
},
SourceIP: "10.1.2.3",
Disposition: dmarcrpt.DispositionNone,
AlignedDKIMPass: true,
AlignedSPFPass: true,
EnvelopeTo: "mox.example",
EnvelopeFrom: "sender1.example",
HeaderFrom: "sender1.example",
DKIMResults: []dmarcrpt.DKIMAuthResult{
{
Domain: "sender1.example",
Selector: "test",
Result: dmarcrpt.DKIMPass,
},
},
SPFResults: []dmarcrpt.SPFAuthResult{
{
Domain: "sender1.example",
Scope: dmarcrpt.SPFDomainScopeMailFrom,
Result: dmarcrpt.SPFPass,
},
},
}
e1 := e0
e2 := parseJSON(strings.ReplaceAll(packJSON(e0), "sender1.example", "sender2.example"))
e3 := parseJSON(strings.ReplaceAll(packJSON(e0), "10.1.2.3", "10.3.2.1"))
e3.Optional = true
for i, e := range []*Evaluation{&e0, &e1, &e2, &e3} {
e.Evaluated = e.Evaluated.Add(time.Duration(i) * time.Second)
err = AddEvaluation(ctxbg, 3600, e)
tcheckf(t, err, "add evaluation")
}
expStats := map[string]EvaluationStat{
"sender1.example": {
Count: 3,
SendReport: true,
Domain: dns.Domain{ASCII: "sender1.example"},
},
"sender2.example": {
Count: 1,
SendReport: true,
Domain: dns.Domain{ASCII: "sender2.example"},
},
}
stats, err := EvaluationStats(ctxbg)
tcheckf(t, err, "evaluation stats")
tcompare(t, stats, expStats)
// EvaluationsDomain
evals, err := EvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender1.example"})
tcheckf(t, err, "get evaluations for domain")
tcompare(t, evals, []Evaluation{e0, e1, e3})
evals, err = EvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender2.example"})
tcheckf(t, err, "get evaluations for domain")
tcompare(t, evals, []Evaluation{e2})
evals, err = EvaluationsDomain(ctxbg, dns.Domain{ASCII: "bogus.example"})
tcheckf(t, err, "get evaluations for domain")
tcompare(t, evals, []Evaluation{})
// RemoveEvaluationsDomain
err = RemoveEvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender1.example"})
tcheckf(t, err, "remove evaluations")
expStats = map[string]EvaluationStat{
"sender2.example": {
Count: 1,
SendReport: true,
Domain: dns.Domain{ASCII: "sender2.example"},
},
}
stats, err = EvaluationStats(ctxbg)
tcheckf(t, err, "evaluation stats")
tcompare(t, stats, expStats)
}
func TestSendReports(t *testing.T) {
mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug})
os.RemoveAll("../testdata/dmarcdb/data")
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
mox.MustLoadConfig(true, false)
EvalDB = nil
db, err := evalDB(ctxbg)
tcheckf(t, err, "database")
defer func() {
EvalDB.Close()
EvalDB = nil
}()
resolver := dns.MockResolver{
TXT: map[string][]string{
"_dmarc.sender.example.": {
"v=DMARC1; rua=mailto:dmarcrpt@sender.example; ri=3600",
},
},
}
end := nextWholeHour(time.Now())
eval := Evaluation{
PolicyDomain: "sender.example",
Evaluated: end.Add(-time.Hour / 2),
IntervalHours: 1,
PolicyPublished: dmarcrpt.PolicyPublished{
Domain: "sender.example",
ADKIM: dmarcrpt.AlignmentRelaxed,
ASPF: dmarcrpt.AlignmentRelaxed,
Policy: dmarcrpt.DispositionReject,
SubdomainPolicy: dmarcrpt.DispositionReject,
Percentage: 100,
},
SourceIP: "10.1.2.3",
Disposition: dmarcrpt.DispositionNone,
AlignedDKIMPass: true,
AlignedSPFPass: true,
EnvelopeTo: "mox.example",
EnvelopeFrom: "sender.example",
HeaderFrom: "sender.example",
DKIMResults: []dmarcrpt.DKIMAuthResult{
{
Domain: "sender.example",
Selector: "test",
Result: dmarcrpt.DKIMPass,
},
},
SPFResults: []dmarcrpt.SPFAuthResult{
{
Domain: "sender.example",
Scope: dmarcrpt.SPFDomainScopeMailFrom,
Result: dmarcrpt.SPFPass,
},
},
}
expFeedback := &dmarcrpt.Feedback{
XMLName: xml.Name{Local: "feedback"},
Version: "1.0",
ReportMetadata: dmarcrpt.ReportMetadata{
OrgName: "mail.mox.example",
Email: "postmaster@mail.mox.example",
DateRange: dmarcrpt.DateRange{
Begin: end.Add(-1 * time.Hour).Unix(),
End: end.Add(-time.Second).Unix(),
},
},
PolicyPublished: dmarcrpt.PolicyPublished{
Domain: "sender.example",
ADKIM: dmarcrpt.AlignmentRelaxed,
ASPF: dmarcrpt.AlignmentRelaxed,
Policy: dmarcrpt.DispositionReject,
SubdomainPolicy: dmarcrpt.DispositionReject,
Percentage: 100,
},
Records: []dmarcrpt.ReportRecord{
{
Row: dmarcrpt.Row{
SourceIP: "10.1.2.3",
Count: 1,
PolicyEvaluated: dmarcrpt.PolicyEvaluated{
Disposition: dmarcrpt.DispositionNone,
DKIM: dmarcrpt.DMARCPass,
SPF: dmarcrpt.DMARCPass,
},
},
Identifiers: dmarcrpt.Identifiers{
EnvelopeTo: "mox.example",
EnvelopeFrom: "sender.example",
HeaderFrom: "sender.example",
},
AuthResults: dmarcrpt.AuthResults{
DKIM: []dmarcrpt.DKIMAuthResult{
{
Domain: "sender.example",
Selector: "test",
Result: dmarcrpt.DKIMPass,
},
},
SPF: []dmarcrpt.SPFAuthResult{
{
Domain: "sender.example",
Scope: dmarcrpt.SPFDomainScopeMailFrom,
Result: dmarcrpt.SPFPass,
},
},
},
},
},
}
// Set a timeUntil that we steplock and that causes the actual sleep to return immediately when we want to.
wait := make(chan struct{})
step := make(chan time.Duration)
jitteredTimeUntil = func(_ time.Time) time.Duration {
wait <- struct{}{}
return <-step
}
sleepBetween = func() {}
test := func(evals []Evaluation, expAggrAddrs map[string]struct{}, expErrorAddrs map[string]struct{}, optExpReport *dmarcrpt.Feedback) {
t.Helper()
mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
for _, e := range evals {
err := db.Insert(ctxbg, &e)
tcheckf(t, err, "inserting evaluation")
}
aggrAddrs := map[string]struct{}{}
errorAddrs := map[string]struct{}{}
queueAdd = func(ctx context.Context, log *mlog.Log, qm *queue.Msg, msgFile *os.File) error {
// Read message file. Also write copy to disk for inspection.
buf, err := io.ReadAll(&moxio.AtReader{R: msgFile})
tcheckf(t, err, "read report message")
err = os.WriteFile("../testdata/dmarcdb/data/report.eml", append(append([]byte{}, qm.MsgPrefix...), buf...), 0600)
tcheckf(t, err, "write report message")
var feedback *dmarcrpt.Feedback
addr := qm.Recipient().String()
isErrorReport := strings.Contains(string(buf), "DMARC aggregate reporting error report")
if isErrorReport {
errorAddrs[addr] = struct{}{}
} else {
aggrAddrs[addr] = struct{}{}
feedback, err = dmarcrpt.ParseMessageReport(log, msgFile)
tcheckf(t, err, "parsing generated report message")
}
if optExpReport != nil {
// Parse report in message and compare with expected.
expFeedback.ReportMetadata.ReportID = feedback.ReportMetadata.ReportID
tcompare(t, feedback, expFeedback)
}
return nil
}
Start(resolver)
// Run first loop.
<-wait
step <- 0
<-wait
tcompare(t, aggrAddrs, expAggrAddrs)
tcompare(t, errorAddrs, expErrorAddrs)
// Second loop. Evaluations cleaned, should not result in report messages.
aggrAddrs = map[string]struct{}{}
errorAddrs = map[string]struct{}{}
step <- 0
<-wait
tcompare(t, aggrAddrs, map[string]struct{}{})
tcompare(t, errorAddrs, map[string]struct{}{})
// Caus Start to stop.
mox.ShutdownCancel()
step <- time.Minute
}
// Typical case, with a single address that receives an aggregate report.
test([]Evaluation{eval}, map[string]struct{}{"dmarcrpt@sender.example": {}}, map[string]struct{}{}, expFeedback)
// Only optional evaluations, no report at all.
evalOpt := eval
evalOpt.Optional = true
test([]Evaluation{evalOpt}, map[string]struct{}{}, map[string]struct{}{}, nil)
// Two RUA's, one with a size limit that doesn't pass, and one that does pass.
resolver.TXT["_dmarc.sender.example."] = []string{"v=DMARC1; rua=mailto:dmarcrpt1@sender.example!1,mailto:dmarcrpt2@sender.example!10t; ri=3600"}
test([]Evaluation{eval}, map[string]struct{}{"dmarcrpt2@sender.example": {}}, map[string]struct{}{}, nil)
// Redirect to external domain, without permission, no report sent.
resolver.TXT["_dmarc.sender.example."] = []string{"v=DMARC1; rua=mailto:unauthorized@other.example"}
test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{}, nil)
// Redirect to external domain, with basic permission.
resolver.TXT = map[string][]string{
"_dmarc.sender.example.": {"v=DMARC1; rua=mailto:authorized@other.example"},
"sender.example._report._dmarc.other.example.": {"v=DMARC1"},
}
test([]Evaluation{eval}, map[string]struct{}{"authorized@other.example": {}}, map[string]struct{}{}, nil)
// Redirect to authorized external domain, with 2 allowed replacements and 1 invalid and 1 refusing due to size.
resolver.TXT = map[string][]string{
"_dmarc.sender.example.": {"v=DMARC1; rua=mailto:authorized@other.example"},
"sender.example._report._dmarc.other.example.": {"v=DMARC1; rua=mailto:good1@other.example,mailto:bad1@yetanother.example,mailto:good2@other.example,mailto:badsize@other.example!1"},
}
test([]Evaluation{eval}, map[string]struct{}{"good1@other.example": {}, "good2@other.example": {}}, map[string]struct{}{}, nil)
// Without RUA, we send no message.
resolver.TXT = map[string][]string{
"_dmarc.sender.example.": {"v=DMARC1;"},
}
test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{}, nil)
// If message size limit is reached, an error repor is sent.
resolver.TXT = map[string][]string{
"_dmarc.sender.example.": {"v=DMARC1; rua=mailto:dmarcrpt@sender.example!1"},
}
test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{"dmarcrpt@sender.example": {}}, nil)
}

40
dmarcdb/db.go → dmarcdb/reports.go
View file

@ -1,9 +1,3 @@
// Package dmarcdb stores incoming DMARC reports.
//
// With DMARC, a domain can request emails with DMARC verification results by
// remote mail servers to be sent to a specified address. Mox parses such
// reports, stores them in its database and makes them available through its
// admin web interface.
package dmarcdb
import (
@ -25,9 +19,9 @@ import (
)
var (
DBTypes = []any{DomainFeedback{}} // Types stored in DB.
DB *bstore.DB // Exported for backups.
mutex sync.Mutex
ReportsDBTypes = []any{DomainFeedback{}} // Types stored in DB.
ReportsDB *bstore.DB // Exported for backups.
reportsMutex sync.Mutex
)
var (
@ -65,25 +59,19 @@ type DomainFeedback struct {
dmarcrpt.Feedback
}
func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
mutex.Lock()
defer mutex.Unlock()
if DB == nil {
func reportsDB(ctx context.Context) (rdb *bstore.DB, rerr error) {
reportsMutex.Lock()
defer reportsMutex.Unlock()
if ReportsDB == nil {
p := mox.DataDirPath("dmarcrpt.db")
os.MkdirAll(filepath.Dir(p), 0770)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, ReportsDBTypes...)
if err != nil {
return nil, err
}
DB = db
ReportsDB = db
}
return DB, nil
}
// Init opens the database.
func Init() error {
_, err := database(mox.Shutdown)
return err
return ReportsDB, nil
}
// AddReport adds a DMARC aggregate feedback report from an email to the database,
@ -91,7 +79,7 @@ func Init() error {
//
// fromDomain is the domain in the report message From header.
func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) error {
db, err := database(ctx)
db, err := reportsDB(ctx)
if err != nil {
return err
}
@ -141,7 +129,7 @@ func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain)
// Records returns all reports in the database.
func Records(ctx context.Context) ([]DomainFeedback, error) {
db, err := database(ctx)
db, err := reportsDB(ctx)
if err != nil {
return nil, err
}
@ -151,7 +139,7 @@ func Records(ctx context.Context) ([]DomainFeedback, error) {
// RecordID returns the report for the ID.
func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
db, err := database(ctx)
db, err := reportsDB(ctx)
if err != nil {
return DomainFeedback{}, err
}
@ -164,7 +152,7 @@ func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
// RecordsPeriodDomain returns the reports overlapping start and end, for the given
// domain. If domain is empty, all records match for domain.
func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]DomainFeedback, error) {
db, err := database(ctx)
db, err := reportsDB(ctx)
if err != nil {
return nil, err
}

9
dmarcdb/db_test.go → dmarcdb/reports_test.go
View file

@ -17,8 +17,8 @@ var ctxbg = context.Background()
func TestDMARCDB(t *testing.T) {
mox.Shutdown = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/fake.conf")
mox.Conf.Static.DataDir = "."
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
mox.MustLoadConfig(true, false)
dbpath := mox.DataDirPath("dmarcrpt.db")
os.MkdirAll(filepath.Dir(dbpath), 0770)
@ -27,7 +27,10 @@ func TestDMARCDB(t *testing.T) {
t.Fatalf("init database: %s", err)
}
defer os.Remove(dbpath)
defer DB.Close()
defer func() {
ReportsDB.Close()
ReportsDB = nil
}()
feedback := &dmarcrpt.Feedback{
ReportMetadata: dmarcrpt.ReportMetadata{

8
dmarcrpt/feedback.go
View file

@ -1,9 +1,14 @@
package dmarcrpt
import (
"encoding/xml"
)
// Initially generated by xsdgen, then modified.
// Feedback is the top-level XML field returned.
type Feedback struct {
XMLName xml.Name `xml:"feedback" json:"-"` // todo: removing the json tag triggers bug in sherpadoc, should fix.
Version string `xml:"version"`
ReportMetadata ReportMetadata `xml:"report_metadata"`
PolicyPublished PolicyPublished `xml:"policy_published"`
@ -26,6 +31,9 @@ type DateRange struct {
// PolicyPublished is the policy as found in DNS for the domain.
type PolicyPublished struct {
// Domain is where DMARC record was found, not necessarily message From. Reports we
// generate use unicode names, incoming reports may have either ASCII-only or
// Unicode domains.
Domain string `xml:"domain"`
ADKIM Alignment `xml:"adkim,omitempty"`
ASPF Alignment `xml:"aspf,omitempty"`

2
dmarcrpt/parse.go
View file

@ -17,7 +17,7 @@ import (
"github.com/mjl-/mox/moxio"
)
var ErrNoReport = errors.New("no dmarc report found in message")
var ErrNoReport = errors.New("no dmarc aggregate report found in message")
// ParseReport parses an XML aggregate feedback report.
// The maximum report size is 20MB.

4
dmarcrpt/parse_test.go
View file

@ -1,6 +1,7 @@
package dmarcrpt
import (
"encoding/xml"
"os"
"path/filepath"
"reflect"
@ -62,6 +63,7 @@ const reportExample = `<?xml version="1.0" encoding="UTF-8" ?>
func TestParseReport(t *testing.T) {
var expect = &Feedback{
XMLName: xml.Name{Local: "feedback"},
ReportMetadata: ReportMetadata{
OrgName: "google.com",
Email: "noreply-dmarc-support@google.com",
@ -126,7 +128,7 @@ func TestParseMessageReport(t *testing.T) {
dir := filepath.FromSlash("../testdata/dmarc-reports")
files, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("listing dmarc report emails: %s", err)
t.Fatalf("listing dmarc aggregate report emails: %s", err)
}
for _, file := range files {

7
gentestdata.go
View file

@ -192,9 +192,9 @@ Accounts:
err = dmarcdb.Init()
xcheckf(err, "dmarcdb init")
report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
xcheckf(err, "parsing dmarc report")
xcheckf(err, "parsing dmarc aggregate report")
err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
xcheckf(err, "adding dmarc report")
xcheckf(err, "adding dmarc aggregate report")
// Populate mtasts.db.
err = mtastsdb.Init(false)
@ -233,7 +233,8 @@ Accounts:
const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
_, err = fmt.Fprint(mf, qmsg)
xcheckf(err, "writing message")
_, err = queue.Add(ctxbg, log, "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, mf, nil, nil)
qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil)
err = queue.Add(ctxbg, log, &qm, mf)
xcheckf(err, "enqueue message")
// Create three accounts.

3
localserve.go
View file

@ -124,8 +124,9 @@ during those commands instead of during "data".
queue.Localserve = true
const mtastsdbRefresher = false
const sendDMARCReports = false
const skipForkExec = true
if err := start(mtastsdbRefresher, skipForkExec); err != nil {
if err := start(mtastsdbRefresher, sendDMARCReports, skipForkExec); err != nil {
log.Fatalx("starting mox", err)
}
golog.Printf("mox, version %s", moxvar.Version)

8
main.go
View file

@ -2436,13 +2436,13 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at
return
}
accepts, status, _, txt, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom)
accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom)
var txtstr string
txtaddr := fmt.Sprintf("%s._report._dmarc.%s", domain.ASCII, destdom.ASCII)
if txt == "" {
txtstr = fmt.Sprintf(" (no txt record %s)", txtaddr)
if len(txts) == 0 {
txtstr = fmt.Sprintf(" (no txt records %s)", txtaddr)
} else {
txtstr = fmt.Sprintf(" (txt record %s: %q)", txtaddr, txt)
txtstr = fmt.Sprintf(" (txt record %s: %q)", txtaddr, txts)
}
if status != dmarc.StatusNone {
printResult("fail: %s%s", err, txtstr)

1
metrics/panic.go
View file

@ -22,6 +22,7 @@ const (
Import Panic = "import"
Serve Panic = "serve"
Imapserver Panic = "imapserver"
Dmarcdb Panic = "dmarcdb"
Mtastsdb Panic = "mtastsdb"
Queue Panic = "queue"
Smtpclient Panic = "smtpclient"

11
queue/direct.go
View file

@ -92,7 +92,7 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT
// todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls?
// todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379
if permanent || m.Attempts >= 8 {
if permanent || m.MaxAttempts == 0 && m.Attempts >= 8 || m.MaxAttempts > 0 && m.Attempts >= m.MaxAttempts {
qlog.Errorx("permanent failure delivering from queue", errors.New(errmsg))
deliverDSNFailure(qlog, m, remoteMTA, secodeOpt, errmsg)
@ -230,12 +230,13 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce
permanent, daneRequired, badTLS, secodeOpt, remoteIP, errmsg, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode)
// If we had a TLS-related failure when doing TLS, and we don't have a requirement for MTA-STS/DANE,
// we try again without TLS. This could be an old
// server that only does ancient TLS versions, or has a misconfiguration. Note that
// If we had a TLS-related failure when doing TLS, and we don't have a requirement
// for MTA-STS/DANE, we try again without TLS. This could be an old server that
// only does ancient TLS versions, or has a misconfiguration. Note that
// opportunistic TLS does not do regular certificate verification, so that can't be
// the problem.
if !ok && badTLS && (!enforceMTASTS && tlsMode == smtpclient.TLSOpportunistic && !daneRequired || m.RequireTLS != nil && !*m.RequireTLS) {
// We don't fall back to plain text for DMARC reports. ../rfc/7489:1768 ../rfc/7489:2683
if !ok && badTLS && (!enforceMTASTS && tlsMode == smtpclient.TLSOpportunistic && !daneRequired && !m.IsDMARCReport || m.RequireTLS != nil && !*m.RequireTLS) {
metricPlaintextFallback.Inc()
if m.RequireTLS != nil && !*m.RequireTLS {
metricTLSRequiredNoIgnored.WithLabelValues("badtls").Inc()

36
queue/dsn.go
View file

@ -6,6 +6,9 @@ import (
"os"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dsn"
"github.com/mjl-/mox/message"
@ -15,6 +18,15 @@ import (
"github.com/mjl-/mox/store"
)
var (
metricDMARCReportFailure = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_queue_dmarcreport_failure_total",
Help: "Permanent failures to deliver a DMARC report.",
},
)
)
func deliverDSNFailure(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
const subject = "mail delivery failed"
message := fmt.Sprintf(`
@ -33,6 +45,12 @@ Error during the last delivery attempt:
}
func deliverDSNDelay(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) {
// Should not happen, but doesn't hurt to prevent sending delayed delivery
// notifications for DMARC reports. We don't want to waste postmaster attention.
if m.IsDMARCReport {
return
}
const subject = "mail delivery delayed"
message := fmt.Sprintf(`
Delivery has been delayed of your email to:
@ -133,7 +151,12 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
msgData = append(msgData, []byte("Return-Path: <"+dsnMsg.From.XString(m.SMTPUTF8)+">\r\n")...)
mailbox := "Inbox"
acc, err := store.OpenAccount(m.SenderAccount)
senderAccount := m.SenderAccount
if m.IsDMARCReport {
// senderAccount should already by postmaster, but doesn't hurt to ensure it.
senderAccount = mox.Conf.Static.Postmaster.Account
}
acc, err := store.OpenAccount(senderAccount)
if err != nil {
acc, err = store.OpenAccount(mox.Conf.Static.Postmaster.Account)
if err != nil {
@ -171,6 +194,17 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
Size: msgWriter.Size,
MsgPrefix: []byte{},
}
// If this is a DMARC report, deliver it as seen message to a submailbox of the
// postmaster mailbox. We mark it as seen so it doesn't waste postmaster attention,
// but we deliver them so they can be checked in case of problems.
if m.IsDMARCReport {
mailbox = fmt.Sprintf("%s/dmarc", mox.Conf.Static.Postmaster.Mailbox)
msg.Seen = true
metricDMARCReportFailure.Inc()
log.Info("delivering dsn for failure to deliver outgoing dmarc report")
}
acc.WithWLock(func() {
if err := acc.DeliverMailbox(log, mailbox, msg, msgFile); err != nil {
qlog("delivering dsn to mailbox", err)

87
queue/queue.go
View file

@ -70,6 +70,9 @@ var DB *bstore.DB // Exported for making backups.
var Localserve bool
// Msg is a message in the queue.
//
// Use MakeMsg to make a message with fields that Add needs. Add will further set
// queueing related fields.
type Msg struct {
ID int64
Queued time.Time `bstore:"default now"`
@ -80,15 +83,19 @@ type Msg struct {
RecipientDomain dns.IPDomain
RecipientDomainStr string // For filtering.
Attempts int // Next attempt is based on last attempt and exponential back off based on attempts.
MaxAttempts int // Max number of attempts before giving up. If 0, then the default of 8 attempts is used instead.
DialedIPs map[string][]net.IP // For each host, the IPs that were dialed. Used for IP selection for later attempts.
NextAttempt time.Time // For scheduling.
LastAttempt *time.Time
LastError string
Has8bit bool // Whether message contains bytes with high bit set, determines whether 8BITMIME SMTP extension is needed.
SMTPUTF8 bool // Whether message requires use of SMTPUTF8.
Size int64 // Full size of message, combined MsgPrefix with contents of message file.
MessageID string // Used when composing a DSN, in its References header.
MsgPrefix []byte
Has8bit bool // Whether message contains bytes with high bit set, determines whether 8BITMIME SMTP extension is needed.
SMTPUTF8 bool // Whether message requires use of SMTPUTF8.
IsDMARCReport bool // Delivery failures for DMARC reports are handled differently.
IsTLSReport bool // Delivery failures for TLS reports are handled differently.
Size int64 // Full size of message, combined MsgPrefix with contents of message file.
MessageID string // Used when composing a DSN, in its References header.
MsgPrefix []byte
// If set, this message is a DSN and this is a version using utf-8, for the case
// the remote MTA supports smtputf8. In this case, Size and MsgPrefix are not
@ -188,44 +195,71 @@ func Count(ctx context.Context) (int, error) {
return bstore.QueryDB[Msg](ctx, DB).Count()
}
// MakeMsg is a convenience function that sets the commonly used fields for a Msg.
func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, prefix []byte, requireTLS *bool) Msg {
return Msg{
SenderAccount: mox.Conf.Static.Postmaster.Account,
SenderLocalpart: sender.Localpart,
SenderDomain: sender.IPDomain,
RecipientLocalpart: recipient.Localpart,
RecipientDomain: recipient.IPDomain,
Has8bit: has8bit,
SMTPUTF8: smtputf8,
Size: size,
MessageID: messageID,
MsgPrefix: prefix,
RequireTLS: requireTLS,
}
}
// Add a new message to the queue. The queue is kicked immediately to start a
// first delivery attempt.
//
// dnsutf8Opt is a utf8-version of the message, to be used only for DNSs. If set,
// this data is used as the message when delivering the DSN and the remote SMTP
// server supports SMTPUTF8. If the remote SMTP server does not support SMTPUTF8,
// the regular non-utf8 message is delivered.
func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, requireTLS *bool) (int64, error) {
// ID must be 0 and will be set after inserting in the queue.
//
// Add sets derived fields like RecipientDomainStr, and fields related to queueing,
// such as Queued, NextAttempt, LastAttempt, LastError.
func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error {
// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
if qm.ID != 0 {
return fmt.Errorf("id of queued message must be 0")
}
qm.Queued = time.Now()
qm.DialedIPs = nil
qm.NextAttempt = qm.Queued
qm.LastAttempt = nil
qm.LastError = ""
qm.RecipientDomainStr = formatIPDomain(qm.RecipientDomain)
if Localserve {
if senderAccount == "" {
return 0, fmt.Errorf("cannot queue with localserve without local account")
if qm.SenderAccount == "" {
return fmt.Errorf("cannot queue with localserve without local account")
}
acc, err := store.OpenAccount(senderAccount)
acc, err := store.OpenAccount(qm.SenderAccount)
if err != nil {
return 0, fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err)
return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err)
}
defer func() {
err := acc.Close()
log.Check(err, "closing account")
}()
m := store.Message{Size: size, MsgPrefix: msgPrefix}
m := store.Message{Size: qm.Size, MsgPrefix: qm.MsgPrefix}
conf, _ := acc.Conf()
dest := conf.Destinations[mailFrom.String()]
dest := conf.Destinations[qm.Sender().String()]
acc.WithWLock(func() {
err = acc.DeliverDestination(log, dest, &m, msgFile)
})
if err != nil {
return 0, fmt.Errorf("delivering message: %v", err)
return fmt.Errorf("delivering message: %v", err)
}
log.Debug("immediately delivered from queue to sender")
return 0, nil
return nil
}
tx, err := DB.Begin(ctx, true)
if err != nil {
return 0, fmt.Errorf("begin transaction: %w", err)
return fmt.Errorf("begin transaction: %w", err)
}
defer func() {
if tx != nil {
@ -235,11 +269,8 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
}
}()
now := time.Now()
qm := Msg{0, now, senderAccount, mailFrom.Localpart, mailFrom.IPDomain, rcptTo.Localpart, rcptTo.IPDomain, formatIPDomain(rcptTo.IPDomain), 0, nil, now, nil, "", has8bit, smtputf8, size, messageID, msgPrefix, dsnutf8Opt, "", requireTLS}
if err := tx.Insert(&qm); err != nil {
return 0, err
if err := tx.Insert(qm); err != nil {
return err
}
dst := qm.MessagePath()
@ -252,19 +283,19 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
dstDir := filepath.Dir(dst)
os.MkdirAll(dstDir, 0770)
if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil {
return 0, fmt.Errorf("linking/copying message to new file: %s", err)
return fmt.Errorf("linking/copying message to new file: %s", err)
} else if err := moxio.SyncDir(dstDir); err != nil {
return 0, fmt.Errorf("sync directory: %v", err)
return fmt.Errorf("sync directory: %v", err)
}
if err := tx.Commit(); err != nil {
return 0, fmt.Errorf("commit transaction: %s", err)
return fmt.Errorf("commit transaction: %s", err)
}
tx = nil
dst = ""
queuekick()
return qm.ID, nil
return nil
}
func formatIPDomain(d dns.IPDomain) string {

77
queue/queue_test.go
View file

@ -110,10 +110,14 @@ func TestQueue(t *testing.T) {
defer os.Remove(mf.Name())
defer mf.Close()
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
var qm Msg
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg)
@ -440,7 +444,8 @@ func TestQueue(t *testing.T) {
// Add a message to be delivered with submit because of its route.
topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}}
_, err = Add(ctxbg, xlog, "mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
wasNetDialer = testDeliver(fakeSubmitServer)
if !wasNetDialer {
@ -448,10 +453,11 @@ func TestQueue(t *testing.T) {
}
// Add a message to be delivered with submit because of explicitly configured transport, that uses TLS.
msgID, err := Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
transportSubmitTLS := "submittls"
n, err = Kick(ctxbg, msgID, "", "", &transportSubmitTLS)
n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -472,10 +478,11 @@ func TestQueue(t *testing.T) {
}
// Add a message to be delivered with socks.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
transportSocks := "socks"
n, err = Kick(ctxbg, msgID, "", "", &transportSocks)
n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -486,9 +493,10 @@ func TestQueue(t *testing.T) {
}
// Add message to be delivered with opportunistic TLS verification.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -496,9 +504,10 @@ func TestQueue(t *testing.T) {
testDeliver(fakeSMTPSTARTTLSServer)
// Test fallback to plain text with TLS handshake fails.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -512,9 +521,10 @@ func TestQueue(t *testing.T) {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: moxCert.Leaf.RawSubjectPublicKeyInfo},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -530,9 +540,10 @@ func TestQueue(t *testing.T) {
// Add message to be delivered with verified TLS and REQUIRETLS.
yes := true
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, mf, nil, &yes)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -545,9 +556,10 @@ func TestQueue(t *testing.T) {
{},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -562,9 +574,10 @@ func TestQueue(t *testing.T) {
{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: make([]byte, sha256.Size)},
},
}
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -581,9 +594,10 @@ func TestQueue(t *testing.T) {
// Check that message is delivered with TLS-Required: No and non-matching DANE record.
no := false
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, mf, nil, &no)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -591,9 +605,10 @@ func TestQueue(t *testing.T) {
testDeliver(fakeSMTPSTARTTLSServer)
// Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, mf, nil, &no)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -601,9 +616,10 @@ func TestQueue(t *testing.T) {
testDeliver(makeBadFakeSMTPSTARTTLSServer(true))
// Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, mf, nil, &yes)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -615,9 +631,10 @@ func TestQueue(t *testing.T) {
resolver.TLSA = nil
// Add message with requiretls that fails immediately due to no verification policy for recipient domain.
msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, mf, nil, &yes)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
n, err = Kick(ctxbg, msgID, "", "", nil)
n, err = Kick(ctxbg, qm.ID, "", "", nil)
tcheck(t, err, "kick queue")
if n != 1 {
t.Fatalf("kick changed %d messages, expected 1", n)
@ -629,7 +646,8 @@ func TestQueue(t *testing.T) {
})
// Add another message that we'll fail to deliver entirely.
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
msgs, err = List(ctxbg)
@ -788,7 +806,8 @@ func TestQueueStart(t *testing.T) {
mf := prepareFile(t)
defer os.Remove(mf.Name())
defer mf.Close()
_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
err = Add(ctxbg, xlog, &qm, mf)
tcheck(t, err, "add message to queue for delivery")
checkDialed(true)

14
serve.go
View file

@ -52,7 +52,7 @@ func shutdown(log *mlog.Log) {
// start initializes all packages, starts all listeners and the switchboard
// goroutine, then returns.
func start(mtastsdbRefresher, skipForkExec bool) error {
func start(mtastsdbRefresher, sendDMARCReports, skipForkExec bool) error {
smtpserver.Listen()
imapserver.Listen()
http.Listen()
@ -69,10 +69,6 @@ func start(mtastsdbRefresher, skipForkExec bool) error {
}
}
if err := dmarcdb.Init(); err != nil {
return fmt.Errorf("dmarc init: %s", err)
}
if err := mtastsdb.Init(mtastsdbRefresher); err != nil {
return fmt.Errorf("mtasts init: %s", err)
}
@ -86,6 +82,14 @@ func start(mtastsdbRefresher, skipForkExec bool) error {
return fmt.Errorf("queue start: %s", err)
}
// dmarcdb starts after queue because it may start sending reports through the queue.
if err := dmarcdb.Init(); err != nil {
return fmt.Errorf("dmarc init: %s", err)
}
if sendDMARCReports {
dmarcdb.Start(dns.StrictResolver{Pkg: "dmarcdb"})
}
store.StartAuthCache()
smtpserver.Serve()
imapserver.Serve()

2
serve_unix.go
View file

@ -224,7 +224,7 @@ Only implemented on unix systems, not Windows.
// taken.
const mtastsdbRefresher = true
const skipForkExec = false
if err := start(mtastsdbRefresher, skipForkExec); err != nil {
if err := start(mtastsdbRefresher, !mox.Conf.Static.NoOutgoingDMARCReports, skipForkExec); err != nil {
log.Fatalx("start", err)
}
log.Print("ready to serve")

4
smtp/address.go
View file

@ -107,6 +107,10 @@ func NewAddress(localpart Localpart, domain dns.Domain) Address {
return Address{localpart, domain}
}
func (a Address) Path() Path {
return Path{Localpart: a.Localpart, IPDomain: dns.IPDomain{Domain: a.Domain}}
}
func (a Address) IsZero() bool {
return a == Address{}
}

56
smtpserver/analyze.go
View file

@ -37,16 +37,17 @@ type delivery struct {
}
type analysis struct {
accept bool
mailbox string
code int
secode string
userError bool
errmsg string
err error // For our own logging, not sent to remote.
dmarcReport *dmarcrpt.Feedback // Validated dmarc aggregate report, not yet stored.
tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.
reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.
accept bool
mailbox string
code int
secode string
userError bool
errmsg string
err error // For our own logging, not sent to remote.
dmarcReport *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored.
tlsReport *tlsrpt.Report // Validated TLS report, not yet stored.
reason string // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.
dmarcOverrideReason string // If set, one of dmarcrpt.PolicyOverride
}
const (
@ -64,7 +65,7 @@ const (
reasonDNSBlocklisted = "dns-blocklisted"
reasonSubjectpass = "subjectpass"
reasonSubjectpassError = "subjectpass-error"
reasonIPrev = "iprev" // No or mil junk reputation signals, and bad iprev.
reasonIPrev = "iprev" // No or mild junk reputation signals, and bad iprev.
)
func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis {
@ -83,15 +84,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
ld := rs.ListAllowDNSDomain
// todo: on temporary failures, reject temporarily?
if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {
return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow}
return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList)}
}
for _, r := range d.dkimResults {
if r.Status == dkim.StatusPass && r.Sig.Domain == ld {
return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow}
return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList)}
}
}
}
var dmarcOverrideReason string
// For forwarded messages, we have different junk analysis. We don't reject for
// failing DMARC, and we clear fields that could implicate the forwarding mail
// server during future classifications on incoming messages (the forwarding mail
@ -113,6 +116,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
}
}
d.m.DKIMDomains = dkimdoms
dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded)
log.Info("forwarded message, clearing identifying signals of forwarding mail server")
}
@ -154,7 +158,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
})
})
if mberr != nil {
return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError}
return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, dmarcOverrideReason}
}
d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID.
}
@ -168,7 +172,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
d.m.Seen = true
log.Info("accepting reject to configured mailbox due to ruleset")
}
return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason}
return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason, dmarcOverrideReason}
}
if d.dmarcUse && d.dmarcResult.Reject {
@ -180,17 +184,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
// track of the report. We'll check reputation, defaulting to accept.
var dmarcReport *dmarcrpt.Feedback
if d.rcptAcc.destination.DMARCReports {
// Messages with DMARC aggregate reports must have a dmarc pass. ../rfc/7489:1866
// Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866
if d.dmarcResult.Status != dmarc.StatusPass {
log.Info("received dmarc report without dmarc pass, not processing as dmarc report")
log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")
} else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
log.Infox("parsing dmarc report", err)
log.Infox("parsing dmarc aggregate report", err)
} else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {
log.Infox("parsing domain in dmarc report", err)
log.Infox("parsing domain in dmarc aggregate report", err)
} else if _, ok := mox.Conf.Domain(d); !ok {
log.Info("dmarc report for domain not configured, ignoring", mlog.Field("domain", d))
log.Info("dmarc aggregate report for domain not configured, ignoring", mlog.Field("domain", d))
} else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {
log.Info("dmarc report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
log.Info("dmarc aggregate report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
} else {
dmarcReport = report
}
@ -261,12 +265,12 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method)))
if conclusive {
if !*isjunk {
return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason}
return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason}
}
return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))
} else if dmarcReport != nil || tlsReport != nil {
log.Info("accepting dmarc reporting or tlsrpt message without reputation")
return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting}
log.Info("accepting message with dmarc aggregate report or tls report without reputation")
return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting, dmarcOverrideReason: dmarcOverrideReason}
}
// If there was no previous message from sender or its domain, and we have an SPF
// (soft)fail, reject the message.
@ -302,7 +306,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
pass := err == nil
log.Infox("pass by subject token", err, mlog.Field("pass", pass))
if pass {
return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass}
return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason}
}
}
@ -382,7 +386,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
}
if accept {
return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals}
return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals, dmarcOverrideReason: dmarcOverrideReason}
}
if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {

4
smtpserver/dsn.go
View file

@ -50,7 +50,9 @@ func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message, req
if requireTLS {
reqTLS = &requireTLS
}
if _, err := queue.Add(ctx, c.log, "", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, f, bufUTF8, reqTLS); err != nil {
qm := queue.MakeMsg("", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, reqTLS)
qm.DSNUTF8 = bufUTF8
if err := queue.Add(ctx, c.log, &qm, f); err != nil {
return err
}
return nil

216
smtpserver/server.go
View file

@ -37,6 +37,7 @@ import (
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dmarc"
"github.com/mjl-/mox/dmarcdb"
"github.com/mjl-/mox/dmarcrpt"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dsn"
"github.com/mjl-/mox/iprev"
@ -1835,7 +1836,8 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...)
msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
if _, err := queue.Add(ctx, c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, dataFile, nil, c.requireTLS); err != nil {
qm := queue.MakeMsg(c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, c.requireTLS)
if err := queue.Add(ctx, c.log, &qm, dataFile); err != nil {
// Aborting the transaction is not great. But continuing and generating DSNs will
// probably result in errors as well...
metricSubmission.WithLabelValues("queueerror").Inc()
@ -2065,7 +2067,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
var comment string
var props []message.AuthProp
if r.Sig != nil {
// todo future: also specify whether dns record was dnssec-signed.
if r.Record != nil && r.Record.PublicKey != nil {
if pubkey, ok := r.Record.PublicKey.(*rsa.PublicKey); ok {
comment = fmt.Sprintf("%d bit rsa, ", pubkey.N.BitLen())
@ -2167,6 +2168,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
var dmarcUse bool
var dmarcResult dmarc.Result
const applyRandomPercentage = true
// dmarcMethod is added to authResults when delivering to recipients: accounts can
// have different policy override rules.
var dmarcMethod message.AuthMethod
var msgFromValidation = store.ValidationNone
if msgFrom.IsZero() {
@ -2178,6 +2181,15 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
} else {
msgFromValidation = alignment(ctx, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity)
// We are doing the DMARC evaluation now. But we only store it for inclusion in an
// aggregate report when we actually use it. We use an evaluation for each
// recipient, with each a potentially different result due to mailing
// list/forwarding configuration. If we reject a message due to being spam, we
// don't want to spend any resources for the sender domain, and we don't want to
// give the sender any more information about us, so we won't record the
// evaluation.
// todo future: also not send for first-time senders? they could be spammers getting through our filter, don't want to give them insights either. though we currently would have no reasonable way to decide if they are still reputationless at the time we are composing/sending aggregate reports.
dmarcctx, dmarccancel := context.WithTimeout(ctx, time.Minute)
defer dmarccancel()
dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage)
@ -2202,9 +2214,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
msgFromValidation = store.ValidationDMARC
}
// todo future: consider enforcing an spf fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507
// todo future: consider enforcing an spf (soft)fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507
}
authResults.Methods = append(authResults.Methods, dmarcMethod)
c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain))
// Prepare for analyzing content, calculating reputation.
@ -2366,16 +2377,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
continue
}
// ../rfc/5321:3204
// Received-SPF header goes before Received. ../rfc/7208:2038
msgPrefix := []byte(
"Delivered-To: " + rcptAcc.rcptTo.XString(c.smtputf8) + "\r\n" + // ../rfc/9228:274
"Return-Path: <" + c.mailFrom.String() + ">\r\n" + // ../rfc/5321:3300
authResults.Header() +
receivedSPF.Header() +
recvHdrFor(rcptAcc.rcptTo.String()),
)
m := &store.Message{
Received: time.Now(),
RemoteIP: c.remoteIP.String(),
@ -2398,16 +2399,187 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
MailFromValidation: mailFromValidation,
MsgFromValidation: msgFromValidation,
DKIMDomains: verifiedDKIMDomains,
Size: int64(len(msgPrefix)) + msgWriter.Size,
MsgPrefix: msgPrefix,
Size: msgWriter.Size,
}
d := delivery{m, dataFile, rcptAcc, acc, msgFrom, c.dnsBLs, dmarcUse, dmarcResult, dkimResults, iprevStatus}
a := analyze(ctx, log, c.resolver, d)
if a.reason != "" {
xmoxreason := "X-Mox-Reason: " + a.reason + "\r\n"
m.MsgPrefix = append([]byte(xmoxreason), m.MsgPrefix...)
m.Size += int64(len(xmoxreason))
// Any DMARC result override is stored in the evaluation for outgoing DMARC
// aggregate reports, and added to the Authentication-Results message header.
var dmarcOverride string
if dmarcResult.Record != nil {
if !dmarcUse {
dmarcOverride = string(dmarcrpt.PolicyOverrideSampledOut)
} else if a.dmarcOverrideReason != "" && (a.accept && !m.IsReject) == dmarcResult.Reject {
dmarcOverride = a.dmarcOverrideReason
}
}
// Add per-recipient DMARC method to Authentication-Results. Each account can have
// their own override rules, e.g. based on configured mailing lists/forwards.
// ../rfc/7489:1486
rcptDMARCMethod := dmarcMethod
if dmarcOverride != "" {
if rcptDMARCMethod.Comment != "" {
rcptDMARCMethod.Comment += ", "
}
rcptDMARCMethod.Comment += "override " + dmarcOverride
}
rcptAuthResults := authResults
rcptAuthResults.Methods = append([]message.AuthMethod{}, authResults.Methods...)
rcptAuthResults.Methods = append(rcptAuthResults.Methods, rcptDMARCMethod)
// Prepend reason as message header, for easy display in mail clients.
var xmoxreason string
if a.reason != "" {
xmoxreason = "X-Mox-Reason: " + a.reason + "\r\n"
}
// ../rfc/5321:3204
// Received-SPF header goes before Received. ../rfc/7208:2038
m.MsgPrefix = []byte(
xmoxreason +
"Delivered-To: " + rcptAcc.rcptTo.XString(c.smtputf8) + "\r\n" + // ../rfc/9228:274
"Return-Path: <" + c.mailFrom.String() + ">\r\n" + // ../rfc/5321:3300
rcptAuthResults.Header() +
receivedSPF.Header() +
recvHdrFor(rcptAcc.rcptTo.String()),
)
m.Size += int64(len(m.MsgPrefix))
// Store DMARC evaluation for inclusion in an aggregate report. Only if there is at
// least one reporting address: We don't want to needlessly store a row in a
// database for each delivery attempt. If we reject a message for being junk, we
// are also not going to send it a DMARC report. The DMARC check is done early in
// the analysis, we will report on rejects because of DMARC, because it could be
// valuable feedback about forwarded or mailing list messages.
// ../rfc/7489:1492
if !mox.Conf.Static.NoOutgoingDMARCReports && dmarcResult.Record != nil && len(dmarcResult.Record.AggregateReportAddresses) > 0 && (a.accept && !m.IsReject || a.reason == reasonDMARCPolicy) {
// Disposition holds our decision on whether to accept the message. Not what the
// DMARC evaluation resulted in. We can override, e.g. because of mailing lists,
// forwarding, or local policy.
// We treat quarantine as reject, so never claim to quarantine.
// ../rfc/7489:1691
disposition := dmarcrpt.DispositionNone
if !a.accept {
disposition = dmarcrpt.DispositionReject
}
// unknownDomain returns whether the sender is domain with which this account has
// not had positive interaction.
unknownDomain := func() (unknown bool) {
err := acc.DB.Read(ctx, func(tx *bstore.Tx) (err error) {
// See if we received a non-junk message from this organizational domain.
q := bstore.QueryTx[store.Message](tx)
q.FilterNonzero(store.Message{MsgFromOrgDomain: m.MsgFromOrgDomain})
q.FilterEqual("Notjunk", false)
exists, err := q.Exists()
if err != nil {
return fmt.Errorf("querying for non-junk message from organizational domain: %v", err)
}
if exists {
return nil
}
// See if we sent a message to this organizational domain.
qr := bstore.QueryTx[store.Recipient](tx)
qr.FilterNonzero(store.Recipient{OrgDomain: m.MsgFromOrgDomain})
exists, err = qr.Exists()
if err != nil {
return fmt.Errorf("querying for message sent to organizational domain: %v", err)
}
if !exists {
unknown = true
}
return nil
})
if err != nil {
log.Errorx("checking if sender is unknown domain, for dmarc aggregate report evaluation", err)
}
return
}
r := dmarcResult.Record
addresses := make([]string, len(r.AggregateReportAddresses))
for i, a := range r.AggregateReportAddresses {
addresses[i] = a.String()
}
sp := dmarcrpt.Disposition(r.SubdomainPolicy)
if r.SubdomainPolicy == dmarc.PolicyEmpty {
sp = dmarcrpt.Disposition(r.Policy)
}
eval := dmarcdb.Evaluation{
// Evaluated and IntervalHours set by AddEvaluation.
PolicyDomain: dmarcResult.Domain.Name(),
// Optional evaluations don't cause a report to be sent, but will be included.
// Useful for automated inter-mailer messages, we don't want to get in a reporting
// loop. We also don't want to be used for sending reports to unsuspecting domains
// we have no relation with.
// todo: would it make sense to also mark some percentage of mailing-list-policy-overrides optional? to lower the load on mail servers of folks sending to large mailing lists.
Optional: rcptAcc.destination.DMARCReports || rcptAcc.destination.TLSReports || a.reason == reasonDMARCPolicy && unknownDomain(),
Addresses: addresses,
PolicyPublished: dmarcrpt.PolicyPublished{
Domain: dmarcResult.Domain.Name(),
ADKIM: dmarcrpt.Alignment(r.ADKIM),
ASPF: dmarcrpt.Alignment(r.ASPF),
Policy: dmarcrpt.Disposition(r.Policy),
SubdomainPolicy: sp,
Percentage: r.Percentage,
// We don't save ReportingOptions, we don't do per-message failure reporting.
},
SourceIP: c.remoteIP.String(),
Disposition: disposition,
AlignedDKIMPass: dmarcResult.AlignedDKIMPass,
AlignedSPFPass: dmarcResult.AlignedSPFPass,
EnvelopeTo: rcptAcc.rcptTo.IPDomain.String(),
EnvelopeFrom: c.mailFrom.IPDomain.String(),
HeaderFrom: msgFrom.Domain.Name(),
}
if dmarcOverride != "" {
eval.OverrideReasons = []dmarcrpt.PolicyOverrideReason{
{Type: dmarcrpt.PolicyOverride(dmarcOverride)},
}
}
// We'll include all signatures for the organizational domain, even if they weren't
// relevant due to strict alignment requirement.
for _, dkimResult := range dkimResults {
if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, msgFrom.Domain) != publicsuffix.Lookup(ctx, dkimResult.Sig.Domain) {
continue
}
r := dmarcrpt.DKIMAuthResult{
Domain: dkimResult.Sig.Domain.Name(),
Selector: dkimResult.Sig.Selector.ASCII,
Result: dmarcrpt.DKIMResult(dkimResult.Status),
}
eval.DKIMResults = append(eval.DKIMResults, r)
}
switch receivedSPF.Identity {
case spf.ReceivedHELO:
spfAuthResult := dmarcrpt.SPFAuthResult{
Domain: spfArgs.HelloDomain.String(), // Can be unicode and also IP.
Scope: dmarcrpt.SPFDomainScopeHelo,
Result: dmarcrpt.SPFResult(receivedSPF.Result),
}
eval.SPFResults = []dmarcrpt.SPFAuthResult{spfAuthResult}
case spf.ReceivedMailFrom:
spfAuthResult := dmarcrpt.SPFAuthResult{
Domain: spfArgs.MailFromDomain.Name(), // Can be unicode.
Scope: dmarcrpt.SPFDomainScopeMailFrom,
Result: dmarcrpt.SPFResult(receivedSPF.Result),
}
eval.SPFResults = []dmarcrpt.SPFAuthResult{spfAuthResult}
}
err := dmarcdb.AddEvaluation(ctx, dmarcResult.Record.AggregateReportingInterval, &eval)
log.Check(err, "adding dmarc evaluation to database for aggregate report")
}
if !a.accept {
conf, _ := acc.Conf()
if conf.RejectsMailbox != "" {
@ -2455,9 +2627,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
if a.dmarcReport != nil {
// todo future: add rate limiting to prevent DoS attacks. ../rfc/7489:2570
if err := dmarcdb.AddReport(ctx, a.dmarcReport, msgFrom.Domain); err != nil {
log.Errorx("saving dmarc report in database", err)
log.Errorx("saving dmarc aggregate report in database", err)
} else {
log.Info("dmarc report processed")
log.Info("dmarc aggregate report processed")
m.Flags.Seen = true
delayFirstTime = false
}

100
smtpserver/server_test.go
View file

@ -100,6 +100,11 @@ func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *test
ts := testserver{t: t, cid: 1, resolver: resolver, tlsmode: smtpclient.TLSOpportunistic}
if dmarcdb.EvalDB != nil {
dmarcdb.EvalDB.Close()
dmarcdb.EvalDB = nil
}
mox.Context = ctxbg
mox.ConfigStaticPath = configPath
mox.MustLoadConfig(true, false)
@ -192,6 +197,15 @@ func fakeCert(t *testing.T) tls.Certificate {
return c
}
// check expected dmarc evaluations for outgoing aggregate reports.
func checkEvaluationCount(t *testing.T, n int) []dmarcdb.Evaluation {
t.Helper()
l, err := dmarcdb.Evaluations(ctxbg)
tcheck(t, err, "get dmarc evaluations")
tcompare(t, len(l), n)
return l
}
// Test submission from authenticated user.
func TestSubmission(t *testing.T) {
ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/mox.conf"), dns.MockResolver{})
@ -229,6 +243,7 @@ func TestSubmission(t *testing.T) {
if expErr == nil && err != nil || expErr != nil && (err == nil || !errors.As(err, &cerr) || cerr.Secode != expErr.Secode) {
t.Fatalf("got err %#v (%q), expected %#v", err, err, expErr)
}
checkEvaluationCount(t, 0)
})
}
@ -329,6 +344,8 @@ func TestDelivery(t *testing.T) {
t.Fatalf("no delivery in 1s")
}
})
checkEvaluationCount(t, 0)
}
func tinsertmsg(t *testing.T, acc *store.Account, mailbox string, m *store.Message, msg string) {
@ -392,7 +409,7 @@ func TestSpam(t *testing.T) {
},
TXT: map[string][]string{
"example.org.": {"v=spf1 ip4:127.0.0.10 -all"},
"_dmarc.example.org.": {"v=DMARC1;p=reject"},
"_dmarc.example.org.": {"v=DMARC1;p=reject; rua=mailto:dmarcrpt@example.org"},
},
}
ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver)
@ -451,6 +468,7 @@ func TestSpam(t *testing.T) {
}
checkCount("Rejects", 1)
checkEvaluationCount(t, 0) // No positive interactions yet.
})
// Delivery from sender with bad reputation matching AcceptRejectsToMailbox should
@ -463,8 +481,9 @@ func TestSpam(t *testing.T) {
}
tcheck(t, err, "deliver")
checkCount("mjl2junk", 1) // In ruleset rejects mailbox.
checkCount("Rejects", 1) // Same as before.
checkCount("mjl2junk", 1) // In ruleset rejects mailbox.
checkCount("Rejects", 1) // Same as before.
checkEvaluationCount(t, 0) // This is not an actual accept.
})
// Mark the messages as having good reputation.
@ -485,6 +504,7 @@ func TestSpam(t *testing.T) {
// Message should now be removed from Rejects mailboxes.
checkCount("Rejects", 0)
checkCount("mjl2junk", 1)
checkEvaluationCount(t, 1)
})
// Undo dmarc pass, mark messages as junk, and train the filter.
@ -506,6 +526,7 @@ func TestSpam(t *testing.T) {
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
}
checkEvaluationCount(t, 1) // No new evaluation, this isn't a DMARC reject.
})
}
@ -525,9 +546,9 @@ func TestForward(t *testing.T) {
"bad.example.": {"v=spf1 ip4:127.0.0.1 -all"},
"good.example.": {"v=spf1 ip4:127.0.0.1 -all"},
"forward.example.": {"v=spf1 ip4:127.0.0.10 -all"},
"_dmarc.bad.example.": {"v=DMARC1;p=reject"},
"_dmarc.good.example.": {"v=DMARC1;p=reject"},
"_dmarc.forward.example.": {"v=DMARC1;p=reject"},
"_dmarc.bad.example.": {"v=DMARC1;p=reject; rua=mailto:dmarc@bad.example"},
"_dmarc.good.example.": {"v=DMARC1;p=reject; rua=mailto:dmarc@good.example"},
"_dmarc.forward.example.": {"v=DMARC1;p=reject; rua=mailto:dmarc@forward.example"},
},
PTR: map[string][]string{
"127.0.0.10": {"forward.example."}, // For iprev check.
@ -544,6 +565,8 @@ func TestForward(t *testing.T) {
ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver)
defer ts.close()
totalEvaluations := 0
var msgBad = strings.ReplaceAll(`From: <remote@bad.example>
To: <mjl3@mox.example>
Subject: test
@ -580,6 +603,7 @@ happens to come from forwarding mail server.
err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgBad)), strings.NewReader(msgBad), false, false, false)
tcheck(t, err, "deliver message")
}
totalEvaluations += 10
n, err := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB).UpdateFields(map[string]any{"Junk": true, "MsgFromValidated": true})
tcheck(t, err, "marking messages as junk")
@ -591,6 +615,8 @@ happens to come from forwarding mail server.
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
t.Fatalf("delivery by bad sender, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
}
checkEvaluationCount(t, totalEvaluations)
})
// Delivery from different "message From" without reputation, but from same
@ -607,12 +633,14 @@ happens to come from forwarding mail server.
err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgOK)), strings.NewReader(msgOK), false, false, false)
if forward {
tcheck(t, err, "deliver")
totalEvaluations += 1
} else {
var cerr smtpclient.Error
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
t.Fatalf("delivery by bad ip, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
}
}
checkEvaluationCount(t, totalEvaluations)
})
// Delivery from forwarding server that isn't a forward should get same treatment.
@ -624,12 +652,14 @@ happens to come from forwarding mail server.
err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgOK2)), strings.NewReader(msgOK2), false, false, false)
if forward {
tcheck(t, err, "deliver")
totalEvaluations += 1
} else {
var cerr smtpclient.Error
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
t.Fatalf("delivery by bad ip, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
}
}
checkEvaluationCount(t, totalEvaluations)
})
}
@ -644,13 +674,31 @@ func TestDMARCSent(t *testing.T) {
"example.org.": {"127.0.0.1"}, // For mx check.
},
TXT: map[string][]string{
"example.org.": {"v=spf1 ip4:127.0.0.10 -all"},
"_dmarc.example.org.": {"v=DMARC1;p=reject"},
"example.org.": {"v=spf1 ip4:127.0.0.1 -all"},
"_dmarc.example.org.": {"v=DMARC1;p=reject;rua=mailto:dmarcrpt@example.org"},
},
}
ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver)
defer ts.close()
// First check that DMARC policy rejects message and results in optional evaluation.
ts.run(func(err error, client *smtpclient.Client) {
mailFrom := "remote@example.org"
rcptTo := "mjl@mox.example"
if err == nil {
err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false)
}
var cerr smtpclient.Error
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C550MailboxUnavail)
}
l := checkEvaluationCount(t, 1)
tcompare(t, l[0].Optional, true)
})
// Update DNS for an SPF pass, and DMARC pass.
resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.10 -all"}
// Insert spammy messages not related to the test message.
m := store.Message{
MailFrom: "remote@test.example",
@ -676,6 +724,7 @@ func TestDMARCSent(t *testing.T) {
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
}
checkEvaluationCount(t, 1) // No new evaluation.
})
// Insert a message that we sent to the address that is about to send to us.
@ -684,7 +733,26 @@ func TestDMARCSent(t *testing.T) {
err := ts.acc.DB.Insert(ctxbg, &store.Recipient{MessageID: sentMsg.ID, Localpart: "remote", Domain: "example.org", OrgDomain: "example.org", Sent: time.Now()})
tcheck(t, err, "inserting message recipient")
// Reject a message due to DMARC again. Since we sent a message to the domain, it
// is no longer unknown and we should see a non-optional evaluation that will
// result in a DMARC report.
resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.1 -all"}
ts.run(func(err error, client *smtpclient.Client) {
mailFrom := "remote@example.org"
rcptTo := "mjl@mox.example"
if err == nil {
err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false)
}
var cerr smtpclient.Error
if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C550MailboxUnavail)
}
l := checkEvaluationCount(t, 2) // New evaluation.
tcompare(t, l[1].Optional, false)
})
// We should now be accepting the message because we recently sent a message.
resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.10 -all"}
ts.run(func(err error, client *smtpclient.Client) {
mailFrom := "remote@example.org"
rcptTo := "mjl@mox.example"
@ -692,6 +760,8 @@ func TestDMARCSent(t *testing.T) {
err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false)
}
tcheck(t, err, "deliver")
l := checkEvaluationCount(t, 3) // New evaluation.
tcompare(t, l[2].Optional, false)
})
}
@ -773,7 +843,7 @@ func TestDMARCReport(t *testing.T) {
},
TXT: map[string][]string{
"example.org.": {"v=spf1 ip4:127.0.0.10 -all"},
"_dmarc.example.org.": {"v=DMARC1;p=reject"},
"_dmarc.example.org.": {"v=DMARC1;p=reject; rua=mailto:dmarcrpt@example.org"},
},
PTR: map[string][]string{
"127.0.0.10": {"example.org."}, // For iprev check.
@ -815,6 +885,11 @@ func TestDMARCReport(t *testing.T) {
run(dmarcReport, 0)
run(strings.ReplaceAll(dmarcReport, "xmox.nl", "mox.example"), 1)
// We always store as an evaluation, but as optional for reports.
evals := checkEvaluationCount(t, 2)
tcompare(t, evals[0].Optional, true)
tcompare(t, evals[1].Optional, true)
}
const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
@ -896,7 +971,7 @@ func TestTLSReport(t *testing.T) {
TXT: map[string][]string{
"testsel._domainkey.example.org.": {dkimTxt},
"example.org.": {"v=spf1 ip4:127.0.0.10 -all"},
"_dmarc.example.org.": {"v=DMARC1;p=reject"},
"_dmarc.example.org.": {"v=DMARC1;p=reject;rua=mailto:dmarcrpt@example.org"},
},
PTR: map[string][]string{
"127.0.0.10": {"example.org."}, // For iprev check.
@ -939,6 +1014,11 @@ func TestTLSReport(t *testing.T) {
run(tlsrpt, 0)
run(strings.ReplaceAll(tlsrpt, "xmox.nl", "mox.example"), 1)
// We always store as an evaluation, but as optional for reports.
evals := checkEvaluationCount(t, 2)
tcompare(t, evals[0].Optional, true)
tcompare(t, evals[1].Optional, true)
}
func TestRatelimitConnectionrate(t *testing.T) {

17
store/cleanuptemp.go Normal file
View file

@ -0,0 +1,17 @@
package store
import (
"os"
"github.com/mjl-/mox/mlog"
)
// CloseRemoveTempFile closes and removes f, a file described by descr. Often
// used in a defer after creating a temporary file.
func CloseRemoveTempFile(log *mlog.Log, f *os.File, descr string) {
name := f.Name()
err := f.Close()
log.Check(err, "closing temporary file", mlog.Field("kind", descr))
err = os.Remove(name)
log.Check(err, "removing temporary file", mlog.Field("kind", descr))
}

28
testdata/dmarcdb/domains.conf vendored Normal file
View file

@ -0,0 +1,28 @@
Domains:
mox.example:
DKIM:
Selectors:
testsel:
PrivateKeyFile: testsel.rsakey.pkcs8.pem
Sign:
- testsel
Accounts:
other:
Domain: mox.example
Destinations:
other@mox.example: nil
mjl:
MaxOutgoingMessagesPerDay: 30
MaxFirstTimeRecipientsPerDay: 10
Domain: mox.example
Destinations:
mjl@mox.example: nil
møx@mox.example: nil
RejectsMailbox: Rejects
JunkFilter:
Threshold: 0.95
Params:
Twograms: true
MaxPower: 0.1
TopWords: 10
IgnoreWords: 0.1

11
testdata/dmarcdb/mox.conf vendored Normal file
View file

@ -0,0 +1,11 @@
DataDir: data
User: 1000
LogLevel: trace
Hostname: mail.mox.example
Listeners:
local:
IPs:
- 0.0.0.0
Postmaster:
Account: mjl
Mailbox: postmaster

30
testdata/dmarcdb/testsel.rsakey.pkcs8.pem vendored Normal file
View file

@ -0,0 +1,30 @@
-----BEGIN PRIVATE KEY-----
Note: RSA private key for use with DKIM, generated by mox
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDdkh3fKzvRUWym
n9UwVrEw6s2Mc0+DTg04TWJKGKHXpvcTHuEcE6ALVS9MZKasyVsIHU7FNeS9/qNb
pLihhGdlhU3KAfrMpTBhiFpJoYiDXED98Of4iBxNHIuheLMxSBSClMbLGE2vAgha
/6LuONuzdMqk/c1TijBD+vGjCZI2qD58cgXWWKRK9e+WNhKNoVdedZ9iJtbtN0MI
UWk3iwHmjXf5qzS7i8vDoy86Ln0HW0vKl7UtwemLVv09/E23OdNN163eQvSlrEhx
a0odPQsM9SizxhiaI9rmcZtSqULt37hhPaNA+/AbELCzWijZPDqePVRqKGd5gYDK
8STLj0UHAgMBAAECggEBAKVkJJgplYUx2oCmXmSu0aVKIBTvHjNNV+DnIq9co7Ju
F5BWRILIw3ayJ5RGrYPc6e6ssdfT2uNX6GjIFGm8g9HsJ5zazXNk+zBSr9K2mUg0
3O6xnPaP41BMNo5ZoqjuvSCcHagMhDBWvBXxLJXWK2lRjNKMAXCSfmTANQ8WXeYd
XG2nYTPtBu6UgY8W6sKAx1xetxBrzk8q6JTxb5eVG22BSiUniWYif+XVmAj1u6TH
0m6X0Kb6zsMYYgKPC2hmDsxD3uZ7qBNxxJzzLjpK6eP9aeFKzNyfnaoO4s+9K6Di
31oxTBpqLI4dcrvg4xWl+YkEknXXaomMqM8hyDzfcAECgYEA9/zmjRpoTAoY3fu9
mn16wxReFXZZZhqV0+c+gyYtao2Kf2pUNAdhD62HQv7KtAPPHKvLfL8PH0u7bzK0
vVNzBUukwxGI7gsoTMdc3L5x4v9Yb6jUx7RrDZn93sDod/1f/sb56ARCFQoqbUck
dSjnVUyF/l5oeh6CgKhvtghJ/AcCgYEA5Lq4kL82qWjIuNUT/C3lzjPfQVU+WvQ9
wa+x4B4mxm5r4na3AU1T8H+peh4YstAJUgscGfYnLzxuMGuP1ReIuWYy29eDptKl
WTzVZDcZrAPciP1FOL6jm03PT2UAEuoPRr4OHLg8DxoOqG8pxqk1izDSHG2Tof6l
0ToafeIALwECgYEA8wvLTgnOpI/U1WNP7aUDd0Rz/WbzsW1m4Lsn+lOleWPllIE6
q4974mi5Q8ECG7IL/9aj5cw/XvXTauVwXIn4Ff2QKpr58AvBYJaX/cUtS0PlgfIf
MOczcK43MWUxscADoGmVLn9V4NcIw/dQ1P7U0zXfsXEHxoA2eTAb5HV1RWsCgYBd
TcXoVfgIV1Q6AcGrR1XNLd/OmOVc2PEwR2l6ERKkM3sS4HZ6s36gRpNt20Ub/D0x
GJMYDA+j9zTDz7zWokkFyCjLATkVHiyRIH2z6b4xK0oVH6vTIAFBYxZEPuEu1gfx
RaogEQ9+4ZRFJUOXZIMRCpNLQW/Nz0D4/oi7/SsyAQKBgHEA27Js8ivt+EFCBjwB
UbkW+LonDAXuUbw91lh5jICCigqUg73HNmV5xpoYI9JNPc6fy6wLyInVUC2w9tpO
eH2Rl8n79vQMLbzsFClGEC/Q1kAbK5bwUjlfvKBZjvE0RknWX9e1ZY04DSsunSrM
prS2eHVZ24hecd7j9XfAbHLC
-----END PRIVATE KEY-----

22
verifydata.go
View file

@ -97,9 +97,12 @@ possibly making them potentially no longer readable by the previous version.
// Check a database file by opening it with BoltDB and bstore and lightly checking
// its contents.
checkDB := func(path string, types []any) {
checkDB := func(required bool, path string, types []any) {
_, err := os.Stat(path)
checkf(err, path, "checking if file exists")
if !required && err != nil && errors.Is(err, fs.ErrNotExist) {
return
}
checkf(err, path, "checking if database file exists")
if err != nil {
return
}
@ -156,7 +159,7 @@ possibly making them potentially no longer readable by the previous version.
checkQueue := func() {
dbpath := filepath.Join(dataDir, "queue/index.db")
checkDB(dbpath, queue.DBTypes)
checkDB(true, dbpath, queue.DBTypes)
// Check that all messages present in the database also exist on disk.
seen := map[string]struct{}{}
@ -222,12 +225,12 @@ possibly making them potentially no longer readable by the previous version.
// Check an account, with its database file and messages.
checkAccount := func(name string) {
accdir := filepath.Join(dataDir, "accounts", name)
checkDB(filepath.Join(accdir, "index.db"), store.DBTypes)
checkDB(true, filepath.Join(accdir, "index.db"), store.DBTypes)
jfdbpath := filepath.Join(accdir, "junkfilter.db")
jfbloompath := filepath.Join(accdir, "junkfilter.bloom")
if exists(jfdbpath) || exists(jfbloompath) {
checkDB(jfdbpath, junk.DBTypes)
checkDB(true, jfdbpath, junk.DBTypes)
}
// todo: add some kind of check for the bloom filter?
@ -399,7 +402,7 @@ possibly making them potentially no longer readable by the previous version.
p = p[len(dataDir)+1:]
}
switch p {
case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion":
case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion":
return nil
case "acme", "queue", "accounts", "tmp", "moved":
return fs.SkipDir
@ -417,9 +420,10 @@ possibly making them potentially no longer readable by the previous version.
checkf(err, dataDir, "walking data directory")
}
checkDB(filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.DBTypes)
checkDB(filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)
checkDB(filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes)
checkDB(true, filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.ReportsDBTypes)
checkDB(false, filepath.Join(dataDir, "dmarceval.db"), dmarcdb.EvalDBTypes) // After v0.0.7.
checkDB(true, filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)
checkDB(true, filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes)
checkQueue()
checkAccounts()
checkOther()

36
webadmin/admin.go
View file

@ -1546,7 +1546,7 @@ func (Admin) TLSRPTSummaries(ctx context.Context, start, end time.Time, domain s
// end (most recent first), then by domain.
func (Admin) DMARCReports(ctx context.Context, start, end time.Time, domain string) (reports []dmarcdb.DomainFeedback) {
reports, err := dmarcdb.RecordsPeriodDomain(ctx, start, end, domain)
xcheckf(ctx, err, "fetching dmarc reports from database")
xcheckf(ctx, err, "fetching dmarc aggregate reports from database")
sort.Slice(reports, func(i, j int) bool {
iend := reports[i].ReportMetadata.DateRange.End
jend := reports[j].ReportMetadata.DateRange.End
@ -1565,9 +1565,9 @@ func (Admin) DMARCReportID(ctx context.Context, domain string, reportID int64) (
err = bstore.ErrAbsent
}
if err == bstore.ErrAbsent {
xcheckuserf(ctx, err, "fetching dmarc report from database")
xcheckuserf(ctx, err, "fetching dmarc aggregate report from database")
}
xcheckf(ctx, err, "fetching dmarc report from database")
xcheckf(ctx, err, "fetching dmarc aggregate report from database")
return report
}
@ -1589,7 +1589,7 @@ type DMARCSummary struct {
// The returned summaries are ordered by domain name.
func (Admin) DMARCSummaries(ctx context.Context, start, end time.Time, domain string) (domainSummaries []DMARCSummary) {
reports, err := dmarcdb.RecordsPeriodDomain(ctx, start, end, domain)
xcheckf(ctx, err, "fetching dmarc reports from database")
xcheckf(ctx, err, "fetching dmarc aggregate reports from database")
summaries := map[string]DMARCSummary{}
for _, r := range reports {
sum := summaries[r.Domain]
@ -1932,3 +1932,31 @@ func (Admin) WebserverConfigSave(ctx context.Context, oldConf, newConf Webserver
func (Admin) Transports(ctx context.Context) map[string]config.Transport {
return mox.Conf.Static.Transports
}
// DMARCEvaluationStats returns a map of all domains with evaluations to a count of
// the evaluations and whether those evaluations will cause a report to be sent.
func (Admin) DMARCEvaluationStats(ctx context.Context) map[string]dmarcdb.EvaluationStat {
stats, err := dmarcdb.EvaluationStats(ctx)
xcheckf(ctx, err, "get evaluation stats")
return stats
}
// DMARCEvaluationsDomain returns all evaluations for aggregate reports for the
// domain, sorted from oldest to most recent.
func (Admin) DMARCEvaluationsDomain(ctx context.Context, domain string) (dns.Domain, []dmarcdb.Evaluation) {
dom, err := dns.ParseDomain(domain)
xcheckf(ctx, err, "parsing domain")
evals, err := dmarcdb.EvaluationsDomain(ctx, dom)
xcheckf(ctx, err, "get evaluations for domain")
return dom, evals
}
// DMARCRemoveEvaluations removes evaluations for a domain.
func (Admin) DMARCRemoveEvaluations(ctx context.Context, domain string) {
dom, err := dns.ParseDomain(domain)
xcheckf(ctx, err, "parsing domain")
err = dmarcdb.RemoveEvaluationsDomain(ctx, dom)
xcheckf(ctx, err, "removing evaluations for domain")
}

210
webadmin/admin.html
View file

@ -260,11 +260,13 @@ const index = async () => {
),
),
dom.br(),
dom.h2('Reporting'),
dom.div(dom.a('DMARC', attr({href: '#dmarc'}))),
dom.h2('Reports'),
dom.div(dom.a('DMARC', attr({href: '#dmarc/reports'}))),
dom.div(dom.a('TLS', attr({href: '#tlsrpt'}))),
dom.br(),
dom.h2('Operations'),
dom.div(dom.a('MTA-STS policies', attr({href: '#mtasts'}))),
// todo: outgoing DMARC findings
dom.div(dom.a('DMARC evaluations', attr({href: '#dmarc/evaluations'}))),
// todo: outgoing TLSRPT findings
// todo: routing, globally, per domain and per account
dom.br(),
@ -418,6 +420,16 @@ const box = (color, ...l) => [
),
dom.br(),
]
const inlineBox = (color, ...l) =>
dom.span(
style({
display: 'inline-block',
padding: color ? '0.05em 0.2em' : '',
backgroundColor: color,
borderRadius: '3px',
}),
l,
)
const accounts = async () => {
const accounts = await api.Accounts()
@ -1004,7 +1016,25 @@ const domainDNSCheck = async (d) => {
)
}
const dmarc = async () => {
const dmarcIndex = async () => {
const page = document.getElementById('page')
dom._kids(page,
crumbs(
crumblink('Mox Admin', '#'),
'DMARC reports and evaluations',
),
dom.ul(
dom.li(
dom.a(attr({href: '#dmarc/reports'}), 'Reports'), ', incoming DMARC aggregate reports.',
),
dom.li(
dom.a(attr({href: '#dmarc/evaluations'}), 'Evaluations'), ', for outgoing DMARC aggregate reports.',
),
),
)
}
const dmarcReports = async () => {
const end = new Date().toISOString()
const start = new Date(new Date().getTime() - 30*24*3600*1000).toISOString()
const summaries = await api.DMARCSummaries(start, end, "")
@ -1013,7 +1043,8 @@ const dmarc = async () => {
dom._kids(page,
crumbs(
crumblink('Mox Admin', '#'),
'DMARC aggregate reporting summary',
crumblink('DMARC', '#dmarc'),
'Aggregate reporting summary',
),
dom.p('DMARC reports are periodically sent by other mail servers that received an email message with a "From" header with our domain. Domains can have a DMARC DNS record that asks other mail servers to send these aggregate reports for analysis.'),
renderDMARCSummaries(summaries),
@ -1051,6 +1082,167 @@ const renderDMARCSummaries = (summaries) => {
]
}
const dmarcEvaluations = async () => {
const evalStats = await api.DMARCEvaluationStats()
const isEmpty = (o) => {
for (const e in o) {
return false
}
return true
}
const page = document.getElementById('page')
dom._kids(page,
crumbs(
crumblink('Mox Admin', '#'),
crumblink('DMARC', '#dmarc'),
'Evaluations',
),
dom.p('Incoming messages are checked against the DMARC policy of the domain in the message From header. If the policy requests reporting on the resulting evaluations, they are stored in the database. Each interval of 1 to 24 hours, the evaluations may be sent to a reporting address specified in the domain\'s DMARC policy. Not all evaluations are a reason to send a report, but if a report is sent all evaluations are included.'),
dom.table(
dom.thead(
dom.tr(
dom.th('Domain', attr({title: 'Domain in the message From header. Keep in mind these can be forged, so this does not necessarily mean someone from this domain authentically tried delivering email.'})),
dom.th('Evaluations', attr({title: 'Total number of message delivery attempts, including retries.'})),
dom.th('Send report', attr({title: 'Whether the current evaluations will cause a report to be sent.'})),
),
),
dom.tbody(
Object.entries(evalStats).sort((a, b) => a[0] < b[0] ? -1 : 1).map(t =>
dom.tr(
dom.td(dom.a(attr({href: '#dmarc/evaluations/'+domainName(t[1].Domain)}), domainString(t[1].Domain))),
dom.td(style({textAlign: 'right'}), ''+t[1].Count),
dom.td(style({textAlign: 'right'}), t[1].SendReport ? '✓' : ''),
),
),
isEmpty(evalStats) ? dom.tr(dom.td(attr({colspan: '3'}), 'No evaluations.')) : [],
),
),
)
}
const dmarcEvaluationsDomain = async (domain) => {
const [d, evaluations] = await api.DMARCEvaluationsDomain(domain)
let lastInterval = ''
let lastAddresses = ''
const formatPolicy = (e) => {
const p = e.PolicyPublished
let s = ''
const add = (k, v) => {
if (v) {
s += k+'='+v+'; '
}
}
add('p', p.Policy)
add('sp', p.SubdomainPolicy)
add('adkim', p.ADKIM)
add('aspf', p.ASPF)
add('pct', ''+p.Percentage)
add('fo', ''+p.ReportingOptions)
return s
}
let lastPolicy = ''
const authStatus = (v) => inlineBox(v ? '' : yellow, v ? 'pass' : 'fail')
const formatDKIMResults = (results) => results.map(r => dom.div('selector '+r.Selector+(r.Domain !== domain ? ', domain '+r.Domain : '') + ': ', inlineBox(r.Result === "pass" ? '' : yellow, r.Result)))
const formatSPFResults = (results) => results.map(r => dom.div(''+r.Scope+(r.Domain !== domain ? ', domain '+r.Domain : '') + ': ', inlineBox(r.Result === "pass" ? '' : yellow, r.Result)))
const sourceIP = (ip) => {
const r = dom.span(ip, attr({title: 'Click to do a reverse lookup of the IP.'}), style({cursor: 'pointer'}), async function click(e) {
e.preventDefault()
try {
const rev = await api.LookupIP(ip)
r.innerText = ip + '\n' + rev.Hostnames.join('\n')
} catch (err) {
r.innerText = ip + '\nerror: ' +err.message
}
})
return r
}
const page = document.getElementById('page')
dom._kids(page,
crumbs(
crumblink('Mox Admin', '#'),
crumblink('DMARC', '#dmarc'),
crumblink('Evaluations', '#dmarc/evaluations'),
'Domain '+domainString(d),
),
dom.div(
dom.button('Remove evaluations', async function click(e) {
e.target.disabled = true
try {
await api.DMARCRemoveEvaluations(domain)
window.location.reload() // todo: only clear the table?
} catch (err) {
console.log({err})
window.alert('Error: ' + err.message)
} finally {
e.target.disabled = false
}
}),
),
dom.br(),
dom.p('The evaluations below will be sent in a DMARC aggregate report to the addresses found in the published DMARC DNS record, which is fetched again before sending the report. The fields Interval hours, Addresses and Policy are only filled for the first row and whenever a new value in the published DMARC record is encountered.'),
dom.table(
dom.thead(
dom.tr(
dom.th('ID'),
dom.th('Evaluated'),
dom.th('Optional', attr({title: 'Some evaluations will not cause a DMARC aggregate report to be sent. But if a report is sent, optional records are included.'})),
dom.th('Interval hours', attr({title: 'DMARC policies published by a domain can specify how often they would like to receive reports. The default is 24 hours, but can be as often as each hour. To keep reports comparable between different mail servers that send reports, reports are sent at rounded up intervals of whole hours that can divide a 24 hour day, and are aligned with the start of a day at UTC.'})),
dom.th('Addresses', attr({title: 'Addresses that will receive the report. An address can have a maximum report size configured. If there is no address, no report will be sent.'})),
dom.th('Policy', attr({title: 'Summary of the policy as encountered in the DMARC DNS record of the domain, and used for evaluation.'})),
dom.th('IP', attr({title: 'IP address of delivery attempt that was evaluated, relevant for SPF.'})),
dom.th('Disposition', attr({title: 'Our decision to accept/reject this message. It may be different than requested by the published policy. For example, when overriding due to delivery from a mailing list or forwarded address.'})),
dom.th('DKIM/SPF', attr({title: 'Whether DKIM and SPF had an aligned pass, where strict/relaxed alignment means whether the domain of an SPF pass and DKIM pass matches the exact domain (strict) or optionally a subdomain (relaxed). A DMARC pass requires at least one pass.'})),
dom.th('Envelope to', attr({title: 'Domain used in SMTP RCPT TO during delivery.'})),
dom.th('Envelope from', attr({title: 'Domain used in SMTP MAIL FROM during delivery.'})),
dom.th('Message from', attr({title: 'Domain in "From" message header.'})),
dom.th('DKIM details', attr({title: 'Results of verifying DKIM-Signature headers in message. Only signatures with matching organizational domain are included, regardless of strict/relaxed DKIM alignment in DMARC policy.'})),
dom.th('SPF details', attr({title: 'Results of SPF check used in DMARC evaluation. "mfrom" indicates the "SMTP MAIL FROM" domain was used, "helo" indicates the SMTP EHLO domain was used.'})),
),
),
dom.tbody(
evaluations.map(e => {
const ival = e.IntervalHours + 'h'
const interval = ival === lastInterval ? '' : ival
lastInterval = ival
const a = (e.Addresses || []).join('\n')
const addresses = a === lastAddresses ? '' : a
lastAddresses = a
const p = formatPolicy(e)
const policy = p === lastPolicy ? '' : p
lastPolicy = p
return dom.tr(
dom.td(''+e.ID),
dom.td(new Date(e.Evaluated).toUTCString()),
dom.td(e.Optional ? 'Yes' : ''),
dom.td(interval),
dom.td(addresses),
dom.td(policy),
dom.td(sourceIP(e.SourceIP)),
dom.td(inlineBox(e.Disposition === 'none' ? '' : 'red', e.Disposition), (e.OverrideReasons || []).length > 0 ? ' ('+e.OverrideReasons.map(r => r.Type).join(', ')+')' : ''),
dom.td(authStatus(e.AlignedDKIMPass), '/', authStatus(e.AlignedSPFPass)),
dom.td(e.EnvelopeTo),
dom.td(e.EnvelopeFrom),
dom.td(e.HeaderFrom),
dom.td(formatDKIMResults(e.DKIMResults || [])),
dom.td(formatSPFResults(e.SPFResults || [])),
)
}),
evaluations.length === 0 ? dom.tr(dom.td(attr({colspan: '14'}), 'No evaluations.')) : [],
),
),
)
}
const utcDate = (dt) => new Date(Date.UTC(dt.getUTCFullYear(), dt.getUTCMonth(), dt.getUTCDate(), dt.getUTCHours(), dt.getUTCMinutes(), dt.getUTCSeconds()))
const utcDateStr = (dt) => [dt.getUTCFullYear(), 1+dt.getUTCMonth(), dt.getUTCDate()].join('-')
const isDayChange = (dt) => utcDateStr(new Date(dt.getTime() - 2*60*1000)) !== utcDateStr(new Date(dt.getTime() + 2*60*1000))
@ -2241,7 +2433,13 @@ const init = async () => {
} else if (h === 'tlsrpt') {
await tlsrpt()
} else if (h === 'dmarc') {
await dmarc()
await dmarcIndex()
} else if (h === 'dmarc/reports') {
await dmarcReports()
} else if (h === 'dmarc/evaluations') {
await dmarcEvaluations()
} else if (t[0] == 'dmarc' && t[1] == 'evaluations' && t.length === 3) {
await dmarcEvaluationsDomain(t[2])
} else if (h === 'mtasts') {
await mtasts()
} else if (h === 'dnsbl') {

235
webadmin/adminapi.json
View file

@ -753,6 +753,60 @@
]
}
]
},
{
"Name": "DMARCEvaluationStats",
"Docs": "DMARCEvaluationStats returns a map of all domains with evaluations to a count of\nthe evaluations and whether those evaluations will cause a report to be sent.",
"Params": [],
"Returns": [
{
"Name": "r0",
"Typewords": [
"{}",
"EvaluationStat"
]
}
]
},
{
"Name": "DMARCEvaluationsDomain",
"Docs": "DMARCEvaluationsDomain returns all evaluations for aggregate reports for the\ndomain, sorted from oldest to most recent.",
"Params": [
{
"Name": "domain",
"Typewords": [
"string"
]
}
],
"Returns": [
{
"Name": "r0",
"Typewords": [
"Domain"
]
},
{
"Name": "r1",
"Typewords": [
"[]",
"Evaluation"
]
}
]
},
{
"Name": "DMARCRemoveEvaluations",
"Docs": "DMARCRemoveEvaluations removes evaluations for a domain.",
"Params": [
{
"Name": "domain",
"Typewords": [
"string"
]
}
],
"Returns": []
}
],
"Sections": [],
@ -2512,7 +2566,7 @@
"Fields": [
{
"Name": "Domain",
"Docs": "",
"Docs": "Domain is where DMARC record was found, not necessarily message From. Reports we generate use unicode names, incoming reports may have either ASCII-only or Unicode domains.",
"Typewords": [
"string"
]
@ -2914,7 +2968,7 @@
},
{
"Name": "Msg",
"Docs": "Msg is a message in the queue.",
"Docs": "Msg is a message in the queue.\n\nUse MakeMsg to make a message with fields that Add needs. Add will further set\nqueueing related fields.",
"Fields": [
{
"Name": "ID",
@ -2979,6 +3033,13 @@
"int32"
]
},
{
"Name": "MaxAttempts",
"Docs": "Max number of attempts before giving up. If 0, then the default of 8 attempts is used instead.",
"Typewords": [
"int32"
]
},
{
"Name": "DialedIPs",
"Docs": "For each host, the IPs that were dialed. Used for IP selection for later attempts.",
@ -3024,6 +3085,20 @@
"bool"
]
},
{
"Name": "IsDMARCReport",
"Docs": "Delivery failures for DMARC reports are handled differently.",
"Typewords": [
"bool"
]
},
{
"Name": "IsTLSReport",
"Docs": "Delivery failures for TLS reports are handled differently.",
"Typewords": [
"bool"
]
},
{
"Name": "Size",
"Docs": "Full size of message, combined MsgPrefix with contents of message file.",
@ -3441,6 +3516,162 @@
]
}
]
},
{
"Name": "EvaluationStat",
"Docs": "EvaluationStat summarizes stored evaluations, for inclusion in an upcoming\naggregate report, for a domain.",
"Fields": [
{
"Name": "Count",
"Docs": "",
"Typewords": [
"int32"
]
},
{
"Name": "SendReport",
"Docs": "",
"Typewords": [
"bool"
]
},
{
"Name": "Domain",
"Docs": "",
"Typewords": [
"Domain"
]
}
]
},
{
"Name": "Evaluation",
"Docs": "Evaluation is the result of an evaluation of a DMARC policy, to be included\nin a DMARC report.",
"Fields": [
{
"Name": "ID",
"Docs": "",
"Typewords": [
"int64"
]
},
{
"Name": "PolicyDomain",
"Docs": "Domain where DMARC policy was found, could be the organizational domain while evaluation was for a subdomain. Unicode. Same as domain found in PolicyPublished. A separate field for its index.",
"Typewords": [
"string"
]
},
{
"Name": "Evaluated",
"Docs": "Time of evaluation, determines which report (covering whole hours) this evaluation will be included in.",
"Typewords": [
"timestamp"
]
},
{
"Name": "Optional",
"Docs": "If optional, this evaluation is not a reason to send a DMARC report, but it will be included when a report is sent due to other non-optional evaluations. Set for evaluations of incoming DMARC reports. We don't want such deliveries causing us to send a report, or we would keep exchanging reporting messages forever. Also set for when evaluation is a DMARC reject for domains we haven't positively interacted with, to prevent being used to flood an unsuspecting domain with reports.",
"Typewords": [
"bool"
]
},
{
"Name": "IntervalHours",
"Docs": "Effective aggregate reporting interval in hours. Between 1 and 24, rounded up from seconds from policy to first number that can divide 24.",
"Typewords": [
"int32"
]
},
{
"Name": "Addresses",
"Docs": "\"rua\" in DMARC record, we only store evaluations for records with aggregate reporting addresses, so always non-empty.",
"Typewords": [
"[]",
"string"
]
},
{
"Name": "PolicyPublished",
"Docs": "Policy used for evaluation. We don't store the \"fo\" field for failure reporting options, since we don't send failure reports for individual messages.",
"Typewords": [
"PolicyPublished"
]
},
{
"Name": "SourceIP",
"Docs": "For \"row\" in a report record.",
"Typewords": [
"string"
]
},
{
"Name": "Disposition",
"Docs": "",
"Typewords": [
"Disposition"
]
},
{
"Name": "AlignedDKIMPass",
"Docs": "",
"Typewords": [
"bool"
]
},
{
"Name": "AlignedSPFPass",
"Docs": "",
"Typewords": [
"bool"
]
},
{
"Name": "OverrideReasons",
"Docs": "",
"Typewords": [
"[]",
"PolicyOverrideReason"
]
},
{
"Name": "EnvelopeTo",
"Docs": "For \"identifiers\" in a report record.",
"Typewords": [
"string"
]
},
{
"Name": "EnvelopeFrom",
"Docs": "",
"Typewords": [
"string"
]
},
{
"Name": "HeaderFrom",
"Docs": "",
"Typewords": [
"string"
]
},
{
"Name": "DKIMResults",
"Docs": "For \"auth_results\" in a report record.",
"Typewords": [
"[]",
"DKIMAuthResult"
]
},
{
"Name": "SPFResults",
"Docs": "",
"Typewords": [
"[]",
"SPFAuthResult"
]
}
]
}
],
"Ints": [],

3
webmail/api.go
View file

@ -691,7 +691,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
Localpart: rcpt.Localpart,
IPDomain: dns.IPDomain{Domain: rcpt.Domain},
}
_, err := queue.Add(ctx, log, reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), dataFile, nil, m.RequireTLS)
qm := queue.MakeMsg(reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS)
err := queue.Add(ctx, log, &qm, dataFile)
if err != nil {
metricSubmission.WithLabelValues("queueerror").Inc()
}