implement outgoing dmarc aggregate reporting

in smtpserver, we store dmarc evaluations (under the right conditions).
in dmarcdb, we periodically (hourly) send dmarc reports if there are
evaluations. for failed deliveries, we deliver the dsn quietly to a submailbox
of the postmaster mailbox.

this is on by default, but can be disabled in mox.conf.

.gitignore

@@ -5,6 +5,7 @@
 /testdata/check/
 /testdata/ctl/data/
 /testdata/ctl/dkim/
+/testdata/dmarcdb/data/
 /testdata/empty/
 /testdata/exportmaildir/
 /testdata/exportmbox/

README.md

@@ -8,7 +8,8 @@ See Quickstart below to get started.
 - SMTP (with extensions) for receiving, submitting and delivering email.
 - IMAP4 (with extensions) for giving email clients access to email.
 - Webmail for reading/sending email from the browser.
-- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC reports.
+- SPF/DKIM/DMARC for authenticating messages/delivery, also DMARC aggregate
+  reports.
 - Reputation tracking, learning (per user) host-, domain- and
   sender address-based reputation from (Non-)Junk email classification.
 - Bayesian spam filtering that learns (per user) from (Non-)Junk email.
@@ -113,7 +114,7 @@ https://nlnet.nl/project/Mox/.
 
 ## Roadmap
 
-- Sending DMARC and TLS reports (currently only receiving)
+- Sending TLS reports (currently only receiving)
 - Authentication other than HTTP-basic for webmail/webadmin/webaccount
 - Per-domain webmail and IMAP/SMTP host name (and TLS cert) and client settings
 - Make mox Go packages more easily reusable, each pulling in fewer (internal)

backup.go

@@ -279,7 +279,8 @@ func backupctl(ctx context.Context, ctl *ctl) {
 	if err := os.WriteFile(filepath.Join(dstDataDir, "moxversion"), []byte(moxvar.Version), 0660); err != nil {
 		xerrx("writing moxversion", err)
 	}
-	backupDB(dmarcdb.DB, "dmarcrpt.db")
+	backupDB(dmarcdb.ReportsDB, "dmarcrpt.db")
+	backupDB(dmarcdb.EvalDB, "dmarceval.db")
 	backupDB(mtastsdb.DB, "mtasts.db")
 	backupDB(tlsrptdb.DB, "tlsrpt.db")
 	backupFile("receivedid.key")
@@ -529,7 +530,7 @@ func backupctl(ctx context.Context, ctl *ctl) {
 		}
 
 		switch p {
-		case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
+		case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "ctl":
 			// Already handled.
 			return nil
 		case "lastknownversion": // Optional file, not yet handled.

config/config.go

@@ -60,6 +60,7 @@ type Static struct {
 	InitialMailboxes       InitialMailboxes     `sconf:"optional" sconf-doc:"Mailboxes to create for new accounts. Inbox is always created. Mailboxes can be given a 'special-use' role, which are understood by most mail clients. If absent/empty, the following mailboxes are created: Sent, Archive, Trash, Drafts and Junk."`
 	DefaultMailboxes       []string             `sconf:"optional" sconf-doc:"Deprecated in favor of InitialMailboxes. Mailboxes to create when adding an account. Inbox is always created. If no mailboxes are specified, the following are automatically created: Sent, Archive, Trash, Drafts and Junk."`
 	Transports             map[string]Transport `sconf:"optional" sconf-doc:"Transport are mechanisms for delivering messages. Transports can be referenced from Routes in accounts, domains and the global configuration. There is always an implicit/fallback delivery transport doing direct delivery with SMTP from the outgoing message queue. Transports are typically only configured when using smarthosts, i.e. when delivering through another SMTP server. Zero or one transport methods must be set in a transport, never multiple. When using an external party to send email for a domain, keep in mind you may have to add their IP address to your domain's SPF record, and possibly additional DKIM records."`
+	NoOutgoingDMARCReports bool                 `sconf:"optional" sconf-doc:"Do not send DMARC reports (aggregate only). By default, aggregate reports on DMARC evaluations are sent to domains if their DMARC policy requests them. Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24 hours, rounded up so a whole number of intervals cover 24 hours, aligned at whole days in UTC."`
 
 	// All IPs that were explicitly listen on for external SMTP. Only set when there
 	// are no unspecified external SMTP listeners and there is at most one for IPv4 and

config/doc.go

@@ -549,6 +549,13 @@ describe-static" and "mox config describe-domains":
 				# typically the hostname of the host in the Address field.
 				RemoteHostname:
 
+	# Do not send DMARC reports (aggregate only). By default, aggregate reports on
+	# DMARC evaluations are sent to domains if their DMARC policy requests them.
+	# Reports are sent at whole hours, with a minimum of 1 hour and maximum of 24
+	# hours, rounded up so a whole number of intervals cover 24 hours, aligned at
+	# whole days in UTC. (optional)
+	NoOutgoingDMARCReports: false
+
 # domains.conf
 
 	# NOTE: This config file is in 'sconf' format. Indent with tabs. Comments must be

dkim/dkim.go

@@ -382,14 +382,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, smtputf8 bool, policy fu
 
 		h, canonHeaderSimple, canonDataSimple, err := checkSignatureParams(ctx, sig)
 		if err != nil {
-			results = append(results, Result{StatusPermerror, nil, nil, false, err})
+			results = append(results, Result{StatusPermerror, sig, nil, false, err})
 			continue
 		}
 
 		// ../rfc/6376:2560
 		if err := policy(sig); err != nil {
 			err := fmt.Errorf("%w: %s", ErrPolicy, err)
-			results = append(results, Result{StatusPolicy, nil, nil, false, err})
+			results = append(results, Result{StatusPolicy, sig, nil, false, err})
 			continue
 		}
 

dmarc/dmarc.go

@@ -76,6 +76,8 @@ type Result struct {
 	// Result of DMARC validation. A message can fail validation, but still
 	// not be rejected, e.g. if the policy is "none".
 	Status          Status
+	AlignedSPFPass  bool
+	AlignedDKIMPass bool
 	// Domain with the DMARC DNS record. May be the organizational domain instead of
 	// the domain in the From-header.
 	Domain dns.Domain
@@ -142,7 +144,7 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain)
 			return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
 		}
 		if record != nil {
-			// ../ ../rfc/7489:1388
+			// ../rfc/7489:1388
 			return StatusNone, nil, "", result.Authentic, ErrMultipleRecords
 		}
 		text = txt
@@ -152,14 +154,15 @@ func lookupRecord(ctx context.Context, resolver dns.Resolver, domain dns.Domain)
 	return StatusNone, record, text, result.Authentic, rerr
 }
 
-func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, *Record, string, bool, error) {
+func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain, extDestDomain dns.Domain) (Status, []*Record, []string, bool, error) {
+	// ../rfc/7489:1566
 	name := dmarcDomain.ASCII + "._report._dmarc." + extDestDomain.ASCII + "."
 	txts, result, err := dns.WithPackage(resolver, "dmarc").LookupTXT(ctx, name)
 	if err != nil && !dns.IsNotFound(err) {
-		return StatusTemperror, nil, "", result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err)
+		return StatusTemperror, nil, nil, result.Authentic, fmt.Errorf("%w: %s", ErrDNS, err)
 	}
-	var record *Record
+	var records []*Record
-	var text string
+	var texts []string
 	var rerr error = ErrNoRecord
 	for _, txt := range txts {
 		r, isdmarc, err := ParseRecordNoRequired(txt)
@@ -171,44 +174,44 @@ func lookupReportsRecord(ctx context.Context, resolver dns.Resolver, dmarcDomain
 			r, isdmarc, err = &xr, true, nil
 		}
 		if !isdmarc {
-			// ../rfc/7489:1374
+			// ../rfc/7489:1586
 			continue
-		} else if err != nil {
-			return StatusPermerror, nil, text, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
 		}
-		if record != nil {
+		texts = append(texts, txt)
-			// ../ ../rfc/7489:1388
+		records = append(records, r)
-			return StatusNone, nil, "", result.Authentic, ErrMultipleRecords
+		if err != nil {
+			return StatusPermerror, records, texts, result.Authentic, fmt.Errorf("%w: %s", ErrSyntax, err)
 		}
-		text = txt
+		// Multiple records are allowed for the _report record, unlike for policies. ../rfc/7489:1593
-		record = r
 		rerr = nil
 	}
-	return StatusNone, record, text, result.Authentic, rerr
+	return StatusNone, records, texts, result.Authentic, rerr
 }
 
 // LookupExternalReportsAccepted returns whether the extDestDomain has opted in
 // to receiving dmarc reports for dmarcDomain (where the dmarc record was found),
 // through a "._report._dmarc." DNS TXT DMARC record.
 //
-// Callers should look at status for interpretation, not err, because err will
+// accepts is true if the external domain has opted in.
-// be set to ErrNoRecord when the DNS TXT record isn't present, which means the
+// If a temporary error occurred, the returned status is StatusTemperror, and a
-// extDestDomain does not opt in (not a failure condition).
+// later retry may give an authoritative result.
+// The returned error is ErrNoRecord if no opt-in DNS record exists, which is
+// not a failure condition.
 //
 // The normally invalid "v=DMARC1" record is accepted since it is used as
 // example in RFC 7489.
 //
 // authentic indicates if the DNS results were DNSSEC-verified.
-func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, record *Record, txt string, authentic bool, rerr error) {
+func LookupExternalReportsAccepted(ctx context.Context, resolver dns.Resolver, dmarcDomain dns.Domain, extDestDomain dns.Domain) (accepts bool, status Status, records []*Record, txts []string, authentic bool, rerr error) {
 	log := xlog.WithContext(ctx)
 	start := time.Now()
 	defer func() {
-		log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("record", record), mlog.Field("duration", time.Since(start)))
+		log.Debugx("dmarc externalreports result", rerr, mlog.Field("accepts", accepts), mlog.Field("dmarcdomain", dmarcDomain), mlog.Field("extdestdomain", extDestDomain), mlog.Field("records", records), mlog.Field("duration", time.Since(start)))
 	}()
 
-	status, record, txt, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
+	status, records, txts, authentic, rerr = lookupReportsRecord(ctx, resolver, dmarcDomain, extDestDomain)
 	accepts = rerr == nil
-	return accepts, status, record, txt, authentic, rerr
+	return accepts, status, records, txts, authentic, rerr
 }
 
 // Verify evaluates the DMARC policy for the domain in the From-header of a
@@ -241,7 +244,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 
 	status, recordDomain, record, _, authentic, err := Lookup(ctx, resolver, from)
 	if record == nil {
-		return false, Result{false, status, recordDomain, record, authentic, err}
+		return false, Result{false, status, false, false, recordDomain, record, authentic, err}
 	}
 	result.Domain = recordDomain
 	result.Record = record
@@ -251,8 +254,8 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 	// See ../rfc/7489:1432
 	useResult = !applyRandomPercentage || record.Percentage == 100 || mathrand.Intn(100) < record.Percentage
 
-	// We reject treat "quarantine" and "reject" the same. Thus, we also don't
+	// We treat "quarantine" and "reject" the same. Thus, we also don't "downgrade"
-	// "downgrade" from reject to quarantine if this message was sampled out.
+	// from reject to quarantine if this message was sampled out.
 	// ../rfc/7489:1446 ../rfc/7489:1024
 	if recordDomain != from && record.SubdomainPolicy != PolicyEmpty {
 		result.Reject = record.SubdomainPolicy != PolicyNone
@@ -282,9 +285,7 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 	// ../rfc/7489:1319
 	// ../rfc/7489:544
 	if spfResult == spf.StatusPass && spfIdentity != nil && (*spfIdentity == from || result.Record.ASPF == "r" && pubsuffix(from) == pubsuffix(*spfIdentity)) {
-		result.Reject = false
+		result.AlignedSPFPass = true
-		result.Status = StatusPass
-		return
 	}
 
 	for _, dkimResult := range dkimResults {
@@ -296,10 +297,14 @@ func Verify(ctx context.Context, resolver dns.Resolver, from dns.Domain, dkimRes
 		// ../rfc/7489:511
 		if dkimResult.Status == dkim.StatusPass && dkimResult.Sig != nil && (dkimResult.Sig.Domain == from || result.Record.ADKIM == "r" && pubsuffix(from) == pubsuffix(dkimResult.Sig.Domain)) {
 			// ../rfc/7489:535
+			result.AlignedDKIMPass = true
+			break
+		}
+	}
+
+	if result.AlignedSPFPass || result.AlignedDKIMPass {
 		result.Reject = false
 		result.Status = StatusPass
-			return
-		}
 	}
 	return
 }

dmarc/dmarc_test.go

@@ -84,7 +84,7 @@ func TestLookupExternalReportsAccepted(t *testing.T) {
 	test("example.com", "simple2.example", StatusNone, true, nil)
 	test("example.com", "one.example", StatusNone, true, nil)
 	test("example.com", "absent.example", StatusNone, false, ErrNoRecord)
-	test("example.com", "multiple.example", StatusNone, false, ErrMultipleRecords)
+	test("example.com", "multiple.example", StatusNone, true, nil)
 	test("example.com", "malformed.example", StatusPermerror, false, ErrSyntax)
 	test("example.com", "temperror.example", StatusTemperror, false, ErrDNS)
 }
@@ -137,7 +137,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		true, Result{true, StatusFail, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Accept with spf pass.
@@ -145,7 +145,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "sub.reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Accept with dkim pass.
@@ -161,7 +161,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusFail,
 		&dns.Domain{ASCII: "reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Reject due to spf and dkim "strict".
@@ -181,7 +181,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "sub.strict.example"},
-		true, Result{true, StatusFail, dns.Domain{ASCII: "strict.example"}, &strict, false, nil},
+		true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "strict.example"}, &strict, false, nil},
 	)
 
 	// No dmarc policy, nothing to say.
@@ -189,7 +189,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
+		false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
 	)
 
 	// No dmarc policy, spf pass does nothing.
@@ -197,7 +197,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "absent.example"},
-		false, Result{false, StatusNone, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
+		false, Result{false, StatusNone, false, false, dns.Domain{ASCII: "absent.example"}, nil, false, ErrNoRecord},
 	)
 
 	none := DefaultRecord
@@ -207,7 +207,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "none.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "none.example"}, &none, false, nil},
+		true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "none.example"}, &none, false, nil},
 	)
 
 	// No actual reject due to pct=0.
@@ -218,7 +218,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		false, Result{true, StatusFail, dns.Domain{ASCII: "test.example"}, &testr, false, nil},
+		false, Result{true, StatusFail, false, false, dns.Domain{ASCII: "test.example"}, &testr, false, nil},
 	)
 
 	// No reject if subdomain has "none" policy.
@@ -229,7 +229,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusFail,
 		&dns.Domain{ASCII: "sub.subnone.example"},
-		true, Result{false, StatusFail, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil},
+		true, Result{false, StatusFail, false, false, dns.Domain{ASCII: "subnone.example"}, &sub, false, nil},
 	)
 
 	// No reject if spf temperror and no other pass.
@@ -237,7 +237,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusTemperror,
 		&dns.Domain{ASCII: "mail.reject.example"},
-		true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// No reject if dkim temperror and no other pass.
@@ -253,7 +253,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusNone,
 		nil,
-		true, Result{false, StatusTemperror, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusTemperror, false, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// No reject if spf temperror but still dkim pass.
@@ -269,7 +269,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusTemperror,
 		&dns.Domain{ASCII: "mail.reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, false, true, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// No reject if dkim temperror but still spf pass.
@@ -285,7 +285,7 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusPass,
 		&dns.Domain{ASCII: "mail.reject.example"},
-		true, Result{false, StatusPass, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
+		true, Result{false, StatusPass, true, false, dns.Domain{ASCII: "reject.example"}, &reject, false, nil},
 	)
 
 	// Bad DMARC record results in permerror without reject.
@@ -293,7 +293,7 @@ func TestVerify(t *testing.T) {
 		[]dkim.Result{},
 		spf.StatusNone,
 		nil,
-		false, Result{false, StatusPermerror, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax},
+		false, Result{false, StatusPermerror, false, false, dns.Domain{ASCII: "malformed.example"}, nil, false, ErrSyntax},
 	)
 
 	// DKIM domain that is higher-level than organizational can not result in a pass. ../rfc/7489:525
@@ -309,6 +309,6 @@ func TestVerify(t *testing.T) {
 		},
 		spf.StatusNone,
 		nil,
-		true, Result{true, StatusFail, dns.Domain{ASCII: "example.com"}, &reject, false, nil},
+		true, Result{true, StatusFail, false, false, dns.Domain{ASCII: "example.com"}, &reject, false, nil},
 	)
 }

dmarcdb/dmarcdb.go

@@ -0,0 +1,29 @@
+// Package dmarcdb stores incoming DMARC aggrate reports and evaluations for outgoing aggregate reports.
+//
+// With DMARC, a domain can request reports with DMARC evaluation results to be
+// sent to a specified address. Mox parses such reports, stores them in its
+// database and makes them available through its admin web interface. Mox also
+// keeps track of the evaluations it does for incoming messages and sends reports
+// to mail servers that request reports.
+//
+// Only aggregate reports are stored and sent. Failure reports about individual
+// messages are not implemented.
+package dmarcdb
+
+import (
+	"github.com/mjl-/mox/mox-"
+)
+
+// Init opens the databases.
+//
+// The incoming reports and evaluations for outgoing reports are in separate
+// databases for simpler file-based handling of the databases.
+func Init() error {
+	if _, err := reportsDB(mox.Shutdown); err != nil {
+		return err
+	}
+	if _, err := evalDB(mox.Shutdown); err != nil {
+		return err
+	}
+	return nil
+}

dmarcdb/eval_test.go

@@ -0,0 +1,384 @@
+package dmarcdb
+
+import (
+	"context"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mjl-/mox/dmarcrpt"
+	"github.com/mjl-/mox/dns"
+	"github.com/mjl-/mox/mlog"
+	"github.com/mjl-/mox/mox-"
+	"github.com/mjl-/mox/moxio"
+	"github.com/mjl-/mox/queue"
+)
+
+func tcheckf(t *testing.T, err error, format string, args ...any) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("%s: %s", fmt.Sprintf(format, args...), err)
+	}
+}
+
+func tcompare(t *testing.T, got, expect any) {
+	t.Helper()
+	if !reflect.DeepEqual(got, expect) {
+		t.Fatalf("got:\n%v\nexpected:\n%v", got, expect)
+	}
+}
+
+func TestEvaluations(t *testing.T) {
+	os.RemoveAll("../testdata/dmarcdb/data")
+	mox.Context = ctxbg
+	mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
+	mox.MustLoadConfig(true, false)
+	EvalDB = nil
+
+	_, err := evalDB(ctxbg)
+	tcheckf(t, err, "database")
+	defer func() {
+		EvalDB.Close()
+		EvalDB = nil
+	}()
+
+	parseJSON := func(s string) (e Evaluation) {
+		t.Helper()
+		err := json.Unmarshal([]byte(s), &e)
+		tcheckf(t, err, "unmarshal")
+		return
+	}
+	packJSON := func(e Evaluation) string {
+		t.Helper()
+		buf, err := json.Marshal(e)
+		tcheckf(t, err, "marshal")
+		return string(buf)
+	}
+
+	e0 := Evaluation{
+		PolicyDomain:  "sender1.example",
+		Evaluated:     time.Now().Round(0),
+		IntervalHours: 1,
+		PolicyPublished: dmarcrpt.PolicyPublished{
+			Domain:          "sender1.example",
+			ADKIM:           dmarcrpt.AlignmentRelaxed,
+			ASPF:            dmarcrpt.AlignmentRelaxed,
+			Policy:          dmarcrpt.DispositionReject,
+			SubdomainPolicy: dmarcrpt.DispositionReject,
+			Percentage:      100,
+		},
+		SourceIP:        "10.1.2.3",
+		Disposition:     dmarcrpt.DispositionNone,
+		AlignedDKIMPass: true,
+		AlignedSPFPass:  true,
+		EnvelopeTo:      "mox.example",
+		EnvelopeFrom:    "sender1.example",
+		HeaderFrom:      "sender1.example",
+		DKIMResults: []dmarcrpt.DKIMAuthResult{
+			{
+				Domain:   "sender1.example",
+				Selector: "test",
+				Result:   dmarcrpt.DKIMPass,
+			},
+		},
+		SPFResults: []dmarcrpt.SPFAuthResult{
+			{
+				Domain: "sender1.example",
+				Scope:  dmarcrpt.SPFDomainScopeMailFrom,
+				Result: dmarcrpt.SPFPass,
+			},
+		},
+	}
+	e1 := e0
+	e2 := parseJSON(strings.ReplaceAll(packJSON(e0), "sender1.example", "sender2.example"))
+	e3 := parseJSON(strings.ReplaceAll(packJSON(e0), "10.1.2.3", "10.3.2.1"))
+	e3.Optional = true
+
+	for i, e := range []*Evaluation{&e0, &e1, &e2, &e3} {
+		e.Evaluated = e.Evaluated.Add(time.Duration(i) * time.Second)
+		err = AddEvaluation(ctxbg, 3600, e)
+		tcheckf(t, err, "add evaluation")
+	}
+
+	expStats := map[string]EvaluationStat{
+		"sender1.example": {
+			Count:      3,
+			SendReport: true,
+			Domain:     dns.Domain{ASCII: "sender1.example"},
+		},
+		"sender2.example": {
+			Count:      1,
+			SendReport: true,
+			Domain:     dns.Domain{ASCII: "sender2.example"},
+		},
+	}
+	stats, err := EvaluationStats(ctxbg)
+	tcheckf(t, err, "evaluation stats")
+	tcompare(t, stats, expStats)
+
+	// EvaluationsDomain
+	evals, err := EvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender1.example"})
+	tcheckf(t, err, "get evaluations for domain")
+	tcompare(t, evals, []Evaluation{e0, e1, e3})
+
+	evals, err = EvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender2.example"})
+	tcheckf(t, err, "get evaluations for domain")
+	tcompare(t, evals, []Evaluation{e2})
+
+	evals, err = EvaluationsDomain(ctxbg, dns.Domain{ASCII: "bogus.example"})
+	tcheckf(t, err, "get evaluations for domain")
+	tcompare(t, evals, []Evaluation{})
+
+	// RemoveEvaluationsDomain
+	err = RemoveEvaluationsDomain(ctxbg, dns.Domain{ASCII: "sender1.example"})
+	tcheckf(t, err, "remove evaluations")
+
+	expStats = map[string]EvaluationStat{
+		"sender2.example": {
+			Count:      1,
+			SendReport: true,
+			Domain:     dns.Domain{ASCII: "sender2.example"},
+		},
+	}
+	stats, err = EvaluationStats(ctxbg)
+	tcheckf(t, err, "evaluation stats")
+	tcompare(t, stats, expStats)
+}
+
+func TestSendReports(t *testing.T) {
+	mlog.SetConfig(map[string]mlog.Level{"": mlog.LevelDebug})
+
+	os.RemoveAll("../testdata/dmarcdb/data")
+	mox.Context = ctxbg
+	mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
+	mox.MustLoadConfig(true, false)
+	EvalDB = nil
+
+	db, err := evalDB(ctxbg)
+	tcheckf(t, err, "database")
+	defer func() {
+		EvalDB.Close()
+		EvalDB = nil
+	}()
+
+	resolver := dns.MockResolver{
+		TXT: map[string][]string{
+			"_dmarc.sender.example.": {
+				"v=DMARC1; rua=mailto:dmarcrpt@sender.example; ri=3600",
+			},
+		},
+	}
+
+	end := nextWholeHour(time.Now())
+
+	eval := Evaluation{
+		PolicyDomain:  "sender.example",
+		Evaluated:     end.Add(-time.Hour / 2),
+		IntervalHours: 1,
+		PolicyPublished: dmarcrpt.PolicyPublished{
+			Domain:          "sender.example",
+			ADKIM:           dmarcrpt.AlignmentRelaxed,
+			ASPF:            dmarcrpt.AlignmentRelaxed,
+			Policy:          dmarcrpt.DispositionReject,
+			SubdomainPolicy: dmarcrpt.DispositionReject,
+			Percentage:      100,
+		},
+		SourceIP:        "10.1.2.3",
+		Disposition:     dmarcrpt.DispositionNone,
+		AlignedDKIMPass: true,
+		AlignedSPFPass:  true,
+		EnvelopeTo:      "mox.example",
+		EnvelopeFrom:    "sender.example",
+		HeaderFrom:      "sender.example",
+		DKIMResults: []dmarcrpt.DKIMAuthResult{
+			{
+				Domain:   "sender.example",
+				Selector: "test",
+				Result:   dmarcrpt.DKIMPass,
+			},
+		},
+		SPFResults: []dmarcrpt.SPFAuthResult{
+			{
+				Domain: "sender.example",
+				Scope:  dmarcrpt.SPFDomainScopeMailFrom,
+				Result: dmarcrpt.SPFPass,
+			},
+		},
+	}
+
+	expFeedback := &dmarcrpt.Feedback{
+		XMLName: xml.Name{Local: "feedback"},
+		Version: "1.0",
+		ReportMetadata: dmarcrpt.ReportMetadata{
+			OrgName: "mail.mox.example",
+			Email:   "postmaster@mail.mox.example",
+			DateRange: dmarcrpt.DateRange{
+				Begin: end.Add(-1 * time.Hour).Unix(),
+				End:   end.Add(-time.Second).Unix(),
+			},
+		},
+		PolicyPublished: dmarcrpt.PolicyPublished{
+			Domain:          "sender.example",
+			ADKIM:           dmarcrpt.AlignmentRelaxed,
+			ASPF:            dmarcrpt.AlignmentRelaxed,
+			Policy:          dmarcrpt.DispositionReject,
+			SubdomainPolicy: dmarcrpt.DispositionReject,
+			Percentage:      100,
+		},
+		Records: []dmarcrpt.ReportRecord{
+			{
+				Row: dmarcrpt.Row{
+					SourceIP: "10.1.2.3",
+					Count:    1,
+					PolicyEvaluated: dmarcrpt.PolicyEvaluated{
+						Disposition: dmarcrpt.DispositionNone,
+						DKIM:        dmarcrpt.DMARCPass,
+						SPF:         dmarcrpt.DMARCPass,
+					},
+				},
+				Identifiers: dmarcrpt.Identifiers{
+					EnvelopeTo:   "mox.example",
+					EnvelopeFrom: "sender.example",
+					HeaderFrom:   "sender.example",
+				},
+				AuthResults: dmarcrpt.AuthResults{
+					DKIM: []dmarcrpt.DKIMAuthResult{
+						{
+							Domain:   "sender.example",
+							Selector: "test",
+							Result:   dmarcrpt.DKIMPass,
+						},
+					},
+					SPF: []dmarcrpt.SPFAuthResult{
+						{
+							Domain: "sender.example",
+							Scope:  dmarcrpt.SPFDomainScopeMailFrom,
+							Result: dmarcrpt.SPFPass,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	// Set a timeUntil that we steplock and that causes the actual sleep to return immediately when we want to.
+	wait := make(chan struct{})
+	step := make(chan time.Duration)
+	jitteredTimeUntil = func(_ time.Time) time.Duration {
+		wait <- struct{}{}
+		return <-step
+	}
+
+	sleepBetween = func() {}
+
+	test := func(evals []Evaluation, expAggrAddrs map[string]struct{}, expErrorAddrs map[string]struct{}, optExpReport *dmarcrpt.Feedback) {
+		t.Helper()
+
+		mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
+
+		for _, e := range evals {
+			err := db.Insert(ctxbg, &e)
+			tcheckf(t, err, "inserting evaluation")
+		}
+
+		aggrAddrs := map[string]struct{}{}
+		errorAddrs := map[string]struct{}{}
+
+		queueAdd = func(ctx context.Context, log *mlog.Log, qm *queue.Msg, msgFile *os.File) error {
+			// Read message file. Also write copy to disk for inspection.
+			buf, err := io.ReadAll(&moxio.AtReader{R: msgFile})
+			tcheckf(t, err, "read report message")
+			err = os.WriteFile("../testdata/dmarcdb/data/report.eml", append(append([]byte{}, qm.MsgPrefix...), buf...), 0600)
+			tcheckf(t, err, "write report message")
+
+			var feedback *dmarcrpt.Feedback
+			addr := qm.Recipient().String()
+			isErrorReport := strings.Contains(string(buf), "DMARC aggregate reporting error report")
+			if isErrorReport {
+				errorAddrs[addr] = struct{}{}
+			} else {
+				aggrAddrs[addr] = struct{}{}
+
+				feedback, err = dmarcrpt.ParseMessageReport(log, msgFile)
+				tcheckf(t, err, "parsing generated report message")
+			}
+
+			if optExpReport != nil {
+				// Parse report in message and compare with expected.
+				expFeedback.ReportMetadata.ReportID = feedback.ReportMetadata.ReportID
+				tcompare(t, feedback, expFeedback)
+			}
+
+			return nil
+		}
+
+		Start(resolver)
+		// Run first loop.
+		<-wait
+		step <- 0
+		<-wait
+		tcompare(t, aggrAddrs, expAggrAddrs)
+		tcompare(t, errorAddrs, expErrorAddrs)
+
+		// Second loop. Evaluations cleaned, should not result in report messages.
+		aggrAddrs = map[string]struct{}{}
+		errorAddrs = map[string]struct{}{}
+		step <- 0
+		<-wait
+		tcompare(t, aggrAddrs, map[string]struct{}{})
+		tcompare(t, errorAddrs, map[string]struct{}{})
+
+		// Caus Start to stop.
+		mox.ShutdownCancel()
+		step <- time.Minute
+	}
+
+	// Typical case, with a single address that receives an aggregate report.
+	test([]Evaluation{eval}, map[string]struct{}{"dmarcrpt@sender.example": {}}, map[string]struct{}{}, expFeedback)
+
+	// Only optional evaluations, no report at all.
+	evalOpt := eval
+	evalOpt.Optional = true
+	test([]Evaluation{evalOpt}, map[string]struct{}{}, map[string]struct{}{}, nil)
+
+	// Two RUA's, one with a size limit that doesn't pass, and one that does pass.
+	resolver.TXT["_dmarc.sender.example."] = []string{"v=DMARC1; rua=mailto:dmarcrpt1@sender.example!1,mailto:dmarcrpt2@sender.example!10t; ri=3600"}
+	test([]Evaluation{eval}, map[string]struct{}{"dmarcrpt2@sender.example": {}}, map[string]struct{}{}, nil)
+
+	// Redirect to external domain, without permission, no report sent.
+	resolver.TXT["_dmarc.sender.example."] = []string{"v=DMARC1; rua=mailto:unauthorized@other.example"}
+	test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{}, nil)
+
+	// Redirect to external domain, with basic permission.
+	resolver.TXT = map[string][]string{
+		"_dmarc.sender.example.":                       {"v=DMARC1; rua=mailto:authorized@other.example"},
+		"sender.example._report._dmarc.other.example.": {"v=DMARC1"},
+	}
+	test([]Evaluation{eval}, map[string]struct{}{"authorized@other.example": {}}, map[string]struct{}{}, nil)
+
+	// Redirect to authorized external domain, with 2 allowed replacements and 1 invalid and 1 refusing due to size.
+	resolver.TXT = map[string][]string{
+		"_dmarc.sender.example.":                       {"v=DMARC1; rua=mailto:authorized@other.example"},
+		"sender.example._report._dmarc.other.example.": {"v=DMARC1; rua=mailto:good1@other.example,mailto:bad1@yetanother.example,mailto:good2@other.example,mailto:badsize@other.example!1"},
+	}
+	test([]Evaluation{eval}, map[string]struct{}{"good1@other.example": {}, "good2@other.example": {}}, map[string]struct{}{}, nil)
+
+	// Without RUA, we send no message.
+	resolver.TXT = map[string][]string{
+		"_dmarc.sender.example.": {"v=DMARC1;"},
+	}
+	test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{}, nil)
+
+	// If message size limit is reached, an error repor is sent.
+	resolver.TXT = map[string][]string{
+		"_dmarc.sender.example.": {"v=DMARC1; rua=mailto:dmarcrpt@sender.example!1"},
+	}
+	test([]Evaluation{eval}, map[string]struct{}{}, map[string]struct{}{"dmarcrpt@sender.example": {}}, nil)
+}

dmarcdb/reports.go

@@ -1,9 +1,3 @@
-// Package dmarcdb stores incoming DMARC reports.
-//
-// With DMARC, a domain can request emails with DMARC verification results by
-// remote mail servers to be sent to a specified address. Mox parses such
-// reports, stores them in its database and makes them available through its
-// admin web interface.
 package dmarcdb
 
 import (
@@ -25,9 +19,9 @@ import (
 )
 
 var (
-	DBTypes = []any{DomainFeedback{}} // Types stored in DB.
+	ReportsDBTypes = []any{DomainFeedback{}} // Types stored in DB.
-	DB      *bstore.DB                // Exported for backups.
+	ReportsDB      *bstore.DB                // Exported for backups.
-	mutex   sync.Mutex
+	reportsMutex   sync.Mutex
 )
 
 var (
@@ -65,25 +59,19 @@ type DomainFeedback struct {
 	dmarcrpt.Feedback
 }
 
-func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
+func reportsDB(ctx context.Context) (rdb *bstore.DB, rerr error) {
-	mutex.Lock()
+	reportsMutex.Lock()
-	defer mutex.Unlock()
+	defer reportsMutex.Unlock()
-	if DB == nil {
+	if ReportsDB == nil {
 		p := mox.DataDirPath("dmarcrpt.db")
 		os.MkdirAll(filepath.Dir(p), 0770)
-		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
+		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, ReportsDBTypes...)
 		if err != nil {
 			return nil, err
 		}
-		DB = db
+		ReportsDB = db
 	}
-	return DB, nil
+	return ReportsDB, nil
-}
-
-// Init opens the database.
-func Init() error {
-	_, err := database(mox.Shutdown)
-	return err
 }
 
 // AddReport adds a DMARC aggregate feedback report from an email to the database,
@@ -91,7 +79,7 @@ func Init() error {
 //
 // fromDomain is the domain in the report message From header.
 func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) error {
-	db, err := database(ctx)
+	db, err := reportsDB(ctx)
 	if err != nil {
 		return err
 	}
@@ -141,7 +129,7 @@ func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain)
 
 // Records returns all reports in the database.
 func Records(ctx context.Context) ([]DomainFeedback, error) {
-	db, err := database(ctx)
+	db, err := reportsDB(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -151,7 +139,7 @@ func Records(ctx context.Context) ([]DomainFeedback, error) {
 
 // RecordID returns the report for the ID.
 func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
-	db, err := database(ctx)
+	db, err := reportsDB(ctx)
 	if err != nil {
 		return DomainFeedback{}, err
 	}
@@ -164,7 +152,7 @@ func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
 // RecordsPeriodDomain returns the reports overlapping start and end, for the given
 // domain. If domain is empty, all records match for domain.
 func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]DomainFeedback, error) {
-	db, err := database(ctx)
+	db, err := reportsDB(ctx)
 	if err != nil {
 		return nil, err
 	}

dmarcdb/reports_test.go

@@ -17,8 +17,8 @@ var ctxbg = context.Background()
 
 func TestDMARCDB(t *testing.T) {
 	mox.Shutdown = ctxbg
-	mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/fake.conf")
+	mox.ConfigStaticPath = filepath.FromSlash("../testdata/dmarcdb/mox.conf")
-	mox.Conf.Static.DataDir = "."
+	mox.MustLoadConfig(true, false)
 
 	dbpath := mox.DataDirPath("dmarcrpt.db")
 	os.MkdirAll(filepath.Dir(dbpath), 0770)
@@ -27,7 +27,10 @@ func TestDMARCDB(t *testing.T) {
 		t.Fatalf("init database: %s", err)
 	}
 	defer os.Remove(dbpath)
-	defer DB.Close()
+	defer func() {
+		ReportsDB.Close()
+		ReportsDB = nil
+	}()
 
 	feedback := &dmarcrpt.Feedback{
 		ReportMetadata: dmarcrpt.ReportMetadata{

dmarcrpt/feedback.go

@@ -1,9 +1,14 @@
 package dmarcrpt
 
+import (
+	"encoding/xml"
+)
+
 // Initially generated by xsdgen, then modified.
 
 // Feedback is the top-level XML field returned.
 type Feedback struct {
+	XMLName         xml.Name        `xml:"feedback" json:"-"` // todo: removing the json tag triggers bug in sherpadoc, should fix.
 	Version         string          `xml:"version"`
 	ReportMetadata  ReportMetadata  `xml:"report_metadata"`
 	PolicyPublished PolicyPublished `xml:"policy_published"`
@@ -26,6 +31,9 @@ type DateRange struct {
 
 // PolicyPublished is the policy as found in DNS for the domain.
 type PolicyPublished struct {
+	// Domain is where DMARC record was found, not necessarily message From. Reports we
+	// generate use unicode names, incoming reports may have either ASCII-only or
+	// Unicode domains.
 	Domain           string      `xml:"domain"`
 	ADKIM            Alignment   `xml:"adkim,omitempty"`
 	ASPF             Alignment   `xml:"aspf,omitempty"`

dmarcrpt/parse.go

@@ -17,7 +17,7 @@ import (
 	"github.com/mjl-/mox/moxio"
 )
 
-var ErrNoReport = errors.New("no dmarc report found in message")
+var ErrNoReport = errors.New("no dmarc aggregate report found in message")
 
 // ParseReport parses an XML aggregate feedback report.
 // The maximum report size is 20MB.

dmarcrpt/parse_test.go

@@ -1,6 +1,7 @@
 package dmarcrpt
 
 import (
+	"encoding/xml"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -62,6 +63,7 @@ const reportExample = `<?xml version="1.0" encoding="UTF-8" ?>
 
 func TestParseReport(t *testing.T) {
 	var expect = &Feedback{
+		XMLName: xml.Name{Local: "feedback"},
 		ReportMetadata: ReportMetadata{
 			OrgName:          "google.com",
 			Email:            "noreply-dmarc-support@google.com",
@@ -126,7 +128,7 @@ func TestParseMessageReport(t *testing.T) {
 	dir := filepath.FromSlash("../testdata/dmarc-reports")
 	files, err := os.ReadDir(dir)
 	if err != nil {
-		t.Fatalf("listing dmarc report emails: %s", err)
+		t.Fatalf("listing dmarc aggregate report emails: %s", err)
 	}
 
 	for _, file := range files {

gentestdata.go

@@ -192,9 +192,9 @@ Accounts:
 	err = dmarcdb.Init()
 	xcheckf(err, "dmarcdb init")
 	report, err := dmarcrpt.ParseReport(strings.NewReader(dmarcReport))
-	xcheckf(err, "parsing dmarc report")
+	xcheckf(err, "parsing dmarc aggregate report")
 	err = dmarcdb.AddReport(ctxbg, report, dns.Domain{ASCII: "mox.example"})
-	xcheckf(err, "adding dmarc report")
+	xcheckf(err, "adding dmarc aggregate report")
 
 	// Populate mtasts.db.
 	err = mtastsdb.Init(false)
@@ -233,7 +233,8 @@ Accounts:
 	const qmsg = "From: <test0@mox.example>\r\nTo: <other@remote.example>\r\nSubject: test\r\n\r\nthe message...\r\n"
 	_, err = fmt.Fprint(mf, qmsg)
 	xcheckf(err, "writing message")
-	_, err = queue.Add(ctxbg, log, "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, mf, nil, nil)
+	qm := queue.MakeMsg("test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, nil)
+	err = queue.Add(ctxbg, log, &qm, mf)
 	xcheckf(err, "enqueue message")
 
 	// Create three accounts.

localserve.go

@@ -124,8 +124,9 @@ during those commands instead of during "data".
 	queue.Localserve = true
 
 	const mtastsdbRefresher = false
+	const sendDMARCReports = false
 	const skipForkExec = true
-	if err := start(mtastsdbRefresher, skipForkExec); err != nil {
+	if err := start(mtastsdbRefresher, sendDMARCReports, skipForkExec); err != nil {
 		log.Fatalx("starting mox", err)
 	}
 	golog.Printf("mox, version %s", moxvar.Version)

main.go

@@ -2436,13 +2436,13 @@ address must opt-in to receiving DMARC reports by creating a DMARC record at
 			return
 		}
 
-		accepts, status, _, txt, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom)
+		accepts, status, _, txts, authentic, err := dmarc.LookupExternalReportsAccepted(context.Background(), dns.StrictResolver{}, domain, destdom)
 		var txtstr string
 		txtaddr := fmt.Sprintf("%s._report._dmarc.%s", domain.ASCII, destdom.ASCII)
-		if txt == "" {
+		if len(txts) == 0 {
-			txtstr = fmt.Sprintf(" (no txt record %s)", txtaddr)
+			txtstr = fmt.Sprintf(" (no txt records %s)", txtaddr)
 		} else {
-			txtstr = fmt.Sprintf(" (txt record %s: %q)", txtaddr, txt)
+			txtstr = fmt.Sprintf(" (txt record %s: %q)", txtaddr, txts)
 		}
 		if status != dmarc.StatusNone {
 			printResult("fail: %s%s", err, txtstr)

metrics/panic.go

@@ -22,6 +22,7 @@ const (
 	Import           Panic = "import"
 	Serve            Panic = "serve"
 	Imapserver       Panic = "imapserver"
+	Dmarcdb          Panic = "dmarcdb"
 	Mtastsdb         Panic = "mtastsdb"
 	Queue            Panic = "queue"
 	Smtpclient       Panic = "smtpclient"

queue/direct.go

@@ -92,7 +92,7 @@ func fail(qlog *mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMT
 	// todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls?
 	// todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379
 
-	if permanent || m.Attempts >= 8 {
+	if permanent || m.MaxAttempts == 0 && m.Attempts >= 8 || m.MaxAttempts > 0 && m.Attempts >= m.MaxAttempts {
 		qlog.Errorx("permanent failure delivering from queue", errors.New(errmsg))
 		deliverDSNFailure(qlog, m, remoteMTA, secodeOpt, errmsg)
 
@@ -230,12 +230,13 @@ func deliverDirect(cid int64, qlog *mlog.Log, resolver dns.Resolver, dialer smtp
 		enforceMTASTS := policy != nil && policy.Mode == mtasts.ModeEnforce
 		permanent, daneRequired, badTLS, secodeOpt, remoteIP, errmsg, ok = deliverHost(nqlog, resolver, dialer, cid, ourHostname, transportName, h, enforceMTASTS, haveMX, origNextHopAuthentic, origNextHop, expandedNextHopAuthentic, expandedNextHop, &m, tlsMode)
 
-		// If we had a TLS-related failure when doing TLS, and we don't have a requirement for MTA-STS/DANE,
+		// If we had a TLS-related failure when doing TLS, and we don't have a requirement
-		// we try again without TLS. This could be an old
+		// for MTA-STS/DANE, we try again without TLS. This could be an old server that
-		// server that only does ancient TLS versions, or has a misconfiguration. Note that
+		// only does ancient TLS versions, or has a misconfiguration. Note that
 		// opportunistic TLS does not do regular certificate verification, so that can't be
 		// the problem.
-		if !ok && badTLS && (!enforceMTASTS && tlsMode == smtpclient.TLSOpportunistic && !daneRequired || m.RequireTLS != nil && !*m.RequireTLS) {
+		// We don't fall back to plain text for DMARC reports. ../rfc/7489:1768 ../rfc/7489:2683
+		if !ok && badTLS && (!enforceMTASTS && tlsMode == smtpclient.TLSOpportunistic && !daneRequired && !m.IsDMARCReport || m.RequireTLS != nil && !*m.RequireTLS) {
 			metricPlaintextFallback.Inc()
 			if m.RequireTLS != nil && !*m.RequireTLS {
 				metricTLSRequiredNoIgnored.WithLabelValues("badtls").Inc()

queue/dsn.go

@@ -6,6 +6,9 @@ import (
 	"os"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
 	"github.com/mjl-/mox/dns"
 	"github.com/mjl-/mox/dsn"
 	"github.com/mjl-/mox/message"
@@ -15,6 +18,15 @@ import (
 	"github.com/mjl-/mox/store"
 )
 
+var (
+	metricDMARCReportFailure = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "mox_queue_dmarcreport_failure_total",
+			Help: "Permanent failures to deliver a DMARC report.",
+		},
+	)
+)
+
 func deliverDSNFailure(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
 	const subject = "mail delivery failed"
 	message := fmt.Sprintf(`
@@ -33,6 +45,12 @@ Error during the last delivery attempt:
 }
 
 func deliverDSNDelay(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) {
+	// Should not happen, but doesn't hurt to prevent sending delayed delivery
+	// notifications for DMARC reports. We don't want to waste postmaster attention.
+	if m.IsDMARCReport {
+		return
+	}
+
 	const subject = "mail delivery delayed"
 	message := fmt.Sprintf(`
 Delivery has been delayed of your email to:
@@ -133,7 +151,12 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
 	msgData = append(msgData, []byte("Return-Path: <"+dsnMsg.From.XString(m.SMTPUTF8)+">\r\n")...)
 
 	mailbox := "Inbox"
-	acc, err := store.OpenAccount(m.SenderAccount)
+	senderAccount := m.SenderAccount
+	if m.IsDMARCReport {
+		// senderAccount should already by postmaster, but doesn't hurt to ensure it.
+		senderAccount = mox.Conf.Static.Postmaster.Account
+	}
+	acc, err := store.OpenAccount(senderAccount)
 	if err != nil {
 		acc, err = store.OpenAccount(mox.Conf.Static.Postmaster.Account)
 		if err != nil {
@@ -171,6 +194,17 @@ func deliverDSN(log *mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg st
 		Size:      msgWriter.Size,
 		MsgPrefix: []byte{},
 	}
+
+	// If this is a DMARC report, deliver it as seen message to a submailbox of the
+	// postmaster mailbox. We mark it as seen so it doesn't waste postmaster attention,
+	// but we deliver them so they can be checked in case of problems.
+	if m.IsDMARCReport {
+		mailbox = fmt.Sprintf("%s/dmarc", mox.Conf.Static.Postmaster.Mailbox)
+		msg.Seen = true
+		metricDMARCReportFailure.Inc()
+		log.Info("delivering dsn for failure to deliver outgoing dmarc report")
+	}
+
 	acc.WithWLock(func() {
 		if err := acc.DeliverMailbox(log, mailbox, msg, msgFile); err != nil {
 			qlog("delivering dsn to mailbox", err)

queue/queue.go

@@ -70,6 +70,9 @@ var DB *bstore.DB          // Exported for making backups.
 var Localserve bool
 
 // Msg is a message in the queue.
+//
+// Use MakeMsg to make a message with fields that Add needs. Add will further set
+// queueing related fields.
 type Msg struct {
 	ID                 int64
 	Queued             time.Time      `bstore:"default now"`
@@ -80,12 +83,16 @@ type Msg struct {
 	RecipientDomain    dns.IPDomain
 	RecipientDomainStr string              // For filtering.
 	Attempts           int                 // Next attempt is based on last attempt and exponential back off based on attempts.
+	MaxAttempts        int                 // Max number of attempts before giving up. If 0, then the default of 8 attempts is used instead.
 	DialedIPs          map[string][]net.IP // For each host, the IPs that were dialed. Used for IP selection for later attempts.
 	NextAttempt        time.Time           // For scheduling.
 	LastAttempt        *time.Time
 	LastError          string
+
 	Has8bit       bool   // Whether message contains bytes with high bit set, determines whether 8BITMIME SMTP extension is needed.
 	SMTPUTF8      bool   // Whether message requires use of SMTPUTF8.
+	IsDMARCReport bool   // Delivery failures for DMARC reports are handled differently.
+	IsTLSReport   bool   // Delivery failures for TLS reports are handled differently.
 	Size          int64  // Full size of message, combined MsgPrefix with contents of message file.
 	MessageID     string // Used when composing a DSN, in its References header.
 	MsgPrefix     []byte
@@ -188,44 +195,71 @@ func Count(ctx context.Context) (int, error) {
 	return bstore.QueryDB[Msg](ctx, DB).Count()
 }
 
+// MakeMsg is a convenience function that sets the commonly used fields for a Msg.
+func MakeMsg(senderAccount string, sender, recipient smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, prefix []byte, requireTLS *bool) Msg {
+	return Msg{
+		SenderAccount:      mox.Conf.Static.Postmaster.Account,
+		SenderLocalpart:    sender.Localpart,
+		SenderDomain:       sender.IPDomain,
+		RecipientLocalpart: recipient.Localpart,
+		RecipientDomain:    recipient.IPDomain,
+		Has8bit:            has8bit,
+		SMTPUTF8:           smtputf8,
+		Size:               size,
+		MessageID:          messageID,
+		MsgPrefix:          prefix,
+		RequireTLS:         requireTLS,
+	}
+}
+
 // Add a new message to the queue. The queue is kicked immediately to start a
 // first delivery attempt.
 //
-// dnsutf8Opt is a utf8-version of the message, to be used only for DNSs. If set,
+// ID must be 0 and will be set after inserting in the queue.
-// this data is used as the message when delivering the DSN and the remote SMTP
+//
-// server supports SMTPUTF8. If the remote SMTP server does not support SMTPUTF8,
+// Add sets derived fields like RecipientDomainStr, and fields related to queueing,
-// the regular non-utf8 message is delivered.
+// such as Queued, NextAttempt, LastAttempt, LastError.
-func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, messageID string, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, requireTLS *bool) (int64, error) {
+func Add(ctx context.Context, log *mlog.Log, qm *Msg, msgFile *os.File) error {
 	// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
 
-	if Localserve {
+	if qm.ID != 0 {
-		if senderAccount == "" {
+		return fmt.Errorf("id of queued message must be 0")
-			return 0, fmt.Errorf("cannot queue with localserve without local account")
 	}
-		acc, err := store.OpenAccount(senderAccount)
+	qm.Queued = time.Now()
+	qm.DialedIPs = nil
+	qm.NextAttempt = qm.Queued
+	qm.LastAttempt = nil
+	qm.LastError = ""
+	qm.RecipientDomainStr = formatIPDomain(qm.RecipientDomain)
+
+	if Localserve {
+		if qm.SenderAccount == "" {
+			return fmt.Errorf("cannot queue with localserve without local account")
+		}
+		acc, err := store.OpenAccount(qm.SenderAccount)
 		if err != nil {
-			return 0, fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err)
+			return fmt.Errorf("opening sender account for immediate delivery with localserve: %v", err)
 		}
 		defer func() {
 			err := acc.Close()
 			log.Check(err, "closing account")
 		}()
-		m := store.Message{Size: size, MsgPrefix: msgPrefix}
+		m := store.Message{Size: qm.Size, MsgPrefix: qm.MsgPrefix}
 		conf, _ := acc.Conf()
-		dest := conf.Destinations[mailFrom.String()]
+		dest := conf.Destinations[qm.Sender().String()]
 		acc.WithWLock(func() {
 			err = acc.DeliverDestination(log, dest, &m, msgFile)
 		})
 		if err != nil {
-			return 0, fmt.Errorf("delivering message: %v", err)
+			return fmt.Errorf("delivering message: %v", err)
 		}
 		log.Debug("immediately delivered from queue to sender")
-		return 0, nil
+		return nil
 	}
 
 	tx, err := DB.Begin(ctx, true)
 	if err != nil {
-		return 0, fmt.Errorf("begin transaction: %w", err)
+		return fmt.Errorf("begin transaction: %w", err)
 	}
 	defer func() {
 		if tx != nil {
@@ -235,11 +269,8 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
 		}
 	}()
 
-	now := time.Now()
+	if err := tx.Insert(qm); err != nil {
-	qm := Msg{0, now, senderAccount, mailFrom.Localpart, mailFrom.IPDomain, rcptTo.Localpart, rcptTo.IPDomain, formatIPDomain(rcptTo.IPDomain), 0, nil, now, nil, "", has8bit, smtputf8, size, messageID, msgPrefix, dsnutf8Opt, "", requireTLS}
+		return err
-
-	if err := tx.Insert(&qm); err != nil {
-		return 0, err
 	}
 
 	dst := qm.MessagePath()
@@ -252,19 +283,19 @@ func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcp
 	dstDir := filepath.Dir(dst)
 	os.MkdirAll(dstDir, 0770)
 	if err := moxio.LinkOrCopy(log, dst, msgFile.Name(), nil, true); err != nil {
-		return 0, fmt.Errorf("linking/copying message to new file: %s", err)
+		return fmt.Errorf("linking/copying message to new file: %s", err)
 	} else if err := moxio.SyncDir(dstDir); err != nil {
-		return 0, fmt.Errorf("sync directory: %v", err)
+		return fmt.Errorf("sync directory: %v", err)
 	}
 
 	if err := tx.Commit(); err != nil {
-		return 0, fmt.Errorf("commit transaction: %s", err)
+		return fmt.Errorf("commit transaction: %s", err)
 	}
 	tx = nil
 	dst = ""
 
 	queuekick()
-	return qm.ID, nil
+	return nil
 }
 
 func formatIPDomain(d dns.IPDomain) string {

queue/queue_test.go

@@ -110,10 +110,14 @@ func TestQueue(t *testing.T) {
 	defer os.Remove(mf.Name())
 	defer mf.Close()
 
-	_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
+	var qm Msg
+
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
 
-	_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
 
 	msgs, err = List(ctxbg)
@@ -440,7 +444,8 @@ func TestQueue(t *testing.T) {
 
 	// Add a message to be delivered with submit because of its route.
 	topath := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "submit.example"}}}
-	_, err = Add(ctxbg, xlog, "mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, topath, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
 	wasNetDialer = testDeliver(fakeSubmitServer)
 	if !wasNetDialer {
@@ -448,10 +453,11 @@ func TestQueue(t *testing.T) {
 	}
 
 	// Add a message to be delivered with submit because of explicitly configured transport, that uses TLS.
-	msgID, err := Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
 	transportSubmitTLS := "submittls"
-	n, err = Kick(ctxbg, msgID, "", "", &transportSubmitTLS)
+	n, err = Kick(ctxbg, qm.ID, "", "", &transportSubmitTLS)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -472,10 +478,11 @@ func TestQueue(t *testing.T) {
 	}
 
 	// Add a message to be delivered with socks.
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<socks@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
 	transportSocks := "socks"
-	n, err = Kick(ctxbg, msgID, "", "", &transportSocks)
+	n, err = Kick(ctxbg, qm.ID, "", "", &transportSocks)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -486,9 +493,10 @@ func TestQueue(t *testing.T) {
 	}
 
 	// Add message to be delivered with opportunistic TLS verification.
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -496,9 +504,10 @@ func TestQueue(t *testing.T) {
 	testDeliver(fakeSMTPSTARTTLSServer)
 
 	// Test fallback to plain text with TLS handshake fails.
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<badtls@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -512,9 +521,10 @@ func TestQueue(t *testing.T) {
 			{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: moxCert.Leaf.RawSubjectPublicKeyInfo},
 		},
 	}
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<dane@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -530,9 +540,10 @@ func TestQueue(t *testing.T) {
 
 	// Add message to be delivered with verified TLS and REQUIRETLS.
 	yes := true
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, mf, nil, &yes)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<opportunistictls@localhost>", nil, &yes)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -545,9 +556,10 @@ func TestQueue(t *testing.T) {
 			{},
 		},
 	}
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneunusable@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -562,9 +574,10 @@ func TestQueue(t *testing.T) {
 			{Usage: adns.TLSAUsageDANEEE, Selector: adns.TLSASelectorSPKI, MatchType: adns.TLSAMatchTypeFull, CertAssoc: make([]byte, sha256.Size)},
 		},
 	}
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<daneinsecure@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -581,9 +594,10 @@ func TestQueue(t *testing.T) {
 
 	// Check that message is delivered with TLS-Required: No and non-matching DANE record.
 	no := false
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, mf, nil, &no)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednostarttls@localhost>", nil, &no)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -591,9 +605,10 @@ func TestQueue(t *testing.T) {
 	testDeliver(fakeSMTPSTARTTLSServer)
 
 	// Check that message is delivered with TLS-Required: No and bad TLS, falling back to plain text.
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, mf, nil, &no)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednoplaintext@localhost>", nil, &no)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -601,9 +616,10 @@ func TestQueue(t *testing.T) {
 	testDeliver(makeBadFakeSMTPSTARTTLSServer(true))
 
 	// Add message with requiretls that fails immediately due to no REQUIRETLS support in all servers.
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, mf, nil, &yes)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequiredunsupported@localhost>", nil, &yes)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -615,9 +631,10 @@ func TestQueue(t *testing.T) {
 	resolver.TLSA = nil
 
 	// Add message with requiretls that fails immediately due to no verification policy for recipient domain.
-	msgID, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, mf, nil, &yes)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<tlsrequirednopolicy@localhost>", nil, &yes)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
-	n, err = Kick(ctxbg, msgID, "", "", nil)
+	n, err = Kick(ctxbg, qm.ID, "", "", nil)
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -629,7 +646,8 @@ func TestQueue(t *testing.T) {
 	})
 
 	// Add another message that we'll fail to deliver entirely.
-	_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
+	qm = MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
 
 	msgs, err = List(ctxbg)
@@ -788,7 +806,8 @@ func TestQueueStart(t *testing.T) {
 	mf := prepareFile(t)
 	defer os.Remove(mf.Name())
 	defer mf.Close()
-	_, err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, mf, nil, nil)
+	qm := MakeMsg("mjl", path, path, false, false, int64(len(testmsg)), "<test@localhost>", nil, nil)
+	err = Add(ctxbg, xlog, &qm, mf)
 	tcheck(t, err, "add message to queue for delivery")
 	checkDialed(true)
 

serve.go

@@ -52,7 +52,7 @@ func shutdown(log *mlog.Log) {
 
 // start initializes all packages, starts all listeners and the switchboard
 // goroutine, then returns.
-func start(mtastsdbRefresher, skipForkExec bool) error {
+func start(mtastsdbRefresher, sendDMARCReports, skipForkExec bool) error {
 	smtpserver.Listen()
 	imapserver.Listen()
 	http.Listen()
@@ -69,10 +69,6 @@ func start(mtastsdbRefresher, skipForkExec bool) error {
 		}
 	}
 
-	if err := dmarcdb.Init(); err != nil {
-		return fmt.Errorf("dmarc init: %s", err)
-	}
-
 	if err := mtastsdb.Init(mtastsdbRefresher); err != nil {
 		return fmt.Errorf("mtasts init: %s", err)
 	}
@@ -86,6 +82,14 @@ func start(mtastsdbRefresher, skipForkExec bool) error {
 		return fmt.Errorf("queue start: %s", err)
 	}
 
+	// dmarcdb starts after queue because it may start sending reports through the queue.
+	if err := dmarcdb.Init(); err != nil {
+		return fmt.Errorf("dmarc init: %s", err)
+	}
+	if sendDMARCReports {
+		dmarcdb.Start(dns.StrictResolver{Pkg: "dmarcdb"})
+	}
+
 	store.StartAuthCache()
 	smtpserver.Serve()
 	imapserver.Serve()

serve_unix.go

@@ -224,7 +224,7 @@ Only implemented on unix systems, not Windows.
 	// taken.
 	const mtastsdbRefresher = true
 	const skipForkExec = false
-	if err := start(mtastsdbRefresher, skipForkExec); err != nil {
+	if err := start(mtastsdbRefresher, !mox.Conf.Static.NoOutgoingDMARCReports, skipForkExec); err != nil {
 		log.Fatalx("start", err)
 	}
 	log.Print("ready to serve")

smtp/address.go

@@ -107,6 +107,10 @@ func NewAddress(localpart Localpart, domain dns.Domain) Address {
 	return Address{localpart, domain}
 }
 
+func (a Address) Path() Path {
+	return Path{Localpart: a.Localpart, IPDomain: dns.IPDomain{Domain: a.Domain}}
+}
+
 func (a Address) IsZero() bool {
 	return a == Address{}
 }

smtpserver/analyze.go

@@ -44,9 +44,10 @@ type analysis struct {
 	userError           bool
 	errmsg              string
 	err                 error              // For our own logging, not sent to remote.
-	dmarcReport *dmarcrpt.Feedback // Validated dmarc aggregate report, not yet stored.
+	dmarcReport         *dmarcrpt.Feedback // Validated DMARC aggregate report, not yet stored.
 	tlsReport           *tlsrpt.Report     // Validated TLS report, not yet stored.
 	reason              string             // If non-empty, reason for this decision. Can be one of reputationMethod and a few other tokens.
+	dmarcOverrideReason string             // If set, one of dmarcrpt.PolicyOverride
 }
 
 const (
@@ -64,7 +65,7 @@ const (
 	reasonDNSBlocklisted    = "dns-blocklisted"
 	reasonSubjectpass       = "subjectpass"
 	reasonSubjectpassError  = "subjectpass-error"
-	reasonIPrev             = "iprev" // No or mil junk reputation signals, and bad iprev.
+	reasonIPrev             = "iprev" // No or mild junk reputation signals, and bad iprev.
 )
 
 func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delivery) analysis {
@@ -83,15 +84,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 		ld := rs.ListAllowDNSDomain
 		// todo: on temporary failures, reject temporarily?
 		if d.m.MailFromValidated && ld.Name() == d.m.MailFromDomain {
-			return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow}
+			return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList)}
 		}
 		for _, r := range d.dkimResults {
 			if r.Status == dkim.StatusPass && r.Sig.Domain == ld {
-				return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow}
+				return analysis{accept: true, mailbox: mailbox, reason: reasonListAllow, dmarcOverrideReason: string(dmarcrpt.PolicyOverrideMailingList)}
 			}
 		}
 	}
 
+	var dmarcOverrideReason string
+
 	// For forwarded messages, we have different junk analysis. We don't reject for
 	// failing DMARC, and we clear fields that could implicate the forwarding mail
 	// server during future classifications on incoming messages (the forwarding mail
@@ -113,6 +116,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 			}
 		}
 		d.m.DKIMDomains = dkimdoms
+		dmarcOverrideReason = string(dmarcrpt.PolicyOverrideForwarded)
 		log.Info("forwarded message, clearing identifying signals of forwarding mail server")
 	}
 
@@ -154,7 +158,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 				})
 			})
 			if mberr != nil {
-				return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError}
+				return analysis{false, mailbox, smtp.C451LocalErr, smtp.SeSys3Other0, false, "error processing", err, nil, nil, reasonReputationError, dmarcOverrideReason}
 			}
 			d.m.MailboxID = 0 // We plan to reject, no need to set intended MailboxID.
 		}
@@ -168,7 +172,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 			d.m.Seen = true
 			log.Info("accepting reject to configured mailbox due to ruleset")
 		}
-		return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason}
+		return analysis{accept, mailbox, code, secode, err == nil, errmsg, err, nil, nil, reason, dmarcOverrideReason}
 	}
 
 	if d.dmarcUse && d.dmarcResult.Reject {
@@ -180,17 +184,17 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 	// track of the report. We'll check reputation, defaulting to accept.
 	var dmarcReport *dmarcrpt.Feedback
 	if d.rcptAcc.destination.DMARCReports {
-		// Messages with DMARC aggregate reports must have a dmarc pass. ../rfc/7489:1866
+		// Messages with DMARC aggregate reports must have a DMARC pass. ../rfc/7489:1866
 		if d.dmarcResult.Status != dmarc.StatusPass {
-			log.Info("received dmarc report without dmarc pass, not processing as dmarc report")
+			log.Info("received dmarc aggregate report without dmarc pass, not processing as dmarc report")
 		} else if report, err := dmarcrpt.ParseMessageReport(log, store.FileMsgReader(d.m.MsgPrefix, d.dataFile)); err != nil {
-			log.Infox("parsing dmarc report", err)
+			log.Infox("parsing dmarc aggregate report", err)
 		} else if d, err := dns.ParseDomain(report.PolicyPublished.Domain); err != nil {
-			log.Infox("parsing domain in dmarc report", err)
+			log.Infox("parsing domain in dmarc aggregate report", err)
 		} else if _, ok := mox.Conf.Domain(d); !ok {
-			log.Info("dmarc report for domain not configured, ignoring", mlog.Field("domain", d))
+			log.Info("dmarc aggregate report for domain not configured, ignoring", mlog.Field("domain", d))
 		} else if report.ReportMetadata.DateRange.End > time.Now().Unix()+60 {
-			log.Info("dmarc report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
+			log.Info("dmarc aggregate report with end date in the future, ignoring", mlog.Field("domain", d), mlog.Field("end", time.Unix(report.ReportMetadata.DateRange.End, 0)))
 		} else {
 			dmarcReport = report
 		}
@@ -261,12 +265,12 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 	log.Info("reputation analyzed", mlog.Field("conclusive", conclusive), mlog.Field("isjunk", isjunk), mlog.Field("method", string(method)))
 	if conclusive {
 		if !*isjunk {
-			return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason}
+			return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reason, dmarcOverrideReason: dmarcOverrideReason}
 		}
 		return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, string(method))
 	} else if dmarcReport != nil || tlsReport != nil {
-		log.Info("accepting dmarc reporting or tlsrpt message without reputation")
+		log.Info("accepting message with dmarc aggregate report or tls report without reputation")
-		return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting}
+		return analysis{accept: true, mailbox: mailbox, dmarcReport: dmarcReport, tlsReport: tlsReport, reason: reasonReporting, dmarcOverrideReason: dmarcOverrideReason}
 	}
 	// If there was no previous message from sender or its domain, and we have an SPF
 	// (soft)fail, reject the message.
@@ -302,7 +306,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 		pass := err == nil
 		log.Infox("pass by subject token", err, mlog.Field("pass", pass))
 		if pass {
-			return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass}
+			return analysis{accept: true, mailbox: mailbox, reason: reasonSubjectpass, dmarcOverrideReason: dmarcOverrideReason}
 		}
 	}
 
@@ -382,7 +386,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 	}
 
 	if accept {
-		return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals}
+		return analysis{accept: true, mailbox: mailbox, reason: reasonNoBadSignals, dmarcOverrideReason: dmarcOverrideReason}
 	}
 
 	if subjectpassKey != "" && d.dmarcResult.Status == dmarc.StatusPass && method == methodNone && (dnsblocklisted || junkSubjectpass) {

smtpserver/dsn.go

@@ -50,7 +50,9 @@ func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message, req
 	if requireTLS {
 		reqTLS = &requireTLS
 	}
-	if _, err := queue.Add(ctx, c.log, "", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, f, bufUTF8, reqTLS); err != nil {
+	qm := queue.MakeMsg("", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), m.MessageID, nil, reqTLS)
+	qm.DSNUTF8 = bufUTF8
+	if err := queue.Add(ctx, c.log, &qm, f); err != nil {
 		return err
 	}
 	return nil

smtpserver/server.go

@@ -37,6 +37,7 @@ import (
 	"github.com/mjl-/mox/dkim"
 	"github.com/mjl-/mox/dmarc"
 	"github.com/mjl-/mox/dmarcdb"
+	"github.com/mjl-/mox/dmarcrpt"
 	"github.com/mjl-/mox/dns"
 	"github.com/mjl-/mox/dsn"
 	"github.com/mjl-/mox/iprev"
@@ -1835,7 +1836,8 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
 		xmsgPrefix := append([]byte(recvHdrFor(rcptAcc.rcptTo.String())), msgPrefix...)
 
 		msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
-		if _, err := queue.Add(ctx, c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, dataFile, nil, c.requireTLS); err != nil {
+		qm := queue.MakeMsg(c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, messageID, xmsgPrefix, c.requireTLS)
+		if err := queue.Add(ctx, c.log, &qm, dataFile); err != nil {
 			// Aborting the transaction is not great. But continuing and generating DSNs will
 			// probably result in errors as well...
 			metricSubmission.WithLabelValues("queueerror").Inc()
@@ -2065,7 +2067,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 		var comment string
 		var props []message.AuthProp
 		if r.Sig != nil {
-			// todo future: also specify whether dns record was dnssec-signed.
 			if r.Record != nil && r.Record.PublicKey != nil {
 				if pubkey, ok := r.Record.PublicKey.(*rsa.PublicKey); ok {
 					comment = fmt.Sprintf("%d bit rsa, ", pubkey.N.BitLen())
@@ -2167,6 +2168,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 	var dmarcUse bool
 	var dmarcResult dmarc.Result
 	const applyRandomPercentage = true
+	// dmarcMethod is added to authResults when delivering to recipients: accounts can
+	// have different policy override rules.
 	var dmarcMethod message.AuthMethod
 	var msgFromValidation = store.ValidationNone
 	if msgFrom.IsZero() {
@@ -2178,6 +2181,15 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 	} else {
 		msgFromValidation = alignment(ctx, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity)
 
+		// We are doing the DMARC evaluation now. But we only store it for inclusion in an
+		// aggregate report when we actually use it. We use an evaluation for each
+		// recipient, with each a potentially different result due to mailing
+		// list/forwarding configuration. If we reject a message due to being spam, we
+		// don't want to spend any resources for the sender domain, and we don't want to
+		// give the sender any more information about us, so we won't record the
+		// evaluation.
+		// todo future: also not send for first-time senders? they could be spammers getting through our filter, don't want to give them insights either. though we currently would have no reasonable way to decide if they are still reputationless at the time we are composing/sending aggregate reports.
+
 		dmarcctx, dmarccancel := context.WithTimeout(ctx, time.Minute)
 		defer dmarccancel()
 		dmarcUse, dmarcResult = dmarc.Verify(dmarcctx, c.resolver, msgFrom.Domain, dkimResults, receivedSPF.Result, spfIdentity, applyRandomPercentage)
@@ -2202,9 +2214,8 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 			msgFromValidation = store.ValidationDMARC
 		}
 
-		// todo future: consider enforcing an spf fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507
+		// todo future: consider enforcing an spf (soft)fail if there is no dmarc policy or the dmarc policy is none. ../rfc/7489:1507
 	}
-	authResults.Methods = append(authResults.Methods, dmarcMethod)
 	c.log.Debug("dmarc verification", mlog.Field("result", dmarcResult.Status), mlog.Field("domain", msgFrom.Domain))
 
 	// Prepare for analyzing content, calculating reputation.
@@ -2366,16 +2377,6 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 			continue
 		}
 
-		// ../rfc/5321:3204
-		// Received-SPF header goes before Received. ../rfc/7208:2038
-		msgPrefix := []byte(
-			"Delivered-To: " + rcptAcc.rcptTo.XString(c.smtputf8) + "\r\n" + // ../rfc/9228:274
-				"Return-Path: <" + c.mailFrom.String() + ">\r\n" + // ../rfc/5321:3300
-				authResults.Header() +
-				receivedSPF.Header() +
-				recvHdrFor(rcptAcc.rcptTo.String()),
-		)
-
 		m := &store.Message{
 			Received:           time.Now(),
 			RemoteIP:           c.remoteIP.String(),
@@ -2398,16 +2399,187 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 			MailFromValidation: mailFromValidation,
 			MsgFromValidation:  msgFromValidation,
 			DKIMDomains:        verifiedDKIMDomains,
-			Size:               int64(len(msgPrefix)) + msgWriter.Size,
+			Size:               msgWriter.Size,
-			MsgPrefix:          msgPrefix,
 		}
 		d := delivery{m, dataFile, rcptAcc, acc, msgFrom, c.dnsBLs, dmarcUse, dmarcResult, dkimResults, iprevStatus}
 		a := analyze(ctx, log, c.resolver, d)
-		if a.reason != "" {
+
-			xmoxreason := "X-Mox-Reason: " + a.reason + "\r\n"
+		// Any DMARC result override is stored in the evaluation for outgoing DMARC
-			m.MsgPrefix = append([]byte(xmoxreason), m.MsgPrefix...)
+		// aggregate reports, and added to the Authentication-Results message header.
-			m.Size += int64(len(xmoxreason))
+		var dmarcOverride string
+		if dmarcResult.Record != nil {
+			if !dmarcUse {
+				dmarcOverride = string(dmarcrpt.PolicyOverrideSampledOut)
+			} else if a.dmarcOverrideReason != "" && (a.accept && !m.IsReject) == dmarcResult.Reject {
+				dmarcOverride = a.dmarcOverrideReason
 			}
+		}
+
+		// Add per-recipient DMARC method to Authentication-Results. Each account can have
+		// their own override rules, e.g. based on configured mailing lists/forwards.
+		// ../rfc/7489:1486
+		rcptDMARCMethod := dmarcMethod
+		if dmarcOverride != "" {
+			if rcptDMARCMethod.Comment != "" {
+				rcptDMARCMethod.Comment += ", "
+			}
+			rcptDMARCMethod.Comment += "override " + dmarcOverride
+		}
+		rcptAuthResults := authResults
+		rcptAuthResults.Methods = append([]message.AuthMethod{}, authResults.Methods...)
+		rcptAuthResults.Methods = append(rcptAuthResults.Methods, rcptDMARCMethod)
+
+		// Prepend reason as message header, for easy display in mail clients.
+		var xmoxreason string
+		if a.reason != "" {
+			xmoxreason = "X-Mox-Reason: " + a.reason + "\r\n"
+		}
+
+		// ../rfc/5321:3204
+		// Received-SPF header goes before Received. ../rfc/7208:2038
+		m.MsgPrefix = []byte(
+			xmoxreason +
+				"Delivered-To: " + rcptAcc.rcptTo.XString(c.smtputf8) + "\r\n" + // ../rfc/9228:274
+				"Return-Path: <" + c.mailFrom.String() + ">\r\n" + // ../rfc/5321:3300
+				rcptAuthResults.Header() +
+				receivedSPF.Header() +
+				recvHdrFor(rcptAcc.rcptTo.String()),
+		)
+		m.Size += int64(len(m.MsgPrefix))
+
+		// Store DMARC evaluation for inclusion in an aggregate report. Only if there is at
+		// least one reporting address: We don't want to needlessly store a row in a
+		// database for each delivery attempt. If we reject a message for being junk, we
+		// are also not going to send it a DMARC report. The DMARC check is done early in
+		// the analysis, we will report on rejects because of DMARC, because it could be
+		// valuable feedback about forwarded or mailing list messages.
+		// ../rfc/7489:1492
+		if !mox.Conf.Static.NoOutgoingDMARCReports && dmarcResult.Record != nil && len(dmarcResult.Record.AggregateReportAddresses) > 0 && (a.accept && !m.IsReject || a.reason == reasonDMARCPolicy) {
+			// Disposition holds our decision on whether to accept the message. Not what the
+			// DMARC evaluation resulted in. We can override, e.g. because of mailing lists,
+			// forwarding, or local policy.
+			// We treat quarantine as reject, so never claim to quarantine.
+			// ../rfc/7489:1691
+			disposition := dmarcrpt.DispositionNone
+			if !a.accept {
+				disposition = dmarcrpt.DispositionReject
+			}
+
+			// unknownDomain returns whether the sender is domain with which this account has
+			// not had positive interaction.
+			unknownDomain := func() (unknown bool) {
+				err := acc.DB.Read(ctx, func(tx *bstore.Tx) (err error) {
+					// See if we received a non-junk message from this organizational domain.
+					q := bstore.QueryTx[store.Message](tx)
+					q.FilterNonzero(store.Message{MsgFromOrgDomain: m.MsgFromOrgDomain})
+					q.FilterEqual("Notjunk", false)
+					exists, err := q.Exists()
+					if err != nil {
+						return fmt.Errorf("querying for non-junk message from organizational domain: %v", err)
+					}
+					if exists {
+						return nil
+					}
+
+					// See if we sent a message to this organizational domain.
+					qr := bstore.QueryTx[store.Recipient](tx)
+					qr.FilterNonzero(store.Recipient{OrgDomain: m.MsgFromOrgDomain})
+					exists, err = qr.Exists()
+					if err != nil {
+						return fmt.Errorf("querying for message sent to organizational domain: %v", err)
+					}
+					if !exists {
+						unknown = true
+					}
+					return nil
+				})
+				if err != nil {
+					log.Errorx("checking if sender is unknown domain, for dmarc aggregate report evaluation", err)
+				}
+				return
+			}
+
+			r := dmarcResult.Record
+			addresses := make([]string, len(r.AggregateReportAddresses))
+			for i, a := range r.AggregateReportAddresses {
+				addresses[i] = a.String()
+			}
+			sp := dmarcrpt.Disposition(r.SubdomainPolicy)
+			if r.SubdomainPolicy == dmarc.PolicyEmpty {
+				sp = dmarcrpt.Disposition(r.Policy)
+			}
+			eval := dmarcdb.Evaluation{
+				// Evaluated and IntervalHours set by AddEvaluation.
+				PolicyDomain: dmarcResult.Domain.Name(),
+
+				// Optional evaluations don't cause a report to be sent, but will be included.
+				// Useful for automated inter-mailer messages, we don't want to get in a reporting
+				// loop. We also don't want to be used for sending reports to unsuspecting domains
+				// we have no relation with.
+				// todo: would it make sense to also mark some percentage of mailing-list-policy-overrides optional? to lower the load on mail servers of folks sending to large mailing lists.
+				Optional: rcptAcc.destination.DMARCReports || rcptAcc.destination.TLSReports || a.reason == reasonDMARCPolicy && unknownDomain(),
+
+				Addresses: addresses,
+
+				PolicyPublished: dmarcrpt.PolicyPublished{
+					Domain:          dmarcResult.Domain.Name(),
+					ADKIM:           dmarcrpt.Alignment(r.ADKIM),
+					ASPF:            dmarcrpt.Alignment(r.ASPF),
+					Policy:          dmarcrpt.Disposition(r.Policy),
+					SubdomainPolicy: sp,
+					Percentage:      r.Percentage,
+					// We don't save ReportingOptions, we don't do per-message failure reporting.
+				},
+				SourceIP:        c.remoteIP.String(),
+				Disposition:     disposition,
+				AlignedDKIMPass: dmarcResult.AlignedDKIMPass,
+				AlignedSPFPass:  dmarcResult.AlignedSPFPass,
+				EnvelopeTo:      rcptAcc.rcptTo.IPDomain.String(),
+				EnvelopeFrom:    c.mailFrom.IPDomain.String(),
+				HeaderFrom:      msgFrom.Domain.Name(),
+			}
+
+			if dmarcOverride != "" {
+				eval.OverrideReasons = []dmarcrpt.PolicyOverrideReason{
+					{Type: dmarcrpt.PolicyOverride(dmarcOverride)},
+				}
+			}
+
+			// We'll include all signatures for the organizational domain, even if they weren't
+			// relevant due to strict alignment requirement.
+			for _, dkimResult := range dkimResults {
+				if dkimResult.Sig == nil || publicsuffix.Lookup(ctx, msgFrom.Domain) != publicsuffix.Lookup(ctx, dkimResult.Sig.Domain) {
+					continue
+				}
+				r := dmarcrpt.DKIMAuthResult{
+					Domain:   dkimResult.Sig.Domain.Name(),
+					Selector: dkimResult.Sig.Selector.ASCII,
+					Result:   dmarcrpt.DKIMResult(dkimResult.Status),
+				}
+				eval.DKIMResults = append(eval.DKIMResults, r)
+			}
+
+			switch receivedSPF.Identity {
+			case spf.ReceivedHELO:
+				spfAuthResult := dmarcrpt.SPFAuthResult{
+					Domain: spfArgs.HelloDomain.String(), // Can be unicode and also IP.
+					Scope:  dmarcrpt.SPFDomainScopeHelo,
+					Result: dmarcrpt.SPFResult(receivedSPF.Result),
+				}
+				eval.SPFResults = []dmarcrpt.SPFAuthResult{spfAuthResult}
+			case spf.ReceivedMailFrom:
+				spfAuthResult := dmarcrpt.SPFAuthResult{
+					Domain: spfArgs.MailFromDomain.Name(), // Can be unicode.
+					Scope:  dmarcrpt.SPFDomainScopeMailFrom,
+					Result: dmarcrpt.SPFResult(receivedSPF.Result),
+				}
+				eval.SPFResults = []dmarcrpt.SPFAuthResult{spfAuthResult}
+			}
+
+			err := dmarcdb.AddEvaluation(ctx, dmarcResult.Record.AggregateReportingInterval, &eval)
+			log.Check(err, "adding dmarc evaluation to database for aggregate report")
+		}
+
 		if !a.accept {
 			conf, _ := acc.Conf()
 			if conf.RejectsMailbox != "" {
@@ -2455,9 +2627,9 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 		if a.dmarcReport != nil {
 			// todo future: add rate limiting to prevent DoS attacks. ../rfc/7489:2570
 			if err := dmarcdb.AddReport(ctx, a.dmarcReport, msgFrom.Domain); err != nil {
-				log.Errorx("saving dmarc report in database", err)
+				log.Errorx("saving dmarc aggregate report in database", err)
 			} else {
-				log.Info("dmarc report processed")
+				log.Info("dmarc aggregate report processed")
 				m.Flags.Seen = true
 				delayFirstTime = false
 			}

smtpserver/server_test.go

@@ -100,6 +100,11 @@ func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *test
 
 	ts := testserver{t: t, cid: 1, resolver: resolver, tlsmode: smtpclient.TLSOpportunistic}
 
+	if dmarcdb.EvalDB != nil {
+		dmarcdb.EvalDB.Close()
+		dmarcdb.EvalDB = nil
+	}
+
 	mox.Context = ctxbg
 	mox.ConfigStaticPath = configPath
 	mox.MustLoadConfig(true, false)
@@ -192,6 +197,15 @@ func fakeCert(t *testing.T) tls.Certificate {
 	return c
 }
 
+// check expected dmarc evaluations for outgoing aggregate reports.
+func checkEvaluationCount(t *testing.T, n int) []dmarcdb.Evaluation {
+	t.Helper()
+	l, err := dmarcdb.Evaluations(ctxbg)
+	tcheck(t, err, "get dmarc evaluations")
+	tcompare(t, len(l), n)
+	return l
+}
+
 // Test submission from authenticated user.
 func TestSubmission(t *testing.T) {
 	ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/mox.conf"), dns.MockResolver{})
@@ -229,6 +243,7 @@ func TestSubmission(t *testing.T) {
 			if expErr == nil && err != nil || expErr != nil && (err == nil || !errors.As(err, &cerr) || cerr.Secode != expErr.Secode) {
 				t.Fatalf("got err %#v (%q), expected %#v", err, err, expErr)
 			}
+			checkEvaluationCount(t, 0)
 		})
 	}
 
@@ -329,6 +344,8 @@ func TestDelivery(t *testing.T) {
 			t.Fatalf("no delivery in 1s")
 		}
 	})
+
+	checkEvaluationCount(t, 0)
 }
 
 func tinsertmsg(t *testing.T, acc *store.Account, mailbox string, m *store.Message, msg string) {
@@ -392,7 +409,7 @@ func TestSpam(t *testing.T) {
 		},
 		TXT: map[string][]string{
 			"example.org.":        {"v=spf1 ip4:127.0.0.10 -all"},
-			"_dmarc.example.org.": {"v=DMARC1;p=reject"},
+			"_dmarc.example.org.": {"v=DMARC1;p=reject; rua=mailto:dmarcrpt@example.org"},
 		},
 	}
 	ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver)
@@ -451,6 +468,7 @@ func TestSpam(t *testing.T) {
 		}
 
 		checkCount("Rejects", 1)
+		checkEvaluationCount(t, 0) // No positive interactions yet.
 	})
 
 	// Delivery from sender with bad reputation matching AcceptRejectsToMailbox should
@@ -465,6 +483,7 @@ func TestSpam(t *testing.T) {
 
 		checkCount("mjl2junk", 1)  // In ruleset rejects mailbox.
 		checkCount("Rejects", 1)   // Same as before.
+		checkEvaluationCount(t, 0) // This is not an actual accept.
 	})
 
 	// Mark the messages as having good reputation.
@@ -485,6 +504,7 @@ func TestSpam(t *testing.T) {
 		// Message should now be removed from Rejects mailboxes.
 		checkCount("Rejects", 0)
 		checkCount("mjl2junk", 1)
+		checkEvaluationCount(t, 1)
 	})
 
 	// Undo dmarc pass, mark messages as junk, and train the filter.
@@ -506,6 +526,7 @@ func TestSpam(t *testing.T) {
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
 			t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
 		}
+		checkEvaluationCount(t, 1) // No new evaluation, this isn't a DMARC reject.
 	})
 }
 
@@ -525,9 +546,9 @@ func TestForward(t *testing.T) {
 				"bad.example.":            {"v=spf1 ip4:127.0.0.1 -all"},
 				"good.example.":           {"v=spf1 ip4:127.0.0.1 -all"},
 				"forward.example.":        {"v=spf1 ip4:127.0.0.10 -all"},
-				"_dmarc.bad.example.":     {"v=DMARC1;p=reject"},
+				"_dmarc.bad.example.":     {"v=DMARC1;p=reject; rua=mailto:dmarc@bad.example"},
-				"_dmarc.good.example.":    {"v=DMARC1;p=reject"},
+				"_dmarc.good.example.":    {"v=DMARC1;p=reject; rua=mailto:dmarc@good.example"},
-				"_dmarc.forward.example.": {"v=DMARC1;p=reject"},
+				"_dmarc.forward.example.": {"v=DMARC1;p=reject; rua=mailto:dmarc@forward.example"},
 			},
 			PTR: map[string][]string{
 				"127.0.0.10": {"forward.example."}, // For iprev check.
@@ -544,6 +565,8 @@ func TestForward(t *testing.T) {
 		ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver)
 		defer ts.close()
 
+		totalEvaluations := 0
+
 		var msgBad = strings.ReplaceAll(`From: <remote@bad.example>
 To: <mjl3@mox.example>
 Subject: test
@@ -580,6 +603,7 @@ happens to come from forwarding mail server.
 				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgBad)), strings.NewReader(msgBad), false, false, false)
 				tcheck(t, err, "deliver message")
 			}
+			totalEvaluations += 10
 
 			n, err := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB).UpdateFields(map[string]any{"Junk": true, "MsgFromValidated": true})
 			tcheck(t, err, "marking messages as junk")
@@ -591,6 +615,8 @@ happens to come from forwarding mail server.
 			if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
 				t.Fatalf("delivery by bad sender, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
 			}
+
+			checkEvaluationCount(t, totalEvaluations)
 		})
 
 		// Delivery from different "message From" without reputation, but from same
@@ -607,12 +633,14 @@ happens to come from forwarding mail server.
 			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgOK)), strings.NewReader(msgOK), false, false, false)
 			if forward {
 				tcheck(t, err, "deliver")
+				totalEvaluations += 1
 			} else {
 				var cerr smtpclient.Error
 				if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
 					t.Fatalf("delivery by bad ip, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
 				}
 			}
+			checkEvaluationCount(t, totalEvaluations)
 		})
 
 		// Delivery from forwarding server that isn't a forward should get same treatment.
@@ -624,12 +652,14 @@ happens to come from forwarding mail server.
 			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msgOK2)), strings.NewReader(msgOK2), false, false, false)
 			if forward {
 				tcheck(t, err, "deliver")
+				totalEvaluations += 1
 			} else {
 				var cerr smtpclient.Error
 				if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
 					t.Fatalf("delivery by bad ip, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
 				}
 			}
+			checkEvaluationCount(t, totalEvaluations)
 		})
 	}
 
@@ -644,13 +674,31 @@ func TestDMARCSent(t *testing.T) {
 			"example.org.": {"127.0.0.1"}, // For mx check.
 		},
 		TXT: map[string][]string{
-			"example.org.":        {"v=spf1 ip4:127.0.0.10 -all"},
+			"example.org.":        {"v=spf1 ip4:127.0.0.1 -all"},
-			"_dmarc.example.org.": {"v=DMARC1;p=reject"},
+			"_dmarc.example.org.": {"v=DMARC1;p=reject;rua=mailto:dmarcrpt@example.org"},
 		},
 	}
 	ts := newTestServer(t, filepath.FromSlash("../testdata/smtp/junk/mox.conf"), resolver)
 	defer ts.close()
 
+	// First check that DMARC policy rejects message and results in optional evaluation.
+	ts.run(func(err error, client *smtpclient.Client) {
+		mailFrom := "remote@example.org"
+		rcptTo := "mjl@mox.example"
+		if err == nil {
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false)
+		}
+		var cerr smtpclient.Error
+		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
+			t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C550MailboxUnavail)
+		}
+		l := checkEvaluationCount(t, 1)
+		tcompare(t, l[0].Optional, true)
+	})
+
+	// Update DNS for an SPF pass, and DMARC pass.
+	resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.10 -all"}
+
 	// Insert spammy messages not related to the test message.
 	m := store.Message{
 		MailFrom:        "remote@test.example",
@@ -676,6 +724,7 @@ func TestDMARCSent(t *testing.T) {
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
 			t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C451LocalErr)
 		}
+		checkEvaluationCount(t, 1) // No new evaluation.
 	})
 
 	// Insert a message that we sent to the address that is about to send to us.
@@ -684,7 +733,26 @@ func TestDMARCSent(t *testing.T) {
 	err := ts.acc.DB.Insert(ctxbg, &store.Recipient{MessageID: sentMsg.ID, Localpart: "remote", Domain: "example.org", OrgDomain: "example.org", Sent: time.Now()})
 	tcheck(t, err, "inserting message recipient")
 
+	// Reject a message due to DMARC again. Since we sent a message to the domain, it
+	// is no longer unknown and we should see a non-optional evaluation that will
+	// result in a DMARC report.
+	resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.1 -all"}
+	ts.run(func(err error, client *smtpclient.Client) {
+		mailFrom := "remote@example.org"
+		rcptTo := "mjl@mox.example"
+		if err == nil {
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false)
+		}
+		var cerr smtpclient.Error
+		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
+			t.Fatalf("attempt to deliver spamy message, got err %v, expected smtpclient.Error with code %d", err, smtp.C550MailboxUnavail)
+		}
+		l := checkEvaluationCount(t, 2) // New evaluation.
+		tcompare(t, l[1].Optional, false)
+	})
+
 	// We should now be accepting the message because we recently sent a message.
+	resolver.TXT["example.org."] = []string{"v=spf1 ip4:127.0.0.10 -all"}
 	ts.run(func(err error, client *smtpclient.Client) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
@@ -692,6 +760,8 @@ func TestDMARCSent(t *testing.T) {
 			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false, false)
 		}
 		tcheck(t, err, "deliver")
+		l := checkEvaluationCount(t, 3) // New evaluation.
+		tcompare(t, l[2].Optional, false)
 	})
 }
 
@@ -773,7 +843,7 @@ func TestDMARCReport(t *testing.T) {
 		},
 		TXT: map[string][]string{
 			"example.org.":        {"v=spf1 ip4:127.0.0.10 -all"},
-			"_dmarc.example.org.": {"v=DMARC1;p=reject"},
+			"_dmarc.example.org.": {"v=DMARC1;p=reject; rua=mailto:dmarcrpt@example.org"},
 		},
 		PTR: map[string][]string{
 			"127.0.0.10": {"example.org."}, // For iprev check.
@@ -815,6 +885,11 @@ func TestDMARCReport(t *testing.T) {
 
 	run(dmarcReport, 0)
 	run(strings.ReplaceAll(dmarcReport, "xmox.nl", "mox.example"), 1)
+
+	// We always store as an evaluation, but as optional for reports.
+	evals := checkEvaluationCount(t, 2)
+	tcompare(t, evals[0].Optional, true)
+	tcompare(t, evals[1].Optional, true)
 }
 
 const dmarcReport = `<?xml version="1.0" encoding="UTF-8" ?>
@@ -896,7 +971,7 @@ func TestTLSReport(t *testing.T) {
 		TXT: map[string][]string{
 			"testsel._domainkey.example.org.": {dkimTxt},
 			"example.org.":                    {"v=spf1 ip4:127.0.0.10 -all"},
-			"_dmarc.example.org.":             {"v=DMARC1;p=reject"},
+			"_dmarc.example.org.":             {"v=DMARC1;p=reject;rua=mailto:dmarcrpt@example.org"},
 		},
 		PTR: map[string][]string{
 			"127.0.0.10": {"example.org."}, // For iprev check.
@@ -939,6 +1014,11 @@ func TestTLSReport(t *testing.T) {
 
 	run(tlsrpt, 0)
 	run(strings.ReplaceAll(tlsrpt, "xmox.nl", "mox.example"), 1)
+
+	// We always store as an evaluation, but as optional for reports.
+	evals := checkEvaluationCount(t, 2)
+	tcompare(t, evals[0].Optional, true)
+	tcompare(t, evals[1].Optional, true)
 }
 
 func TestRatelimitConnectionrate(t *testing.T) {

store/cleanuptemp.go

@@ -0,0 +1,17 @@
+package store
+
+import (
+	"os"
+
+	"github.com/mjl-/mox/mlog"
+)
+
+// CloseRemoveTempFile closes and removes f, a file described by descr. Often
+// used in a defer after creating a temporary file.
+func CloseRemoveTempFile(log *mlog.Log, f *os.File, descr string) {
+	name := f.Name()
+	err := f.Close()
+	log.Check(err, "closing temporary file", mlog.Field("kind", descr))
+	err = os.Remove(name)
+	log.Check(err, "removing temporary file", mlog.Field("kind", descr))
+}

testdata/dmarcdb/domains.conf

@@ -0,0 +1,28 @@
+Domains:
+	mox.example:
+		DKIM:
+			Selectors:
+				testsel:
+					PrivateKeyFile: testsel.rsakey.pkcs8.pem
+			Sign:
+				- testsel
+Accounts:
+	other:
+		Domain: mox.example
+		Destinations:
+			other@mox.example: nil
+	mjl:
+		MaxOutgoingMessagesPerDay: 30
+		MaxFirstTimeRecipientsPerDay: 10
+		Domain: mox.example
+		Destinations:
+			mjl@mox.example: nil
+			møx@mox.example: nil
+		RejectsMailbox: Rejects
+		JunkFilter:
+			Threshold: 0.95
+			Params:
+				Twograms: true
+				MaxPower: 0.1
+				TopWords: 10
+				IgnoreWords: 0.1

testdata/dmarcdb/mox.conf

@@ -0,0 +1,11 @@
+DataDir: data
+User: 1000
+LogLevel: trace
+Hostname: mail.mox.example
+Listeners:
+	local:
+		IPs:
+			- 0.0.0.0
+Postmaster:
+	Account: mjl
+	Mailbox: postmaster

testdata/dmarcdb/testsel.rsakey.pkcs8.pem

@@ -0,0 +1,30 @@
+-----BEGIN PRIVATE KEY-----
+Note: RSA private key for use with DKIM, generated by mox
+
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDdkh3fKzvRUWym
+n9UwVrEw6s2Mc0+DTg04TWJKGKHXpvcTHuEcE6ALVS9MZKasyVsIHU7FNeS9/qNb
+pLihhGdlhU3KAfrMpTBhiFpJoYiDXED98Of4iBxNHIuheLMxSBSClMbLGE2vAgha
+/6LuONuzdMqk/c1TijBD+vGjCZI2qD58cgXWWKRK9e+WNhKNoVdedZ9iJtbtN0MI
+UWk3iwHmjXf5qzS7i8vDoy86Ln0HW0vKl7UtwemLVv09/E23OdNN163eQvSlrEhx
+a0odPQsM9SizxhiaI9rmcZtSqULt37hhPaNA+/AbELCzWijZPDqePVRqKGd5gYDK
+8STLj0UHAgMBAAECggEBAKVkJJgplYUx2oCmXmSu0aVKIBTvHjNNV+DnIq9co7Ju
+F5BWRILIw3ayJ5RGrYPc6e6ssdfT2uNX6GjIFGm8g9HsJ5zazXNk+zBSr9K2mUg0
+3O6xnPaP41BMNo5ZoqjuvSCcHagMhDBWvBXxLJXWK2lRjNKMAXCSfmTANQ8WXeYd
+XG2nYTPtBu6UgY8W6sKAx1xetxBrzk8q6JTxb5eVG22BSiUniWYif+XVmAj1u6TH
+0m6X0Kb6zsMYYgKPC2hmDsxD3uZ7qBNxxJzzLjpK6eP9aeFKzNyfnaoO4s+9K6Di
+31oxTBpqLI4dcrvg4xWl+YkEknXXaomMqM8hyDzfcAECgYEA9/zmjRpoTAoY3fu9
+mn16wxReFXZZZhqV0+c+gyYtao2Kf2pUNAdhD62HQv7KtAPPHKvLfL8PH0u7bzK0
+vVNzBUukwxGI7gsoTMdc3L5x4v9Yb6jUx7RrDZn93sDod/1f/sb56ARCFQoqbUck
+dSjnVUyF/l5oeh6CgKhvtghJ/AcCgYEA5Lq4kL82qWjIuNUT/C3lzjPfQVU+WvQ9
+wa+x4B4mxm5r4na3AU1T8H+peh4YstAJUgscGfYnLzxuMGuP1ReIuWYy29eDptKl
+WTzVZDcZrAPciP1FOL6jm03PT2UAEuoPRr4OHLg8DxoOqG8pxqk1izDSHG2Tof6l
+0ToafeIALwECgYEA8wvLTgnOpI/U1WNP7aUDd0Rz/WbzsW1m4Lsn+lOleWPllIE6
+q4974mi5Q8ECG7IL/9aj5cw/XvXTauVwXIn4Ff2QKpr58AvBYJaX/cUtS0PlgfIf
+MOczcK43MWUxscADoGmVLn9V4NcIw/dQ1P7U0zXfsXEHxoA2eTAb5HV1RWsCgYBd
+TcXoVfgIV1Q6AcGrR1XNLd/OmOVc2PEwR2l6ERKkM3sS4HZ6s36gRpNt20Ub/D0x
+GJMYDA+j9zTDz7zWokkFyCjLATkVHiyRIH2z6b4xK0oVH6vTIAFBYxZEPuEu1gfx
+RaogEQ9+4ZRFJUOXZIMRCpNLQW/Nz0D4/oi7/SsyAQKBgHEA27Js8ivt+EFCBjwB
+UbkW+LonDAXuUbw91lh5jICCigqUg73HNmV5xpoYI9JNPc6fy6wLyInVUC2w9tpO
+eH2Rl8n79vQMLbzsFClGEC/Q1kAbK5bwUjlfvKBZjvE0RknWX9e1ZY04DSsunSrM
+prS2eHVZ24hecd7j9XfAbHLC
+-----END PRIVATE KEY-----

verifydata.go

@@ -97,9 +97,12 @@ possibly making them potentially no longer readable by the previous version.
 
 	// Check a database file by opening it with BoltDB and bstore and lightly checking
 	// its contents.
-	checkDB := func(path string, types []any) {
+	checkDB := func(required bool, path string, types []any) {
 		_, err := os.Stat(path)
-		checkf(err, path, "checking if file exists")
+		if !required && err != nil && errors.Is(err, fs.ErrNotExist) {
+			return
+		}
+		checkf(err, path, "checking if database file exists")
 		if err != nil {
 			return
 		}
@@ -156,7 +159,7 @@ possibly making them potentially no longer readable by the previous version.
 
 	checkQueue := func() {
 		dbpath := filepath.Join(dataDir, "queue/index.db")
-		checkDB(dbpath, queue.DBTypes)
+		checkDB(true, dbpath, queue.DBTypes)
 
 		// Check that all messages present in the database also exist on disk.
 		seen := map[string]struct{}{}
@@ -222,12 +225,12 @@ possibly making them potentially no longer readable by the previous version.
 	// Check an account, with its database file and messages.
 	checkAccount := func(name string) {
 		accdir := filepath.Join(dataDir, "accounts", name)
-		checkDB(filepath.Join(accdir, "index.db"), store.DBTypes)
+		checkDB(true, filepath.Join(accdir, "index.db"), store.DBTypes)
 
 		jfdbpath := filepath.Join(accdir, "junkfilter.db")
 		jfbloompath := filepath.Join(accdir, "junkfilter.bloom")
 		if exists(jfdbpath) || exists(jfbloompath) {
-			checkDB(jfdbpath, junk.DBTypes)
+			checkDB(true, jfdbpath, junk.DBTypes)
 		}
 		// todo: add some kind of check for the bloom filter?
 
@@ -399,7 +402,7 @@ possibly making them potentially no longer readable by the previous version.
 				p = p[len(dataDir)+1:]
 			}
 			switch p {
-			case "dmarcrpt.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion":
+			case "dmarcrpt.db", "dmarceval.db", "mtasts.db", "tlsrpt.db", "receivedid.key", "lastknownversion":
 				return nil
 			case "acme", "queue", "accounts", "tmp", "moved":
 				return fs.SkipDir
@@ -417,9 +420,10 @@ possibly making them potentially no longer readable by the previous version.
 		checkf(err, dataDir, "walking data directory")
 	}
 
-	checkDB(filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.DBTypes)
+	checkDB(true, filepath.Join(dataDir, "dmarcrpt.db"), dmarcdb.ReportsDBTypes)
-	checkDB(filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)
+	checkDB(false, filepath.Join(dataDir, "dmarceval.db"), dmarcdb.EvalDBTypes) // After v0.0.7.
-	checkDB(filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes)
+	checkDB(true, filepath.Join(dataDir, "mtasts.db"), mtastsdb.DBTypes)
+	checkDB(true, filepath.Join(dataDir, "tlsrpt.db"), tlsrptdb.DBTypes)
 	checkQueue()
 	checkAccounts()
 	checkOther()

webadmin/admin.go

@@ -1546,7 +1546,7 @@ func (Admin) TLSRPTSummaries(ctx context.Context, start, end time.Time, domain s
 // end (most recent first), then by domain.
 func (Admin) DMARCReports(ctx context.Context, start, end time.Time, domain string) (reports []dmarcdb.DomainFeedback) {
 	reports, err := dmarcdb.RecordsPeriodDomain(ctx, start, end, domain)
-	xcheckf(ctx, err, "fetching dmarc reports from database")
+	xcheckf(ctx, err, "fetching dmarc aggregate reports from database")
 	sort.Slice(reports, func(i, j int) bool {
 		iend := reports[i].ReportMetadata.DateRange.End
 		jend := reports[j].ReportMetadata.DateRange.End
@@ -1565,9 +1565,9 @@ func (Admin) DMARCReportID(ctx context.Context, domain string, reportID int64) (
 		err = bstore.ErrAbsent
 	}
 	if err == bstore.ErrAbsent {
-		xcheckuserf(ctx, err, "fetching dmarc report from database")
+		xcheckuserf(ctx, err, "fetching dmarc aggregate report from database")
 	}
-	xcheckf(ctx, err, "fetching dmarc report from database")
+	xcheckf(ctx, err, "fetching dmarc aggregate report from database")
 	return report
 }
 
@@ -1589,7 +1589,7 @@ type DMARCSummary struct {
 // The returned summaries are ordered by domain name.
 func (Admin) DMARCSummaries(ctx context.Context, start, end time.Time, domain string) (domainSummaries []DMARCSummary) {
 	reports, err := dmarcdb.RecordsPeriodDomain(ctx, start, end, domain)
-	xcheckf(ctx, err, "fetching dmarc reports from database")
+	xcheckf(ctx, err, "fetching dmarc aggregate reports from database")
 	summaries := map[string]DMARCSummary{}
 	for _, r := range reports {
 		sum := summaries[r.Domain]
@@ -1932,3 +1932,31 @@ func (Admin) WebserverConfigSave(ctx context.Context, oldConf, newConf Webserver
 func (Admin) Transports(ctx context.Context) map[string]config.Transport {
 	return mox.Conf.Static.Transports
 }
+
+// DMARCEvaluationStats returns a map of all domains with evaluations to a count of
+// the evaluations and whether those evaluations will cause a report to be sent.
+func (Admin) DMARCEvaluationStats(ctx context.Context) map[string]dmarcdb.EvaluationStat {
+	stats, err := dmarcdb.EvaluationStats(ctx)
+	xcheckf(ctx, err, "get evaluation stats")
+	return stats
+}
+
+// DMARCEvaluationsDomain returns all evaluations for aggregate reports for the
+// domain, sorted from oldest to most recent.
+func (Admin) DMARCEvaluationsDomain(ctx context.Context, domain string) (dns.Domain, []dmarcdb.Evaluation) {
+	dom, err := dns.ParseDomain(domain)
+	xcheckf(ctx, err, "parsing domain")
+
+	evals, err := dmarcdb.EvaluationsDomain(ctx, dom)
+	xcheckf(ctx, err, "get evaluations for domain")
+	return dom, evals
+}
+
+// DMARCRemoveEvaluations removes evaluations for a domain.
+func (Admin) DMARCRemoveEvaluations(ctx context.Context, domain string) {
+	dom, err := dns.ParseDomain(domain)
+	xcheckf(ctx, err, "parsing domain")
+
+	err = dmarcdb.RemoveEvaluationsDomain(ctx, dom)
+	xcheckf(ctx, err, "removing evaluations for domain")
+}

webadmin/admin.html

@@ -260,11 +260,13 @@ const index = async () => {
 			),
 		),
 		dom.br(),
-		dom.h2('Reporting'),
+		dom.h2('Reports'),
-		dom.div(dom.a('DMARC', attr({href: '#dmarc'}))),
+		dom.div(dom.a('DMARC', attr({href: '#dmarc/reports'}))),
 		dom.div(dom.a('TLS', attr({href: '#tlsrpt'}))),
+		dom.br(),
+		dom.h2('Operations'),
 		dom.div(dom.a('MTA-STS policies', attr({href: '#mtasts'}))),
-		// todo: outgoing DMARC findings
+		dom.div(dom.a('DMARC evaluations', attr({href: '#dmarc/evaluations'}))),
 		// todo: outgoing TLSRPT findings
 		// todo: routing, globally, per domain and per account
 		dom.br(),
@@ -418,6 +420,16 @@ const box = (color, ...l) => [
 	),
 	dom.br(),
 ]
+const inlineBox = (color, ...l) =>
+	dom.span(
+		style({
+			display: 'inline-block',
+			padding: color ? '0.05em 0.2em' : '',
+			backgroundColor: color,
+			borderRadius: '3px',
+		}),
+		l,
+	)
 
 const accounts = async () => {
 	const accounts = await api.Accounts()
@@ -1004,7 +1016,25 @@ const domainDNSCheck = async (d) => {
 	)
 }
 
-const dmarc = async () => {
+const dmarcIndex = async () => {
+	const page = document.getElementById('page')
+	dom._kids(page,
+		crumbs(
+			crumblink('Mox Admin', '#'),
+			'DMARC reports and evaluations',
+		),
+		dom.ul(
+			dom.li(
+				dom.a(attr({href: '#dmarc/reports'}), 'Reports'), ', incoming DMARC aggregate reports.',
+			),
+			dom.li(
+				dom.a(attr({href: '#dmarc/evaluations'}), 'Evaluations'), ', for outgoing DMARC aggregate reports.',
+			),
+		),
+	)
+}
+
+const dmarcReports = async () => {
 	const end = new Date().toISOString()
 	const start = new Date(new Date().getTime() - 30*24*3600*1000).toISOString()
 	const summaries = await api.DMARCSummaries(start, end, "")
@@ -1013,7 +1043,8 @@ const dmarc = async () => {
 	dom._kids(page,
 		crumbs(
 			crumblink('Mox Admin', '#'),
-			'DMARC aggregate reporting summary',
+			crumblink('DMARC', '#dmarc'),
+			'Aggregate reporting summary',
 		),
 		dom.p('DMARC reports are periodically sent by other mail servers that received an email message with a "From" header with our domain. Domains can have a DMARC DNS record that asks other mail servers to send these aggregate reports for analysis.'),
 		renderDMARCSummaries(summaries),
@@ -1051,6 +1082,167 @@ const renderDMARCSummaries = (summaries) => {
 	]
 }
 
+const dmarcEvaluations = async () => {
+	const evalStats = await api.DMARCEvaluationStats()
+
+	const isEmpty = (o) => {
+		for (const e in o) {
+			return false
+		}
+		return true
+	}
+
+	const page = document.getElementById('page')
+	dom._kids(page,
+		crumbs(
+			crumblink('Mox Admin', '#'),
+			crumblink('DMARC', '#dmarc'),
+			'Evaluations',
+		),
+		dom.p('Incoming messages are checked against the DMARC policy of the domain in the message From header. If the policy requests reporting on the resulting evaluations, they are stored in the database. Each interval of 1 to 24 hours, the evaluations may be sent to a reporting address specified in the domain\'s DMARC policy. Not all evaluations are a reason to send a report, but if a report is sent all evaluations are included.'),
+		dom.table(
+			dom.thead(
+				dom.tr(
+					dom.th('Domain', attr({title: 'Domain in the message From header. Keep in mind these can be forged, so this does not necessarily mean someone from this domain authentically tried delivering email.'})),
+					dom.th('Evaluations', attr({title: 'Total number of message delivery attempts, including retries.'})),
+					dom.th('Send report', attr({title: 'Whether the current evaluations will cause a report to be sent.'})),
+				),
+			),
+			dom.tbody(
+				Object.entries(evalStats).sort((a, b) => a[0] < b[0] ? -1 : 1).map(t =>
+					dom.tr(
+						dom.td(dom.a(attr({href: '#dmarc/evaluations/'+domainName(t[1].Domain)}), domainString(t[1].Domain))),
+						dom.td(style({textAlign: 'right'}), ''+t[1].Count),
+						dom.td(style({textAlign: 'right'}), t[1].SendReport ? '✓' : ''),
+					),
+				),
+				isEmpty(evalStats) ? dom.tr(dom.td(attr({colspan: '3'}), 'No evaluations.')) : [],
+			),
+		),
+	)
+}
+
+const dmarcEvaluationsDomain = async (domain) => {
+	const [d, evaluations] = await api.DMARCEvaluationsDomain(domain)
+
+	let lastInterval = ''
+	let lastAddresses = ''
+
+	const formatPolicy = (e) => {
+		const p = e.PolicyPublished
+		let s = ''
+		const add = (k, v) => {
+			if (v) {
+				s += k+'='+v+'; '
+			}
+		}
+		add('p', p.Policy)
+		add('sp', p.SubdomainPolicy)
+		add('adkim', p.ADKIM)
+		add('aspf', p.ASPF)
+		add('pct', ''+p.Percentage)
+		add('fo', ''+p.ReportingOptions)
+		return s
+	}
+	let lastPolicy = ''
+
+	const authStatus = (v) => inlineBox(v ? '' : yellow, v ? 'pass' : 'fail')
+	const formatDKIMResults = (results) => results.map(r => dom.div('selector '+r.Selector+(r.Domain !== domain ? ', domain '+r.Domain : '') + ': ', inlineBox(r.Result === "pass" ? '' : yellow, r.Result)))
+	const formatSPFResults = (results) => results.map(r => dom.div(''+r.Scope+(r.Domain !== domain ? ', domain '+r.Domain : '') + ': ', inlineBox(r.Result === "pass" ? '' : yellow, r.Result)))
+
+	const sourceIP = (ip) => {
+		const r = dom.span(ip, attr({title: 'Click to do a reverse lookup of the IP.'}), style({cursor: 'pointer'}), async function click(e) {
+			e.preventDefault()
+			try {
+				const rev = await api.LookupIP(ip)
+				r.innerText = ip + '\n' + rev.Hostnames.join('\n')
+			} catch (err) {
+				r.innerText = ip + '\nerror: ' +err.message
+			}
+		})
+		return r
+	}
+
+	const page = document.getElementById('page')
+	dom._kids(page,
+		crumbs(
+			crumblink('Mox Admin', '#'),
+			crumblink('DMARC', '#dmarc'),
+			crumblink('Evaluations', '#dmarc/evaluations'),
+			'Domain '+domainString(d),
+		),
+		dom.div(
+			dom.button('Remove evaluations', async function click(e) {
+				e.target.disabled = true
+				try {
+					await api.DMARCRemoveEvaluations(domain)
+					window.location.reload() // todo: only clear the table?
+				} catch (err) {
+					console.log({err})
+					window.alert('Error: ' + err.message)
+				} finally {
+					e.target.disabled = false
+				}
+			}),
+		),
+		dom.br(),
+		dom.p('The evaluations below will be sent in a DMARC aggregate report to the addresses found in the published DMARC DNS record, which is fetched again before sending the report. The fields Interval hours, Addresses and Policy are only filled for the first row and whenever a new value in the published DMARC record is encountered.'),
+		dom.table(
+			dom.thead(
+				dom.tr(
+					dom.th('ID'),
+					dom.th('Evaluated'),
+					dom.th('Optional', attr({title: 'Some evaluations will not cause a DMARC aggregate report to be sent. But if a report is sent, optional records are included.'})),
+					dom.th('Interval hours', attr({title: 'DMARC policies published by a domain can specify how often they would like to receive reports. The default is 24 hours, but can be as often as each hour. To keep reports comparable between different mail servers that send reports, reports are sent at rounded up intervals of whole hours that can divide a 24 hour day, and are aligned with the start of a day at UTC.'})),
+					dom.th('Addresses', attr({title: 'Addresses that will receive the report. An address can have a maximum report size configured. If there is no address, no report will be sent.'})),
+					dom.th('Policy', attr({title: 'Summary of the policy as encountered in the DMARC DNS record of the domain, and used for evaluation.'})),
+					dom.th('IP', attr({title: 'IP address of delivery attempt that was evaluated, relevant for SPF.'})),
+					dom.th('Disposition', attr({title: 'Our decision to accept/reject this message. It may be different than requested by the published policy. For example, when overriding due to delivery from a mailing list or forwarded address.'})),
+					dom.th('DKIM/SPF', attr({title: 'Whether DKIM and SPF had an aligned pass, where strict/relaxed alignment means whether the domain of an SPF pass and DKIM pass matches the exact domain (strict) or optionally a subdomain (relaxed). A DMARC pass requires at least one pass.'})),
+					dom.th('Envelope to', attr({title: 'Domain used in SMTP RCPT TO during delivery.'})),
+					dom.th('Envelope from', attr({title: 'Domain used in SMTP MAIL FROM during delivery.'})),
+					dom.th('Message from', attr({title: 'Domain in "From" message header.'})),
+					dom.th('DKIM details', attr({title: 'Results of verifying DKIM-Signature headers in message. Only signatures with matching organizational domain are included, regardless of strict/relaxed DKIM alignment in DMARC policy.'})),
+					dom.th('SPF details', attr({title: 'Results of SPF check used in DMARC evaluation. "mfrom" indicates the "SMTP MAIL FROM" domain was used, "helo" indicates the SMTP EHLO domain was used.'})),
+				),
+			),
+			dom.tbody(
+				evaluations.map(e => {
+					const ival = e.IntervalHours + 'h'
+					const interval = ival === lastInterval ? '' : ival
+					lastInterval = ival
+
+					const a = (e.Addresses || []).join('\n')
+					const addresses = a === lastAddresses ? '' : a
+					lastAddresses = a
+
+					const p = formatPolicy(e)
+					const policy = p === lastPolicy ? '' : p
+					lastPolicy = p
+
+					return dom.tr(
+						dom.td(''+e.ID),
+						dom.td(new Date(e.Evaluated).toUTCString()),
+						dom.td(e.Optional ? 'Yes' : ''),
+						dom.td(interval),
+						dom.td(addresses),
+						dom.td(policy),
+						dom.td(sourceIP(e.SourceIP)),
+						dom.td(inlineBox(e.Disposition === 'none' ? '' : 'red', e.Disposition), (e.OverrideReasons || []).length > 0 ? ' ('+e.OverrideReasons.map(r => r.Type).join(', ')+')' : ''),
+						dom.td(authStatus(e.AlignedDKIMPass), '/', authStatus(e.AlignedSPFPass)),
+						dom.td(e.EnvelopeTo),
+						dom.td(e.EnvelopeFrom),
+						dom.td(e.HeaderFrom),
+						dom.td(formatDKIMResults(e.DKIMResults || [])),
+						dom.td(formatSPFResults(e.SPFResults || [])),
+					)
+				}),
+				evaluations.length === 0 ? dom.tr(dom.td(attr({colspan: '14'}), 'No evaluations.')) : [],
+			),
+		),
+	)
+}
+
 const utcDate = (dt) => new Date(Date.UTC(dt.getUTCFullYear(), dt.getUTCMonth(), dt.getUTCDate(), dt.getUTCHours(), dt.getUTCMinutes(), dt.getUTCSeconds()))
 const utcDateStr = (dt) => [dt.getUTCFullYear(), 1+dt.getUTCMonth(), dt.getUTCDate()].join('-')
 const isDayChange = (dt) => utcDateStr(new Date(dt.getTime() - 2*60*1000)) !== utcDateStr(new Date(dt.getTime() + 2*60*1000))
@@ -2241,7 +2433,13 @@ const init = async () => {
 			} else if (h === 'tlsrpt') {
 				await tlsrpt()
 			} else if (h === 'dmarc') {
-				await dmarc()
+				await dmarcIndex()
+			} else if (h === 'dmarc/reports') {
+				await dmarcReports()
+			} else if (h === 'dmarc/evaluations') {
+				await dmarcEvaluations()
+			} else if (t[0] == 'dmarc' && t[1] == 'evaluations' && t.length === 3) {
+				await dmarcEvaluationsDomain(t[2])
 			} else if (h === 'mtasts') {
 				await mtasts()
 			} else if (h === 'dnsbl') {

webadmin/adminapi.json

@@ -753,6 +753,60 @@
 					]
 				}
 			]
+		},
+		{
+			"Name": "DMARCEvaluationStats",
+			"Docs": "DMARCEvaluationStats returns a map of all domains with evaluations to a count of\nthe evaluations and whether those evaluations will cause a report to be sent.",
+			"Params": [],
+			"Returns": [
+				{
+					"Name": "r0",
+					"Typewords": [
+						"{}",
+						"EvaluationStat"
+					]
+				}
+			]
+		},
+		{
+			"Name": "DMARCEvaluationsDomain",
+			"Docs": "DMARCEvaluationsDomain returns all evaluations for aggregate reports for the\ndomain, sorted from oldest to most recent.",
+			"Params": [
+				{
+					"Name": "domain",
+					"Typewords": [
+						"string"
+					]
+				}
+			],
+			"Returns": [
+				{
+					"Name": "r0",
+					"Typewords": [
+						"Domain"
+					]
+				},
+				{
+					"Name": "r1",
+					"Typewords": [
+						"[]",
+						"Evaluation"
+					]
+				}
+			]
+		},
+		{
+			"Name": "DMARCRemoveEvaluations",
+			"Docs": "DMARCRemoveEvaluations removes evaluations for a domain.",
+			"Params": [
+				{
+					"Name": "domain",
+					"Typewords": [
+						"string"
+					]
+				}
+			],
+			"Returns": []
 		}
 	],
 	"Sections": [],
@@ -2512,7 +2566,7 @@
 			"Fields": [
 				{
 					"Name": "Domain",
-					"Docs": "",
+					"Docs": "Domain is where DMARC record was found, not necessarily message From. Reports we generate use unicode names, incoming reports may have either ASCII-only or Unicode domains.",
 					"Typewords": [
 						"string"
 					]
@@ -2914,7 +2968,7 @@
 		},
 		{
 			"Name": "Msg",
-			"Docs": "Msg is a message in the queue.",
+			"Docs": "Msg is a message in the queue.\n\nUse MakeMsg to make a message with fields that Add needs. Add will further set\nqueueing related fields.",
 			"Fields": [
 				{
 					"Name": "ID",
@@ -2979,6 +3033,13 @@
 						"int32"
 					]
 				},
+				{
+					"Name": "MaxAttempts",
+					"Docs": "Max number of attempts before giving up. If 0, then the default of 8 attempts is used instead.",
+					"Typewords": [
+						"int32"
+					]
+				},
 				{
 					"Name": "DialedIPs",
 					"Docs": "For each host, the IPs that were dialed. Used for IP selection for later attempts.",
@@ -3024,6 +3085,20 @@
 						"bool"
 					]
 				},
+				{
+					"Name": "IsDMARCReport",
+					"Docs": "Delivery failures for DMARC reports are handled differently.",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "IsTLSReport",
+					"Docs": "Delivery failures for TLS reports are handled differently.",
+					"Typewords": [
+						"bool"
+					]
+				},
 				{
 					"Name": "Size",
 					"Docs": "Full size of message, combined MsgPrefix with contents of message file.",
@@ -3441,6 +3516,162 @@
 					]
 				}
 			]
+		},
+		{
+			"Name": "EvaluationStat",
+			"Docs": "EvaluationStat summarizes stored evaluations, for inclusion in an upcoming\naggregate report, for a domain.",
+			"Fields": [
+				{
+					"Name": "Count",
+					"Docs": "",
+					"Typewords": [
+						"int32"
+					]
+				},
+				{
+					"Name": "SendReport",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "Domain",
+					"Docs": "",
+					"Typewords": [
+						"Domain"
+					]
+				}
+			]
+		},
+		{
+			"Name": "Evaluation",
+			"Docs": "Evaluation is the result of an evaluation of a DMARC policy, to be included\nin a DMARC report.",
+			"Fields": [
+				{
+					"Name": "ID",
+					"Docs": "",
+					"Typewords": [
+						"int64"
+					]
+				},
+				{
+					"Name": "PolicyDomain",
+					"Docs": "Domain where DMARC policy was found, could be the organizational domain while evaluation was for a subdomain. Unicode. Same as domain found in PolicyPublished. A separate field for its index.",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "Evaluated",
+					"Docs": "Time of evaluation, determines which report (covering whole hours) this evaluation will be included in.",
+					"Typewords": [
+						"timestamp"
+					]
+				},
+				{
+					"Name": "Optional",
+					"Docs": "If optional, this evaluation is not a reason to send a DMARC report, but it will be included when a report is sent due to other non-optional evaluations. Set for evaluations of incoming DMARC reports. We don't want such deliveries causing us to send a report, or we would keep exchanging reporting messages forever. Also set for when evaluation is a DMARC reject for domains we haven't positively interacted with, to prevent being used to flood an unsuspecting domain with reports.",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "IntervalHours",
+					"Docs": "Effective aggregate reporting interval in hours. Between 1 and 24, rounded up from seconds from policy to first number that can divide 24.",
+					"Typewords": [
+						"int32"
+					]
+				},
+				{
+					"Name": "Addresses",
+					"Docs": "\"rua\" in DMARC record, we only store evaluations for records with aggregate reporting addresses, so always non-empty.",
+					"Typewords": [
+						"[]",
+						"string"
+					]
+				},
+				{
+					"Name": "PolicyPublished",
+					"Docs": "Policy used for evaluation. We don't store the \"fo\" field for failure reporting options, since we don't send failure reports for individual messages.",
+					"Typewords": [
+						"PolicyPublished"
+					]
+				},
+				{
+					"Name": "SourceIP",
+					"Docs": "For \"row\" in a report record.",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "Disposition",
+					"Docs": "",
+					"Typewords": [
+						"Disposition"
+					]
+				},
+				{
+					"Name": "AlignedDKIMPass",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "AlignedSPFPass",
+					"Docs": "",
+					"Typewords": [
+						"bool"
+					]
+				},
+				{
+					"Name": "OverrideReasons",
+					"Docs": "",
+					"Typewords": [
+						"[]",
+						"PolicyOverrideReason"
+					]
+				},
+				{
+					"Name": "EnvelopeTo",
+					"Docs": "For \"identifiers\" in a report record.",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "EnvelopeFrom",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "HeaderFrom",
+					"Docs": "",
+					"Typewords": [
+						"string"
+					]
+				},
+				{
+					"Name": "DKIMResults",
+					"Docs": "For \"auth_results\" in a report record.",
+					"Typewords": [
+						"[]",
+						"DKIMAuthResult"
+					]
+				},
+				{
+					"Name": "SPFResults",
+					"Docs": "",
+					"Typewords": [
+						"[]",
+						"SPFAuthResult"
+					]
+				}
+			]
 		}
 	],
 	"Ints": [],

webmail/api.go

@@ -691,7 +691,8 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
 			Localpart: rcpt.Localpart,
 			IPDomain:  dns.IPDomain{Domain: rcpt.Domain},
 		}
-		_, err := queue.Add(ctx, log, reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), dataFile, nil, m.RequireTLS)
+		qm := queue.MakeMsg(reqInfo.AccountName, fromPath, toPath, has8bit, smtputf8, msgSize, messageID, []byte(rcptMsgPrefix), m.RequireTLS)
+		err := queue.Add(ctx, log, &qm, dataFile)
 		if err != nil {
 			metricSubmission.WithLabelValues("queueerror").Inc()
 		}