tlsrpt improvements

- accept incoming tls reports for the host, with policy-domain the host name.
  instead of not storing the domain because it is not a configured (recipient)
  domain.
- in tlsrpt summaries, rename domain to policy domain for clarity.
- in webadmin, fix html for table that lists tls reports in case of multiple
  policies and/or multiple failure details.

tlsrptdb/report.go

@@ -49,7 +49,7 @@ var (
 // todo: should be named just Record, but it would cause a sherpa type name conflict.
 type TLSReportRecord struct {
 	ID         int64  `bstore:"typename Record"`
-	Domain     string `bstore:"index"` // Domain to which the TLS report applies.
+	Domain     string `bstore:"index"` // Policy domain to which the TLS report applies. Unicode.
 	FromDomain string
 	MailFrom   string
 	HostReport bool // Report for host TLSRPT record, as opposed to domain TLSRPT record.
@@ -99,16 +99,19 @@ func AddReport(ctx context.Context, verifiedFromDomain dns.Domain, mailFrom stri
 	for _, p := range r.Policies {
 		pp := p.Policy
 
-		// Check domain, they must all be the same for now (in future, with DANE, this may
+		// Check domain, they must all be the same for now. We are not expecting senders to
-		// no longer apply).
+		// coalesce TLS results for different policy domains in a single report.
 		d, err := dns.ParseDomain(pp.Domain)
 		if err != nil {
 			log.Errorx("invalid domain in tls report", err, mlog.Field("domain", pp.Domain), mlog.Field("mailfrom", mailFrom))
 			continue
 		}
-		if _, ok := mox.Conf.Domain(d); !ok {
+		if hostReport && d != mox.Conf.Static.HostnameDomain {
-			log.Info("unknown domain in tls report, not storing", mlog.Field("domain", d), mlog.Field("mailfrom", mailFrom))
+			log.Info("unknown mail host policy domain in tls report, not storing", mlog.Field("domain", d), mlog.Field("mailfrom", mailFrom))
-			return fmt.Errorf("unknown domain")
+			return fmt.Errorf("unknown mail host policy domain")
+		} else if _, ok := mox.Conf.Domain(d); !hostReport && !ok {
+			log.Info("unknown recipient policy domain in tls report, not storing", mlog.Field("domain", d), mlog.Field("mailfrom", mailFrom))
+			return fmt.Errorf("unknown recipient policy domain")
 		}
 		if reportdom != zerodom && d != reportdom {
 			return fmt.Errorf("multiple domains in report %s and %s", reportdom, d)
@@ -151,17 +154,19 @@ func RecordID(ctx context.Context, id int64) (TLSReportRecord, error) {
 	return e, err
 }
 
-// RecordsPeriodDomain returns the reports overlapping start and end, for the given
+// RecordsPeriodPolicyDomain returns the reports overlapping start and end, for the
-// domain. If domain is empty, all records match for domain.
+// given policy domain. If policy domain is empty, records for all domains are
-func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]TLSReportRecord, error) {
+// returned.
+func RecordsPeriodDomain(ctx context.Context, start, end time.Time, policyDomain dns.Domain) ([]TLSReportRecord, error) {
 	db, err := reportDB(ctx)
 	if err != nil {
 		return nil, err
 	}
 
 	q := bstore.QueryDB[TLSReportRecord](ctx, db)
-	if domain != "" {
+	var zerodom dns.Domain
-		q.FilterNonzero(TLSReportRecord{Domain: domain})
+	if policyDomain != zerodom {
+		q.FilterNonzero(TLSReportRecord{Domain: policyDomain.Name()})
 	}
 	q.FilterFn(func(r TLSReportRecord) bool {
 		dr := r.Report.DateRange

tlsrptdb/report_test.go

@@ -124,7 +124,7 @@ func TestReport(t *testing.T) {
 
 	start, _ := time.Parse(time.RFC3339, "2016-04-01T00:00:00Z")
 	end, _ := time.Parse(time.RFC3339, "2016-04-01T23:59:59Z")
-	records, err = RecordsPeriodDomain(ctxbg, start, end, "test.xmox.nl")
+	records, err = RecordsPeriodDomain(ctxbg, start, end, dns.Domain{ASCII: "test.xmox.nl"})
 	if err != nil || len(records) != 1 {
 		t.Fatalf("got err %v, records %#v, expected no error with 1 record", err, records)
 	}

webadmin/admin.go

@@ -1517,10 +1517,17 @@ func (Admin) MTASTSPolicies(ctx context.Context) (records []mtastsdb.PolicyRecor
 }
 
 // TLSReports returns TLS reports overlapping with period start/end, for the given
-// domain (or all domains if empty). The reports are sorted first by period end
+// policy domain (or all domains if empty). The reports are sorted first by period
-// (most recent first), then by domain.
+// end (most recent first), then by policy domain.
-func (Admin) TLSReports(ctx context.Context, start, end time.Time, domain string) (reports []tlsrptdb.TLSReportRecord) {
+func (Admin) TLSReports(ctx context.Context, start, end time.Time, policyDomain string) (reports []tlsrptdb.TLSReportRecord) {
-	records, err := tlsrptdb.RecordsPeriodDomain(ctx, start, end, domain)
+	var polDom dns.Domain
+	if policyDomain != "" {
+		var err error
+		polDom, err = dns.ParseDomain(policyDomain)
+		xcheckuserf(ctx, err, "parsing domain %q", policyDomain)
+	}
+
+	records, err := tlsrptdb.RecordsPeriodDomain(ctx, start, end, polDom)
 	xcheckf(ctx, err, "fetching tlsrpt report records from database")
 	sort.Slice(records, func(i, j int) bool {
 		iend := records[i].Report.DateRange.End
@@ -1549,7 +1556,7 @@ func (Admin) TLSReportID(ctx context.Context, domain string, reportID int64) tls
 // TLSRPTSummary presents TLS reporting statistics for a single domain
 // over a period.
 type TLSRPTSummary struct {
-	Domain           string
+	PolicyDomain     dns.Domain
 	Success          int64
 	Failure          int64
 	ResultTypeCounts map[tlsrpt.ResultType]int
@@ -1558,13 +1565,23 @@ type TLSRPTSummary struct {
 // TLSRPTSummaries returns a summary of received TLS reports overlapping with
 // period start/end for one or all domains (when domain is empty).
 // The returned summaries are ordered by domain name.
-func (Admin) TLSRPTSummaries(ctx context.Context, start, end time.Time, domain string) (domainSummaries []TLSRPTSummary) {
+func (Admin) TLSRPTSummaries(ctx context.Context, start, end time.Time, policyDomain string) (domainSummaries []TLSRPTSummary) {
-	reports, err := tlsrptdb.RecordsPeriodDomain(ctx, start, end, domain)
+	var polDom dns.Domain
+	if policyDomain != "" {
+		var err error
+		polDom, err = dns.ParseDomain(policyDomain)
+		xcheckuserf(ctx, err, "parsing policy domain")
+	}
+	reports, err := tlsrptdb.RecordsPeriodDomain(ctx, start, end, polDom)
 	xcheckf(ctx, err, "fetching tlsrpt reports from database")
-	summaries := map[string]TLSRPTSummary{}
+
+	summaries := map[dns.Domain]TLSRPTSummary{}
 	for _, r := range reports {
-		sum := summaries[r.Domain]
+		dom, err := dns.ParseDomain(r.Domain)
-		sum.Domain = r.Domain
+		xcheckf(ctx, err, "parsing domain %q", r.Domain)
+
+		sum := summaries[dom]
+		sum.PolicyDomain = dom
 		for _, result := range r.Report.Policies {
 			sum.Success += result.Summary.TotalSuccessfulSessionCount
 			sum.Failure += result.Summary.TotalFailureSessionCount
@@ -1575,14 +1592,14 @@ func (Admin) TLSRPTSummaries(ctx context.Context, start, end time.Time, domain s
 				sum.ResultTypeCounts[details.ResultType]++
 			}
 		}
-		summaries[r.Domain] = sum
+		summaries[dom] = sum
 	}
 	sums := make([]TLSRPTSummary, 0, len(summaries))
 	for _, sum := range summaries {
 		sums = append(sums, sum)
 	}
 	sort.Slice(sums, func(i, j int) bool {
-		return sums[i].Domain < sums[j].Domain
+		return sums[i].PolicyDomain.Name() < sums[j].PolicyDomain.Name()
 	})
 	return sums
 }

webadmin/admin.html

@@ -1102,7 +1102,7 @@ const dmarcEvaluations = async () => {
 			'Evaluations',
 		),
 		dom.p('Incoming messages are checked against the DMARC policy of the domain in the message From header. If the policy requests reporting on the resulting evaluations, they are stored in the database. Each interval of 1 to 24 hours, the evaluations may be sent to a reporting address specified in the domain\'s DMARC policy. Not all evaluations are a reason to send a report, but if a report is sent all evaluations are included.'),
-		dom.table(
+		dom('table.hover',
 			dom.thead(
 				dom.tr(
 					dom.th('Domain', attr({title: 'Domain in the message From header. Keep in mind these can be forged, so this does not necessarily mean someone from this domain authentically tried delivering email.'})),
@@ -1189,7 +1189,7 @@ const dmarcEvaluationsDomain = async (domain) => {
 		),
 		dom.br(),
 		dom.p('The evaluations below will be sent in a DMARC aggregate report to the addresses found in the published DMARC DNS record, which is fetched again before sending the report. The fields Interval hours, Addresses and Policy are only filled for the first row and whenever a new value in the published DMARC record is encountered.'),
-		dom.table(
+		dom('table.hover',
 			dom.thead(
 				dom.tr(
 					dom.th('ID'),
@@ -1290,7 +1290,7 @@ const domainDMARC = async (d) => {
 		dom.p('DMARC reports are periodically sent by other mail servers that received an email message with a "From" header with our domain. Domains can have a DMARC DNS record that asks other mail servers to send these aggregate reports for analysis.'),
 		dom.p('Below the DMARC aggregate reports for the past 30 days.'),
 		reports.length === 0 ? dom.div('No DMARC reports for domain.') :
-		dom.table(
+		dom('table.hover',
 			dom.thead(
 				dom.tr(
 					dom.th('ID'),
@@ -1631,19 +1631,19 @@ const renderTLSRPTSummaries = (summaries) => {
 	return [
 		dom.p('Below a summary of TLS reports for the past 30 days.'),
 		summaries.length === 0 ? dom.div(box(yellow, 'No domains with TLS reports.')) :
-		dom.table(
+		dom('table.hover',
 			dom.thead(
 				dom.tr(
-					dom.th('Domain', attr({title: ''})),
+					dom.th('Policy domain', attr({title: 'Policy domain the report is about. The recipient domain for MTA-STS, the TLSA base domain for DANE.'})),
-					dom.th('Successes', attr({title: ''})),
+					dom.th('Successes', attr({title: 'Number of successful SMTP STARTTLS sessions.'})),
-					dom.th('Failures', attr({title: ''})),
+					dom.th('Failures', attr({title: 'Number of failed SMTP STARTTLS sessions.'})),
-					dom.th('Failure details', attr({title: ''})),
+					dom.th('Failure details', attr({title: 'Details about connection failures.'})),
 				)
 			),
 			dom.tbody(
 				summaries.map(r =>
 					dom.tr(
-						dom.td(dom.a(attr({href: '#domains/' + r.Domain + '/tlsrpt', title: 'See report details.'}), r.Domain)),
+						dom.td(dom.a(attr({href: '#domains/' + domainName(r.PolicyDomain) + '/tlsrpt', title: 'See report details.'}), domainName(r.PolicyDomain))),
 						dom.td(style({textAlign: 'right'}), '' + r.Success),
 						dom.td(style({textAlign: 'right'}), '' + r.Failure),
 						dom.td(!r.ResultTypeCounts ? [] : Object.entries(r.ResultTypeCounts).map(kv => kv[0] + ': ' + kv[1]).join('; ')),
@@ -1683,7 +1683,7 @@ const domainTLSRPT = async (d) => {
 		dom.p('TLSRPT (TLS reporting) is a mechanism to request feedback from other mail servers about TLS connections to your mail server. If is typically used along with MTA-STS and/or DANE to enforce that SMTP connections are protected with TLS. Mail servers implementing TLSRPT will typically send a daily report with both successful and failed connection counts, including details about failures.'),
 		dom.p('Below the TLS reports for the past 30 days.'),
 		records.length === 0 ? dom.div('No TLS reports for domain.') :
-		dom.table(
+		dom('table.hover',
 			dom.thead(
 				dom.tr(
 					dom.th('Report', attr({colspan: '3'})),
@@ -1712,21 +1712,23 @@ const domainTLSRPT = async (d) => {
 			dom.tbody(
 				records.map(record => {
 					const r = record.Report
-					const reportRowSpan = attr({rowspan: ''+r.policies.length})
+					let nrows = 0
+					r.policies.forEach(pr => nrows += (pr['failure-details'] || []).length || 1)
+					const reportRowSpan = attr({rowspan: ''+nrows})
 					const valignTop = style({verticalAlign: 'top'})
 					const alignRight = style({textAlign: 'right'})
 					return r.policies.map((result, index) => {
 						const rows = []
 						const details = result['failure-details'] || []
 						const resultRowSpan = attr({rowspan: ''+(details.length || 1)})
-						const addRow = (d) => {
+						const addRow = (d, di) => {
 							const row = dom.tr(
 								index > 0 || rows.length > 0 ? [] : [
 									dom.td(reportRowSpan, valignTop, dom.a(''+record.ID, attr({href: '#domains/' + record.Domain + '/tlsrpt/'+record.ID}))),
 									dom.td(reportRowSpan, valignTop, r['organization-name'] || r['contact-info'] || record.MailFrom || '', attr({title: 'Organization: ' +r['organization-name'] + '; \nContact info: ' + r['contact-info'] + '; \nReport ID: ' + r['report-id'] + '; \nMail from: ' + record.MailFrom, })),
 									dom.td(reportRowSpan, valignTop, period(new Date(r['date-range']['start-datetime']), new Date(r['date-range']['end-datetime']))),
 								],
-								index > 0 ? [] : [
+								di > 0 ? [] : [
 									dom.td(resultRowSpan, valignTop, policyType(result.policy), attr({title: (result.policy['policy-string'] || []).join('\n')})),
 									dom.td(resultRowSpan, valignTop, alignRight, '' + result.summary['total-successful-session-count']),
 									dom.td(resultRowSpan, valignTop, alignRight, '' + result.summary['total-failure-session-count']),
@@ -1735,7 +1737,7 @@ const domainTLSRPT = async (d) => {
 									dom.td(d['result-type']),
 									dom.td(d['sending-mta-ip']),
 									dom.td(d['receiving-mx-hostname']),
-									dom.td(d['receiving-mx-helo']),
+									dom.td(d['receiving-mx-helo'] || ''),
 									dom.td(d['receiving-ip']),
 									dom.td(alignRight, '' + d['failed-session-count']),
 									dom.td(d['additional-information']),
@@ -1745,11 +1747,13 @@ const domainTLSRPT = async (d) => {
 							)
 							rows.push(row)
 						}
+						let di = 0
 						for (const d of details) {
-							addRow(d)
+							addRow(d, di)
+							di++
 						}
-						if (!details.length) {
+						if (details.length === 0) {
-							addRow()
+							addRow(undefined, 0)
 						}
 						return rows
 					})
@@ -1819,7 +1823,7 @@ const makeMTASTSTable = items => {
 		["Inserted", "", "Time when the policy was first inserted."],
 	]
 	const nowSecs = new Date().getTime()/1000
-	return dom.table(
+	return dom('table.hover',
 		dom.thead(
 			dom.tr(keys.map(kt => dom.th(dom.span(attr({title: kt[2]}), kt[1] || kt[0])))),
 		),
@@ -1903,7 +1907,7 @@ const queueList = async () => {
 		msgs.length === 0 ? 'Currently no messages in the queue.' : [
 			dom.p('The messages below are currently in the queue.'),
 			// todo: sorting by address/timestamps/attempts. perhaps filtering.
-			dom.table(
+			dom('table.hover',
 				dom.thead(
 					dom.tr(
 						dom.th('ID'),

webadmin/adminapi.json

@@ -159,7 +159,7 @@
 		},
 		{
 			"Name": "TLSReports",
-			"Docs": "TLSReports returns TLS reports overlapping with period start/end, for the given\ndomain (or all domains if empty). The reports are sorted first by period end\n(most recent first), then by domain.",
+			"Docs": "TLSReports returns TLS reports overlapping with period start/end, for the given\npolicy domain (or all domains if empty). The reports are sorted first by period\nend (most recent first), then by policy domain.",
 			"Params": [
 				{
 					"Name": "start",
@@ -174,7 +174,7 @@
 					]
 				},
 				{
-					"Name": "domain",
+					"Name": "policyDomain",
 					"Typewords": [
 						"string"
 					]
@@ -233,7 +233,7 @@
 					]
 				},
 				{
-					"Name": "domain",
+					"Name": "policyDomain",
 					"Typewords": [
 						"string"
 					]
@@ -2271,7 +2271,7 @@
 				},
 				{
 					"Name": "Domain",
-					"Docs": "Domain to which the TLS report applies.",
+					"Docs": "Policy domain to which the TLS report applies. Unicode.",
 					"Typewords": [
 						"string"
 					]
@@ -2520,10 +2520,10 @@
 			"Docs": "TLSRPTSummary presents TLS reporting statistics for a single domain\nover a period.",
 			"Fields": [
 				{
-					"Name": "Domain",
+					"Name": "PolicyDomain",
 					"Docs": "",
 					"Typewords": [
-						"string"
+						"Domain"
 					]
 				},
 				{