mox/tlsrpt/report.go

package tlsrpt

import (
	"compress/gzip"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"reflect"
	"sort"
	"strings"
	"time"

	"golang.org/x/exp/slices"
	"golang.org/x/exp/slog"

	"github.com/mjl-/adns"

	"github.com/mjl-/mox/dns"
	"github.com/mjl-/mox/message"
	"github.com/mjl-/mox/mlog"
	"github.com/mjl-/mox/moxio"
)

var ErrNoReport = errors.New("no tlsrpt report found")

// ../rfc/8460:628

// Report is a TLSRPT report.
type Report struct {
	OrganizationName string
	DateRange        TLSRPTDateRange
	ContactInfo      string
	ReportID         string
	Policies         []Result
}

// ReportJSON is a TLS report with field names as used in the specification. These field names are inconvenient to use in JavaScript, so after parsing a ReportJSON is turned into a Report.
type ReportJSON struct {
	OrganizationName string              `json:"organization-name"`
	DateRange        TLSRPTDateRangeJSON `json:"date-range"`
	ContactInfo      string              `json:"contact-info"` // Email address.
	ReportID         string              `json:"report-id"`
	Policies         []ResultJSON        `json:"policies"`
}

func convertSlice[T interface{ Convert() S }, S any](l []T) []S {
	if l == nil {
		return nil
	}
	r := make([]S, len(l))
	for i, e := range l {
		r[i] = e.Convert()
	}
	return r
}

func (v Report) Convert() ReportJSON {
	return ReportJSON{v.OrganizationName, v.DateRange.Convert(), v.ContactInfo, v.ReportID, convertSlice[Result, ResultJSON](v.Policies)}
}

func (v ReportJSON) Convert() Report {
	return Report{v.OrganizationName, v.DateRange.Convert(), v.ContactInfo, v.ReportID, convertSlice[ResultJSON, Result](v.Policies)}
}

// Merge combines the counts and failure details of results into the report.
// Policies are merged if identical and added otherwise. Same for failure details
// within a result.
func (r *Report) Merge(results ...Result) {
Merge:
	for _, nr := range results {
		for i, p := range r.Policies {
			if !p.Policy.equal(nr.Policy) {
				continue
			}

			r.Policies[i].Add(nr.Summary.TotalSuccessfulSessionCount, nr.Summary.TotalFailureSessionCount, nr.FailureDetails...)
			continue Merge
		}

		r.Policies = append(r.Policies, nr)
	}
}

// Add increases the success/failure counts of a result, and adds any failure
// details.
func (r *Result) Add(success, failure int64, fds ...FailureDetails) {
	r.Summary.TotalSuccessfulSessionCount += success
	r.Summary.TotalFailureSessionCount += failure

	// In smtpclient we can compensate with a negative success, after failed read after
	// successful handshake. Sanity check that we never get negative counts.
	if r.Summary.TotalSuccessfulSessionCount < 0 {
		r.Summary.TotalSuccessfulSessionCount = 0
	}
	if r.Summary.TotalFailureSessionCount < 0 {
		r.Summary.TotalFailureSessionCount = 0
	}

Merge:
	for _, nfd := range fds {
		for i, fd := range r.FailureDetails {
			if !fd.equalKey(nfd) {
				continue
			}

			fd.FailedSessionCount += nfd.FailedSessionCount
			r.FailureDetails[i] = fd
			continue Merge
		}
		r.FailureDetails = append(r.FailureDetails, nfd)
	}
}

// Add is a convenience function for merging making a Result and merging it into
// the report.
func (r *Report) Add(policy ResultPolicy, success, failure int64, fds ...FailureDetails) {
	r.Merge(Result{policy, Summary{success, failure}, fds})
}

// TLSAPolicy returns a policy for DANE.
func TLSAPolicy(records []adns.TLSA, tlsaBaseDomain dns.Domain) ResultPolicy {
	// The policy domain is the TLSA base domain. ../rfc/8460:251

	l := make([]string, len(records))
	for i, r := range records {
		l[i] = r.Record()
	}
	sort.Strings(l) // For consistent equals.
	return ResultPolicy{
		Type:   TLSA,
		String: l,
		Domain: tlsaBaseDomain.ASCII,
		MXHost: []string{},
	}
}

func MakeResult(policyType PolicyType, domain dns.Domain, fds ...FailureDetails) Result {
	if fds == nil {
		fds = []FailureDetails{}
	}
	return Result{
		Policy:         ResultPolicy{Type: policyType, Domain: domain.ASCII, String: []string{}, MXHost: []string{}},
		FailureDetails: fds,
	}
}

// note: with TLSRPT prefix to prevent clash in sherpadoc types.
type TLSRPTDateRange struct {
	Start time.Time
	End   time.Time
}

func (v TLSRPTDateRange) Convert() TLSRPTDateRangeJSON {
	return TLSRPTDateRangeJSON(v)
}

type TLSRPTDateRangeJSON struct {
	Start time.Time `json:"start-datetime"`
	End   time.Time `json:"end-datetime"`
}

func (v TLSRPTDateRangeJSON) Convert() TLSRPTDateRange {
	return TLSRPTDateRange(v)
}

// UnmarshalJSON is defined on the date range, not the individual time.Time fields
// because it is easier to keep the unmodified time.Time fields stored in the
// database.
func (dr *TLSRPTDateRangeJSON) UnmarshalJSON(buf []byte) error {
	var v struct {
		Start xtime `json:"start-datetime"`
		End   xtime `json:"end-datetime"`
	}
	if err := json.Unmarshal(buf, &v); err != nil {
		return err
	}
	dr.Start = time.Time(v.Start)
	dr.End = time.Time(v.End)
	return nil
}

// xtime and its UnmarshalJSON exists to work around a specific invalid date-time encoding seen in the wild.
type xtime time.Time

func (x *xtime) UnmarshalJSON(buf []byte) error {
	var t time.Time
	err := t.UnmarshalJSON(buf)
	if err == nil {
		*x = xtime(t)
		return nil
	}

	// Microsoft is sending reports with invalid start-datetime/end-datetime (missing
	// timezone, ../rfc/8460:682 ../rfc/3339:415). We compensate.
	var s string
	if err := json.Unmarshal(buf, &s); err != nil {
		return err
	}
	t, err = time.Parse("2006-01-02T15:04:05", s)
	if err != nil {
		return err
	}
	*x = xtime(t)
	return nil
}

type Result struct {
	Policy         ResultPolicy
	Summary        Summary
	FailureDetails []FailureDetails
}

func (r Result) Convert() ResultJSON {
	return ResultJSON{ResultPolicyJSON(r.Policy), SummaryJSON(r.Summary), convertSlice[FailureDetails, FailureDetailsJSON](r.FailureDetails)}
}

type ResultJSON struct {
	Policy         ResultPolicyJSON     `json:"policy"`
	Summary        SummaryJSON          `json:"summary"`
	FailureDetails []FailureDetailsJSON `json:"failure-details"`
}

func (r ResultJSON) Convert() Result {
	return Result{ResultPolicy(r.Policy), Summary(r.Summary), convertSlice[FailureDetailsJSON, FailureDetails](r.FailureDetails)}
}

// todo spec: ../rfc/8460:437 says policy is a string, with rules for turning dane records into a single string. perhaps a remnant of an earlier version (for mtasts a single string would have made more sense). i doubt the intention is to always have a single element in policy-string (though the field name is singular).

type ResultPolicy struct {
	Type   PolicyType
	String []string
	Domain string // ASCII/A-labels, ../rfc/8460:704
	MXHost []string
}

type ResultPolicyJSON struct {
	Type   PolicyType `json:"policy-type"`
	String []string   `json:"policy-string"`
	Domain string     `json:"policy-domain"`
	MXHost []string   `json:"mx-host"` // Example in RFC has errata, it originally was a single string. ../rfc/8460-eid6241 ../rfc/8460:1779
}

// PolicyType indicates the policy success/failure results are for.
type PolicyType string

const (
	// For DANE, against a mail host (not recipient domain).
	TLSA PolicyType = "tlsa"

	// For MTA-STS, against a recipient domain (not a mail host).
	STS PolicyType = "sts"

	// Recipient domain did not have MTA-STS policy, or mail host (TSLA base domain)
	// did not have DANE TLSA records.
	NoPolicyFound PolicyType = "no-policy-found"
	// todo spec: ../rfc/8460:440 ../rfc/8460:697 suggest to replace with values like "no-sts-found" and "no-tlsa-found" to make it explicit which policy isn't found. also easier to implement, because you don't have to handle leaving out an sts no-policy-found result for a mail host when a tlsa policy is present.
)

func (rp ResultPolicy) equal(orp ResultPolicy) bool {
	return rp.Type == orp.Type && slices.Equal(rp.String, orp.String) && rp.Domain == orp.Domain && slices.Equal(rp.MXHost, orp.MXHost)
}

type Summary struct {
	TotalSuccessfulSessionCount int64
	TotalFailureSessionCount    int64
}

type SummaryJSON struct {
	TotalSuccessfulSessionCount int64 `json:"total-successful-session-count"`
	TotalFailureSessionCount    int64 `json:"total-failure-session-count"`
}

// ResultType represents a TLS error.
type ResultType string

// ../rfc/8460:1377
// https://www.iana.org/assignments/starttls-validation-result-types/starttls-validation-result-types.xhtml

const (
	ResultSTARTTLSNotSupported    ResultType = "starttls-not-supported"
	ResultCertificateHostMismatch ResultType = "certificate-host-mismatch"
	ResultCertificateExpired      ResultType = "certificate-expired"
	ResultTLSAInvalid             ResultType = "tlsa-invalid"
	ResultDNSSECInvalid           ResultType = "dnssec-invalid"
	ResultDANERequired            ResultType = "dane-required"
	ResultCertificateNotTrusted   ResultType = "certificate-not-trusted"
	ResultSTSPolicyInvalid        ResultType = "sts-policy-invalid"
	ResultSTSWebPKIInvalid        ResultType = "sts-webpki-invalid"
	ResultValidationFailure       ResultType = "validation-failure" // Other error.
	ResultSTSPolicyFetch          ResultType = "sts-policy-fetch-error"
)

// todo spec: ../rfc/8460:719 more of these fields should be optional. some sts failure details, like failed policy fetches, won't have an ip or mx, the failure happens earlier in the delivery process.

type FailureDetails struct {
	ResultType            ResultType
	SendingMTAIP          string
	ReceivingMXHostname   string
	ReceivingMXHelo       string
	ReceivingIP           string
	FailedSessionCount    int64
	AdditionalInformation string
	FailureReasonCode     string
}

func (v FailureDetails) Convert() FailureDetailsJSON { return FailureDetailsJSON(v) }

type FailureDetailsJSON struct {
	ResultType            ResultType `json:"result-type"`
	SendingMTAIP          string     `json:"sending-mta-ip"`
	ReceivingMXHostname   string     `json:"receiving-mx-hostname"`
	ReceivingMXHelo       string     `json:"receiving-mx-helo,omitempty"`
	ReceivingIP           string     `json:"receiving-ip"`
	FailedSessionCount    int64      `json:"failed-session-count"`
	AdditionalInformation string     `json:"additional-information"`
	FailureReasonCode     string     `json:"failure-reason-code"`
}

func (v FailureDetailsJSON) Convert() FailureDetails { return FailureDetails(v) }

// equalKey returns whether FailureDetails have the same values, expect for
// FailedSessionCount. Useful for aggregating FailureDetails.
func (fd FailureDetails) equalKey(ofd FailureDetails) bool {
	fd.FailedSessionCount = 0
	ofd.FailedSessionCount = 0
	return fd == ofd
}

// Details is a convenience function to compose a FailureDetails.
func Details(t ResultType, r string) FailureDetails {
	return FailureDetails{ResultType: t, FailedSessionCount: 1, FailureReasonCode: r}
}

var invalidReasons = map[x509.InvalidReason]string{
	x509.NotAuthorizedToSign:           "not-authorized-to-sign",
	x509.Expired:                       "certificate-expired",
	x509.CANotAuthorizedForThisName:    "ca-not-authorized-for-this-name",
	x509.TooManyIntermediates:          "too-many-intermediates",
	x509.IncompatibleUsage:             "incompatible-key-usage",
	x509.NameMismatch:                  "parent-subject-child-issuer-mismatch",
	x509.NameConstraintsWithoutSANs:    "name-constraint-without-sans",
	x509.UnconstrainedName:             "unconstrained-name",
	x509.TooManyConstraints:            "too-many-constraints",
	x509.CANotAuthorizedForExtKeyUsage: "ca-not-authorized-for-ext-key-usage",
}

// TLSFailureDetails turns errors encountered during TLS handshakes into a result
// type and failure reason code for use with FailureDetails.
//
// Errors from crypto/tls, including local and remote alerts, from crypto/x509,
// and generic i/o and timeout errors are recognized.
func TLSFailureDetails(err error) (ResultType, string) {
	var invalidErr x509.CertificateInvalidError
	var hostErr x509.HostnameError
	var unknownAuthErr x509.UnknownAuthorityError
	var rootsErr x509.SystemRootsError
	var verifyErr *tls.CertificateVerificationError
	var netErr *net.OpError
	var recordHdrErr tls.RecordHeaderError
	if errors.As(err, &invalidErr) {
		if invalidErr.Reason == x509.Expired {
			// Result: ../rfc/8460:546
			return ResultCertificateExpired, ""
		}
		s, ok := invalidReasons[invalidErr.Reason]
		if !ok {
			s = fmt.Sprintf("go-x509-invalid-reason-%d", invalidErr.Reason)
		}
		// Result: ../rfc/8460:549
		return ResultCertificateNotTrusted, s
	} else if errors.As(err, &hostErr) {
		// Result: ../rfc/8460:541
		return ResultCertificateHostMismatch, ""
	} else if errors.As(err, &unknownAuthErr) {
		// Result: ../rfc/8460:549
		return ResultCertificateNotTrusted, ""
	} else if errors.As(err, &rootsErr) {
		// Result: ../rfc/8460:549
		return ResultCertificateNotTrusted, "no-system-roots"
	} else if errors.As(err, &verifyErr) {
		// We don't know a more specific error. ../rfc/8460:610
		// Result: ../rfc/8460:567
		return ResultValidationFailure, "unknown-go-certificate-verification-error"
	} else if errors.As(err, &netErr) && netErr.Op == "remote error" {
		// This is how TLS errors from the server (through an alert) are represented by
		// crypto/tls. Err will usually be tls.alert error that is a type around uint8.
		reasonCode := "tls-remote-error"
		if netErr.Err != nil {
			// todo: ideally, crypto/tls would let us check if this is an alert. it could be another uint8-typed error.
			v := reflect.ValueOf(netErr.Err)
			if v.Kind() == reflect.Uint8 && v.Type().Name() == "alert" {
				reasonCode = "tls-remote-" + formatAlert(uint8(v.Uint()))
			}
		}
		return ResultValidationFailure, reasonCode
	} else if errors.As(err, &recordHdrErr) {
		// Like for AlertError, not a lot of details, but better than nothing.
		// Result: ../rfc/8460:567
		return ResultValidationFailure, "tls-record-header-error"
	}

	// Consider not adding failure details at all for transient errors? It probably
	// isn't very common to have an accidental connection failure during STARTTL setup
	// after having completed SMTP TCP setup and having exchanged commands. Seems best
	// to report on them. ../rfc/8460:625
	// Could be any other kind of error, we try to report on i/o errors, but best not to claim any
	// other reason we don't know about. ../rfc/8460:610
	// Result: ../rfc/8460:567
	var reasonCode string
	if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) {
		reasonCode = "io-timeout-during-handshake"
	} else if moxio.IsClosed(err) || errors.Is(err, io.ErrClosedPipe) {
		reasonCode = "connection-closed-during-handshake"
	} else {
		// Attempt to get a local, outgoing TLS alert.
		// We unwrap the error to the end (not multiple errors), and check for uint8 of a
		// type named "alert".
		for {
			uerr := errors.Unwrap(err)
			if uerr == nil {
				break
			}
			err = uerr
		}
		v := reflect.ValueOf(err)
		if v.Kind() == reflect.Uint8 && v.Type().Name() == "alert" {
			reasonCode = "tls-local-" + formatAlert(uint8(v.Uint()))
		}
	}
	return ResultValidationFailure, reasonCode
}

// Parse parses a Report.
// The maximum size is 20MB.
func Parse(r io.Reader) (*ReportJSON, error) {
	r = &moxio.LimitReader{R: r, Limit: 20 * 1024 * 1024}
	var report ReportJSON
	if err := json.NewDecoder(r).Decode(&report); err != nil {
		return nil, err
	}
	// note: there may be leftover data, we ignore it.
	return &report, nil
}

// ParseMessage parses a Report from a mail message.
// The maximum size of the message is 15MB, the maximum size of the
// decompressed report is 20MB.
func ParseMessage(elog *slog.Logger, r io.ReaderAt) (*ReportJSON, error) {
	log := mlog.New("tlsrpt", elog)

	// ../rfc/8460:905
	p, err := message.Parse(log.Logger, true, &moxio.LimitAtReader{R: r, Limit: 15 * 1024 * 1024})
	if err != nil {
		return nil, fmt.Errorf("parsing mail message: %s", err)
	}

	// Using multipart appears optional, and similar to DMARC someone may decide to
	// send it like that, so accept a report if it's the entire message.
	const allow = true
	return parseMessageReport(log, p, allow)
}

func parseMessageReport(log mlog.Log, p message.Part, allow bool) (*ReportJSON, error) {
	if p.MediaType != "MULTIPART" {
		if !allow {
			return nil, ErrNoReport
		}
		return parseReport(p)
	}

	for {
		sp, err := p.ParseNextPart(log.Logger)
		if err == io.EOF {
			return nil, ErrNoReport
		}
		if err != nil {
			return nil, err
		}
		if p.MediaSubType == "REPORT" && p.ContentTypeParams["report-type"] != "tlsrpt" {
			return nil, fmt.Errorf("unknown report-type parameter %q", p.ContentTypeParams["report-type"])
		}
		report, err := parseMessageReport(log, *sp, p.MediaSubType == "REPORT")
		if err == ErrNoReport {
			continue
		} else if err != nil || report != nil {
			return report, err
		}
	}
}

func parseReport(p message.Part) (*ReportJSON, error) {
	mt := strings.ToLower(p.MediaType + "/" + p.MediaSubType)
	switch mt {
	case "application/tlsrpt+json":
		return Parse(p.Reader())
	case "application/tlsrpt+gzip":
		gzr, err := gzip.NewReader(p.Reader())
		if err != nil {
			return nil, fmt.Errorf("decoding gzip TLSRPT report: %s", err)
		}
		return Parse(gzr)
	}
	return nil, ErrNoReport
}