expose fewer internals in packages, for easier software reuse

- prometheus is now behind an interface, they aren't dependencies for the
  reusable components anymore.
- some dependencies have been inverted: instead of packages importing a main
  package to get configuration, the main package now sets configuration in
  these packages. that means fewer internals are pulled in.
- some functions now have new parameters for values that were retrieved from
  package "mox-".
This commit is contained in:
Mechiel Lukkien 2023-12-05 21:13:57 +01:00
parent fcaa504878
commit 72ac1fde29
No known key found for this signature in database
51 changed files with 696 additions and 568 deletions

View file

@ -61,29 +61,16 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/adns" "github.com/mjl-/adns"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/stub"
) )
var ( var (
metricVerify = promauto.NewCounter( MetricVerify stub.Counter = stub.CounterIgnore{}
prometheus.CounterOpts{ MetricVerifyErrors stub.Counter = stub.CounterIgnore{}
Name: "mox_dane_verify_total",
Help: "Total number of DANE verification attempts, including mox_dane_verify_errors_total.",
},
)
metricVerifyErrors = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_dane_verify_errors_total",
Help: "Total number of DANE verification failures, causing connections to fail.",
},
)
) )
var ( var (
@ -134,7 +121,7 @@ func (e VerifyError) Unwrap() error {
// indicate DNSSEC errors. // indicate DNSSEC errors.
// - ErrInsecure // - ErrInsecure
// - VerifyError, potentially wrapping errors from crypto/x509. // - VerifyError, potentially wrapping errors from crypto/x509.
func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage) (net.Conn, adns.TLSA, error) { func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network, address string, allowedUsages []adns.TLSAUsage, pkixRoots *x509.CertPool) (net.Conn, adns.TLSA, error) {
log := mlog.New("dane", elog) log := mlog.New("dane", elog)
// Split host and port. // Split host and port.
@ -274,7 +261,7 @@ func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network
} }
var verifiedRecord adns.TLSA var verifiedRecord adns.TLSA
config := TLSClientConfig(log.Logger, records, baseDom, moreAllowedHosts, &verifiedRecord) config := TLSClientConfig(log.Logger, records, baseDom, moreAllowedHosts, &verifiedRecord, pkixRoots)
tlsConn := tls.Client(conn, &config) tlsConn := tls.Client(conn, &config)
if err := tlsConn.HandshakeContext(ctx); err != nil { if err := tlsConn.HandshakeContext(ctx); err != nil {
conn.Close() conn.Close()
@ -297,13 +284,13 @@ func Dial(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, network
// //
// If verifiedRecord is not nil, it is set to the record that was successfully // If verifiedRecord is not nil, it is set to the record that was successfully
// verified, if any. // verified, if any.
func TLSClientConfig(elog *slog.Logger, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config { func TLSClientConfig(elog *slog.Logger, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA, pkixRoots *x509.CertPool) tls.Config {
log := mlog.New("dane", elog) log := mlog.New("dane", elog)
return tls.Config{ return tls.Config{
ServerName: allowedHost.ASCII, // For SNI. ServerName: allowedHost.ASCII, // For SNI.
InsecureSkipVerify: true, InsecureSkipVerify: true,
VerifyConnection: func(cs tls.ConnectionState) error { VerifyConnection: func(cs tls.ConnectionState) error {
verified, record, err := Verify(log.Logger, records, cs, allowedHost, moreAllowedHosts) verified, record, err := Verify(log.Logger, records, cs, allowedHost, moreAllowedHosts, pkixRoots)
log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record)) log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record))
if verified { if verified {
if verifiedRecord != nil { if verifiedRecord != nil {
@ -335,23 +322,23 @@ func TLSClientConfig(elog *slog.Logger, records []adns.TLSA, allowedHost dns.Dom
// If an error is encountered while verifying a record, e.g. for x509 // If an error is encountered while verifying a record, e.g. for x509
// trusted-anchor verification, an error may be returned, typically one or more // trusted-anchor verification, an error may be returned, typically one or more
// (wrapped) errors of type VerifyError. // (wrapped) errors of type VerifyError.
func Verify(elog *slog.Logger, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, matching adns.TLSA, rerr error) { func Verify(elog *slog.Logger, records []adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, pkixRoots *x509.CertPool) (verified bool, matching adns.TLSA, rerr error) {
log := mlog.New("dane", elog) log := mlog.New("dane", elog)
metricVerify.Inc() MetricVerify.Inc()
if len(records) == 0 { if len(records) == 0 {
metricVerifyErrors.Inc() MetricVerifyErrors.Inc()
return false, adns.TLSA{}, fmt.Errorf("verify requires at least one tlsa record") return false, adns.TLSA{}, fmt.Errorf("verify requires at least one tlsa record")
} }
var errs []error var errs []error
for _, r := range records { for _, r := range records {
ok, err := verifySingle(log, r, cs, allowedHost, moreAllowedHosts) ok, err := verifySingle(log, r, cs, allowedHost, moreAllowedHosts, pkixRoots)
if err != nil { if err != nil {
errs = append(errs, VerifyError{err, r}) errs = append(errs, VerifyError{err, r})
} else if ok { } else if ok {
return true, r, nil return true, r, nil
} }
} }
metricVerifyErrors.Inc() MetricVerifyErrors.Inc()
return false, adns.TLSA{}, errors.Join(errs...) return false, adns.TLSA{}, errors.Join(errs...)
} }
@ -364,7 +351,7 @@ func Verify(elog *slog.Logger, records []adns.TLSA, cs tls.ConnectionState, allo
// errors while verifying certificates against a trust-anchor, an error can be // errors while verifying certificates against a trust-anchor, an error can be
// returned with one or more underlying x509 verification errors. A nil-nil error // returned with one or more underlying x509 verification errors. A nil-nil error
// is only returned when verified is false. // is only returned when verified is false.
func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain) (verified bool, rerr error) { func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, pkixRoots *x509.CertPool) (verified bool, rerr error) {
if len(cs.PeerCertificates) == 0 { if len(cs.PeerCertificates) == 0 {
return false, fmt.Errorf("no server certificate") return false, fmt.Errorf("no server certificate")
} }
@ -401,7 +388,7 @@ func verifySingle(log mlog.Log, tlsa adns.TLSA, cs tls.ConnectionState, allowedH
opts := x509.VerifyOptions{ opts := x509.VerifyOptions{
DNSName: host.ASCII, DNSName: host.ASCII,
Intermediates: x509.NewCertPool(), Intermediates: x509.NewCertPool(),
Roots: mox.Conf.Static.TLS.CertPool, Roots: pkixRoots,
} }
for _, cert := range cs.PeerCertificates[1:] { for _, cert := range cs.PeerCertificates[1:] {
opts.Intermediates.AddCert(cert) opts.Intermediates.AddCert(cert)

View file

@ -26,7 +26,6 @@ import (
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
) )
func tcheckf(t *testing.T, err error, format string, args ...any) { func tcheckf(t *testing.T, err error, format string, args ...any) {
@ -137,11 +136,13 @@ func TestDial(t *testing.T) {
dialHost := "localhost" dialHost := "localhost"
var allowedUsages []adns.TLSAUsage var allowedUsages []adns.TLSAUsage
pkixRoots := x509.NewCertPool()
// Helper function for dialing with DANE. // Helper function for dialing with DANE.
test := func(resolver dns.Resolver, expRecord adns.TLSA, expErr any) { test := func(resolver dns.Resolver, expRecord adns.TLSA, expErr any) {
t.Helper() t.Helper()
conn, record, err := Dial(context.Background(), log.Logger, resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages) conn, record, err := Dial(context.Background(), log.Logger, resolver, "tcp", net.JoinHostPort(dialHost, portstr), allowedUsages, pkixRoots)
if err == nil { if err == nil {
conn.Close() conn.Close()
} }
@ -457,11 +458,8 @@ func TestDial(t *testing.T) {
} }
test(resolver, zeroRecord, &x509.UnknownAuthorityError{}) test(resolver, zeroRecord, &x509.UnknownAuthorityError{})
// Now we add the TA to the "system" trusted roots and try again. // Now we add the TA to the "pkix" trusted roots and try again.
pool, err := x509.SystemCertPool() pkixRoots.AddCert(taCert)
tcheckf(t, err, "get system certificate pool")
mox.Conf.Static.TLS.CertPool = pool
pool.AddCert(taCert)
// PKIXEE, will now succeed. // PKIXEE, will now succeed.
resolver = dns.MockResolver{ resolver = dns.MockResolver{

View file

@ -26,38 +26,17 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/stub"
) )
var ( var (
metricSign = promauto.NewCounterVec( MetricSign stub.CounterVec = stub.CounterVecIgnore{}
prometheus.CounterOpts{ MetricVerify stub.HistogramVec = stub.HistogramVecIgnore{}
Name: "mox_dkim_sign_total",
Help: "DKIM messages signings, label key is the type of key, rsa or ed25519.",
},
[]string{
"key",
},
)
metricVerify = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_dkim_verify_duration_seconds",
Help: "DKIM verify, including lookup, duration and result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"algorithm",
"status",
},
)
) )
var timeNow = time.Now // Replaced during tests. var timeNow = time.Now // Replaced during tests.
@ -122,8 +101,28 @@ type Result struct {
// todo: use some io.Writer to hash the body and the header. // todo: use some io.Writer to hash the body and the header.
// Selector holds selectors and key material to generate DKIM signatures.
type Selector struct {
Hash string // "sha256" or the older "sha1".
HeaderRelaxed bool // If the header is canonicalized in relaxed instead of simple mode.
BodyRelaxed bool // If the body is canonicalized in relaxed instead of simple mode.
Headers []string // Headers to include in signature.
// Whether to "oversign" headers, ensuring additional/new values of existing
// headers cannot be added.
SealHeaders bool
// If > 0, period a signature is valid after signing, as duration, e.g. 72h. The
// period should be enough for delivery at the final destination, potentially with
// several hops/relays. In the order of days at least.
Expiration time.Duration
PrivateKey crypto.Signer // Either an *rsa.PrivateKey or ed25519.PrivateKey.
Domain dns.Domain // Of selector only, not FQDN.
}
// Sign returns line(s) with DKIM-Signature headers, generated according to the configuration. // Sign returns line(s) with DKIM-Signature headers, generated according to the configuration.
func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, domain dns.Domain, c config.DKIM, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) { func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, domain dns.Domain, selectors []Selector, smtputf8 bool, msg io.ReaderAt) (headers string, rerr error) {
log := mlog.New("dkim", elog) log := mlog.New("dkim", elog)
start := timeNow() start := timeNow()
defer func() { defer func() {
@ -155,26 +154,25 @@ func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, doma
var bodyHashes = map[hashKey][]byte{} var bodyHashes = map[hashKey][]byte{}
for _, sign := range c.Sign { for _, sel := range selectors {
sel := c.Selectors[sign]
sig := newSigWithDefaults() sig := newSigWithDefaults()
sig.Version = 1 sig.Version = 1
switch sel.Key.(type) { switch sel.PrivateKey.(type) {
case *rsa.PrivateKey: case *rsa.PrivateKey:
sig.AlgorithmSign = "rsa" sig.AlgorithmSign = "rsa"
metricSign.WithLabelValues("rsa").Inc() MetricSign.IncLabels("rsa")
case ed25519.PrivateKey: case ed25519.PrivateKey:
sig.AlgorithmSign = "ed25519" sig.AlgorithmSign = "ed25519"
metricSign.WithLabelValues("ed25519").Inc() MetricSign.IncLabels("ed25519")
default: default:
return "", fmt.Errorf("internal error, unknown pivate key %T", sel.Key) return "", fmt.Errorf("internal error, unknown pivate key %T", sel.PrivateKey)
} }
sig.AlgorithmHash = sel.HashEffective sig.AlgorithmHash = sel.Hash
sig.Domain = domain sig.Domain = domain
sig.Selector = sel.Domain sig.Selector = sel.Domain
sig.Identity = &Identity{&localpart, domain} sig.Identity = &Identity{&localpart, domain}
sig.SignedHeaders = append([]string{}, sel.HeadersEffective...) sig.SignedHeaders = append([]string{}, sel.Headers...)
if !sel.DontSealHeaders { if sel.SealHeaders {
// ../rfc/6376:2156 // ../rfc/6376:2156
// Each time a header name is added to the signature, the next unused value is // Each time a header name is added to the signature, the next unused value is
// signed (in reverse order as they occur in the message). So we can add each // signed (in reverse order as they occur in the message). So we can add each
@ -184,23 +182,23 @@ func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, doma
for _, h := range hdrs { for _, h := range hdrs {
counts[h.lkey]++ counts[h.lkey]++
} }
for _, h := range sel.HeadersEffective { for _, h := range sel.Headers {
for j := counts[strings.ToLower(h)]; j > 0; j-- { for j := counts[strings.ToLower(h)]; j > 0; j-- {
sig.SignedHeaders = append(sig.SignedHeaders, h) sig.SignedHeaders = append(sig.SignedHeaders, h)
} }
} }
} }
sig.SignTime = timeNow().Unix() sig.SignTime = timeNow().Unix()
if sel.ExpirationSeconds > 0 { if sel.Expiration > 0 {
sig.ExpireTime = sig.SignTime + int64(sel.ExpirationSeconds) sig.ExpireTime = sig.SignTime + int64(sel.Expiration/time.Second)
} }
sig.Canonicalization = "simple" sig.Canonicalization = "simple"
if sel.Canonicalization.HeaderRelaxed { if sel.HeaderRelaxed {
sig.Canonicalization = "relaxed" sig.Canonicalization = "relaxed"
} }
sig.Canonicalization += "/" sig.Canonicalization += "/"
if sel.Canonicalization.BodyRelaxed { if sel.BodyRelaxed {
sig.Canonicalization += "relaxed" sig.Canonicalization += "relaxed"
} else { } else {
sig.Canonicalization += "simple" sig.Canonicalization += "simple"
@ -217,12 +215,12 @@ func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, doma
// DKIM-Signature header. // DKIM-Signature header.
// ../rfc/6376:1700 // ../rfc/6376:1700
hk := hashKey{!sel.Canonicalization.BodyRelaxed, strings.ToLower(sig.AlgorithmHash)} hk := hashKey{!sel.BodyRelaxed, strings.ToLower(sig.AlgorithmHash)}
if bh, ok := bodyHashes[hk]; ok { if bh, ok := bodyHashes[hk]; ok {
sig.BodyHash = bh sig.BodyHash = bh
} else { } else {
br := bufio.NewReader(&moxio.AtReader{R: msg, Offset: int64(bodyOffset)}) br := bufio.NewReader(&moxio.AtReader{R: msg, Offset: int64(bodyOffset)})
bh, err = bodyHash(h.New(), !sel.Canonicalization.BodyRelaxed, br) bh, err = bodyHash(h.New(), !sel.BodyRelaxed, br)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -236,12 +234,12 @@ func Sign(ctx context.Context, elog *slog.Logger, localpart smtp.Localpart, doma
} }
verifySig := []byte(strings.TrimSuffix(sigh, "\r\n")) verifySig := []byte(strings.TrimSuffix(sigh, "\r\n"))
dh, err := dataHash(h.New(), !sel.Canonicalization.HeaderRelaxed, sig, hdrs, verifySig) dh, err := dataHash(h.New(), !sel.HeaderRelaxed, sig, hdrs, verifySig)
if err != nil { if err != nil {
return "", err return "", err
} }
switch key := sel.Key.(type) { switch key := sel.PrivateKey.(type) {
case *rsa.PrivateKey: case *rsa.PrivateKey:
sig.Signature, err = key.Sign(cryptorand.Reader, dh, h) sig.Signature, err = key.Sign(cryptorand.Reader, dh, h)
if err != nil { if err != nil {
@ -358,7 +356,7 @@ func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, smtpu
alg = r.Sig.Algorithm() alg = r.Sig.Algorithm()
} }
status := string(r.Status) status := string(r.Status)
metricVerify.WithLabelValues(alg, status).Observe(duration) MetricVerify.ObserveLabels(duration, alg, status)
} }
if len(results) == 0 { if len(results) == 0 {

View file

@ -15,7 +15,6 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
) )
@ -222,50 +221,42 @@ test
rsaKey := getRSAKey(t) rsaKey := getRSAKey(t)
ed25519Key := ed25519.NewKeyFromSeed(make([]byte, 32)) ed25519Key := ed25519.NewKeyFromSeed(make([]byte, 32))
selrsa := config.Selector{ selrsa := Selector{
HashEffective: "sha256", Hash: "sha256",
Key: rsaKey, PrivateKey: rsaKey,
HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","), Headers: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
Domain: dns.Domain{ASCII: "testrsa"}, Domain: dns.Domain{ASCII: "testrsa"},
} }
// Now with sha1 and relaxed canonicalization. // Now with sha1 and relaxed canonicalization.
selrsa2 := config.Selector{ selrsa2 := Selector{
HashEffective: "sha1", Hash: "sha1",
Key: rsaKey, PrivateKey: rsaKey,
HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","), Headers: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
Domain: dns.Domain{ASCII: "testrsa2"}, Domain: dns.Domain{ASCII: "testrsa2"},
} }
selrsa2.Canonicalization.HeaderRelaxed = true selrsa2.HeaderRelaxed = true
selrsa2.Canonicalization.BodyRelaxed = true selrsa2.BodyRelaxed = true
// Ed25519 key. // Ed25519 key.
seled25519 := config.Selector{ seled25519 := Selector{
HashEffective: "sha256", Hash: "sha256",
Key: ed25519Key, PrivateKey: ed25519Key,
HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","), Headers: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
Domain: dns.Domain{ASCII: "tested25519"}, Domain: dns.Domain{ASCII: "tested25519"},
} }
// Again ed25519, but without sealing headers. Use sha256 again, for reusing the body hash from the previous dkim-signature. // Again ed25519, but without sealing headers. Use sha256 again, for reusing the body hash from the previous dkim-signature.
seled25519b := config.Selector{ seled25519b := Selector{
HashEffective: "sha256", Hash: "sha256",
Key: ed25519Key, PrivateKey: ed25519Key,
HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,Subject,Date", ","), Headers: strings.Split("From,To,Cc,Bcc,Reply-To,Subject,Date", ","),
DontSealHeaders: true, SealHeaders: true,
Domain: dns.Domain{ASCII: "tested25519b"}, Domain: dns.Domain{ASCII: "tested25519b"},
}
dkimConf := config.DKIM{
Selectors: map[string]config.Selector{
"testrsa": selrsa,
"testrsa2": selrsa2,
"tested25519": seled25519,
"tested25519b": seled25519b,
},
Sign: []string{"testrsa", "testrsa2", "tested25519", "tested25519b"},
} }
selectors := []Selector{selrsa, selrsa2, seled25519, seled25519b}
ctx := context.Background() ctx := context.Background()
headers, err := Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(message)) headers, err := Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, selectors, false, strings.NewReader(message))
if err != nil { if err != nil {
t.Fatalf("sign: %v", err) t.Fatalf("sign: %v", err)
} }
@ -307,31 +298,31 @@ test
//log.Infof("nmsg\n%s", nmsg) //log.Infof("nmsg\n%s", nmsg)
// Multiple From headers. // Multiple From headers.
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From: <mjl@mox.example>\r\nFrom: <mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, selectors, false, strings.NewReader("From: <mjl@mox.example>\r\nFrom: <mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrFrom) { if !errors.Is(err, ErrFrom) {
t.Fatalf("sign, got err %v, expected ErrFrom", err) t.Fatalf("sign, got err %v, expected ErrFrom", err)
} }
// No From header. // No From header.
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Brom: <mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, selectors, false, strings.NewReader("Brom: <mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrFrom) { if !errors.Is(err, ErrFrom) {
t.Fatalf("sign, got err %v, expected ErrFrom", err) t.Fatalf("sign, got err %v, expected ErrFrom", err)
} }
// Malformed headers. // Malformed headers.
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(":\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, selectors, false, strings.NewReader(":\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader(" From:<mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, selectors, false, strings.NewReader(" From:<mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("Frøm:<mjl@mox.example>\r\n\r\ntest")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, selectors, false, strings.NewReader("Frøm:<mjl@mox.example>\r\n\r\ntest"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
_, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, dkimConf, false, strings.NewReader("From:<mjl@mox.example>")) _, err = Sign(ctx, pkglog.Logger, "mjl", dns.Domain{ASCII: "mox.example"}, selectors, false, strings.NewReader("From:<mjl@mox.example>"))
if !errors.Is(err, ErrHeaderMalformed) { if !errors.Is(err, ErrHeaderMalformed) {
t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err) t.Fatalf("sign, got err %v, expected ErrHeaderMalformed", err)
} }
@ -358,9 +349,9 @@ test
var record *Record var record *Record
var recordTxt string var recordTxt string
var msg string var msg string
var sel config.Selector
var dkimConf config.DKIM
var policy func(*Sig) error var policy func(*Sig) error
var sel Selector
var selectors []Selector
var signed bool var signed bool
var signDomain dns.Domain var signDomain dns.Domain
@ -389,18 +380,13 @@ test
}, },
} }
sel = config.Selector{ sel = Selector{
HashEffective: "sha256", Hash: "sha256",
Key: key, PrivateKey: key,
HeadersEffective: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","), Headers: strings.Split("From,To,Cc,Bcc,Reply-To,References,In-Reply-To,Subject,Date,Message-ID,Content-Type", ","),
Domain: dns.Domain{ASCII: "test"}, Domain: dns.Domain{ASCII: "test"},
}
dkimConf = config.DKIM{
Selectors: map[string]config.Selector{
"test": sel,
},
Sign: []string{"test"},
} }
selectors = []Selector{sel}
msg = message msg = message
signed = false signed = false
@ -411,7 +397,7 @@ test
msg = strings.ReplaceAll(msg, "\n", "\r\n") msg = strings.ReplaceAll(msg, "\n", "\r\n")
headers, err := Sign(context.Background(), pkglog.Logger, "mjl", signDomain, dkimConf, false, strings.NewReader(msg)) headers, err := Sign(context.Background(), pkglog.Logger, "mjl", signDomain, selectors, false, strings.NewReader(msg))
if err != nil { if err != nil {
t.Fatalf("sign: %v", err) t.Fatalf("sign: %v", err)
} }
@ -515,11 +501,9 @@ test
}) })
// Unknown canonicalization. // Unknown canonicalization.
test(nil, StatusPermerror, ErrCanonicalizationUnknown, func() { test(nil, StatusPermerror, ErrCanonicalizationUnknown, func() {
sel.Canonicalization.HeaderRelaxed = true sel.HeaderRelaxed = true
sel.Canonicalization.BodyRelaxed = true sel.BodyRelaxed = true
dkimConf.Selectors = map[string]config.Selector{ selectors = []Selector{sel}
"test": sel,
}
sign() sign()
msg = strings.ReplaceAll(msg, "relaxed/relaxed", "bogus/bogus") msg = strings.ReplaceAll(msg, "relaxed/relaxed", "bogus/bogus")
@ -577,10 +561,8 @@ test
resolver.TXT = map[string][]string{ resolver.TXT = map[string][]string{
"test._domainkey.mox.example.": {txt}, "test._domainkey.mox.example.": {txt},
} }
sel.Key = key sel.PrivateKey = key
dkimConf.Selectors = map[string]config.Selector{ selectors = []Selector{sel}
"test": sel,
}
}) })
// Key not allowed for email by DNS record. ../rfc/6376:1541 // Key not allowed for email by DNS record. ../rfc/6376:1541
test(nil, StatusPermerror, ErrKeyNotForEmail, func() { test(nil, StatusPermerror, ErrKeyNotForEmail, func() {
@ -603,18 +585,14 @@ test
// Check that last-occurring header field is used. // Check that last-occurring header field is used.
test(nil, StatusFail, ErrSigVerify, func() { test(nil, StatusFail, ErrSigVerify, func() {
sel.DontSealHeaders = true sel.SealHeaders = false
dkimConf.Selectors = map[string]config.Selector{ selectors = []Selector{sel}
"test": sel,
}
sign() sign()
msg = strings.ReplaceAll(msg, "\r\n\r\n", "\r\nsubject: another\r\n\r\n") msg = strings.ReplaceAll(msg, "\r\n\r\n", "\r\nsubject: another\r\n\r\n")
}) })
test(nil, StatusPass, nil, func() { test(nil, StatusPass, nil, func() {
sel.DontSealHeaders = true sel.SealHeaders = false
dkimConf.Selectors = map[string]config.Selector{ selectors = []Selector{sel}
"test": sel,
}
sign() sign()
msg = "subject: another\r\n" + msg msg = "subject: another\r\n" + msg
}) })

View file

@ -7,10 +7,12 @@ import (
"strings" "strings"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
// Pedantic enables stricter parsing.
var Pedantic bool
type parseErr string type parseErr string
func (e parseErr) Error() string { func (e parseErr) Error() string {
@ -200,7 +202,7 @@ func (p *parser) xdomainselector(isselector bool) dns.Domain {
// domain names must always be a-labels, ../rfc/6376:1115 ../rfc/6376:1187 ../rfc/6376:1303 // domain names must always be a-labels, ../rfc/6376:1115 ../rfc/6376:1187 ../rfc/6376:1303
// dkim selectors with underscores happen in the wild, accept them when not in // dkim selectors with underscores happen in the wild, accept them when not in
// pedantic mode. ../rfc/6376:581 ../rfc/5321:2303 // pedantic mode. ../rfc/6376:581 ../rfc/5321:2303
return isalphadigit(c) || (i > 0 && (c == '-' || isselector && !moxvar.Pedantic && c == '_') && p.o+1 < len(p.s)) return isalphadigit(c) || (i > 0 && (c == '-' || isselector && !Pedantic && c == '_') && p.o+1 < len(p.s))
} }
s := p.xtakefn1(false, subdomain) s := p.xtakefn1(false, subdomain)
for p.hasPrefix(".") { for p.hasPrefix(".") {
@ -273,7 +275,7 @@ func (p *parser) xlocalpart() smtp.Localpart {
} }
} }
// In the wild, some services use large localparts for generated (bounce) addresses. // In the wild, some services use large localparts for generated (bounce) addresses.
if moxvar.Pedantic && len(s) > 64 || len(s) > 128 { if Pedantic && len(s) > 64 || len(s) > 128 {
// ../rfc/5321:3486 // ../rfc/5321:3486
p.xerrorf("localpart longer than 64 octets") p.xerrorf("localpart longer than 64 octets")
} }

View file

@ -17,31 +17,18 @@ import (
mathrand "math/rand" mathrand "math/rand"
"time" "time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dkim" "github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/spf" "github.com/mjl-/mox/spf"
"github.com/mjl-/mox/stub"
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
) )
var ( var (
metricDMARCVerify = promauto.NewHistogramVec( MetricVerify stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{
Name: "mox_dmarc_verify_duration_seconds",
Help: "DMARC verify, including lookup, duration and result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"status",
"reject", // yes/no
"use", // yes/no, if policy is used after random selection
},
)
) )
// link errata: // link errata:
@ -248,7 +235,7 @@ func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, from
if result.Reject { if result.Reject {
reject = "yes" reject = "yes"
} }
metricDMARCVerify.WithLabelValues(string(result.Status), reject, use).Observe(float64(time.Since(start)) / float64(time.Second)) MetricVerify.ObserveLabels(float64(time.Since(start))/float64(time.Second), string(result.Status), reject, use)
log.Debugx("dmarc verify result", result.Err, log.Debugx("dmarc verify result", result.Err,
slog.Any("fromdomain", from), slog.Any("fromdomain", from),
slog.Any("dkimresults", dkimResults), slog.Any("dkimresults", dkimResults),

View file

@ -1068,8 +1068,9 @@ func dkimSign(ctx context.Context, log mlog.Log, fromAddr smtp.Address, smtputf8
var zerodom dns.Domain var zerodom dns.Domain
for fd != zerodom { for fd != zerodom {
confDom, ok := mox.Conf.Domain(fd) confDom, ok := mox.Conf.Domain(fd)
if len(confDom.DKIM.Sign) > 0 { selectors := mox.DKIMSelectors(confDom.DKIM)
dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fd, confDom.DKIM, smtputf8, mf) if len(selectors) > 0 {
dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fd, selectors, smtputf8, mf)
if err != nil { if err != nil {
log.Errorx("dkim-signing dmarc report, continuing without signature", err) log.Errorx("dkim-signing dmarc report, continuing without signature", err)
metricReportError.Inc() metricReportError.Inc()

View file

@ -10,10 +10,11 @@ import (
"golang.org/x/net/idna" "golang.org/x/net/idna"
"github.com/mjl-/adns" "github.com/mjl-/adns"
"github.com/mjl-/mox/moxvar"
) )
// Pedantic enables stricter parsing.
var Pedantic bool
var ( var (
errTrailingDot = errors.New("dns name has trailing dot") errTrailingDot = errors.New("dns name has trailing dot")
errUnderscore = errors.New("domain name with underscore") errUnderscore = errors.New("domain name with underscore")
@ -113,7 +114,7 @@ func ParseDomain(s string) (Domain, error) {
// is not enabled. Used for interoperability, e.g. domains may specify MX // is not enabled. Used for interoperability, e.g. domains may specify MX
// targets with underscores. // targets with underscores.
func ParseDomainLax(s string) (Domain, error) { func ParseDomainLax(s string) (Domain, error) {
if moxvar.Pedantic || !strings.Contains(s, "_") { if Pedantic || !strings.Contains(s, "_") {
return ParseDomain(s) return ParseDomain(s)
} }

View file

@ -12,12 +12,10 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/adns" "github.com/mjl-/adns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/stub"
) )
// todo future: replace with a dnssec capable resolver // todo future: replace with a dnssec capable resolver
@ -29,18 +27,7 @@ func init() {
} }
var ( var (
metricLookup = promauto.NewHistogramVec( MetricLookup stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{
Name: "mox_dns_lookup_duration_seconds",
Help: "DNS lookups.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{
"pkg",
"type", // Lower-case Resolver method name without leading Lookup.
"result", // ok, nxdomain, temporary, timeout, canceled, error
},
)
) )
// Resolver is the interface strict resolver implements. // Resolver is the interface strict resolver implements.
@ -106,7 +93,7 @@ func metricLookupObserve(pkg, typ string, err error, start time.Time) {
default: default:
result = "error" result = "error"
} }
metricLookup.WithLabelValues(pkg, typ, result).Observe(float64(time.Since(start)) / float64(time.Second)) MetricLookup.ObserveLabels(float64(time.Since(start))/float64(time.Second), pkg, typ, result)
} }
func (r StrictResolver) WithPackage(name string) Resolver { func (r StrictResolver) WithPackage(name string) Resolver {

View file

@ -12,25 +12,13 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/stub"
) )
var ( var (
metricLookup = promauto.NewHistogramVec( MetricLookup stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{
Name: "mox_dnsbl_lookup_duration_seconds",
Help: "DNSBL lookup",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"zone",
"status",
},
)
) )
var ErrDNS = errors.New("dnsbl: dns error") var ErrDNS = errors.New("dnsbl: dns error")
@ -49,7 +37,7 @@ func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, zone
log := mlog.New("dnsbl", elog) log := mlog.New("dnsbl", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
metricLookup.WithLabelValues(zone.Name(), string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second)) MetricLookup.ObserveLabels(float64(time.Since(start))/float64(time.Second), zone.Name(), string(rstatus))
log.Debugx("dnsbl lookup result", rerr, log.Debugx("dnsbl lookup result", rerr,
slog.Any("zone", zone), slog.Any("zone", zone),
slog.Any("ip", ip), slog.Any("ip", ip),

View file

@ -5,7 +5,6 @@ package dsn
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"context"
"encoding/base64" "encoding/base64"
"errors" "errors"
"fmt" "fmt"
@ -16,13 +15,8 @@ import (
"strings" "strings"
"time" "time"
"golang.org/x/exp/slog"
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
@ -48,7 +42,6 @@ type Message struct {
// Message subject header, e.g. describing mail delivery failure. // Message subject header, e.g. describing mail delivery failure.
Subject string Subject string
// Set when message is composed.
MessageID string MessageID string
// References header, with Message-ID of original message this DSN is about. So // References header, with Message-ID of original message this DSN is about. So
@ -136,7 +129,7 @@ type Recipient struct {
// supports smtputf8. This influences the message media (sub)types used for the // supports smtputf8. This influences the message media (sub)types used for the
// DSN. // DSN.
// //
// DKIM signatures are added if DKIM signing is configured for the "from" domain. // Called may want to add DKIM-Signature headers.
func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) { func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) {
// ../rfc/3462:119 // ../rfc/3462:119
// ../rfc/3464:377 // ../rfc/3464:377
@ -168,7 +161,9 @@ func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) {
header("From", fmt.Sprintf("<%s>", m.From.XString(smtputf8))) // todo: would be good to have a local ascii-only name for this address. header("From", fmt.Sprintf("<%s>", m.From.XString(smtputf8))) // todo: would be good to have a local ascii-only name for this address.
header("To", fmt.Sprintf("<%s>", m.To.XString(smtputf8))) // todo: we could just leave this out if it has utf-8 and remote does not support utf-8. header("To", fmt.Sprintf("<%s>", m.To.XString(smtputf8))) // todo: we could just leave this out if it has utf-8 and remote does not support utf-8.
header("Subject", m.Subject) header("Subject", m.Subject)
m.MessageID = mox.MessageIDGen(smtputf8) if m.MessageID == "" {
return nil, fmt.Errorf("missing message-id")
}
header("Message-Id", fmt.Sprintf("<%s>", m.MessageID)) header("Message-Id", fmt.Sprintf("<%s>", m.MessageID))
if m.References != "" { if m.References != "" {
header("References", m.References) header("References", m.References)
@ -367,31 +362,6 @@ func (m *Message) Compose(log mlog.Log, smtputf8 bool) ([]byte, error) {
} }
data := msgw.w.Bytes() data := msgw.w.Bytes()
// Add DKIM signature for domain, even if higher up than the full mail hostname.
// This helps with an assumed (because default) relaxed DKIM policy. If the DMARC
// policy happens to be strict, the signature won't help, but won't hurt either.
fd := m.From.IPDomain.Domain
var zerodom dns.Domain
for fd != zerodom {
confDom, ok := mox.Conf.Domain(fd)
if !ok {
var nfd dns.Domain
_, nfd.ASCII, _ = strings.Cut(fd.ASCII, ".")
_, nfd.Unicode, _ = strings.Cut(fd.Unicode, ".")
fd = nfd
continue
}
dkimHeaders, err := dkim.Sign(context.Background(), log.Logger, m.From.Localpart, fd, confDom.DKIM, smtputf8, bytes.NewReader(data))
if err != nil {
log.Errorx("dsn: dkim sign for domain, returning unsigned dsn", err, slog.Any("domain", fd))
} else {
data = append([]byte(dkimHeaders), data...)
}
break
}
return data, nil return data, nil
} }

View file

@ -2,21 +2,17 @@ package dsn
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"io" "io"
"net" "net"
"path/filepath"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
@ -83,10 +79,11 @@ func TestDSN(t *testing.T) {
m := Message{ m := Message{
SMTPUTF8: false, SMTPUTF8: false,
From: smtp.Path{Localpart: "postmaster", IPDomain: xparseIPDomain("mox.example")}, From: smtp.Path{Localpart: "postmaster", IPDomain: xparseIPDomain("mox.example")},
To: smtp.Path{Localpart: "mjl", IPDomain: xparseIPDomain("remote.example")}, To: smtp.Path{Localpart: "mjl", IPDomain: xparseIPDomain("remote.example")},
Subject: "dsn", Subject: "dsn",
TextBody: "delivery failure\n", MessageID: "test@localhost",
TextBody: "delivery failure\n",
ReportingMTA: "mox.example", ReportingMTA: "mox.example",
ReceivedFromMTA: smtp.Ehlo{Name: xparseIPDomain("relay.example"), ConnIP: net.ParseIP("10.10.10.10")}, ReceivedFromMTA: smtp.Ehlo{Name: xparseIPDomain("relay.example"), ConnIP: net.ParseIP("10.10.10.10")},
@ -107,6 +104,7 @@ func TestDSN(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("composing dsn: %v", err) t.Fatalf("composing dsn: %v", err)
} }
pmsg, part := tparseMessage(t, msgbuf, 3) pmsg, part := tparseMessage(t, msgbuf, 3)
tcheckType(t, part, "multipart", "report", "") tcheckType(t, part, "multipart", "report", "")
tcheckType(t, &part.Parts[0], "text", "plain", "7bit") tcheckType(t, &part.Parts[0], "text", "plain", "7bit")
@ -130,35 +128,15 @@ func TestDSN(t *testing.T) {
tcompareReader(t, part.Parts[2].Reader(), m.Original) tcompareReader(t, part.Parts[2].Reader(), m.Original)
tcompare(t, pmsg.Recipients[0].FinalRecipient, m.Recipients[0].FinalRecipient) tcompare(t, pmsg.Recipients[0].FinalRecipient, m.Recipients[0].FinalRecipient)
// Test for valid DKIM signature.
mox.Context = context.Background()
mox.ConfigStaticPath = filepath.FromSlash("../testdata/dsn/mox.conf")
mox.MustLoadConfig(true, false)
msgbuf, err = m.Compose(log, false)
if err != nil {
t.Fatalf("composing utf-8 dsn with utf-8 support: %v", err)
}
resolver := &dns.MockResolver{
TXT: map[string][]string{
"testsel._domainkey.mox.example.": {"v=DKIM1;h=sha256;t=s;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3ZId3ys70VFspp/VMFaxMOrNjHNPg04NOE1iShih16b3Ex7hHBOgC1UvTGSmrMlbCB1OxTXkvf6jW6S4oYRnZYVNygH6zKUwYYhaSaGIg1xA/fDn+IgcTRyLoXizMUgUgpTGyxhNrwIIWv+i7jjbs3TKpP3NU4owQ/rxowmSNqg+fHIF1likSvXvljYS" + "jaFXXnWfYibW7TdDCFFpN4sB5o13+as0u4vLw6MvOi59B1tLype1LcHpi1b9PfxNtznTTdet3kL0paxIcWtKHT0LDPUos8YYmiPa5nGbUqlC7d+4YT2jQPvwGxCws1oo2Tw6nj1UaihneYGAyvEky49FBwIDAQAB"},
},
}
results, err := dkim.Verify(context.Background(), log.Logger, resolver, false, func(*dkim.Sig) error { return nil }, bytes.NewReader(msgbuf), false)
if err != nil {
t.Fatalf("dkim verify: %v", err)
}
if len(results) != 1 || results[0].Status != dkim.StatusPass {
t.Fatalf("dkim result not pass, %#v", results)
}
// An utf-8 message. // An utf-8 message.
m = Message{ m = Message{
SMTPUTF8: true, SMTPUTF8: true,
From: smtp.Path{Localpart: "postmæster", IPDomain: xparseIPDomain("møx.example")}, From: smtp.Path{Localpart: "postmæster", IPDomain: xparseIPDomain("møx.example")},
To: smtp.Path{Localpart: "møx", IPDomain: xparseIPDomain("remøte.example")}, To: smtp.Path{Localpart: "møx", IPDomain: xparseIPDomain("remøte.example")},
Subject: "dsn¡", Subject: "dsn¡",
TextBody: "delivery failure¿\n", MessageID: "test@localhost",
TextBody: "delivery failure¿\n",
ReportingMTA: "mox.example", ReportingMTA: "mox.example",
ReceivedFromMTA: smtp.Ehlo{Name: xparseIPDomain("reläy.example"), ConnIP: net.ParseIP("10.10.10.10")}, ReceivedFromMTA: smtp.Ehlo{Name: xparseIPDomain("reläy.example"), ConnIP: net.ParseIP("10.10.10.10")},

View file

@ -8,7 +8,7 @@ import (
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/imapclient" "github.com/mjl-/mox/imapclient"
"github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
@ -150,9 +150,9 @@ func testCondstoreQresync(t *testing.T, qresync bool) {
imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq + 4)}}, imapclient.UntaggedFetch{Seq: 6, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(6), noflags, imapclient.FetchModSeq(clientModseq + 4)}},
) )
moxvar.Pedantic = true mox.SetPedantic(true)
tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax. tc.transactf("bad", `Fetch 1 Flags (Changedsince 0)`) // 0 not allowed in syntax.
moxvar.Pedantic = false mox.SetPedantic(false)
tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)") tc.transactf("ok", "Uid fetch 1 (Flags) (Changedsince 0)")
tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(clientModseq)}}) tc.xuntagged(imapclient.UntaggedFetch{Seq: 1, Attrs: []imapclient.FetchAttr{imapclient.FetchUID(1), noflags, imapclient.FetchModSeq(clientModseq)}})

View file

@ -17,8 +17,8 @@ import (
"github.com/mjl-/bstore" "github.com/mjl-/bstore"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
@ -102,7 +102,7 @@ func (c *conn) cmdxFetch(isUID bool, tag, cmdstr string, p *parser) {
p.xspace() p.xspace()
changedSince = p.xnumber64() changedSince = p.xnumber64()
// workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled. // workaround: ios mail (16.5.1) was seen sending changedSince 0 on an existing account that got condstore enabled.
if changedSince == 0 && moxvar.Pedantic { if changedSince == 0 && mox.Pedantic {
// ../rfc/7162:2551 // ../rfc/7162:2551
xsyntaxErrorf("changedsince modseq must be > 0") xsyntaxErrorf("changedsince modseq must be > 0")
} }

View file

@ -1456,7 +1456,7 @@ func (c *conn) cmdStarttls(tag, cmd string, p *parser) {
panic(fmt.Errorf("starttls handshake: %s (%w)", err, errIO)) panic(fmt.Errorf("starttls handshake: %s (%w)", err, errIO))
} }
cancel() cancel()
tlsversion, ciphersuite := mox.TLSInfo(tlsConn) tlsversion, ciphersuite := moxio.TLSInfo(tlsConn)
c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite)) c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite))
c.conn = tlsConn c.conn = tlsConn

View file

@ -11,24 +11,15 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/stub"
) )
var xlog = mlog.New("iprev", nil) var xlog = mlog.New("iprev", nil)
var ( var (
metricIPRev = promauto.NewHistogramVec( MetricIPRev stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{
Name: "mox_iprev_lookup_total",
Help: "Number of iprev lookups.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"status"},
)
) )
// Lookup errors. // Lookup errors.
@ -62,7 +53,7 @@ func Lookup(ctx context.Context, resolver dns.Resolver, ip net.IP) (rstatus Stat
log := xlog.WithContext(ctx) log := xlog.WithContext(ctx)
start := time.Now() start := time.Now()
defer func() { defer func() {
metricIPRev.WithLabelValues(string(rstatus)).Observe(float64(time.Since(start)) / float64(time.Second)) MetricIPRev.ObserveLabels(float64(time.Since(start))/float64(time.Second), string(rstatus))
log.Debugx("iprev lookup result", rerr, log.Debugx("iprev lookup result", rerr,
slog.Any("ip", ip), slog.Any("ip", ip),
slog.Any("status", rstatus), slog.Any("status", rstatus),

14
main.go
View file

@ -394,7 +394,7 @@ func mustLoadConfig() {
log.Fatal("unknown loglevel", slog.String("loglevel", loglevel)) log.Fatal("unknown loglevel", slog.String("loglevel", loglevel))
} }
if pedantic { if pedantic {
moxvar.Pedantic = true mox.SetPedantic(true)
} }
} }
@ -445,7 +445,7 @@ func main() {
defer profile(cpuprofile, memprofile)() defer profile(cpuprofile, memprofile)()
if pedantic { if pedantic {
moxvar.Pedantic = true mox.SetPedantic(true)
} }
mox.ConfigDynamicPath = filepath.Join(filepath.Dir(mox.ConfigStaticPath), "domains.conf") mox.ConfigDynamicPath = filepath.Join(filepath.Dir(mox.ConfigStaticPath), "domains.conf")
@ -1600,8 +1600,11 @@ connection.
} }
} }
pkixRoots, err := x509.SystemCertPool()
xcheckf(err, "get system pkix certificate pool")
resolver := dns.StrictResolver{Pkg: "danedial"} resolver := dns.StrictResolver{Pkg: "danedial"}
conn, record, err := dane.Dial(context.Background(), c.log.Logger, resolver, "tcp", args[0], allowedUsages) conn, record, err := dane.Dial(context.Background(), c.log.Logger, resolver, "tcp", args[0], allowedUsages, pkixRoots)
xcheckf(err, "dial") xcheckf(err, "dial")
log.Printf("(connected, verified with %s)", record) log.Printf("(connected, verified with %s)", record)
@ -1756,7 +1759,7 @@ sharing most of its code.
log.Printf("gathered valid tls certificate names for potential verification with dane-ta: %s", strings.Join(l, ", ")) log.Printf("gathered valid tls certificate names for potential verification with dane-ta: %s", strings.Join(l, ", "))
dialer := &net.Dialer{Timeout: 5 * time.Second} dialer := &net.Dialer{Timeout: 5 * time.Second}
conn, _, err := smtpclient.Dial(ctxbg, c.log.Logger, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs) conn, _, err := smtpclient.Dial(ctxbg, c.log.Logger, dialer, dns.IPDomain{Domain: expandedHost}, ips, 25, dialedIPs, nil)
if err != nil { if err != nil {
log.Printf("dial %s: %v, skipping", expandedHost, err) log.Printf("dial %s: %v, skipping", expandedHost, err)
continue continue
@ -2238,7 +2241,8 @@ headers prepended.
log.Fatalf("domain %s not configured", dom) log.Fatalf("domain %s not configured", dom)
} }
headers, err := dkim.Sign(context.Background(), c.log.Logger, localpart, dom, domConf.DKIM, false, msgf) selectors := mox.DKIMSelectors(domConf.DKIM)
headers, err := dkim.Sign(context.Background(), c.log.Logger, localpart, dom, selectors, false, msgf)
xcheckf(err, "signing message with dkim") xcheckf(err, "signing message with dkim")
if headers == "" { if headers == "" {
log.Fatalf("no DKIM configured for domain %s", dom) log.Fatalf("no DKIM configured for domain %s", dom)

View file

@ -1,4 +1,4 @@
package moxio package message
import ( import (
"io" "io"

View file

@ -1,4 +1,4 @@
package moxio package message
import ( import (
"io" "io"
@ -10,7 +10,7 @@ func TestDecodeReader(t *testing.T) {
check := func(charset, input, output string) { check := func(charset, input, output string) {
t.Helper() t.Helper()
buf, err := io.ReadAll(DecodeReader(charset, strings.NewReader(input))) buf, err := io.ReadAll(DecodeReader(charset, strings.NewReader(input)))
tcheckf(t, err, "decode") tcheck(t, err, "decode")
if string(buf) != output { if string(buf) != output {
t.Fatalf("decoding %q with charset %q, got %q, expected %q", input, charset, buf, output) t.Fatalf("decoding %q with charset %q, got %q, expected %q", input, charset, buf, output)
} }

View file

@ -5,7 +5,6 @@ import (
"fmt" "fmt"
"strings" "strings"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
@ -29,7 +28,7 @@ func MessageIDCanonical(s string) (string, bool, error) {
// Seen in practice: Message-ID: <valid@valid.example> (added by postmaster@some.example) // Seen in practice: Message-ID: <valid@valid.example> (added by postmaster@some.example)
// Doesn't seem valid, but we allow it. // Doesn't seem valid, but we allow it.
s, rem, have := strings.Cut(s, ">") s, rem, have := strings.Cut(s, ">")
if !have || (rem != "" && (moxvar.Pedantic || !strings.HasPrefix(rem, " "))) { if !have || (rem != "" && (Pedantic || !strings.HasPrefix(rem, " "))) {
return "", false, fmt.Errorf("%w: missing >", errBadMessageID) return "", false, fmt.Errorf("%w: missing >", errBadMessageID)
} }
// We canonicalize the Message-ID: lower-case, no unneeded quoting. // We canonicalize the Message-ID: lower-case, no unneeded quoting.

View file

@ -25,11 +25,12 @@ import (
"golang.org/x/text/encoding/ianaindex" "golang.org/x/text/encoding/ianaindex"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
// Pedantic enables stricter parsing.
var Pedantic bool
var ( var (
ErrBadContentType = errors.New("bad content-type") ErrBadContentType = errors.New("bad content-type")
) )
@ -207,7 +208,7 @@ func (p *Part) Walk(elog *slog.Logger, parent *Part) error {
// If this is a DSN and we are not in pedantic mode, accept unexpected end of // If this is a DSN and we are not in pedantic mode, accept unexpected end of
// message. This is quite common because MTA's sometimes just truncate the original // message. This is quite common because MTA's sometimes just truncate the original
// message in a place that makes the message invalid. // message in a place that makes the message invalid.
if errors.Is(err, errUnexpectedEOF) && !moxvar.Pedantic && parent != nil && len(parent.Parts) >= 3 && p == &parent.Parts[2] && parent.MediaType == "MULTIPART" && parent.MediaSubType == "REPORT" { if errors.Is(err, errUnexpectedEOF) && !Pedantic && parent != nil && len(parent.Parts) >= 3 && p == &parent.Parts[2] && parent.MediaType == "MULTIPART" && parent.MediaSubType == "REPORT" {
mp, err = fallbackPart(mp, br, int64(len(buf))) mp, err = fallbackPart(mp, br, int64(len(buf)))
if err != nil { if err != nil {
return fmt.Errorf("parsing invalid embedded message: %w", err) return fmt.Errorf("parsing invalid embedded message: %w", err)
@ -305,7 +306,7 @@ func newPart(log mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Par
ct := p.header.Get("Content-Type") ct := p.header.Get("Content-Type")
mt, params, err := mime.ParseMediaType(ct) mt, params, err := mime.ParseMediaType(ct)
if err != nil && ct != "" { if err != nil && ct != "" {
if moxvar.Pedantic || strict { if Pedantic || strict {
return p, fmt.Errorf("%w: %s: %q", ErrBadContentType, err, ct) return p, fmt.Errorf("%w: %s: %q", ErrBadContentType, err, ct)
} }
@ -337,7 +338,7 @@ func newPart(log mlog.Log, strict bool, r io.ReaderAt, offset int64, parent *Par
} else if mt != "" { } else if mt != "" {
t := strings.SplitN(strings.ToUpper(mt), "/", 2) t := strings.SplitN(strings.ToUpper(mt), "/", 2)
if len(t) != 2 { if len(t) != 2 {
if moxvar.Pedantic || strict { if Pedantic || strict {
return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct) return p, fmt.Errorf("bad content-type: %q (content-type %q)", mt, ct)
} }
log.Debug("malformed media-type, ignoring and continuing", slog.String("type", mt)) log.Debug("malformed media-type, ignoring and continuing", slog.String("type", mt))
@ -584,7 +585,7 @@ func (p *Part) Reader() io.Reader {
// already). For unknown or missing character sets/encodings, the original reader // already). For unknown or missing character sets/encodings, the original reader
// is returned. // is returned.
func (p *Part) ReaderUTF8OrBinary() io.Reader { func (p *Part) ReaderUTF8OrBinary() io.Reader {
return moxio.DecodeReader(p.ContentTypeParams["charset"], p.Reader()) return DecodeReader(p.ContentTypeParams["charset"], p.Reader())
} }
func (p *Part) bodyReader(r io.Reader) io.Reader { func (p *Part) bodyReader(r io.Reader) io.Reader {
@ -689,7 +690,7 @@ func (r *crlfReader) Read(buf []byte) (int, error) {
if b == '\n' && !r.prevcr { if b == '\n' && !r.prevcr {
err = errBareLF err = errBareLF
break break
} else if b != '\n' && r.prevcr && (r.strict || moxvar.Pedantic) { } else if b != '\n' && r.prevcr && (r.strict || Pedantic) {
err = errBareCR err = errBareCR
break break
} }
@ -719,7 +720,7 @@ type bufAt struct {
const maxLineLength = 8 * 1024 const maxLineLength = 8 * 1024
func (b *bufAt) maxLineLength() int { func (b *bufAt) maxLineLength() int {
if b.strict || moxvar.Pedantic { if b.strict || Pedantic {
return 1000 return 1000
} }
return maxLineLength return maxLineLength
@ -777,7 +778,7 @@ func (b *bufAt) line(consume, requirecrlf bool) (buf []byte, crlf bool, err erro
} }
i++ i++
if i >= b.nbuf || b.buf[i] != '\n' { if i >= b.nbuf || b.buf[i] != '\n' {
if b.strict || moxvar.Pedantic { if b.strict || Pedantic {
return nil, false, errBareCR return nil, false, errBareCR
} }
continue continue
@ -833,7 +834,7 @@ func (r *offsetReader) Read(buf []byte) (int, error) {
if n > 0 { if n > 0 {
r.offset += int64(n) r.offset += int64(n)
max := maxLineLength max := maxLineLength
if r.strict || moxvar.Pedantic { if r.strict || Pedantic {
max = 1000 max = 1000
} }
@ -844,7 +845,7 @@ func (r *offsetReader) Read(buf []byte) (int, error) {
if err == nil || err == io.EOF { if err == nil || err == io.EOF {
if c == '\n' && !r.prevcr { if c == '\n' && !r.prevcr {
err = errBareLF err = errBareLF
} else if c != '\n' && r.prevcr && (r.strict || moxvar.Pedantic) { } else if c != '\n' && r.prevcr && (r.strict || Pedantic) {
err = errBareCR err = errBareCR
} }
} }

View file

@ -12,7 +12,6 @@ import (
"testing" "testing"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxvar"
) )
var pkglog = mlog.New("message", nil) var pkglog = mlog.New("message", nil)
@ -54,7 +53,7 @@ func TestBadContentType(t *testing.T) {
expBody := "test" expBody := "test"
// Pedantic is like strict. // Pedantic is like strict.
moxvar.Pedantic = true Pedantic = true
s := "content-type: text/html;;\r\n\r\ntest" s := "content-type: text/html;;\r\n\r\ntest"
p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, ErrBadContentType) tfail(t, err, ErrBadContentType)
@ -63,7 +62,7 @@ func TestBadContentType(t *testing.T) {
tcompare(t, string(buf), expBody) tcompare(t, string(buf), expBody)
tcompare(t, p.MediaType, "APPLICATION") tcompare(t, p.MediaType, "APPLICATION")
tcompare(t, p.MediaSubType, "OCTET-STREAM") tcompare(t, p.MediaSubType, "OCTET-STREAM")
moxvar.Pedantic = false Pedantic = false
// Strict // Strict
s = "content-type: text/html;;\r\n\r\ntest" s = "content-type: text/html;;\r\n\r\ntest"
@ -111,12 +110,12 @@ func TestBareCR(t *testing.T) {
expBody := "bare\rcr\r\n" expBody := "bare\rcr\r\n"
// Pedantic is like strict. // Pedantic is like strict.
moxvar.Pedantic = true Pedantic = true
p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s))) p, err := EnsurePart(pkglog.Logger, false, strings.NewReader(s), int64(len(s)))
tfail(t, err, errBareCR) tfail(t, err, errBareCR)
_, err = io.ReadAll(p.Reader()) _, err = io.ReadAll(p.Reader())
tfail(t, err, errBareCR) tfail(t, err, errBareCR)
moxvar.Pedantic = false Pedantic = false
// Strict. // Strict.
p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s))) p, err = EnsurePart(pkglog.Logger, true, strings.NewReader(s), int64(len(s)))
@ -291,12 +290,12 @@ func TestBareCrLf(t *testing.T) {
err = parse(false, "\r\ntest\ntest\r\n") err = parse(false, "\r\ntest\ntest\r\n")
tfail(t, err, errBareLF) tfail(t, err, errBareLF)
moxvar.Pedantic = true Pedantic = true
err = parse(false, "subject: test\rtest\r\n") err = parse(false, "subject: test\rtest\r\n")
tfail(t, err, errBareCR) tfail(t, err, errBareCR)
err = parse(false, "\r\ntest\rtest\r\n") err = parse(false, "\r\ntest\rtest\r\n")
tfail(t, err, errBareCR) tfail(t, err, errBareCR)
moxvar.Pedantic = false Pedantic = false
err = parse(true, "subject: test\rtest\r\n") err = parse(true, "subject: test\rtest\r\n")
tfail(t, err, errBareCR) tfail(t, err, errBareCR)

267
metrics.go Normal file
View file

@ -0,0 +1,267 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dane"
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dmarc"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/dnsbl"
"github.com/mjl-/mox/iprev"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mtasts"
"github.com/mjl-/mox/smtpclient"
"github.com/mjl-/mox/spf"
"github.com/mjl-/mox/subjectpass"
"github.com/mjl-/mox/tlsrpt"
"github.com/mjl-/mox/updates"
)
var metricHTTPClient = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_httpclient_request_duration_seconds",
Help: "HTTP requests lookups.",
Buckets: []float64{0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{
"pkg",
"method",
"code",
"result",
},
)
// httpClientObserve tracks the result of an HTTP transaction in a metric, and
// logs the result.
func httpClientObserve(ctx context.Context, elog *slog.Logger, pkg, method string, statusCode int, err error, start time.Time) {
log := mlog.New("metrics", elog)
var result string
switch {
case err == nil:
switch statusCode / 100 {
case 2:
result = "ok"
case 4:
result = "usererror"
case 5:
result = "servererror"
default:
result = "other"
}
case errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded):
result = "timeout"
case errors.Is(err, context.Canceled):
result = "canceled"
default:
result = "error"
}
metricHTTPClient.WithLabelValues(pkg, method, result, fmt.Sprintf("%d", statusCode)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("httpclient result", err,
slog.String("pkg", pkg),
slog.String("method", method),
slog.Int("code", statusCode),
slog.Duration("duration", time.Since(start)))
}
func init() {
dane.MetricVerify = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_dane_verify_total",
Help: "Total number of DANE verification attempts, including mox_dane_verify_errors_total.",
},
)
dane.MetricVerifyErrors = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_dane_verify_errors_total",
Help: "Total number of DANE verification failures, causing connections to fail.",
},
)
dkim.MetricSign = counterVec{promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "mox_dkim_sign_total",
Help: "DKIM messages signings, label key is the type of key, rsa or ed25519.",
},
[]string{
"key",
},
)}
dkim.MetricVerify = histogramVec{
promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_dkim_verify_duration_seconds",
Help: "DKIM verify, including lookup, duration and result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"algorithm",
"status",
},
),
}
dmarc.MetricVerify = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_dmarc_verify_duration_seconds",
Help: "DMARC verify, including lookup, duration and result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"status",
"reject", // yes/no
"use", // yes/no, if policy is used after random selection
},
)}
dns.MetricLookup = histogramVec{
promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_dns_lookup_duration_seconds",
Help: "DNS lookups.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{
"pkg",
"type", // Lower-case Resolver method name without leading Lookup.
"result", // ok, nxdomain, temporary, timeout, canceled, error
},
),
}
dnsbl.MetricLookup = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_dnsbl_lookup_duration_seconds",
Help: "DNSBL lookup",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"zone",
"status",
},
)}
iprev.MetricIPRev = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_iprev_lookup_total",
Help: "Number of iprev lookups.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"status"},
)}
mtasts.MetricGet = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_mtasts_get_duration_seconds",
Help: "MTA-STS get of policy, including lookup, duration and result.",
Buckets: []float64{0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"result", // ok, lookuperror, fetcherror
},
)}
mtasts.HTTPClientObserve = httpClientObserve
smtpclient.MetricCommands = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_smtpclient_command_duration_seconds",
Help: "SMTP client command duration and result codes in seconds.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30, 60, 120},
},
[]string{
"cmd",
"code",
"secode",
},
)}
smtpclient.MetricTLSRequiredNoIgnored = counterVec{promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "mox_smtpclient_tlsrequiredno_ignored_total",
Help: "Connection attempts with TLS policy findings ignored due to message with TLS-Required: No header. Does not cover case where TLS certificate cannot be PKIX-verified.",
},
[]string{
"ignored", // daneverification (no matching tlsa record)
},
)}
smtpclient.MetricPanicInc = func() {
metrics.PanicInc(metrics.Smtpclient)
}
spf.MetricVerify = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_spf_verify_duration_seconds",
Help: "SPF verify, including lookup, duration and result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"status",
},
)}
subjectpass.MetricGenerate = promauto.NewCounter(
prometheus.CounterOpts{
Name: "mox_subjectpass_generate_total",
Help: "Number of generated subjectpass challenges.",
},
)
subjectpass.MetricVerify = counterVec{promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "mox_subjectpass_verify_total",
Help: "Number of subjectpass verifications.",
},
[]string{
"result", // ok, fail
},
)}
tlsrpt.MetricLookup = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_tlsrpt_lookup_duration_seconds",
Help: "TLSRPT lookups with result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"result"},
)}
updates.MetricLookup = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_updates_lookup_duration_seconds",
Help: "Updates lookup with result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"result"},
)}
updates.MetricFetchChangelog = histogramVec{promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_updates_fetchchangelog_duration_seconds",
Help: "Fetch changelog with result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"result"},
)}
}
type counterVec struct {
*prometheus.CounterVec
}
func (m counterVec) IncLabels(labels ...string) {
m.CounterVec.WithLabelValues(labels...).Inc()
}
type histogramVec struct {
*prometheus.HistogramVec
}
func (m histogramVec) ObserveLabels(v float64, labels ...string) {
m.HistogramVec.WithLabelValues(labels...).Observe(v)
}

View file

@ -1,65 +0,0 @@
// Package metrics has prometheus metric variables/functions.
package metrics
import (
"context"
"errors"
"fmt"
"os"
"time"
"golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/mlog"
)
var (
metricHTTPClient = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_httpclient_request_duration_seconds",
Help: "HTTP requests lookups.",
Buckets: []float64{0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{
"pkg",
"method",
"code",
"result",
},
)
)
// HTTPClientObserve tracks the result of an HTTP transaction in a metric, and
// logs the result.
func HTTPClientObserve(ctx context.Context, log mlog.Log, pkg, method string, statusCode int, err error, start time.Time) {
log = log.WithPkg("metrics")
var result string
switch {
case err == nil:
switch statusCode / 100 {
case 2:
result = "ok"
case 4:
result = "usererror"
case 5:
result = "servererror"
default:
result = "other"
}
case errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded):
result = "timeout"
case errors.Is(err, context.Canceled):
result = "canceled"
default:
result = "error"
}
metricHTTPClient.WithLabelValues(pkg, method, result, fmt.Sprintf("%d", statusCode)).Observe(float64(time.Since(start)) / float64(time.Second))
log.Debugx("httpclient result", err,
slog.String("pkg", pkg),
slog.String("method", method),
slog.Int("code", statusCode),
slog.Duration("duration", time.Since(start)))
}

View file

@ -37,16 +37,20 @@ import (
"github.com/mjl-/mox/autotls" "github.com/mjl-/mox/autotls"
"github.com/mjl-/mox/config" "github.com/mjl-/mox/config"
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/mtasts" "github.com/mjl-/mox/mtasts"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
var pkglog = mlog.New("mox", nil) var pkglog = mlog.New("mox", nil)
// Pedantic enables stricter parsing.
var Pedantic bool
// Config paths are set early in program startup. They will point to files in // Config paths are set early in program startup. They will point to files in
// the same directory. // the same directory.
var ( var (
@ -397,7 +401,16 @@ func SetConfig(c *Config) {
} }
} }
moxvar.Pedantic = c.Static.Pedantic SetPedantic(c.Static.Pedantic)
}
// Set pedantic in all packages.
func SetPedantic(p bool) {
dkim.Pedantic = p
dns.Pedantic = p
message.Pedantic = p
smtp.Pedantic = p
Pedantic = p
} }
// ParseConfig parses the static config at path p. If checkOnly is true, no changes // ParseConfig parses the static config at path p. If checkOnly is true, no changes

65
mox-/dkimsign.go Normal file
View file

@ -0,0 +1,65 @@
package mox
import (
"bytes"
"context"
"fmt"
"strings"
"time"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dkim"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp"
)
// DKIMSelectors returns the selectors to use for signing.
func DKIMSelectors(dkimConf config.DKIM) []dkim.Selector {
var l []dkim.Selector
for _, sign := range dkimConf.Sign {
sel := dkimConf.Selectors[sign]
s := dkim.Selector{
Hash: sel.HashEffective,
HeaderRelaxed: sel.Canonicalization.HeaderRelaxed,
BodyRelaxed: sel.Canonicalization.BodyRelaxed,
Headers: sel.HeadersEffective,
SealHeaders: !sel.DontSealHeaders,
Expiration: time.Duration(sel.ExpirationSeconds) * time.Second,
PrivateKey: sel.Key,
Domain: sel.Domain,
}
l = append(l, s)
}
return l
}
// DKIMSign looks up the domain for "from", and uses its DKIM configuration to
// generate DKIM-Signature headers, for inclusion in a message. The
// DKIM-Signatur headers, are returned. If no domain was found an empty string and
// nil error is returned.
func DKIMSign(ctx context.Context, log mlog.Log, from smtp.Path, smtputf8 bool, data []byte) (string, error) {
// Add DKIM signature for domain, even if higher up than the full mail hostname.
// This helps with an assumed (because default) relaxed DKIM policy. If the DMARC
// policy happens to be strict, the signature won't help, but won't hurt either.
fd := from.IPDomain.Domain
var zerodom dns.Domain
for fd != zerodom {
confDom, ok := Conf.Domain(fd)
if !ok {
var nfd dns.Domain
_, nfd.ASCII, _ = strings.Cut(fd.ASCII, ".")
_, nfd.Unicode, _ = strings.Cut(fd.Unicode, ".")
fd = nfd
continue
}
selectors := DKIMSelectors(confDom.DKIM)
dkimHeaders, err := dkim.Sign(ctx, log.Logger, from.Localpart, fd, selectors, smtputf8, bytes.NewReader(data))
if err != nil {
return "", fmt.Errorf("dkim sign for domain %s: %v", fd, err)
}
return dkimHeaders, nil
}
return "", nil
}

View file

@ -1,4 +1,4 @@
package mox package moxio
import ( import (
"crypto/tls" "crypto/tls"

View file

@ -1,4 +0,0 @@
package moxvar
// Pendantic mode, in moxvar to prevent cyclic imports.
var Pedantic bool

View file

@ -22,28 +22,17 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/adns" "github.com/mjl-/adns"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/stub"
) )
var ( var (
metricGet = promauto.NewHistogramVec( MetricGet stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{ HTTPClientObserve func(ctx context.Context, log *slog.Logger, pkg, method string, statusCode int, err error, start time.Time) = stub.HTTPClientObserveIgnore
Name: "mox_mtasts_get_duration_seconds",
Help: "MTA-STS get of policy, including lookup, duration and result.",
Buckets: []float64{0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"result", // ok, lookuperror, fetcherror
},
)
) )
// Pair is an extension key/value pair in a MTA-STS DNS record or policy. // Pair is an extension key/value pair in a MTA-STS DNS record or policy.
@ -298,7 +287,7 @@ func FetchPolicy(ctx context.Context, elog *slog.Logger, domain dns.Domain) (pol
// We pass along underlying TLS certificate errors. // We pass along underlying TLS certificate errors.
return nil, "", fmt.Errorf("%w: http get: %w", ErrPolicyFetch, err) return nil, "", fmt.Errorf("%w: http get: %w", ErrPolicyFetch, err)
} }
metrics.HTTPClientObserve(ctx, log, "mtasts", req.Method, resp.StatusCode, err, start) HTTPClientObserve(ctx, log.Logger, "mtasts", req.Method, resp.StatusCode, err, start)
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound { if resp.StatusCode == http.StatusNotFound {
return nil, "", ErrNoPolicy return nil, "", ErrNoPolicy
@ -341,7 +330,7 @@ func Get(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domain d
start := time.Now() start := time.Now()
result := "lookuperror" result := "lookuperror"
defer func() { defer func() {
metricGet.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) MetricGet.ObserveLabels(float64(time.Since(start))/float64(time.Second), result)
log.Debugx("mtasts get result", err, log.Debugx("mtasts get result", err,
slog.Any("domain", domain), slog.Any("domain", domain),
slog.Any("record", record), slog.Any("record", record),

View file

@ -90,14 +90,14 @@ var (
) )
// todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time? // todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time?
func fail(qlog mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { func fail(ctx context.Context, qlog mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
// todo future: when we implement relaying, we should be able to send DSNs to non-local users. and possibly specify a null mailfrom. ../rfc/5321:1503 // todo future: when we implement relaying, we should be able to send DSNs to non-local users. and possibly specify a null mailfrom. ../rfc/5321:1503
// todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls? // todo future: when we implement relaying, and a dsn cannot be delivered, and requiretls was active, we cannot drop the message. instead deliver to local postmaster? though ../rfc/8689:383 may intend to say the dsn should be delivered without requiretls?
// todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379 // todo future: when we implement smtp dsn extension, parameter RET=FULL must be disregarded for messages with REQUIRETLS. ../rfc/8689:379
if permanent || m.MaxAttempts == 0 && m.Attempts >= 8 || m.MaxAttempts > 0 && m.Attempts >= m.MaxAttempts { if permanent || m.MaxAttempts == 0 && m.Attempts >= 8 || m.MaxAttempts > 0 && m.Attempts >= m.MaxAttempts {
qlog.Errorx("permanent failure delivering from queue", errors.New(errmsg)) qlog.Errorx("permanent failure delivering from queue", errors.New(errmsg))
deliverDSNFailure(qlog, m, remoteMTA, secodeOpt, errmsg) deliverDSNFailure(ctx, qlog, m, remoteMTA, secodeOpt, errmsg)
if err := queueDelete(context.Background(), m.ID); err != nil { if err := queueDelete(context.Background(), m.ID); err != nil {
qlog.Errorx("deleting message from queue after permanent failure", err) qlog.Errorx("deleting message from queue after permanent failure", err)
@ -117,7 +117,7 @@ func fail(qlog mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA
qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), slog.Duration("backoff", backoff)) qlog.Errorx("temporary failure delivering from queue, sending delayed dsn", errors.New(errmsg), slog.Duration("backoff", backoff))
retryUntil := m.LastAttempt.Add((4 + 8 + 16) * time.Hour) retryUntil := m.LastAttempt.Add((4 + 8 + 16) * time.Hour)
deliverDSNDelay(qlog, m, remoteMTA, secodeOpt, errmsg, retryUntil) deliverDSNDelay(ctx, qlog, m, remoteMTA, secodeOpt, errmsg, retryUntil)
} else { } else {
qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), slog.Duration("backoff", backoff), slog.Time("nextattempt", m.NextAttempt)) qlog.Errorx("temporary failure delivering from queue", errors.New(errmsg), slog.Duration("backoff", backoff), slog.Time("nextattempt", m.NextAttempt))
} }
@ -169,7 +169,7 @@ func deliverDirect(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
recipientDomainResult.Summary.TotalFailureSessionCount++ recipientDomainResult.Summary.TotalFailureSessionCount++
} }
fail(qlog, m, backoff, permanent, dsn.NameIP{}, "", err.Error()) fail(ctx, qlog, m, backoff, permanent, dsn.NameIP{}, "", err.Error())
return return
} }
@ -189,7 +189,7 @@ func deliverDirect(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
} else { } else {
qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, slog.Any("domain", origNextHop)) qlog.Infox("mtasts lookup temporary error, aborting delivery attempt", err, slog.Any("domain", origNextHop))
recipientDomainResult.Summary.TotalFailureSessionCount++ recipientDomainResult.Summary.TotalFailureSessionCount++
fail(qlog, m, backoff, false, dsn.NameIP{}, "", err.Error()) fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", err.Error())
return return
} }
} }
@ -333,7 +333,7 @@ func deliverDirect(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
permanent = true permanent = true
} }
fail(qlog, m, backoff, permanent, remoteMTA, secodeOpt, errmsg) fail(ctx, qlog, m, backoff, permanent, remoteMTA, secodeOpt, errmsg)
return return
} }
@ -527,7 +527,7 @@ func deliverHost(log mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
if m.DialedIPs == nil { if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{} m.DialedIPs = map[string][]net.IP{}
} }
conn, remoteIP, err = smtpclient.Dial(ctx, log.Logger, dialer, host, ips, 25, m.DialedIPs) conn, remoteIP, err = smtpclient.Dial(ctx, log.Logger, dialer, host, ips, 25, m.DialedIPs, mox.Conf.Static.SpecifiedSMTPListenIPs)
} }
cancel() cancel()

View file

@ -2,6 +2,7 @@ package queue
import ( import (
"bufio" "bufio"
"context"
"fmt" "fmt"
"os" "os"
"time" "time"
@ -29,7 +30,7 @@ var (
) )
) )
func deliverDSNFailure(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) { func deliverDSNFailure(ctx context.Context, log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string) {
const subject = "mail delivery failed" const subject = "mail delivery failed"
message := fmt.Sprintf(` message := fmt.Sprintf(`
Delivery has failed permanently for your email to: Delivery has failed permanently for your email to:
@ -43,10 +44,10 @@ Error during the last delivery attempt:
%s %s
`, m.Recipient().XString(m.SMTPUTF8), errmsg) `, m.Recipient().XString(m.SMTPUTF8), errmsg)
deliverDSN(log, m, remoteMTA, secodeOpt, errmsg, true, nil, subject, message) deliverDSN(ctx, log, m, remoteMTA, secodeOpt, errmsg, true, nil, subject, message)
} }
func deliverDSNDelay(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) { func deliverDSNDelay(ctx context.Context, log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, retryUntil time.Time) {
// Should not happen, but doesn't hurt to prevent sending delayed delivery // Should not happen, but doesn't hurt to prevent sending delayed delivery
// notifications for DMARC reports. We don't want to waste postmaster attention. // notifications for DMARC reports. We don't want to waste postmaster attention.
if m.IsDMARCReport { if m.IsDMARCReport {
@ -67,14 +68,14 @@ Error during the last delivery attempt:
%s %s
`, m.Recipient().XString(false), errmsg) `, m.Recipient().XString(false), errmsg)
deliverDSN(log, m, remoteMTA, secodeOpt, errmsg, false, &retryUntil, subject, message) deliverDSN(ctx, log, m, remoteMTA, secodeOpt, errmsg, false, &retryUntil, subject, message)
} }
// We only queue DSNs for delivery failures for emails submitted by authenticated // We only queue DSNs for delivery failures for emails submitted by authenticated
// users. So we are delivering to local users. ../rfc/5321:1466 // users. So we are delivering to local users. ../rfc/5321:1466
// ../rfc/5321:1494 // ../rfc/5321:1494
// ../rfc/7208:490 // ../rfc/7208:490
func deliverDSN(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) { func deliverDSN(ctx context.Context, log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg string, permanent bool, retryUntil *time.Time, subject, textBody string) {
kind := "delayed delivery" kind := "delayed delivery"
if permanent { if permanent {
kind = "failure" kind = "failure"
@ -124,6 +125,7 @@ func deliverDSN(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg str
From: smtp.Path{Localpart: "postmaster", IPDomain: dns.IPDomain{Domain: mox.Conf.Static.HostnameDomain}}, From: smtp.Path{Localpart: "postmaster", IPDomain: dns.IPDomain{Domain: mox.Conf.Static.HostnameDomain}},
To: m.Sender(), To: m.Sender(),
Subject: subject, Subject: subject,
MessageID: mox.MessageIDGen(false),
References: m.MessageID, References: m.MessageID,
TextBody: textBody, TextBody: textBody,
@ -150,7 +152,7 @@ func deliverDSN(log mlog.Log, m Msg, remoteMTA dsn.NameIP, secodeOpt, errmsg str
return return
} }
msgData = append(msgData, []byte("Return-Path: <"+dsnMsg.From.XString(m.SMTPUTF8)+">\r\n")...) msgData = append([]byte("Return-Path: <"+dsnMsg.From.XString(m.SMTPUTF8)+">\r\n"), msgData...)
mailbox := "Inbox" mailbox := "Inbox"
senderAccount := m.SenderAccount senderAccount := m.SenderAccount

View file

@ -525,6 +525,8 @@ func queueDelete(ctx context.Context, msgID int64) error {
// The queue is updated, either by removing a delivered or permanently failed // The queue is updated, either by removing a delivered or permanently failed
// message, or updating the time for the next attempt. A DSN may be sent. // message, or updating the time for the next attempt. A DSN may be sent.
func deliver(log mlog.Log, resolver dns.Resolver, m Msg) { func deliver(log mlog.Log, resolver dns.Resolver, m Msg) {
ctx := mox.Shutdown
qlog := log.WithCid(mox.Cid()).With(slog.Any("from", m.Sender()), qlog := log.WithCid(mox.Cid()).With(slog.Any("from", m.Sender()),
slog.Any("recipient", m.Recipient()), slog.Any("recipient", m.Recipient()),
slog.Int("attempts", m.Attempts), slog.Int("attempts", m.Attempts),
@ -572,7 +574,7 @@ func deliver(log mlog.Log, resolver dns.Resolver, m Msg) {
transport, ok = mox.Conf.Static.Transports[m.Transport] transport, ok = mox.Conf.Static.Transports[m.Transport]
if !ok { if !ok {
var remoteMTA dsn.NameIP // Zero value, will not be included in DSN. ../rfc/3464:1027 var remoteMTA dsn.NameIP // Zero value, will not be included in DSN. ../rfc/3464:1027
fail(qlog, m, backoff, false, remoteMTA, "", fmt.Sprintf("cannot find transport %q", m.Transport)) fail(ctx, qlog, m, backoff, false, remoteMTA, "", fmt.Sprintf("cannot find transport %q", m.Transport))
return return
} }
transportName = m.Transport transportName = m.Transport
@ -683,10 +685,10 @@ func deliver(log mlog.Log, resolver dns.Resolver, m Msg) {
if transport.Socks != nil { if transport.Socks != nil {
socksdialer, err := proxy.SOCKS5("tcp", transport.Socks.Address, nil, &net.Dialer{}) socksdialer, err := proxy.SOCKS5("tcp", transport.Socks.Address, nil, &net.Dialer{})
if err != nil { if err != nil {
fail(qlog, m, backoff, false, dsn.NameIP{}, "", fmt.Sprintf("socks dialer: %v", err)) fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", fmt.Sprintf("socks dialer: %v", err))
return return
} else if d, ok := socksdialer.(smtpclient.Dialer); !ok { } else if d, ok := socksdialer.(smtpclient.Dialer); !ok {
fail(qlog, m, backoff, false, dsn.NameIP{}, "", "socks dialer is not a contextdialer") fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", "socks dialer is not a contextdialer")
return return
} else { } else {
dialer = d dialer = d

View file

@ -70,16 +70,18 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
// todo: for submission, understand SRV records, and even DANE. // todo: for submission, understand SRV records, and even DANE.
ctx := mox.Shutdown
// If submit was done with REQUIRETLS extension for SMTP, we must verify TLS // If submit was done with REQUIRETLS extension for SMTP, we must verify TLS
// certificates. If our submission connection is not configured that way, abort. // certificates. If our submission connection is not configured that way, abort.
requireTLS := m.RequireTLS != nil && *m.RequireTLS requireTLS := m.RequireTLS != nil && *m.RequireTLS
if requireTLS && (tlsMode != smtpclient.TLSRequiredStartTLS && tlsMode != smtpclient.TLSImmediate || !tlsPKIX) { if requireTLS && (tlsMode != smtpclient.TLSRequiredStartTLS && tlsMode != smtpclient.TLSImmediate || !tlsPKIX) {
errmsg = fmt.Sprintf("transport %s: message requires verified tls but transport does not verify tls", transportName) errmsg = fmt.Sprintf("transport %s: message requires verified tls but transport does not verify tls", transportName)
fail(qlog, m, backoff, true, dsn.NameIP{}, smtp.SePol7MissingReqTLS, errmsg) fail(ctx, qlog, m, backoff, true, dsn.NameIP{}, smtp.SePol7MissingReqTLS, errmsg)
return return
} }
dialctx, dialcancel := context.WithTimeout(context.Background(), 30*time.Second) dialctx, dialcancel := context.WithTimeout(ctx, 30*time.Second)
defer dialcancel() defer dialcancel()
if m.DialedIPs == nil { if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{} m.DialedIPs = map[string][]net.IP{}
@ -90,7 +92,7 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
if m.DialedIPs == nil { if m.DialedIPs == nil {
m.DialedIPs = map[string][]net.IP{} m.DialedIPs = map[string][]net.IP{}
} }
conn, _, err = smtpclient.Dial(dialctx, qlog.Logger, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs) conn, _, err = smtpclient.Dial(dialctx, qlog.Logger, dialer, dns.IPDomain{Domain: transport.DNSHost}, ips, port, m.DialedIPs, mox.Conf.Static.SpecifiedSMTPListenIPs)
} }
addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port)) addr := net.JoinHostPort(transport.Host, fmt.Sprintf("%d", port))
var result string var result string
@ -112,7 +114,7 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
} }
qlog.Errorx("dialing for submission", err, slog.String("remote", addr)) qlog.Errorx("dialing for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: dialing %s for submission: %v", transportName, addr, err) errmsg = fmt.Sprintf("transport %s: dialing %s for submission: %v", transportName, addr, err)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return return
} }
dialcancel() dialcancel()
@ -134,7 +136,7 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
// Should not happen. // Should not happen.
qlog.Error("missing smtp authentication mechanisms implementation", slog.String("mechanism", mech)) qlog.Error("missing smtp authentication mechanisms implementation", slog.String("mechanism", mech))
errmsg = fmt.Sprintf("transport %s: authentication mechanisms %q not implemented", transportName, mech) errmsg = fmt.Sprintf("transport %s: authentication mechanisms %q not implemented", transportName, mech)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return return
} }
} }
@ -155,7 +157,7 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
qlog.Errorx("establishing smtp session for submission", err, slog.String("remote", addr)) qlog.Errorx("establishing smtp session for submission", err, slog.String("remote", addr))
errmsg = fmt.Sprintf("transport %s: establishing smtp session with %s for submission: %v", transportName, addr, err) errmsg = fmt.Sprintf("transport %s: establishing smtp session with %s for submission: %v", transportName, addr, err)
secodeOpt = smtperr.Secode secodeOpt = smtperr.Secode
fail(qlog, m, backoff, false, remoteMTA, secodeOpt, errmsg) fail(ctx, qlog, m, backoff, false, remoteMTA, secodeOpt, errmsg)
return return
} }
defer func() { defer func() {
@ -180,7 +182,7 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
if err != nil { if err != nil {
qlog.Errorx("opening message for delivery", err, slog.String("remote", addr), slog.String("path", p)) qlog.Errorx("opening message for delivery", err, slog.String("remote", addr), slog.String("path", p))
errmsg = fmt.Sprintf("transport %s: opening message file for submission: %v", transportName, err) errmsg = fmt.Sprintf("transport %s: opening message file for submission: %v", transportName, err)
fail(qlog, m, backoff, false, dsn.NameIP{}, "", errmsg) fail(ctx, qlog, m, backoff, false, dsn.NameIP{}, "", errmsg)
return return
} }
msgr = store.FileMsgReader(m.MsgPrefix, f) msgr = store.FileMsgReader(m.MsgPrefix, f)
@ -223,7 +225,7 @@ func deliverSubmit(qlog mlog.Log, resolver dns.Resolver, dialer smtpclient.Diale
permanent = smtperr.Permanent permanent = smtperr.Permanent
secodeOpt = smtperr.Secode secodeOpt = smtperr.Secode
errmsg = fmt.Sprintf("transport %s: submitting email to %s: %v", transportName, addr, err) errmsg = fmt.Sprintf("transport %s: submitting email to %s: %v", transportName, addr, err)
fail(qlog, m, backoff, permanent, remoteMTA, secodeOpt, errmsg) fail(ctx, qlog, m, backoff, permanent, remoteMTA, secodeOpt, errmsg)
return return
} }
qlog.Info("delivered from queue with transport") qlog.Info("delivered from queue with transport")

View file

@ -7,9 +7,11 @@ import (
"strings" "strings"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/moxvar"
) )
// Pedantic enables stricter parsing.
var Pedantic bool
var ErrBadAddress = errors.New("invalid email address") var ErrBadAddress = errors.New("invalid email address")
// Localpart is a decoded local part of an email address, before the "@". // Localpart is a decoded local part of an email address, before the "@".
@ -262,7 +264,7 @@ func (p *parser) xlocalpart() Localpart {
} }
} }
// In the wild, some services use large localparts for generated (bounce) addresses. // In the wild, some services use large localparts for generated (bounce) addresses.
if moxvar.Pedantic && len(s) > 64 || len(s) > 128 { if Pedantic && len(s) > 64 || len(s) > 128 {
// ../rfc/5321:3486 // ../rfc/5321:3486
p.xerrorf("localpart longer than 64 octets") p.xerrorf("localpart longer than 64 octets")
} }

View file

@ -18,46 +18,24 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/adns" "github.com/mjl-/adns"
"github.com/mjl-/mox/dane" "github.com/mjl-/mox/dane"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/sasl" "github.com/mjl-/mox/sasl"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/stub"
"github.com/mjl-/mox/tlsrpt" "github.com/mjl-/mox/tlsrpt"
) )
// todo future: add function to deliver message to multiple recipients. requires more elaborate return value, indicating success per message: some recipients may succeed, others may fail, and we should still deliver. to prevent backscatter, we also sometimes don't allow multiple recipients. ../rfc/5321:1144 // todo future: add function to deliver message to multiple recipients. requires more elaborate return value, indicating success per message: some recipients may succeed, others may fail, and we should still deliver. to prevent backscatter, we also sometimes don't allow multiple recipients. ../rfc/5321:1144
var ( var (
metricCommands = promauto.NewHistogramVec( MetricCommands stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{ MetricTLSRequiredNoIgnored stub.CounterVec = stub.CounterVecIgnore{}
Name: "mox_smtpclient_command_duration_seconds", MetricPanicInc = func() {}
Help: "SMTP client command duration and result codes in seconds.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30, 60, 120},
},
[]string{
"cmd",
"code",
"secode",
},
)
metricTLSRequiredNoIgnored = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "mox_smtpclient_tlsrequiredno_ignored_total",
Help: "Connection attempts with TLS policy findings ignored due to message with TLS-Required: No header. Does not cover case where TLS certificate cannot be PKIX-verified.",
},
[]string{
"ignored", // daneverification (no matching tlsa record)
},
)
) )
var ( var (
@ -281,7 +259,7 @@ func New(ctx context.Context, elog *slog.Logger, conn net.Conn, tlsMode TLSMode,
c.firstReadAfterHandshake = true c.firstReadAfterHandshake = true
c.tlsResultAdd(1, 0, nil) c.tlsResultAdd(1, 0, nil)
c.conn = tlsconn c.conn = tlsconn
tlsversion, ciphersuite := mox.TLSInfo(tlsconn) tlsversion, ciphersuite := moxio.TLSInfo(tlsconn)
c.log.Debug("tls client handshake done", c.log.Debug("tls client handshake done",
slog.String("tls", tlsversion), slog.String("tls", tlsversion),
slog.String("ciphersuite", ciphersuite), slog.String("ciphersuite", ciphersuite),
@ -334,7 +312,7 @@ func (c *Client) tlsConfig() *tls.Config {
// DANE verification. // DANE verification.
// daneRecords can be non-nil and empty, that's intended. // daneRecords can be non-nil and empty, that's intended.
if c.daneRecords != nil { if c.daneRecords != nil {
verified, record, err := dane.Verify(c.log.Logger, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames) verified, record, err := dane.Verify(c.log.Logger, c.daneRecords, cs, c.remoteHostname, c.daneMoreHostnames, c.rootCAs)
c.log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record)) c.log.Debugx("dane verification", err, slog.Bool("verified", verified), slog.Any("record", record))
if verified { if verified {
if c.daneVerifiedRecord != nil { if c.daneVerifiedRecord != nil {
@ -355,7 +333,7 @@ func (c *Client) tlsConfig() *tls.Config {
if c.ignoreTLSVerifyErrors { if c.ignoreTLSVerifyErrors {
// We ignore the failure and continue the connection. // We ignore the failure and continue the connection.
c.log.Infox("verifying dane failed, continuing with connection", err) c.log.Infox("verifying dane failed, continuing with connection", err)
metricTLSRequiredNoIgnored.WithLabelValues("daneverification").Inc() MetricTLSRequiredNoIgnored.IncLabels("daneverification")
} else { } else {
// This connection will fail. // This connection will fail.
daneErr = dane.ErrNoMatch daneErr = dane.ErrNoMatch
@ -547,7 +525,7 @@ func (c *Client) readecode(ecodes bool) (code int, secode, lastLine string, text
c.cmds = c.cmds[1:] c.cmds = c.cmds[1:]
} }
} }
metricCommands.WithLabelValues(cmd, fmt.Sprintf("%d", co), sec).Observe(float64(time.Since(c.cmdStart)) / float64(time.Second)) MetricCommands.ObserveLabels(float64(time.Since(c.cmdStart))/float64(time.Second), cmd, fmt.Sprintf("%d", co), sec)
c.log.Debug("smtpclient command result", c.log.Debug("smtpclient command result",
slog.String("cmd", cmd), slog.String("cmd", cmd),
slog.Int("code", co), slog.Int("code", co),
@ -651,7 +629,7 @@ func (c *Client) recover(rerr *error) {
} }
cerr, ok := x.(Error) cerr, ok := x.(Error)
if !ok { if !ok {
metrics.PanicInc(metrics.Smtpclient) MetricPanicInc()
panic(x) panic(x)
} }
*rerr = cerr *rerr = cerr
@ -779,7 +757,7 @@ func (c *Client) hello(ctx context.Context, tlsMode TLSMode, ehloHostname dns.Do
c.r = bufio.NewReader(c.tr) c.r = bufio.NewReader(c.tr)
c.w = bufio.NewWriter(c.tw) c.w = bufio.NewWriter(c.tw)
tlsversion, ciphersuite := mox.TLSInfo(nconn) tlsversion, ciphersuite := moxio.TLSInfo(nconn)
c.log.Debug("starttls client handshake done", c.log.Debug("starttls client handshake done",
slog.Any("tlsmode", tlsMode), slog.Any("tlsmode", tlsMode),
slog.Bool("verifypkix", c.tlsVerifyPKIX), slog.Bool("verifypkix", c.tlsVerifyPKIX),

View file

@ -10,7 +10,6 @@ import (
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
) )
// DialHook can be used during tests to override the regular dialer from being used. // DialHook can be used during tests to override the regular dialer from being used.
@ -50,10 +49,9 @@ type Dialer interface {
// Dial updates dialedIPs, callers may want to save it so it can be taken into // Dial updates dialedIPs, callers may want to save it so it can be taken into
// account for future delivery attempts. // account for future delivery attempts.
// //
// If we have fully specified local SMTP listener IPs, we set those for the // The first matching protocol family from localIPs is set for the local side
// outgoing connection. The admin probably configured these same IPs in SPF, but // of the TCP connection.
// others possibly not. func Dial(ctx context.Context, elog *slog.Logger, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP, localIPs []net.IP) (conn net.Conn, ip net.IP, rerr error) {
func Dial(ctx context.Context, elog *slog.Logger, dialer Dialer, host dns.IPDomain, ips []net.IP, port int, dialedIPs map[string][]net.IP) (conn net.Conn, ip net.IP, rerr error) {
log := mlog.New("smtpclient", elog) log := mlog.New("smtpclient", elog)
timeout := 30 * time.Second timeout := 30 * time.Second
if deadline, ok := ctx.Deadline(); ok && len(ips) > 0 { if deadline, ok := ctx.Deadline(); ok && len(ips) > 0 {
@ -66,7 +64,7 @@ func Dial(ctx context.Context, elog *slog.Logger, dialer Dialer, host dns.IPDoma
addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", port)) addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", port))
log.Debug("dialing host", slog.String("addr", addr)) log.Debug("dialing host", slog.String("addr", addr))
var laddr net.Addr var laddr net.Addr
for _, lip := range mox.Conf.Static.SpecifiedSMTPListenIPs { for _, lip := range localIPs {
ipIs4 := ip.To4() != nil ipIs4 := ip.To4() != nil
lipIs4 := lip.To4() != nil lipIs4 := lip.To4() != nil
if ipIs4 == lipIs4 { if ipIs4 == lipIs4 {

View file

@ -41,7 +41,7 @@ func TestDialHost(t *testing.T) {
if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}) || !dualstack { if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}) || !dualstack {
t.Fatalf("expected err nil, address 10.0.0.1,2001:db8::1, dualstack true, got %v %v %v", err, ips, dualstack) t.Fatalf("expected err nil, address 10.0.0.1,2001:db8::1, dualstack true, got %v %v %v", err, ips, dualstack)
} }
_, ip, err := Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) _, ip, err := Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs, nil)
if err != nil || ip.String() != "10.0.0.1" { if err != nil || ip.String() != "10.0.0.1" {
t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack) t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack)
} }
@ -50,7 +50,7 @@ func TestDialHost(t *testing.T) {
if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("10.0.0.1")}) || !dualstack { if err != nil || !reflect.DeepEqual(ips, []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("10.0.0.1")}) || !dualstack {
t.Fatalf("expected err nil, address 2001:db8::1,10.0.0.1, dualstack true, got %v %v %v", err, ips, dualstack) t.Fatalf("expected err nil, address 2001:db8::1,10.0.0.1, dualstack true, got %v %v %v", err, ips, dualstack)
} }
_, ip, err = Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs) _, ip, err = Dial(ctxbg, log.Logger, nil, ipdomain("dualstack.example"), ips, 25, dialedIPs, nil)
if err != nil || ip.String() != "2001:db8::1" { if err != nil || ip.String() != "2001:db8::1" {
t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack) t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack)
} }

View file

@ -5,22 +5,32 @@ import (
"fmt" "fmt"
"github.com/mjl-/mox/dsn" "github.com/mjl-/mox/dsn"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/queue" "github.com/mjl-/mox/queue"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
// compose dsn message and add it to the queue for delivery to rcptTo. // compose dsn message and add it to the queue for delivery to rcptTo.
func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message, requireTLS bool) error { func queueDSN(ctx context.Context, log mlog.Log, c *conn, rcptTo smtp.Path, m dsn.Message, requireTLS bool) error {
buf, err := m.Compose(c.log, false) buf, err := m.Compose(c.log, false)
if err != nil { if err != nil {
return err return err
} }
bufDKIM, err := mox.DKIMSign(ctx, c.log, m.From, false, buf)
log.Check(err, "dkim signing dsn")
buf = append([]byte(bufDKIM), buf...)
var bufUTF8 []byte var bufUTF8 []byte
if c.smtputf8 { if c.smtputf8 {
bufUTF8, err = m.Compose(c.log, true) bufUTF8, err = m.Compose(c.log, true)
if err != nil { if err != nil {
c.log.Errorx("composing dsn with utf-8 for incoming delivery for unknown user, continuing with ascii-only dsn", err) c.log.Errorx("composing dsn with utf-8 for incoming delivery for unknown user, continuing with ascii-only dsn", err)
} else {
bufUTF8DKIM, err := mox.DKIMSign(ctx, log, m.From, true, bufUTF8)
log.Check(err, "dkim signing dsn with utf8")
bufUTF8 = append([]byte(bufUTF8DKIM), bufUTF8...)
} }
} }

View file

@ -8,7 +8,7 @@ import (
"strings" "strings"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/moxvar" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
) )
@ -308,7 +308,7 @@ func (p *parser) xipdomain(isehlo bool) dns.IPDomain {
// Mail user agents that submit are relatively likely to use IPs in EHLO and forget // Mail user agents that submit are relatively likely to use IPs in EHLO and forget
// that an IPv6 address needs to be tagged as such. We can forgive them. For // that an IPv6 address needs to be tagged as such. We can forgive them. For
// SMTP servers we are strict. // SMTP servers we are strict.
return isehlo && p.conn.submission && !moxvar.Pedantic && ip.To16() != nil return isehlo && p.conn.submission && !mox.Pedantic && ip.To16() != nil
} }
if ipv6 && isv4 { if ipv6 && isv4 {
p.xerrorf("ip address is not ipv6") p.xerrorf("ip address is not ipv6")
@ -337,7 +337,7 @@ func (p *parser) xlocalpart() smtp.Localpart {
} }
} }
// In the wild, some services use large localparts for generated (bounce) addresses. // In the wild, some services use large localparts for generated (bounce) addresses.
if moxvar.Pedantic && len(s) > 64 || len(s) > 128 { if mox.Pedantic && len(s) > 64 || len(s) > 128 {
// ../rfc/5321:3486 // ../rfc/5321:3486
p.xerrorf("localpart longer than 64 octets") p.xerrorf("localpart longer than 64 octets")
} }

View file

@ -804,7 +804,7 @@ func (c *conn) cmdEhlo(p *parser) {
// ../rfc/5321:1783 // ../rfc/5321:1783
func (c *conn) cmdHello(p *parser, ehlo bool) { func (c *conn) cmdHello(p *parser, ehlo bool) {
var remote dns.IPDomain var remote dns.IPDomain
if c.submission && !moxvar.Pedantic { if c.submission && !mox.Pedantic {
// Mail clients regularly put bogus information in the hostname/ip. For submission, // Mail clients regularly put bogus information in the hostname/ip. For submission,
// the value is of no use, so there is not much point in annoying the user with // the value is of no use, so there is not much point in annoying the user with
// errors they cannot fix themselves. Except when in pedantic mode. // errors they cannot fix themselves. Except when in pedantic mode.
@ -907,7 +907,7 @@ func (c *conn) cmdStarttls(p *parser) {
panic(fmt.Errorf("starttls handshake: %s (%w)", err, errIO)) panic(fmt.Errorf("starttls handshake: %s (%w)", err, errIO))
} }
cancel() cancel()
tlsversion, ciphersuite := mox.TLSInfo(tlsConn) tlsversion, ciphersuite := moxio.TLSInfo(tlsConn)
c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite)) c.log.Debug("tls server handshake done", slog.String("tls", tlsversion), slog.String("ciphersuite", ciphersuite))
c.conn = tlsConn c.conn = tlsConn
c.tr = moxio.NewTraceReader(c.log, "RC: ", c) c.tr = moxio.NewTraceReader(c.log, "RC: ", c)
@ -982,7 +982,7 @@ func (c *conn) cmdAuth(p *parser) {
} }
} else { } else {
p.xspace() p.xspace()
if !moxvar.Pedantic { if !mox.Pedantic {
// Windows Mail 16005.14326.21606.0 sends two spaces between "AUTH PLAIN" and the // Windows Mail 16005.14326.21606.0 sends two spaces between "AUTH PLAIN" and the
// base64 data. // base64 data.
for p.space() { for p.space() {
@ -1304,7 +1304,7 @@ func (c *conn) cmdMail(p *parser) {
// note: no space allowed after colon. ../rfc/5321:1093 // note: no space allowed after colon. ../rfc/5321:1093
// Microsoft Outlook 365 Apps for Enterprise sends it with submission. For delivery // Microsoft Outlook 365 Apps for Enterprise sends it with submission. For delivery
// it is mostly used by spammers, but has been seen with legitimate senders too. // it is mostly used by spammers, but has been seen with legitimate senders too.
if !moxvar.Pedantic { if !mox.Pedantic {
p.space() p.space()
} }
rawRevPath := p.xrawReversePath() rawRevPath := p.xrawReversePath()
@ -1445,7 +1445,7 @@ func (c *conn) cmdRcpt(p *parser) {
// note: no space allowed after colon. ../rfc/5321:1093 // note: no space allowed after colon. ../rfc/5321:1093
// Microsoft Outlook 365 Apps for Enterprise sends it with submission. For delivery // Microsoft Outlook 365 Apps for Enterprise sends it with submission. For delivery
// it is mostly used by spammers, but has been seen with legitimate senders too. // it is mostly used by spammers, but has been seen with legitimate senders too.
if !moxvar.Pedantic { if !mox.Pedantic {
p.space() p.space()
} }
var fpath smtp.Path var fpath smtp.Path
@ -1639,13 +1639,13 @@ func (c *conn) cmdData(p *parser) {
} }
// Check only for pedantic mode because ios mail will attempt to send smtputf8 with // Check only for pedantic mode because ios mail will attempt to send smtputf8 with
// non-ascii in message from localpart without using 8bitmime. // non-ascii in message from localpart without using 8bitmime.
if moxvar.Pedantic && msgWriter.Has8bit && !c.has8bitmime { if mox.Pedantic && msgWriter.Has8bit && !c.has8bitmime {
// ../rfc/5321:906 // ../rfc/5321:906
xsmtpUserErrorf(smtp.C500BadSyntax, smtp.SeMsg6Other0, "message with non-us-ascii requires 8bitmime extension") xsmtpUserErrorf(smtp.C500BadSyntax, smtp.SeMsg6Other0, "message with non-us-ascii requires 8bitmime extension")
} }
} }
if Localserve && moxvar.Pedantic { if Localserve && mox.Pedantic {
// Require that message can be parsed fully. // Require that message can be parsed fully.
p, err := message.Parse(c.log.Logger, false, dataFile) p, err := message.Parse(c.log.Logger, false, dataFile)
if err == nil { if err == nil {
@ -1855,11 +1855,11 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "internal error") xsmtpServerErrorf(codes{smtp.C451LocalErr, smtp.SeSys3Other0}, "internal error")
} }
dkimConfig := confDom.DKIM selectors := mox.DKIMSelectors(confDom.DKIM)
if len(dkimConfig.Sign) > 0 { if len(selectors) > 0 {
if canonical, err := mox.CanonicalLocalpart(msgFrom.Localpart, confDom); err != nil { if canonical, err := mox.CanonicalLocalpart(msgFrom.Localpart, confDom); err != nil {
c.log.Errorx("determining canonical localpart for dkim signing", err, slog.Any("localpart", msgFrom.Localpart)) c.log.Errorx("determining canonical localpart for dkim signing", err, slog.Any("localpart", msgFrom.Localpart))
} else if dkimHeaders, err := dkim.Sign(ctx, c.log.Logger, canonical, msgFrom.Domain, dkimConfig, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil { } else if dkimHeaders, err := dkim.Sign(ctx, c.log.Logger, canonical, msgFrom.Domain, selectors, c.smtputf8, store.FileMsgReader(msgPrefix, dataFile)); err != nil {
c.log.Errorx("dkim sign for domain", err, slog.Any("domain", msgFrom.Domain)) c.log.Errorx("dkim sign for domain", err, slog.Any("domain", msgFrom.Domain))
metricServerErrors.WithLabelValues("dkimsign").Inc() metricServerErrors.WithLabelValues("dkimsign").Inc()
} else { } else {
@ -2841,6 +2841,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
From: smtp.Path{Localpart: "postmaster", IPDomain: deliverErrors[0].rcptTo.IPDomain}, From: smtp.Path{Localpart: "postmaster", IPDomain: deliverErrors[0].rcptTo.IPDomain},
To: *c.mailFrom, To: *c.mailFrom,
Subject: "mail delivery failure", Subject: "mail delivery failure",
MessageID: mox.MessageIDGen(false),
References: messageID, References: messageID,
// Per-message details. // Per-message details.
@ -2876,7 +2877,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
if Localserve { if Localserve {
c.log.Error("not queueing dsn for incoming delivery due to localserve") c.log.Error("not queueing dsn for incoming delivery due to localserve")
} else if err := queueDSN(context.TODO(), c, *c.mailFrom, dsnMsg, c.requireTLS != nil && *c.requireTLS); err != nil { } else if err := queueDSN(context.TODO(), c.log, c, *c.mailFrom, dsnMsg, c.requireTLS != nil && *c.requireTLS); err != nil {
metricServerErrors.WithLabelValues("queuedsn").Inc() metricServerErrors.WithLabelValues("queuedsn").Inc()
c.log.Errorx("queuing DSN for incoming delivery, no DSN sent", err) c.log.Errorx("queuing DSN for incoming delivery, no DSN sent", err)
} }

View file

@ -1008,7 +1008,8 @@ func TestTLSReport(t *testing.T) {
tcheck(t, xerr, "write msg") tcheck(t, xerr, "write msg")
msg := msgb.String() msg := msgb.String()
headers, xerr := dkim.Sign(ctxbg, pkglog.Logger, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg)) selectors := mox.DKIMSelectors(dkimConf)
headers, xerr := dkim.Sign(ctxbg, pkglog.Logger, "remote", dns.Domain{ASCII: "example.org"}, selectors, false, strings.NewReader(msg))
tcheck(t, xerr, "dkim sign") tcheck(t, xerr, "dkim sign")
msg = headers + msg msg = headers + msg

View file

@ -18,12 +18,10 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/stub"
) )
// The net package always returns DNS names in absolute, lower-case form. We make // The net package always returns DNS names in absolute, lower-case form. We make
@ -31,16 +29,7 @@ import (
// verify names relative to our local search domain. // verify names relative to our local search domain.
var ( var (
metricSPFVerify = promauto.NewHistogramVec( MetricVerify stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{
Name: "mox_spf_verify_duration_seconds",
Help: "SPF verify, including lookup, duration and result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20},
},
[]string{
"status",
},
)
) )
// cross-link rfc and errata // cross-link rfc and errata
@ -202,7 +191,7 @@ func Verify(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, args
log := mlog.New("spf", elog) log := mlog.New("spf", elog)
start := time.Now() start := time.Now()
defer func() { defer func() {
metricSPFVerify.WithLabelValues(string(received.Result)).Observe(float64(time.Since(start)) / float64(time.Second)) MetricVerify.ObserveLabels(float64(time.Since(start))/float64(time.Second), string(received.Result))
log.Debugx("spf verify result", rerr, log.Debugx("spf verify result", rerr,
slog.Any("domain", args.domain), slog.Any("domain", args.domain),
slog.Any("ip", args.RemoteIP), slog.Any("ip", args.RemoteIP),

View file

@ -56,7 +56,6 @@ import (
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-" "github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/publicsuffix" "github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/scram" "github.com/mjl-/mox/scram"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
@ -2086,7 +2085,7 @@ func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error)
if field, ok := fields[f]; ok { if field, ok := fields[f]; ok {
*field = true *field = true
} else if seen[f] { } else if seen[f] {
if moxvar.Pedantic { if mox.Pedantic {
return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f) return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)
} }
} else { } else {

7
stub/doc.go Normal file
View file

@ -0,0 +1,7 @@
// Package stub provides interfaces and stub implementations.
//
// Packages in mox use these interfaces and implementations so other software
// reusing these packages won't have to take on unwanted dependencies.
//
// Stubs are provided for: metrics (prometheus).
package stub

43
stub/metrics.go Normal file
View file

@ -0,0 +1,43 @@
package stub
import (
"context"
"time"
"golang.org/x/exp/slog"
)
func HTTPClientObserveIgnore(ctx context.Context, log *slog.Logger, pkg, method string, statusCode int, err error, start time.Time) {
}
type Counter interface {
Inc()
}
type CounterIgnore struct{}
func (CounterIgnore) Inc() {}
type CounterVec interface {
IncLabels(labels ...string)
}
type CounterVecIgnore struct{}
func (CounterVecIgnore) IncLabels(labels ...string) {}
type Histogram interface {
Observe(float64)
}
type HistogramIgnore struct{}
func (HistogramIgnore) Observe(float64) {}
type HistogramVec interface {
ObserveLabels(v float64, labels ...string)
}
type HistogramVecIgnore struct{}
func (HistogramVecIgnore) ObserveLabels(v float64, labels ...string) {}

View file

@ -13,31 +13,16 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/stub"
) )
var ( var (
metricGenerate = promauto.NewCounter( MetricGenerate stub.Counter = stub.CounterIgnore{}
prometheus.CounterOpts{ MetricVerify stub.CounterVec = stub.CounterVecIgnore{}
Name: "mox_subjectpass_generate_total",
Help: "Number of generated subjectpass challenges.",
},
)
metricVerify = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "mox_subjectpass_verify_total",
Help: "Number of subjectpass verifications.",
},
[]string{
"result", // ok, fail
},
)
) )
var ( var (
@ -57,7 +42,7 @@ var Explanation = "Your message resembles spam. If your email is legitimate, ple
func Generate(elog *slog.Logger, mailFrom smtp.Address, key []byte, tm time.Time) string { func Generate(elog *slog.Logger, mailFrom smtp.Address, key []byte, tm time.Time) string {
log := mlog.New("subjectpass", elog) log := mlog.New("subjectpass", elog)
metricGenerate.Inc() MetricGenerate.Inc()
log.Debug("subjectpass generate", slog.Any("mailfrom", mailFrom)) log.Debug("subjectpass generate", slog.Any("mailfrom", mailFrom))
// We discard the lower 8 bits of the time, we can do with less precision. // We discard the lower 8 bits of the time, we can do with less precision.
@ -88,7 +73,7 @@ func Verify(elog *slog.Logger, r io.ReaderAt, key []byte, period time.Duration)
if rerr == nil { if rerr == nil {
result = "ok" result = "ok"
} }
metricVerify.WithLabelValues(result).Inc() MetricVerify.IncLabels(result)
log.Debugx("subjectpass verify result", rerr, slog.String("token", token), slog.Duration("period", period)) log.Debugx("subjectpass verify result", rerr, slog.String("token", token), slog.Duration("period", period))
}() }()

View file

@ -8,22 +8,13 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/stub"
) )
var ( var (
metricLookup = promauto.NewHistogramVec( MetricLookup stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{
Name: "mox_tlsrpt_lookup_duration_seconds",
Help: "TLSRPT lookups with result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"result"},
)
) )
var ( var (
@ -53,7 +44,7 @@ func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domai
result = "error" result = "error"
} }
} }
metricLookup.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) MetricLookup.ObserveLabels(float64(time.Since(start))/float64(time.Second), result)
log.Debugx("tlsrpt lookup result", rerr, log.Debugx("tlsrpt lookup result", rerr,
slog.Any("domain", domain), slog.Any("domain", domain),
slog.Any("record", rrecord), slog.Any("record", rrecord),

View file

@ -688,15 +688,14 @@ func composeMessage(ctx context.Context, log mlog.Log, mf *os.File, policyDomain
xc.Flush() xc.Flush()
selectors := map[string]config.Selector{} selectors := mox.DKIMSelectors(confDKIM)
for name, sel := range confDKIM.Selectors { for i, sel := range selectors {
// Also sign the TLS-Report headers. ../rfc/8460:940 // Also sign the TLS-Report headers. ../rfc/8460:940
sel.HeadersEffective = append(append([]string{}, sel.HeadersEffective...), "TLS-Report-Domain", "TLS-Report-Submitter") sel.Headers = append(append([]string{}, sel.Headers...), "TLS-Report-Domain", "TLS-Report-Submitter")
selectors[name] = sel selectors[i] = sel
} }
confDKIM.Selectors = selectors
dkimHeader, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fromAddr.Domain, confDKIM, smtputf8, mf) dkimHeader, err := dkim.Sign(ctx, log.Logger, fromAddr.Localpart, fromAddr.Domain, selectors, smtputf8, mf)
xc.Checkf(err, "dkim-signing report message") xc.Checkf(err, "dkim-signing report message")
return dkimHeader, xc.Has8bit, xc.SMTPUTF8, messageID, nil return dkimHeader, xc.Has8bit, xc.SMTPUTF8, messageID, nil

View file

@ -23,32 +23,16 @@ import (
"golang.org/x/exp/slog" "golang.org/x/exp/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/stub"
) )
var ( var (
metricLookup = promauto.NewHistogramVec( MetricLookup stub.HistogramVec = stub.HistogramVecIgnore{}
prometheus.HistogramOpts{ MetricFetchChangelog stub.HistogramVec = stub.HistogramVecIgnore{}
Name: "mox_updates_lookup_duration_seconds", HTTPClientObserve func(ctx context.Context, log *slog.Logger, pkg, method string, statusCode int, err error, start time.Time) = stub.HTTPClientObserveIgnore
Help: "Updates lookup with result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"result"},
)
metricFetchChangelog = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "mox_updates_fetchchangelog_duration_seconds",
Help: "Fetch changelog with result.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.100, 0.5, 1, 5, 10, 20, 30},
},
[]string{"result"},
)
) )
var ( var (
@ -89,7 +73,7 @@ func Lookup(ctx context.Context, elog *slog.Logger, resolver dns.Resolver, domai
if rerr != nil { if rerr != nil {
result = "error" result = "error"
} }
metricLookup.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) MetricLookup.ObserveLabels(float64(time.Since(start))/float64(time.Second), result)
log.Debugx("updates lookup result", rerr, log.Debugx("updates lookup result", rerr,
slog.Any("domain", domain), slog.Any("domain", domain),
slog.Any("version", rversion), slog.Any("version", rversion),
@ -144,7 +128,7 @@ func FetchChangelog(ctx context.Context, elog *slog.Logger, baseURL string, base
if rerr != nil { if rerr != nil {
result = "error" result = "error"
} }
metricFetchChangelog.WithLabelValues(result).Observe(float64(time.Since(start)) / float64(time.Second)) MetricFetchChangelog.ObserveLabels(float64(time.Since(start))/float64(time.Second), result)
log.Debugx("updates fetch changelog result", rerr, log.Debugx("updates fetch changelog result", rerr,
slog.String("baseurl", baseURL), slog.String("baseurl", baseURL),
slog.Any("base", base), slog.Any("base", base),
@ -163,7 +147,7 @@ func FetchChangelog(ctx context.Context, elog *slog.Logger, baseURL string, base
if resp == nil { if resp == nil {
resp = &http.Response{StatusCode: 0} resp = &http.Response{StatusCode: 0}
} }
metrics.HTTPClientObserve(ctx, log, "updates", req.Method, resp.StatusCode, err, start) HTTPClientObserve(ctx, log.Logger, "updates", req.Method, resp.StatusCode, err, start)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: making http request: %s", ErrChangelogFetch, err) return nil, fmt.Errorf("%w: making http request: %s", ErrChangelogFetch, err)
} }

View file

@ -557,8 +557,9 @@ func (w Webmail) MessageSubmit(ctx context.Context, m SubmitMessage) {
var msgPrefix string var msgPrefix string
fd := fromAddr.Address.Domain fd := fromAddr.Address.Domain
confDom, _ := mox.Conf.Domain(fd) confDom, _ := mox.Conf.Domain(fd)
if len(confDom.DKIM.Sign) > 0 { selectors := mox.DKIMSelectors(confDom.DKIM)
dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Address.Localpart, fd, confDom.DKIM, smtputf8, dataFile) if len(selectors) > 0 {
dkimHeaders, err := dkim.Sign(ctx, log.Logger, fromAddr.Address.Localpart, fd, selectors, smtputf8, dataFile)
if err != nil { if err != nil {
metricServerErrors.WithLabelValues("dkimsign").Inc() metricServerErrors.WithLabelValues("dkimsign").Inc()
} }

View file

@ -13,8 +13,8 @@ import (
"github.com/mjl-/mox/dns" "github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message" "github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog" "github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio" "github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/moxvar"
"github.com/mjl-/mox/smtp" "github.com/mjl-/mox/smtp"
"github.com/mjl-/mox/store" "github.com/mjl-/mox/store"
) )
@ -39,7 +39,7 @@ func tryDecodeParam(log mlog.Log, name string) string {
return name return name
} }
// todo: find where this is allowed. it seems quite common. perhaps we should remove the pedantic check? // todo: find where this is allowed. it seems quite common. perhaps we should remove the pedantic check?
if moxvar.Pedantic { if mox.Pedantic {
log.Debug("attachment contains rfc2047 q/b-word-encoded mime parameter instead of rfc2231-encoded", slog.String("name", name)) log.Debug("attachment contains rfc2047 q/b-word-encoded mime parameter instead of rfc2231-encoded", slog.String("name", name))
return name return name
} }